def __init__(self, conf, logger): self.logger = logger self.devices = conf.get('devices', '/srv/node/') self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) self.keep_cache_size = int(conf.get('keep_cache_size', 5242880)) self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024 self.mount_check = config_true_value(conf.get('mount_check', 'true')) threads_per_disk = int(conf.get('threads_per_disk', '0')) self.threadpools = defaultdict( lambda: ThreadPool(nthreads=threads_per_disk))
def __init__(self, conf, logger): self.logger = logger self.disk_chunk_size = int( conf.get('disk_chunk_size', DEFAULT_DISK_CHUNK_SIZE)) self.keep_cache_size = int( conf.get('keep_cache_size', DEFAULT_KEEP_CACHE_SIZE)) self.bytes_per_sync = int(conf.get('mb_per_sync', DEFAULT_MB_PER_SYNC)) * 1024 * 1024 self.devices = conf.get('devices', '/srv/node/') self.mount_check = config_true_value(conf.get('mount_check', 'true')) threads_per_disk = int(conf.get('threads_per_disk', '0')) self.threadpools = defaultdict( lambda: ThreadPool(nthreads=threads_per_disk))
def __init__(self, conf): """ Creates a new WSGI application for the Swift Object Server. An example configuration is given at <source-dir>/etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ self.logger = get_logger(conf, log_route='object-server') self.devices = conf.get('devices', '/srv/node/') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) self.keep_cache_size = int(conf.get('keep_cache_size', 5242880)) self.keep_cache_private = \ config_true_value(conf.get('keep_cache_private', 'false')) self.log_requests = config_true_value(conf.get('log_requests', 'true')) self.max_upload_time = int(conf.get('max_upload_time', 86400)) self.slow = int(conf.get('slow', 0)) self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024 replication_server = conf.get('replication_server', None) if replication_server is not None: replication_server = config_true_value(replication_server) self.replication_server = replication_server self.threads_per_disk = int(conf.get('threads_per_disk', '0')) self.threadpools = defaultdict( lambda: ThreadPool(nthreads=self.threads_per_disk)) default_allowed_headers = ''' content-disposition, content-encoding, x-delete-at, x-object-manifest, x-static-large-object, ''' extra_allowed_headers = [ header.strip().lower() for header in conf.get( 'allowed_headers', default_allowed_headers).split(',') if header.strip() ] self.allowed_headers = set() for header in extra_allowed_headers: if header not in DATAFILE_SYSTEM_META: self.allowed_headers.add(header) self.expiring_objects_account = \ (conf.get('auto_create_account_prefix') or '.') + \ 'expiring_objects' self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400)
def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, bytes_per_sync=(512 * 1024 * 1024), iter_hook=None, threadpool=None, obj_dir='objects', mount_check=False): if mount_check and not check_mount(path, device): raise DiskFileDeviceUnavailable() self.disk_chunk_size = disk_chunk_size self.bytes_per_sync = bytes_per_sync self.iter_hook = iter_hook self.name = '/' + '/'.join((account, container, obj)) name_hash = hash_path(account, container, obj) self.datadir = join(path, device, storage_directory(obj_dir, partition, name_hash)) self.device_path = join(path, device) self.tmpdir = join(path, device, 'tmp') self.logger = logger self.metadata = {} self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.suppress_file_closing = False self.threadpool = threadpool or ThreadPool(nthreads=0) data_file, meta_file, ts_file = self._get_ondisk_file() if not data_file: if ts_file: self._construct_from_ts_file(ts_file) else: fp = self._construct_from_data_file(data_file, meta_file) if keep_data_fp: self.fp = fp else: fp.close()
def __init__(self, mgr, dev_path, threadpool, partition, account=None, container=None, obj=None, policy=None, uid=DEFAULT_UID, gid=DEFAULT_GID, **kwargs): # Variables partition and policy is currently unused. self._mgr = mgr self._device_path = dev_path self._threadpool = threadpool or ThreadPool(nthreads=0) self._uid = int(uid) self._gid = int(gid) self._is_dir = False self._metadata = None self._fd = None # This fd attribute is not used in PUT path. fd used in PUT path # is encapsulated inside DiskFileWriter object. self._stat = None # Don't store a value for data_file until we know it exists. self._data_file = None # Account name contains resller_prefix which is retained and not # stripped. This to conform to Swift's behavior where account name # entry in Account DBs contain resller_prefix. self._account = account self._container = container self._container_path = \ os.path.join(self._device_path, self._account, self._container) obj = obj.strip(os.path.sep) obj_path, self._obj = os.path.split(obj) if obj_path: self._obj_path = obj_path.strip(os.path.sep) self._put_datadir = os.path.join(self._container_path, self._obj_path) else: self._obj_path = '' self._put_datadir = self._container_path self._data_file = os.path.join(self._put_datadir, self._obj) self._disk_file_open = False
def __init__(self, mgr, dev_path, threadpool, partition, account=None, container=None, obj=None, policy_idx=0, uid=DEFAULT_UID, gid=DEFAULT_GID): # Variables partition and policy_idx is currently unused. self._mgr = mgr self._device_path = dev_path self._threadpool = threadpool or ThreadPool(nthreads=0) self._uid = int(uid) self._gid = int(gid) self._is_dir = False self._metadata = None self._fd = None # Don't store a value for data_file until we know it exists. self._data_file = None # Account name contains resller_prefix which is retained and not # stripped. This to conform to Swift's behavior where account name # entry in Account DBs contain resller_prefix. self._account = account self._container = container self._container_path = \ os.path.join(self._device_path, self._account, self._container) obj = obj.strip(os.path.sep) obj_path, self._obj = os.path.split(obj) if obj_path: self._obj_path = obj_path.strip(os.path.sep) self._datadir = os.path.join(self._container_path, self._obj_path) else: self._obj_path = '' self._datadir = self._container_path if _use_put_mount: self._put_datadir = os.path.join(self._device_path + '_PUT', container, self._obj_path) else: self._put_datadir = self._datadir self._data_file = os.path.join(self._put_datadir, self._obj)
def __init__(self, mgr, device_path, threadpool, partition, account, container, obj): self._mgr = mgr self._device_path = device_path self._threadpool = threadpool or ThreadPool(nthreads=0) self._logger = mgr.logger self._disk_chunk_size = mgr.disk_chunk_size self._bytes_per_sync = mgr.bytes_per_sync self._name = '/' + '/'.join((account, container, obj)) name_hash = hash_path(account, container, obj) self._datadir = join(device_path, storage_directory(DATADIR, partition, name_hash)) self._tmpdir = join(device_path, 'tmp') self._metadata = None self._data_file = None self._fp = None self._quarantined_dir = None
def __init__(self, path, device, partition, account, container, obj, logger, disk_chunk_size=65536, bytes_per_sync=(512 * 1024 * 1024), iter_hook=None, threadpool=None, obj_dir='objects', mount_check=False): if mount_check and not check_mount(path, device): raise DiskFileDeviceUnavailable() self.disk_chunk_size = disk_chunk_size self.bytes_per_sync = bytes_per_sync self.iter_hook = iter_hook self.name = '/' + '/'.join((account, container, obj)) name_hash = hash_path(account, container, obj) self.datadir = join(path, device, storage_directory(obj_dir, partition, name_hash)) self.device_path = join(path, device) self.tmpdir = join(path, device, 'tmp') self.logger = logger self._metadata = None self.data_file = None self._data_file_size = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.suppress_file_closing = False self._verify_close = False self.threadpool = threadpool or ThreadPool(nthreads=0) # FIXME(clayg): this attribute is set after open and affects the # behavior of the class (i.e. public interface) self.keep_cache = False
def __init__(self, mgr, dev_path, threadpool, account, container, obj, uid=DEFAULT_UID, gid=DEFAULT_GID): self._mgr = mgr self._device_path = dev_path self._threadpool = threadpool or ThreadPool(nthreads=0) self._uid = int(uid) self._gid = int(gid) self._is_dir = False self._logger = mgr.logger self._metadata = None self._fd = None # Don't store a value for data_file until we know it exists. self._data_file = None self._container_path = os.path.join(self._device_path, container) obj = obj.strip(os.path.sep) obj_path, self._obj = os.path.split(obj) if obj_path: self._obj_path = obj_path.strip(os.path.sep) self._datadir = os.path.join(self._container_path, self._obj_path) else: self._obj_path = '' self._datadir = self._container_path if _use_put_mount: self._put_datadir = os.path.join(self._device_path + '_PUT', container, self._obj_path) else: self._put_datadir = self._datadir self._data_file = os.path.join(self._put_datadir, self._obj)
def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, bytes_per_sync=(512 * 1024 * 1024), iter_hook=None, threadpool=None, obj_dir='objects', mount_check=False, disallowed_metadata_keys=None): if mount_check and not check_mount(path, device): raise DiskFileDeviceUnavailable() self.disk_chunk_size = disk_chunk_size self.bytes_per_sync = bytes_per_sync self.iter_hook = iter_hook self.name = '/' + '/'.join((account, container, obj)) name_hash = hash_path(account, container, obj) self.datadir = join(path, device, storage_directory(obj_dir, partition, name_hash)) self.device_path = join(path, device) self.tmpdir = join(path, device, 'tmp') self.logger = logger self.disallowed_metadata_keys = disallowed_metadata_keys or [] self.metadata = {} self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.suppress_file_closing = False self.threadpool = threadpool or ThreadPool(nthreads=0) if not exists(self.datadir): return files = sorted(os.listdir(self.datadir), reverse=True) meta_file = None for afile in files: if afile.endswith('.ts'): self.data_file = None with open(join(self.datadir, afile)) as mfp: self.metadata = read_metadata(mfp) self.metadata['deleted'] = True break if afile.endswith('.meta') and not meta_file: meta_file = join(self.datadir, afile) if afile.endswith('.data') and not self.data_file: self.data_file = join(self.datadir, afile) break if not self.data_file: return self.fp = open(self.data_file, 'rb') self.metadata = read_metadata(self.fp) if not keep_data_fp: self.close(verify_file=False) if meta_file: with open(meta_file) as mfp: for key in self.metadata.keys(): if key.lower() not in self.disallowed_metadata_keys: del self.metadata[key] self.metadata.update(read_metadata(mfp)) if 'name' in self.metadata: if self.metadata['name'] != self.name: self.logger.error( _('Client path %(client)s does not match ' 'path stored in object metadata %(meta)s'), { 'client': self.name, 'meta': self.metadata['name'] }) raise DiskFileCollision('Client path does not match path ' 'stored in object metadata')