class ObjectExpirer(SwiftObjectExpirer):
def __init__(self, conf, logger=None, swift=None):
conf_path = conf.get('__file__') or '/etc/swift/object-expirer.conf'
self.devices = conf.get('devices', '/mnt/gluster-object')
# Do not retry DELETEs on getting 404. Hence default is set to 1.
request_tries = int(conf.get('request_tries') or 1)
# Use our extended version of InternalClient
swift = GlusterSwiftInternalClient(
conf_path, 'Gluster Swift Object Expirer', request_tries,
devices=self.devices)
# Let the parent class initialize self.swift
super(ObjectExpirer, self).__init__(conf, logger=logger, swift=swift)
self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip()
if not self.reseller_prefix.endswith('_'):
self.reseller_prefix = self.reseller_prefix + '_'
# nthread=0 is intentional. This ensures that no green pool is
# used. Call to force_run_in_thread() will ensure that the method
# passed as arg is run in a real external thread using eventlet.tpool
# which has a threadpool of 20 threads (default)
self.threadpool = ThreadPool(nthreads=0)
def pop_queue(self, container, obj):
"""
In Swift, this method removes tracker object entry directly from
container database. In gluster-swift, this method deletes tracker
object directly from filesystem.
"""
container_path = os.path.join(self.devices,
self.expiring_objects_account,
container)
self.threadpool.force_run_in_thread(delete_tracker_object,
container_path, obj)
def delete_actual_object(self, actual_obj, timestamp):
"""
Swift's expirer will re-attempt expiring if the source object is not
available (404 or ANY other error) up to self.reclaim_age seconds
before it gives up and deletes the entry in the queue.
Don't do this in gluster-swift. GlusterFS isn't eventually consistent
and has no concept of hand-off nodes. If actual data object doesn't
exist (404), remove tracker object from the queue (filesystem).
However if DELETE fails due a reason other than 404, do not remove
tracker object yet, follow Swift's behaviour of waiting till
self.reclaim_age seconds.
This method is just a wrapper around parent class's method. All this
wrapper does is ignore 404 failures.
"""
try:
super(ObjectExpirer, self).delete_actual_object(
actual_obj, timestamp)
except UnexpectedResponse as err:
if err.resp.status_int != HTTP_NOT_FOUND:
raise
def __init__(self, conf, logger):
self.logger = logger
self.devices = conf.get('devices', '/srv/node/')
self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536))
self.keep_cache_size = int(conf.get('keep_cache_size', 5242880))
self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
threads_per_disk = int(conf.get('threads_per_disk', '0'))
self.threadpools = defaultdict(
lambda: ThreadPool(nthreads=threads_per_disk))
def __init__(self, conf, logger):
self.logger = logger
self.disk_chunk_size = int(
conf.get('disk_chunk_size', DEFAULT_DISK_CHUNK_SIZE))
self.keep_cache_size = int(
conf.get('keep_cache_size', DEFAULT_KEEP_CACHE_SIZE))
self.bytes_per_sync = int(conf.get('mb_per_sync',
DEFAULT_MB_PER_SYNC)) * 1024 * 1024
self.devices = conf.get('devices', '/srv/node/')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
threads_per_disk = int(conf.get('threads_per_disk', '0'))
self.threadpools = defaultdict(
lambda: ThreadPool(nthreads=threads_per_disk))
def __init__(self, conf):
"""
Creates a new WSGI application for the Swift Object Server. An
example configuration is given at
<source-dir>/etc/object-server.conf-sample or
/etc/swift/object-server.conf-sample.
"""
self.logger = get_logger(conf, log_route='object-server')
self.devices = conf.get('devices', '/srv/node/')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.node_timeout = int(conf.get('node_timeout', 3))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536))
self.network_chunk_size = int(conf.get('network_chunk_size', 65536))
self.keep_cache_size = int(conf.get('keep_cache_size', 5242880))
self.keep_cache_private = \
config_true_value(conf.get('keep_cache_private', 'false'))
self.log_requests = config_true_value(conf.get('log_requests', 'true'))
self.max_upload_time = int(conf.get('max_upload_time', 86400))
self.slow = int(conf.get('slow', 0))
self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024
replication_server = conf.get('replication_server', None)
if replication_server is not None:
replication_server = config_true_value(replication_server)
self.replication_server = replication_server
self.threads_per_disk = int(conf.get('threads_per_disk', '0'))
self.threadpools = defaultdict(
lambda: ThreadPool(nthreads=self.threads_per_disk))
default_allowed_headers = '''
content-disposition,
content-encoding,
x-delete-at,
x-object-manifest,
x-static-large-object,
'''
extra_allowed_headers = [
header.strip().lower() for header in conf.get(
'allowed_headers', default_allowed_headers).split(',')
if header.strip()
]
self.allowed_headers = set()
for header in extra_allowed_headers:
if header not in DATAFILE_SYSTEM_META:
self.allowed_headers.add(header)
self.expiring_objects_account = \
(conf.get('auto_create_account_prefix') or '.') + \
'expiring_objects'
self.expiring_objects_container_divisor = \
int(conf.get('expiring_objects_container_divisor') or 86400)
def __init__(self,
path,
device,
partition,
account,
container,
obj,
logger,
keep_data_fp=False,
disk_chunk_size=65536,
bytes_per_sync=(512 * 1024 * 1024),
iter_hook=None,
threadpool=None,
obj_dir='objects',
mount_check=False):
if mount_check and not check_mount(path, device):
raise DiskFileDeviceUnavailable()
self.disk_chunk_size = disk_chunk_size
self.bytes_per_sync = bytes_per_sync
self.iter_hook = iter_hook
self.name = '/' + '/'.join((account, container, obj))
name_hash = hash_path(account, container, obj)
self.datadir = join(path, device,
storage_directory(obj_dir, partition, name_hash))
self.device_path = join(path, device)
self.tmpdir = join(path, device, 'tmp')
self.logger = logger
self.metadata = {}
self.data_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.suppress_file_closing = False
self.threadpool = threadpool or ThreadPool(nthreads=0)
data_file, meta_file, ts_file = self._get_ondisk_file()
if not data_file:
if ts_file:
self._construct_from_ts_file(ts_file)
else:
fp = self._construct_from_data_file(data_file, meta_file)
if keep_data_fp:
self.fp = fp
else:
fp.close()
def __init__(self,
mgr,
dev_path,
threadpool,
partition,
account=None,
container=None,
obj=None,
policy=None,
uid=DEFAULT_UID,
gid=DEFAULT_GID,
**kwargs):
# Variables partition and policy is currently unused.
self._mgr = mgr
self._device_path = dev_path
self._threadpool = threadpool or ThreadPool(nthreads=0)
self._uid = int(uid)
self._gid = int(gid)
self._is_dir = False
self._metadata = None
self._fd = None
# This fd attribute is not used in PUT path. fd used in PUT path
# is encapsulated inside DiskFileWriter object.
self._stat = None
# Don't store a value for data_file until we know it exists.
self._data_file = None
# Account name contains resller_prefix which is retained and not
# stripped. This to conform to Swift's behavior where account name
# entry in Account DBs contain resller_prefix.
self._account = account
self._container = container
self._container_path = \
os.path.join(self._device_path, self._account, self._container)
obj = obj.strip(os.path.sep)
obj_path, self._obj = os.path.split(obj)
if obj_path:
self._obj_path = obj_path.strip(os.path.sep)
self._put_datadir = os.path.join(self._container_path,
self._obj_path)
else:
self._obj_path = ''
self._put_datadir = self._container_path
self._data_file = os.path.join(self._put_datadir, self._obj)
self._disk_file_open = False
def __init__(self,
mgr,
dev_path,
threadpool,
partition,
account=None,
container=None,
obj=None,
policy_idx=0,
uid=DEFAULT_UID,
gid=DEFAULT_GID):
# Variables partition and policy_idx is currently unused.
self._mgr = mgr
self._device_path = dev_path
self._threadpool = threadpool or ThreadPool(nthreads=0)
self._uid = int(uid)
self._gid = int(gid)
self._is_dir = False
self._metadata = None
self._fd = None
# Don't store a value for data_file until we know it exists.
self._data_file = None
# Account name contains resller_prefix which is retained and not
# stripped. This to conform to Swift's behavior where account name
# entry in Account DBs contain resller_prefix.
self._account = account
self._container = container
self._container_path = \
os.path.join(self._device_path, self._account, self._container)
obj = obj.strip(os.path.sep)
obj_path, self._obj = os.path.split(obj)
if obj_path:
self._obj_path = obj_path.strip(os.path.sep)
self._datadir = os.path.join(self._container_path, self._obj_path)
else:
self._obj_path = ''
self._datadir = self._container_path
if _use_put_mount:
self._put_datadir = os.path.join(self._device_path + '_PUT',
container, self._obj_path)
else:
self._put_datadir = self._datadir
self._data_file = os.path.join(self._put_datadir, self._obj)
def __init__(self, mgr, device_path, threadpool, partition, account,
container, obj):
self._mgr = mgr
self._device_path = device_path
self._threadpool = threadpool or ThreadPool(nthreads=0)
self._logger = mgr.logger
self._disk_chunk_size = mgr.disk_chunk_size
self._bytes_per_sync = mgr.bytes_per_sync
self._name = '/' + '/'.join((account, container, obj))
name_hash = hash_path(account, container, obj)
self._datadir = join(device_path,
storage_directory(DATADIR, partition, name_hash))
self._tmpdir = join(device_path, 'tmp')
self._metadata = None
self._data_file = None
self._fp = None
self._quarantined_dir = None
def __init__(self,
path,
device,
partition,
account,
container,
obj,
logger,
disk_chunk_size=65536,
bytes_per_sync=(512 * 1024 * 1024),
iter_hook=None,
threadpool=None,
obj_dir='objects',
mount_check=False):
if mount_check and not check_mount(path, device):
raise DiskFileDeviceUnavailable()
self.disk_chunk_size = disk_chunk_size
self.bytes_per_sync = bytes_per_sync
self.iter_hook = iter_hook
self.name = '/' + '/'.join((account, container, obj))
name_hash = hash_path(account, container, obj)
self.datadir = join(path, device,
storage_directory(obj_dir, partition, name_hash))
self.device_path = join(path, device)
self.tmpdir = join(path, device, 'tmp')
self.logger = logger
self._metadata = None
self.data_file = None
self._data_file_size = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.suppress_file_closing = False
self._verify_close = False
self.threadpool = threadpool or ThreadPool(nthreads=0)
# FIXME(clayg): this attribute is set after open and affects the
# behavior of the class (i.e. public interface)
self.keep_cache = False
def __init__(self, conf, logger=None, swift=None):
conf_path = conf.get('__file__') or '/etc/swift/object-expirer.conf'
self.devices = conf.get('devices', '/mnt/gluster-object')
# Do not retry DELETEs on getting 404. Hence default is set to 1.
request_tries = int(conf.get('request_tries') or 1)
# Use our extended version of InternalClient
swift = GlusterSwiftInternalClient(
conf_path, 'Gluster Swift Object Expirer', request_tries,
devices=self.devices)
# Let the parent class initialize self.swift
super(ObjectExpirer, self).__init__(conf, logger=logger, swift=swift)
self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip()
if not self.reseller_prefix.endswith('_'):
self.reseller_prefix = self.reseller_prefix + '_'
# nthread=0 is intentional. This ensures that no green pool is
# used. Call to force_run_in_thread() will ensure that the method
# passed as arg is run in a real external thread using eventlet.tpool
# which has a threadpool of 20 threads (default)
self.threadpool = ThreadPool(nthreads=0)
def __init__(self,
mgr,
dev_path,
threadpool,
account,
container,
obj,
uid=DEFAULT_UID,
gid=DEFAULT_GID):
self._mgr = mgr
self._device_path = dev_path
self._threadpool = threadpool or ThreadPool(nthreads=0)
self._uid = int(uid)
self._gid = int(gid)
self._is_dir = False
self._logger = mgr.logger
self._metadata = None
self._fd = None
# Don't store a value for data_file until we know it exists.
self._data_file = None
self._container_path = os.path.join(self._device_path, container)
obj = obj.strip(os.path.sep)
obj_path, self._obj = os.path.split(obj)
if obj_path:
self._obj_path = obj_path.strip(os.path.sep)
self._datadir = os.path.join(self._container_path, self._obj_path)
else:
self._obj_path = ''
self._datadir = self._container_path
if _use_put_mount:
self._put_datadir = os.path.join(self._device_path + '_PUT',
container, self._obj_path)
else:
self._put_datadir = self._datadir
self._data_file = os.path.join(self._put_datadir, self._obj)
def __init__(self,
path,
device,
partition,
account,
container,
obj,
logger,
keep_data_fp=False,
disk_chunk_size=65536,
bytes_per_sync=(512 * 1024 * 1024),
iter_hook=None,
threadpool=None,
obj_dir='objects',
mount_check=False,
disallowed_metadata_keys=None):
if mount_check and not check_mount(path, device):
raise DiskFileDeviceUnavailable()
self.disk_chunk_size = disk_chunk_size
self.bytes_per_sync = bytes_per_sync
self.iter_hook = iter_hook
self.name = '/' + '/'.join((account, container, obj))
name_hash = hash_path(account, container, obj)
self.datadir = join(path, device,
storage_directory(obj_dir, partition, name_hash))
self.device_path = join(path, device)
self.tmpdir = join(path, device, 'tmp')
self.logger = logger
self.disallowed_metadata_keys = disallowed_metadata_keys or []
self.metadata = {}
self.data_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.suppress_file_closing = False
self.threadpool = threadpool or ThreadPool(nthreads=0)
if not exists(self.datadir):
return
files = sorted(os.listdir(self.datadir), reverse=True)
meta_file = None
for afile in files:
if afile.endswith('.ts'):
self.data_file = None
with open(join(self.datadir, afile)) as mfp:
self.metadata = read_metadata(mfp)
self.metadata['deleted'] = True
break
if afile.endswith('.meta') and not meta_file:
meta_file = join(self.datadir, afile)
if afile.endswith('.data') and not self.data_file:
self.data_file = join(self.datadir, afile)
break
if not self.data_file:
return
self.fp = open(self.data_file, 'rb')
self.metadata = read_metadata(self.fp)
if not keep_data_fp:
self.close(verify_file=False)
if meta_file:
with open(meta_file) as mfp:
for key in self.metadata.keys():
if key.lower() not in self.disallowed_metadata_keys:
del self.metadata[key]
self.metadata.update(read_metadata(mfp))
if 'name' in self.metadata:
if self.metadata['name'] != self.name:
self.logger.error(
_('Client path %(client)s does not match '
'path stored in object metadata %(meta)s'), {
'client': self.name,
'meta': self.metadata['name']
})
raise DiskFileCollision('Client path does not match path '
'stored in object metadata')
