class InternalBrainClient(object):
def __init__(self, conf_file, account='AUTH_test'):
self.swift = InternalClient(conf_file, 'probe-test', 3)
self.account = account
@translate_client_exception
def put_container(self, container_name, headers):
return self.swift.create_container(self.account, container_name,
headers=headers)
@translate_client_exception
def post_container(self, container_name, headers):
return self.swift.set_container_metadata(self.account, container_name,
headers)
@translate_client_exception
def delete_container(self, container_name):
return self.swift.delete_container(self.account, container_name)
def parse_qs(self, query_string):
if query_string is not None:
return {k: v[-1] for k, v in parse_qs(query_string).items()}
@translate_client_exception
def put_object(self, container_name, object_name, headers, contents,
query_string=None):
return self.swift.upload_object(StringIO(contents), self.account,
container_name, object_name,
headers=headers,
params=self.parse_qs(query_string))
@translate_client_exception
def delete_object(self, container_name, object_name):
return self.swift.delete_object(
self.account, container_name, object_name)
@translate_client_exception
def head_object(self, container_name, object_name):
return self.swift.get_object_metadata(
self.account, container_name, object_name)
@translate_client_exception
def get_object(self, container_name, object_name, query_string=None):
status, headers, resp_iter = self.swift.get_object(
self.account, container_name, object_name,
params=self.parse_qs(query_string))
return headers, ''.join(resp_iter)
class ContainerReconciler(Daemon):
"""
Move objects that are in the wrong storage policy.
"""
def __init__(self, conf):
self.conf = conf
# This option defines how long an un-processable misplaced object
# marker will be retried before it is abandoned. It is not coupled
# with the tombstone reclaim age in the consistency engine.
self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7))
self.interval = int(conf.get('interval', 30))
conf_path = conf.get('__file__') or \
'/etc/swift/container-reconciler.conf'
self.logger = get_logger(conf, log_route='container-reconciler')
request_tries = int(conf.get('request_tries') or 3)
self.swift = InternalClient(conf_path, 'Swift Container Reconciler',
request_tries)
self.stats = defaultdict(int)
self.last_stat_time = time.time()
def stats_log(self, metric, msg, *args, **kwargs):
"""
Update stats tracking for metric and emit log message.
"""
level = kwargs.pop('level', logging.DEBUG)
log_message = '%s: ' % metric + msg
self.logger.log(level, log_message, *args, **kwargs)
self.stats[metric] += 1
def log_stats(self, force=False):
"""
Dump stats to logger, noop when stats have been already been
logged in the last minute.
"""
now = time.time()
should_log = force or (now - self.last_stat_time > 60)
if should_log:
self.last_stat_time = now
self.logger.info('Reconciler Stats: %r', dict(**self.stats))
def pop_queue(self, container, obj, q_ts, q_record):
"""
Issue a delete object request to the container for the misplaced
object queue entry.
:param container: the misplaced objects container
:param obj: the name of the misplaced object
:param q_ts: the timestamp of the misplaced object
:param q_record: the timestamp of the queue entry
N.B. q_ts will normally be the same time as q_record except when
an object was manually re-enqued.
"""
q_path = '/%s/%s/%s' % (MISPLACED_OBJECTS_ACCOUNT, container, obj)
x_timestamp = slightly_later_timestamp(max(q_record, q_ts))
self.stats_log('pop_queue', 'remove %r (%f) from the queue (%s)',
q_path, q_ts, x_timestamp)
headers = {'X-Timestamp': x_timestamp}
direct_delete_container_entry(self.swift.container_ring,
MISPLACED_OBJECTS_ACCOUNT,
container,
obj,
headers=headers)
def throw_tombstones(self, account, container, obj, timestamp,
policy_index, path):
"""
Issue a delete object request to the given storage_policy.
:param account: the account name
:param container: the container name
:param obj: the object name
:param timestamp: the timestamp of the object to delete
:param policy_index: the policy index to direct the request
:param path: the path to be used for logging
"""
x_timestamp = slightly_later_timestamp(timestamp)
self.stats_log('cleanup_attempt', '%r (%f) from policy_index '
'%s (%s) will be deleted', path, timestamp,
policy_index, x_timestamp)
headers = {
'X-Timestamp': x_timestamp,
'X-Backend-Storage-Policy-Index': policy_index,
}
success = False
try:
self.swift.delete_object(account,
container,
obj,
acceptable_statuses=(2, 404),
headers=headers)
except UnexpectedResponse as err:
self.stats_log(
'cleanup_failed', '%r (%f) was not cleaned up '
'in storage_policy %s (%s)', path, timestamp, policy_index,
err)
else:
success = True
self.stats_log(
'cleanup_success', '%r (%f) was successfully '
'removed from policy_index %s', path, timestamp, policy_index)
return success
def _reconcile_object(self, account, container, obj, q_policy_index, q_ts,
q_op, path, **kwargs):
"""
Perform object reconciliation.
:param account: the account name of the misplaced object
:param container: the container name of the misplaced object
:param obj: the object name
:param q_policy_index: the policy index of the source indicated by the
queue entry.
:param q_ts: the timestamp of the misplaced object
:param q_op: the operation of the misplaced request
:param path: the full path of the misplaced object for logging
:returns: True to indicate the request is fully processed
successfully, otherwise False.
"""
container_policy_index = direct_get_container_policy_index(
self.swift.container_ring, account, container)
if container_policy_index is None:
self.stats_log(
'unavailable_container', '%r (%f) unable to '
'determine the destination policy_index', path, q_ts)
return False
if container_policy_index == q_policy_index:
self.stats_log(
'noop_object', '%r (%f) container policy_index '
'%s matches queue policy index %s', path, q_ts,
container_policy_index, q_policy_index)
return True
# check if object exists in the destination already
self.logger.debug(
'checking for %r (%f) in destination '
'policy_index %s', path, q_ts, container_policy_index)
headers = {'X-Backend-Storage-Policy-Index': container_policy_index}
dest_obj = self.swift.get_object_metadata(account,
container,
obj,
headers=headers,
acceptable_statuses=(2, 4))
dest_ts = Timestamp(dest_obj.get('x-backend-timestamp', 0))
if dest_ts >= q_ts:
self.stats_log(
'found_object', '%r (%f) in policy_index %s '
'is newer than queue (%f)', path, dest_ts,
container_policy_index, q_ts)
return self.throw_tombstones(account, container, obj, q_ts,
q_policy_index, path)
# object is misplaced
self.stats_log(
'misplaced_object', '%r (%f) in policy_index %s '
'should be in policy_index %s', path, q_ts, q_policy_index,
container_policy_index)
# fetch object from the source location
self.logger.debug('fetching %r (%f) from storage policy %s', path,
q_ts, q_policy_index)
headers = {'X-Backend-Storage-Policy-Index': q_policy_index}
try:
source_obj_status, source_obj_info, source_obj_iter = \
self.swift.get_object(account, container, obj,
headers=headers,
acceptable_statuses=(2, 4))
except UnexpectedResponse as err:
source_obj_status = err.resp.status_int
source_obj_info = {}
source_obj_iter = None
source_ts = Timestamp(source_obj_info.get('x-backend-timestamp', 0))
if source_obj_status == 404 and q_op == 'DELETE':
return self.ensure_tombstone_in_right_location(
q_policy_index, account, container, obj, q_ts, path,
container_policy_index, source_ts)
else:
return self.ensure_object_in_right_location(
q_policy_index, account, container, obj, q_ts, path,
container_policy_index, source_ts, source_obj_status,
source_obj_info, source_obj_iter)
def ensure_object_in_right_location(self, q_policy_index, account,
container, obj, q_ts, path,
container_policy_index, source_ts,
source_obj_status, source_obj_info,
source_obj_iter, **kwargs):
"""
Validate source object will satisfy the misplaced object queue entry
and move to destination.
:param q_policy_index: the policy_index for the source object
:param account: the account name of the misplaced object
:param container: the container name of the misplaced object
:param obj: the name of the misplaced object
:param q_ts: the timestamp of the misplaced object
:param path: the full path of the misplaced object for logging
:param container_policy_index: the policy_index of the destination
:param source_ts: the timestamp of the source object
:param source_obj_status: the HTTP status source object request
:param source_obj_info: the HTTP headers of the source object request
:param source_obj_iter: the body iter of the source object request
"""
if source_obj_status // 100 != 2 or source_ts < q_ts:
if q_ts < time.time() - self.reclaim_age:
# it's old and there are no tombstones or anything; give up
self.stats_log('lost_source', '%r (%s) was not available in '
'policy_index %s and has expired',
path,
q_ts.internal,
q_policy_index,
level=logging.CRITICAL)
return True
# the source object is unavailable or older than the queue
# entry; a version that will satisfy the queue entry hopefully
# exists somewhere in the cluster, so wait and try again
self.stats_log('unavailable_source', '%r (%s) in '
'policy_index %s responded %s (%s)',
path,
q_ts.internal,
q_policy_index,
source_obj_status,
source_ts.internal,
level=logging.WARNING)
return False
# optimistically move any source with a timestamp >= q_ts
ts = max(Timestamp(source_ts), q_ts)
# move the object
put_timestamp = slightly_later_timestamp(ts, offset=2)
self.stats_log(
'copy_attempt', '%r (%f) in policy_index %s will be '
'moved to policy_index %s (%s)', path, source_ts, q_policy_index,
container_policy_index, put_timestamp)
headers = source_obj_info.copy()
headers['X-Backend-Storage-Policy-Index'] = container_policy_index
headers['X-Timestamp'] = put_timestamp
try:
self.swift.upload_object(FileLikeIter(source_obj_iter),
account,
container,
obj,
headers=headers)
except UnexpectedResponse as err:
self.stats_log('copy_failed', 'upload %r (%f) from '
'policy_index %s to policy_index %s '
'returned %s',
path,
source_ts,
q_policy_index,
container_policy_index,
err,
level=logging.WARNING)
return False
except: # noqa
self.stats_log('unhandled_error', 'unable to upload %r (%f) '
'from policy_index %s to policy_index %s ',
path,
source_ts,
q_policy_index,
container_policy_index,
level=logging.ERROR,
exc_info=True)
return False
self.stats_log(
'copy_success', '%r (%f) moved from policy_index %s '
'to policy_index %s (%s)', path, source_ts, q_policy_index,
container_policy_index, put_timestamp)
return self.throw_tombstones(account, container, obj, q_ts,
q_policy_index, path)
def ensure_tombstone_in_right_location(self, q_policy_index, account,
container, obj, q_ts, path,
container_policy_index, source_ts,
**kwargs):
"""
Issue a DELETE request against the destination to match the
misplaced DELETE against the source.
"""
delete_timestamp = slightly_later_timestamp(q_ts, offset=2)
self.stats_log(
'delete_attempt', '%r (%f) in policy_index %s '
'will be deleted from policy_index %s (%s)', path, source_ts,
q_policy_index, container_policy_index, delete_timestamp)
headers = {
'X-Backend-Storage-Policy-Index': container_policy_index,
'X-Timestamp': delete_timestamp,
}
try:
self.swift.delete_object(account, container, obj, headers=headers)
except UnexpectedResponse as err:
self.stats_log('delete_failed', 'delete %r (%f) from '
'policy_index %s (%s) returned %s',
path,
source_ts,
container_policy_index,
delete_timestamp,
err,
level=logging.WARNING)
return False
except: # noqa
self.stats_log('unhandled_error', 'unable to delete %r (%f) '
'from policy_index %s (%s)',
path,
source_ts,
container_policy_index,
delete_timestamp,
level=logging.ERROR,
exc_info=True)
return False
self.stats_log('delete_success', '%r (%f) deleted from '
'policy_index %s (%s)',
path,
source_ts,
container_policy_index,
delete_timestamp,
level=logging.INFO)
return self.throw_tombstones(account, container, obj, q_ts,
q_policy_index, path)
def reconcile_object(self, info):
"""
Process a possibly misplaced object write request. Determine correct
destination storage policy by checking with primary containers. Check
source and destination, copying or deleting into destination and
cleaning up the source as needed.
This method wraps _reconcile_object for exception handling.
:param info: a queue entry dict
:returns: True to indicate the request is fully processed
successfully, otherwise False.
"""
self.logger.debug(
'checking placement for %r (%f) '
'in policy_index %s', info['path'], info['q_ts'],
info['q_policy_index'])
success = False
try:
success = self._reconcile_object(**info)
except: # noqa
self.logger.exception(
'Unhandled Exception trying to '
'reconcile %r (%f) in policy_index %s', info['path'],
info['q_ts'], info['q_policy_index'])
if success:
metric = 'success'
msg = 'was handled successfully'
else:
metric = 'retry'
msg = 'must be retried'
msg = '%(path)r (%(q_ts)f) in policy_index %(q_policy_index)s ' + msg
self.stats_log(metric, msg, info, level=logging.INFO)
self.log_stats()
return success
def _iter_containers(self):
"""
Generate a list of containers to process.
"""
# hit most recent container first instead of waiting on the updaters
current_container = get_reconciler_container_name(time.time())
yield current_container
container_gen = self.swift.iter_containers(MISPLACED_OBJECTS_ACCOUNT)
self.logger.debug('looking for containers in %s',
MISPLACED_OBJECTS_ACCOUNT)
while True:
one_page = None
try:
one_page = list(
itertools.islice(container_gen,
constraints.CONTAINER_LISTING_LIMIT))
except UnexpectedResponse as err:
self.logger.error(
'Error listing containers in '
'account %s (%s)', MISPLACED_OBJECTS_ACCOUNT, err)
if not one_page:
# don't generally expect more than one page
break
# reversed order since we expect older containers to be empty
for c in reversed(one_page):
# encoding here is defensive
container = c['name'].encode('utf8')
if container == current_container:
continue # we've already hit this one this pass
yield container
def _iter_objects(self, container):
"""
Generate a list of objects to process.
:param container: the name of the container to process
If the given container is empty and older than reclaim_age this
processor will attempt to reap it.
"""
self.logger.debug('looking for objects in %s', container)
found_obj = False
try:
for raw_obj in self.swift.iter_objects(MISPLACED_OBJECTS_ACCOUNT,
container):
found_obj = True
yield raw_obj
except UnexpectedResponse as err:
self.logger.error('Error listing objects in container %s (%s)',
container, err)
if float(container) < time.time() - self.reclaim_age and \
not found_obj:
# Try to delete old empty containers so the queue doesn't
# grow without bound. It's ok if there's a conflict.
self.swift.delete_container(MISPLACED_OBJECTS_ACCOUNT,
container,
acceptable_statuses=(2, 404, 409, 412))
def reconcile(self):
"""
Main entry point for processing misplaced objects.
Iterate over all queue entries and delegate to reconcile_object.
"""
self.logger.debug('pulling items from the queue')
for container in self._iter_containers():
for raw_obj in self._iter_objects(container):
try:
obj_info = parse_raw_obj(raw_obj)
except Exception:
self.stats_log('invalid_record',
'invalid queue record: %r',
raw_obj,
level=logging.ERROR,
exc_info=True)
continue
finished = self.reconcile_object(obj_info)
if finished:
self.pop_queue(container, raw_obj['name'],
obj_info['q_ts'], obj_info['q_record'])
self.log_stats()
self.logger.debug('finished container %s', container)
def run_once(self, *args, **kwargs):
"""
Process every entry in the queue.
"""
try:
self.reconcile()
except: # noqa
self.logger.exception('Unhandled Exception trying to reconcile')
self.log_stats(force=True)
def run_forever(self, *args, **kwargs):
while True:
self.run_once(*args, **kwargs)
self.stats = defaultdict(int)
self.logger.info('sleeping between intervals (%ss)', self.interval)
time.sleep(self.interval)
class ContainerReconciler(Daemon):
"""
Move objects that are in the wrong storage policy.
"""
def __init__(self, conf):
self.conf = conf
self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7))
self.interval = int(conf.get('interval', 30))
conf_path = conf.get('__file__') or \
'/etc/swift/container-reconciler.conf'
self.logger = get_logger(conf, log_route='container-reconciler')
request_tries = int(conf.get('request_tries') or 3)
self.swift = InternalClient(conf_path,
'Swift Container Reconciler',
request_tries)
self.stats = defaultdict(int)
self.last_stat_time = time.time()
def stats_log(self, metric, msg, *args, **kwargs):
"""
Update stats tracking for metric and emit log message.
"""
level = kwargs.pop('level', logging.DEBUG)
log_message = '%s: ' % metric + msg
self.logger.log(level, log_message, *args, **kwargs)
self.stats[metric] += 1
def log_stats(self, force=False):
"""
Dump stats to logger, noop when stats have been already been
logged in the last minute.
"""
now = time.time()
should_log = force or (now - self.last_stat_time > 60)
if should_log:
self.last_stat_time = now
self.logger.info('Reconciler Stats: %r', dict(**self.stats))
def pop_queue(self, container, obj, q_ts, q_record):
"""
Issue a delete object request to the container for the misplaced
object queue entry.
:param container: the misplaced objects container
:param q_ts: the timestamp of the misplaced object
:param q_record: the timestamp of the queue entry
N.B. q_ts will normally be the same time as q_record except when
an object was manually re-enqued.
"""
q_path = '/%s/%s/%s' % (MISPLACED_OBJECTS_ACCOUNT, container, obj)
x_timestamp = slightly_later_timestamp(max(q_record, q_ts))
self.stats_log('pop_queue', 'remove %r (%f) from the queue (%s)',
q_path, q_ts, x_timestamp)
headers = {'X-Timestamp': x_timestamp}
direct_delete_container_entry(
self.swift.container_ring, MISPLACED_OBJECTS_ACCOUNT,
container, obj, headers=headers)
def throw_tombstones(self, account, container, obj, timestamp,
policy_index, path):
"""
Issue a delete object request to the given storage_policy.
:param account: the account name
:param container: the container name
:param account: the object name
:param timestamp: the timestamp of the object to delete
:param policy_index: the policy index to direct the request
:param path: the path to be used for logging
"""
x_timestamp = slightly_later_timestamp(timestamp)
self.stats_log('cleanup_attempt', '%r (%f) from policy_index '
'%s (%s) will be deleted',
path, timestamp, policy_index, x_timestamp)
headers = {
'X-Timestamp': x_timestamp,
'X-Backend-Storage-Policy-Index': policy_index,
}
success = False
try:
self.swift.delete_object(account, container, obj,
acceptable_statuses=(2, 404),
headers=headers)
except UnexpectedResponse as err:
self.stats_log('cleanup_failed', '%r (%f) was not cleaned up '
'in storage_policy %s (%s)', path, timestamp,
policy_index, err)
else:
success = True
self.stats_log('cleanup_success', '%r (%f) was successfully '
'removed from policy_index %s', path, timestamp,
policy_index)
return success
def _reconcile_object(self, account, container, obj, q_policy_index, q_ts,
q_op, path, **kwargs):
"""
Perform object reconciliation.
:param account: the account name of the misplaced object
:param container: the container name of the misplaced object
:param obj: the object name
:param q_policy_index: the policy index of the source indicated by the
queue entry.
:param q_ts: the timestamp of the misplaced object
:param q_op: the operation of the misplaced request
:param path: the full path of the misplaced object for logging
:returns: True to indicate the request is fully processed
successfully, otherwise False.
"""
container_policy_index = direct_get_container_policy_index(
self.swift.container_ring, account, container)
if container_policy_index is None:
self.stats_log('unavailable_container', '%r (%f) unable to '
'determine the destination policy_index',
path, q_ts)
return False
if container_policy_index == q_policy_index:
self.stats_log('noop_object', '%r (%f) container policy_index '
'%s matches queue policy index %s', path, q_ts,
container_policy_index, q_policy_index)
return True
# check if object exists in the destination already
self.logger.debug('checking for %r (%f) in destination '
'policy_index %s', path, q_ts,
container_policy_index)
headers = {
'X-Backend-Storage-Policy-Index': container_policy_index}
dest_obj = self.swift.get_object_metadata(account, container, obj,
headers=headers,
acceptable_statuses=(2, 4))
dest_ts = Timestamp(dest_obj.get('x-backend-timestamp', 0))
if dest_ts >= q_ts:
self.stats_log('found_object', '%r (%f) in policy_index %s '
'is newer than queue (%f)', path, dest_ts,
container_policy_index, q_ts)
return self.throw_tombstones(account, container, obj, q_ts,
q_policy_index, path)
# object is misplaced
self.stats_log('misplaced_object', '%r (%f) in policy_index %s '
'should be in policy_index %s', path, q_ts,
q_policy_index, container_policy_index)
# fetch object from the source location
self.logger.debug('fetching %r (%f) from storage policy %s', path,
q_ts, q_policy_index)
headers = {
'X-Backend-Storage-Policy-Index': q_policy_index}
try:
source_obj_status, source_obj_info, source_obj_iter = \
self.swift.get_object(account, container, obj,
headers=headers,
acceptable_statuses=(2, 4))
except UnexpectedResponse as err:
source_obj_status = err.resp.status_int
source_obj_info = {}
source_obj_iter = None
source_ts = Timestamp(source_obj_info.get('x-backend-timestamp', 0))
if source_obj_status == 404 and q_op == 'DELETE':
return self.ensure_tombstone_in_right_location(
q_policy_index, account, container, obj, q_ts, path,
container_policy_index, source_ts)
else:
return self.ensure_object_in_right_location(
q_policy_index, account, container, obj, q_ts, path,
container_policy_index, source_ts, source_obj_status,
source_obj_info, source_obj_iter)
def ensure_object_in_right_location(self, q_policy_index, account,
container, obj, q_ts, path,
container_policy_index, source_ts,
source_obj_status, source_obj_info,
source_obj_iter, **kwargs):
"""
Validate source object will satisfy the misplaced object queue entry
and move to destination.
:param q_policy_index: the policy_index for the source object
:param account: the account name of the misplaced object
:param container: the container name of the misplaced object
:param obj: the name of the misplaced object
:param q_ts: the timestamp of the misplaced object
:param path: the full path of the misplaced object for logging
:param container_policy_index: the policy_index of the destination
:param source_ts: the timestamp of the source object
:param source_obj_status: the HTTP status source object request
:param source_obj_info: the HTTP headers of the source object request
:param source_obj_iter: the body iter of the source object request
"""
if source_obj_status // 100 != 2 or source_ts < q_ts:
if q_ts < time.time() - self.reclaim_age:
# it's old and there are no tombstones or anything; give up
self.stats_log('lost_source', '%r (%s) was not available in '
'policy_index %s and has expired', path,
q_ts.internal, q_policy_index,
level=logging.CRITICAL)
return True
# the source object is unavailable or older than the queue
# entry; a version that will satisfy the queue entry hopefully
# exists somewhere in the cluster, so wait and try again
self.stats_log('unavailable_source', '%r (%s) in '
'policy_index %s responded %s (%s)', path,
q_ts.internal, q_policy_index, source_obj_status,
source_ts.internal, level=logging.WARNING)
return False
# optimistically move any source with a timestamp >= q_ts
ts = max(Timestamp(source_ts), q_ts)
# move the object
put_timestamp = slightly_later_timestamp(ts, offset=2)
self.stats_log('copy_attempt', '%r (%f) in policy_index %s will be '
'moved to policy_index %s (%s)', path, source_ts,
q_policy_index, container_policy_index, put_timestamp)
headers = source_obj_info.copy()
headers['X-Backend-Storage-Policy-Index'] = container_policy_index
headers['X-Timestamp'] = put_timestamp
try:
self.swift.upload_object(
FileLikeIter(source_obj_iter), account, container, obj,
headers=headers)
except UnexpectedResponse as err:
self.stats_log('copy_failed', 'upload %r (%f) from '
'policy_index %s to policy_index %s '
'returned %s', path, source_ts, q_policy_index,
container_policy_index, err, level=logging.WARNING)
return False
except: # noqa
self.stats_log('unhandled_error', 'unable to upload %r (%f) '
'from policy_index %s to policy_index %s ', path,
source_ts, q_policy_index, container_policy_index,
level=logging.ERROR, exc_info=True)
return False
self.stats_log('copy_success', '%r (%f) moved from policy_index %s '
'to policy_index %s (%s)', path, source_ts,
q_policy_index, container_policy_index, put_timestamp)
return self.throw_tombstones(account, container, obj, q_ts,
q_policy_index, path)
def ensure_tombstone_in_right_location(self, q_policy_index, account,
container, obj, q_ts, path,
container_policy_index, source_ts,
**kwargs):
"""
Issue a DELETE request against the destination to match the
misplaced DELETE against the source.
"""
delete_timestamp = slightly_later_timestamp(q_ts, offset=2)
self.stats_log('delete_attempt', '%r (%f) in policy_index %s '
'will be deleted from policy_index %s (%s)', path,
source_ts, q_policy_index, container_policy_index,
delete_timestamp)
headers = {
'X-Backend-Storage-Policy-Index': container_policy_index,
'X-Timestamp': delete_timestamp,
}
try:
self.swift.delete_object(account, container, obj,
headers=headers)
except UnexpectedResponse as err:
self.stats_log('delete_failed', 'delete %r (%f) from '
'policy_index %s (%s) returned %s', path,
source_ts, container_policy_index,
delete_timestamp, err, level=logging.WARNING)
return False
except: # noqa
self.stats_log('unhandled_error', 'unable to delete %r (%f) '
'from policy_index %s (%s)', path, source_ts,
container_policy_index, delete_timestamp,
level=logging.ERROR, exc_info=True)
return False
self.stats_log('delete_success', '%r (%f) deleted from '
'policy_index %s (%s)', path, source_ts,
container_policy_index, delete_timestamp,
level=logging.INFO)
return self.throw_tombstones(account, container, obj, q_ts,
q_policy_index, path)
def reconcile_object(self, info):
"""
Process a possibly misplaced object write request. Determine correct
destination storage policy by checking with primary containers. Check
source and destination, copying or deleting into destination and
cleaning up the source as needed.
This method wraps _reconcile_object for exception handling.
:param info: a queue entry dict
:returns: True to indicate the request is fully processed
successfully, otherwise False.
"""
self.logger.debug('checking placement for %r (%f) '
'in policy_index %s', info['path'],
info['q_ts'], info['q_policy_index'])
success = False
try:
success = self._reconcile_object(**info)
except: # noqa
self.logger.exception('Unhandled Exception trying to '
'reconcile %r (%f) in policy_index %s',
info['path'], info['q_ts'],
info['q_policy_index'])
if success:
metric = 'success'
msg = 'was handled successfully'
else:
metric = 'retry'
msg = 'must be retried'
msg = '%(path)r (%(q_ts)f) in policy_index %(q_policy_index)s ' + msg
self.stats_log(metric, msg, info, level=logging.INFO)
self.log_stats()
return success
def _iter_containers(self):
"""
Generate a list of containers to process.
"""
# hit most recent container first instead of waiting on the updaters
current_container = get_reconciler_container_name(time.time())
yield current_container
container_gen = self.swift.iter_containers(MISPLACED_OBJECTS_ACCOUNT)
self.logger.debug('looking for containers in %s',
MISPLACED_OBJECTS_ACCOUNT)
while True:
one_page = None
try:
one_page = list(itertools.islice(
container_gen, constraints.CONTAINER_LISTING_LIMIT))
except UnexpectedResponse as err:
self.logger.error('Error listing containers in '
'account %s (%s)',
MISPLACED_OBJECTS_ACCOUNT, err)
if not one_page:
# don't generally expect more than one page
break
# reversed order since we expect older containers to be empty
for c in reversed(one_page):
# encoding here is defensive
container = c['name'].encode('utf8')
if container == current_container:
continue # we've already hit this one this pass
yield container
def _iter_objects(self, container):
"""
Generate a list of objects to process.
:param container: the name of the container to process
If the given container is empty and older than reclaim_age this
processor will attempt to reap it.
"""
self.logger.debug('looking for objects in %s', container)
found_obj = False
try:
for raw_obj in self.swift.iter_objects(
MISPLACED_OBJECTS_ACCOUNT, container):
found_obj = True
yield raw_obj
except UnexpectedResponse as err:
self.logger.error('Error listing objects in container %s (%s)',
container, err)
if float(container) < time.time() - self.reclaim_age and \
not found_obj:
# Try to delete old empty containers so the queue doesn't
# grow without bound. It's ok if there's a conflict.
self.swift.delete_container(
MISPLACED_OBJECTS_ACCOUNT, container,
acceptable_statuses=(2, 404, 409, 412))
def reconcile(self):
"""
Main entry point for processing misplaced objects.
Iterate over all queue entries and delegate to reconcile_object.
"""
self.logger.debug('pulling items from the queue')
for container in self._iter_containers():
for raw_obj in self._iter_objects(container):
try:
obj_info = parse_raw_obj(raw_obj)
except Exception:
self.stats_log('invalid_record',
'invalid queue record: %r', raw_obj,
level=logging.ERROR, exc_info=True)
continue
finished = self.reconcile_object(obj_info)
if finished:
self.pop_queue(container, raw_obj['name'],
obj_info['q_ts'],
obj_info['q_record'])
self.log_stats()
self.logger.debug('finished container %s', container)
def run_once(self, *args, **kwargs):
"""
Process every entry in the queue.
"""
try:
self.reconcile()
except:
self.logger.exception('Unhandled Exception trying to reconcile')
self.log_stats(force=True)
def run_forever(self, *args, **kwargs):
while True:
self.run_once(*args, **kwargs)
self.stats = defaultdict(int)
self.logger.info('sleeping between intervals (%ss)', self.interval)
time.sleep(self.interval)
class TestObjectExpirer(ReplProbeTest):
def setUp(self):
self.expirer = Manager(['object-expirer'])
self.expirer.start()
err = self.expirer.stop()
if err:
raise unittest.SkipTest('Unable to verify object-expirer service')
conf_files = []
for server in self.expirer.servers:
conf_files.extend(server.conf_files())
conf_file = conf_files[0]
self.client = InternalClient(conf_file, 'probe-test', 3)
super(TestObjectExpirer, self).setUp()
self.container_name = 'container-%s' % uuid.uuid4()
self.object_name = 'object-%s' % uuid.uuid4()
self.brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name)
def _check_obj_in_container_listing(self):
for obj in self.client.iter_objects(self.account,
self.container_name):
if self.object_name == obj['name']:
return True
return False
@unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy")
def test_expirer_object_split_brain(self):
old_policy = random.choice(ENABLED_POLICIES)
wrong_policy = random.choice([p for p in ENABLED_POLICIES
if p != old_policy])
# create an expiring object and a container with the wrong policy
self.brain.stop_primary_half()
self.brain.put_container(int(old_policy))
self.brain.put_object(headers={'X-Delete-After': 2})
# get the object timestamp
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
create_timestamp = Timestamp(metadata['x-timestamp'])
self.brain.start_primary_half()
# get the expiring object updates in their queue, while we have all
# the servers up
Manager(['object-updater']).once()
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
# don't start handoff servers, only wrong policy is available
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# this guy should no-op since it's unable to expire the object
self.expirer.once()
self.brain.start_handoff_half()
self.get_to_final_state()
# validate object is expired
found_in_policy = None
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
self.assertIn('x-backend-timestamp', metadata)
self.assertEqual(Timestamp(metadata['x-backend-timestamp']),
create_timestamp)
# but it is still in the listing
self.assertTrue(self._check_obj_in_container_listing(),
msg='Did not find listing for %s' % self.object_name)
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# run the expirer again after replication
self.expirer.once()
# object is not in the listing
self.assertFalse(self._check_obj_in_container_listing(),
msg='Found listing for %s' % self.object_name)
# and validate object is tombstoned
found_in_policy = None
for policy in ENABLED_POLICIES:
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(policy)})
if 'x-backend-timestamp' in metadata:
if found_in_policy:
self.fail('found object in %s and also %s' %
(found_in_policy, policy))
found_in_policy = policy
self.assertIn('x-backend-timestamp', metadata)
self.assertGreater(Timestamp(metadata['x-backend-timestamp']),
create_timestamp)
def test_expirer_doesnt_make_async_pendings(self):
# The object expirer cleans up its own queue. The inner loop
# basically looks like this:
#
# for obj in stuff_to_delete:
# delete_the_object(obj)
# remove_the_queue_entry(obj)
#
# By default, upon receipt of a DELETE request for an expiring
# object, the object servers will create async_pending records to
# clean the expirer queue. Since the expirer cleans its own queue,
# this is unnecessary. The expirer can make requests in such a way
# tha the object server does not write out any async pendings; this
# test asserts that this is the case.
# Make an expiring object in each policy
for policy in ENABLED_POLICIES:
container_name = "expirer-test-%d" % policy.idx
container_headers = {'X-Storage-Policy': policy.name}
client.put_container(self.url, self.token, container_name,
headers=container_headers)
now = time.time()
delete_at = int(now + 2.0)
client.put_object(
self.url, self.token, container_name, "some-object",
headers={'X-Delete-At': str(delete_at),
'X-Timestamp': Timestamp(now).normal},
contents='dontcare')
time.sleep(2.0)
# make sure auto-created expirer-queue containers get in the account
# listing so the expirer can find them
Manager(['container-updater']).once()
# Make sure there's no async_pendings anywhere. Probe tests only run
# on single-node installs anyway, so this set should be small enough
# that an exhaustive check doesn't take too long.
all_obj_nodes = self.get_all_object_nodes()
pendings_before = self.gather_async_pendings(all_obj_nodes)
# expire the objects
Manager(['object-expirer']).once()
pendings_after = self.gather_async_pendings(all_obj_nodes)
self.assertEqual(pendings_after, pendings_before)
def test_expirer_object_should_not_be_expired(self):
# Current object-expirer checks the correctness via x-if-delete-at
# header that it can be deleted by expirer. If there are objects
# either which doesn't have x-delete-at header as metadata or which
# has different x-delete-at value from x-if-delete-at value,
# object-expirer's delete will fail as 412 PreconditionFailed.
# However, if some of the objects are in handoff nodes, the expirer
# can put the tombstone with the timestamp as same as x-delete-at and
# the object consistency will be resolved as the newer timestamp will
# be winner (in particular, overwritten case w/o x-delete-at). This
# test asserts such a situation that, at least, the overwriten object
# which have larger timestamp than the original expirered date should
# be safe.
def put_object(headers):
# use internal client to PUT objects so that X-Timestamp in headers
# is effective
headers['Content-Length'] = '0'
path = self.client.make_path(
self.account, self.container_name, self.object_name)
try:
self.client.make_request('PUT', path, headers, (2,))
except UnexpectedResponse as e:
self.fail(
'Expected 201 for PUT object but got %s' % e.resp.status)
obj_brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name, 'object', self.policy)
# T(obj_created) < T(obj_deleted with x-delete-at) < T(obj_recreated)
# < T(expirer_executed)
# Recreated obj should be appeared in any split brain case
obj_brain.put_container()
# T(obj_deleted with x-delete-at)
# object-server accepts req only if X-Delete-At is later than 'now'
# so here, T(obj_created) < T(obj_deleted with x-delete-at)
now = time.time()
delete_at = int(now + 2.0)
recreate_at = delete_at + 1.0
put_object(headers={'X-Delete-At': str(delete_at),
'X-Timestamp': Timestamp(now).normal})
# some object servers stopped to make a situation that the
# object-expirer can put tombstone in the primary nodes.
obj_brain.stop_primary_half()
# increment the X-Timestamp explicitly
# (will be T(obj_deleted with x-delete-at) < T(obj_recreated))
put_object(headers={'X-Object-Meta-Expired': 'False',
'X-Timestamp': Timestamp(recreate_at).normal})
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# sanity, the newer object is still there
try:
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name)
except UnexpectedResponse as e:
self.fail(
'Expected 200 for HEAD object but got %s' % e.resp.status)
self.assertIn('x-object-meta-expired', metadata)
# some object servers recovered
obj_brain.start_primary_half()
# sleep until after recreated_at
while time.time() <= recreate_at:
time.sleep(0.1)
# Now, expirer runs at the time after obj is recreated
self.expirer.once()
# verify that original object was deleted by expirer
obj_brain.stop_handoff_half()
try:
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,))
except UnexpectedResponse as e:
self.fail(
'Expected 404 for HEAD object but got %s' % e.resp.status)
obj_brain.start_handoff_half()
# and inconsistent state of objects is recovered by replicator
Manager(['object-replicator']).once()
# check if you can get recreated object
try:
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name)
except UnexpectedResponse as e:
self.fail(
'Expected 200 for HEAD object but got %s' % e.resp.status)
self.assertIn('x-object-meta-expired', metadata)
def _test_expirer_delete_outdated_object_version(self, object_exists):
# This test simulates a case where the expirer tries to delete
# an outdated version of an object.
# One case is where the expirer gets a 404, whereas the newest version
# of the object is offline.
# Another case is where the expirer gets a 412, since the old version
# of the object mismatches the expiration time sent by the expirer.
# In any of these cases, the expirer should retry deleting the object
# later, for as long as a reclaim age has not passed.
obj_brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name, 'object', self.policy)
obj_brain.put_container()
if object_exists:
obj_brain.put_object()
# currently, the object either doesn't exist, or does not have
# an expiration
# stop primary servers and put a newer version of the object, this
# time with an expiration. only the handoff servers will have
# the new version
obj_brain.stop_primary_half()
now = time.time()
delete_at = int(now + 2.0)
obj_brain.put_object({'X-Delete-At': str(delete_at)})
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# update object record in the container listing
Manager(['container-replicator']).once()
# take handoff servers down, and bring up the outdated primary servers
obj_brain.start_primary_half()
obj_brain.stop_handoff_half()
# wait until object expiration time
while time.time() <= delete_at:
time.sleep(0.1)
# run expirer against the outdated servers. it should fail since
# the outdated version does not match the expiration time
self.expirer.once()
# bring all servers up, and run replicator to update servers
obj_brain.start_handoff_half()
Manager(['object-replicator']).once()
# verify the deletion has failed by checking the container listing
self.assertTrue(self._check_obj_in_container_listing(),
msg='Did not find listing for %s' % self.object_name)
# run expirer again, delete should now succeed
self.expirer.once()
# verify the deletion by checking the container listing
self.assertFalse(self._check_obj_in_container_listing(),
msg='Found listing for %s' % self.object_name)
def test_expirer_delete_returns_outdated_404(self):
self._test_expirer_delete_outdated_object_version(object_exists=False)
def test_expirer_delete_returns_outdated_412(self):
self._test_expirer_delete_outdated_object_version(object_exists=True)
def test_reconciler_move_object_twice(self):
# select some policies
old_policy = random.choice(ENABLED_POLICIES)
new_policy = random.choice([p for p in ENABLED_POLICIES
if p != old_policy])
# setup a split brain
self.brain.stop_handoff_half()
# get old_policy on two primaries
self.brain.put_container(policy_index=int(old_policy))
self.brain.start_handoff_half()
self.brain.stop_primary_half()
# force a recreate on handoffs
self.brain.put_container(policy_index=int(old_policy))
self.brain.delete_container()
self.brain.put_container(policy_index=int(new_policy))
self.brain.put_object() # populate memcache with new_policy
self.brain.start_primary_half()
# at this point two primaries have old policy
container_part, container_nodes = self.container_ring.get_nodes(
self.account, self.container_name)
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
old_container_node_ids = [
node['id'] for node, metadata in head_responses
if int(old_policy) ==
int(metadata['X-Backend-Storage-Policy-Index'])]
self.assertEqual(2, len(old_container_node_ids))
# hopefully memcache still has the new policy cached
self.brain.put_object(headers={'x-object-meta-test': 'custom-meta'},
contents='VERIFY')
# double-check object correctly written to new policy
conf_files = []
for server in Manager(['container-reconciler']).servers:
conf_files.extend(server.conf_files())
conf_file = conf_files[0]
int_client = InternalClient(conf_file, 'probe-test', 3)
int_client.get_object_metadata(
self.account, self.container_name, self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(new_policy)})
int_client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
# shutdown the containers that know about the new policy
self.brain.stop_handoff_half()
# and get rows enqueued from old nodes
for server_type in ('container-replicator', 'container-updater'):
server = Manager([server_type])
tuple(server.once(number=n + 1) for n in old_container_node_ids)
# verify entry in the queue for the "misplaced" new_policy
for container in int_client.iter_containers('.misplaced_objects'):
for obj in int_client.iter_objects('.misplaced_objects',
container['name']):
expected = '%d:/%s/%s/%s' % (new_policy, self.account,
self.container_name,
self.object_name)
self.assertEqual(obj['name'], expected)
Manager(['container-reconciler']).once()
# verify object in old_policy
int_client.get_object_metadata(
self.account, self.container_name, self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
# verify object is *not* in new_policy
int_client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(new_policy)})
self.get_to_final_state()
# verify entry in the queue
for container in int_client.iter_containers('.misplaced_objects'):
for obj in int_client.iter_objects('.misplaced_objects',
container['name']):
expected = '%d:/%s/%s/%s' % (old_policy, self.account,
self.container_name,
self.object_name)
self.assertEqual(obj['name'], expected)
Manager(['container-reconciler']).once()
# and now it flops back
int_client.get_object_metadata(
self.account, self.container_name, self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(new_policy)})
int_client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
# make sure the queue is settled
self.get_to_final_state()
for container in int_client.iter_containers('.misplaced_objects'):
for obj in int_client.iter_objects('.misplaced_objects',
container['name']):
self.fail('Found unexpected object %r in the queue' % obj)
# verify that the object data read by external client is correct
headers, data = self._get_object_patiently(int(new_policy))
self.assertEqual('VERIFY', data)
self.assertEqual('custom-meta', headers['x-object-meta-test'])
class TestObjectExpirer(ReplProbeTest):
def setUp(self):
self.expirer = Manager(['object-expirer'])
self.expirer.start()
err = self.expirer.stop()
if err:
raise unittest.SkipTest('Unable to verify object-expirer service')
conf_files = []
for server in self.expirer.servers:
conf_files.extend(server.conf_files())
conf_file = conf_files[0]
self.client = InternalClient(conf_file, 'probe-test', 3)
super(TestObjectExpirer, self).setUp()
self.container_name = 'container-%s' % uuid.uuid4()
self.object_name = 'object-%s' % uuid.uuid4()
self.brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name)
def _check_obj_in_container_listing(self):
for obj in self.client.iter_objects(self.account, self.container_name):
if self.object_name == obj['name']:
return True
return False
@unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy")
def test_expirer_object_split_brain(self):
old_policy = random.choice(ENABLED_POLICIES)
wrong_policy = random.choice(
[p for p in ENABLED_POLICIES if p != old_policy])
# create an expiring object and a container with the wrong policy
self.brain.stop_primary_half()
self.brain.put_container(int(old_policy))
self.brain.put_object(headers={'X-Delete-After': 2})
# get the object timestamp
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
create_timestamp = Timestamp(metadata['x-timestamp'])
self.brain.start_primary_half()
# get the expiring object updates in their queue, while we have all
# the servers up
Manager(['object-updater']).once()
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
# don't start handoff servers, only wrong policy is available
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# this guy should no-op since it's unable to expire the object
self.expirer.once()
self.brain.start_handoff_half()
self.get_to_final_state()
# validate object is expired
found_in_policy = None
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4, ),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
self.assertIn('x-backend-timestamp', metadata)
self.assertEqual(Timestamp(metadata['x-backend-timestamp']),
create_timestamp)
# but it is still in the listing
self.assertTrue(self._check_obj_in_container_listing(),
msg='Did not find listing for %s' % self.object_name)
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# run the expirier again after replication
self.expirer.once()
# object is not in the listing
self.assertFalse(self._check_obj_in_container_listing(),
msg='Found listing for %s' % self.object_name)
# and validate object is tombstoned
found_in_policy = None
for policy in ENABLED_POLICIES:
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4, ),
headers={'X-Backend-Storage-Policy-Index': int(policy)})
if 'x-backend-timestamp' in metadata:
if found_in_policy:
self.fail('found object in %s and also %s' %
(found_in_policy, policy))
found_in_policy = policy
self.assertIn('x-backend-timestamp', metadata)
self.assertGreater(Timestamp(metadata['x-backend-timestamp']),
create_timestamp)
def test_expirer_object_should_not_be_expired(self):
# Current object-expirer checks the correctness via x-if-delete-at
# header that it can be deleted by expirer. If there are objects
# either which doesn't have x-delete-at header as metadata or which
# has different x-delete-at value from x-if-delete-at value,
# object-expirer's delete will fail as 412 PreconditionFailed.
# However, if some of the objects are in handoff nodes, the expirer
# can put the tombstone with the timestamp as same as x-delete-at and
# the object consistency will be resolved as the newer timestamp will
# be winner (in particular, overwritten case w/o x-delete-at). This
# test asserts such a situation that, at least, the overwriten object
# which have larger timestamp than the original expirered date should
# be safe.
def put_object(headers):
# use internal client to PUT objects so that X-Timestamp in headers
# is effective
headers['Content-Length'] = '0'
path = self.client.make_path(self.account, self.container_name,
self.object_name)
try:
self.client.make_request('PUT', path, headers, (2, ))
except UnexpectedResponse as e:
self.fail('Expected 201 for PUT object but got %s' %
e.resp.status)
obj_brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name, 'object', self.policy)
# T(obj_created) < T(obj_deleted with x-delete-at) < T(obj_recreated)
# < T(expirer_executed)
# Recreated obj should be appeared in any split brain case
obj_brain.put_container()
# T(obj_deleted with x-delete-at)
# object-server accepts req only if X-Delete-At is later than 'now'
# so here, T(obj_created) < T(obj_deleted with x-delete-at)
now = time.time()
delete_at = int(now + 2.0)
recreate_at = delete_at + 1.0
put_object(headers={
'X-Delete-At': str(delete_at),
'X-Timestamp': Timestamp(now).normal
})
# some object servers stopped to make a situation that the
# object-expirer can put tombstone in the primary nodes.
obj_brain.stop_primary_half()
# increment the X-Timestamp explicitly
# (will be T(obj_deleted with x-delete-at) < T(obj_recreated))
put_object(
headers={
'X-Object-Meta-Expired': 'False',
'X-Timestamp': Timestamp(recreate_at).normal
})
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# sanity, the newer object is still there
try:
metadata = self.client.get_object_metadata(self.account,
self.container_name,
self.object_name)
except UnexpectedResponse as e:
self.fail('Expected 200 for HEAD object but got %s' %
e.resp.status)
self.assertIn('x-object-meta-expired', metadata)
# some object servers recovered
obj_brain.start_primary_half()
# sleep until after recreated_at
while time.time() <= recreate_at:
time.sleep(0.1)
# Now, expirer runs at the time after obj is recreated
self.expirer.once()
# verify that original object was deleted by expirer
obj_brain.stop_handoff_half()
try:
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4, ))
except UnexpectedResponse as e:
self.fail('Expected 404 for HEAD object but got %s' %
e.resp.status)
obj_brain.start_handoff_half()
# and inconsistent state of objects is recovered by replicator
Manager(['object-replicator']).once()
# check if you can get recreated object
try:
metadata = self.client.get_object_metadata(self.account,
self.container_name,
self.object_name)
except UnexpectedResponse as e:
self.fail('Expected 200 for HEAD object but got %s' %
e.resp.status)
self.assertIn('x-object-meta-expired', metadata)
def _test_expirer_delete_outdated_object_version(self, object_exists):
# This test simulates a case where the expirer tries to delete
# an outdated version of an object.
# One case is where the expirer gets a 404, whereas the newest version
# of the object is offline.
# Another case is where the expirer gets a 412, since the old version
# of the object mismatches the expiration time sent by the expirer.
# In any of these cases, the expirer should retry deleting the object
# later, for as long as a reclaim age has not passed.
obj_brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name, 'object', self.policy)
obj_brain.put_container()
if object_exists:
obj_brain.put_object()
# currently, the object either doesn't exist, or does not have
# an expiration
# stop primary servers and put a newer version of the object, this
# time with an expiration. only the handoff servers will have
# the new version
obj_brain.stop_primary_half()
now = time.time()
delete_at = int(now + 2.0)
obj_brain.put_object({'X-Delete-At': str(delete_at)})
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# update object record in the container listing
Manager(['container-replicator']).once()
# take handoff servers down, and bring up the outdated primary servers
obj_brain.start_primary_half()
obj_brain.stop_handoff_half()
# wait until object expiration time
while time.time() <= delete_at:
time.sleep(0.1)
# run expirer against the outdated servers. it should fail since
# the outdated version does not match the expiration time
self.expirer.once()
# bring all servers up, and run replicator to update servers
obj_brain.start_handoff_half()
Manager(['object-replicator']).once()
# verify the deletion has failed by checking the container listing
self.assertTrue(self._check_obj_in_container_listing(),
msg='Did not find listing for %s' % self.object_name)
# run expirer again, delete should now succeed
self.expirer.once()
# this is mainly to paper over lp bug #1652323
self.get_to_final_state()
# verify the deletion by checking the container listing
self.assertFalse(self._check_obj_in_container_listing(),
msg='Found listing for %s' % self.object_name)
def test_expirer_delete_returns_outdated_404(self):
self._test_expirer_delete_outdated_object_version(object_exists=False)
def test_expirer_delete_returns_outdated_412(self):
self._test_expirer_delete_outdated_object_version(object_exists=True)
def test_reconciler_move_object_twice(self):
# select some policies
old_policy = random.choice(ENABLED_POLICIES)
new_policy = random.choice(
[p for p in ENABLED_POLICIES if p != old_policy])
# setup a split brain
self.brain.stop_handoff_half()
# get old_policy on two primaries
self.brain.put_container(policy_index=int(old_policy))
self.brain.start_handoff_half()
self.brain.stop_primary_half()
# force a recreate on handoffs
self.brain.put_container(policy_index=int(old_policy))
self.brain.delete_container()
self.brain.put_container(policy_index=int(new_policy))
self.brain.put_object() # populate memcache with new_policy
self.brain.start_primary_half()
# at this point two primaries have old policy
container_part, container_nodes = self.container_ring.get_nodes(
self.account, self.container_name)
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
old_container_node_ids = [
node['id'] for node, metadata in head_responses if int(old_policy)
== int(metadata['X-Backend-Storage-Policy-Index'])
]
self.assertEqual(2, len(old_container_node_ids))
# hopefully memcache still has the new policy cached
self.brain.put_object(headers={'x-object-meta-test': 'custom-meta'},
contents='VERIFY')
# double-check object correctly written to new policy
conf_files = []
for server in Manager(['container-reconciler']).servers:
conf_files.extend(server.conf_files())
conf_file = conf_files[0]
int_client = InternalClient(conf_file, 'probe-test', 3)
int_client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(new_policy)})
int_client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4, ),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
# shutdown the containers that know about the new policy
self.brain.stop_handoff_half()
# and get rows enqueued from old nodes
for server_type in ('container-replicator', 'container-updater'):
server = Manager([server_type])
tuple(server.once(number=n + 1) for n in old_container_node_ids)
# verify entry in the queue for the "misplaced" new_policy
for container in int_client.iter_containers('.misplaced_objects'):
for obj in int_client.iter_objects('.misplaced_objects',
container['name']):
expected = '%d:/%s/%s/%s' % (new_policy, self.account,
self.container_name,
self.object_name)
self.assertEqual(obj['name'], expected)
Manager(['container-reconciler']).once()
# verify object in old_policy
int_client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
# verify object is *not* in new_policy
int_client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4, ),
headers={'X-Backend-Storage-Policy-Index': int(new_policy)})
self.get_to_final_state()
# verify entry in the queue
for container in int_client.iter_containers('.misplaced_objects'):
for obj in int_client.iter_objects('.misplaced_objects',
container['name']):
expected = '%d:/%s/%s/%s' % (old_policy, self.account,
self.container_name,
self.object_name)
self.assertEqual(obj['name'], expected)
Manager(['container-reconciler']).once()
# and now it flops back
int_client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(new_policy)})
int_client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4, ),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
# make sure the queue is settled
self.get_to_final_state()
for container in int_client.iter_containers('.misplaced_objects'):
for obj in int_client.iter_objects('.misplaced_objects',
container['name']):
self.fail('Found unexpected object %r in the queue' % obj)
# verify that the object data read by external client is correct
headers, data = self._get_object_patiently(int(new_policy))
self.assertEqual('VERIFY', data)
self.assertEqual('custom-meta', headers['x-object-meta-test'])
class TestObjectExpirer(unittest.TestCase):
def setUp(self):
if len(POLICIES) < 2:
raise SkipTest('Need more than one policy')
self.expirer = Manager(['object-expirer'])
self.expirer.start()
err = self.expirer.stop()
if err:
raise SkipTest('Unable to verify object-expirer service')
conf_files = []
for server in self.expirer.servers:
conf_files.extend(server.conf_files())
conf_file = conf_files[0]
self.client = InternalClient(conf_file, 'probe-test', 3)
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
self.container_name = 'container-%s' % uuid.uuid4()
self.object_name = 'object-%s' % uuid.uuid4()
self.brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name)
def test_expirer_object_split_brain(self):
old_policy = random.choice(list(POLICIES))
wrong_policy = random.choice([p for p in POLICIES if p != old_policy])
# create an expiring object and a container with the wrong policy
self.brain.stop_primary_half()
self.brain.put_container(int(old_policy))
self.brain.put_object(headers={'X-Delete-After': 2})
# get the object timestamp
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
create_timestamp = Timestamp(metadata['x-timestamp'])
self.brain.start_primary_half()
# get the expiring object updates in their queue, while we have all
# the servers up
Manager(['object-updater']).once()
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
# don't start handoff servers, only wrong policy is available
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# this guy should no-op since it's unable to expire the object
self.expirer.once()
self.brain.start_handoff_half()
get_to_final_state()
# validate object is expired
found_in_policy = None
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
self.assert_('x-backend-timestamp' in metadata)
self.assertEqual(Timestamp(metadata['x-backend-timestamp']),
create_timestamp)
# but it is still in the listing
for obj in self.client.iter_objects(self.account,
self.container_name):
if self.object_name == obj['name']:
break
else:
self.fail('Did not find listing for %s' % self.object_name)
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# run the expirier again after replication
self.expirer.once()
# object is not in the listing
for obj in self.client.iter_objects(self.account,
self.container_name):
if self.object_name == obj['name']:
self.fail('Found listing for %s' % self.object_name)
# and validate object is tombstoned
found_in_policy = None
for policy in POLICIES:
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(policy)})
if 'x-backend-timestamp' in metadata:
if found_in_policy:
self.fail('found object in %s and also %s' %
(found_in_policy, policy))
found_in_policy = policy
self.assert_('x-backend-timestamp' in metadata)
self.assert_(Timestamp(metadata['x-backend-timestamp']) >
create_timestamp)
class ObjectRestorer(Daemon):
"""
Daemon that queries the internal hidden expiring_objects_account to
discover objects that need to be deleted.
:param conf: The daemon configuration.
"""
def __init__(self, conf):
self.conf = conf
self.container_ring = Ring('/etc/swift', ring_name='container')
self.logger = get_logger(conf, log_route='object-restorer')
self.logger.set_statsd_prefix('s3-object-restorer')
self.interval = int(conf.get('interval') or 300)
self.restoring_object_account = '.s3_restoring_objects'
self.expiring_restored_account = '.s3_expiring_restored_objects'
self.glacier_account_prefix = '.glacier_'
self.todo_container = 'todo'
self.restoring_container = 'restoring'
conf_path = '/etc/swift/s3-object-restorer.conf'
request_tries = int(conf.get('request_tries') or 3)
self.glacier = self._init_glacier()
self.glacier_tmpdir = conf.get('temp_path', '/var/cache/s3/')
self.swift = InternalClient(conf_path,
'Swift Object Restorer',
request_tries)
self.report_interval = int(conf.get('report_interval') or 300)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = join(self.recon_cache_path, 'object.recon')
self.concurrency = int(conf.get('concurrency', 1))
if self.concurrency < 1:
raise ValueError("concurrency must be set to at least 1")
self.processes = int(self.conf.get('processes', 0))
self.process = int(self.conf.get('process', 0))
self.client = Client(self.conf.get('sentry_sdn', ''))
def _init_glacier(self):
con = Layer2(region_name='ap-northeast-1')
return con.get_vault('swift-s3-transition')
def report(self, final=False):
"""
Emits a log line report of the progress so far, or the final progress
is final=True.
:param final: Set to True for the last report once the expiration pass
has completed.
"""
if final:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass completed in %ds; %d objects restored') %
(elapsed, self.report_objects))
dump_recon_cache({'object_expiration_pass': elapsed,
'expired_last_pass': self.report_objects},
self.rcache, self.logger)
elif time() - self.report_last_time >= self.report_interval:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass so far %ds; %d objects restored') %
(elapsed, self.report_objects))
self.report_last_time = time()
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
processes, process = self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug(_('Run begin'))
for o in self.swift.iter_objects(self.restoring_object_account,
self.todo_container):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' % (self.todo_container, obj)).
hexdigest(), 16)
if obj_process % processes != process:
continue
pool.spawn_n(self.start_object_restoring, obj)
pool.waitall()
for o in self.swift.iter_objects(self.restoring_object_account,
self.restoring_container):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' % (self.restoring_container, obj)).
hexdigest(), 16)
if obj_process % processes != process:
continue
pool.spawn_n(self.check_object_restored, obj)
pool.waitall()
self.logger.debug(_('Run end'))
self.report(final=True)
except (Exception, Timeout) as e:
report_exception(self.logger, _('Unhandled exception'), self.client)
def run_forever(self, *args, **kwargs):
"""
Executes passes forever, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon has no additional keyword args.
"""
sleep(random() * self.interval)
while True:
begin = time()
try:
self.run_once(*args, **kwargs)
except (Exception, Timeout):
report_exception(self.logger, _('Unhandled exception'), self.client)
elapsed = time() - begin
if elapsed < self.interval:
sleep(random() * (self.interval - elapsed))
def get_process_values(self, kwargs):
"""
Gets the processes, process from the kwargs if those values exist.
Otherwise, return processes, process set in the config file.
:param kwargs: Keyword args passed into the run_forever(), run_once()
methods. They have values specified on the command
line when the daemon is run.
"""
if kwargs.get('processes') is not None:
processes = int(kwargs['processes'])
else:
processes = self.processes
if kwargs.get('process') is not None:
process = int(kwargs['process'])
else:
process = self.process
if process < 0:
raise ValueError(
'process must be an integer greater than or equal to 0')
if processes < 0:
raise ValueError(
'processes must be an integer greater than or equal to 0')
if processes and process >= processes:
raise ValueError(
'process must be less than or equal to processes')
return processes, process
def start_object_restoring(self, obj):
start_time = time()
try:
actual_obj = obj
account, container, obj = actual_obj.split('/', 2)
archiveId = self.get_archiveid(account, container, obj)
if archiveId is None:
self.swift.delete_object(self.restoring_object_account,
self.todo_container, actual_obj)
return
jobId = self.glacier.retrieve_archive(archiveId).id
restoring_obj = make_glacier_hidden_object_name(actual_obj, jobId)
meta_prefix = 'X-Object-Meta'
meta = self.swift.get_object_metadata(account, container, obj,
metadata_prefix=meta_prefix)
meta = {'X-Object-Meta' + key: value for key, value in
meta.iteritems()}
self.update_action_hidden(self.restoring_object_account,
self.restoring_container,
restoring_obj, metadata=meta)
self.swift.delete_object(self.restoring_object_account,
self.todo_container, actual_obj)
self.report_objects += 1
self.logger.increment('start')
except (Exception, Timeout) as err:
self.logger.increment('errors')
report_exception(self.logger.exception,
_('Exception while restoring object %s. %s') %
(obj, str(err)), self.client)
self.logger.timing_since('timing', start_time)
self.report()
def get_archiveid(self, account, container, obj):
glacier_account = '%s%s' % (self.glacier_account_prefix, account)
glacier_obj = None
for o in get_objects_by_prefix(glacier_account, container, obj,
swift_client=self.swift):
name = get_glacier_objname_from_hidden_object(o)
if name == obj:
glacier_obj = o
break
if glacier_obj is None:
return None
return get_glacier_key_from_hidden_object(glacier_obj)
def check_object_restored(self, restoring_object):
actual_obj = get_glacier_objname_from_hidden_object(restoring_object)
jobId = get_glacier_key_from_hidden_object(restoring_object)
try:
path = '/v1/%s' % actual_obj
resp = self.swift.make_request('GET', path, {}, (2, 4,))
if resp.status_int == 404:
raise Exception('Object Not Found: %s' % actual_obj)
job = self.glacier.get_job(job_id=jobId)
if not job.completed:
return
self.complete_restore(actual_obj, job)
except Exception as e:
# Job ID가 만료될 경우 다시 restore 를 시도한다.
if not e.message.startswith('Object Not Found:'):
self.start_object_restoring(actual_obj)
self.logger.info(e)
self.swift.delete_object(self.restoring_object_account,
self.restoring_container, restoring_object)
def complete_restore(self, actual_obj, job):
tmppath = tempfile.NamedTemporaryFile(bufsize=0, delete=False,
dir=self.glacier_tmpdir).name
try:
job.download_to_file(filename=tmppath)
prefix = 'X-Object-Meta'
a, c, o = actual_obj.split('/', 2)
metadata = self.swift.get_object_metadata(a, c, o,
metadata_prefix=prefix)
metadata = {'X-Object-Meta' + key: value for key, value in metadata
.iteritems()}
days = int(metadata['X-Object-Meta-s3-restore-expire-days'])
exp_time = normalize_delete_at_timestamp(calc_nextDay(time()) +
(days - 1) * 86400)
# send restored object to proxy server
path = '/v1/%s' % actual_obj
metadata['X-Object-Meta-S3-Restored'] = True
exp_date = strftime("%a, %d %b %Y %H:%M:%S GMT",
gmtime(float(exp_time)))
metadata['X-Object-Meta-s3-restore'] = 'ongoing-request="false", ' \
'expiry-date="%s"' % exp_date
metadata['Content-Length'] = os.path.getsize(tmppath)
del metadata['X-Object-Meta-s3-restore-expire-days']
obj_body = open(tmppath, 'r')
self.swift.make_request('PUT', path, metadata, (2,),
body_file=obj_body)
# Add to .s3_expiring_restored_objects
self.update_action_hidden(self.expiring_restored_account,
exp_time, actual_obj)
obj_body.close()
self.logger.increment('done')
except UnexpectedResponse as e:
if e.resp.status_int == 404:
self.logger.error('Restoring object not found - %s' %
actual_obj)
except Exception as e:
self.logger.increment('errors')
self.logger.debug(e)
finally:
os.remove(tmppath)
def compute_obj_md5(self, obj):
etag = hashlib.md5()
etag.update(obj)
etag = etag.hexdigest()
return etag
def update_action_hidden(self, account, container, obj, metadata=None):
hidden_path = '/%s/%s/%s' % (account, container, obj)
part, nodes = self.container_ring.get_nodes(account, container)
for node in nodes:
ip = node['ip']
port = node['port']
dev = node['device']
action_headers = dict()
action_headers['user-agent'] = 'restore-daemon'
action_headers['X-Timestamp'] = normalize_timestamp(time())
action_headers['referer'] = 'restore-daemon'
action_headers['x-size'] = '0'
action_headers['x-content-type'] = "text/plain"
action_headers['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'
if metadata:
action_headers.update(metadata)
conn = http_connect(ip, port, dev, part, 'PUT', hidden_path,
action_headers)
response = conn.getresponse()
response.read()
class TestObjectExpirer(ReplProbeTest):
def setUp(self):
if len(ENABLED_POLICIES) < 2:
raise SkipTest('Need more than one policy')
self.expirer = Manager(['object-expirer'])
self.expirer.start()
err = self.expirer.stop()
if err:
raise SkipTest('Unable to verify object-expirer service')
conf_files = []
for server in self.expirer.servers:
conf_files.extend(server.conf_files())
conf_file = conf_files[0]
self.client = InternalClient(conf_file, 'probe-test', 3)
super(TestObjectExpirer, self).setUp()
self.container_name = 'container-%s' % uuid.uuid4()
self.object_name = 'object-%s' % uuid.uuid4()
self.brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name)
def test_expirer_object_split_brain(self):
old_policy = random.choice(ENABLED_POLICIES)
wrong_policy = random.choice([p for p in ENABLED_POLICIES
if p != old_policy])
# create an expiring object and a container with the wrong policy
self.brain.stop_primary_half()
self.brain.put_container(int(old_policy))
self.brain.put_object(headers={'X-Delete-After': 2})
# get the object timestamp
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
create_timestamp = Timestamp(metadata['x-timestamp'])
self.brain.start_primary_half()
# get the expiring object updates in their queue, while we have all
# the servers up
Manager(['object-updater']).once()
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
# don't start handoff servers, only wrong policy is available
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# this guy should no-op since it's unable to expire the object
self.expirer.once()
self.brain.start_handoff_half()
self.get_to_final_state()
# validate object is expired
found_in_policy = None
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
self.assertTrue('x-backend-timestamp' in metadata)
self.assertEqual(Timestamp(metadata['x-backend-timestamp']),
create_timestamp)
# but it is still in the listing
for obj in self.client.iter_objects(self.account,
self.container_name):
if self.object_name == obj['name']:
break
else:
self.fail('Did not find listing for %s' % self.object_name)
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# run the expirier again after replication
self.expirer.once()
# object is not in the listing
for obj in self.client.iter_objects(self.account,
self.container_name):
if self.object_name == obj['name']:
self.fail('Found listing for %s' % self.object_name)
# and validate object is tombstoned
found_in_policy = None
for policy in ENABLED_POLICIES:
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(policy)})
if 'x-backend-timestamp' in metadata:
if found_in_policy:
self.fail('found object in %s and also %s' %
(found_in_policy, policy))
found_in_policy = policy
self.assertTrue('x-backend-timestamp' in metadata)
self.assertTrue(Timestamp(metadata['x-backend-timestamp']) >
create_timestamp)
def test_expirer_object_should_not_be_expired(self):
obj_brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name, 'object', self.policy)
# T(obj_created) < T(obj_deleted with x-delete-at) < T(obj_recreated)
# < T(expirer_executed)
# Recreated obj should be appeared in any split brain case
# T(obj_created)
first_created_at = time.time()
# T(obj_deleted with x-delete-at)
# object-server accepts req only if X-Delete-At is later than 'now'
delete_at = int(time.time() + 1.5)
# T(obj_recreated)
recreated_at = time.time() + 2.0
# T(expirer_executed) - 'now'
sleep_for_expirer = 2.01
obj_brain.put_container(int(self.policy))
obj_brain.put_object(
headers={'X-Delete-At': delete_at,
'X-Timestamp': Timestamp(first_created_at).internal})
# some object servers stopped
obj_brain.stop_primary_half()
obj_brain.put_object(
headers={'X-Timestamp': Timestamp(recreated_at).internal,
'X-Object-Meta-Expired': 'False'})
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# some object servers recovered
obj_brain.start_primary_half()
# sleep to make sure expirer runs at the time after obj is recreated
time.sleep(sleep_for_expirer)
self.expirer.once()
# inconsistent state of objects is recovered
Manager(['object-replicator']).once()
# check if you can get recreated object
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name)
self.assertIn('x-object-meta-expired', metadata)
class TestObjectExpirer(unittest.TestCase):
def setUp(self):
if len(POLICIES) < 2:
raise SkipTest('Need more than one policy')
self.expirer = Manager(['object-expirer'])
self.expirer.start()
err = self.expirer.stop()
if err:
raise SkipTest('Unable to verify object-expirer service')
conf_files = []
for server in self.expirer.servers:
conf_files.extend(server.conf_files())
conf_file = conf_files[0]
self.client = InternalClient(conf_file, 'probe-test', 3)
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.policy, self.url, self.token, self.account,
self.configs) = reset_environment()
self.container_name = 'container-%s' % uuid.uuid4()
self.object_name = 'object-%s' % uuid.uuid4()
self.brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name)
def test_expirer_object_split_brain(self):
old_policy = random.choice(list(POLICIES))
wrong_policy = random.choice([p for p in POLICIES if p != old_policy])
# create an expiring object and a container with the wrong policy
self.brain.stop_primary_half()
self.brain.put_container(int(old_policy))
self.brain.put_object(headers={'X-Delete-After': 2})
# get the object timestamp
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
create_timestamp = Timestamp(metadata['x-timestamp'])
self.brain.start_primary_half()
# get the expiring object updates in their queue, while we have all
# the servers up
Manager(['object-updater']).once()
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
# don't start handoff servers, only wrong policy is available
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# this guy should no-op since it's unable to expire the object
self.expirer.once()
self.brain.start_handoff_half()
get_to_final_state()
# validate object is expired
found_in_policy = None
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4, ),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
self.assert_('x-backend-timestamp' in metadata)
self.assertEqual(Timestamp(metadata['x-backend-timestamp']),
create_timestamp)
# but it is still in the listing
for obj in self.client.iter_objects(self.account, self.container_name):
if self.object_name == obj['name']:
break
else:
self.fail('Did not find listing for %s' % self.object_name)
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# run the expirier again after replication
self.expirer.once()
# object is not in the listing
for obj in self.client.iter_objects(self.account, self.container_name):
if self.object_name == obj['name']:
self.fail('Found listing for %s' % self.object_name)
# and validate object is tombstoned
found_in_policy = None
for policy in POLICIES:
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4, ),
headers={'X-Backend-Storage-Policy-Index': int(policy)})
if 'x-backend-timestamp' in metadata:
if found_in_policy:
self.fail('found object in %s and also %s' %
(found_in_policy, policy))
found_in_policy = policy
self.assert_('x-backend-timestamp' in metadata)
self.assert_(
Timestamp(metadata['x-backend-timestamp']) >
create_timestamp)
class TestObjectExpirer(ReplProbeTest):
def setUp(self):
if len(ENABLED_POLICIES) < 2:
raise SkipTest("Need more than one policy")
self.expirer = Manager(["object-expirer"])
self.expirer.start()
err = self.expirer.stop()
if err:
raise SkipTest("Unable to verify object-expirer service")
conf_files = []
for server in self.expirer.servers:
conf_files.extend(server.conf_files())
conf_file = conf_files[0]
self.client = InternalClient(conf_file, "probe-test", 3)
super(TestObjectExpirer, self).setUp()
self.container_name = "container-%s" % uuid.uuid4()
self.object_name = "object-%s" % uuid.uuid4()
self.brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name)
def test_expirer_object_split_brain(self):
old_policy = random.choice(ENABLED_POLICIES)
wrong_policy = random.choice([p for p in ENABLED_POLICIES if p != old_policy])
# create an expiring object and a container with the wrong policy
self.brain.stop_primary_half()
self.brain.put_container(int(old_policy))
self.brain.put_object(headers={"X-Delete-After": 2})
# get the object timestamp
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
headers={"X-Backend-Storage-Policy-Index": int(old_policy)},
)
create_timestamp = Timestamp(metadata["x-timestamp"])
self.brain.start_primary_half()
# get the expiring object updates in their queue, while we have all
# the servers up
Manager(["object-updater"]).once()
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
# don't start handoff servers, only wrong policy is available
# make sure auto-created containers get in the account listing
Manager(["container-updater"]).once()
# this guy should no-op since it's unable to expire the object
self.expirer.once()
self.brain.start_handoff_half()
self.get_to_final_state()
# validate object is expired
found_in_policy = None
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4,),
headers={"X-Backend-Storage-Policy-Index": int(old_policy)},
)
self.assertTrue("x-backend-timestamp" in metadata)
self.assertEqual(Timestamp(metadata["x-backend-timestamp"]), create_timestamp)
# but it is still in the listing
for obj in self.client.iter_objects(self.account, self.container_name):
if self.object_name == obj["name"]:
break
else:
self.fail("Did not find listing for %s" % self.object_name)
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# run the expirier again after replication
self.expirer.once()
# object is not in the listing
for obj in self.client.iter_objects(self.account, self.container_name):
if self.object_name == obj["name"]:
self.fail("Found listing for %s" % self.object_name)
# and validate object is tombstoned
found_in_policy = None
for policy in ENABLED_POLICIES:
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4,),
headers={"X-Backend-Storage-Policy-Index": int(policy)},
)
if "x-backend-timestamp" in metadata:
if found_in_policy:
self.fail("found object in %s and also %s" % (found_in_policy, policy))
found_in_policy = policy
self.assertTrue("x-backend-timestamp" in metadata)
self.assertTrue(Timestamp(metadata["x-backend-timestamp"]) > create_timestamp)
class RestoredObjectExpirer(Daemon):
"""
Daemon that queries the internal hidden expiring_objects_account to
discover objects that need to be deleted.
:param conf: The daemon configuration.
"""
def __init__(self, conf):
self.conf = conf
self.logger = get_logger(conf, log_route='restored-object-expirer')
self.logger.set_statsd_prefix('s3-restored-object-expirer')
self.interval = int(conf.get('interval') or 300)
self.expire_restored_account = '.s3_expiring_restored_objects'
conf_path = '/etc/swift/s3-restored-object-expirer.conf'
request_tries = int(conf.get('request_tries') or 3)
self.swift = InternalClient(conf_path,
'Swift Restored Object Expirer',
request_tries)
self.report_interval = int(conf.get('report_interval') or 300)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = join(self.recon_cache_path, 'object.recon')
self.concurrency = int(conf.get('concurrency', 1))
if self.concurrency < 1:
raise ValueError("concurrency must be set to at least 1")
self.processes = int(self.conf.get('processes', 0))
self.process = int(self.conf.get('process', 0))
self.client = Client(self.conf.get('sentry_sdn', ''))
def report(self, final=False):
"""
Emits a log line report of the progress so far, or the final progress
is final=True.
:param final: Set to True for the last report once the expiration pass
has completed.
"""
if final:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass completed in %ds; %d objects expired') %
(elapsed, self.report_objects))
dump_recon_cache({'object_expiration_pass': elapsed,
'expired_last_pass': self.report_objects},
self.rcache, self.logger)
elif time() - self.report_last_time >= self.report_interval:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass so far %ds; %d objects expired') %
(elapsed, self.report_objects))
self.report_last_time = time()
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
processes, process = self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
containers_to_delete = []
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug(_('Run begin'))
containers, objects = \
self.swift.get_account_info(
self.expire_restored_account)
self.logger.info(_('Pass beginning; %s possible containers; %s '
'possible objects') % (containers, objects))
for c in self.swift.iter_containers(self.expire_restored_account):
container = c['name']
timestamp = int(container)
if timestamp > int(time()):
break
containers_to_delete.append(container)
for o in self.swift.iter_objects(self.expire_restored_account,
container):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' % (container, obj)).
hexdigest(), 16)
if obj_process % processes != process:
continue
pool.spawn_n(self.delete_object, container, obj)
pool.waitall()
for container in containers_to_delete:
try:
self.swift.delete_container(self.expire_restored_account,
container, (2, 4))
except (Exception, Timeout) as err:
report_exception(self.logger,
_('Exception while deleting container %s %s') %
(container, str(err)), self.client)
self.logger.debug(_('Run end'))
self.report(final=True)
except (Exception, Timeout):
report_exception(self.logger, _('Unhandled exception'), self.client)
def run_forever(self, *args, **kwargs):
"""
Executes passes forever, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon has no additional keyword args.
"""
sleep(random() * self.interval)
while True:
begin = time()
try:
self.run_once(*args, **kwargs)
except (Exception, Timeout):
report_exception(self.logger, _('Unhandled exception'), self.client)
elapsed = time() - begin
if elapsed < self.interval:
sleep(random() * (self.interval - elapsed))
def get_process_values(self, kwargs):
"""
Gets the processes, process from the kwargs if those values exist.
Otherwise, return processes, process set in the config file.
:param kwargs: Keyword args passed into the run_forever(), run_once()
methods. They have values specified on the command
line when the daemon is run.
"""
if kwargs.get('processes') is not None:
processes = int(kwargs['processes'])
else:
processes = self.processes
if kwargs.get('process') is not None:
process = int(kwargs['process'])
else:
process = self.process
if process < 0:
raise ValueError(
'process must be an integer greater than or equal to 0')
if processes < 0:
raise ValueError(
'processes must be an integer greater than or equal to 0')
if processes and process >= processes:
raise ValueError(
'process must be less than or equal to processes')
return processes, process
def delete_object(self, container, obj):
start_time = time()
try:
self.delete_actual_object(obj)
self.swift.delete_object(self.expire_restored_account,
container, obj)
self.report_objects += 1
self.logger.increment('objects')
except (Exception, Timeout) as err:
self.logger.increment('errors')
report_exception(self.logger,
_('Exception while deleting object %s %s %s') %
(container, obj, str(err)), self.client)
self.logger.timing_since('timing', start_time)
self.report()
def delete_actual_object(self, actual_obj):
"""
Deletes the end-user object indicated by the actual object name given
'<account>/<container>/<object>' if and only if the X-Delete-At value
of the object is exactly the timestamp given.
:param actual_obj: The name of the end-user object to delete:
'<account>/<container>/<object>'
:param timestamp: The timestamp the X-Delete-At value must match to
perform the actual delete.
"""
path = '/v1/' + urllib.quote(actual_obj.lstrip('/'))
account, container, object = actual_obj.split('/', 2)
try:
metadata = self.swift.get_object_metadata(account, container, object,
'X-Object-Meta')
except UnexpectedResponse as e:
if e.resp.status_int == 404:
return
metadata = {'X-Object-Meta' + key: value for key, value in metadata
.iteritems()}
del metadata['X-Object-Meta-s3-restore']
self.swift.make_request('POST', path, metadata,
(2, HTTP_NOT_FOUND, HTTP_PRECONDITION_FAILED))
class TestObjectExpirer(ReplProbeTest):
def setUp(self):
self.expirer = Manager(['object-expirer'])
self.expirer.start()
err = self.expirer.stop()
if err:
raise unittest.SkipTest('Unable to verify object-expirer service')
conf_files = []
for server in self.expirer.servers:
conf_files.extend(server.conf_files())
conf_file = conf_files[0]
self.client = InternalClient(conf_file, 'probe-test', 3)
super(TestObjectExpirer, self).setUp()
self.container_name = 'container-%s' % uuid.uuid4()
self.object_name = 'object-%s' % uuid.uuid4()
self.brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name)
def _check_obj_in_container_listing(self):
for obj in self.client.iter_objects(self.account, self.container_name):
if self.object_name == obj['name']:
return True
return False
@unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy")
def test_expirer_object_split_brain(self):
old_policy = random.choice(ENABLED_POLICIES)
wrong_policy = random.choice(
[p for p in ENABLED_POLICIES if p != old_policy])
# create an expiring object and a container with the wrong policy
self.brain.stop_primary_half()
self.brain.put_container(int(old_policy))
self.brain.put_object(headers={'X-Delete-After': 2})
# get the object timestamp
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
create_timestamp = Timestamp(metadata['x-timestamp'])
self.brain.start_primary_half()
# get the expiring object updates in their queue, while we have all
# the servers up
Manager(['object-updater']).once()
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
# don't start handoff servers, only wrong policy is available
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# this guy should no-op since it's unable to expire the object
self.expirer.once()
self.brain.start_handoff_half()
self.get_to_final_state()
# validate object is expired
found_in_policy = None
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4, ),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
self.assertIn('x-backend-timestamp', metadata)
self.assertEqual(Timestamp(metadata['x-backend-timestamp']),
create_timestamp)
# but it is still in the listing
self.assertTrue(self._check_obj_in_container_listing(),
msg='Did not find listing for %s' % self.object_name)
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# run the expirer again after replication
self.expirer.once()
# object is not in the listing
self.assertFalse(self._check_obj_in_container_listing(),
msg='Found listing for %s' % self.object_name)
# and validate object is tombstoned
found_in_policy = None
for policy in ENABLED_POLICIES:
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4, ),
headers={'X-Backend-Storage-Policy-Index': int(policy)})
if 'x-backend-timestamp' in metadata:
if found_in_policy:
self.fail('found object in %s and also %s' %
(found_in_policy, policy))
found_in_policy = policy
self.assertIn('x-backend-timestamp', metadata)
self.assertGreater(Timestamp(metadata['x-backend-timestamp']),
create_timestamp)
def test_expirer_doesnt_make_async_pendings(self):
# The object expirer cleans up its own queue. The inner loop
# basically looks like this:
#
# for obj in stuff_to_delete:
# delete_the_object(obj)
# remove_the_queue_entry(obj)
#
# By default, upon receipt of a DELETE request for an expiring
# object, the object servers will create async_pending records to
# clean the expirer queue. Since the expirer cleans its own queue,
# this is unnecessary. The expirer can make requests in such a way
# tha the object server does not write out any async pendings; this
# test asserts that this is the case.
# Make an expiring object in each policy
for policy in ENABLED_POLICIES:
container_name = "expirer-test-%d" % policy.idx
container_headers = {'X-Storage-Policy': policy.name}
client.put_container(self.url,
self.token,
container_name,
headers=container_headers)
now = time.time()
delete_at = int(now + 2.0)
client.put_object(self.url,
self.token,
container_name,
"some-object",
headers={
'X-Delete-At': str(delete_at),
'X-Timestamp': Timestamp(now).normal
},
contents='dontcare')
time.sleep(2.0)
# make sure auto-created expirer-queue containers get in the account
# listing so the expirer can find them
Manager(['container-updater']).once()
# Make sure there's no async_pendings anywhere. Probe tests only run
# on single-node installs anyway, so this set should be small enough
# that an exhaustive check doesn't take too long.
all_obj_nodes = self.get_all_object_nodes()
pendings_before = self.gather_async_pendings(all_obj_nodes)
# expire the objects
Manager(['object-expirer']).once()
pendings_after = self.gather_async_pendings(all_obj_nodes)
self.assertEqual(pendings_after, pendings_before)
def test_expirer_object_should_not_be_expired(self):
# Current object-expirer checks the correctness via x-if-delete-at
# header that it can be deleted by expirer. If there are objects
# either which doesn't have x-delete-at header as metadata or which
# has different x-delete-at value from x-if-delete-at value,
# object-expirer's delete will fail as 412 PreconditionFailed.
# However, if some of the objects are in handoff nodes, the expirer
# can put the tombstone with the timestamp as same as x-delete-at and
# the object consistency will be resolved as the newer timestamp will
# be winner (in particular, overwritten case w/o x-delete-at). This
# test asserts such a situation that, at least, the overwriten object
# which have larger timestamp than the original expirered date should
# be safe.
def put_object(headers):
# use internal client to PUT objects so that X-Timestamp in headers
# is effective
headers['Content-Length'] = '0'
path = self.client.make_path(self.account, self.container_name,
self.object_name)
try:
self.client.make_request('PUT', path, headers, (2, ))
except UnexpectedResponse as e:
self.fail('Expected 201 for PUT object but got %s' %
e.resp.status)
obj_brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name, 'object', self.policy)
# T(obj_created) < T(obj_deleted with x-delete-at) < T(obj_recreated)
# < T(expirer_executed)
# Recreated obj should be appeared in any split brain case
obj_brain.put_container()
# T(obj_deleted with x-delete-at)
# object-server accepts req only if X-Delete-At is later than 'now'
# so here, T(obj_created) < T(obj_deleted with x-delete-at)
now = time.time()
delete_at = int(now + 2.0)
recreate_at = delete_at + 1.0
put_object(headers={
'X-Delete-At': str(delete_at),
'X-Timestamp': Timestamp(now).normal
})
# some object servers stopped to make a situation that the
# object-expirer can put tombstone in the primary nodes.
obj_brain.stop_primary_half()
# increment the X-Timestamp explicitly
# (will be T(obj_deleted with x-delete-at) < T(obj_recreated))
put_object(
headers={
'X-Object-Meta-Expired': 'False',
'X-Timestamp': Timestamp(recreate_at).normal
})
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# sanity, the newer object is still there
try:
metadata = self.client.get_object_metadata(self.account,
self.container_name,
self.object_name)
except UnexpectedResponse as e:
self.fail('Expected 200 for HEAD object but got %s' %
e.resp.status)
self.assertIn('x-object-meta-expired', metadata)
# some object servers recovered
obj_brain.start_primary_half()
# sleep until after recreated_at
while time.time() <= recreate_at:
time.sleep(0.1)
# Now, expirer runs at the time after obj is recreated
self.expirer.once()
# verify that original object was deleted by expirer
obj_brain.stop_handoff_half()
try:
metadata = self.client.get_object_metadata(
self.account,
self.container_name,
self.object_name,
acceptable_statuses=(4, ))
except UnexpectedResponse as e:
self.fail('Expected 404 for HEAD object but got %s' %
e.resp.status)
obj_brain.start_handoff_half()
# and inconsistent state of objects is recovered by replicator
Manager(['object-replicator']).once()
# check if you can get recreated object
try:
metadata = self.client.get_object_metadata(self.account,
self.container_name,
self.object_name)
except UnexpectedResponse as e:
self.fail('Expected 200 for HEAD object but got %s' %
e.resp.status)
self.assertIn('x-object-meta-expired', metadata)
def _test_expirer_delete_outdated_object_version(self, object_exists):
# This test simulates a case where the expirer tries to delete
# an outdated version of an object.
# One case is where the expirer gets a 404, whereas the newest version
# of the object is offline.
# Another case is where the expirer gets a 412, since the old version
# of the object mismatches the expiration time sent by the expirer.
# In any of these cases, the expirer should retry deleting the object
# later, for as long as a reclaim age has not passed.
obj_brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name, 'object', self.policy)
obj_brain.put_container()
if object_exists:
obj_brain.put_object()
# currently, the object either doesn't exist, or does not have
# an expiration
# stop primary servers and put a newer version of the object, this
# time with an expiration. only the handoff servers will have
# the new version
obj_brain.stop_primary_half()
now = time.time()
delete_at = int(now + 2.0)
obj_brain.put_object({'X-Delete-At': str(delete_at)})
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# update object record in the container listing
Manager(['container-replicator']).once()
# take handoff servers down, and bring up the outdated primary servers
obj_brain.start_primary_half()
obj_brain.stop_handoff_half()
# wait until object expiration time
while time.time() <= delete_at:
time.sleep(0.1)
# run expirer against the outdated servers. it should fail since
# the outdated version does not match the expiration time
self.expirer.once()
# bring all servers up, and run replicator to update servers
obj_brain.start_handoff_half()
Manager(['object-replicator']).once()
# verify the deletion has failed by checking the container listing
self.assertTrue(self._check_obj_in_container_listing(),
msg='Did not find listing for %s' % self.object_name)
# run expirer again, delete should now succeed
self.expirer.once()
# verify the deletion by checking the container listing
self.assertFalse(self._check_obj_in_container_listing(),
msg='Found listing for %s' % self.object_name)
def test_expirer_delete_returns_outdated_404(self):
self._test_expirer_delete_outdated_object_version(object_exists=False)
def test_expirer_delete_returns_outdated_412(self):
self._test_expirer_delete_outdated_object_version(object_exists=True)
def test_slo_async_delete(self):
if not self.cluster_info.get('slo', {}).get('allow_async_delete'):
raise unittest.SkipTest('allow_async_delete not enabled')
segment_container = self.container_name + '_segments'
client.put_container(self.url, self.token, self.container_name, {})
client.put_container(self.url, self.token, segment_container, {})
client.put_object(self.url, self.token, segment_container, 'segment_1',
b'1234')
client.put_object(self.url, self.token, segment_container, 'segment_2',
b'5678')
client.put_object(self.url,
self.token,
self.container_name,
'slo',
json.dumps([
{
'path': segment_container + '/segment_1'
},
{
'data': 'Cg=='
},
{
'path': segment_container + '/segment_2'
},
]),
query_string='multipart-manifest=put')
_, body = client.get_object(self.url, self.token, self.container_name,
'slo')
self.assertEqual(body, b'1234\n5678')
client.delete_object(
self.url,
self.token,
self.container_name,
'slo',
query_string='multipart-manifest=delete&async=true')
# Object's deleted
_, objects = client.get_container(self.url, self.token,
self.container_name)
self.assertEqual(objects, [])
with self.assertRaises(client.ClientException) as caught:
client.get_object(self.url, self.token, self.container_name, 'slo')
self.assertEqual(404, caught.exception.http_status)
# But segments are still around and accessible
_, objects = client.get_container(self.url, self.token,
segment_container)
self.assertEqual([o['name'] for o in objects],
['segment_1', 'segment_2'])
_, body = client.get_object(self.url, self.token, segment_container,
'segment_1')
self.assertEqual(body, b'1234')
_, body = client.get_object(self.url, self.token, segment_container,
'segment_2')
self.assertEqual(body, b'5678')
# make sure auto-created expirer-queue containers get in the account
# listing so the expirer can find them
Manager(['container-updater']).once()
self.expirer.once()
# Now the expirer has cleaned up the segments
_, objects = client.get_container(self.url, self.token,
segment_container)
self.assertEqual(objects, [])
with self.assertRaises(client.ClientException) as caught:
client.get_object(self.url, self.token, segment_container,
'segment_1')
self.assertEqual(404, caught.exception.http_status)
with self.assertRaises(client.ClientException) as caught:
client.get_object(self.url, self.token, segment_container,
'segment_2')
self.assertEqual(404, caught.exception.http_status)
account = sys.argv[1]
container = sys.argv[2]
obj = sys.argv[3]
post_container = False
if len(sys.argv) == 5:
if sys.argv[4] in ['y', 'Y', 'yes', 'YES']:
post_container = True
client = InternalClient('/etc/swift/internal-client.conf', 'check-cont', 3)
for p in POLICIES:
print('Checking policy name: %s (%d)' % (p.name, p.idx))
headers = { 'X-Backend-Storage-Policy-Index': p.idx}
meta = client.get_object_metadata(account, container, obj,
headers=headers,
acceptable_statuses=(2, 4))
if 'x-timestamp' in meta:
print(' >> Find object %s in policy %s' % (obj, p.name) )
if post_container:
print('create container in policy %s' % p.name )
headers = { 'X-Storage-Policy': p.name}
client.create_container(account, container, headers)
break
else:
print(' >> Can not find %s in policy %s' % (obj, p.name))
