def test_enable(self):
broker = self._make_broker()
broker.update_metadata({'X-Container-Sysmeta-Sharding':
(True, Timestamp.now().internal)})
# no shard ranges
out = StringIO()
err = StringIO()
with self.assertRaises(SystemExit):
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
main([broker.db_file, 'enable'])
expected = ["WARNING: invalid shard ranges: ['No shard ranges.'].",
'Aborting.']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
# success
shard_ranges = []
for data in self.shard_data:
path = ShardRange.make_path(
'.shards_a', 'c', 'c', Timestamp.now(), data['index'])
shard_ranges.append(
ShardRange(path, Timestamp.now(), data['lower'],
data['upper'], data['object_count']))
broker.merge_shard_ranges(shard_ranges)
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
with mock_timestamp_now() as now:
main([broker.db_file, 'enable'])
expected = [
"Container moved to state 'sharding' with epoch %s." %
now.internal,
'Run container-sharder on all nodes to shard the container.']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self._assert_enabled(broker, now)
# already enabled
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
main([broker.db_file, 'enable'])
expected = [
"Container already in state 'sharding' with epoch %s." %
now.internal,
'No action required.',
'Run container-sharder on all nodes to shard the container.']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self._assert_enabled(broker, now)
def get_info(self):
now = Timestamp.now().internal
return {'container_count': 0,
'object_count': 0,
'bytes_used': 0,
'created_at': now,
'put_timestamp': now}
def create_account_stat_table(self, conn, put_timestamp):
"""
Create account_stat table which is specific to the account DB.
Not a part of Pluggable Back-ends, internal to the baseline code.
:param conn: DB connection object
:param put_timestamp: put timestamp
"""
conn.executescript("""
CREATE TABLE account_stat (
account TEXT,
created_at TEXT,
put_timestamp TEXT DEFAULT '0',
delete_timestamp TEXT DEFAULT '0',
container_count INTEGER,
object_count INTEGER DEFAULT 0,
bytes_used INTEGER DEFAULT 0,
hash TEXT default '00000000000000000000000000000000',
id TEXT,
status TEXT DEFAULT '',
status_changed_at TEXT DEFAULT '0',
metadata TEXT DEFAULT ''
);
INSERT INTO account_stat (container_count) VALUES (0);
""")
conn.execute('''
UPDATE account_stat SET account = ?, created_at = ?, id = ?,
put_timestamp = ?, status_changed_at = ?
''', (self.account, Timestamp.now().internal, str(uuid4()),
put_timestamp, put_timestamp))
def _make_diskfile(self, device='dev', partition='9',
account='a', container='c', obj='o', body='test',
extra_metadata=None, policy=None,
frag_index=None, timestamp=None, df_mgr=None,
commit=True, verify=True):
policy = policy or POLICIES.legacy
object_parts = account, container, obj
timestamp = Timestamp.now() if timestamp is None else timestamp
if df_mgr is None:
df_mgr = self.daemon._df_router[policy]
df = df_mgr.get_diskfile(
device, partition, *object_parts, policy=policy,
frag_index=frag_index)
write_diskfile(df, timestamp, data=body, extra_metadata=extra_metadata,
commit=commit)
if commit and verify:
# when we write and commit stub data, sanity check it's readable
# and not quarantined because of any validation check
with df.open():
self.assertEqual(b''.join(df.reader()), body)
# sanity checks
listing = os.listdir(df._datadir)
self.assertTrue(listing)
for filename in listing:
self.assertTrue(filename.startswith(timestamp.internal))
return df
def iter_task_to_expire(self, task_account_container_list,
my_index, divisor):
"""
Yields task expire info dict which consists of task_account,
task_container, task_object, timestamp_to_delete, and target_path
"""
for task_account, task_container in task_account_container_list:
for o in self.swift.iter_objects(task_account, task_container):
task_object = o['name'].encode('utf8')
try:
delete_timestamp, target_account, target_container, \
target_object = self.parse_task_obj(task_object)
except ValueError:
self.logger.exception('Unexcepted error handling task %r' %
task_object)
continue
if delete_timestamp > Timestamp.now():
# we shouldn't yield the object that doesn't reach
# the expiration date yet.
break
# Only one expirer daemon assigned for one task
if self.hash_mod('%s/%s' % (task_container, task_object),
divisor) != my_index:
continue
yield {'task_account': task_account,
'task_container': task_container,
'task_object': task_object,
'target_path': '/'.join([
target_account, target_container, target_object]),
'delete_timestamp': delete_timestamp}
def main():
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--config', default='/etc/swift/internal-client.conf',
help=('internal-client config file '
'(default: /etc/swift/internal-client.conf'))
parser.add_argument('--request-tries', type=int, default=3,
help='(default: 3)')
parser.add_argument('account', help='account from which to delete')
parser.add_argument('container', help='container from which to delete')
parser.add_argument(
'--prefix', default='',
help='only delete objects with this prefix (default: none)')
parser.add_argument(
'--marker', default='',
help='only delete objects after this marker (default: none)')
parser.add_argument(
'--end-marker', default='',
help='only delete objects before this end-marker (default: none)')
parser.add_argument(
'--timestamp', type=Timestamp, default=Timestamp.now(),
help='delete all objects as of this time (default: now)')
args = parser.parse_args()
swift = InternalClient(
args.config, 'Swift Container Deleter', args.request_tries)
for deleted, marker in mark_for_deletion(
swift, args.account, args.container,
args.marker, args.end_marker, args.prefix, args.timestamp):
if marker is None:
print('Finished. Marked %d objects for deletion.' % deleted)
else:
print('Marked %d objects for deletion, through %r' % (
deleted, marker))
def test_get_shard_ranges_invalid_shard_range(self):
sr = ShardRange('a/c', Timestamp.now())
bad_sr_data = dict(sr, name='bad_name')
body = json.dumps([bad_sr_data]).encode('ascii')
error_lines = self._check_get_shard_ranges_bad_data(body)
self.assertIn('Failed to get shard ranges', error_lines[0])
self.assertIn('ValueError', error_lines[0])
self.assertFalse(error_lines[1:])
def iter_task_containers_to_expire(self, task_account):
"""
Yields task_container names under the task_account if the delete at
timestamp of task_container is past.
"""
for c in self.swift.iter_containers(task_account,
prefix=self.task_container_prefix):
task_container = str(c['name'])
timestamp = self.delete_at_time_of_task_container(task_container)
if timestamp > Timestamp.now():
break
yield task_container
def test_get_shard_ranges_missing_record_type(self):
base = Controller(self.app)
req = Request.blank('/v1/a/c/o', method='PUT')
sr = ShardRange('a/c', Timestamp.now())
body = json.dumps([dict(sr)]).encode('ascii')
with mocked_http_conn(
200, 200, body_iter=iter([b'', body])):
actual = base._get_shard_ranges(req, 'a', 'c', '1_test')
self.assertIsNone(actual)
error_lines = self.app.logger.get_lines_for_level('error')
self.assertIn('Failed to get shard ranges', error_lines[0])
self.assertIn('unexpected record type', error_lines[0])
self.assertIn('/a/c', error_lines[0])
self.assertFalse(error_lines[1:])
def gen_headers(hdrs_in=None, add_ts=True):
"""
Get the headers ready for a request. All requests should have a User-Agent
string, but if one is passed in don't over-write it. Not all requests will
need an X-Timestamp, but if one is passed in do not over-write it.
:param headers: dict or None, base for HTTP headers
:param add_ts: boolean, should be True for any "unsafe" HTTP request
:returns: HeaderKeyDict based on headers and ready for the request
"""
hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict()
if add_ts and 'X-Timestamp' not in hdrs_out:
hdrs_out['X-Timestamp'] = Timestamp.now().internal
if 'user-agent' not in hdrs_out:
hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid()
return hdrs_out
def test_direct_head_container_deleted(self):
important_timestamp = Timestamp.now().internal
headers = HeaderKeyDict({'X-Backend-Important-Timestamp':
important_timestamp})
with mocked_http_conn(404, headers) as conn:
with self.assertRaises(ClientException) as raised:
direct_client.direct_head_container(
self.node, self.part, self.account, self.container)
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(raised.exception.http_status, 404)
self.assertEqual(raised.exception.http_headers, headers)
def test_direct_head_object_not_found(self):
important_timestamp = Timestamp.now().internal
stub_headers = {'X-Backend-Important-Timestamp': important_timestamp}
with mocked_http_conn(404, headers=stub_headers) as conn:
with self.assertRaises(ClientException) as raised:
direct_client.direct_head_object(
self.node, self.part, self.account, self.container,
self.obj)
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(raised.exception.http_status, 404)
self.assertEqual(
raised.exception.http_headers['x-backend-important-timestamp'],
important_timestamp)
def _get_synced_replication_info(self, broker, remote_info):
"""
Sync the remote_info storage_policy_index if needed and return the
newly synced replication info.
:param broker: the database broker
:param remote_info: the remote replication info
:returns: local broker replication info
"""
info = broker.get_replication_info()
if incorrect_policy_index(info, remote_info):
status_changed_at = Timestamp.now().internal
broker.set_storage_policy_index(
remote_info['storage_policy_index'],
timestamp=status_changed_at)
info = broker.get_replication_info()
return info
def _handle_sync_response(self, node, response, info, broker, http,
different_region):
parent = super(ContainerReplicator, self)
if is_success(response.status):
remote_info = json.loads(response.data)
if incorrect_policy_index(info, remote_info):
status_changed_at = Timestamp.now()
broker.set_storage_policy_index(
remote_info['storage_policy_index'],
timestamp=status_changed_at.internal)
sync_timestamps = ('created_at', 'put_timestamp',
'delete_timestamp')
if any(info[key] != remote_info[key] for key in sync_timestamps):
broker.merge_timestamps(*(remote_info[key] for key in
sync_timestamps))
rv = parent._handle_sync_response(
node, response, info, broker, http, different_region)
return rv
def merge_timestamps(self, created_at, put_timestamp, delete_timestamp):
"""
Used in replication to handle updating timestamps.
:param created_at: create timestamp
:param put_timestamp: put timestamp
:param delete_timestamp: delete timestamp
"""
with self.get() as conn:
old_status = self._is_deleted(conn)
conn.execute('''
UPDATE %s_stat SET created_at=MIN(?, created_at),
put_timestamp=MAX(?, put_timestamp),
delete_timestamp=MAX(?, delete_timestamp)
''' % self.db_type, (created_at, put_timestamp, delete_timestamp))
if old_status != self._is_deleted(conn):
timestamp = Timestamp.now()
self._update_status_changed_at(conn, timestamp.internal)
conn.commit()
def _handle_sync_response(self, node, response, info, broker, http,
different_region=False):
if is_success(response.status):
remote_info = json.loads(response.data)
if incorrect_policy_index(info, remote_info):
status_changed_at = Timestamp.now()
broker.set_storage_policy_index(
remote_info['storage_policy_index'],
timestamp=status_changed_at.internal)
sync_timestamps = ('created_at', 'put_timestamp',
'delete_timestamp')
if any(info[key] != remote_info[key] for key in sync_timestamps):
broker.merge_timestamps(*(remote_info[key] for key in
sync_timestamps))
if 'shard_max_row' in remote_info:
# Grab remote's shard ranges, too
self._fetch_and_merge_shard_ranges(http, broker)
return super(ContainerReplicator, self)._handle_sync_response(
node, response, info, broker, http, different_region)
def test_direct_head_container_deleted(self):
important_timestamp = Timestamp.now().internal
headers = HeaderKeyDict({'X-Backend-Important-Timestamp':
important_timestamp})
with mocked_http_conn(404, headers) as conn:
try:
direct_client.direct_head_container(
self.node, self.part, self.account, self.container)
except Exception as err:
self.assertTrue(isinstance(err, ClientException))
else:
self.fail('ClientException not raised')
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(err.http_status, 404)
self.assertEqual(err.http_headers, headers)
def merge_timestamps(self, created_at, put_timestamp, delete_timestamp):
"""
Used in replication to handle updating timestamps.
:param created_at: create timestamp
:param put_timestamp: put timestamp
:param delete_timestamp: delete timestamp
"""
with self.get() as conn:
old_status = self._is_deleted(conn)
conn.execute(
'''
UPDATE %s_stat SET created_at=MIN(?, created_at),
put_timestamp=MAX(?, put_timestamp),
delete_timestamp=MAX(?, delete_timestamp)
''' % self.db_type, (created_at, put_timestamp, delete_timestamp))
if old_status != self._is_deleted(conn):
timestamp = Timestamp.now()
self._update_status_changed_at(conn, timestamp.internal)
conn.commit()
def _make_diskfile(self,
device='dev',
partition='9',
account='a',
container='c',
obj='o',
body='test',
extra_metadata=None,
policy=None,
frag_index=None,
timestamp=None,
df_mgr=None,
commit=True,
verify=True):
policy = policy or POLICIES.legacy
object_parts = account, container, obj
timestamp = Timestamp.now() if timestamp is None else timestamp
if df_mgr is None:
df_mgr = self.daemon._df_router[policy]
df = df_mgr.get_diskfile(device,
partition,
*object_parts,
policy=policy,
frag_index=frag_index)
write_diskfile(df,
timestamp,
data=body,
extra_metadata=extra_metadata,
commit=commit)
if commit and verify:
# when we write and commit stub data, sanity check it's readable
# and not quarantined because of any validation check
with df.open():
self.assertEqual(''.join(df.reader()), body)
# sanity checks
listing = os.listdir(df._datadir)
self.assertTrue(listing)
for filename in listing:
self.assertTrue(filename.startswith(timestamp.internal))
return df
def POST(self, request):
"""Handle HTTP POST requests for the Swift Object Server."""
device, partition, account, container, obj, policy_idx = \
get_name_and_placement(request, 5, 5, True)
req_timestamp = valid_timestamp(request)
new_delete_at = int(request.headers.get('X-Delete-At') or 0)
if new_delete_at and new_delete_at < time.time():
return HTTPBadRequest(body='X-Delete-At in past', request=request,
content_type='text/plain')
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj,
policy_idx=policy_idx)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
orig_metadata = disk_file.read_metadata()
except (DiskFileNotExist, DiskFileQuarantined):
return HTTPNotFound(request=request)
orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))
if orig_timestamp >= req_timestamp:
return HTTPConflict(request=request)
metadata = {'X-Timestamp': req_timestamp.internal}
metadata.update(val for val in request.headers.iteritems()
if is_user_meta('object', val[0]))
for header_key in self.allowed_headers:
if header_key in request.headers:
header_caps = header_key.title()
metadata[header_caps] = request.headers[header_key]
orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
if orig_delete_at != new_delete_at:
if new_delete_at:
self.delete_at_update('PUT', new_delete_at, account, container,
obj, request, device, policy_idx)
if orig_delete_at:
self.delete_at_update('DELETE', orig_delete_at, account,
container, obj, request, device,
policy_idx)
disk_file.write_metadata(metadata)
return HTTPAccepted(request=request)
def test_direct_head_container_deleted(self):
important_timestamp = Timestamp.now().internal
headers = HeaderKeyDict(
{'X-Backend-Important-Timestamp': important_timestamp})
with mocked_http_conn(404, headers) as conn:
try:
direct_client.direct_head_container(self.node, self.part,
self.account,
self.container)
except Exception as err:
self.assertTrue(isinstance(err, ClientException))
else:
self.fail('ClientException not raised')
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(err.http_status, 404)
self.assertEqual(err.http_headers, headers)
def account_listing_bucket_response(account,
req,
response_content_type,
listing=None):
if response_content_type != 'application/json':
# AWS S3 is always call wit format=json
# check method GET in ServiceController (swift3/controllers/service.py)
return HTTPPreconditionFailed(body='Invalid content type')
data = []
for entry in listing:
data.append({
'name': entry['name'],
'count': entry['objects'],
'bytes': entry['bytes'],
'last_modified': Timestamp(entry['mtime']).isoformat
})
account_list = json.dumps(data, encoding="utf-8")
ret = HTTPOk(body=account_list, request=req, headers={})
ret.content_type = response_content_type
ret.charset = 'utf-8'
return ret
def DELETE(self, req):
"""Handle HTTP DELETE request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
req_timestamp = valid_timestamp(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
# policy index is only relevant for delete_obj (and transitively for
# auto create accounts)
obj_policy_index = self.get_and_validate_policy_index(req) or 0
broker = self._get_container_broker(drive, part, account, container)
if account.startswith(self.auto_create_account_prefix) and obj and \
not os.path.exists(broker.db_file):
try:
broker.initialize(req_timestamp.internal, obj_policy_index)
except DatabaseAlreadyExists:
pass
if not os.path.exists(broker.db_file):
return HTTPNotFound()
if obj: # delete object
broker.delete_object(obj, req.headers.get('x-timestamp'),
obj_policy_index)
return HTTPNoContent(request=req)
else:
# delete container
if not broker.empty():
return HTTPConflict(request=req)
existed = Timestamp(broker.get_info()['put_timestamp']) and \
not broker.is_deleted()
broker.delete_db(req_timestamp.internal)
if not broker.is_deleted():
return HTTPConflict(request=req)
self._update_sync_store(broker, 'DELETE')
resp = self.account_update(req, account, container, broker)
if resp:
return resp
if existed:
return HTTPNoContent(request=req)
return HTTPNotFound()
def test_replace(self):
broker = self._make_broker()
broker.update_metadata({'X-Container-Sysmeta-Sharding':
(True, Timestamp.now().internal)})
input_file = os.path.join(self.testdir, 'shards')
with open(input_file, 'wb') as fd:
json.dump(self.shard_data, fd)
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
main([broker.db_file, 'replace', input_file])
expected = [
'No shard ranges found to delete.',
'Injected 10 shard ranges.',
'Run container-replicator to replicate them to other nodes.',
'Use the enable sub-command to enable sharding.']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self.assertEqual(
[(data['lower'], data['upper']) for data in self.shard_data],
[(sr.lower_str, sr.upper_str) for sr in broker.get_shard_ranges()])
def test_sync_bogus_db_quarantines(self):
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
policy = random.choice(list(POLICIES))
# create "local" broker
local_broker = self._get_broker('a', 'c', node_index=0)
local_broker.initialize(ts.next(), policy.idx)
# create "remote" broker
remote_broker = self._get_broker('a', 'c', node_index=1)
remote_broker.initialize(ts.next(), policy.idx)
db_path = local_broker.db_file
self.assertTrue(os.path.exists(db_path)) # sanity check
old_inode = os.stat(db_path).st_ino
_orig_get_info = backend.ContainerBroker.get_info
def fail_like_bad_db(broker):
if broker.db_file == local_broker.db_file:
raise sqlite3.OperationalError("no such table: container_info")
else:
return _orig_get_info(broker)
part, node = self._get_broker_part_node(remote_broker)
with mock.patch('swift.container.backend.ContainerBroker.get_info',
fail_like_bad_db):
# Have the remote node replicate to local; local should see its
# corrupt DB, quarantine it, and act like the DB wasn't ever there
# in the first place.
daemon = self._run_once(node)
self.assertTrue(os.path.exists(db_path))
# Make sure we didn't just keep the old DB, but quarantined it and
# made a fresh copy.
new_inode = os.stat(db_path).st_ino
self.assertNotEqual(old_inode, new_inode)
self.assertEqual(daemon.stats['failure'], 0)
def delete_reverted_objs(self, job, objects, frag_index):
"""
For EC we can potentially revert only some of a partition
so we'll delete reverted objects here. Note that we delete
the fragment index of the file we sent to the remote node.
:param job: the job being processed
:param objects: a dict of objects to be deleted, each entry maps
hash=>timestamp
:param frag_index: (int) the fragment index of data files to be deleted
"""
df_mgr = self._df_router[job['policy']]
for object_hash, timestamp in objects.items():
try:
df = df_mgr.get_diskfile_from_hash(job['local_dev']['device'],
job['partition'],
object_hash,
job['policy'],
frag_index=frag_index)
df.purge(Timestamp(timestamp), frag_index)
except DiskFileError:
continue
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
device, partition, account, container, obj, policy_idx = \
get_name_and_placement(request, 5, 5, True)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj,
policy_idx=policy_idx)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
metadata = disk_file.read_metadata()
except DiskFileXattrNotSupported:
return HTTPInsufficientStorage(drive=device, request=request)
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, 'timestamp'):
headers['X-Backend-Timestamp'] = e.timestamp.internal
return HTTPNotFound(request=request, headers=headers,
conditional_response=True)
response = Response(request=request, conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_sys_or_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
ts = Timestamp(metadata['X-Timestamp'])
response.last_modified = math.ceil(float(ts))
# Needed for container sync feature
response.headers['X-Timestamp'] = ts.normal
response.headers['X-Backend-Timestamp'] = ts.internal
response.content_length = int(metadata['Content-Length'])
try:
response.content_encoding = metadata['Content-Encoding']
except KeyError:
pass
return response
def test_reap_object(self):
conf = {
'mount_check': 'false',
}
r = reaper.AccountReaper(conf, logger=unit.debug_logger())
mock_path = 'swift.account.reaper.direct_delete_object'
for policy in POLICIES:
r.reset_stats()
with patch(mock_path) as fake_direct_delete:
with patch('swift.common.utils.Timestamp.now') as mock_now:
mock_now.return_value = Timestamp(1429117638.86767)
r.reap_object('a', 'c', 'partition', cont_nodes, 'o',
policy.idx)
mock_now.assert_called_once_with()
for i, call_args in enumerate(
fake_direct_delete.call_args_list):
cnode = cont_nodes[i % len(cont_nodes)]
host = '%(ip)s:%(port)s' % cnode
device = cnode['device']
headers = {
'X-Container-Host': host,
'X-Container-Partition': 'partition',
'X-Container-Device': device,
'X-Backend-Storage-Policy-Index': policy.idx,
'X-Timestamp': '1429117638.86767'
}
ring = r.get_object_ring(policy.idx)
expected = call(dict(ring.devs[i], index=i),
0,
'a',
'c',
'o',
headers=headers,
conn_timeout=0.5,
response_timeout=10)
self.assertEqual(call_args, expected)
self.assertEqual(policy.object_ring.replicas - 1, i)
self.assertEqual(r.stats_objects_deleted,
policy.object_ring.replicas)
def iter_task_to_expire(self, task_account_container_list, my_index,
divisor):
"""
Yields task expire info dict which consists of task_account,
task_container, task_object, timestamp_to_delete, and target_path
"""
for task_account, task_container in task_account_container_list:
for o in self.swift.iter_objects(task_account, task_container):
task_object = o['name'].encode('utf8')
try:
delete_timestamp, target_account, target_container, \
target_object = self.parse_task_obj(task_object)
except ValueError:
self.logger.exception('Unexcepted error handling task %r' %
task_object)
continue
if delete_timestamp > Timestamp.now():
# we shouldn't yield the object that doesn't reach
# the expiration date yet.
break
# Only one expirer daemon assigned for one task
if self.hash_mod('%s/%s' % (task_container, task_object),
divisor) != my_index:
continue
yield {
'task_account':
task_account,
'task_container':
task_container,
'task_object':
task_object,
'target_path':
'/'.join([target_account, target_container,
target_object]),
'delete_timestamp':
delete_timestamp
}
def make_object_response(self, req, metadata, stream=None, ranges=None):
conditional_etag = None
if 'X-Backend-Etag-Is-At' in req.headers:
conditional_etag = metadata.get(
req.headers['X-Backend-Etag-Is-At'])
resp = Response(request=req,
conditional_response=True,
conditional_etag=conditional_etag)
if config_true_value(metadata['deleted']):
resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
else:
resp.headers['Content-Type'] = metadata.get(
'mime_type', 'application/octet-stream')
properties = metadata.get('properties')
if properties:
for k, v in properties.iteritems():
if is_sys_or_user_meta('object', k) or \
is_object_transient_sysmeta(k) or \
k.lower() in self.allowed_headers:
resp.headers[str(k)] = v
resp.headers['etag'] = metadata['hash'].lower()
resp.headers['x-object-sysmeta-version-id'] = metadata['version']
ts = Timestamp(metadata['ctime'])
resp.last_modified = math.ceil(float(ts))
if stream:
if ranges:
resp.app_iter = StreamRangeIterator(stream)
else:
resp.app_iter = stream
resp.content_length = int(metadata['length'])
try:
resp.content_encoding = metadata['encoding']
except KeyError:
pass
resp.accept_ranges = 'bytes'
return resp
def test_replace(self):
broker = self._make_broker()
broker.update_metadata({'X-Container-Sysmeta-Sharding':
(True, Timestamp.now().internal)})
input_file = os.path.join(self.testdir, 'shards')
with open(input_file, 'w') as fd:
json.dump(self.shard_data, fd)
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
main([broker.db_file, 'replace', input_file])
expected = [
'No shard ranges found to delete.',
'Injected 10 shard ranges.',
'Run container-replicator to replicate them to other nodes.',
'Use the enable sub-command to enable sharding.']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self.assertEqual(
[(data['lower'], data['upper']) for data in self.shard_data],
[(sr.lower_str, sr.upper_str) for sr in broker.get_shard_ranges()])
def test_enumerator_handling_rows_errors(self, num_from_row):
def my_num_from_row(row):
return row['ROWID']
num_from_row.side_effect = my_num_from_row
rows = 10
items = [{'ROWID': x, 'name': str(x),
'created_at': Timestamp(time.time())}
for x in range(rows)]
self.mock_broker.get_items_since.return_value = items
error = RuntimeError('oops')
for node_id in (0, 1):
all_nodes = [{'ip': '1.2.3.4',
'port': 1234,
'device': self.device_dir}
for _ in range(2)]
all_nodes[node_id]['ip'] = '127.0.0.1'
self.mock_ring.get_nodes.return_value = ('deadbeef', all_nodes)
self.mock_handler.handle.reset_mock()
self.mock_handler.handle.side_effect = error
with self._patch_broker():
self.crawler.run_once()
handle_rows = filter(lambda x: x % 2 == node_id, range(rows))
verify_rows = filter(lambda x: x % 2 != node_id, range(rows))
expected = [mock.call(items[row_id], self.mock_ic)
for row_id in handle_rows + verify_rows]
self.assertEqual(expected, self.mock_handler.handle.call_args_list)
expected_errors = [
mock.call('Failed to handle row {} ({}): {}'.format(
row_id, items[row_id]['name'], repr(error)))
for row_id in handle_rows + verify_rows]
self.assertEqual(expected_errors, self.logger.error.mock_calls)
self.logger.error.reset_mock()
def update_data_record(self, record):
"""
Perform any mutations to container listing records that are common to
all serialization formats, and returns it as a dict.
Converts created time to iso timestamp.
Replaces size with 'swift_bytes' content type parameter.
:params record: object entry record
:returns: modified record
"""
(name, created, size, content_type, etag) = record[:5]
if content_type is None:
return {'subdir': name}
response = {
'bytes': size,
'hash': etag,
'name': name,
'content_type': content_type
}
response['last_modified'] = Timestamp(created).isoformat
override_bytes_from_content_type(response, logger=self.logger)
return response
def test_handle_retry_error(self):
rows = [{'name': 'foo', 'ROWID': 1,
'created_at': Timestamp(time.time())}]
self.mock_broker.get_items_since.return_value = rows
self.crawler.submit_items = mock.Mock()
self.crawler.submit_items.side_effect = RetryError
settings = {
'account': 'foo',
'container': 'bar'
}
self.crawler.conf = {'containers': [settings]}
with self._patch_broker():
self.crawler.run_once()
self.mock_handler.get_last_processed_row\
.assert_called_once_with('hash')
self.assertEqual(
[], self.mock_handler.save_last_processed_row.mock_calls)
self.crawler.submit_items.assert_called_once_with(
self.mock_handler, rows, mock.ANY)
def setUp(self):
self._imported_create_container_table = \
AccountBroker.create_container_table
AccountBroker.create_container_table = \
prespi_create_container_table
self._imported_initialize = AccountBroker._initialize
AccountBroker._initialize = prespi_AccountBroker_initialize
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
exc = None
with broker.get() as conn:
try:
conn.execute('SELECT storage_policy_index FROM container')
except BaseException as err:
exc = err
self.assert_('no such column: storage_policy_index' in str(exc))
with broker.get() as conn:
try:
conn.execute('SELECT * FROM policy_stat')
except sqlite3.OperationalError as err:
self.assert_('no such table: policy_stat' in str(err))
else:
self.fail('database created with policy_stat table')
def test_bulk_handling(self, num_from_row):
def my_num_from_row(row):
return row['ROWID']
num_from_row.side_effect = my_num_from_row
self.conf['bulk_process'] = 'true'
self._setup_mocked_crawler()
self.assertEqual(self.crawler.bulk, True)
total_rows = 20
items = [{'ROWID': x, 'created_at': Timestamp(time.time())}
for x in range(total_rows)]
self.mock_broker.get_items_since.return_value = items
expected = [mock.call([items[x] for x in range(total_rows)],
self.mock_ic)]
with self._patch_broker():
self.crawler.run_once()
self.assertEqual(
expected, self.mock_handler.handle.call_args_list)
def test_retain_copy(self, factory_mock):
factory_mock.return_value = mock.Mock()
base = base_sync.BaseSync(self.settings, max_conns=1)
swift_client = mock.Mock()
swift_client.get_object_metadata.return_value = {}
row = {
'deleted': 0,
'created_at': str(time.time() - 5),
'name': 'foo',
'storage_policy_index': 99
}
_, _, swift_ts = decode_timestamps(row['created_at'])
base.delete_local_object(swift_client, row, swift_ts, False)
swift_ts.offset += 1
swift_client.delete_object.assert_called_once_with(
self.settings['account'],
self.settings['container'],
row['name'],
acceptable_statuses=(2, 404, 409),
headers={'X-Timestamp': Timestamp(swift_ts).internal})
def delete_shard_ranges(broker, args):
shard_ranges = broker.get_shard_ranges()
if not shard_ranges:
print("No shard ranges found to delete.")
return 0
while not args.force:
print('This will delete existing %d shard ranges.' % len(shard_ranges))
if broker.get_db_state() != UNSHARDED:
print('WARNING: Be very cautious about deleting existing shard '
'ranges. Deleting all ranges in this db does not guarantee '
'deletion of all ranges on all replicas of the db.')
print(' - this db is in state %s' % broker.get_db_state())
print(' - %d existing shard ranges have started sharding' %
[sr.state != ShardRange.FOUND
for sr in shard_ranges].count(True))
choice = input('Do you want to show the existing ranges [s], '
'delete the existing ranges [yes] '
'or quit without deleting [q]? ')
if choice == 's':
show_shard_ranges(broker, args)
continue
elif choice == 'q':
return 1
elif choice == 'yes':
break
else:
print('Please make a valid choice.')
print()
now = Timestamp.now()
for sr in shard_ranges:
sr.deleted = 1
sr.timestamp = now
broker.merge_shard_ranges(shard_ranges)
print('Deleted %s existing shard ranges.' % len(shard_ranges))
return 0
def _make_open_diskfile(self,
device='dev',
partition='9',
account='a',
container='c',
obj='o',
body='test',
extra_metadata=None,
policy=None,
frag_index=None,
timestamp=None,
df_mgr=None):
policy = policy or POLICIES.legacy
object_parts = account, container, obj
timestamp = Timestamp(time.time()) if timestamp is None else timestamp
if df_mgr is None:
df_mgr = self.daemon._diskfile_router[policy]
df = df_mgr.get_diskfile(device,
partition,
*object_parts,
policy=policy,
frag_index=frag_index)
content_length = len(body)
etag = hashlib.md5(body).hexdigest()
with df.create() as writer:
writer.write(body)
metadata = {
'X-Timestamp': timestamp.internal,
'Content-Length': str(content_length),
'ETag': etag,
}
if extra_metadata:
metadata.update(extra_metadata)
writer.put(metadata)
writer.commit(timestamp)
df.open()
return df
def repair_shard_ranges(broker, args):
if not broker.is_root_container():
print('WARNING: Shard containers cannot be repaired.')
print('This command should be used on a root container.')
return 2
shard_ranges = broker.get_shard_ranges()
if not shard_ranges:
print('No shards found, nothing to do.')
return 0
own_sr = broker.get_own_shard_range()
try:
acceptor_path, overlapping_donors = find_repair_solution(
shard_ranges, own_sr, args)
except ManageShardRangesException:
return 1
if not acceptor_path:
return 0
if not args.yes:
choice = input('Do you want to apply these changes to the container '
'DB? [yes/N]')
if choice != 'yes':
print('No changes applied')
return 0
# merge changes to the broker...
# note: acceptors do not need to be modified since they already span the
# complete range
ts_now = Timestamp.now()
finalize_shrinking(broker, [], overlapping_donors, ts_now)
print('Updated %s donor shard ranges.' % len(overlapping_donors))
print('Run container-replicator to replicate the changes to other nodes.')
print('Run container-sharder on all nodes to repair shards.')
return 0
def test_report_up_to_date(self):
broker = self._get_broker('a', 'c', node_index=0)
broker.initialize(Timestamp(1).internal, int(POLICIES.default))
info = broker.get_info()
broker.reported(info['put_timestamp'], info['delete_timestamp'],
info['object_count'], info['bytes_used'])
full_info = broker.get_replication_info()
expected_info = {
'put_timestamp': Timestamp(1).internal,
'delete_timestamp': '0',
'count': 0,
'bytes_used': 0,
'reported_put_timestamp': Timestamp(1).internal,
'reported_delete_timestamp': '0',
'reported_object_count': 0,
'reported_bytes_used': 0
}
for key, value in expected_info.items():
msg = 'expected value for %r, %r != %r' % (key, full_info[key],
value)
self.assertEqual(full_info[key], value, msg)
repl = replicator.ContainerReplicator({})
self.assertTrue(repl.report_up_to_date(full_info))
full_info['delete_timestamp'] = Timestamp(2).internal
self.assertFalse(repl.report_up_to_date(full_info))
full_info['reported_delete_timestamp'] = Timestamp(2).internal
self.assertTrue(repl.report_up_to_date(full_info))
full_info['count'] = 1
self.assertFalse(repl.report_up_to_date(full_info))
full_info['reported_object_count'] = 1
self.assertTrue(repl.report_up_to_date(full_info))
full_info['bytes_used'] = 1
self.assertFalse(repl.report_up_to_date(full_info))
full_info['reported_bytes_used'] = 1
self.assertTrue(repl.report_up_to_date(full_info))
full_info['put_timestamp'] = Timestamp(3).internal
self.assertFalse(repl.report_up_to_date(full_info))
full_info['reported_put_timestamp'] = Timestamp(3).internal
self.assertTrue(repl.report_up_to_date(full_info))
def test_put_and_get_multirange(self):
# put object
headers = {
'x-timestamp': Timestamp(time.time()).internal,
'content-type': 'application/octet-stream',
}
req = Request.blank(self._get_path(), method='PUT', headers=headers)
num_chunks = 10
req.body = ''.join(chr(i + 97) * self.disk_chunk_size for i in
range(num_chunks))
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
# get object with range
req = Request.blank(self._get_path())
req.range = 'bytes=301-455,686-792'
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 206)
msg = email.message_from_string(
'Content-Type: %s\r\n' % resp.headers['Content-Type'] + resp.body)
parts = [p for p in msg.walk()][1:]
self.assertEqual(2, len(parts))
self.assertEqual(parts[0].get_payload(), 'd' * 99 + 'e' * 56)
self.assertEqual(parts[1].get_payload(), 'g' * 14 + 'h' * 93)
def _backend_requests(self, req, n_outgoing, account_partition, accounts,
policy_index=None):
additional = {'X-Timestamp': Timestamp.now().internal}
if policy_index is None:
additional['X-Backend-Storage-Policy-Default'] = \
int(POLICIES.default)
else:
additional['X-Backend-Storage-Policy-Index'] = str(policy_index)
headers = [self.generate_request_headers(req, transfer=True,
additional=additional)
for _junk in range(n_outgoing)]
for i, account in enumerate(accounts):
i = i % len(headers)
headers[i]['X-Account-Partition'] = account_partition
headers[i]['X-Account-Host'] = csv_append(
headers[i].get('X-Account-Host'),
'%(ip)s:%(port)s' % account)
headers[i]['X-Account-Device'] = csv_append(
headers[i].get('X-Account-Device'),
account['device'])
return headers
def update_data_record(self, record, versions=False):
if 'subdir' in record:
return {'subdir': record['name']}
props = record.get('properties', {})
# This metadata is added by encryption middleware.
if 'x-object-sysmeta-container-update-override-etag' in props:
hash_ = props['x-object-sysmeta-container-update-override-etag']
else:
hash_ = record['hash'].lower()
response = {
'name': record['name'],
'bytes': record['size'],
'hash': hash_,
'last_modified': Timestamp(record['ctime']).isoformat,
'content_type': record.get('mime_type', 'application/octet-stream')
}
if record.get('deleted', False):
response['content_type'] = DELETE_MARKER_CONTENT_TYPE
if versions:
response['version'] = record.get('version', 'null')
override_bytes_from_content_type(response)
return response
def _backend_requests(self, req, n_outgoing, account_partition, accounts,
policy_index=None):
additional = {'X-Timestamp': Timestamp(time.time()).internal}
if policy_index is None:
additional['X-Backend-Storage-Policy-Default'] = \
int(POLICIES.default)
else:
additional['X-Backend-Storage-Policy-Index'] = str(policy_index)
headers = [self.generate_request_headers(req, transfer=True,
additional=additional)
for _junk in range(n_outgoing)]
for i, account in enumerate(accounts):
i = i % len(headers)
headers[i]['X-Account-Partition'] = account_partition
headers[i]['X-Account-Host'] = csv_append(
headers[i].get('X-Account-Host'),
'%(ip)s:%(port)s' % account)
headers[i]['X-Account-Device'] = csv_append(
headers[i].get('X-Account-Device'),
account['device'])
return headers
def test_build_task_obj_round_trip(self):
ts = next(self.ts)
a = 'a1'
c = 'c2'
o = 'obj1'
args = (ts, a, c, o)
self.assertEqual(
args, expirer.parse_task_obj(expirer.build_task_obj(ts, a, c, o)))
self.assertEqual(
args,
expirer.parse_task_obj(
expirer.build_task_obj(ts, a, c, o, high_precision=True)))
ts = Timestamp(next(self.ts), delta=1234)
a = u'\N{SNOWMAN}'
c = u'\N{SNOWFLAKE}'
o = u'\U0001F334'
args = (ts, a, c, o)
self.assertNotEqual(
args, expirer.parse_task_obj(expirer.build_task_obj(ts, a, c, o)))
self.assertEqual(
args,
expirer.parse_task_obj(
expirer.build_task_obj(ts, a, c, o, high_precision=True)))
def test_delete_db_status(self):
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
start = ts.next()
broker = AccountBroker(':memory:', account='a')
broker.initialize(start)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
self.assert_(Timestamp(info['created_at']) >= start)
self.assertEqual(info['delete_timestamp'], '0')
if self.__class__ == TestAccountBrokerBeforeMetadata:
self.assertEqual(info['status_changed_at'], '0')
else:
self.assertEqual(info['status_changed_at'],
Timestamp(start).internal)
# delete it
delete_timestamp = ts.next()
broker.delete_db(delete_timestamp)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
self.assert_(Timestamp(info['created_at']) >= start)
self.assertEqual(info['delete_timestamp'], delete_timestamp)
self.assertEqual(info['status_changed_at'], delete_timestamp)
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container = split_and_validate_path(req, 3, 4)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
if container: # put account container
if 'x-timestamp' not in req.headers:
timestamp = Timestamp.now()
else:
timestamp = valid_timestamp(req)
pending_timeout = None
container_policy_index = \
req.headers.get('X-Backend-Storage-Policy-Index', 0)
if 'x-trans-id' in req.headers:
pending_timeout = 3
broker = self._get_account_broker(drive, part, account,
pending_timeout=pending_timeout)
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
except DatabaseAlreadyExists:
pass
if req.headers.get('x-account-override-deleted', 'no').lower() != \
'yes' and broker.is_deleted():
return HTTPNotFound(request=req)
broker.put_container(container, req.headers['x-put-timestamp'],
req.headers['x-delete-timestamp'],
req.headers['x-object-count'],
req.headers['x-bytes-used'],
container_policy_index)
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
timestamp = valid_timestamp(req)
broker = self._get_account_broker(drive, part, account)
if not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
created = True
except DatabaseAlreadyExists:
created = False
elif broker.is_status_deleted():
return self._deleted_response(broker, req, HTTPForbidden,
body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp.internal)
if broker.is_deleted():
return HTTPConflict(request=req)
metadata = {}
metadata.update((key, (value, timestamp.internal))
for key, value in req.headers.items()
if is_sys_or_user_meta('account', key))
if metadata:
broker.update_metadata(metadata, validate_metadata=True)
if created:
return HTTPCreated(request=req)
else:
return HTTPAccepted(request=req)
def mock_timestamp_now(now=None):
if now is None:
now = Timestamp.now()
with mocklib.patch('swift.common.utils.Timestamp.now',
classmethod(lambda c: now)):
yield now
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container = split_and_validate_path(req, 3, 4)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
if not self.check_free_space(drive):
return HTTPInsufficientStorage(drive=drive, request=req)
if container: # put account container
if 'x-timestamp' not in req.headers:
timestamp = Timestamp.now()
else:
timestamp = valid_timestamp(req)
pending_timeout = None
container_policy_index = \
req.headers.get('X-Backend-Storage-Policy-Index', 0)
if 'x-trans-id' in req.headers:
pending_timeout = 3
broker = self._get_account_broker(drive, part, account,
pending_timeout=pending_timeout)
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
except DatabaseAlreadyExists:
pass
if req.headers.get('x-account-override-deleted', 'no').lower() != \
'yes' and broker.is_deleted():
return HTTPNotFound(request=req)
broker.put_container(container, req.headers['x-put-timestamp'],
req.headers['x-delete-timestamp'],
req.headers['x-object-count'],
req.headers['x-bytes-used'],
container_policy_index)
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
timestamp = valid_timestamp(req)
broker = self._get_account_broker(drive, part, account)
if not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
created = True
except DatabaseAlreadyExists:
created = False
elif broker.is_status_deleted():
return self._deleted_response(broker, req, HTTPForbidden,
body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp.internal)
if broker.is_deleted():
return HTTPConflict(request=req)
metadata = {}
if six.PY2:
metadata.update((key, (value, timestamp.internal))
for key, value in req.headers.items()
if is_sys_or_user_meta('account', key))
else:
for key, value in req.headers.items():
if is_sys_or_user_meta('account', key):
# Cast to native strings, so that json inside
# updata_metadata eats the data.
try:
value = value.encode('latin-1').decode('utf-8')
except UnicodeDecodeError:
raise HTTPBadRequest(
'Metadata must be valid UTF-8')
metadata[key] = (value, timestamp.internal)
if metadata:
broker.update_metadata(metadata, validate_metadata=True)
if created:
return HTTPCreated(request=req)
else:
return HTTPAccepted(request=req)
def test_info(self):
broker = self._make_broker()
broker.update_metadata({'X-Container-Sysmeta-Sharding':
(True, Timestamp.now().internal)})
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
main([broker.db_file, 'info'])
expected = ['Sharding enabled = True',
'Own shard range: None',
'db_state = unsharded',
'Metadata:',
' X-Container-Sysmeta-Sharding = True']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
retiring_db_id = broker.get_info()['id']
broker.merge_shard_ranges(ShardRange('.shards/cc', Timestamp.now()))
epoch = Timestamp.now()
with mock_timestamp_now(epoch) as now:
broker.enable_sharding(epoch)
self.assertTrue(broker.set_sharding_state())
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
with mock_timestamp_now(now):
main([broker.db_file, 'info'])
expected = ['Sharding enabled = True',
'Own shard range: {',
' "bytes_used": 0, ',
' "deleted": 0, ',
' "epoch": "%s", ' % epoch.internal,
' "lower": "", ',
' "meta_timestamp": "%s", ' % now.internal,
' "name": "a/c", ',
' "object_count": 0, ',
' "state": "sharding", ',
' "state_timestamp": "%s", ' % now.internal,
' "timestamp": "%s", ' % now.internal,
' "upper": ""',
'}',
'db_state = sharding',
'Retiring db id: %s' % retiring_db_id,
'Cleaving context: {',
' "cleave_to_row": null, ',
' "cleaving_done": false, ',
' "cursor": "", ',
' "last_cleave_to_row": null, ',
' "max_row": -1, ',
' "misplaced_done": false, ',
' "ranges_done": 0, ',
' "ranges_todo": 0, ',
' "ref": "%s"' % retiring_db_id,
'}',
'Metadata:',
' X-Container-Sysmeta-Sharding = True']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self.assertTrue(broker.set_sharded_state())
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
with mock_timestamp_now(now):
main([broker.db_file, 'info'])
expected = ['Sharding enabled = True',
'Own shard range: {',
' "bytes_used": 0, ',
' "deleted": 0, ',
' "epoch": "%s", ' % epoch.internal,
' "lower": "", ',
' "meta_timestamp": "%s", ' % now.internal,
' "name": "a/c", ',
' "object_count": 0, ',
' "state": "sharding", ',
' "state_timestamp": "%s", ' % now.internal,
' "timestamp": "%s", ' % now.internal,
' "upper": ""',
'}',
'db_state = sharded',
'Metadata:',
' X-Container-Sysmeta-Sharding = True']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
def mark_for_deletion(swift, account, container, marker, end_marker,
prefix, timestamp=None, yield_time=10):
'''
Enqueue jobs to async-delete some portion of a container's namespace
:param swift: InternalClient to use
:param account: account to delete from
:param container: container to delete from
:param marker: only delete objects after this name
:param end_marker: only delete objects before this name. Use ``None`` or
empty string to delete to the end of the namespace.
:param prefix: only delete objects starting with this prefix
:param timestamp: delete all objects as of this time. If ``None``, the
current time will be used.
:param yield_time: approximate period with which intermediate results
should be returned. If ``None``, disable intermediate
results.
:returns: If ``yield_time`` is ``None``, the number of objects marked for
deletion. Otherwise, a generator that will yield out tuples of
``(number of marked objects, last object name)`` approximately
every ``yield_time`` seconds. The final tuple will have ``None``
as the second element. This form allows you to retry when an
error occurs partway through while minimizing duplicate work.
'''
if timestamp is None:
timestamp = Timestamp.now()
def enqueue_deletes():
deleted = 0
obj_iter = swift.iter_objects(
account, container,
marker=marker, end_marker=end_marker, prefix=prefix)
time_marker = time.time()
while True:
to_delete = [obj['name'] for obj in itertools.islice(
obj_iter, OBJECTS_PER_UPDATE)]
if not to_delete:
break
delete_jobs = make_delete_jobs(
account, container, to_delete, timestamp)
swift.make_request(
'UPDATE',
swift.make_path('.expiring_objects', str(int(timestamp))),
headers={'X-Backend-Allow-Private-Methods': 'True',
'X-Backend-Storage-Policy-Index': '0',
'X-Timestamp': timestamp.internal},
acceptable_statuses=(2,),
body_file=io.BytesIO(json.dumps(delete_jobs).encode('ascii')))
deleted += len(delete_jobs)
if yield_time is not None and \
time.time() - time_marker > yield_time:
yield deleted, to_delete[-1]
time_marker = time.time()
yield deleted, None
if yield_time is None:
for deleted, marker in enqueue_deletes():
if marker is None:
return deleted
else:
return enqueue_deletes()
def reap_object(self, account, container, container_partition,
container_nodes, obj, policy_index):
"""
Deletes the given object by issuing a delete request to each node for
the object. The format of the delete request is such that each object
server will update a corresponding container server, removing the
object from the container's listing.
This function returns nothing and should raise no exception but only
update various self.stats_* values for what occurs.
:param account: The name of the account for the object.
:param container: The name of the container for the object.
:param container_partition: The partition for the container on the
container ring.
:param container_nodes: The primary node dicts for the container.
:param obj: The name of the object to delete.
:param policy_index: The storage policy index of the object's container
* See also: :func:`swift.common.ring.Ring.get_nodes` for a description
of the container node dicts.
"""
cnodes = itertools.cycle(container_nodes)
try:
ring = self.get_object_ring(policy_index)
except PolicyError:
self.stats_objects_remaining += 1
self.logger.increment('objects_remaining')
return
part, nodes = ring.get_nodes(account, container, obj)
successes = 0
failures = 0
timestamp = Timestamp.now()
for node in nodes:
cnode = next(cnodes)
try:
direct_delete_object(
node, part, account, container, obj,
conn_timeout=self.conn_timeout,
response_timeout=self.node_timeout,
headers={'X-Container-Host': '%(ip)s:%(port)s' % cnode,
'X-Container-Partition': str(container_partition),
'X-Container-Device': cnode['device'],
'X-Backend-Storage-Policy-Index': policy_index,
'X-Timestamp': timestamp.internal})
successes += 1
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1
self.logger.increment('return_codes.2')
except ClientException as err:
if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception(
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
failures += 1
self.logger.increment('objects_failures')
self.stats_return_codes[err.http_status // 100] = \
self.stats_return_codes.get(err.http_status // 100, 0) + 1
self.logger.increment(
'return_codes.%d' % (err.http_status // 100,))
except (Timeout, socket.error) as err:
failures += 1
self.logger.increment('objects_failures')
self.logger.error(
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
node)
if successes > failures:
self.stats_objects_deleted += 1
self.logger.increment('objects_deleted')
elif not successes:
self.stats_objects_remaining += 1
self.logger.increment('objects_remaining')
else:
self.stats_objects_possibly_remaining += 1
self.logger.increment('objects_possibly_remaining')
def reap_container(self, account, account_partition, account_nodes,
container):
"""
Deletes the data and the container itself for the given container. This
will call :func:`reap_object` up to sqrt(self.concurrency) times
concurrently for the objects in the container.
If there is any exception while deleting a single object, the process
will continue for any other objects in the container and the failed
objects will be tried again the next time this function is called with
the same parameters.
If there is any exception while listing the objects for deletion, the
process will stop (but will obviously be tried again the next time this
function is called with the same parameters). This is a possibility
since the listing comes from querying just the primary remote container
server.
Once all objects have been attempted to be deleted, the container
itself will be attempted to be deleted by sending a delete request to
all container nodes. The format of the delete request is such that each
container server will update a corresponding account server, removing
the container from the account's listing.
This function returns nothing and should raise no exception but only
update various self.stats_* values for what occurs.
:param account: The name of the account for the container.
:param account_partition: The partition for the account on the account
ring.
:param account_nodes: The primary node dicts for the account.
:param container: The name of the container to delete.
* See also: :func:`swift.common.ring.Ring.get_nodes` for a description
of the account node dicts.
"""
account_nodes = list(account_nodes)
part, nodes = self.get_container_ring().get_nodes(account, container)
node = nodes[-1]
pool = GreenPool(size=self.object_concurrency)
marker = ''
while True:
objects = None
try:
headers, objects = direct_get_container(
node, part, account, container,
marker=marker,
conn_timeout=self.conn_timeout,
response_timeout=self.node_timeout)
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1
self.logger.increment('return_codes.2')
except ClientException as err:
if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception(
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
self.stats_return_codes[err.http_status // 100] = \
self.stats_return_codes.get(err.http_status // 100, 0) + 1
self.logger.increment(
'return_codes.%d' % (err.http_status // 100,))
except (Timeout, socket.error) as err:
self.logger.error(
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
node)
if not objects:
break
try:
policy_index = headers.get('X-Backend-Storage-Policy-Index', 0)
policy = POLICIES.get_by_index(policy_index)
if not policy:
self.logger.error('ERROR: invalid storage policy index: %r'
% policy_index)
for obj in objects:
obj_name = obj['name']
if isinstance(obj_name, six.text_type):
obj_name = obj_name.encode('utf8')
pool.spawn(self.reap_object, account, container, part,
nodes, obj_name, policy_index)
pool.waitall()
except (Exception, Timeout):
self.logger.exception(_('Exception with objects for container '
'%(container)s for account %(account)s'
),
{'container': container,
'account': account})
marker = objects[-1]['name']
if marker == '':
break
successes = 0
failures = 0
timestamp = Timestamp.now()
for node in nodes:
anode = account_nodes.pop()
try:
direct_delete_container(
node, part, account, container,
conn_timeout=self.conn_timeout,
response_timeout=self.node_timeout,
headers={'X-Account-Host': '%(ip)s:%(port)s' % anode,
'X-Account-Partition': str(account_partition),
'X-Account-Device': anode['device'],
'X-Account-Override-Deleted': 'yes',
'X-Timestamp': timestamp.internal})
successes += 1
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1
self.logger.increment('return_codes.2')
except ClientException as err:
if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception(
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
failures += 1
self.logger.increment('containers_failures')
self.stats_return_codes[err.http_status // 100] = \
self.stats_return_codes.get(err.http_status // 100, 0) + 1
self.logger.increment(
'return_codes.%d' % (err.http_status // 100,))
except (Timeout, socket.error) as err:
self.logger.error(
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
node)
failures += 1
self.logger.increment('containers_failures')
if successes > failures:
self.stats_containers_deleted += 1
self.logger.increment('containers_deleted')
elif not successes:
self.stats_containers_remaining += 1
self.logger.increment('containers_remaining')
else:
self.stats_containers_possibly_remaining += 1
self.logger.increment('containers_possibly_remaining')
def gen_headers(hdrs_in=None, add_ts=False):
hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict()
if add_ts:
hdrs_out['X-Timestamp'] = Timestamp.now().internal
hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid()
return hdrs_out
def add_to_reconciler_queue(container_ring, account, container, obj,
obj_policy_index, obj_timestamp, op,
force=False, conn_timeout=5, response_timeout=15):
"""
Add an object to the container reconciler's queue. This will cause the
container reconciler to move it from its current storage policy index to
the correct storage policy index.
:param container_ring: container ring
:param account: the misplaced object's account
:param container: the misplaced object's container
:param obj: the misplaced object
:param obj_policy_index: the policy index where the misplaced object
currently is
:param obj_timestamp: the misplaced object's X-Timestamp. We need this to
ensure that the reconciler doesn't overwrite a newer
object with an older one.
:param op: the method of the operation (DELETE or PUT)
:param force: over-write queue entries newer than obj_timestamp
:param conn_timeout: max time to wait for connection to container server
:param response_timeout: max time to wait for response from container
server
:returns: .misplaced_object container name, False on failure. "Success"
means a majority of containers got the update.
"""
container_name = get_reconciler_container_name(obj_timestamp)
object_name = get_reconciler_obj_name(obj_policy_index, account,
container, obj)
if force:
# this allows an operator to re-enqueue an object that has
# already been popped from the queue to be reprocessed, but
# could potentially prevent out of order updates from making it
# into the queue
x_timestamp = Timestamp.now().internal
else:
x_timestamp = obj_timestamp
q_op_type = get_reconciler_content_type(op)
headers = {
'X-Size': 0,
'X-Etag': obj_timestamp,
'X-Timestamp': x_timestamp,
'X-Content-Type': q_op_type,
}
def _check_success(*args, **kwargs):
try:
direct_put_container_object(*args, **kwargs)
return 1
except (ClientException, Timeout, socket.error):
return 0
pile = GreenPile()
part, nodes = container_ring.get_nodes(MISPLACED_OBJECTS_ACCOUNT,
container_name)
for node in nodes:
pile.spawn(_check_success, node, part, MISPLACED_OBJECTS_ACCOUNT,
container_name, object_name, headers=headers,
conn_timeout=conn_timeout,
response_timeout=response_timeout)
successes = sum(pile)
if successes >= majority_size(len(nodes)):
return container_name
else:
return False
def blank(cls, *args, **kwargs):
req = super(Request, cls).blank(*args, **kwargs)
if 'X-Timestamp' not in req.headers:
req.headers['X-Timestamp'] = Timestamp.now().normal
return req
