def _enable_sharding(broker, own_shard_range, args):
if own_shard_range.update_state(ShardRange.SHARDING):
own_shard_range.epoch = Timestamp.now()
own_shard_range.state_timestamp = own_shard_range.epoch
with broker.updated_timeout(args.enable_timeout):
broker.merge_shard_ranges([own_shard_range])
broker.update_metadata({'X-Container-Sysmeta-Sharding':
('True', Timestamp.now().normal)})
return own_shard_range
def test_enable(self):
broker = self._make_broker()
broker.update_metadata(
{'X-Container-Sysmeta-Sharding': (True, Timestamp.now().internal)})
# no shard ranges
out = StringIO()
err = StringIO()
with self.assertRaises(SystemExit):
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
main([broker.db_file, 'enable'])
expected = [
"WARNING: invalid shard ranges: ['No shard ranges.'].", 'Aborting.'
]
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
# success
shard_ranges = []
for data in self.shard_data:
path = ShardRange.make_path('.shards_a', 'c', 'c', Timestamp.now(),
data['index'])
shard_ranges.append(
ShardRange(path, Timestamp.now(), data['lower'], data['upper'],
data['object_count']))
broker.merge_shard_ranges(shard_ranges)
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
with mock_timestamp_now() as now:
main([broker.db_file, 'enable'])
expected = [
"Container moved to state 'sharding' with epoch %s." %
now.internal,
'Run container-sharder on all nodes to shard the container.'
]
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self._assert_enabled(broker, now)
# already enabled
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
main([broker.db_file, 'enable'])
expected = [
"Container already in state 'sharding' with epoch %s." %
now.internal, 'No action required.',
'Run container-sharder on all nodes to shard the container.'
]
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self._assert_enabled(broker, now)
def test_enable(self):
broker = self._make_broker()
broker.update_metadata({'X-Container-Sysmeta-Sharding':
(True, Timestamp.now().internal)})
# no shard ranges
out = StringIO()
err = StringIO()
with self.assertRaises(SystemExit):
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
main([broker.db_file, 'enable'])
expected = ["WARNING: invalid shard ranges: ['No shard ranges.'].",
'Aborting.']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
# success
shard_ranges = []
for data in self.shard_data:
path = ShardRange.make_path(
'.shards_a', 'c', 'c', Timestamp.now(), data['index'])
shard_ranges.append(
ShardRange(path, Timestamp.now(), data['lower'],
data['upper'], data['object_count']))
broker.merge_shard_ranges(shard_ranges)
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
with mock_timestamp_now() as now:
main([broker.db_file, 'enable'])
expected = [
"Container moved to state 'sharding' with epoch %s." %
now.internal,
'Run container-sharder on all nodes to shard the container.']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self._assert_enabled(broker, now)
# already enabled
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
main([broker.db_file, 'enable'])
expected = [
"Container already in state 'sharding' with epoch %s." %
now.internal,
'No action required.',
'Run container-sharder on all nodes to shard the container.']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self._assert_enabled(broker, now)
def iter_task_to_expire(self, task_containers):
"""
Yields task expire info dict which consists of task_container,
target_path, timestamp_to_delete, and target_path
"""
for task_container in task_containers:
for o in self.swift.iter_objects(self.expiring_objects_account,
task_container):
task_object = o['name'].encode('utf8')
delete_timestamp, target_path = task_object.split('-', 1)
delete_timestamp = Timestamp(delete_timestamp)
if delete_timestamp > Timestamp.now():
# we shouldn't yield the object that doesn't reach
# the expiration date yet.
break
if self.processes > 0:
obj_process = int(
hashlib.md5('%s/%s' %
(task_container, task_object)).hexdigest(),
16)
if obj_process % self.processes != self.process:
continue
yield {
'task_container': task_container,
'task_object': task_object,
'target_path': target_path,
'delete_timestamp': delete_timestamp
}
def _backend_requests(self,
req,
n_outgoing,
account_partition,
accounts,
policy_index=None):
additional = {'X-Timestamp': Timestamp.now().internal}
if policy_index is None:
additional['X-Backend-Storage-Policy-Default'] = \
int(POLICIES.default)
else:
additional['X-Backend-Storage-Policy-Index'] = str(policy_index)
headers = [
self.generate_request_headers(req,
transfer=True,
additional=additional)
for _junk in range(n_outgoing)
]
for i, account in enumerate(accounts):
i = i % len(headers)
headers[i]['X-Account-Partition'] = account_partition
headers[i]['X-Account-Host'] = csv_append(
headers[i].get('X-Account-Host'), '%(ip)s:%(port)s' % account)
headers[i]['X-Account-Device'] = csv_append(
headers[i].get('X-Account-Device'), account['device'])
return headers
def gen_headers(hdrs_in=None, add_ts=False, add_user_agent=True):
hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict()
if add_ts:
hdrs_out['X-Timestamp'] = Timestamp.now().internal
if add_user_agent:
hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid()
return hdrs_out
def create_account_stat_table(self, conn, put_timestamp):
"""
Create account_stat table which is specific to the account DB.
Not a part of Pluggable Back-ends, internal to the baseline code.
:param conn: DB connection object
:param put_timestamp: put timestamp
"""
conn.executescript("""
CREATE TABLE account_stat (
account TEXT,
created_at TEXT,
put_timestamp TEXT DEFAULT '0',
delete_timestamp TEXT DEFAULT '0',
container_count INTEGER,
object_count INTEGER DEFAULT 0,
bytes_used INTEGER DEFAULT 0,
hash TEXT default '00000000000000000000000000000000',
id TEXT,
status TEXT DEFAULT '',
status_changed_at TEXT DEFAULT '0',
metadata TEXT DEFAULT ''
);
INSERT INTO account_stat (container_count) VALUES (0);
""")
conn.execute(
'''
UPDATE account_stat SET account = ?, created_at = ?, id = ?,
put_timestamp = ?, status_changed_at = ?
''', (self.account, Timestamp.now().internal, str(
uuid4()), put_timestamp, put_timestamp))
def _handle_sync_response(self,
node,
response,
info,
broker,
http,
different_region=False):
if is_success(response.status):
remote_info = json.loads(response.data)
if incorrect_policy_index(info, remote_info):
status_changed_at = Timestamp.now()
broker.set_storage_policy_index(
remote_info['storage_policy_index'],
timestamp=status_changed_at.internal)
sync_timestamps = ('created_at', 'put_timestamp',
'delete_timestamp')
if any(info[key] != remote_info[key] for key in sync_timestamps):
broker.merge_timestamps(*(remote_info[key]
for key in sync_timestamps))
# Grab remote's shard ranges, too
self._fetch_and_merge_shard_ranges(http, broker)
return super(ContainerReplicator,
self)._handle_sync_response(node, response, info, broker,
http, different_region)
def main():
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--config', default='/etc/swift/internal-client.conf',
help=('internal-client config file '
'(default: /etc/swift/internal-client.conf'))
parser.add_argument('--request-tries', type=int, default=3,
help='(default: 3)')
parser.add_argument('account', help='account from which to delete')
parser.add_argument('container', help='container from which to delete')
parser.add_argument(
'--prefix', default='',
help='only delete objects with this prefix (default: none)')
parser.add_argument(
'--marker', default='',
help='only delete objects after this marker (default: none)')
parser.add_argument(
'--end-marker', default='',
help='only delete objects before this end-marker (default: none)')
parser.add_argument(
'--timestamp', type=Timestamp, default=Timestamp.now(),
help='delete all objects as of this time (default: now)')
args = parser.parse_args()
swift = InternalClient(
args.config, 'Swift Container Deleter', args.request_tries)
for deleted, marker in mark_for_deletion(
swift, args.account, args.container,
args.marker, args.end_marker, args.prefix, args.timestamp):
if marker is None:
print('Finished. Marked %d objects for deletion.' % deleted)
else:
print('Marked %d objects for deletion, through %r' % (
deleted, marker))
def create_account_stat_table(self, conn, put_timestamp):
"""
Create account_stat table which is specific to the account DB.
Not a part of Pluggable Back-ends, internal to the baseline code.
:param conn: DB connection object
:param put_timestamp: put timestamp
"""
conn.executescript("""
CREATE TABLE account_stat (
account TEXT,
created_at TEXT,
put_timestamp TEXT DEFAULT '0',
delete_timestamp TEXT DEFAULT '0',
container_count INTEGER,
object_count INTEGER DEFAULT 0,
bytes_used INTEGER DEFAULT 0,
hash TEXT default '00000000000000000000000000000000',
id TEXT,
status TEXT DEFAULT '',
status_changed_at TEXT DEFAULT '0',
metadata TEXT DEFAULT ''
);
INSERT INTO account_stat (container_count) VALUES (0);
""")
conn.execute('''
UPDATE account_stat SET account = ?, created_at = ?, id = ?,
put_timestamp = ?, status_changed_at = ?
''', (self.account, Timestamp.now().internal, str(uuid4()),
put_timestamp, put_timestamp))
def set_storage_policy_index(self, policy_index, timestamp=None):
"""
Update the container_stat policy_index and status_changed_at.
"""
if timestamp is None:
timestamp = Timestamp.now().internal
def _setit(conn):
conn.execute('''
INSERT OR IGNORE INTO policy_stat (storage_policy_index)
VALUES (?)
''', (policy_index,))
conn.execute('''
UPDATE container_stat
SET storage_policy_index = ?,
status_changed_at = MAX(?, status_changed_at)
WHERE storage_policy_index <> ?
''', (policy_index, timestamp, policy_index))
conn.commit()
with self.get() as conn:
try:
_setit(conn)
except sqlite3.OperationalError as err:
if not any(msg in str(err) for msg in (
"no such column: storage_policy_index",
"no such table: policy_stat")):
raise
self._migrate_add_storage_policy(conn)
_setit(conn)
self._storage_policy_index = policy_index
def _make_diskfile(self, device='dev', partition='9',
account='a', container='c', obj='o', body='test',
extra_metadata=None, policy=None,
frag_index=None, timestamp=None, df_mgr=None,
commit=True, verify=True):
policy = policy or POLICIES.legacy
object_parts = account, container, obj
timestamp = Timestamp.now() if timestamp is None else timestamp
if df_mgr is None:
df_mgr = self.daemon._df_router[policy]
df = df_mgr.get_diskfile(
device, partition, *object_parts, policy=policy,
frag_index=frag_index)
write_diskfile(df, timestamp, data=body, extra_metadata=extra_metadata,
commit=commit)
if commit and verify:
# when we write and commit stub data, sanity check it's readable
# and not quarantined because of any validation check
with df.open():
self.assertEqual(b''.join(df.reader()), body)
# sanity checks
listing = os.listdir(df._datadir)
self.assertTrue(listing)
for filename in listing:
self.assertTrue(filename.startswith(timestamp.internal))
return df
def iter_task_to_expire(self, task_account_container_list,
my_index, divisor):
"""
Yields task expire info dict which consists of task_account,
task_container, task_object, timestamp_to_delete, and target_path
"""
for task_account, task_container in task_account_container_list:
for o in self.swift.iter_objects(task_account, task_container):
task_object = o['name'].encode('utf8')
try:
delete_timestamp, target_account, target_container, \
target_object = self.parse_task_obj(task_object)
except ValueError:
self.logger.exception('Unexcepted error handling task %r' %
task_object)
continue
if delete_timestamp > Timestamp.now():
# we shouldn't yield the object that doesn't reach
# the expiration date yet.
break
# Only one expirer daemon assigned for one task
if self.hash_mod('%s/%s' % (task_container, task_object),
divisor) != my_index:
continue
yield {'task_account': task_account,
'task_container': task_container,
'task_object': task_object,
'target_path': '/'.join([
target_account, target_container, target_object]),
'delete_timestamp': delete_timestamp}
def get_info(self):
now = Timestamp.now().internal
return {'container_count': 0,
'object_count': 0,
'bytes_used': 0,
'created_at': now,
'put_timestamp': now}
def _make_diskfile(self, device='dev', partition='9',
account='a', container='c', obj='o', body='test',
extra_metadata=None, policy=None,
frag_index=None, timestamp=None, df_mgr=None,
commit=True, verify=True):
policy = policy or POLICIES.legacy
object_parts = account, container, obj
timestamp = Timestamp.now() if timestamp is None else timestamp
if df_mgr is None:
df_mgr = self.daemon._df_router[policy]
df = df_mgr.get_diskfile(
device, partition, *object_parts, policy=policy,
frag_index=frag_index)
write_diskfile(df, timestamp, data=body, extra_metadata=extra_metadata,
commit=commit)
if commit and verify:
# when we write and commit stub data, sanity check it's readable
# and not quarantined because of any validation check
with df.open():
self.assertEqual(b''.join(df.reader()), body)
# sanity checks
listing = os.listdir(df._datadir)
self.assertTrue(listing)
for filename in listing:
self.assertTrue(filename.startswith(timestamp.internal))
return df
def main():
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--config', default='/etc/swift/internal-client.conf',
help=('internal-client config file '
'(default: /etc/swift/internal-client.conf'))
parser.add_argument('--request-tries', type=int, default=3,
help='(default: 3)')
parser.add_argument('account', help='account from which to delete')
parser.add_argument('container', help='container from which to delete')
parser.add_argument(
'--prefix', default='',
help='only delete objects with this prefix (default: none)')
parser.add_argument(
'--marker', default='',
help='only delete objects after this marker (default: none)')
parser.add_argument(
'--end-marker', default='',
help='only delete objects before this end-marker (default: none)')
parser.add_argument(
'--timestamp', type=Timestamp, default=Timestamp.now(),
help='delete all objects as of this time (default: now)')
args = parser.parse_args()
swift = InternalClient(
args.config, 'Swift Container Deleter', args.request_tries)
for deleted, marker in mark_for_deletion(
swift, args.account, args.container,
args.marker, args.end_marker, args.prefix, args.timestamp):
if marker is None:
print('Finished. Marked %d objects for deletion.' % deleted)
else:
print('Marked %d objects for deletion, through %r' % (
deleted, marker))
def repair_shard_ranges(broker, args):
if not broker.is_root_container():
print('WARNING: Shard containers cannot be repaired.')
print('This command should be used on a root container.')
return EXIT_ERROR
shard_ranges = broker.get_shard_ranges()
if not shard_ranges:
print('No shards found, nothing to do.')
return EXIT_SUCCESS
own_sr = broker.get_own_shard_range()
try:
acceptor_path, overlapping_donors = find_repair_solution(
shard_ranges, own_sr, args)
except ManageShardRangesException:
return EXIT_ERROR
if not acceptor_path:
return EXIT_SUCCESS
if not _proceed(args):
return EXIT_USER_QUIT
# merge changes to the broker...
# note: acceptors do not need to be modified since they already span the
# complete range
ts_now = Timestamp.now()
finalize_shrinking(broker, [], overlapping_donors, ts_now)
print('Updated %s donor shard ranges.' % len(overlapping_donors))
print('Run container-replicator to replicate the changes to other nodes.')
print('Run container-sharder on all nodes to repair shards.')
return EXIT_SUCCESS
def test_get_shard_ranges_invalid_shard_range(self):
sr = ShardRange('a/c', Timestamp.now())
bad_sr_data = dict(sr, name='bad_name')
body = json.dumps([bad_sr_data]).encode('ascii')
error_lines = self._check_get_shard_ranges_bad_data(body)
self.assertIn('Failed to get shard ranges', error_lines[0])
self.assertIn('ValueError', error_lines[0])
self.assertFalse(error_lines[1:])
def compact_shard_ranges(broker, args):
if not broker.is_root_container():
print('WARNING: Shard containers cannot be compacted.')
print('This command should be used on a root container.')
return 2
if not broker.is_sharded():
print('WARNING: Container is not yet sharded so cannot be compacted.')
return 2
shard_ranges = broker.get_shard_ranges()
if find_overlapping_ranges(
[sr for sr in shard_ranges if sr.state != ShardRange.SHRINKING]):
print('WARNING: Container has overlapping shard ranges so cannot be '
'compacted.')
return 2
compactible = find_compactible_shard_sequences(broker,
args.shrink_threshold,
args.expansion_limit,
args.max_shrinking,
args.max_expanding)
if not compactible:
print('No shards identified for compaction.')
return 0
for sequence in compactible:
if sequence[-1].state not in (ShardRange.ACTIVE, ShardRange.SHARDED):
print('ERROR: acceptor not in correct state: %s' % sequence[-1],
file=sys.stderr)
return 1
if not args.yes:
for sequence in compactible:
acceptor = sequence[-1]
donors = sequence[:-1]
print('Donor shard range(s) with total of %d objects:' %
donors.object_count)
for donor in donors:
_print_shard_range(donor, level=1)
print('can be compacted into acceptor shard range:')
_print_shard_range(acceptor, level=1)
print('Once applied to the broker these changes will result in shard '
'range compaction the next time the sharder runs.')
choice = input('Do you want to apply these changes? [y/N]')
if choice != 'y':
print('No changes applied')
return 0
timestamp = Timestamp.now()
acceptor_ranges, shrinking_ranges = process_compactible_shard_sequences(
compactible, timestamp)
finalize_shrinking(broker, acceptor_ranges, shrinking_ranges, timestamp)
print('Updated %s shard sequences for compaction.' % len(compactible))
print('Run container-replicator to replicate the changes to other '
'nodes.')
print('Run container-sharder on all nodes to compact shards.')
return 0
def _move_broker_to_sharded_state(self, broker):
epoch = Timestamp.now()
broker.enable_sharding(epoch)
self.assertTrue(broker.set_sharding_state())
self.assertTrue(broker.set_sharded_state())
own_sr = broker.get_own_shard_range()
own_sr.update_state(ShardRange.SHARDED, epoch)
broker.merge_shard_ranges([own_sr])
return epoch
def get_info(self):
now = Timestamp.now().internal
return {
'container_count': 0,
'object_count': 0,
'bytes_used': 0,
'created_at': now,
'put_timestamp': now
}
def iter_task_containers_to_expire(self):
"""
Yields container name under the expiring_objects_account if
the container name (i.e. timestamp) is past.
"""
for c in self.swift.iter_containers(self.expiring_objects_account):
task_container = str(c['name'])
timestamp = Timestamp(task_container)
if timestamp > Timestamp.now():
break
yield task_container
def iter_task_containers_to_expire(self, task_account):
"""
Yields task_container names under the task_account if the delete at
timestamp of task_container is past.
"""
for c in self.swift.iter_containers(task_account,
prefix=self.task_container_prefix):
task_container = str(c['name'])
timestamp = self.delete_at_time_of_task_container(task_container)
if timestamp > Timestamp.now():
break
yield task_container
def iter_task_containers_to_expire(self, task_account):
"""
Yields task_container names under the task_account if the delete at
timestamp of task_container is past.
"""
for c in self.swift.iter_containers(task_account,
prefix=self.task_container_prefix):
task_container = str(c['name'])
timestamp = self.delete_at_time_of_task_container(task_container)
if timestamp > Timestamp.now():
break
yield task_container
def test_get_shard_ranges_missing_record_type(self):
base = Controller(self.app)
req = Request.blank('/v1/a/c/o', method='PUT')
sr = ShardRange('a/c', Timestamp.now())
body = json.dumps([dict(sr)]).encode('ascii')
with mocked_http_conn(
200, 200, body_iter=iter([b'', body])):
actual = base._get_shard_ranges(req, 'a', 'c', '1_test')
self.assertIsNone(actual)
error_lines = self.app.logger.get_lines_for_level('error')
self.assertIn('Failed to get shard ranges', error_lines[0])
self.assertIn('unexpected record type', error_lines[0])
self.assertIn('/a/c', error_lines[0])
self.assertFalse(error_lines[1:])
def gen_headers(hdrs_in=None, add_ts=True):
"""
Get the headers ready for a request. All requests should have a User-Agent
string, but if one is passed in don't over-write it. Not all requests will
need an X-Timestamp, but if one is passed in do not over-write it.
:param headers: dict or None, base for HTTP headers
:param add_ts: boolean, should be True for any "unsafe" HTTP request
:returns: HeaderKeyDict based on headers and ready for the request
"""
hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict()
if add_ts and 'X-Timestamp' not in hdrs_out:
hdrs_out['X-Timestamp'] = Timestamp.now().internal
if 'user-agent' not in hdrs_out:
hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid()
return hdrs_out
def gen_headers(hdrs_in=None, add_ts=True):
"""
Get the headers ready for a request. All requests should have a User-Agent
string, but if one is passed in don't over-write it. Not all requests will
need an X-Timestamp, but if one is passed in do not over-write it.
:param headers: dict or None, base for HTTP headers
:param add_ts: boolean, should be True for any "unsafe" HTTP request
:returns: HeaderKeyDict based on headers and ready for the request
"""
hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict()
if add_ts and 'X-Timestamp' not in hdrs_out:
hdrs_out['X-Timestamp'] = Timestamp.now().internal
if 'user-agent' not in hdrs_out:
hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid()
return hdrs_out
def test_direct_head_object_not_found(self):
important_timestamp = Timestamp.now().internal
stub_headers = {'X-Backend-Important-Timestamp': important_timestamp}
with mocked_http_conn(404, headers=stub_headers) as conn:
with self.assertRaises(ClientException) as raised:
direct_client.direct_head_object(self.node, self.part,
self.account, self.container,
self.obj)
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(raised.exception.http_status, 404)
self.assertEqual(
raised.exception.http_headers['x-backend-important-timestamp'],
important_timestamp)
def test_direct_head_object_not_found(self):
important_timestamp = Timestamp.now().internal
stub_headers = {'X-Backend-Important-Timestamp': important_timestamp}
with mocked_http_conn(404, headers=stub_headers) as conn:
with self.assertRaises(ClientException) as raised:
direct_client.direct_head_object(
self.node, self.part, self.account, self.container,
self.obj)
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(raised.exception.http_status, 404)
self.assertEqual(
raised.exception.http_headers['x-backend-important-timestamp'],
important_timestamp)
def test_direct_head_container_deleted(self):
important_timestamp = Timestamp.now().internal
headers = HeaderKeyDict({'X-Backend-Important-Timestamp':
important_timestamp})
with mocked_http_conn(404, headers) as conn:
with self.assertRaises(ClientException) as raised:
direct_client.direct_head_container(
self.node, self.part, self.account, self.container)
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(raised.exception.http_status, 404)
self.assertEqual(raised.exception.http_headers, headers)
def iter_task_to_expire(self, task_account_container_list, my_index,
divisor):
"""
Yields task expire info dict which consists of task_account,
task_container, task_object, timestamp_to_delete, and target_path
"""
for task_account, task_container in task_account_container_list:
for o in self.swift.iter_objects(task_account, task_container):
if six.PY2:
task_object = o['name'].encode('utf8')
else:
task_object = o['name']
try:
delete_timestamp, target_account, target_container, \
target_object = parse_task_obj(task_object)
except ValueError:
self.logger.exception('Unexcepted error handling task %r' %
task_object)
continue
if delete_timestamp > Timestamp.now():
# we shouldn't yield the object that doesn't reach
# the expiration date yet.
break
# Only one expirer daemon assigned for one task
if self.hash_mod('%s/%s' % (task_container, task_object),
divisor) != my_index:
continue
is_async = o.get('content_type') == ASYNC_DELETE_TYPE
yield {
'task_account':
task_account,
'task_container':
task_container,
'task_object':
task_object,
'target_path':
'/'.join([target_account, target_container,
target_object]),
'delete_timestamp':
delete_timestamp,
'is_async_delete':
is_async
}
def _get_synced_replication_info(self, broker, remote_info):
"""
Sync the remote_info storage_policy_index if needed and return the
newly synced replication info.
:param broker: the database broker
:param remote_info: the remote replication info
:returns: local broker replication info
"""
info = broker.get_replication_info()
if incorrect_policy_index(info, remote_info):
status_changed_at = Timestamp.now().internal
broker.set_storage_policy_index(
remote_info['storage_policy_index'],
timestamp=status_changed_at)
info = broker.get_replication_info()
return info
def _handle_sync_response(self, node, response, info, broker, http,
different_region):
parent = super(ContainerReplicator, self)
if is_success(response.status):
remote_info = json.loads(response.data)
if incorrect_policy_index(info, remote_info):
status_changed_at = Timestamp.now()
broker.set_storage_policy_index(
remote_info['storage_policy_index'],
timestamp=status_changed_at.internal)
sync_timestamps = ('created_at', 'put_timestamp',
'delete_timestamp')
if any(info[key] != remote_info[key] for key in sync_timestamps):
broker.merge_timestamps(*(remote_info[key] for key in
sync_timestamps))
rv = parent._handle_sync_response(
node, response, info, broker, http, different_region)
return rv
def _get_synced_replication_info(self, broker, remote_info):
"""
Sync the remote_info storage_policy_index if needed and return the
newly synced replication info.
:param broker: the database broker
:param remote_info: the remote replication info
:returns: local broker replication info
"""
info = broker.get_replication_info()
if incorrect_policy_index(info, remote_info):
status_changed_at = Timestamp.now().internal
broker.set_storage_policy_index(
remote_info['storage_policy_index'],
timestamp=status_changed_at)
info = broker.get_replication_info()
return info
def _handle_sync_response(self, node, response, info, broker, http,
different_region):
parent = super(ContainerReplicator, self)
if is_success(response.status):
remote_info = json.loads(response.data)
if incorrect_policy_index(info, remote_info):
status_changed_at = Timestamp.now()
broker.set_storage_policy_index(
remote_info['storage_policy_index'],
timestamp=status_changed_at.internal)
sync_timestamps = ('created_at', 'put_timestamp',
'delete_timestamp')
if any(info[key] != remote_info[key] for key in sync_timestamps):
broker.merge_timestamps(*(remote_info[key]
for key in sync_timestamps))
rv = parent._handle_sync_response(node, response, info, broker, http,
different_region)
return rv
def test_direct_head_container_deleted(self):
important_timestamp = Timestamp.now().internal
headers = HeaderKeyDict(
{'X-Backend-Important-Timestamp': important_timestamp})
with mocked_http_conn(404, headers) as conn:
with self.assertRaises(ClientException) as raised:
direct_client.direct_head_container(self.node, self.part,
self.account,
self.container)
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(raised.exception.http_status, 404)
self.assertEqual(raised.exception.http_headers, headers)
def merge_timestamps(self, created_at, put_timestamp, delete_timestamp):
"""
Used in replication to handle updating timestamps.
:param created_at: create timestamp
:param put_timestamp: put timestamp
:param delete_timestamp: delete timestamp
"""
with self.get() as conn:
old_status = self._is_deleted(conn)
conn.execute('''
UPDATE %s_stat SET created_at=MIN(?, created_at),
put_timestamp=MAX(?, put_timestamp),
delete_timestamp=MAX(?, delete_timestamp)
''' % self.db_type, (created_at, put_timestamp, delete_timestamp))
if old_status != self._is_deleted(conn):
timestamp = Timestamp.now()
self._update_status_changed_at(conn, timestamp.internal)
conn.commit()
def merge_timestamps(self, created_at, put_timestamp, delete_timestamp):
"""
Used in replication to handle updating timestamps.
:param created_at: create timestamp
:param put_timestamp: put timestamp
:param delete_timestamp: delete timestamp
"""
with self.get() as conn:
old_status = self._is_deleted(conn)
conn.execute('''
UPDATE %s_stat SET created_at=MIN(?, created_at),
put_timestamp=MAX(?, put_timestamp),
delete_timestamp=MAX(?, delete_timestamp)
''' % self.db_type, (created_at, put_timestamp, delete_timestamp))
if old_status != self._is_deleted(conn):
timestamp = Timestamp.now()
self._update_status_changed_at(conn, timestamp.internal)
conn.commit()
def test_direct_head_container_deleted(self):
important_timestamp = Timestamp.now().internal
headers = HeaderKeyDict({'X-Backend-Important-Timestamp':
important_timestamp})
with mocked_http_conn(404, headers) as conn:
try:
direct_client.direct_head_container(
self.node, self.part, self.account, self.container)
except Exception as err:
self.assertTrue(isinstance(err, ClientException))
else:
self.fail('ClientException not raised')
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(err.http_status, 404)
self.assertEqual(err.http_headers, headers)
def test_direct_head_container_deleted(self):
important_timestamp = Timestamp.now().internal
headers = HeaderKeyDict({'X-Backend-Important-Timestamp':
important_timestamp})
with mocked_http_conn(404, headers) as conn:
try:
direct_client.direct_head_container(
self.node, self.part, self.account, self.container)
except Exception as err:
self.assertTrue(isinstance(err, ClientException))
else:
self.fail('ClientException not raised')
self.assertEqual(conn.host, self.node['ip'])
self.assertEqual(conn.port, self.node['port'])
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(err.http_status, 404)
self.assertEqual(err.http_headers, headers)
def _handle_sync_response(self, node, response, info, broker, http,
different_region=False):
if is_success(response.status):
remote_info = json.loads(response.data)
if incorrect_policy_index(info, remote_info):
status_changed_at = Timestamp.now()
broker.set_storage_policy_index(
remote_info['storage_policy_index'],
timestamp=status_changed_at.internal)
sync_timestamps = ('created_at', 'put_timestamp',
'delete_timestamp')
if any(info[key] != remote_info[key] for key in sync_timestamps):
broker.merge_timestamps(*(remote_info[key] for key in
sync_timestamps))
if 'shard_max_row' in remote_info:
# Grab remote's shard ranges, too
self._fetch_and_merge_shard_ranges(http, broker)
return super(ContainerReplicator, self)._handle_sync_response(
node, response, info, broker, http, different_region)
def test_replace(self):
broker = self._make_broker()
broker.update_metadata({'X-Container-Sysmeta-Sharding':
(True, Timestamp.now().internal)})
input_file = os.path.join(self.testdir, 'shards')
with open(input_file, 'w') as fd:
json.dump(self.shard_data, fd)
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
main([broker.db_file, 'replace', input_file])
expected = [
'No shard ranges found to delete.',
'Injected 10 shard ranges.',
'Run container-replicator to replicate them to other nodes.',
'Use the enable sub-command to enable sharding.']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self.assertEqual(
[(data['lower'], data['upper']) for data in self.shard_data],
[(sr.lower_str, sr.upper_str) for sr in broker.get_shard_ranges()])
def test_replace(self):
broker = self._make_broker()
broker.update_metadata({'X-Container-Sysmeta-Sharding':
(True, Timestamp.now().internal)})
input_file = os.path.join(self.testdir, 'shards')
with open(input_file, 'wb') as fd:
json.dump(self.shard_data, fd)
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
main([broker.db_file, 'replace', input_file])
expected = [
'No shard ranges found to delete.',
'Injected 10 shard ranges.',
'Run container-replicator to replicate them to other nodes.',
'Use the enable sub-command to enable sharding.']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self.assertEqual(
[(data['lower'], data['upper']) for data in self.shard_data],
[(sr.lower_str, sr.upper_str) for sr in broker.get_shard_ranges()])
def delete_shard_ranges(broker, args):
shard_ranges = broker.get_shard_ranges()
if not shard_ranges:
print("No shard ranges found to delete.")
return 0
while not args.force:
print('This will delete existing %d shard ranges.' % len(shard_ranges))
if broker.get_db_state() != UNSHARDED:
print('WARNING: Be very cautious about deleting existing shard '
'ranges. Deleting all ranges in this db does not guarantee '
'deletion of all ranges on all replicas of the db.')
print(' - this db is in state %s' % broker.get_db_state())
print(' - %d existing shard ranges have started sharding' %
[sr.state != ShardRange.FOUND
for sr in shard_ranges].count(True))
choice = input('Do you want to show the existing ranges [s], '
'delete the existing ranges [yes] '
'or quit without deleting [q]? ')
if choice == 's':
show_shard_ranges(broker, args)
continue
elif choice == 'q':
return 1
elif choice == 'yes':
break
else:
print('Please make a valid choice.')
print()
now = Timestamp.now()
for sr in shard_ranges:
sr.deleted = 1
sr.timestamp = now
broker.merge_shard_ranges(shard_ranges)
print('Deleted %s existing shard ranges.' % len(shard_ranges))
return 0
def create_container_info_table(self, conn, put_timestamp,
storage_policy_index):
"""
Create the container_info table which is specific to the container DB.
Not a part of Pluggable Back-ends, internal to the baseline code.
Also creates the container_stat view.
:param conn: DB connection object
:param put_timestamp: put timestamp
:param storage_policy_index: storage policy index
"""
if put_timestamp is None:
put_timestamp = Timestamp(0).internal
# The container_stat view is for compatibility; old versions of Swift
# expected a container_stat table with columns "object_count" and
# "bytes_used", but when that stuff became per-storage-policy and
# moved to the policy_stat table, we stopped creating those columns in
# container_stat.
#
# To retain compatibility, we create the container_stat view with some
# triggers to make it behave like the old container_stat table. This
# way, if an old version of Swift encounters a database with the new
# schema, it can still work.
#
# Note that this can occur during a rolling Swift upgrade if a DB gets
# rsynced from an old node to a new, so it's necessary for
# availability during upgrades. The fact that it enables downgrades is
# a nice bonus.
conn.executescript(CONTAINER_INFO_TABLE_SCRIPT +
CONTAINER_STAT_VIEW_SCRIPT)
conn.execute("""
INSERT INTO container_info (account, container, created_at, id,
put_timestamp, status_changed_at, storage_policy_index)
VALUES (?, ?, ?, ?, ?, ?, ?);
""", (self.account, self.container, Timestamp.now().internal,
str(uuid4()), put_timestamp, put_timestamp,
storage_policy_index))
def repair_shard_ranges(broker, args):
if not broker.is_root_container():
print('WARNING: Shard containers cannot be repaired.')
print('This command should be used on a root container.')
return 2
shard_ranges = broker.get_shard_ranges()
if not shard_ranges:
print('No shards found, nothing to do.')
return 0
own_sr = broker.get_own_shard_range()
try:
acceptor_path, overlapping_donors = find_repair_solution(
shard_ranges, own_sr, args)
except ManageShardRangesException:
return 1
if not acceptor_path:
return 0
if not args.yes:
choice = input('Do you want to apply these changes to the container '
'DB? [yes/N]')
if choice != 'yes':
print('No changes applied')
return 0
# merge changes to the broker...
# note: acceptors do not need to be modified since they already span the
# complete range
ts_now = Timestamp.now()
finalize_shrinking(broker, [], overlapping_donors, ts_now)
print('Updated %s donor shard ranges.' % len(overlapping_donors))
print('Run container-replicator to replicate the changes to other nodes.')
print('Run container-sharder on all nodes to repair shards.')
return 0
def _backend_requests(self, req, n_outgoing, account_partition, accounts,
policy_index=None):
additional = {'X-Timestamp': Timestamp.now().internal}
if policy_index is None:
additional['X-Backend-Storage-Policy-Default'] = \
int(POLICIES.default)
else:
additional['X-Backend-Storage-Policy-Index'] = str(policy_index)
headers = [self.generate_request_headers(req, transfer=True,
additional=additional)
for _junk in range(n_outgoing)]
for i, account in enumerate(accounts):
i = i % len(headers)
headers[i]['X-Account-Partition'] = account_partition
headers[i]['X-Account-Host'] = csv_append(
headers[i].get('X-Account-Host'),
'%(ip)s:%(port)s' % account)
headers[i]['X-Account-Device'] = csv_append(
headers[i].get('X-Account-Device'),
account['device'])
return headers
def gen_headers(hdrs_in=None, add_ts=False):
hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict()
if add_ts:
hdrs_out['X-Timestamp'] = Timestamp.now().internal
hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid()
return hdrs_out
def mock_timestamp_now(now=None):
if now is None:
now = Timestamp.now()
with mocklib.patch('swift.common.utils.Timestamp.now',
classmethod(lambda c: now)):
yield now
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container = split_and_validate_path(req, 3, 4)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
if container: # put account container
if 'x-timestamp' not in req.headers:
timestamp = Timestamp.now()
else:
timestamp = valid_timestamp(req)
pending_timeout = None
container_policy_index = \
req.headers.get('X-Backend-Storage-Policy-Index', 0)
if 'x-trans-id' in req.headers:
pending_timeout = 3
broker = self._get_account_broker(drive, part, account,
pending_timeout=pending_timeout)
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
except DatabaseAlreadyExists:
pass
if req.headers.get('x-account-override-deleted', 'no').lower() != \
'yes' and broker.is_deleted():
return HTTPNotFound(request=req)
broker.put_container(container, req.headers['x-put-timestamp'],
req.headers['x-delete-timestamp'],
req.headers['x-object-count'],
req.headers['x-bytes-used'],
container_policy_index)
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
timestamp = valid_timestamp(req)
broker = self._get_account_broker(drive, part, account)
if not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
created = True
except DatabaseAlreadyExists:
created = False
elif broker.is_status_deleted():
return self._deleted_response(broker, req, HTTPForbidden,
body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp.internal)
if broker.is_deleted():
return HTTPConflict(request=req)
metadata = {}
metadata.update((key, (value, timestamp.internal))
for key, value in req.headers.items()
if is_sys_or_user_meta('account', key))
if metadata:
broker.update_metadata(metadata, validate_metadata=True)
if created:
return HTTPCreated(request=req)
else:
return HTTPAccepted(request=req)
def blank(cls, *args, **kwargs):
req = super(Request, cls).blank(*args, **kwargs)
if 'X-Timestamp' not in req.headers:
req.headers['X-Timestamp'] = Timestamp.now().normal
return req
def mock_timestamp_now(now=None):
if now is None:
now = Timestamp.now()
with mocklib.patch('swift.common.utils.Timestamp.now',
classmethod(lambda c: now)):
yield now
def add_to_reconciler_queue(container_ring, account, container, obj,
obj_policy_index, obj_timestamp, op,
force=False, conn_timeout=5, response_timeout=15):
"""
Add an object to the container reconciler's queue. This will cause the
container reconciler to move it from its current storage policy index to
the correct storage policy index.
:param container_ring: container ring
:param account: the misplaced object's account
:param container: the misplaced object's container
:param obj: the misplaced object
:param obj_policy_index: the policy index where the misplaced object
currently is
:param obj_timestamp: the misplaced object's X-Timestamp. We need this to
ensure that the reconciler doesn't overwrite a newer
object with an older one.
:param op: the method of the operation (DELETE or PUT)
:param force: over-write queue entries newer than obj_timestamp
:param conn_timeout: max time to wait for connection to container server
:param response_timeout: max time to wait for response from container
server
:returns: .misplaced_object container name, False on failure. "Success"
means a majority of containers got the update.
"""
container_name = get_reconciler_container_name(obj_timestamp)
object_name = get_reconciler_obj_name(obj_policy_index, account,
container, obj)
if force:
# this allows an operator to re-enqueue an object that has
# already been popped from the queue to be reprocessed, but
# could potentially prevent out of order updates from making it
# into the queue
x_timestamp = Timestamp.now().internal
else:
x_timestamp = obj_timestamp
q_op_type = get_reconciler_content_type(op)
headers = {
'X-Size': 0,
'X-Etag': obj_timestamp,
'X-Timestamp': x_timestamp,
'X-Content-Type': q_op_type,
}
def _check_success(*args, **kwargs):
try:
direct_put_container_object(*args, **kwargs)
return 1
except (ClientException, Timeout, socket.error):
return 0
pile = GreenPile()
part, nodes = container_ring.get_nodes(MISPLACED_OBJECTS_ACCOUNT,
container_name)
for node in nodes:
pile.spawn(_check_success, node, part, MISPLACED_OBJECTS_ACCOUNT,
container_name, object_name, headers=headers,
conn_timeout=conn_timeout,
response_timeout=response_timeout)
successes = sum(pile)
if successes >= majority_size(len(nodes)):
return container_name
else:
return False
def add_to_reconciler_queue(container_ring,
account,
container,
obj,
obj_policy_index,
obj_timestamp,
op,
force=False,
conn_timeout=5,
response_timeout=15):
"""
Add an object to the container reconciler's queue. This will cause the
container reconciler to move it from its current storage policy index to
the correct storage policy index.
:param container_ring: container ring
:param account: the misplaced object's account
:param container: the misplaced object's container
:param obj: the misplaced object
:param obj_policy_index: the policy index where the misplaced object
currently is
:param obj_timestamp: the misplaced object's X-Timestamp. We need this to
ensure that the reconciler doesn't overwrite a newer
object with an older one.
:param op: the method of the operation (DELETE or PUT)
:param force: over-write queue entries newer than obj_timestamp
:param conn_timeout: max time to wait for connection to container server
:param response_timeout: max time to wait for response from container
server
:returns: .misplaced_object container name, False on failure. "Success"
means a majority of containers got the update.
"""
container_name = get_reconciler_container_name(obj_timestamp)
object_name = get_reconciler_obj_name(obj_policy_index, account, container,
obj)
if force:
# this allows an operator to re-enqueue an object that has
# already been popped from the queue to be reprocessed, but
# could potentially prevent out of order updates from making it
# into the queue
x_timestamp = Timestamp.now().internal
else:
x_timestamp = obj_timestamp
q_op_type = get_reconciler_content_type(op)
headers = {
'X-Size': 0,
'X-Etag': obj_timestamp,
'X-Timestamp': x_timestamp,
'X-Content-Type': q_op_type,
}
def _check_success(*args, **kwargs):
try:
direct_put_container_object(*args, **kwargs)
return 1
except (ClientException, Timeout, socket.error):
return 0
pile = GreenPile()
part, nodes = container_ring.get_nodes(MISPLACED_OBJECTS_ACCOUNT,
container_name)
for node in nodes:
pile.spawn(_check_success,
node,
part,
MISPLACED_OBJECTS_ACCOUNT,
container_name,
object_name,
headers=headers,
conn_timeout=conn_timeout,
response_timeout=response_timeout)
successes = sum(pile)
if successes >= majority_size(len(nodes)):
return container_name
else:
return False
def reap_object(self, account, container, container_partition,
container_nodes, obj, policy_index):
"""
Deletes the given object by issuing a delete request to each node for
the object. The format of the delete request is such that each object
server will update a corresponding container server, removing the
object from the container's listing.
This function returns nothing and should raise no exception but only
update various self.stats_* values for what occurs.
:param account: The name of the account for the object.
:param container: The name of the container for the object.
:param container_partition: The partition for the container on the
container ring.
:param container_nodes: The primary node dicts for the container.
:param obj: The name of the object to delete.
:param policy_index: The storage policy index of the object's container
* See also: :func:`swift.common.ring.Ring.get_nodes` for a description
of the container node dicts.
"""
cnodes = itertools.cycle(container_nodes)
try:
ring = self.get_object_ring(policy_index)
except PolicyError:
self.stats_objects_remaining += 1
self.logger.increment('objects_remaining')
return
part, nodes = ring.get_nodes(account, container, obj)
successes = 0
failures = 0
timestamp = Timestamp.now()
for node in nodes:
cnode = next(cnodes)
try:
direct_delete_object(
node, part, account, container, obj,
conn_timeout=self.conn_timeout,
response_timeout=self.node_timeout,
headers={'X-Container-Host': '%(ip)s:%(port)s' % cnode,
'X-Container-Partition': str(container_partition),
'X-Container-Device': cnode['device'],
'X-Backend-Storage-Policy-Index': policy_index,
'X-Timestamp': timestamp.internal})
successes += 1
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1
self.logger.increment('return_codes.2')
except ClientException as err:
if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception(
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
failures += 1
self.logger.increment('objects_failures')
self.stats_return_codes[err.http_status // 100] = \
self.stats_return_codes.get(err.http_status // 100, 0) + 1
self.logger.increment(
'return_codes.%d' % (err.http_status // 100,))
except (Timeout, socket.error) as err:
failures += 1
self.logger.increment('objects_failures')
self.logger.error(
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
node)
if successes > failures:
self.stats_objects_deleted += 1
self.logger.increment('objects_deleted')
elif not successes:
self.stats_objects_remaining += 1
self.logger.increment('objects_remaining')
else:
self.stats_objects_possibly_remaining += 1
self.logger.increment('objects_possibly_remaining')
def reap_container(self, account, account_partition, account_nodes,
container):
"""
Deletes the data and the container itself for the given container. This
will call :func:`reap_object` up to sqrt(self.concurrency) times
concurrently for the objects in the container.
If there is any exception while deleting a single object, the process
will continue for any other objects in the container and the failed
objects will be tried again the next time this function is called with
the same parameters.
If there is any exception while listing the objects for deletion, the
process will stop (but will obviously be tried again the next time this
function is called with the same parameters). This is a possibility
since the listing comes from querying just the primary remote container
server.
Once all objects have been attempted to be deleted, the container
itself will be attempted to be deleted by sending a delete request to
all container nodes. The format of the delete request is such that each
container server will update a corresponding account server, removing
the container from the account's listing.
This function returns nothing and should raise no exception but only
update various self.stats_* values for what occurs.
:param account: The name of the account for the container.
:param account_partition: The partition for the account on the account
ring.
:param account_nodes: The primary node dicts for the account.
:param container: The name of the container to delete.
* See also: :func:`swift.common.ring.Ring.get_nodes` for a description
of the account node dicts.
"""
account_nodes = list(account_nodes)
part, nodes = self.get_container_ring().get_nodes(account, container)
node = nodes[-1]
pool = GreenPool(size=self.object_concurrency)
marker = ''
while True:
objects = None
try:
headers, objects = direct_get_container(
node, part, account, container,
marker=marker,
conn_timeout=self.conn_timeout,
response_timeout=self.node_timeout)
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1
self.logger.increment('return_codes.2')
except ClientException as err:
if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception(
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
self.stats_return_codes[err.http_status // 100] = \
self.stats_return_codes.get(err.http_status // 100, 0) + 1
self.logger.increment(
'return_codes.%d' % (err.http_status // 100,))
except (Timeout, socket.error) as err:
self.logger.error(
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
node)
if not objects:
break
try:
policy_index = headers.get('X-Backend-Storage-Policy-Index', 0)
policy = POLICIES.get_by_index(policy_index)
if not policy:
self.logger.error('ERROR: invalid storage policy index: %r'
% policy_index)
for obj in objects:
obj_name = obj['name']
if isinstance(obj_name, six.text_type):
obj_name = obj_name.encode('utf8')
pool.spawn(self.reap_object, account, container, part,
nodes, obj_name, policy_index)
pool.waitall()
except (Exception, Timeout):
self.logger.exception(_('Exception with objects for container '
'%(container)s for account %(account)s'
),
{'container': container,
'account': account})
marker = objects[-1]['name']
if marker == '':
break
successes = 0
failures = 0
timestamp = Timestamp.now()
for node in nodes:
anode = account_nodes.pop()
try:
direct_delete_container(
node, part, account, container,
conn_timeout=self.conn_timeout,
response_timeout=self.node_timeout,
headers={'X-Account-Host': '%(ip)s:%(port)s' % anode,
'X-Account-Partition': str(account_partition),
'X-Account-Device': anode['device'],
'X-Account-Override-Deleted': 'yes',
'X-Timestamp': timestamp.internal})
successes += 1
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1
self.logger.increment('return_codes.2')
except ClientException as err:
if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception(
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
failures += 1
self.logger.increment('containers_failures')
self.stats_return_codes[err.http_status // 100] = \
self.stats_return_codes.get(err.http_status // 100, 0) + 1
self.logger.increment(
'return_codes.%d' % (err.http_status // 100,))
except (Timeout, socket.error) as err:
self.logger.error(
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
node)
failures += 1
self.logger.increment('containers_failures')
if successes > failures:
self.stats_containers_deleted += 1
self.logger.increment('containers_deleted')
elif not successes:
self.stats_containers_remaining += 1
self.logger.increment('containers_remaining')
else:
self.stats_containers_possibly_remaining += 1
self.logger.increment('containers_possibly_remaining')
def test_info(self):
broker = self._make_broker()
broker.update_metadata({'X-Container-Sysmeta-Sharding':
(True, Timestamp.now().internal)})
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
main([broker.db_file, 'info'])
expected = ['Sharding enabled = True',
'Own shard range: None',
'db_state = unsharded',
'Metadata:',
' X-Container-Sysmeta-Sharding = True']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
retiring_db_id = broker.get_info()['id']
broker.merge_shard_ranges(ShardRange('.shards/cc', Timestamp.now()))
epoch = Timestamp.now()
with mock_timestamp_now(epoch) as now:
broker.enable_sharding(epoch)
self.assertTrue(broker.set_sharding_state())
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
with mock_timestamp_now(now):
main([broker.db_file, 'info'])
expected = ['Sharding enabled = True',
'Own shard range: {',
' "bytes_used": 0, ',
' "deleted": 0, ',
' "epoch": "%s", ' % epoch.internal,
' "lower": "", ',
' "meta_timestamp": "%s", ' % now.internal,
' "name": "a/c", ',
' "object_count": 0, ',
' "state": "sharding", ',
' "state_timestamp": "%s", ' % now.internal,
' "timestamp": "%s", ' % now.internal,
' "upper": ""',
'}',
'db_state = sharding',
'Retiring db id: %s' % retiring_db_id,
'Cleaving context: {',
' "cleave_to_row": null, ',
' "cleaving_done": false, ',
' "cursor": "", ',
' "last_cleave_to_row": null, ',
' "max_row": -1, ',
' "misplaced_done": false, ',
' "ranges_done": 0, ',
' "ranges_todo": 0, ',
' "ref": "%s"' % retiring_db_id,
'}',
'Metadata:',
' X-Container-Sysmeta-Sharding = True']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
self.assertTrue(broker.set_sharded_state())
out = StringIO()
err = StringIO()
with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
with mock_timestamp_now(now):
main([broker.db_file, 'info'])
expected = ['Sharding enabled = True',
'Own shard range: {',
' "bytes_used": 0, ',
' "deleted": 0, ',
' "epoch": "%s", ' % epoch.internal,
' "lower": "", ',
' "meta_timestamp": "%s", ' % now.internal,
' "name": "a/c", ',
' "object_count": 0, ',
' "state": "sharding", ',
' "state_timestamp": "%s", ' % now.internal,
' "timestamp": "%s", ' % now.internal,
' "upper": ""',
'}',
'db_state = sharded',
'Metadata:',
' X-Container-Sysmeta-Sharding = True']
self.assertEqual(expected, out.getvalue().splitlines())
self.assertEqual(['Loaded db broker for a/c.'],
err.getvalue().splitlines())
def mark_for_deletion(swift, account, container, marker, end_marker,
prefix, timestamp=None, yield_time=10):
'''
Enqueue jobs to async-delete some portion of a container's namespace
:param swift: InternalClient to use
:param account: account to delete from
:param container: container to delete from
:param marker: only delete objects after this name
:param end_marker: only delete objects before this name. Use ``None`` or
empty string to delete to the end of the namespace.
:param prefix: only delete objects starting with this prefix
:param timestamp: delete all objects as of this time. If ``None``, the
current time will be used.
:param yield_time: approximate period with which intermediate results
should be returned. If ``None``, disable intermediate
results.
:returns: If ``yield_time`` is ``None``, the number of objects marked for
deletion. Otherwise, a generator that will yield out tuples of
``(number of marked objects, last object name)`` approximately
every ``yield_time`` seconds. The final tuple will have ``None``
as the second element. This form allows you to retry when an
error occurs partway through while minimizing duplicate work.
'''
if timestamp is None:
timestamp = Timestamp.now()
def enqueue_deletes():
deleted = 0
obj_iter = swift.iter_objects(
account, container,
marker=marker, end_marker=end_marker, prefix=prefix)
time_marker = time.time()
while True:
to_delete = [obj['name'] for obj in itertools.islice(
obj_iter, OBJECTS_PER_UPDATE)]
if not to_delete:
break
delete_jobs = make_delete_jobs(
account, container, to_delete, timestamp)
swift.make_request(
'UPDATE',
swift.make_path('.expiring_objects', str(int(timestamp))),
headers={'X-Backend-Allow-Private-Methods': 'True',
'X-Backend-Storage-Policy-Index': '0',
'X-Timestamp': timestamp.internal},
acceptable_statuses=(2,),
body_file=io.BytesIO(json.dumps(delete_jobs).encode('ascii')))
deleted += len(delete_jobs)
if yield_time is not None and \
time.time() - time_marker > yield_time:
yield deleted, to_delete[-1]
time_marker = time.time()
yield deleted, None
if yield_time is None:
for deleted, marker in enqueue_deletes():
if marker is None:
return deleted
else:
return enqueue_deletes()
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container = split_and_validate_path(req, 3, 4)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
if not self.check_free_space(drive):
return HTTPInsufficientStorage(drive=drive, request=req)
if container: # put account container
if 'x-timestamp' not in req.headers:
timestamp = Timestamp.now()
else:
timestamp = valid_timestamp(req)
pending_timeout = None
container_policy_index = \
req.headers.get('X-Backend-Storage-Policy-Index', 0)
if 'x-trans-id' in req.headers:
pending_timeout = 3
broker = self._get_account_broker(drive, part, account,
pending_timeout=pending_timeout)
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
except DatabaseAlreadyExists:
pass
if req.headers.get('x-account-override-deleted', 'no').lower() != \
'yes' and broker.is_deleted():
return HTTPNotFound(request=req)
broker.put_container(container, req.headers['x-put-timestamp'],
req.headers['x-delete-timestamp'],
req.headers['x-object-count'],
req.headers['x-bytes-used'],
container_policy_index)
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
timestamp = valid_timestamp(req)
broker = self._get_account_broker(drive, part, account)
if not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
created = True
except DatabaseAlreadyExists:
created = False
elif broker.is_status_deleted():
return self._deleted_response(broker, req, HTTPForbidden,
body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp.internal)
if broker.is_deleted():
return HTTPConflict(request=req)
metadata = {}
if six.PY2:
metadata.update((key, (value, timestamp.internal))
for key, value in req.headers.items()
if is_sys_or_user_meta('account', key))
else:
for key, value in req.headers.items():
if is_sys_or_user_meta('account', key):
# Cast to native strings, so that json inside
# updata_metadata eats the data.
try:
value = value.encode('latin-1').decode('utf-8')
except UnicodeDecodeError:
raise HTTPBadRequest(
'Metadata must be valid UTF-8')
metadata[key] = (value, timestamp.internal)
if metadata:
broker.update_metadata(metadata, validate_metadata=True)
if created:
return HTTPCreated(request=req)
else:
return HTTPAccepted(request=req)