def update_data_record(self, record):
"""
Perform any mutations to container listing records that are common to
all serialization formats, and returns it as a dict.
Converts created time to iso timestamp.
Replaces size with 'swift_bytes' content type parameter.
:params record: object entry record
:returns: modified record
"""
(name, created, size, content_type, etag) = record[:5]
if content_type is None:
return {'subdir': name}
response = {
'bytes': size,
'hash': etag,
'name': name,
'content_type': content_type
}
response['last_modified'] = Timestamp(created).isoformat
override_bytes_from_content_type(response, logger=self.logger)
return response
def setUp(self):
self._imported_create_container_table = \
AccountBroker.create_container_table
AccountBroker.create_container_table = \
prespi_create_container_table
self._imported_initialize = AccountBroker._initialize
AccountBroker._initialize = prespi_AccountBroker_initialize
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
exc = None
with broker.get() as conn:
try:
conn.execute('SELECT storage_policy_index FROM container')
except BaseException as err:
exc = err
self.assert_('no such column: storage_policy_index' in str(exc))
with broker.get() as conn:
try:
conn.execute('SELECT * FROM policy_stat')
except sqlite3.OperationalError as err:
self.assert_('no such table: policy_stat' in str(err))
else:
self.fail('database created with policy_stat table')
def create_container_info_table(self, conn, put_timestamp,
storage_policy_index):
"""
Create the container_info table which is specific to the container DB.
Not a part of Pluggable Back-ends, internal to the baseline code.
Also creates the container_stat view.
:param conn: DB connection object
:param put_timestamp: put timestamp
:param storage_policy_index: storage policy index
"""
if put_timestamp is None:
put_timestamp = Timestamp(0).internal
# The container_stat view is for compatibility; old versions of Swift
# expected a container_stat table with columns "object_count" and
# "bytes_used", but when that stuff became per-storage-policy and
# moved to the policy_stat table, we stopped creating those columns in
# container_stat.
#
# To retain compatibility, we create the container_stat view with some
# triggers to make it behave like the old container_stat table. This
# way, if an old version of Swift encounters a database with the new
# schema, it can still work.
#
# Note that this can occur during a rolling Swift upgrade if a DB gets
# rsynced from an old node to a new, so it's necessary for
# availability during upgrades. The fact that it enables downgrades is
# a nice bonus.
conn.executescript(CONTAINER_INFO_TABLE_SCRIPT +
CONTAINER_STAT_VIEW_SCRIPT)
conn.execute("""
INSERT INTO container_info (account, container, created_at, id,
put_timestamp, status_changed_at, storage_policy_index)
VALUES (?, ?, ?, ?, ?, ?, ?);
""", (self.account, self.container, Timestamp.now().internal,
str(uuid4()), put_timestamp, put_timestamp,
storage_policy_index))
def _make_open_diskfile(self,
device='dev',
partition='9',
account='a',
container='c',
obj='o',
body='test',
extra_metadata=None,
policy=None,
frag_index=None,
timestamp=None,
df_mgr=None):
policy = policy or POLICIES.legacy
object_parts = account, container, obj
timestamp = Timestamp(time.time()) if timestamp is None else timestamp
if df_mgr is None:
df_mgr = self.daemon._diskfile_router[policy]
df = df_mgr.get_diskfile(device,
partition,
*object_parts,
policy=policy,
frag_index=frag_index)
content_length = len(body)
etag = hashlib.md5(body).hexdigest()
with df.create() as writer:
writer.write(body)
metadata = {
'X-Timestamp': timestamp.internal,
'Content-Length': str(content_length),
'ETag': etag,
}
if extra_metadata:
metadata.update(extra_metadata)
writer.put(metadata)
writer.commit(timestamp)
df.open()
return df
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
device, partition, account, container, obj, policy_idx = \
get_name_and_placement(request, 5, 5, True)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj,
policy_idx=policy_idx)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
metadata = disk_file.read_metadata()
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, 'timestamp'):
headers['X-Backend-Timestamp'] = e.timestamp.internal
return HTTPNotFound(request=request, headers=headers,
conditional_response=True)
response = Response(request=request, conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_sys_or_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
ts = Timestamp(metadata['X-Timestamp'])
response.last_modified = math.ceil(float(ts))
# Needed for container sync feature
response.headers['X-Timestamp'] = ts.normal
response.headers['X-Backend-Timestamp'] = ts.internal
response.content_length = int(metadata['Content-Length'])
try:
response.content_encoding = metadata['Content-Encoding']
except KeyError:
pass
return response
def test_report_up_to_date(self):
broker = self._get_broker('a', 'c', node_index=0)
broker.initialize(Timestamp(1).internal, int(POLICIES.default))
info = broker.get_info()
broker.reported(info['put_timestamp'], info['delete_timestamp'],
info['object_count'], info['bytes_used'])
full_info = broker.get_replication_info()
expected_info = {
'put_timestamp': Timestamp(1).internal,
'delete_timestamp': '0',
'count': 0,
'bytes_used': 0,
'reported_put_timestamp': Timestamp(1).internal,
'reported_delete_timestamp': '0',
'reported_object_count': 0,
'reported_bytes_used': 0
}
for key, value in expected_info.items():
msg = 'expected value for %r, %r != %r' % (key, full_info[key],
value)
self.assertEqual(full_info[key], value, msg)
repl = replicator.ContainerReplicator({})
self.assertTrue(repl.report_up_to_date(full_info))
full_info['delete_timestamp'] = Timestamp(2).internal
self.assertFalse(repl.report_up_to_date(full_info))
full_info['reported_delete_timestamp'] = Timestamp(2).internal
self.assertTrue(repl.report_up_to_date(full_info))
full_info['count'] = 1
self.assertFalse(repl.report_up_to_date(full_info))
full_info['reported_object_count'] = 1
self.assertTrue(repl.report_up_to_date(full_info))
full_info['bytes_used'] = 1
self.assertFalse(repl.report_up_to_date(full_info))
full_info['reported_bytes_used'] = 1
self.assertTrue(repl.report_up_to_date(full_info))
full_info['put_timestamp'] = Timestamp(3).internal
self.assertFalse(repl.report_up_to_date(full_info))
full_info['reported_put_timestamp'] = Timestamp(3).internal
self.assertTrue(repl.report_up_to_date(full_info))
def test_generic_exception_handling(self):
auditor_worker = auditor.AuditorWorker(self.conf, self.logger,
self.rcache, self.devices)
# pretend that we logged (and reset counters) just now
auditor_worker.last_logged = time.time()
timestamp = str(normalize_timestamp(time.time()))
pre_errors = auditor_worker.errors
data = '0' * 1024
etag = md5()
with self.disk_file.create() as writer:
writer.write(data)
etag.update(data)
etag = etag.hexdigest()
metadata = {
'ETag': etag,
'X-Timestamp': timestamp,
'Content-Length': str(os.fstat(writer._fd).st_size),
}
writer.put(metadata)
writer.commit(Timestamp(timestamp))
with mock.patch('swift.obj.diskfile.DiskFileManager.diskfile_cls',
lambda *_: 1 / 0):
auditor_worker.audit_all_objects()
self.assertEqual(auditor_worker.errors, pre_errors + 1)
def test_reap_object(self):
conf = {
'mount_check': 'false',
}
r = reaper.AccountReaper(conf, logger=debug_logger())
mock_path = 'swift.account.reaper.direct_delete_object'
for policy in POLICIES:
r.reset_stats()
with patch(mock_path) as fake_direct_delete:
with patch('swift.common.utils.Timestamp.now') as mock_now:
mock_now.return_value = Timestamp(1429117638.86767)
r.reap_object('a', 'c', 'partition', cont_nodes, 'o',
policy.idx)
mock_now.assert_called_once_with()
for i, call_args in enumerate(
fake_direct_delete.call_args_list):
cnode = cont_nodes[i % len(cont_nodes)]
host = '%(ip)s:%(port)s' % cnode
device = cnode['device']
headers = {
'X-Container-Host': host,
'X-Container-Partition': 'partition',
'X-Container-Device': device,
'X-Backend-Storage-Policy-Index': policy.idx,
'X-Timestamp': '1429117638.86767',
'x-backend-use-replication-network': 'true',
}
ring = r.get_object_ring(policy.idx)
expected = call(dict(ring.devs[i], index=i), 0,
'a', 'c', 'o',
headers=headers, conn_timeout=0.5,
response_timeout=10)
self.assertEqual(call_args, expected)
self.assertEqual(policy.object_ring.replicas - 1, i)
self.assertEqual(r.stats_objects_deleted,
policy.object_ring.replicas)
def update_data_record(self, record, versions=False):
if 'subdir' in record:
return {'subdir': record['name']}
props = record.get('properties', {})
# This metadata is added by encryption middleware.
if 'x-object-sysmeta-container-update-override-etag' in props:
hash_ = props['x-object-sysmeta-container-update-override-etag']
else:
hash_ = record['hash'].lower()
response = {
'name': record['name'],
'bytes': record['size'],
'hash': hash_,
'last_modified': Timestamp(record['ctime']).isoformat,
'content_type': record.get('mime_type', 'application/octet-stream')
}
if record.get('deleted', False):
response['content_type'] = DELETE_MARKER_CONTENT_TYPE
if versions:
response['version'] = record.get('version', 'null')
override_bytes_from_content_type(response)
return response
def test_put_and_get_multirange(self):
# put object
headers = {
'x-timestamp': Timestamp(time.time()).internal,
'content-type': 'application/octet-stream',
}
req = Request.blank(self._get_path(), method='PUT', headers=headers)
num_chunks = 10
req.body = ''.join(chr(i + 97) * self.disk_chunk_size for i in
range(num_chunks))
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
# get object with range
req = Request.blank(self._get_path())
req.range = 'bytes=301-455,686-792'
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 206)
msg = email.message_from_string(
'Content-Type: %s\r\n' % resp.headers['Content-Type'] + resp.body)
parts = [p for p in msg.walk()][1:]
self.assertEqual(2, len(parts))
self.assertEqual(parts[0].get_payload(), 'd' * 99 + 'e' * 56)
self.assertEqual(parts[1].get_payload(), 'g' * 14 + 'h' * 93)
def test_build_task_obj_round_trip(self):
ts = next(self.ts)
a = 'a1'
c = 'c2'
o = 'obj1'
args = (ts, a, c, o)
self.assertEqual(
args, expirer.parse_task_obj(expirer.build_task_obj(ts, a, c, o)))
self.assertEqual(
args,
expirer.parse_task_obj(
expirer.build_task_obj(ts, a, c, o, high_precision=True)))
ts = Timestamp(next(self.ts), delta=1234)
a = u'\N{SNOWMAN}'
c = u'\N{SNOWFLAKE}'
o = u'\U0001F334'
args = (ts, a, c, o)
self.assertNotEqual(
args, expirer.parse_task_obj(expirer.build_task_obj(ts, a, c, o)))
self.assertEqual(
args,
expirer.parse_task_obj(
expirer.build_task_obj(ts, a, c, o, high_precision=True)))
def _backend_requests(self, req, n_outgoing, account_partition, accounts,
policy_index=None):
additional = {'X-Timestamp': Timestamp(time.time()).internal}
if policy_index is None:
additional['X-Backend-Storage-Policy-Default'] = \
int(POLICIES.default)
else:
additional['X-Backend-Storage-Policy-Index'] = str(policy_index)
headers = [self.generate_request_headers(req, transfer=True,
additional=additional)
for _junk in range(n_outgoing)]
for i, account in enumerate(accounts):
i = i % len(headers)
headers[i]['X-Account-Partition'] = account_partition
headers[i]['X-Account-Host'] = csv_append(
headers[i].get('X-Account-Host'),
'%(ip)s:%(port)s' % account)
headers[i]['X-Account-Device'] = csv_append(
headers[i].get('X-Account-Device'),
account['device'])
return headers
def test_delete_db_status(self):
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
start = ts.next()
broker = AccountBroker(':memory:', account='a')
broker.initialize(start)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
self.assert_(Timestamp(info['created_at']) >= start)
self.assertEqual(info['delete_timestamp'], '0')
if self.__class__ == TestAccountBrokerBeforeMetadata:
self.assertEqual(info['status_changed_at'], '0')
else:
self.assertEqual(info['status_changed_at'],
Timestamp(start).internal)
# delete it
delete_timestamp = ts.next()
broker.delete_db(delete_timestamp)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
self.assert_(Timestamp(info['created_at']) >= start)
self.assertEqual(info['delete_timestamp'], delete_timestamp)
self.assertEqual(info['status_changed_at'], delete_timestamp)
def timestamp(self):
if self._metadata is None:
raise DiskFileNotOpen()
return Timestamp(self._metadata.get('X-Timestamp'))
def _replicate_object(self, partition, object_file, node_id):
"""
Replicate the db, choosing method based on whether or not it
already exists on peers.
:param partition: partition to be replicated to
:param object_file: DB file name to be replicated
:param node_id: node id of the node to be replicated to
"""
start_time = now = time.time()
self.logger.debug('Replicating db %s', object_file)
self.stats['attempted'] += 1
self.logger.increment('attempts')
shouldbehere = True
try:
broker = self.brokerclass(object_file, pending_timeout=30)
broker.reclaim(now - self.reclaim_age,
now - (self.reclaim_age * 2))
info = broker.get_replication_info()
bpart = self.ring.get_part(info['account'], info.get('container'))
if bpart != int(partition):
partition = bpart
# Important to set this false here since the later check only
# checks if it's on the proper device, not partition.
shouldbehere = False
name = '/' + quote(info['account'])
if 'container' in info:
name += '/' + quote(info['container'])
self.logger.error(
'Found %s for %s when it should be on partition %s; will '
'replicate out and remove.' % (object_file, name, bpart))
except (Exception, Timeout) as e:
if 'no such table' in str(e):
self.logger.error(_('Quarantining DB %s'), object_file)
quarantine_db(broker.db_file, broker.db_type)
else:
self.logger.exception(_('ERROR reading db %s'), object_file)
nodes = self.ring.get_part_nodes(int(partition))
self._add_failure_stats([(failure_dev['replication_ip'],
failure_dev['device'])
for failure_dev in nodes])
self.logger.increment('failures')
return
# The db is considered deleted if the delete_timestamp value is greater
# than the put_timestamp, and there are no objects.
delete_timestamp = Timestamp(info.get('delete_timestamp') or 0)
put_timestamp = Timestamp(info.get('put_timestamp') or 0)
if (now - self.reclaim_age) > delete_timestamp > put_timestamp and \
info['count'] in (None, '', 0, '0'):
if self.report_up_to_date(info):
self.delete_db(broker)
self.logger.timing_since('timing', start_time)
return
responses = []
failure_devs_info = set()
nodes = self.ring.get_part_nodes(int(partition))
local_dev = None
for node in nodes:
if node['id'] == node_id:
local_dev = node
break
if shouldbehere:
shouldbehere = bool([n for n in nodes if n['id'] == node_id])
# See Footnote [1] for an explanation of the repl_nodes assignment.
if len(nodes) > 1:
i = 0
while i < len(nodes) and nodes[i]['id'] != node_id:
i += 1
repl_nodes = nodes[i + 1:] + nodes[:i]
else: # Special case if using only a single replica
repl_nodes = nodes
more_nodes = self.ring.get_more_nodes(int(partition))
if not local_dev:
# Check further if local device is a handoff node
for node in self.ring.get_more_nodes(int(partition)):
if node['id'] == node_id:
local_dev = node
break
for node in repl_nodes:
different_region = False
if local_dev and local_dev['region'] != node['region']:
# This additional information will help later if we
# want to handle syncing to a node in different
# region with some optimizations.
different_region = True
success = False
try:
success = self._repl_to_node(node, broker, partition, info,
different_region)
except DriveNotMounted:
try:
repl_nodes.append(next(more_nodes))
except StopIteration:
self.logger.error(
_('ERROR There are not enough handoff nodes to reach '
'replica count for partition %s'), partition)
self.logger.error(_('ERROR Remote drive not mounted %s'), node)
except (Exception, Timeout):
self.logger.exception(
_('ERROR syncing %(file)s with node'
' %(node)s'), {
'file': object_file,
'node': node
})
if not success:
failure_devs_info.add((node['replication_ip'], node['device']))
self.logger.increment('successes' if success else 'failures')
responses.append(success)
try:
self._post_replicate_hook(broker, info, responses)
except (Exception, Timeout):
self.logger.exception(
'UNHANDLED EXCEPTION: in post replicate '
'hook for %s', broker.db_file)
if not shouldbehere and responses and all(responses):
# If the db shouldn't be on this node and has been successfully
# synced to all of its peers, it can be removed.
if not self.delete_db(broker):
failure_devs_info.update([(failure_dev['replication_ip'],
failure_dev['device'])
for failure_dev in repl_nodes])
target_devs_info = set([(target_dev['replication_ip'],
target_dev['device'])
for target_dev in repl_nodes])
self.stats['success'] += len(target_devs_info - failure_devs_info)
self._add_failure_stats(failure_devs_info)
self.logger.timing_since('timing', start_time)
def _prepare_headers(self, req):
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
headers = self.generate_request_headers(req, additional=req.headers)
return headers
def delete_at_time_of_task_container(self, task_container):
"""
get delete_at timestamp from task_container name
"""
# task_container name is timestamp
return Timestamp(task_container)
def make_timestamp_iter(offset=0):
return iter(
Timestamp(t) for t in itertools.count(int(time.time()) + offset))
def print_db_info_metadata(db_type, info, metadata):
"""
print out data base info/metadata based on its type
:param db_type: database type, account or container
:param info: dict of data base info
:param metadata: dict of data base metadata
"""
if info is None:
raise ValueError('DB info is None')
if db_type not in ['container', 'account']:
raise ValueError('Wrong DB type')
try:
account = info['account']
container = None
if db_type == 'container':
container = info['container']
path = '/%s/%s' % (account, container)
else:
path = '/%s' % account
print 'Path: %s' % path
print ' Account: %s' % account
if db_type == 'container':
print ' Container: %s' % container
path_hash = hash_path(account, container)
if db_type == 'container':
print ' Container Hash: %s' % path_hash
else:
print ' Account Hash: %s' % path_hash
print 'Metadata:'
print (' Created at: %s (%s)' %
(Timestamp(info['created_at']).isoformat,
info['created_at']))
print (' Put Timestamp: %s (%s)' %
(Timestamp(info['put_timestamp']).isoformat,
info['put_timestamp']))
print (' Delete Timestamp: %s (%s)' %
(Timestamp(info['delete_timestamp']).isoformat,
info['delete_timestamp']))
print (' Status Timestamp: %s (%s)' %
(Timestamp(info['status_changed_at']).isoformat,
info['status_changed_at']))
if db_type == 'account':
print ' Container Count: %s' % info['container_count']
print ' Object Count: %s' % info['object_count']
print ' Bytes Used: %s' % info['bytes_used']
if db_type == 'container':
try:
policy_name = POLICIES[info['storage_policy_index']].name
except KeyError:
policy_name = 'Unknown'
print (' Storage Policy: %s (%s)' % (
policy_name, info['storage_policy_index']))
print (' Reported Put Timestamp: %s (%s)' %
(Timestamp(info['reported_put_timestamp']).isoformat,
info['reported_put_timestamp']))
print (' Reported Delete Timestamp: %s (%s)' %
(Timestamp(info['reported_delete_timestamp']).isoformat,
info['reported_delete_timestamp']))
print ' Reported Object Count: %s' % info['reported_object_count']
print ' Reported Bytes Used: %s' % info['reported_bytes_used']
print ' Chexor: %s' % info['hash']
print ' UUID: %s' % info['id']
except KeyError as e:
raise ValueError('Info is incomplete: %s' % e)
meta_prefix = 'x_' + db_type + '_'
for key, value in info.iteritems():
if key.lower().startswith(meta_prefix):
title = key.replace('_', '-').title()
print ' %s: %s' % (title, value)
user_metadata = {}
sys_metadata = {}
for key, (value, timestamp) in metadata.iteritems():
if is_user_meta(db_type, key):
user_metadata[strip_user_meta_prefix(db_type, key)] = value
elif is_sys_meta(db_type, key):
sys_metadata[strip_sys_meta_prefix(db_type, key)] = value
else:
title = key.replace('_', '-').title()
print ' %s: %s' % (title, value)
if sys_metadata:
print ' System Metadata: %s' % sys_metadata
else:
print 'No system metadata found in db file'
if user_metadata:
print ' User Metadata: %s' % user_metadata
else:
print 'No user metadata found in db file'
def test_container_table_migration(self, tempdir):
db_path = os.path.join(tempdir, 'account.db')
# first init an acct DB without the policy_stat table present
broker = AccountBroker(db_path, account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
try:
conn.execute('''
SELECT storage_policy_index FROM container
''').fetchone()[0]
except sqlite3.OperationalError as err:
# confirm that the table doesn't have this column
self.assert_(
'no such column: storage_policy_index' in str(err))
else:
self.fail('broker did not raise sqlite3.OperationalError '
'trying to select from storage_policy_index '
'from container table!')
# manually insert an existing row to avoid migration
with broker.get() as conn:
conn.execute(
'''
INSERT INTO container (name, put_timestamp,
delete_timestamp, object_count, bytes_used,
deleted)
VALUES (?, ?, ?, ?, ?, ?)
''', ('test_name', Timestamp(time()).internal, 0, 1, 2, 0))
conn.commit()
# make sure we can iter containers without the migration
for c in broker.list_containers_iter(1, None, None, None, None):
self.assertEqual(c, ('test_name', 1, 2, 0))
# stats table is mysteriously empty...
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 0)
# now do a PUT with a different value for storage_policy_index
# which will update the DB schema as well as update policy_stats
# for legacy containers in the DB (those without an SPI)
other_policy = [p for p in POLICIES if p.idx != 0][0]
broker.put_container('test_second',
Timestamp(time()).internal, 0, 3, 4,
other_policy.idx)
broker._commit_puts_stale_ok()
with broker.get() as conn:
rows = conn.execute('''
SELECT name, storage_policy_index FROM container
''').fetchall()
for row in rows:
if row[0] == 'test_name':
self.assertEqual(row[1], 0)
else:
self.assertEqual(row[1], other_policy.idx)
# we should have stats for both containers
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 2)
self.assertEqual(stats[0]['object_count'], 1)
self.assertEqual(stats[0]['bytes_used'], 2)
self.assertEqual(stats[1]['object_count'], 3)
self.assertEqual(stats[1]['bytes_used'], 4)
# now lets delete a container and make sure policy_stats is OK
with broker.get() as conn:
conn.execute(
'''
DELETE FROM container WHERE name = ?
''', ('test_name', ))
conn.commit()
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 2)
self.assertEqual(stats[0]['object_count'], 0)
self.assertEqual(stats[0]['bytes_used'], 0)
self.assertEqual(stats[1]['object_count'], 3)
self.assertEqual(stats[1]['bytes_used'], 4)
def test_reclaim(self):
broker = AccountBroker(':memory:', account='test_account')
broker.initialize(Timestamp('1').internal)
broker.put_container('c',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 0").fetchone()[0], 1)
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 0)
broker.reclaim(Timestamp(time() - 999).internal, time())
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 0").fetchone()[0], 1)
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001)
broker.put_container('c', 0,
Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 0").fetchone()[0], 0)
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 1)
broker.reclaim(Timestamp(time() - 999).internal, time())
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 0").fetchone()[0], 0)
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 1)
sleep(.00001)
broker.reclaim(Timestamp(time()).internal, time())
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 0").fetchone()[0], 0)
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 0)
# Test reclaim after deletion. Create 3 test containers
broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx)
broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx)
broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx)
broker.reclaim(Timestamp(time()).internal, time())
# self.assertEqual(len(res), 2)
# self.assert_(isinstance(res, tuple))
# containers, account_name = res
# self.assert_(containers is None)
# self.assert_(account_name is None)
# Now delete the account
broker.delete_db(Timestamp(time()).internal)
broker.reclaim(Timestamp(time()).internal, time())
def test_double_check_trailing_delimiter(self):
# Test AccountBroker.list_containers_iter for an
# account that has an odd container with a trailing delimiter
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('a',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('a-',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('a-a',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('a-a-a',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('a-a-b',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('a-b',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('b',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('b-a',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('b-b',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('c',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
listing = broker.list_containers_iter(15, None, None, None, None)
self.assertEqual(len(listing), 10)
self.assertEqual([row[0] for row in listing], [
'a', 'a-', 'a-a', 'a-a-a', 'a-a-b', 'a-b', 'b', 'b-a', 'b-b', 'c'
])
listing = broker.list_containers_iter(15, None, None, '', '-')
self.assertEqual(len(listing), 5)
self.assertEqual([row[0] for row in listing],
['a', 'a-', 'b', 'b-', 'c'])
listing = broker.list_containers_iter(15, None, None, 'a-', '-')
self.assertEqual(len(listing), 4)
self.assertEqual([row[0] for row in listing],
['a-', 'a-a', 'a-a-', 'a-b'])
listing = broker.list_containers_iter(15, None, None, 'b-', '-')
self.assertEqual(len(listing), 2)
self.assertEqual([row[0] for row in listing], ['b-a', 'b-b'])
def test_chexor(self):
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('a',
Timestamp(1).internal,
Timestamp(0).internal, 0, 0, POLICIES.default.idx)
broker.put_container('b',
Timestamp(2).internal,
Timestamp(0).internal, 0, 0, POLICIES.default.idx)
hasha = hashlib.md5(
'%s-%s' %
('a', "%s-%s-%s-%s" %
(Timestamp(1).internal, Timestamp(0).internal, 0, 0))).digest()
hashb = hashlib.md5(
'%s-%s' %
('b', "%s-%s-%s-%s" %
(Timestamp(2).internal, Timestamp(0).internal, 0, 0))).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
broker.put_container('b',
Timestamp(3).internal,
Timestamp(0).internal, 0, 0, POLICIES.default.idx)
hashb = hashlib.md5(
'%s-%s' %
('b', "%s-%s-%s-%s" %
(Timestamp(3).internal, Timestamp(0).internal, 0, 0))).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
def test_list_containers_iter(self):
# Test AccountBroker.list_containers_iter
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
for cont1 in xrange(4):
for cont2 in xrange(125):
broker.put_container('%d-%04d' % (cont1, cont2),
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
for cont in xrange(125):
broker.put_container('2-0051-%04d' % cont,
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
for cont in xrange(125):
broker.put_container('3-%04d-0049' % cont,
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
listing = broker.list_containers_iter(100, '', None, None, '')
self.assertEqual(len(listing), 100)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0099')
listing = broker.list_containers_iter(100, '', '0-0050', None, '')
self.assertEqual(len(listing), 50)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0049')
listing = broker.list_containers_iter(100, '0-0099', None, None, '')
self.assertEqual(len(listing), 100)
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '1-0074')
listing = broker.list_containers_iter(55, '1-0074', None, None, '')
self.assertEqual(len(listing), 55)
self.assertEqual(listing[0][0], '1-0075')
self.assertEqual(listing[-1][0], '2-0004')
listing = broker.list_containers_iter(10, '', None, '0-01', '')
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '0-0109')
listing = broker.list_containers_iter(10, '', None, '0-01', '-')
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '0-0109')
listing = broker.list_containers_iter(10, '', None, '0-', '-')
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0009')
listing = broker.list_containers_iter(10, '', None, '', '-')
self.assertEqual(len(listing), 4)
self.assertEqual([row[0] for row in listing], ['0-', '1-', '2-', '3-'])
listing = broker.list_containers_iter(10, '2-', None, None, '-')
self.assertEqual(len(listing), 1)
self.assertEqual([row[0] for row in listing], ['3-'])
listing = broker.list_containers_iter(10, '', None, '2', '-')
self.assertEqual(len(listing), 1)
self.assertEqual([row[0] for row in listing], ['2-'])
listing = broker.list_containers_iter(10, '2-0050', None, '2-', '-')
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '2-0051')
self.assertEqual(listing[1][0], '2-0051-')
self.assertEqual(listing[2][0], '2-0052')
self.assertEqual(listing[-1][0], '2-0059')
listing = broker.list_containers_iter(10, '3-0045', None, '3-', '-')
self.assertEqual(len(listing), 10)
self.assertEqual([row[0] for row in listing], [
'3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048',
'3-0048-', '3-0049', '3-0049-', '3-0050'
])
broker.put_container('3-0049-',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
listing = broker.list_containers_iter(10, '3-0048', None, None, None)
self.assertEqual(len(listing), 10)
self.assertEqual([row[0] for row in listing], [
'3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050',
'3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049'
])
listing = broker.list_containers_iter(10, '3-0048', None, '3-', '-')
self.assertEqual(len(listing), 10)
self.assertEqual([row[0] for row in listing], [
'3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051',
'3-0051-', '3-0052', '3-0052-', '3-0053'
])
listing = broker.list_containers_iter(10, None, None, '3-0049-', '-')
self.assertEqual(len(listing), 2)
self.assertEqual([row[0] for row in listing],
['3-0049-', '3-0049-0049'])
def test_put_container(self):
# Test AccountBroker.put_container
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
# Create initial container
timestamp = Timestamp(time()).internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT name FROM container").fetchone()[0],
'"{<container \'&\' name>}"')
self.assertEqual(
conn.execute("SELECT put_timestamp FROM container").fetchone()
[0], timestamp)
self.assertEqual(
conn.execute("SELECT deleted FROM container").fetchone()[0], 0)
# Reput same event
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT name FROM container").fetchone()[0],
'"{<container \'&\' name>}"')
self.assertEqual(
conn.execute("SELECT put_timestamp FROM container").fetchone()
[0], timestamp)
self.assertEqual(
conn.execute("SELECT deleted FROM container").fetchone()[0], 0)
# Put new event
sleep(.00001)
timestamp = Timestamp(time()).internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT name FROM container").fetchone()[0],
'"{<container \'&\' name>}"')
self.assertEqual(
conn.execute("SELECT put_timestamp FROM container").fetchone()
[0], timestamp)
self.assertEqual(
conn.execute("SELECT deleted FROM container").fetchone()[0], 0)
# Put old event
otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_container('"{<container \'&\' name>}"', otimestamp, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT name FROM container").fetchone()[0],
'"{<container \'&\' name>}"')
self.assertEqual(
conn.execute("SELECT put_timestamp FROM container").fetchone()
[0], timestamp)
self.assertEqual(
conn.execute("SELECT deleted FROM container").fetchone()[0], 0)
# Put old delete event
dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_container('"{<container \'&\' name>}"', 0, dtimestamp, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT name FROM container").fetchone()[0],
'"{<container \'&\' name>}"')
self.assertEqual(
conn.execute("SELECT put_timestamp FROM container").fetchone()
[0], timestamp)
self.assertEqual(
conn.execute(
"SELECT delete_timestamp FROM container").fetchone()[0],
dtimestamp)
self.assertEqual(
conn.execute("SELECT deleted FROM container").fetchone()[0], 0)
# Put new delete event
sleep(.00001)
timestamp = Timestamp(time()).internal
broker.put_container('"{<container \'&\' name>}"', 0, timestamp, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT name FROM container").fetchone()[0],
'"{<container \'&\' name>}"')
self.assertEqual(
conn.execute(
"SELECT delete_timestamp FROM container").fetchone()[0],
timestamp)
self.assertEqual(
conn.execute("SELECT deleted FROM container").fetchone()[0], 1)
# Put new event
sleep(.00001)
timestamp = Timestamp(time()).internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT name FROM container").fetchone()[0],
'"{<container \'&\' name>}"')
self.assertEqual(
conn.execute("SELECT put_timestamp FROM container").fetchone()
[0], timestamp)
self.assertEqual(
conn.execute("SELECT deleted FROM container").fetchone()[0], 0)
def has_been_recreated(info):
return (info['put_timestamp'] > info['delete_timestamp'] >
Timestamp(0))
def ensure_object_in_right_location(self, q_policy_index, account,
container, obj, q_ts, path,
container_policy_index, source_ts,
source_obj_status, source_obj_info,
source_obj_iter, **kwargs):
"""
Validate source object will satisfy the misplaced object queue entry
and move to destination.
:param q_policy_index: the policy_index for the source object
:param account: the account name of the misplaced object
:param container: the container name of the misplaced object
:param obj: the name of the misplaced object
:param q_ts: the timestamp of the misplaced object
:param path: the full path of the misplaced object for logging
:param container_policy_index: the policy_index of the destination
:param source_ts: the timestamp of the source object
:param source_obj_status: the HTTP status source object request
:param source_obj_info: the HTTP headers of the source object request
:param source_obj_iter: the body iter of the source object request
"""
if source_obj_status // 100 != 2 or source_ts < q_ts:
if q_ts < time.time() - self.reclaim_age:
# it's old and there are no tombstones or anything; give up
self.stats_log('lost_source', '%r (%s) was not available in '
'policy_index %s and has expired',
path,
q_ts.internal,
q_policy_index,
level=logging.CRITICAL)
return True
# the source object is unavailable or older than the queue
# entry; a version that will satisfy the queue entry hopefully
# exists somewhere in the cluster, so wait and try again
self.stats_log('unavailable_source', '%r (%s) in '
'policy_index %s responded %s (%s)',
path,
q_ts.internal,
q_policy_index,
source_obj_status,
source_ts.internal,
level=logging.WARNING)
return False
# optimistically move any source with a timestamp >= q_ts
ts = max(Timestamp(source_ts), q_ts)
# move the object
put_timestamp = slightly_later_timestamp(ts, offset=2)
self.stats_log(
'copy_attempt', '%r (%f) in policy_index %s will be '
'moved to policy_index %s (%s)', path, source_ts, q_policy_index,
container_policy_index, put_timestamp)
headers = source_obj_info.copy()
headers['X-Backend-Storage-Policy-Index'] = container_policy_index
headers['X-Timestamp'] = put_timestamp
try:
self.swift.upload_object(FileLikeIter(source_obj_iter),
account,
container,
obj,
headers=headers)
except UnexpectedResponse as err:
self.stats_log('copy_failed', 'upload %r (%f) from '
'policy_index %s to policy_index %s '
'returned %s',
path,
source_ts,
q_policy_index,
container_policy_index,
err,
level=logging.WARNING)
return False
except: # noqa
self.stats_log('unhandled_error', 'unable to upload %r (%f) '
'from policy_index %s to policy_index %s ',
path,
source_ts,
q_policy_index,
container_policy_index,
level=logging.ERROR,
exc_info=True)
return False
self.stats_log(
'copy_success', '%r (%f) moved from policy_index %s '
'to policy_index %s (%s)', path, source_ts, q_policy_index,
container_policy_index, put_timestamp)
return self.throw_tombstones(account, container, obj, q_ts,
q_policy_index, path)
def reap_account(self, broker, partition, nodes):
"""
Called once per pass for each account this server is the primary for
and attempts to delete the data for the given account. The reaper will
only delete one account at any given time. It will call
:func:`reap_container` up to sqrt(self.concurrency) times concurrently
while reaping the account.
If there is any exception while deleting a single container, the
process will continue for any other containers and the failed
containers will be tried again the next time this function is called
with the same parameters.
If there is any exception while listing the containers for deletion,
the process will stop (but will obviously be tried again the next time
this function is called with the same parameters). This isn't likely
since the listing comes from the local database.
After the process completes (successfully or not) statistics about what
was accomplished will be logged.
This function returns nothing and should raise no exception but only
update various self.stats_* values for what occurs.
:param broker: The AccountBroker for the account to delete.
:param partition: The partition in the account ring the account is on.
:param nodes: The primary node dicts for the account to delete.
.. seealso::
:class:`swift.account.backend.AccountBroker` for the broker class.
.. seealso::
:func:`swift.common.ring.Ring.get_nodes` for a description
of the node dicts.
"""
begin = time()
info = broker.get_info()
if time() - float(Timestamp(info['delete_timestamp'])) <= \
self.delay_reaping:
return False
account = info['account']
self.logger.info(_('Beginning pass on account %s'), account)
self.reset_stats()
try:
marker = ''
while True:
containers = \
list(broker.list_containers_iter(1000, marker, None, None,
None))
if not containers:
break
try:
for (container, _junk, _junk, _junk) in containers:
self.container_pool.spawn(self.reap_container, account,
partition, nodes, container)
self.container_pool.waitall()
except (Exception, Timeout):
self.logger.exception(
_('Exception with containers for account %s'), account)
marker = containers[-1][0]
if marker == '':
break
log = 'Completed pass on account %s' % account
except (Exception, Timeout):
self.logger.exception(
_('Exception with account %s'), account)
log = _('Incomplete pass on account %s') % account
if self.stats_containers_deleted:
log += _(', %s containers deleted') % self.stats_containers_deleted
if self.stats_objects_deleted:
log += _(', %s objects deleted') % self.stats_objects_deleted
if self.stats_containers_remaining:
log += _(', %s containers remaining') % \
self.stats_containers_remaining
if self.stats_objects_remaining:
log += _(', %s objects remaining') % self.stats_objects_remaining
if self.stats_containers_possibly_remaining:
log += _(', %s containers possibly remaining') % \
self.stats_containers_possibly_remaining
if self.stats_objects_possibly_remaining:
log += _(', %s objects possibly remaining') % \
self.stats_objects_possibly_remaining
if self.stats_return_codes:
log += _(', return codes: ')
for code in sorted(self.stats_return_codes):
log += '%s %sxxs, ' % (self.stats_return_codes[code], code)
log = log[:-2]
log += _(', elapsed: %.02fs') % (time() - begin)
self.logger.info(log)
self.logger.timing_since('timing', self.start_time)
delete_timestamp = Timestamp(info['delete_timestamp'])
if self.stats_containers_remaining and \
begin - float(delete_timestamp) >= self.reap_not_done_after:
self.logger.warn(_('Account %s has not been reaped since %s') %
(account, delete_timestamp.isoformat))
return True
def test_object_run_once_pass(self):
auditor_worker = auditor.AuditorWorker(self.conf, self.logger,
self.rcache, self.devices)
auditor_worker.log_time = 0
timestamp = str(normalize_timestamp(time.time()))
pre_quarantines = auditor_worker.quarantines
data = '0' * 1024
def write_file(df):
with df.create() as writer:
writer.write(data)
metadata = {
'ETag': md5(data).hexdigest(),
'X-Timestamp': timestamp,
'Content-Length': str(os.fstat(writer._fd).st_size),
}
writer.put(metadata)
writer.commit(Timestamp(timestamp))
# policy 0
write_file(self.disk_file)
# policy 1
write_file(self.disk_file_p1)
# policy 2
write_file(self.disk_file_ec)
auditor_worker.audit_all_objects()
self.assertEqual(auditor_worker.quarantines, pre_quarantines)
# 1 object per policy falls into 1024 bucket
self.assertEqual(auditor_worker.stats_buckets[1024], 3)
self.assertEqual(auditor_worker.stats_buckets[10240], 0)
# pick up some additional code coverage, large file
data = '0' * 1024 * 1024
for df in (self.disk_file, self.disk_file_ec):
with df.create() as writer:
writer.write(data)
metadata = {
'ETag': md5(data).hexdigest(),
'X-Timestamp': timestamp,
'Content-Length': str(os.fstat(writer._fd).st_size),
}
writer.put(metadata)
writer.commit(Timestamp(timestamp))
auditor_worker.audit_all_objects(device_dirs=['sda', 'sdb'])
self.assertEqual(auditor_worker.quarantines, pre_quarantines)
# still have the 1024 byte object left in policy-1 (plus the
# stats from the original 3)
self.assertEqual(auditor_worker.stats_buckets[1024], 4)
self.assertEqual(auditor_worker.stats_buckets[10240], 0)
# and then policy-0 disk_file was re-written as a larger object
self.assertEqual(auditor_worker.stats_buckets['OVER'], 2)
# pick up even more additional code coverage, misc paths
auditor_worker.log_time = -1
auditor_worker.stats_sizes = []
auditor_worker.audit_all_objects(device_dirs=['sda', 'sdb'])
self.assertEqual(auditor_worker.quarantines, pre_quarantines)
self.assertEqual(auditor_worker.stats_buckets[1024], 4)
self.assertEqual(auditor_worker.stats_buckets[10240], 0)
self.assertEqual(auditor_worker.stats_buckets['OVER'], 2)
def make_timestamp_iter():
return iter(Timestamp(t) for t in itertools.count(int(time.time())))
