def test_select_container_metadata(self, factory_mock):
factory_mock.return_value = mock.Mock()
test_headers = {
'X-Container-Meta-foo': ('foo', '1545179217.201453'),
'x-delete-at': ('1645179217', '1545179217.201453'),
'someotherkey': ('boo', '1545179217.201453'),
'X-Versions-Location':
('test_versions_location', '1545179217.201453'),
'X-Container-Meta-bar': ('bar', '1545179217.201453'),
'X-Container-Read': ('test2:tester2', '1545179217.201453'),
}
expected_result = {
'X-Container-Meta-foo': 'foo',
'X-Container-Meta-bar': 'bar',
'X-Container-Read': 'test2:tester2',
}
base = base_sync.BaseSync(self.settings, max_conns=1)
self.assertFalse(base.sync_container_metadata)
self.assertEqual({}, base.select_container_metadata(test_headers))
self.settings['sync_container_metadata'] = True
self.settings['sync_container_acl'] = True
base = base_sync.BaseSync(self.settings, max_conns=1)
self.assertTrue(base.sync_container_metadata)
self.assertEqual(expected_result,
base.select_container_metadata(test_headers))
def test_http_pool_locking(self, factory_mock):
factory_mock.return_value = mock.Mock()
base = base_sync.BaseSync(self.settings, max_conns=1)
with base.client_pool.get_client():
self.assertEqual(0, base.client_pool.get_semaphore.balance)
self.assertEqual(0,
base.client_pool.client_pool[0].semaphore.balance)
self.assertEqual(1, base.client_pool.get_semaphore.balance)
def test_fail_upload_segment(self, factory_mock):
factory_mock.return_value = mock.Mock()
base = base_sync.BaseSync(self.settings, max_conns=1)
base.logger = mock.Mock()
swift_client = mock.Mock()
swift_client.get_object_metadata.return_value = {
'x-static-large-object': 'true'
}
def _get_object(account, container, key, **kwargs):
manifest = [{
'name': '/container_segments/part1',
'hash': 'deadbeef'
}, {
'name': '/container_segments/part2',
'hash': 'deadbeef2'
}]
body = json.dumps(manifest)
headers = {'etag': hashlib.md5(body).hexdigest()}
return 200, headers, StringIO.StringIO(body)
def _delete_object(acc, cont, obj, acceptable_statuses):
if obj == 'part1':
raise UnexpectedResponse('foo', None)
swift_client.get_object.side_effect = _get_object
swift_client.delete_object.side_effect = _delete_object
row = {
'deleted': 0,
'created_at': str(time.time() - 5),
'name': 'foo',
'storage_policy_index': 99
}
_, _, swift_ts = decode_timestamps(row['created_at'])
base.delete_local_object(swift_client, row, swift_ts, False)
# manifest should not be deleted
swift_client.delete_object.assert_has_calls([
mock.call(self.settings['account'],
'container_segments',
'part1',
acceptable_statuses=(2, 404, 409)),
mock.call(self.settings['account'],
'container_segments',
'part2',
acceptable_statuses=(2, 404, 409))
])
base.logger.warning.assert_called_once_with(
'Failed to delete segment %s/%s/%s: %s', 'account',
'container_segments', 'part1', 'foo')
base.logger.error.assert_called_once_with(
'Failed to delete %s segments of %s/%s', 1, 'container', 'foo')
def test_double_release(self, factory_mock):
factory_mock.return_value = mock.Mock()
base = base_sync.BaseSync(self.settings, max_conns=1)
client = base.client_pool.get_client()
self.assertEqual(0, base.client_pool.get_semaphore.balance)
self.assertEqual(0, base.client_pool.client_pool[0].semaphore.balance)
client.close()
self.assertEqual(1, base.client_pool.get_semaphore.balance)
self.assertEqual(1, client.semaphore.balance)
with self.assertRaises(RuntimeError):
client.close()
self.assertEqual(1, client.semaphore.balance)
def test_retain_copy_slo(self, factory_mock):
factory_mock.return_value = mock.Mock()
base = base_sync.BaseSync(self.settings, max_conns=1)
swift_client = mock.Mock()
swift_client.get_object_metadata.return_value = {
'x-static-large-object': 'true'
}
def _get_object(account, container, key, **kwargs):
manifest = [{
'name': '/container_segments/part1',
'hash': 'deadbeef'
}, {
'name': '/container_segments/part2',
'hash': 'deadbeef2'
}]
body = json.dumps(manifest)
headers = {'etag': hashlib.md5(body).hexdigest()}
return 200, headers, StringIO.StringIO(body)
swift_client.get_object.side_effect = _get_object
row = {
'deleted': 0,
'created_at': str(time.time() - 5),
'name': 'foo',
'storage_policy_index': 99
}
_, _, swift_ts = decode_timestamps(row['created_at'])
base.delete_local_object(swift_client, row, swift_ts, False)
swift_ts.offset += 1
swift_client.delete_object.assert_has_calls([
mock.call(self.settings['account'],
'container_segments',
'part1',
acceptable_statuses=(2, 404, 409)),
mock.call(self.settings['account'],
'container_segments',
'part2',
acceptable_statuses=(2, 404, 409)),
mock.call(self.settings['account'],
self.settings['container'],
row['name'],
acceptable_statuses=(2, 404, 409),
headers={'X-Timestamp': Timestamp(swift_ts).internal})
])
def test_retain_copy_dlo(self, factory_mock):
factory_mock.return_value = mock.Mock()
base = base_sync.BaseSync(self.settings, max_conns=1)
swift_client = mock.NonCallableMock()
swift_client.get_object_metadata.return_value = {
'x-object-manifest': 'container_segments/segment_'
}
swift_client.make_request.side_effect = (mock.Mock(body=json.dumps([{
'name':
'segments_%d' % (i + 1),
'hash':
'deadbeef'
} for i in range(2)]),
status_int=200),
mock.Mock(body='[]',
status_int=200))
row = {
'deleted': 0,
'created_at': str(time.time() - 5),
'name': 'foo',
'storage_policy_index': 99
}
_, _, swift_ts = decode_timestamps(row['created_at'])
base.delete_local_object(swift_client, row, swift_ts, False)
swift_ts.offset += 1
swift_client.delete_object.assert_has_calls([
mock.call(self.settings['account'],
'container_segments',
'segments_1',
acceptable_statuses=(2, 404, 409)),
mock.call(self.settings['account'],
'container_segments',
'segments_2',
acceptable_statuses=(2, 404, 409)),
mock.call(self.settings['account'],
self.settings['container'],
row['name'],
acceptable_statuses=(2, 404, 409),
headers={'X-Timestamp': Timestamp(swift_ts).internal})
])
def test_retain_copy(self, factory_mock):
factory_mock.return_value = mock.Mock()
base = base_sync.BaseSync(self.settings, max_conns=1)
swift_client = mock.Mock()
swift_client.get_object_metadata.return_value = {}
row = {
'deleted': 0,
'created_at': str(time.time() - 5),
'name': 'foo',
'storage_policy_index': 99
}
_, _, swift_ts = decode_timestamps(row['created_at'])
base.delete_local_object(swift_client, row, swift_ts, False)
swift_ts.offset += 1
swift_client.delete_object.assert_called_once_with(
self.settings['account'],
self.settings['container'],
row['name'],
acceptable_statuses=(2, 404, 409),
headers={'X-Timestamp': Timestamp(swift_ts).internal})