def test_is_sharded_container_error(self):
# first, just a sanity test
metadata = {'x-backend-sharding-state': 'sharded'}
self.mock_ic.get_container_metadata.return_value = metadata
self.assertTrue(self.crawler._is_container_sharded('foo', 'bar'))
# if container doesn't exist just return false, don't log error
logger = mock.Mock()
self.crawler.logger = logger
mock_resp = mock.Mock()
mock_resp.status_int = 404
self.mock_ic.get_container_metadata.side_effect = UnexpectedResponse(
"fail", mock_resp)
self.assertFalse(self.crawler._is_container_sharded('foo', 'bar'))
logger.error.assert_not_called()
# Other unexpected error
logger.reset_mock()
mock_resp.status_int = 501
self.assertFalse(self.crawler._is_container_sharded('foo', 'bar'))
self.assertEqual(
[mock.call(
"Failed to retrieve container metadata for foo/bar: fail")],
self.crawler.logger.error.mock_calls)
# generic Exception
logger.reset_mock()
self.mock_ic.get_container_metadata.side_effect = Exception("genfail")
self.assertFalse(self.crawler._is_container_sharded('foo', 'bar'))
self.assertEqual(
[mock.call(
"Failed to retrieve container metadata for foo/bar: genfail")],
self.crawler.logger.error.mock_calls)
def test_per_account_container_create(self, mock_swift):
mock_ic = mock.Mock()
mock_ic.get_object_metadata.side_effect = UnexpectedResponse(
'404 Not Found', None)
swift_client = mock.Mock()
mock_swift.return_value = swift_client
swift_client.head_container.side_effect = ClientException(
'not found', http_status=404, http_reason='Not Found')
self.sync_swift._per_account = True
self.assertFalse(self.sync_swift.verified_container)
self.sync_swift.upload_object('foo', 'policy', mock_ic)
swift_client.put_container.assert_called_once_with('bucketcontainer')
self.assertTrue(self.sync_swift.verified_container)
swift_client.reset_mock()
self.sync_swift.upload_object('foo', 'policy', mock_ic)
self.assertEqual([mock.call.head_object('bucketcontainer', 'foo')],
swift_client.mock_calls)
def _update_container_headers(self, container, internal_client, headers):
# This explicitly does not use update_container_metadata because it
# needs to be able to update the acls (swift_owner: True).
req = swob.Request.blank(internal_client.make_path(
self.config['account'], container),
environ={
'REQUEST_METHOD': 'POST',
'swift_owner': True
},
headers=headers)
resp = req.get_response(internal_client.app)
if resp.status_int // 100 != 2:
raise UnexpectedResponse(
'Failed to update container headers for '
'"%s": %d' % (container, resp.status_int), resp)
self.logger.info('Updated headers for container "%s"' % container)
def _create_container(self,
container,
internal_client,
aws_bucket,
timeout=1):
if self.config.get('protocol') == 'swift':
try:
headers = get_container_headers(self.provider, aws_bucket)
except RemoteHTTPError as e:
if e.resp.status == 404:
raise ContainerNotFound(self.config['aws_identity'],
aws_bucket)
else:
raise
else:
headers = {}
headers[get_sys_migrator_header('container')] =\
MigrationContainerStates.MIGRATING
req = swob.Request.blank(internal_client.make_path(
self.config['account'], container),
environ={
'REQUEST_METHOD': 'PUT',
'swift_owner': True
},
headers=headers)
resp = req.get_response(internal_client.app)
if resp.status_int // 100 != 2:
raise UnexpectedResponse(
'Failed to create container "%s": %d' %
(container, resp.status_int), resp)
start = time.time()
while time.time() - start < timeout:
if not internal_client.container_exists(self.config['account'],
container):
time.sleep(0.1)
else:
self.logger.info('Created container "%s"' % container)
return
raise MigrationError('Timeout while creating container "%s"' %
container)
def _head_internal_account(self, internal_client):
# This explicitly does not use get_account_metadata because it
# needs to be able to read the temp url key (swift_owner: True).
# It should be noted that this should be something internal client can
# do and there is a patch proposed to allow it.
req = swob.Request.blank(internal_client.make_path(
self.config['account']),
environ={
'REQUEST_METHOD': 'HEAD',
'swift_owner': True
})
resp = req.get_response(internal_client.app)
if resp.status_int // 100 != 2:
raise UnexpectedResponse(
'Failed to read container headers for '
'"%s": %d' % (self.config['account'], resp.status_int), resp)
return dict(resp.headers)
def _process_account_metadata(self):
if self.config.get('protocol') != 'swift':
return
if not self.config.get('propagate_account_metadata'):
return
resp = self.provider.head_account()
if resp.status // 100 != 2:
raise UnexpectedResponse(
'Failed to read container headers for '
'"%s": %d' % (self.config['account'], resp.status_int), resp)
rem_headers = resp.headers
with self.ic_pool.item() as ic:
local_headers = self._head_internal_account(ic)
if rem_headers and local_headers:
header_changes = diff_account_headers(rem_headers,
local_headers)
if header_changes:
ic.set_account_metadata(self.config['account'],
dict(header_changes))
self.logger.info(
'Updated account metadata for %s: %s' %
(self.config['account'], header_changes.keys()))
def _delete_object(acc, cont, obj, acceptable_statuses):
if obj == 'part1':
raise UnexpectedResponse('foo', None)
def test_migrate_slo(self, create_provider_mock):
self.migrator.config['protocol'] = 'swift'
provider = create_provider_mock.return_value
self.migrator.status.get_migration.return_value = {}
segments_container = '/slo-segments'
manifest = [{
'name': '/'.join([segments_container, 'part1'])
}, {
'name': '/'.join([segments_container, 'part2'])
}]
objects = {
'slo': {
'remote_headers': {
'x-object-meta-custom': 'slo-meta',
'last-modified': create_timestamp(1.5e9),
'x-static-large-object': True
},
'expected_headers': {
'x-object-meta-custom': 'slo-meta',
'x-timestamp': Timestamp(1.5e9).internal,
'x-static-large-object': True,
'Content-Length': len(json.dumps(manifest))
}
},
'part1': {
'remote_headers': {
'x-object-meta-part': 'part-1',
'last-modified': create_timestamp(1.4e9)
},
'expected_headers': {
'x-object-meta-part': 'part-1',
'x-timestamp': Timestamp(1.4e9).internal
}
},
'part2': {
'remote_headers': {
'x-object-meta-part': 'part-2',
'last-modified': create_timestamp(1.1e9)
},
'expected_headers': {
'x-object-meta-part': 'part-2',
'x-timestamp': Timestamp(1.1e9).internal
}
}
}
containers = {
segments_container: False,
self.migrator.config['container']: False
}
def container_exists(_, container):
return containers[container]
def create_container(_, container, headers):
containers[container] = True
def get_object(name, **args):
if name not in objects.keys():
raise RuntimeError('Unknown object: %s' % name)
return ProviderResponse(True, 200, objects[name]['remote_headers'],
StringIO('object body'))
def head_object(name, container):
if container != segments_container[1:]:
raise RuntimeError('wrong container: %s' % container)
if name not in objects.keys():
raise RuntimeError('unknown object: %s' % name)
resp = mock.Mock()
resp.status = 200
resp.headers = objects[name]['remote_headers']
return resp
self.swift_client.container_exists.side_effect = container_exists
self.swift_client.create_container.side_effect = create_container
swift_head_resp = mock.Mock()
swift_head_resp.status_int = 404
self.swift_client.get_object_metadata.side_effect = UnexpectedResponse(
'', swift_head_resp)
bucket_resp = mock.Mock()
bucket_resp.status = 200
bucket_resp.headers = {}
provider.head_bucket.return_value = bucket_resp
provider.list_objects.return_value = (200, [{'name': 'slo'}])
provider.get_object.side_effect = get_object
provider.head_object.side_effect = head_object
provider.get_manifest.return_value = manifest
self.swift_client.iter_objects.return_value = iter([])
self.migrator.next_pass()
self.swift_client.upload_object.assert_has_calls([
mock.call(mock.ANY, self.migrator.config['account'],
'slo-segments', 'part1',
objects['part1']['expected_headers']),
mock.call(mock.ANY, self.migrator.config['account'],
'slo-segments', 'part2',
objects['part2']['expected_headers']),
mock.call(mock.ANY, self.migrator.config['account'],
self.migrator.config['container'], 'slo',
objects['slo']['expected_headers'])
])
for call in self.swift_client.upload_object.mock_calls:
body, acct, cont, obj, headers = call[1]
if obj.startswith('part'):
self.assertEqual(segments_container[1:], cont)
self.assertEqual('object body', ''.join(body))
else:
self.assertEqual(self.migrator.config['container'], cont)
self.assertEqual(manifest, json.loads(''.join(body)))