def test_head_object_error(self, mock_provider_factory):
mock_provider = mock_provider_factory.return_value
args = [
'--protocol', 's3', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://s3.amazonaws.com',
'--bucket', 'some-bucket'
]
mock_provider.list_objects.return_value = ProviderResponse(
True, 200, {}, [{
'name': 'foo'
}])
mock_provider.head_object.return_value = ProviderResponse(
False, 500, {}, '')
exit_arg = main(args)
self.assertTrue(
exit_arg.endswith(
mock_provider.head_object.return_value.wsgi_status))
mock_provider.list_objects.assert_called_once_with(None,
1,
None,
bucket=None)
mock_provider.head_object.assert_called_once_with('foo', bucket=None)
mock_provider.get_object.assert_not_called()
def test_read_only_all_buckets(self, mock_provider_factory):
args = [
'--protocol', 'swift', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://saio:8080/auth/v1.0',
'--bucket', '/*'
]
body = self.TrackingStringIO('response')
mock_provider = mock_provider_factory.return_value
mock_provider.list_buckets.return_value = ProviderResponse(
True, 200, {}, [{
'name': 'container'
}])
mock_provider.list_objects.return_value = ProviderResponse(
True, 200, {}, [{
'name': 'foo'
}])
mock_provider.get_object.return_value = ProviderResponse(
True, 200, {}, body)
mock_provider.head_object.return_value = ProviderResponse(
True, 204, {}, '')
exit_arg = main(args)
self.assertEqual(0, exit_arg)
mock_provider.list_buckets.assert_called_once_with(limit=1)
mock_provider.list_objects.assert_called_once_with(None,
1,
None,
bucket='container')
mock_provider.head_object.assert_called_once_with('foo',
bucket='container')
mock_provider.get_object.assert_called_once_with('foo',
bucket='container')
self.assertEqual(0, body.last_pos)
self.assertTrue(body.closed)
def test_list_container_shunt_swift(self):
self.mock_list_swift.side_effect = [
ProviderResponse(True, 200, {},
[{
'name': 'abc',
'hash': 'ffff',
'bytes': 42,
'last_modified': 'date',
'content_type': 'type',
'content_location': 'http://some-swift'
}, {
'name': u'unicod\xe9',
'hash': 'ffff',
'bytes': 1000,
'last_modified': 'date',
'content_type': 'type',
'content_location': 'http://some-swift'
}]),
ProviderResponse(True, 200, {}, [])
]
req = swob.Request.blank('/v1/AUTH_a/sw\xc3\xa9ft',
environ={
'__test__.status': '200 OK',
'__test__.body': '[]',
'swift.trans_id': 'id'
})
status, headers, body_iter = req.call_application(self.app)
self.assertEqual(self.mock_shunt_swift.mock_calls, [])
self.mock_list_swift.assert_has_calls([
mock.call('', 10000, '', ''),
mock.call(u'unicod\xe9'.encode('utf-8'), 10000, '', '')
])
names = body_iter.split('\n')
self.assertEqual(['abc', u'unicod\xe9'.encode('utf-8')], names)
def test_read_only_with_bucket_s3(self, mock_provider_factory):
mock_provider = mock_provider_factory.return_value
args = [
'--protocol', 's3', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://s3.amazonaws.com',
'--bucket', 'some-bucket'
]
body = self.TrackingStringIO('response')
mock_provider.list_objects.return_value = ProviderResponse(
True, 200, {}, [{
'name': 'foo'
}])
mock_provider.get_object.return_value = ProviderResponse(
True, 200, {}, body)
mock_provider.head_object.return_value = ProviderResponse(
True, 204, {}, '')
exit_arg = main(args)
self.assertEqual(0, exit_arg)
mock_provider.list_objects.assert_called_once_with(None,
1,
None,
bucket=None)
mock_provider.head_object.assert_called_once_with('foo', bucket=None)
mock_provider.get_object.assert_called_once_with('foo', bucket=None)
self.assertEqual(0, body.last_pos)
self.assertTrue(body.closed)
def test_list_container_shunt_with_duplicates(self):
self.mock_list_swift.side_effect = [
ProviderResponse(True, 200, {},
[{
'subdir': u'a/',
'content_location': 'http://some-swift'
}, {
'name': u'unicod\xe9',
'hash': 'ffff',
'bytes': 1000,
'last_modified': 'date',
'content_type': 'type',
'content_location': 'http://some-swift'
}, {
'subdir': u'z/',
'content_location': 'http://some-swift'
}, {
'name': u'zzzzz',
'hash': 'ffff',
'bytes': 1000,
'last_modified': 'date',
'content_type': 'type',
'content_location': 'http://some-swift'
}]),
ProviderResponse(True, 200, {}, [])
]
# simulate being partially migrated
local_data = [
{
'name': u'a',
'hash': 'ffff',
'bytes': 42,
'last_modified': 'date',
'content_type': 'type'
},
{
'name': u'unicod\xe9',
'hash': 'ffff',
'bytes': 1000,
'last_modified': 'date',
'content_type': 'type'
},
]
req = swob.Request.blank('/v1/AUTH_a/sw\xc3\xa9ft?delimiter=/&limit=4',
environ={
'__test__.status': '200 OK',
'__test__.body': json.dumps(local_data),
'swift.trans_id': 'id'
})
status, headers, body_iter = req.call_application(self.app)
self.assertEqual(self.mock_shunt_swift.mock_calls, [])
self.mock_list_swift.assert_called_once_with('', 4, '', '/')
names = body_iter.split('\n')
self.assertEqual(names, [
'a',
'a/',
u'unicod\xe9'.encode('utf-8'),
'z/',
])
def test_list_container_accept_xml(self):
elements = [{
'name': 'abc',
'hash': 'ffff',
'bytes': 42,
'last_modified': 'date',
'content_type': 'type',
'content_location': 'mock-s3:bucket'
}, {
'name': u'unicod\xc3\xa9',
'hash': 'ffff',
'bytes': 1000,
'last_modified': 'date',
'content_type': 'type',
'content_location': 'mock-s3:bucket'
}]
mock_result = [dict(entry) for entry in elements]
self.mock_list_s3.side_effect = [
ProviderResponse(True, 200, {}, mock_result),
ProviderResponse(True, 200, {}, [])
]
req = swob.Request.blank('/v1/AUTH_a/s3',
environ={
'__test__.status': '200 OK',
'__test__.body': '[]',
'swift.trans_id': 'id'
},
headers={'Accept': 'application/xml'})
status, headers, body_iter = req.call_application(self.app)
self.assertEqual(self.mock_shunt_swift.mock_calls, [])
self.mock_list_s3.assert_has_calls([
mock.call('', 10000, '', ''),
mock.call(u'unicod\xc3\xa9'.encode('utf-8'), 10000, '', '')
])
root = lxml.etree.fromstring(body_iter)
context = lxml.etree.iterwalk(root, events=("start", "end"))
element_index = 0
cur_elem_properties = {}
for action, elem in context:
if action == 'end':
if elem.tag == 'container':
self.assertEqual('s3', elem.get('name'))
elif elem.tag == 'object':
for k in elements[element_index].keys():
if k == 'content_location':
self.assertTrue(k not in cur_elem_properties)
else:
self.assertEqual(elements[element_index][k],
cur_elem_properties[k])
element_index += 1
else:
try:
int_value = int(elem.text)
cur_elem_properties[elem.tag] = int_value
except ValueError:
cur_elem_properties[elem.tag] = elem.text
def test_get_conf_file_from_s3_unchanged(self, mock_syncs3):
mock_syncs3().get_object.return_value = ProviderResponse(
False, 304, {}, iter(['']))
mock_syncs3.reset_mock()
with self.assertRaises(cc_util.ConfigUnchanged):
cc_util.get_conf_file_from_s3('fefee', {
'AWS_ACCESS_KEY_ID': "ASIAIYLSOW5USUQCZAAQ",
'AWS_SECRET_ACCESS_KEY': 'swell',
'CONF_BUCKET': 'abc',
'CONF_ENDPOINT': '', # always set to at least this
}, if_none_match='nono')
self.assertEqual([
mock.call({
'aws_identity': 'ASIAIYLSOW5USUQCZAAQ',
'aws_secret': 'swell',
'encryption': False,
'custom_prefix': '',
'account': 'notused',
'container': 'notused',
'aws_bucket': 'abc',
}),
mock.call().get_object('fefee', IfNoneMatch='nono'),
], mock_syncs3.mock_calls)
def get_object(name, **args):
if name not in objects.keys():
raise RuntimeError('Unknown object: %s' % name)
if name not in migrated:
raise RuntimeError('Object should not be moved %s' % name)
return ProviderResponse(True, 200,
objects[name]['remote_headers'],
StringIO('object body'))
def test_list_container_accept_json(self):
elements = [{
'name': 'abc',
'hash': 'ffff',
'bytes': 42,
'last_modified': 'date',
'content_type': 'type',
'content_location': 'mock-s3:bucket'
}, {
'name': u'unicod\xc3\xa9',
'hash': 'ffff',
'bytes': 1000,
'last_modified': 'date',
'content_type': 'type',
'content_location': 'mock-s3:bucket'
}]
mock_result = [dict(entry) for entry in elements]
self.mock_list_s3.side_effect = [
ProviderResponse(True, 200, {}, mock_result),
ProviderResponse(True, 200, {}, [])
]
req = swob.Request.blank('/v1/AUTH_a/s3',
environ={
'__test__.status': '200 OK',
'__test__.body': '[]',
'swift.trans_id': 'id'
},
headers={'Accept': 'application/json'})
status, headers, body_iter = req.call_application(self.app)
self.assertEqual(self.mock_shunt_swift.mock_calls, [])
self.mock_list_s3.assert_has_calls([
mock.call('', 10000, '', ''),
mock.call(u'unicod\xc3\xa9'.encode('utf-8'), 10000, '', '')
])
results = json.loads(body_iter)
for i, entry in enumerate(results):
for k in elements[i].keys():
if k == 'content_location':
self.assertEqual([elements[i][k]], entry[k])
else:
self.assertEqual(elements[i][k], entry[k])
def test_read_only_no_buckets(self, mock_provider_factory):
mock_provider = mock_provider_factory.return_value
args = [
'--protocol', 's3', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://s3.amazonaws.com',
'--bucket', '/*'
]
mock_provider = mock_provider_factory.return_value
mock_provider.list_buckets.return_value = ProviderResponse(
True, 200, {}, [])
self.assertEqual('No buckets/containers found', main(args))
mock_provider.list_buckets.assert_called_once_with(limit=1)
def test_no_bucket_bad_creds(self, mock_provider_factory):
mock_provider = mock_provider_factory.return_value
args = [
'--protocol', 's3', '--username', 'id', '--password', 'key',
'--endpoint', 'https://s3.amazonaws.com'
]
mock_provider.list_buckets.return_value = ProviderResponse(
False, 500, {}, 'error')
exit_arg = main(args)
self.assertTrue(
exit_arg.endswith(
mock_provider.list_buckets.return_value.wsgi_status))
def test_list_buckets_error(self, mock_provider_factory):
mock_provider = mock_provider_factory.return_value
args = [
'--protocol', 's3', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://s3.amazonaws.com',
'--bucket', '/*'
]
mock_provider = mock_provider_factory.return_value
mock_provider.list_buckets.return_value = ProviderResponse(
False, 401, {}, [])
self.assertTrue(
main(args).endswith(
mock_provider.list_buckets.return_value.wsgi_status))
mock_provider.list_buckets.assert_called_once_with(limit=1)
def test_read_only_with_bucket_swift(self, mock_provider_factory):
mock_provider = mock.Mock()
def _fake_create(conf, max_conns=1024):
mock_provider.settings = conf
return mock_provider
mock_provider_factory.side_effect = _fake_create
args = [
'--protocol', 'swift', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://some.swift.com/auth/v1.0',
'--bucket', 'some-bucket'
]
body = self.TrackingStringIO('response')
mock_provider.list_objects.return_value = ProviderResponse(
True, 200, {}, [{
'name': 'foo'
}])
mock_provider.get_object.return_value = ProviderResponse(
True, 200, {}, body)
mock_provider.head_object.return_value = ProviderResponse(
True, 204, {}, '')
exit_arg = main(args)
self.assertEqual(0, exit_arg)
mock_provider.list_objects.assert_called_once_with(None,
1,
None,
bucket=None)
mock_provider.head_object.assert_called_once_with('foo', bucket=None)
mock_provider.get_object.assert_called_once_with('foo',
bucket=None,
resp_chunk_size=1)
self.assertEqual(0, body.last_pos)
self.assertTrue(body.closed)
def test_list_buckets_error(self, create_provider_mock):
create_provider_mock.return_value.list_buckets.return_value = \
ProviderResponse(False, 404, [], 'Not Found')
self.migrator.config = {'aws_bucket': '/*'}
with self.assertRaises(s3_sync.migrator.MigrationError):
self.migrator.next_pass()
self.assertEqual({
'aws_bucket': '/*',
'container': '.'
}, self.migrator.config)
create_provider_mock.assert_called_once_with(
{
'aws_bucket': '/*',
'container': '.'
}, self.migrator.ic_pool.max_size, False)
def test_list_container_shunt_all_containers(self, create_mock):
create_mock.return_value = mock.Mock()
create_mock.return_value.list_objects.return_value = ProviderResponse(
True, 200, {}, [])
req = swob.Request.blank('/v1/AUTH_b/s3',
environ={
'__test__.status': '200 OK',
'__test__.body': '[]',
'swift.trans_id': 'id'
})
status, headers, body_iter = req.call_application(self.app)
create_mock.assert_called_once_with(
{
'account': 'AUTH_b',
'container': 's3',
'merge_namespaces': True,
'propagate_delete': False,
'aws_bucket': 'dest-bucket',
'aws_identity': 'user',
'aws_secret': 'key'
},
max_conns=1,
per_account=True)
# Follow it up with another request to a *different* container to make
# sure we didn't bleed state
create_mock.reset_mock()
req = swob.Request.blank('/v1/AUTH_b/s4',
environ={
'__test__.status': '200 OK',
'__test__.body': '[]',
'swift.trans_id': 'id'
})
status, headers, body_iter = req.call_application(self.app)
create_mock.assert_called_once_with(
{
'account': 'AUTH_b',
'container': 's4',
'merge_namespaces': True,
'propagate_delete': False,
'aws_bucket': 'dest-bucket',
'aws_identity': 'user',
'aws_secret': 'key'
},
max_conns=1,
per_account=True)
def test_read_only_no_objects(self, mock_provider_factory):
mock_provider = mock_provider_factory.return_value
args = [
'--protocol', 's3', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://s3.amazonaws.com',
'--bucket', 'some-bucket'
]
mock_provider = mock_provider_factory.return_value
mock_provider.list_objects.return_value = ProviderResponse(
True, 200, {}, [])
self.assertEqual(
'There are no objects in the bucket to validate GET/HEAD access',
main(args))
mock_provider.list_objects.assert_called_once_with(None,
1,
None,
bucket=None)
def test_missing_container(self, create_provider_mock):
tests = [{'protocol': 'swift'}, {}]
provider = create_provider_mock.return_value
config = self.migrator.config
for test in tests:
test_config = dict(config, **test)
self.migrator.config = test_config
provider.reset_mock()
self.swift_client.reset_mock()
provider.list_objects.return_value = (200, [{'name': 'test'}])
provider.get_object.return_value = ProviderResponse(
True, 200, {'last-modified': create_timestamp(1.5e9)},
StringIO(''))
if self.migrator.config.get('protocol') == 'swift':
resp = mock.Mock()
resp.status = 200
resp.headers = {'x-container-meta-foo': 'foo'}
provider.head_bucket.return_value = resp
headers = resp.headers
else:
headers = {}
self.swift_client.iter_objects.return_value = iter([])
self.swift_client.container_exists.side_effect = (False, True)
self.migrator.status.get_migration.return_value = {}
self.migrator.next_pass()
if test.get('protocol') == 'swift':
provider.list_objects.assert_called_once_with(
None, self.migrator.work_chunk, None)
provider.head_bucket.assert_called_once_with(
self.migrator.config['container'])
else:
provider.list_objects.assert_called_once_with(
None, self.migrator.work_chunk, None, native=True)
self.swift_client.create_container.assert_called_once_with(
self.migrator.config['account'],
self.migrator.config['container'], headers)
def test_all_containers(self, create_provider_mock):
provider_mock = mock.Mock()
buckets = [{'name': 'bucket'}]
provider_mock.list_buckets.return_value = ProviderResponse(
True, 200, [], buckets)
def check_provider(config, conns, per_account):
# We have to check the arguments this way, as otherwise the
# dictionary gets mutated and assert_called_once_with check will
# fail.
self.assertEqual('/*', config['aws_bucket'])
self.assertEqual('.', config['container'])
self.assertEqual(self.migrator.ic_pool.max_size, conns)
return provider_mock
create_provider_mock.side_effect = check_provider
self.migrator.config = {'aws_bucket': '/*'}
self.migrator._next_pass = mock.Mock()
self.migrator.next_pass()
self.assertEqual(buckets[0]['name'], self.migrator.config['container'])
self.assertEqual(buckets[0]['name'],
self.migrator.config['aws_bucket'])
self.assertEqual(buckets[0]['name'], provider_mock.aws_bucket)
def _config_reader(*args, **kwargs):
return ProviderResponse(True, 200, {'etag': 'an_etag'},
iter([json.dumps(self.s3_passwd)]))
def _config_reader(*args, **kwargs):
return ProviderResponse(True, 200, {'etag': 'an_etag'},
iter(["I ain't JSON, bro!"]))
def getter(*args, **kwargs):
return ProviderResponse(
True, 200, {'etag': args[0] + '_' + tags.pop(0)},
iter([args[0], 'conf']))
