def test_head_object_error(self, mock_provider_factory): mock_provider = mock_provider_factory.return_value args = [ '--protocol', 's3', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://s3.amazonaws.com', '--bucket', 'some-bucket' ] mock_provider.list_objects.return_value = ProviderResponse( True, 200, {}, [{ 'name': 'foo' }]) mock_provider.head_object.return_value = ProviderResponse( False, 500, {}, '') exit_arg = main(args) self.assertTrue( exit_arg.endswith( mock_provider.head_object.return_value.wsgi_status)) mock_provider.list_objects.assert_called_once_with(None, 1, None, bucket=None) mock_provider.head_object.assert_called_once_with('foo', bucket=None) mock_provider.get_object.assert_not_called()
def test_read_only_all_buckets(self, mock_provider_factory): args = [ '--protocol', 'swift', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://saio:8080/auth/v1.0', '--bucket', '/*' ] body = self.TrackingStringIO('response') mock_provider = mock_provider_factory.return_value mock_provider.list_buckets.return_value = ProviderResponse( True, 200, {}, [{ 'name': 'container' }]) mock_provider.list_objects.return_value = ProviderResponse( True, 200, {}, [{ 'name': 'foo' }]) mock_provider.get_object.return_value = ProviderResponse( True, 200, {}, body) mock_provider.head_object.return_value = ProviderResponse( True, 204, {}, '') exit_arg = main(args) self.assertEqual(0, exit_arg) mock_provider.list_buckets.assert_called_once_with(limit=1) mock_provider.list_objects.assert_called_once_with(None, 1, None, bucket='container') mock_provider.head_object.assert_called_once_with('foo', bucket='container') mock_provider.get_object.assert_called_once_with('foo', bucket='container') self.assertEqual(0, body.last_pos) self.assertTrue(body.closed)
def test_list_container_shunt_swift(self): self.mock_list_swift.side_effect = [ ProviderResponse(True, 200, {}, [{ 'name': 'abc', 'hash': 'ffff', 'bytes': 42, 'last_modified': 'date', 'content_type': 'type', 'content_location': 'http://some-swift' }, { 'name': u'unicod\xe9', 'hash': 'ffff', 'bytes': 1000, 'last_modified': 'date', 'content_type': 'type', 'content_location': 'http://some-swift' }]), ProviderResponse(True, 200, {}, []) ] req = swob.Request.blank('/v1/AUTH_a/sw\xc3\xa9ft', environ={ '__test__.status': '200 OK', '__test__.body': '[]', 'swift.trans_id': 'id' }) status, headers, body_iter = req.call_application(self.app) self.assertEqual(self.mock_shunt_swift.mock_calls, []) self.mock_list_swift.assert_has_calls([ mock.call('', 10000, '', ''), mock.call(u'unicod\xe9'.encode('utf-8'), 10000, '', '') ]) names = body_iter.split('\n') self.assertEqual(['abc', u'unicod\xe9'.encode('utf-8')], names)
def test_read_only_with_bucket_s3(self, mock_provider_factory): mock_provider = mock_provider_factory.return_value args = [ '--protocol', 's3', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://s3.amazonaws.com', '--bucket', 'some-bucket' ] body = self.TrackingStringIO('response') mock_provider.list_objects.return_value = ProviderResponse( True, 200, {}, [{ 'name': 'foo' }]) mock_provider.get_object.return_value = ProviderResponse( True, 200, {}, body) mock_provider.head_object.return_value = ProviderResponse( True, 204, {}, '') exit_arg = main(args) self.assertEqual(0, exit_arg) mock_provider.list_objects.assert_called_once_with(None, 1, None, bucket=None) mock_provider.head_object.assert_called_once_with('foo', bucket=None) mock_provider.get_object.assert_called_once_with('foo', bucket=None) self.assertEqual(0, body.last_pos) self.assertTrue(body.closed)
def test_list_container_shunt_with_duplicates(self): self.mock_list_swift.side_effect = [ ProviderResponse(True, 200, {}, [{ 'subdir': u'a/', 'content_location': 'http://some-swift' }, { 'name': u'unicod\xe9', 'hash': 'ffff', 'bytes': 1000, 'last_modified': 'date', 'content_type': 'type', 'content_location': 'http://some-swift' }, { 'subdir': u'z/', 'content_location': 'http://some-swift' }, { 'name': u'zzzzz', 'hash': 'ffff', 'bytes': 1000, 'last_modified': 'date', 'content_type': 'type', 'content_location': 'http://some-swift' }]), ProviderResponse(True, 200, {}, []) ] # simulate being partially migrated local_data = [ { 'name': u'a', 'hash': 'ffff', 'bytes': 42, 'last_modified': 'date', 'content_type': 'type' }, { 'name': u'unicod\xe9', 'hash': 'ffff', 'bytes': 1000, 'last_modified': 'date', 'content_type': 'type' }, ] req = swob.Request.blank('/v1/AUTH_a/sw\xc3\xa9ft?delimiter=/&limit=4', environ={ '__test__.status': '200 OK', '__test__.body': json.dumps(local_data), 'swift.trans_id': 'id' }) status, headers, body_iter = req.call_application(self.app) self.assertEqual(self.mock_shunt_swift.mock_calls, []) self.mock_list_swift.assert_called_once_with('', 4, '', '/') names = body_iter.split('\n') self.assertEqual(names, [ 'a', 'a/', u'unicod\xe9'.encode('utf-8'), 'z/', ])
def test_list_container_accept_xml(self): elements = [{ 'name': 'abc', 'hash': 'ffff', 'bytes': 42, 'last_modified': 'date', 'content_type': 'type', 'content_location': 'mock-s3:bucket' }, { 'name': u'unicod\xc3\xa9', 'hash': 'ffff', 'bytes': 1000, 'last_modified': 'date', 'content_type': 'type', 'content_location': 'mock-s3:bucket' }] mock_result = [dict(entry) for entry in elements] self.mock_list_s3.side_effect = [ ProviderResponse(True, 200, {}, mock_result), ProviderResponse(True, 200, {}, []) ] req = swob.Request.blank('/v1/AUTH_a/s3', environ={ '__test__.status': '200 OK', '__test__.body': '[]', 'swift.trans_id': 'id' }, headers={'Accept': 'application/xml'}) status, headers, body_iter = req.call_application(self.app) self.assertEqual(self.mock_shunt_swift.mock_calls, []) self.mock_list_s3.assert_has_calls([ mock.call('', 10000, '', ''), mock.call(u'unicod\xc3\xa9'.encode('utf-8'), 10000, '', '') ]) root = lxml.etree.fromstring(body_iter) context = lxml.etree.iterwalk(root, events=("start", "end")) element_index = 0 cur_elem_properties = {} for action, elem in context: if action == 'end': if elem.tag == 'container': self.assertEqual('s3', elem.get('name')) elif elem.tag == 'object': for k in elements[element_index].keys(): if k == 'content_location': self.assertTrue(k not in cur_elem_properties) else: self.assertEqual(elements[element_index][k], cur_elem_properties[k]) element_index += 1 else: try: int_value = int(elem.text) cur_elem_properties[elem.tag] = int_value except ValueError: cur_elem_properties[elem.tag] = elem.text
def test_get_conf_file_from_s3_unchanged(self, mock_syncs3): mock_syncs3().get_object.return_value = ProviderResponse( False, 304, {}, iter([''])) mock_syncs3.reset_mock() with self.assertRaises(cc_util.ConfigUnchanged): cc_util.get_conf_file_from_s3('fefee', { 'AWS_ACCESS_KEY_ID': "ASIAIYLSOW5USUQCZAAQ", 'AWS_SECRET_ACCESS_KEY': 'swell', 'CONF_BUCKET': 'abc', 'CONF_ENDPOINT': '', # always set to at least this }, if_none_match='nono') self.assertEqual([ mock.call({ 'aws_identity': 'ASIAIYLSOW5USUQCZAAQ', 'aws_secret': 'swell', 'encryption': False, 'custom_prefix': '', 'account': 'notused', 'container': 'notused', 'aws_bucket': 'abc', }), mock.call().get_object('fefee', IfNoneMatch='nono'), ], mock_syncs3.mock_calls)
def get_object(name, **args): if name not in objects.keys(): raise RuntimeError('Unknown object: %s' % name) if name not in migrated: raise RuntimeError('Object should not be moved %s' % name) return ProviderResponse(True, 200, objects[name]['remote_headers'], StringIO('object body'))
def test_list_container_accept_json(self): elements = [{ 'name': 'abc', 'hash': 'ffff', 'bytes': 42, 'last_modified': 'date', 'content_type': 'type', 'content_location': 'mock-s3:bucket' }, { 'name': u'unicod\xc3\xa9', 'hash': 'ffff', 'bytes': 1000, 'last_modified': 'date', 'content_type': 'type', 'content_location': 'mock-s3:bucket' }] mock_result = [dict(entry) for entry in elements] self.mock_list_s3.side_effect = [ ProviderResponse(True, 200, {}, mock_result), ProviderResponse(True, 200, {}, []) ] req = swob.Request.blank('/v1/AUTH_a/s3', environ={ '__test__.status': '200 OK', '__test__.body': '[]', 'swift.trans_id': 'id' }, headers={'Accept': 'application/json'}) status, headers, body_iter = req.call_application(self.app) self.assertEqual(self.mock_shunt_swift.mock_calls, []) self.mock_list_s3.assert_has_calls([ mock.call('', 10000, '', ''), mock.call(u'unicod\xc3\xa9'.encode('utf-8'), 10000, '', '') ]) results = json.loads(body_iter) for i, entry in enumerate(results): for k in elements[i].keys(): if k == 'content_location': self.assertEqual([elements[i][k]], entry[k]) else: self.assertEqual(elements[i][k], entry[k])
def test_read_only_no_buckets(self, mock_provider_factory): mock_provider = mock_provider_factory.return_value args = [ '--protocol', 's3', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://s3.amazonaws.com', '--bucket', '/*' ] mock_provider = mock_provider_factory.return_value mock_provider.list_buckets.return_value = ProviderResponse( True, 200, {}, []) self.assertEqual('No buckets/containers found', main(args)) mock_provider.list_buckets.assert_called_once_with(limit=1)
def test_no_bucket_bad_creds(self, mock_provider_factory): mock_provider = mock_provider_factory.return_value args = [ '--protocol', 's3', '--username', 'id', '--password', 'key', '--endpoint', 'https://s3.amazonaws.com' ] mock_provider.list_buckets.return_value = ProviderResponse( False, 500, {}, 'error') exit_arg = main(args) self.assertTrue( exit_arg.endswith( mock_provider.list_buckets.return_value.wsgi_status))
def test_list_buckets_error(self, mock_provider_factory): mock_provider = mock_provider_factory.return_value args = [ '--protocol', 's3', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://s3.amazonaws.com', '--bucket', '/*' ] mock_provider = mock_provider_factory.return_value mock_provider.list_buckets.return_value = ProviderResponse( False, 401, {}, []) self.assertTrue( main(args).endswith( mock_provider.list_buckets.return_value.wsgi_status)) mock_provider.list_buckets.assert_called_once_with(limit=1)
def test_read_only_with_bucket_swift(self, mock_provider_factory): mock_provider = mock.Mock() def _fake_create(conf, max_conns=1024): mock_provider.settings = conf return mock_provider mock_provider_factory.side_effect = _fake_create args = [ '--protocol', 'swift', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://some.swift.com/auth/v1.0', '--bucket', 'some-bucket' ] body = self.TrackingStringIO('response') mock_provider.list_objects.return_value = ProviderResponse( True, 200, {}, [{ 'name': 'foo' }]) mock_provider.get_object.return_value = ProviderResponse( True, 200, {}, body) mock_provider.head_object.return_value = ProviderResponse( True, 204, {}, '') exit_arg = main(args) self.assertEqual(0, exit_arg) mock_provider.list_objects.assert_called_once_with(None, 1, None, bucket=None) mock_provider.head_object.assert_called_once_with('foo', bucket=None) mock_provider.get_object.assert_called_once_with('foo', bucket=None, resp_chunk_size=1) self.assertEqual(0, body.last_pos) self.assertTrue(body.closed)
def test_list_buckets_error(self, create_provider_mock): create_provider_mock.return_value.list_buckets.return_value = \ ProviderResponse(False, 404, [], 'Not Found') self.migrator.config = {'aws_bucket': '/*'} with self.assertRaises(s3_sync.migrator.MigrationError): self.migrator.next_pass() self.assertEqual({ 'aws_bucket': '/*', 'container': '.' }, self.migrator.config) create_provider_mock.assert_called_once_with( { 'aws_bucket': '/*', 'container': '.' }, self.migrator.ic_pool.max_size, False)
def test_list_container_shunt_all_containers(self, create_mock): create_mock.return_value = mock.Mock() create_mock.return_value.list_objects.return_value = ProviderResponse( True, 200, {}, []) req = swob.Request.blank('/v1/AUTH_b/s3', environ={ '__test__.status': '200 OK', '__test__.body': '[]', 'swift.trans_id': 'id' }) status, headers, body_iter = req.call_application(self.app) create_mock.assert_called_once_with( { 'account': 'AUTH_b', 'container': 's3', 'merge_namespaces': True, 'propagate_delete': False, 'aws_bucket': 'dest-bucket', 'aws_identity': 'user', 'aws_secret': 'key' }, max_conns=1, per_account=True) # Follow it up with another request to a *different* container to make # sure we didn't bleed state create_mock.reset_mock() req = swob.Request.blank('/v1/AUTH_b/s4', environ={ '__test__.status': '200 OK', '__test__.body': '[]', 'swift.trans_id': 'id' }) status, headers, body_iter = req.call_application(self.app) create_mock.assert_called_once_with( { 'account': 'AUTH_b', 'container': 's4', 'merge_namespaces': True, 'propagate_delete': False, 'aws_bucket': 'dest-bucket', 'aws_identity': 'user', 'aws_secret': 'key' }, max_conns=1, per_account=True)
def test_read_only_no_objects(self, mock_provider_factory): mock_provider = mock_provider_factory.return_value args = [ '--protocol', 's3', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://s3.amazonaws.com', '--bucket', 'some-bucket' ] mock_provider = mock_provider_factory.return_value mock_provider.list_objects.return_value = ProviderResponse( True, 200, {}, []) self.assertEqual( 'There are no objects in the bucket to validate GET/HEAD access', main(args)) mock_provider.list_objects.assert_called_once_with(None, 1, None, bucket=None)
def test_missing_container(self, create_provider_mock): tests = [{'protocol': 'swift'}, {}] provider = create_provider_mock.return_value config = self.migrator.config for test in tests: test_config = dict(config, **test) self.migrator.config = test_config provider.reset_mock() self.swift_client.reset_mock() provider.list_objects.return_value = (200, [{'name': 'test'}]) provider.get_object.return_value = ProviderResponse( True, 200, {'last-modified': create_timestamp(1.5e9)}, StringIO('')) if self.migrator.config.get('protocol') == 'swift': resp = mock.Mock() resp.status = 200 resp.headers = {'x-container-meta-foo': 'foo'} provider.head_bucket.return_value = resp headers = resp.headers else: headers = {} self.swift_client.iter_objects.return_value = iter([]) self.swift_client.container_exists.side_effect = (False, True) self.migrator.status.get_migration.return_value = {} self.migrator.next_pass() if test.get('protocol') == 'swift': provider.list_objects.assert_called_once_with( None, self.migrator.work_chunk, None) provider.head_bucket.assert_called_once_with( self.migrator.config['container']) else: provider.list_objects.assert_called_once_with( None, self.migrator.work_chunk, None, native=True) self.swift_client.create_container.assert_called_once_with( self.migrator.config['account'], self.migrator.config['container'], headers)
def test_all_containers(self, create_provider_mock): provider_mock = mock.Mock() buckets = [{'name': 'bucket'}] provider_mock.list_buckets.return_value = ProviderResponse( True, 200, [], buckets) def check_provider(config, conns, per_account): # We have to check the arguments this way, as otherwise the # dictionary gets mutated and assert_called_once_with check will # fail. self.assertEqual('/*', config['aws_bucket']) self.assertEqual('.', config['container']) self.assertEqual(self.migrator.ic_pool.max_size, conns) return provider_mock create_provider_mock.side_effect = check_provider self.migrator.config = {'aws_bucket': '/*'} self.migrator._next_pass = mock.Mock() self.migrator.next_pass() self.assertEqual(buckets[0]['name'], self.migrator.config['container']) self.assertEqual(buckets[0]['name'], self.migrator.config['aws_bucket']) self.assertEqual(buckets[0]['name'], provider_mock.aws_bucket)
def _config_reader(*args, **kwargs): return ProviderResponse(True, 200, {'etag': 'an_etag'}, iter([json.dumps(self.s3_passwd)]))
def _config_reader(*args, **kwargs): return ProviderResponse(True, 200, {'etag': 'an_etag'}, iter(["I ain't JSON, bro!"]))
def getter(*args, **kwargs): return ProviderResponse( True, 200, {'etag': args[0] + '_' + tags.pop(0)}, iter([args[0], 'conf']))