def test_keystone_requires_swift_proto(self, mock_validate): exit_arg = main([ '--protocol', 's3', '--endpoint', 'https://s3.amazonaws.com', '--username', 'access id', '--password', 'secret key', "--auth-type", "keystone_v3", "--project-name", "test", "--user-domain-name", "default", "--project-domain-name", "default", ]) self.assertIn('Keystone auth requires swift protocol', exit_arg) exit_arg = main([ '--protocol', 's3', '--endpoint', 'https://s3.amazonaws.com', '--username', 'access id', '--password', 'secret key', "--auth-type", "keystone_v2", "--tenant-name", "test", ]) self.assertIn('Keystone auth requires swift protocol', exit_arg)
def test_google_no_bucket(self, mock_get_client): mock_client = \ mock_get_client.return_value.__enter__.return_value mock_client.list_buckets.return_value = { 'Buckets': [{ 'CreationDate': datetime.datetime.now(), 'Name': 'google-bucket' }], 'ResponseMetadata': { 'HTTPStatusCode': 200, 'HTTPHeaders': {} } } exit_arg = main([ '--protocol', 's3', '--endpoint', 'https://storage.googleapis.com', '--username', 'access id', '--password', 'secret key', ]) self.assertEqual(exit_arg, 0) self.assertEqual(mock_client.mock_calls, [ mock.call.list_buckets(), ])
def test_swift_one_bucket(self, mock_validate): exit_arg = main([ '--protocol', 'swift', '--endpoint', 'https://saio:8080/auth/v1.0', '--username', 'access id', '--password', 'secret key', '--storage-policy', 'better', '--bucket', 'some-bucket', ]) self.assertIs(exit_arg, mock_validate.return_value) self.assertEqual([mock.ANY], mock_validate.mock_calls) provider, swift_key, create_bucket, storage_policy =\ mock_validate.mock_calls[0][1] self.assertEqual( { k: v for k, v in provider.settings.items() if k.startswith('aws_') or k in ('protocol', ) }, { 'protocol': 'swift', 'aws_endpoint': 'https://saio:8080/auth/v1.0', 'aws_identity': 'access id', 'aws_secret': 'secret key', 'aws_bucket': 'some-bucket', }) self.assertEqual(swift_key, 'cloud_sync_test_object') self.assertFalse(create_bucket) self.assertEqual(storage_policy, 'better')
def test_aws_with_prefix(self, mock_validate): exit_arg = main([ '--protocol', 's3', '--endpoint', 'https://s3.amazonaws.com', '--username', 'access id', '--password', 'secret key', '--bucket', 'some-bucket', '--prefix', 'jojo/hoho/', ]) self.assertIs(exit_arg, mock_validate.return_value) self.assertEqual([mock.ANY], mock_validate.mock_calls) provider, swift_key, create_bucket, _storage_policy =\ mock_validate.mock_calls[0][1] self.assertEqual( { 'protocol': 's3', 'aws_endpoint': None, 'aws_identity': 'access id', 'aws_secret': 'secret key', 'aws_bucket': 'some-bucket', 'account': 'verify-auth', 'custom_prefix': 'jojo/hoho/', 'remote_account': None, }, {k: v for k, v in provider.settings.items() if k != 'container'}) self.assertEqual(swift_key, 'cloud_sync_test') self.assertFalse(create_bucket)
def test_head_object_error(self, mock_provider_factory): mock_provider = mock_provider_factory.return_value args = [ '--protocol', 's3', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://s3.amazonaws.com', '--bucket', 'some-bucket' ] mock_provider.list_objects.return_value = ProviderResponse( True, 200, {}, [{ 'name': 'foo' }]) mock_provider.head_object.return_value = ProviderResponse( False, 500, {}, '') exit_arg = main(args) self.assertTrue( exit_arg.endswith( mock_provider.head_object.return_value.wsgi_status)) mock_provider.list_objects.assert_called_once_with(None, 1, None, bucket=None) mock_provider.head_object.assert_called_once_with('foo', bucket=None) mock_provider.get_object.assert_not_called()
def test_read_only_with_bucket_s3(self, mock_provider_factory): mock_provider = mock_provider_factory.return_value args = [ '--protocol', 's3', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://s3.amazonaws.com', '--bucket', 'some-bucket' ] body = self.TrackingStringIO('response') mock_provider.list_objects.return_value = ProviderResponse( True, 200, {}, [{ 'name': 'foo' }]) mock_provider.get_object.return_value = ProviderResponse( True, 200, {}, body) mock_provider.head_object.return_value = ProviderResponse( True, 204, {}, '') exit_arg = main(args) self.assertEqual(0, exit_arg) mock_provider.list_objects.assert_called_once_with(None, 1, None, bucket=None) mock_provider.head_object.assert_called_once_with('foo', bucket=None) mock_provider.get_object.assert_called_once_with('foo', bucket=None) self.assertEqual(0, body.last_pos) self.assertTrue(body.closed)
def test_google_leaves_endpoint_alone(self, mock_validate): exit_arg = main([ '--protocol', 's3', '--endpoint', 'https://storage.googleapis.com', '--username', 'access id', '--password', 'secret key', '--bucket', 'some-bucket', ]) self.assertIs(exit_arg, mock_validate.return_value) self.assertEqual([mock.ANY], mock_validate.mock_calls) provider, swift_key, create_bucket = mock_validate.mock_calls[0][1] self.assertEqual( { k: v for k, v in provider.settings.items() if k.startswith('aws_') or k in ('protocol', ) }, { 'protocol': 's3', 'aws_endpoint': 'https://storage.googleapis.com', 'aws_identity': 'access id', 'aws_secret': 'secret key', 'aws_bucket': 'some-bucket', }) self.assertEqual(swift_key, 'fabcab/cloud_sync_test') self.assertFalse(create_bucket)
def test_swift_all_buckets(self, mock_validate): exit_arg = main([ '--protocol', 'swift', '--endpoint', 'https://saio:8080/auth/v1.0', '--username', 'access id', '--password', 'secret key', '--bucket', '/*', ]) self.assertIs(exit_arg, mock_validate.return_value) self.assertEqual([mock.ANY], mock_validate.mock_calls) provider, swift_key, create_bucket = mock_validate.mock_calls[0][1] self.assertEqual( { k: v for k, v in provider.settings.items() if k.startswith('aws_') or k in ('protocol', ) }, { 'protocol': 'swift', 'aws_endpoint': 'https://saio:8080/auth/v1.0', 'aws_identity': 'access id', 'aws_secret': 'secret key', 'aws_bucket': u'.cloudsync_test_container-\U0001f44d', }) self.assertEqual(swift_key, 'cloud_sync_test_object') self.assertTrue(create_bucket)
def test_read_only_all_buckets(self, mock_provider_factory): args = [ '--protocol', 'swift', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://saio:8080/auth/v1.0', '--bucket', '/*' ] body = self.TrackingStringIO('response') mock_provider = mock_provider_factory.return_value mock_provider.list_buckets.return_value = ProviderResponse( True, 200, {}, [{ 'name': 'container' }]) mock_provider.list_objects.return_value = ProviderResponse( True, 200, {}, [{ 'name': 'foo' }]) mock_provider.get_object.return_value = ProviderResponse( True, 200, {}, body) mock_provider.head_object.return_value = ProviderResponse( True, 204, {}, '') exit_arg = main(args) self.assertEqual(0, exit_arg) mock_provider.list_buckets.assert_called_once_with(limit=1) mock_provider.list_objects.assert_called_once_with(None, 1, None, bucket='container') mock_provider.head_object.assert_called_once_with('foo', bucket='container') mock_provider.get_object.assert_called_once_with('foo', bucket='container') self.assertEqual(0, body.last_pos) self.assertTrue(body.closed)
def test_swift_with_bucket(self, mock_get_client): def _fake_put_object(*args, **kwargs): kwargs['response_dict']['headers'] = {} kwargs['response_dict']['status'] = 201 return 'deadbeef' mock_client = \ mock_get_client.return_value.__enter__.return_value mock_client.put_object.side_effect = _fake_put_object mock_client.head_object.side_effect = [ { 'x-object-meta-cloud-sync': 'fabcab' }, { 'x-object-meta-cloud-sync': 'fabcab' }, # One extra for the DELETE ] mock_client.get_container.return_value = ({}, []) mock_client.post_object.return_value = None mock_client.delete_object.return_value = None exit_arg = main([ '--protocol', 'swift', '--endpoint', 'https://saio:8080/auth/v1.0', '--username', 'access id', '--password', 'secret key', '--bucket', 'some-bucket', ]) self.assertEqual(exit_arg, 0) self.assertEqual(mock_client.mock_calls, [ mock.call.put_object('some-bucket', 'cloud_sync_test_object', contents='1space-test', headers={'content-type': 'text/plain'}, response_dict=mock.ANY), mock.call.post_object('some-bucket', 'cloud_sync_test_object', headers={ 'content-type': 'text/plain', 'x-object-meta-cloud-sync': 'fabcab' }), mock.call.head_object( 'some-bucket', 'cloud_sync_test_object', headers={}), mock.call.get_container('some-bucket', delimiter='', limit=1, marker='', prefix='', headers={}), mock.call.head_object( 'some-bucket', 'cloud_sync_test_object', headers={}), mock.call.delete_object( 'some-bucket', 'cloud_sync_test_object', headers={}), ])
def test_swift_with_bucket(self, mock_get_client): mock_client = \ mock_get_client.return_value.__enter__.return_value mock_client.head_object.side_effect = [ None, { 'x-object-meta-cloud-sync': 'fabcab' }, { 'x-object-meta-cloud-sync': 'fabcab' }, # One extra for the DELETE ] mock_client.get_container.return_value = ({}, []) mock_client.post_object.return_value = None mock_client.delete_object.return_value = None exit_arg = main([ '--protocol', 'swift', '--endpoint', 'https://saio:8080/auth/v1.0', '--username', 'access id', '--password', 'secret key', '--bucket', 'some-bucket', ]) self.assertEqual(exit_arg, 0) self.assertEqual(mock_client.mock_calls, [ mock.call.head_object( 'some-bucket', 'cloud_sync_test_object', headers={}), mock.call.put_object('some-bucket', 'cloud_sync_test_object', mock.ANY, content_length=15, etag=mock.ANY, headers={'content-type': 'text/plain'}), mock.call.post_object('some-bucket', 'cloud_sync_test_object', headers={ 'content-type': 'text/plain', 'X-Object-Meta-Cloud-Sync': 'fabcab' }), mock.call.head_object( 'some-bucket', 'cloud_sync_test_object', headers={}), mock.call.get_container('some-bucket', delimiter='', limit=1, marker='', prefix='', headers={}), mock.call.head_object( 'some-bucket', 'cloud_sync_test_object', headers={}), mock.call.delete_object( 'some-bucket', 'cloud_sync_test_object', headers={}), ])
def _stderr_of_sysexit(self, args): with self.assertRaises(SystemExit), \ mock.patch('sys.stderr', new_callable=BytesIO) as err: got = main(args) # This little adapter lets us also test error conditions where # the message is returned, for the surrounding "exit(...)" to # turn into a SystemExit. if got: print >> sys.stderr, got raise SystemExit() return err.getvalue()
def test_read_only_no_buckets(self, mock_provider_factory): mock_provider = mock_provider_factory.return_value args = [ '--protocol', 's3', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://s3.amazonaws.com', '--bucket', '/*' ] mock_provider = mock_provider_factory.return_value mock_provider.list_buckets.return_value = ProviderResponse( True, 200, {}, []) self.assertEqual('No buckets/containers found', main(args)) mock_provider.list_buckets.assert_called_once_with(limit=1)
def test_list_buckets_error(self, mock_provider_factory): mock_provider = mock_provider_factory.return_value args = [ '--protocol', 's3', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://s3.amazonaws.com', '--bucket', '/*' ] mock_provider = mock_provider_factory.return_value mock_provider.list_buckets.return_value = ProviderResponse( False, 401, {}, []) self.assertTrue( main(args).endswith( mock_provider.list_buckets.return_value.wsgi_status)) mock_provider.list_buckets.assert_called_once_with(limit=1)
def test_no_bucket_bad_creds(self, mock_provider_factory): mock_provider = mock_provider_factory.return_value args = [ '--protocol', 's3', '--username', 'id', '--password', 'key', '--endpoint', 'https://s3.amazonaws.com' ] mock_provider.list_buckets.return_value = ProviderResponse( False, 500, {}, 'error') exit_arg = main(args) self.assertTrue( exit_arg.endswith( mock_provider.list_buckets.return_value.wsgi_status))
def test_bucket_cant_have_slash(self, mock_validate): msg = 'Invalid argument: slash is not allowed in container name' self.assertEqual( msg, main([ '--protocol', 's3', '--endpoint', 'https://s3.amazonaws.com', '--username', 'access id', '--password', 'secret key', '--bucket', 'some/bucket', ])) self.assertEqual(mock_validate.mock_calls, [])
def test_swift_no_bucket(self, mock_get_client): exit_arg = main([ '--protocol', 'swift', '--endpoint', 'https://saio:8080/auth/v1.0', '--username', 'access id', '--password', 'secret key', ]) self.assertEqual(exit_arg, 0) mock_client = \ mock_get_client.return_value.__enter__.return_value self.assertEqual(mock_client.mock_calls, [ mock.call.get_account(), ])
def test_google_no_bucket(self, mock_get_client): exit_arg = main([ '--protocol', 's3', '--endpoint', 'https://storage.googleapis.com', '--username', 'access id', '--password', 'secret key', ]) self.assertEqual(exit_arg, 0) mock_client = \ mock_get_client.return_value.__enter__.return_value self.assertEqual(mock_client.mock_calls, [ mock.call.list_buckets(), ])
def test_read_only_no_objects(self, mock_provider_factory): mock_provider = mock_provider_factory.return_value args = [ '--protocol', 's3', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://s3.amazonaws.com', '--bucket', 'some-bucket' ] mock_provider = mock_provider_factory.return_value mock_provider.list_objects.return_value = ProviderResponse( True, 200, {}, []) self.assertEqual( 'There are no objects in the bucket to validate GET/HEAD access', main(args)) mock_provider.list_objects.assert_called_once_with(None, 1, None, bucket=None)
def test_swift_no_bucket_empty_account(self, mock_get_client): mock_client = \ mock_get_client.return_value.__enter__.return_value mock_client.get_account.return_value = ({}, []) exit_arg = main([ '--protocol', 'swift', '--endpoint', 'https://saio:8080/auth/v1.0', '--username', 'access id', '--password', 'secret key', ]) self.assertEqual(exit_arg, 0) self.assertEqual( mock_client.mock_calls, [mock.call.get_account(headers={}, prefix='', limit=1, marker='')])
def test_account_requires_swift(self, mock_validate): msg = 'Invalid argument: account is only valid with swift protocol' self.assertEqual( msg, main([ '--protocol', 's3', '--endpoint', 'https://s3.amazonaws.com', '--username', 'access id', '--password', 'secret key', '--account', 'AUTH_account', '--bucket', 'some-bucket', ])) self.assertEqual(mock_validate.mock_calls, [])
def test_swift_keystone_v3(self, mock_validate): exit_arg = main([ '--protocol', 'swift', '--endpoint', 'http://1space-keystone:5000/v3', '--username', 'access id', '--password', 'secret key', "--auth-type", "keystone_v3", "--project-name", "test", "--project-domain-name", "wat-wat", "--user-domain-name", 'floo-boo', '--bucket', '/*', ]) self.assertIs(exit_arg, mock_validate.return_value) self.assertEqual([mock.ANY], mock_validate.mock_calls) provider, swift_key, create_bucket, _storage_policy =\ mock_validate.mock_calls[0][1] self.assertEqual( { 'protocol': 'swift', 'aws_endpoint': 'http://1space-keystone:5000/v3', 'aws_identity': 'access id', 'aws_secret': 'secret key', 'aws_bucket': u'.cloudsync_test_container-\u00ef', 'account': 'verify-auth', 'container': u'testing-\u00ef', 'custom_prefix': None, 'remote_account': None, 'auth_type': 'keystone_v3', 'user_domain_name': 'floo-boo', 'project_name': 'test', 'project_domain_name': 'wat-wat', }, provider.settings) self.assertEqual(swift_key, 'cloud_sync_test_object') self.assertTrue(create_bucket)
def test_read_only_with_bucket_swift(self, mock_provider_factory): mock_provider = mock.Mock() def _fake_create(conf, max_conns=1024): mock_provider.settings = conf return mock_provider mock_provider_factory.side_effect = _fake_create args = [ '--protocol', 'swift', '--username', 'id', '--password', 'key', '--read-only', '--endpoint', 'https://some.swift.com/auth/v1.0', '--bucket', 'some-bucket' ] body = self.TrackingStringIO('response') mock_provider.list_objects.return_value = ProviderResponse( True, 200, {}, [{ 'name': 'foo' }]) mock_provider.get_object.return_value = ProviderResponse( True, 200, {}, body) mock_provider.head_object.return_value = ProviderResponse( True, 204, {}, '') exit_arg = main(args) self.assertEqual(0, exit_arg) mock_provider.list_objects.assert_called_once_with(None, 1, None, bucket=None) mock_provider.head_object.assert_called_once_with('foo', bucket=None) mock_provider.get_object.assert_called_once_with('foo', bucket=None, resp_chunk_size=1) self.assertEqual(0, body.last_pos) self.assertTrue(body.closed)
def test_aws_no_bucket_empty_account(self, mock_get_client): mock_client = \ mock_get_client.return_value.__enter__.return_value mock_client.list_buckets.return_value = { 'Buckets': [], 'ResponseMetadata': { 'HTTPStatusCode': 200, 'HTTPHeaders': {} } } exit_arg = main([ '--protocol', 's3', '--endpoint', 'https://s3.amazonaws.com', '--username', 'access id', '--password', 'secret key', ]) self.assertEqual(exit_arg, 0) self.assertEqual(mock_client.mock_calls, [ mock.call.list_buckets(), ])
def do_test(args, missing_arg): with self.assertRaises(SystemExit), \ mock.patch('sys.stderr', new_callable=BytesIO) as err: main(args) self.assertIn('argument %s is required' % missing_arg, err.getvalue())
def test_google_with_bucket(self, mock_get_client): mock_client = \ mock_get_client.return_value.__enter__.return_value mock_client.head_object.side_effect = [ None, { # HEAD after PUT 'Body': [''], 'ResponseMetadata': { 'HTTPStatusCode': 200, 'HTTPHeaders': { 'x-amz-meta-cloud-sync': 'fabcab', }, }, }, ] mock_client.list_objects.return_value = { 'Contents': [], 'ReponseMetadata': { 'HTTPStatusCode': 200, 'HTTPHeaders': { 'x-amz-meta-cloud-sync': 'fabcab', }, }, } exit_arg = main([ '--protocol', 's3', '--endpoint', 'https://storage.googleapis.com', '--username', 'access id', '--password', 'secret key', '--bucket', 'some-bucket', ]) self.assertEqual(exit_arg, 0) key = u'9f9835/verify-auth/testing-\U0001f44d/fabcab/cloud_sync_test' self.assert_calls(mock_client, [ mock.call.head_object(Bucket='some-bucket', Key=key), mock.call.put_object(Body=mock.ANY, Bucket='some-bucket', ContentLength=15, ContentType='text/plain', Key=key, Metadata={}), mock.call.copy_object(Bucket='some-bucket', ContentType='text/plain', CopySource={ 'Bucket': 'some-bucket', 'Key': key }, Key=key, Metadata={'cloud-sync': 'fabcab'}, MetadataDirective='REPLACE'), mock.call.head_object(Bucket='some-bucket', Key=key), mock.call.delete_object(Bucket='some-bucket', Key=key), mock.call.list_objects( Bucket='some-bucket', MaxKeys=1, Prefix=u'9f9835/verify-auth/testing-\U0001f44d/'), ])
def test_aws_with_bucket_and_prefix(self, mock_get_client): mock_client = \ mock_get_client.return_value.__enter__.return_value mock_client.put_object.return_value = { 'Body': [''], 'ResponseMetadata': { 'HTTPStatusCode': 201, 'HTTPHeaders': {} } } mock_client.head_object.return_value = { 'Body': [''], 'ResponseMetadata': { 'HTTPStatusCode': 200, 'HTTPHeaders': { 'x-amz-meta-cloud-sync': 'fabcab', }, }, } mock_client.copy_object.return_value = {'CopyObjectResult': {}} mock_client.list_objects.return_value = { 'Contents': [], 'ResponseMetadata': { 'HTTPStatusCode': 200, 'HTTPHeaders': { 'x-amz-meta-cloud-sync': 'fabcab', }, }, } mock_client.delete_object.return_value = { 'DeleteMarker': False, 'VersionId': '', } exit_arg = main([ '--protocol', 's3', '--endpoint', 'https://s3.amazonaws.com', '--username', 'access id', '--password', 'secret key', '--bucket', 'some-bucket', '--prefix', 'heehee/hawhaw/', ]) self.assertEqual(exit_arg, 0) key = u'heehee/hawhaw/cloud_sync_test' self.assert_calls(mock_client, [ mock.call.put_object(Body='1space-test', Bucket='some-bucket', ContentLength=11, ContentType='text/plain', Key=key, Metadata={}, ServerSideEncryption='AES256'), mock.call.copy_object(Bucket='some-bucket', ContentType='text/plain', CopySource={ 'Bucket': 'some-bucket', 'Key': key }, Key=key, Metadata={'cloud-sync': 'fabcab'}, MetadataDirective='REPLACE', ServerSideEncryption='AES256'), mock.call.head_object(Bucket='some-bucket', Key=key), mock.call.list_objects( Bucket='some-bucket', MaxKeys=1, Prefix=u'heehee/hawhaw/'), mock.call.delete_object(Bucket='some-bucket', Key=key), ])