def test_keystone_requires_swift_proto(self, mock_validate):
exit_arg = main([
'--protocol',
's3',
'--endpoint',
'https://s3.amazonaws.com',
'--username',
'access id',
'--password',
'secret key',
"--auth-type",
"keystone_v3",
"--project-name",
"test",
"--user-domain-name",
"default",
"--project-domain-name",
"default",
])
self.assertIn('Keystone auth requires swift protocol', exit_arg)
exit_arg = main([
'--protocol',
's3',
'--endpoint',
'https://s3.amazonaws.com',
'--username',
'access id',
'--password',
'secret key',
"--auth-type",
"keystone_v2",
"--tenant-name",
"test",
])
self.assertIn('Keystone auth requires swift protocol', exit_arg)
def test_google_no_bucket(self, mock_get_client):
mock_client = \
mock_get_client.return_value.__enter__.return_value
mock_client.list_buckets.return_value = {
'Buckets': [{
'CreationDate': datetime.datetime.now(),
'Name': 'google-bucket'
}],
'ResponseMetadata': {
'HTTPStatusCode': 200,
'HTTPHeaders': {}
}
}
exit_arg = main([
'--protocol',
's3',
'--endpoint',
'https://storage.googleapis.com',
'--username',
'access id',
'--password',
'secret key',
])
self.assertEqual(exit_arg, 0)
self.assertEqual(mock_client.mock_calls, [
mock.call.list_buckets(),
])
def test_swift_one_bucket(self, mock_validate):
exit_arg = main([
'--protocol',
'swift',
'--endpoint',
'https://saio:8080/auth/v1.0',
'--username',
'access id',
'--password',
'secret key',
'--storage-policy',
'better',
'--bucket',
'some-bucket',
])
self.assertIs(exit_arg, mock_validate.return_value)
self.assertEqual([mock.ANY], mock_validate.mock_calls)
provider, swift_key, create_bucket, storage_policy =\
mock_validate.mock_calls[0][1]
self.assertEqual(
{
k: v
for k, v in provider.settings.items()
if k.startswith('aws_') or k in ('protocol', )
}, {
'protocol': 'swift',
'aws_endpoint': 'https://saio:8080/auth/v1.0',
'aws_identity': 'access id',
'aws_secret': 'secret key',
'aws_bucket': 'some-bucket',
})
self.assertEqual(swift_key, 'cloud_sync_test_object')
self.assertFalse(create_bucket)
self.assertEqual(storage_policy, 'better')
def test_aws_with_prefix(self, mock_validate):
exit_arg = main([
'--protocol',
's3',
'--endpoint',
'https://s3.amazonaws.com',
'--username',
'access id',
'--password',
'secret key',
'--bucket',
'some-bucket',
'--prefix',
'jojo/hoho/',
])
self.assertIs(exit_arg, mock_validate.return_value)
self.assertEqual([mock.ANY], mock_validate.mock_calls)
provider, swift_key, create_bucket, _storage_policy =\
mock_validate.mock_calls[0][1]
self.assertEqual(
{
'protocol': 's3',
'aws_endpoint': None,
'aws_identity': 'access id',
'aws_secret': 'secret key',
'aws_bucket': 'some-bucket',
'account': 'verify-auth',
'custom_prefix': 'jojo/hoho/',
'remote_account': None,
},
{k: v
for k, v in provider.settings.items() if k != 'container'})
self.assertEqual(swift_key, 'cloud_sync_test')
self.assertFalse(create_bucket)
def test_head_object_error(self, mock_provider_factory):
mock_provider = mock_provider_factory.return_value
args = [
'--protocol', 's3', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://s3.amazonaws.com',
'--bucket', 'some-bucket'
]
mock_provider.list_objects.return_value = ProviderResponse(
True, 200, {}, [{
'name': 'foo'
}])
mock_provider.head_object.return_value = ProviderResponse(
False, 500, {}, '')
exit_arg = main(args)
self.assertTrue(
exit_arg.endswith(
mock_provider.head_object.return_value.wsgi_status))
mock_provider.list_objects.assert_called_once_with(None,
1,
None,
bucket=None)
mock_provider.head_object.assert_called_once_with('foo', bucket=None)
mock_provider.get_object.assert_not_called()
def test_read_only_with_bucket_s3(self, mock_provider_factory):
mock_provider = mock_provider_factory.return_value
args = [
'--protocol', 's3', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://s3.amazonaws.com',
'--bucket', 'some-bucket'
]
body = self.TrackingStringIO('response')
mock_provider.list_objects.return_value = ProviderResponse(
True, 200, {}, [{
'name': 'foo'
}])
mock_provider.get_object.return_value = ProviderResponse(
True, 200, {}, body)
mock_provider.head_object.return_value = ProviderResponse(
True, 204, {}, '')
exit_arg = main(args)
self.assertEqual(0, exit_arg)
mock_provider.list_objects.assert_called_once_with(None,
1,
None,
bucket=None)
mock_provider.head_object.assert_called_once_with('foo', bucket=None)
mock_provider.get_object.assert_called_once_with('foo', bucket=None)
self.assertEqual(0, body.last_pos)
self.assertTrue(body.closed)
def test_google_leaves_endpoint_alone(self, mock_validate):
exit_arg = main([
'--protocol',
's3',
'--endpoint',
'https://storage.googleapis.com',
'--username',
'access id',
'--password',
'secret key',
'--bucket',
'some-bucket',
])
self.assertIs(exit_arg, mock_validate.return_value)
self.assertEqual([mock.ANY], mock_validate.mock_calls)
provider, swift_key, create_bucket = mock_validate.mock_calls[0][1]
self.assertEqual(
{
k: v
for k, v in provider.settings.items()
if k.startswith('aws_') or k in ('protocol', )
}, {
'protocol': 's3',
'aws_endpoint': 'https://storage.googleapis.com',
'aws_identity': 'access id',
'aws_secret': 'secret key',
'aws_bucket': 'some-bucket',
})
self.assertEqual(swift_key, 'fabcab/cloud_sync_test')
self.assertFalse(create_bucket)
def test_swift_all_buckets(self, mock_validate):
exit_arg = main([
'--protocol',
'swift',
'--endpoint',
'https://saio:8080/auth/v1.0',
'--username',
'access id',
'--password',
'secret key',
'--bucket',
'/*',
])
self.assertIs(exit_arg, mock_validate.return_value)
self.assertEqual([mock.ANY], mock_validate.mock_calls)
provider, swift_key, create_bucket = mock_validate.mock_calls[0][1]
self.assertEqual(
{
k: v
for k, v in provider.settings.items()
if k.startswith('aws_') or k in ('protocol', )
}, {
'protocol': 'swift',
'aws_endpoint': 'https://saio:8080/auth/v1.0',
'aws_identity': 'access id',
'aws_secret': 'secret key',
'aws_bucket': u'.cloudsync_test_container-\U0001f44d',
})
self.assertEqual(swift_key, 'cloud_sync_test_object')
self.assertTrue(create_bucket)
def test_read_only_all_buckets(self, mock_provider_factory):
args = [
'--protocol', 'swift', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://saio:8080/auth/v1.0',
'--bucket', '/*'
]
body = self.TrackingStringIO('response')
mock_provider = mock_provider_factory.return_value
mock_provider.list_buckets.return_value = ProviderResponse(
True, 200, {}, [{
'name': 'container'
}])
mock_provider.list_objects.return_value = ProviderResponse(
True, 200, {}, [{
'name': 'foo'
}])
mock_provider.get_object.return_value = ProviderResponse(
True, 200, {}, body)
mock_provider.head_object.return_value = ProviderResponse(
True, 204, {}, '')
exit_arg = main(args)
self.assertEqual(0, exit_arg)
mock_provider.list_buckets.assert_called_once_with(limit=1)
mock_provider.list_objects.assert_called_once_with(None,
1,
None,
bucket='container')
mock_provider.head_object.assert_called_once_with('foo',
bucket='container')
mock_provider.get_object.assert_called_once_with('foo',
bucket='container')
self.assertEqual(0, body.last_pos)
self.assertTrue(body.closed)
def test_swift_with_bucket(self, mock_get_client):
def _fake_put_object(*args, **kwargs):
kwargs['response_dict']['headers'] = {}
kwargs['response_dict']['status'] = 201
return 'deadbeef'
mock_client = \
mock_get_client.return_value.__enter__.return_value
mock_client.put_object.side_effect = _fake_put_object
mock_client.head_object.side_effect = [
{
'x-object-meta-cloud-sync': 'fabcab'
},
{
'x-object-meta-cloud-sync': 'fabcab'
}, # One extra for the DELETE
]
mock_client.get_container.return_value = ({}, [])
mock_client.post_object.return_value = None
mock_client.delete_object.return_value = None
exit_arg = main([
'--protocol',
'swift',
'--endpoint',
'https://saio:8080/auth/v1.0',
'--username',
'access id',
'--password',
'secret key',
'--bucket',
'some-bucket',
])
self.assertEqual(exit_arg, 0)
self.assertEqual(mock_client.mock_calls, [
mock.call.put_object('some-bucket',
'cloud_sync_test_object',
contents='1space-test',
headers={'content-type': 'text/plain'},
response_dict=mock.ANY),
mock.call.post_object('some-bucket',
'cloud_sync_test_object',
headers={
'content-type': 'text/plain',
'x-object-meta-cloud-sync': 'fabcab'
}),
mock.call.head_object(
'some-bucket', 'cloud_sync_test_object', headers={}),
mock.call.get_container('some-bucket',
delimiter='',
limit=1,
marker='',
prefix='',
headers={}),
mock.call.head_object(
'some-bucket', 'cloud_sync_test_object', headers={}),
mock.call.delete_object(
'some-bucket', 'cloud_sync_test_object', headers={}),
])
def test_swift_with_bucket(self, mock_get_client):
mock_client = \
mock_get_client.return_value.__enter__.return_value
mock_client.head_object.side_effect = [
None,
{
'x-object-meta-cloud-sync': 'fabcab'
},
{
'x-object-meta-cloud-sync': 'fabcab'
}, # One extra for the DELETE
]
mock_client.get_container.return_value = ({}, [])
mock_client.post_object.return_value = None
mock_client.delete_object.return_value = None
exit_arg = main([
'--protocol',
'swift',
'--endpoint',
'https://saio:8080/auth/v1.0',
'--username',
'access id',
'--password',
'secret key',
'--bucket',
'some-bucket',
])
self.assertEqual(exit_arg, 0)
self.assertEqual(mock_client.mock_calls, [
mock.call.head_object(
'some-bucket', 'cloud_sync_test_object', headers={}),
mock.call.put_object('some-bucket',
'cloud_sync_test_object',
mock.ANY,
content_length=15,
etag=mock.ANY,
headers={'content-type': 'text/plain'}),
mock.call.post_object('some-bucket',
'cloud_sync_test_object',
headers={
'content-type': 'text/plain',
'X-Object-Meta-Cloud-Sync': 'fabcab'
}),
mock.call.head_object(
'some-bucket', 'cloud_sync_test_object', headers={}),
mock.call.get_container('some-bucket',
delimiter='',
limit=1,
marker='',
prefix='',
headers={}),
mock.call.head_object(
'some-bucket', 'cloud_sync_test_object', headers={}),
mock.call.delete_object(
'some-bucket', 'cloud_sync_test_object', headers={}),
])
def _stderr_of_sysexit(self, args):
with self.assertRaises(SystemExit), \
mock.patch('sys.stderr', new_callable=BytesIO) as err:
got = main(args)
# This little adapter lets us also test error conditions where
# the message is returned, for the surrounding "exit(...)" to
# turn into a SystemExit.
if got:
print >> sys.stderr, got
raise SystemExit()
return err.getvalue()
def test_read_only_no_buckets(self, mock_provider_factory):
mock_provider = mock_provider_factory.return_value
args = [
'--protocol', 's3', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://s3.amazonaws.com',
'--bucket', '/*'
]
mock_provider = mock_provider_factory.return_value
mock_provider.list_buckets.return_value = ProviderResponse(
True, 200, {}, [])
self.assertEqual('No buckets/containers found', main(args))
mock_provider.list_buckets.assert_called_once_with(limit=1)
def test_list_buckets_error(self, mock_provider_factory):
mock_provider = mock_provider_factory.return_value
args = [
'--protocol', 's3', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://s3.amazonaws.com',
'--bucket', '/*'
]
mock_provider = mock_provider_factory.return_value
mock_provider.list_buckets.return_value = ProviderResponse(
False, 401, {}, [])
self.assertTrue(
main(args).endswith(
mock_provider.list_buckets.return_value.wsgi_status))
mock_provider.list_buckets.assert_called_once_with(limit=1)
def test_no_bucket_bad_creds(self, mock_provider_factory):
mock_provider = mock_provider_factory.return_value
args = [
'--protocol', 's3', '--username', 'id', '--password', 'key',
'--endpoint', 'https://s3.amazonaws.com'
]
mock_provider.list_buckets.return_value = ProviderResponse(
False, 500, {}, 'error')
exit_arg = main(args)
self.assertTrue(
exit_arg.endswith(
mock_provider.list_buckets.return_value.wsgi_status))
def test_bucket_cant_have_slash(self, mock_validate):
msg = 'Invalid argument: slash is not allowed in container name'
self.assertEqual(
msg,
main([
'--protocol',
's3',
'--endpoint',
'https://s3.amazonaws.com',
'--username',
'access id',
'--password',
'secret key',
'--bucket',
'some/bucket',
]))
self.assertEqual(mock_validate.mock_calls, [])
def test_swift_no_bucket(self, mock_get_client):
exit_arg = main([
'--protocol',
'swift',
'--endpoint',
'https://saio:8080/auth/v1.0',
'--username',
'access id',
'--password',
'secret key',
])
self.assertEqual(exit_arg, 0)
mock_client = \
mock_get_client.return_value.__enter__.return_value
self.assertEqual(mock_client.mock_calls, [
mock.call.get_account(),
])
def test_google_no_bucket(self, mock_get_client):
exit_arg = main([
'--protocol',
's3',
'--endpoint',
'https://storage.googleapis.com',
'--username',
'access id',
'--password',
'secret key',
])
self.assertEqual(exit_arg, 0)
mock_client = \
mock_get_client.return_value.__enter__.return_value
self.assertEqual(mock_client.mock_calls, [
mock.call.list_buckets(),
])
def test_read_only_no_objects(self, mock_provider_factory):
mock_provider = mock_provider_factory.return_value
args = [
'--protocol', 's3', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://s3.amazonaws.com',
'--bucket', 'some-bucket'
]
mock_provider = mock_provider_factory.return_value
mock_provider.list_objects.return_value = ProviderResponse(
True, 200, {}, [])
self.assertEqual(
'There are no objects in the bucket to validate GET/HEAD access',
main(args))
mock_provider.list_objects.assert_called_once_with(None,
1,
None,
bucket=None)
def test_swift_no_bucket_empty_account(self, mock_get_client):
mock_client = \
mock_get_client.return_value.__enter__.return_value
mock_client.get_account.return_value = ({}, [])
exit_arg = main([
'--protocol',
'swift',
'--endpoint',
'https://saio:8080/auth/v1.0',
'--username',
'access id',
'--password',
'secret key',
])
self.assertEqual(exit_arg, 0)
self.assertEqual(
mock_client.mock_calls,
[mock.call.get_account(headers={}, prefix='', limit=1, marker='')])
def test_account_requires_swift(self, mock_validate):
msg = 'Invalid argument: account is only valid with swift protocol'
self.assertEqual(
msg,
main([
'--protocol',
's3',
'--endpoint',
'https://s3.amazonaws.com',
'--username',
'access id',
'--password',
'secret key',
'--account',
'AUTH_account',
'--bucket',
'some-bucket',
]))
self.assertEqual(mock_validate.mock_calls, [])
def test_swift_keystone_v3(self, mock_validate):
exit_arg = main([
'--protocol',
'swift',
'--endpoint',
'http://1space-keystone:5000/v3',
'--username',
'access id',
'--password',
'secret key',
"--auth-type",
"keystone_v3",
"--project-name",
"test",
"--project-domain-name",
"wat-wat",
"--user-domain-name",
'floo-boo',
'--bucket',
'/*',
])
self.assertIs(exit_arg, mock_validate.return_value)
self.assertEqual([mock.ANY], mock_validate.mock_calls)
provider, swift_key, create_bucket, _storage_policy =\
mock_validate.mock_calls[0][1]
self.assertEqual(
{
'protocol': 'swift',
'aws_endpoint': 'http://1space-keystone:5000/v3',
'aws_identity': 'access id',
'aws_secret': 'secret key',
'aws_bucket': u'.cloudsync_test_container-\u00ef',
'account': 'verify-auth',
'container': u'testing-\u00ef',
'custom_prefix': None,
'remote_account': None,
'auth_type': 'keystone_v3',
'user_domain_name': 'floo-boo',
'project_name': 'test',
'project_domain_name': 'wat-wat',
}, provider.settings)
self.assertEqual(swift_key, 'cloud_sync_test_object')
self.assertTrue(create_bucket)
def test_read_only_with_bucket_swift(self, mock_provider_factory):
mock_provider = mock.Mock()
def _fake_create(conf, max_conns=1024):
mock_provider.settings = conf
return mock_provider
mock_provider_factory.side_effect = _fake_create
args = [
'--protocol', 'swift', '--username', 'id', '--password', 'key',
'--read-only', '--endpoint', 'https://some.swift.com/auth/v1.0',
'--bucket', 'some-bucket'
]
body = self.TrackingStringIO('response')
mock_provider.list_objects.return_value = ProviderResponse(
True, 200, {}, [{
'name': 'foo'
}])
mock_provider.get_object.return_value = ProviderResponse(
True, 200, {}, body)
mock_provider.head_object.return_value = ProviderResponse(
True, 204, {}, '')
exit_arg = main(args)
self.assertEqual(0, exit_arg)
mock_provider.list_objects.assert_called_once_with(None,
1,
None,
bucket=None)
mock_provider.head_object.assert_called_once_with('foo', bucket=None)
mock_provider.get_object.assert_called_once_with('foo',
bucket=None,
resp_chunk_size=1)
self.assertEqual(0, body.last_pos)
self.assertTrue(body.closed)
def test_aws_no_bucket_empty_account(self, mock_get_client):
mock_client = \
mock_get_client.return_value.__enter__.return_value
mock_client.list_buckets.return_value = {
'Buckets': [],
'ResponseMetadata': {
'HTTPStatusCode': 200,
'HTTPHeaders': {}
}
}
exit_arg = main([
'--protocol',
's3',
'--endpoint',
'https://s3.amazonaws.com',
'--username',
'access id',
'--password',
'secret key',
])
self.assertEqual(exit_arg, 0)
self.assertEqual(mock_client.mock_calls, [
mock.call.list_buckets(),
])
def do_test(args, missing_arg):
with self.assertRaises(SystemExit), \
mock.patch('sys.stderr', new_callable=BytesIO) as err:
main(args)
self.assertIn('argument %s is required' % missing_arg,
err.getvalue())
def test_google_with_bucket(self, mock_get_client):
mock_client = \
mock_get_client.return_value.__enter__.return_value
mock_client.head_object.side_effect = [
None,
{ # HEAD after PUT
'Body': [''],
'ResponseMetadata': {
'HTTPStatusCode': 200,
'HTTPHeaders': {
'x-amz-meta-cloud-sync': 'fabcab',
},
},
},
]
mock_client.list_objects.return_value = {
'Contents': [],
'ReponseMetadata': {
'HTTPStatusCode': 200,
'HTTPHeaders': {
'x-amz-meta-cloud-sync': 'fabcab',
},
},
}
exit_arg = main([
'--protocol',
's3',
'--endpoint',
'https://storage.googleapis.com',
'--username',
'access id',
'--password',
'secret key',
'--bucket',
'some-bucket',
])
self.assertEqual(exit_arg, 0)
key = u'9f9835/verify-auth/testing-\U0001f44d/fabcab/cloud_sync_test'
self.assert_calls(mock_client, [
mock.call.head_object(Bucket='some-bucket', Key=key),
mock.call.put_object(Body=mock.ANY,
Bucket='some-bucket',
ContentLength=15,
ContentType='text/plain',
Key=key,
Metadata={}),
mock.call.copy_object(Bucket='some-bucket',
ContentType='text/plain',
CopySource={
'Bucket': 'some-bucket',
'Key': key
},
Key=key,
Metadata={'cloud-sync': 'fabcab'},
MetadataDirective='REPLACE'),
mock.call.head_object(Bucket='some-bucket', Key=key),
mock.call.delete_object(Bucket='some-bucket', Key=key),
mock.call.list_objects(
Bucket='some-bucket',
MaxKeys=1,
Prefix=u'9f9835/verify-auth/testing-\U0001f44d/'),
])
def test_aws_with_bucket_and_prefix(self, mock_get_client):
mock_client = \
mock_get_client.return_value.__enter__.return_value
mock_client.put_object.return_value = {
'Body': [''],
'ResponseMetadata': {
'HTTPStatusCode': 201,
'HTTPHeaders': {}
}
}
mock_client.head_object.return_value = {
'Body': [''],
'ResponseMetadata': {
'HTTPStatusCode': 200,
'HTTPHeaders': {
'x-amz-meta-cloud-sync': 'fabcab',
},
},
}
mock_client.copy_object.return_value = {'CopyObjectResult': {}}
mock_client.list_objects.return_value = {
'Contents': [],
'ResponseMetadata': {
'HTTPStatusCode': 200,
'HTTPHeaders': {
'x-amz-meta-cloud-sync': 'fabcab',
},
},
}
mock_client.delete_object.return_value = {
'DeleteMarker': False,
'VersionId': '',
}
exit_arg = main([
'--protocol',
's3',
'--endpoint',
'https://s3.amazonaws.com',
'--username',
'access id',
'--password',
'secret key',
'--bucket',
'some-bucket',
'--prefix',
'heehee/hawhaw/',
])
self.assertEqual(exit_arg, 0)
key = u'heehee/hawhaw/cloud_sync_test'
self.assert_calls(mock_client, [
mock.call.put_object(Body='1space-test',
Bucket='some-bucket',
ContentLength=11,
ContentType='text/plain',
Key=key,
Metadata={},
ServerSideEncryption='AES256'),
mock.call.copy_object(Bucket='some-bucket',
ContentType='text/plain',
CopySource={
'Bucket': 'some-bucket',
'Key': key
},
Key=key,
Metadata={'cloud-sync': 'fabcab'},
MetadataDirective='REPLACE',
ServerSideEncryption='AES256'),
mock.call.head_object(Bucket='some-bucket', Key=key),
mock.call.list_objects(
Bucket='some-bucket', MaxKeys=1, Prefix=u'heehee/hawhaw/'),
mock.call.delete_object(Bucket='some-bucket', Key=key),
])
