def test_get_policy_no_statement(self):
'''
Test UploadCredentials _get_policy if statements are bad
'''
upload_creds = UploadCredentials(self._test_bucket, self._test_key,
self._test_name)
for item in [
{
'NotStatement': ['a']
},
{
'Statement': {}
},
{
'Statement': []
},
{
'Statement': None
},
{
'Statement': 'a'
},
]:
upload_creds._external_policy = item
self.assertDictEqual(
upload_creds._get_base_policy(),
upload_creds._get_policy(),
)
def test_check_external_policy_one(self):
'''
Test UploadCredentials _check_external_policy one statement in file
'''
from os import remove
expected_bucket_names = ['test-bucket-one']
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
self._test_profile_name,
)
for s3_transfer_allow, s3_transfer_buckets in [
(True, CREDS_DATA_PATH + '/' + 'external_bucket_list_one'),
]:
upload_creds._check_external_policy(
s3_transfer_allow=s3_transfer_allow,
s3_transfer_buckets=s3_transfer_buckets)
remove(s3_transfer_buckets + '.json')
expected_resources = {
's3:GetObject':
['arn:aws:s3:::%s/*' % item for item in expected_bucket_names],
's3:GetObjectAcl':
['arn:aws:s3:::%s/*' % item for item in expected_bucket_names],
}
result_statements = upload_creds._external_policy['Statement']
self.assertEqual(len(result_statements),
len(EXTERNAL_BUCKET_STATEMENTS))
for result_statement in result_statements:
resources = result_statement['Resource']
self.assertTrue(isinstance(resources, list))
self.assertEqual(len(resources), 1)
self.assertListEqual(
expected_resources[result_statement['Action']],
result_statement['Resource'])
def create(cls, registry, uuid, properties, sheets=None):
if properties.get('status') == 'uploading':
sheets = {} if sheets is None else sheets.copy()
bucket = registry.settings['file_upload_bucket']
mapping = cls.schema['file_format_file_extension']
file_extension = mapping[properties['file_format']]
date = properties['date_created'].split('T')[0].replace('-', '/')
accession_or_external = properties.get(
'accession') or properties['external_accession']
key = '{date}/{uuid}/{accession_or_external}{file_extension}'.format(
accession_or_external=accession_or_external,
date=date,
file_extension=file_extension,
uuid=uuid,
**properties)
name = 'up{time:.6f}-{accession_or_external}'.format(
accession_or_external=accession_or_external,
time=time.time(),
**properties)[:32] # max 32 chars
profile_name = registry.settings.get('file_upload_profile_name')
upload_creds = UploadCredentials(bucket,
key,
name,
profile_name=profile_name)
s3_transfer_allow = registry.settings.get(
'external_aws_s3_transfer_allow', 'false')
sheets['external'] = upload_creds.external_creds(
s3_transfer_allow=asbool(s3_transfer_allow),
s3_transfer_buckets=registry.settings.get(
'external_aws_s3_transfer_buckets'),
)
return super(File, cls).create(registry, uuid, properties, sheets)
def post_upload(context, request):
properties = context.upgrade_properties()
if properties['status'] not in ('uploading', 'upload failed'):
raise HTTPForbidden('status must be "uploading" to issue new credentials')
accession_or_external = properties.get('accession') or properties['external_accession']
file_upload_bucket = request.registry.settings['file_upload_bucket']
external = context.propsheets.get('external', None)
registry = request.registry
if external is None:
# Handle objects initially posted as another state.
bucket = file_upload_bucket
uuid = context.uuid
mapping = context.schema['file_format_file_extension']
file_extension = mapping[properties['file_format']]
date = properties['date_created'].split('T')[0].replace('-', '/')
key = '{date}/{uuid}/{accession_or_external}{file_extension}'.format(
accession_or_external=accession_or_external,
date=date, file_extension=file_extension, uuid=uuid, **properties)
elif external.get('service') == 's3':
bucket = external['bucket']
# Must reset file to point to file_upload_bucket (keep AWS public dataset in sync).
if bucket != file_upload_bucket:
registry.notify(BeforeModified(context, request))
context._set_external_sheet({'bucket': file_upload_bucket})
registry.notify(AfterModified(context, request))
bucket = file_upload_bucket
key = external['key']
else:
raise HTTPNotFound(
detail='External service {} not expected'.format(external.get('service'))
)
name = 'up{time:.6f}-{accession_or_external}'.format(
accession_or_external=accession_or_external,
time=time.time(), **properties)[:32] # max 32 chars
profile_name = request.registry.settings.get('file_upload_profile_name')
upload_creds = UploadCredentials(bucket, key, name, profile_name=profile_name)
s3_transfer_allow = request.registry.settings.get('external_aws_s3_transfer_allow', 'false')
creds = upload_creds.external_creds(
s3_transfer_allow=asbool(s3_transfer_allow),
s3_transfer_buckets=request.registry.settings.get('external_aws_s3_transfer_buckets'),
)
new_properties = None
if properties['status'] == 'upload failed':
new_properties = properties.copy()
new_properties['status'] = 'uploading'
registry.notify(BeforeModified(context, request))
context.update(new_properties, {'external': creds})
registry.notify(AfterModified(context, request))
rendered = request.embed('/%s/@@object' % context.uuid, as_user=True)
result = {
'status': 'success',
'@type': ['result'],
'@graph': [rendered],
}
return result
def test_get_policy_no_external(self):
'''
Test UploadCredentials _get_policy if no external statements
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
self._test_profile_name,
)
self.assertDictEqual(upload_creds._get_base_policy(),
upload_creds._get_policy())
def test_get_token_default(self):
'''
Test UploadCredentials _get_token with good profile_name
-Test how this works on travis
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
)
policy = upload_creds._get_policy()
token = upload_creds._get_token(policy)
self.assertTrue(isinstance(token, dict))
def test_get_policy_no_external(self):
'''
Test UploadCredentials _get_policy if no external statements
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
self._test_profile_name,
)
self.assertDictEqual(
upload_creds._get_base_policy(),
upload_creds._get_policy()
)
def test_get_token_default(self):
'''
Test UploadCredentials _get_token with good profile_name
-Test how this works on travis
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
)
policy = upload_creds._get_policy()
token = upload_creds._get_token(policy)
self.assertTrue(isinstance(token, dict))
def test_external_creds():
'''
Original test from file.py for test_external_creds
'''
upload_creds = UploadCredentials(
'mock_bucket',
'mock_object',
'mock_name',
)
creds = upload_creds.external_creds()
assert 'upload_credentials' in creds
assert creds['bucket'] == 'mock_bucket'
assert creds['key'] == 'mock_object'
assert 'mock_name' in creds['upload_credentials']['federated_user_id']
assert creds['service'] == 's3'
def test_external_creds():
'''
Original test from file.py for test_external_creds
'''
upload_creds = UploadCredentials(
'mock_bucket',
'mock_object',
'mock_name',
)
creds = upload_creds.external_creds()
assert 'upload_credentials' in creds
assert creds['bucket'] == 'mock_bucket'
assert creds['key'] == 'mock_object'
assert 'mock_name' in creds['upload_credentials']['federated_user_id']
assert creds['service'] == 's3'
def setUpClass(cls):
cls.mock_sts = mock_sts()
cls.mock_sts.start()
cls._upload_creds_attrs = [
'_bucket',
'_key',
'_name',
'_profile_name',
'_resource_string',
'_upload_url',
'_external_policy',
]
cls._external_creds_keys = [
'service', 'bucket', 'key', 'upload_credentials'
]
cls._credentials_keys = [
'session_token',
'access_key',
'expiration',
'secret_key',
'upload_url',
'federated_user_arn',
'federated_user_id',
'request_id',
]
cls._test_bucket = 'mock_bucket'
cls._test_key = 'mock_key'
cls._test_name = 'mock_name'
cls._test_profile_name = 'mock_profile_name'
cls._default_upload_creds = UploadCredentials(
cls._test_bucket,
cls._test_key,
cls._test_name,
cls._test_profile_name,
)
def test_external_creds_values(self):
'''
Test UploadCredentials external_creds returns correct values
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
)
creds_dict = upload_creds.external_creds()
self.assertEqual(creds_dict['service'], 's3')
self.assertEqual(creds_dict['bucket'], self._test_bucket)
self.assertEqual(creds_dict['key'], self._test_key)
upload_credentials = creds_dict['upload_credentials']
self.assertEqual(upload_credentials['upload_url'],
upload_creds._upload_url)
def test_external_creds_keys(self):
'''
Test UploadCredentials external_creds returns correct keys
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
)
creds_dict = upload_creds.external_creds()
self.assertListEqual(sorted(self._external_creds_keys),
sorted(list(creds_dict.keys())))
upload_credentials = creds_dict['upload_credentials']
self.assertTrue(isinstance(upload_credentials, dict))
self.assertListEqual(sorted(self._credentials_keys),
sorted(list(upload_credentials.keys())))
def test_init_profile_name_none(self):
'''
Test UploadCredentials profile_name defaults to None
'''
upload_creds = UploadCredentials(self._test_bucket, self._test_key,
self._test_name)
self.assertIsNone(upload_creds._profile_name)
def test_init_external_policy_empty(self):
'''
Test UploadCredentials profile_name defaults to empty dict
'''
upload_creds = UploadCredentials(self._test_bucket, self._test_key,
self._test_name)
self.assertTrue(isinstance(upload_creds._external_policy, dict))
self.assertFalse(upload_creds._external_policy)
def test_get_policy_ext_statements(self):
'''
Test UploadCredentials _get_policy if 1 or more good statements
'''
upload_creds = UploadCredentials(self._test_bucket, self._test_key, self._test_name)
for policy in [
{'Statement': ['a']},
{'Statement': ['a', 'b', 'c']},
]:
upload_creds._external_policy = policy
base = upload_creds._get_base_policy()
for statement in policy['Statement']:
base['Statement'].append(statement)
self.assertDictEqual(
base,
upload_creds._get_policy(),
)
def test_get_policy_no_statement(self):
'''
Test UploadCredentials _get_policy if statements are bad
'''
upload_creds = UploadCredentials(self._test_bucket, self._test_key, self._test_name)
for item in [
{'NotStatement': ['a']},
{'Statement': {}},
{'Statement': []},
{'Statement': None},
{'Statement': 'a'},
]:
upload_creds._external_policy = item
self.assertDictEqual(
upload_creds._get_base_policy(),
upload_creds._get_policy(),
)
def test_external_creds_values(self):
'''
Test UploadCredentials external_creds returns correct values
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
)
creds_dict = upload_creds.external_creds()
self.assertEqual(creds_dict['service'], 's3')
self.assertEqual(creds_dict['bucket'], self._test_bucket)
self.assertEqual(creds_dict['key'], self._test_key)
upload_credentials = creds_dict['upload_credentials']
self.assertEqual(
upload_credentials['upload_url'],
upload_creds._upload_url
)
def test_check_external_policy_many(self):
'''
Test UploadCredentials _check_external_policy many statements in file
'''
import os
expected_bucket_names = [
'test-bucket-one',
'test-bucket-two',
'test-bucket-three',
]
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
self._test_profile_name,
)
for s3_transfer_allow, s3_transfer_buckets in [
(True, CREDS_DATA_PATH + '/' + 'external_bucket_list_many'),
]:
upload_creds._check_external_policy(
s3_transfer_allow=s3_transfer_allow,
s3_transfer_buckets=s3_transfer_buckets
)
os.remove(s3_transfer_buckets + '.json')
expected_resources = {
's3:GetObject': [
'arn:aws:s3:::%s/*' % item for item in expected_bucket_names
],
's3:GetObjectAcl': [
'arn:aws:s3:::%s/*' % item for item in expected_bucket_names
],
}
result_statements = upload_creds._external_policy['Statement']
self.assertEqual(len(result_statements), len(EXTERNAL_BUCKET_STATEMENTS))
for result_statement in result_statements:
resources = result_statement['Resource']
self.assertTrue(isinstance(resources, list))
self.assertEqual(len(resources), 3)
self.assertListEqual(
expected_resources[result_statement['Action']],
result_statement['Resource']
)
def test_external_creds_keys(self):
'''
Test UploadCredentials external_creds returns correct keys
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
)
creds_dict = upload_creds.external_creds()
self.assertListEqual(
sorted(self._external_creds_keys),
sorted(list(creds_dict.keys()))
)
upload_credentials = creds_dict['upload_credentials']
self.assertTrue(isinstance(upload_credentials, dict))
self.assertListEqual(
sorted(self._credentials_keys),
sorted(list(upload_credentials.keys()))
)
def test_check_external_policy_zero(self):
'''
Test UploadCredentials _check_external_policy no statements in file
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
self._test_profile_name,
)
for s3_transfer_allow, s3_transfer_buckets in [
(False, None),
(True, None),
(False, 'some-path'),
(True, 'bad-path'),
(True, CREDS_DATA_PATH + '/' + 'external_bucket_list_empty'),
]:
upload_creds._check_external_policy(
s3_transfer_allow=s3_transfer_allow,
s3_transfer_buckets=s3_transfer_buckets)
self.assertDictEqual(upload_creds._external_policy, {})
def test_get_policy_ext_statements(self):
'''
Test UploadCredentials _get_policy if 1 or more good statements
'''
upload_creds = UploadCredentials(self._test_bucket, self._test_key,
self._test_name)
for policy in [
{
'Statement': ['a']
},
{
'Statement': ['a', 'b', 'c']
},
]:
upload_creds._external_policy = policy
base = upload_creds._get_base_policy()
for statement in policy['Statement']:
base['Statement'].append(statement)
self.assertDictEqual(
base,
upload_creds._get_policy(),
)
def test_check_external_policy_zero(self):
'''
Test UploadCredentials _check_external_policy no statements in file
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
self._test_profile_name,
)
for s3_transfer_allow, s3_transfer_buckets in [
(False, None),
(True, None),
(False, 'some-path'),
(True, 'bad-path'),
(True, CREDS_DATA_PATH + '/' + 'external_bucket_list_empty'),
]:
upload_creds._check_external_policy(
s3_transfer_allow=s3_transfer_allow,
s3_transfer_buckets=s3_transfer_buckets
)
self.assertDictEqual(upload_creds._external_policy, {})