def create(cls, registry, uuid, properties, sheets=None):
if properties.get('status') == 'uploading':
sheets = {} if sheets is None else sheets.copy()
bucket = registry.settings['file_upload_bucket']
mapping = cls.schema['file_format_file_extension']
file_extension = mapping[properties['file_format']]
date = properties['date_created'].split('T')[0].replace('-', '/')
accession_or_external = properties.get(
'accession') or properties['external_accession']
key = '{date}/{uuid}/{accession_or_external}{file_extension}'.format(
accession_or_external=accession_or_external,
date=date,
file_extension=file_extension,
uuid=uuid,
**properties)
name = 'up{time:.6f}-{accession_or_external}'.format(
accession_or_external=accession_or_external,
time=time.time(),
**properties)[:32] # max 32 chars
profile_name = registry.settings.get('file_upload_profile_name')
upload_creds = UploadCredentials(bucket,
key,
name,
profile_name=profile_name)
s3_transfer_allow = registry.settings.get(
'external_aws_s3_transfer_allow', 'false')
sheets['external'] = upload_creds.external_creds(
s3_transfer_allow=asbool(s3_transfer_allow),
s3_transfer_buckets=registry.settings.get(
'external_aws_s3_transfer_buckets'),
)
return super(File, cls).create(registry, uuid, properties, sheets)
def post_upload(context, request):
properties = context.upgrade_properties()
if properties['status'] not in ('uploading', 'upload failed'):
raise HTTPForbidden('status must be "uploading" to issue new credentials')
accession_or_external = properties.get('accession') or properties['external_accession']
file_upload_bucket = request.registry.settings['file_upload_bucket']
external = context.propsheets.get('external', None)
registry = request.registry
if external is None:
# Handle objects initially posted as another state.
bucket = file_upload_bucket
uuid = context.uuid
mapping = context.schema['file_format_file_extension']
file_extension = mapping[properties['file_format']]
date = properties['date_created'].split('T')[0].replace('-', '/')
key = '{date}/{uuid}/{accession_or_external}{file_extension}'.format(
accession_or_external=accession_or_external,
date=date, file_extension=file_extension, uuid=uuid, **properties)
elif external.get('service') == 's3':
bucket = external['bucket']
# Must reset file to point to file_upload_bucket (keep AWS public dataset in sync).
if bucket != file_upload_bucket:
registry.notify(BeforeModified(context, request))
context._set_external_sheet({'bucket': file_upload_bucket})
registry.notify(AfterModified(context, request))
bucket = file_upload_bucket
key = external['key']
else:
raise HTTPNotFound(
detail='External service {} not expected'.format(external.get('service'))
)
name = 'up{time:.6f}-{accession_or_external}'.format(
accession_or_external=accession_or_external,
time=time.time(), **properties)[:32] # max 32 chars
profile_name = request.registry.settings.get('file_upload_profile_name')
upload_creds = UploadCredentials(bucket, key, name, profile_name=profile_name)
s3_transfer_allow = request.registry.settings.get('external_aws_s3_transfer_allow', 'false')
creds = upload_creds.external_creds(
s3_transfer_allow=asbool(s3_transfer_allow),
s3_transfer_buckets=request.registry.settings.get('external_aws_s3_transfer_buckets'),
)
new_properties = None
if properties['status'] == 'upload failed':
new_properties = properties.copy()
new_properties['status'] = 'uploading'
registry.notify(BeforeModified(context, request))
context.update(new_properties, {'external': creds})
registry.notify(AfterModified(context, request))
rendered = request.embed('/%s/@@object' % context.uuid, as_user=True)
result = {
'status': 'success',
'@type': ['result'],
'@graph': [rendered],
}
return result
def test_external_creds():
'''
Original test from file.py for test_external_creds
'''
upload_creds = UploadCredentials(
'mock_bucket',
'mock_object',
'mock_name',
)
creds = upload_creds.external_creds()
assert 'upload_credentials' in creds
assert creds['bucket'] == 'mock_bucket'
assert creds['key'] == 'mock_object'
assert 'mock_name' in creds['upload_credentials']['federated_user_id']
assert creds['service'] == 's3'
def test_external_creds():
'''
Original test from file.py for test_external_creds
'''
upload_creds = UploadCredentials(
'mock_bucket',
'mock_object',
'mock_name',
)
creds = upload_creds.external_creds()
assert 'upload_credentials' in creds
assert creds['bucket'] == 'mock_bucket'
assert creds['key'] == 'mock_object'
assert 'mock_name' in creds['upload_credentials']['federated_user_id']
assert creds['service'] == 's3'
def test_external_creds_values(self):
'''
Test UploadCredentials external_creds returns correct values
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
)
creds_dict = upload_creds.external_creds()
self.assertEqual(creds_dict['service'], 's3')
self.assertEqual(creds_dict['bucket'], self._test_bucket)
self.assertEqual(creds_dict['key'], self._test_key)
upload_credentials = creds_dict['upload_credentials']
self.assertEqual(upload_credentials['upload_url'],
upload_creds._upload_url)
def test_external_creds_keys(self):
'''
Test UploadCredentials external_creds returns correct keys
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
)
creds_dict = upload_creds.external_creds()
self.assertListEqual(sorted(self._external_creds_keys),
sorted(list(creds_dict.keys())))
upload_credentials = creds_dict['upload_credentials']
self.assertTrue(isinstance(upload_credentials, dict))
self.assertListEqual(sorted(self._credentials_keys),
sorted(list(upload_credentials.keys())))
def test_external_creds_values(self):
'''
Test UploadCredentials external_creds returns correct values
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
)
creds_dict = upload_creds.external_creds()
self.assertEqual(creds_dict['service'], 's3')
self.assertEqual(creds_dict['bucket'], self._test_bucket)
self.assertEqual(creds_dict['key'], self._test_key)
upload_credentials = creds_dict['upload_credentials']
self.assertEqual(
upload_credentials['upload_url'],
upload_creds._upload_url
)
def test_external_creds_keys(self):
'''
Test UploadCredentials external_creds returns correct keys
'''
upload_creds = UploadCredentials(
self._test_bucket,
self._test_key,
self._test_name,
)
creds_dict = upload_creds.external_creds()
self.assertListEqual(
sorted(self._external_creds_keys),
sorted(list(creds_dict.keys()))
)
upload_credentials = creds_dict['upload_credentials']
self.assertTrue(isinstance(upload_credentials, dict))
self.assertListEqual(
sorted(self._credentials_keys),
sorted(list(upload_credentials.keys()))
)