def test_no_verify_context(self):
conn = create_connection(host='s3.store.com',
s3_ssl_no_verify=True,
auth_headers=self.auth_headers)
assert 'context' in conn.http_connection_kwargs
conn = create_connection(host='s3.store.com',
auth_headers=self.auth_headers)
assert 'context' not in conn.http_connection_kwargs
def bucket(self):
if not self._bucket:
bucket = app.config['UPLOAD_BUCKET']
conn_kwargs = app.config.get('S3_UPLOAD', {})
conn = create_connection(**conn_kwargs)
self._bucket = conn.get_bucket(bucket, validate=False)
return self._bucket
def s3_upload_file(s3_bucket,
source_file,
target_file_name,
headers,
upload_root_dir,
directory="",
return_key_only=False,
conn_name=DEFAULT_CONN):
"""
Upload a file-type object to S3
:param s3_bucket: AWS S3 bucket name
:param source_file_name: name in local file system of the file to upload
:param target_file_name: file name as should appear in S3
:param headers: a dictionary of headers to set on the S3 object
:param directory: path in S3 where the object needs to be stored
:param return_key_only: return key name instead of full url
"""
filename = secure_filename(target_file_name)
upload_key = form_upload_directory(directory, filename, upload_root_dir)
conn_kwargs = app.config.get(conn_name, {})
conn = create_connection(**conn_kwargs)
bucket = conn.get_bucket(s3_bucket, validate=False)
assert (len(upload_key) < 256)
key = bucket.new_key(upload_key)
key.set_contents_from_file(source_file,
headers=headers,
policy='bucket-owner-full-control')
if return_key_only:
return key.name
url = key.generate_url(0, query_auth=False)
return url.split('?')[0]
def get_s3_bucket_key(s3_bucket, s3_url, conn_name=DEFAULT_CONN):
conn_kwargs = app.config.get(conn_name, {})
conn = create_connection(**conn_kwargs)
bucket = conn.get_bucket(s3_bucket, validate=False)
obj = urlparse(s3_url)
path = obj.path
key = bucket.get_key(path, validate=False)
return bucket, key
def test_custom_headers(self):
header = 'x-custom-access-key'
host = 's3.store.com'
access_key = 'test-access-key'
conn = create_connection(host=host,
aws_access_key_id=access_key,
auth_headers=[(header, 'access_key')])
http = conn.build_base_http_request('GET', '/', None)
http.authorize(conn)
assert header in http.headers
assert http.headers[header] == access_key
def encrypted_file(store, bucket, project_id, path):
"""Proxy encrypted task file in a cloud storage"""
current_app.logger.info('Project id {} decrypt file. {}'.format(
project_id, path))
conn_args = current_app.config.get('S3_TASK_REQUEST', {})
signature = request.args.get('task-signature')
if not signature:
current_app.logger.exception('Project id {} no signature {}'.format(
project_id, path))
raise Forbidden('No signature')
project = get_project_data(project_id)
timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL)
payload = signer.loads(signature, max_age=timeout)
task_id = payload['task_id']
check_allowed(current_user.id, task_id, project, request.path)
## download file
try:
key = '/{}/{}'.format(project_id, path)
conn = create_connection(**conn_args)
_bucket = conn.get_bucket(bucket, validate=False)
_key = _bucket.get_key(key, validate=False)
content = _key.get_contents_as_string()
except S3ResponseError as e:
current_app.logger.exception(
'Project id {} get task file {} {}'.format(project_id, path, e))
if e.error_code == 'NoSuchKey':
raise NotFound('File Does Not Exist')
else:
raise InternalServerError('An Error Occurred')
## decyrpt file
secret = current_app.config.get('FILE_ENCRYPTION_KEY')
cipher = AESWithGCM(secret)
decrypted = cipher.decrypt(content)
response = Response(decrypted, content_type=_key.content_type)
response.headers.add('Content-Encoding', _key.content_encoding)
response.headers.add('Content-Disposition', _key.content_disposition)
return response
def s3_get_file_contents(s3_bucket,
s3_path,
headers=None,
encoding='utf-8',
conn=''):
"""Get the conents of a file from S3.
:param s3_bucket: AWS S3 bucket
:param s3_path: Path to an S3 object
:param headers: Additional headers to send
in the request to S3
:param encoding: The text encoding to use
:return: File contents as a string with the
specified encoding
"""
conn = create_connection(**current_app.config.get(conn, {}))
bucket = conn.get_bucket(s3_bucket, validate=False)
key = bucket.get_key(s3_path)
return key.get_contents_as_string(headers=headers, encoding=encoding)
def test_proxied_connection(self, make_request):
params = {
'host': 's3.test.com',
'object_service': 'tests3',
'client_secret': 'abcd',
'client_id': 'test_id',
'auth_headers': [('test', 'object-service')]
}
conn = create_connection(**params)
conn.make_request('GET', 'test_bucket', 'test_key')
make_request.assert_called()
args, kwargs = make_request.call_args
headers = args[3]
assert headers['x-objectservice-id'] == 'TESTS3'
jwt_payload = jwt.decode(headers['jwt'], 'abcd', algorithm='HS256')
assert jwt_payload['path'] == '/test_bucket/test_key'
bucket = conn.get_bucket('test_bucket', validate=False)
key = bucket.get_key('test_key', validate=False)
assert key.generate_url(0).split(
'?')[0] == 'https://s3.test.com/test_bucket/test_key'
def test_path_key(self):
conn = create_connection(host='s3.store.com',
host_suffix='/test',
auth_headers=self.auth_headers)
path = conn.get_path(path='/bucket/key')
assert path == '/test/bucket/key', path
