def test_no_verify_context(self): conn = create_connection(host='s3.store.com', s3_ssl_no_verify=True, auth_headers=self.auth_headers) assert 'context' in conn.http_connection_kwargs conn = create_connection(host='s3.store.com', auth_headers=self.auth_headers) assert 'context' not in conn.http_connection_kwargs
def bucket(self): if not self._bucket: bucket = app.config['UPLOAD_BUCKET'] conn_kwargs = app.config.get('S3_UPLOAD', {}) conn = create_connection(**conn_kwargs) self._bucket = conn.get_bucket(bucket, validate=False) return self._bucket
def s3_upload_file(s3_bucket, source_file, target_file_name, headers, upload_root_dir, directory="", return_key_only=False, conn_name=DEFAULT_CONN): """ Upload a file-type object to S3 :param s3_bucket: AWS S3 bucket name :param source_file_name: name in local file system of the file to upload :param target_file_name: file name as should appear in S3 :param headers: a dictionary of headers to set on the S3 object :param directory: path in S3 where the object needs to be stored :param return_key_only: return key name instead of full url """ filename = secure_filename(target_file_name) upload_key = form_upload_directory(directory, filename, upload_root_dir) conn_kwargs = app.config.get(conn_name, {}) conn = create_connection(**conn_kwargs) bucket = conn.get_bucket(s3_bucket, validate=False) assert (len(upload_key) < 256) key = bucket.new_key(upload_key) key.set_contents_from_file(source_file, headers=headers, policy='bucket-owner-full-control') if return_key_only: return key.name url = key.generate_url(0, query_auth=False) return url.split('?')[0]
def get_s3_bucket_key(s3_bucket, s3_url, conn_name=DEFAULT_CONN): conn_kwargs = app.config.get(conn_name, {}) conn = create_connection(**conn_kwargs) bucket = conn.get_bucket(s3_bucket, validate=False) obj = urlparse(s3_url) path = obj.path key = bucket.get_key(path, validate=False) return bucket, key
def test_custom_headers(self): header = 'x-custom-access-key' host = 's3.store.com' access_key = 'test-access-key' conn = create_connection(host=host, aws_access_key_id=access_key, auth_headers=[(header, 'access_key')]) http = conn.build_base_http_request('GET', '/', None) http.authorize(conn) assert header in http.headers assert http.headers[header] == access_key
def encrypted_file(store, bucket, project_id, path): """Proxy encrypted task file in a cloud storage""" current_app.logger.info('Project id {} decrypt file. {}'.format( project_id, path)) conn_args = current_app.config.get('S3_TASK_REQUEST', {}) signature = request.args.get('task-signature') if not signature: current_app.logger.exception('Project id {} no signature {}'.format( project_id, path)) raise Forbidden('No signature') project = get_project_data(project_id) timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL) payload = signer.loads(signature, max_age=timeout) task_id = payload['task_id'] check_allowed(current_user.id, task_id, project, request.path) ## download file try: key = '/{}/{}'.format(project_id, path) conn = create_connection(**conn_args) _bucket = conn.get_bucket(bucket, validate=False) _key = _bucket.get_key(key, validate=False) content = _key.get_contents_as_string() except S3ResponseError as e: current_app.logger.exception( 'Project id {} get task file {} {}'.format(project_id, path, e)) if e.error_code == 'NoSuchKey': raise NotFound('File Does Not Exist') else: raise InternalServerError('An Error Occurred') ## decyrpt file secret = current_app.config.get('FILE_ENCRYPTION_KEY') cipher = AESWithGCM(secret) decrypted = cipher.decrypt(content) response = Response(decrypted, content_type=_key.content_type) response.headers.add('Content-Encoding', _key.content_encoding) response.headers.add('Content-Disposition', _key.content_disposition) return response
def s3_get_file_contents(s3_bucket, s3_path, headers=None, encoding='utf-8', conn=''): """Get the conents of a file from S3. :param s3_bucket: AWS S3 bucket :param s3_path: Path to an S3 object :param headers: Additional headers to send in the request to S3 :param encoding: The text encoding to use :return: File contents as a string with the specified encoding """ conn = create_connection(**current_app.config.get(conn, {})) bucket = conn.get_bucket(s3_bucket, validate=False) key = bucket.get_key(s3_path) return key.get_contents_as_string(headers=headers, encoding=encoding)
def test_proxied_connection(self, make_request): params = { 'host': 's3.test.com', 'object_service': 'tests3', 'client_secret': 'abcd', 'client_id': 'test_id', 'auth_headers': [('test', 'object-service')] } conn = create_connection(**params) conn.make_request('GET', 'test_bucket', 'test_key') make_request.assert_called() args, kwargs = make_request.call_args headers = args[3] assert headers['x-objectservice-id'] == 'TESTS3' jwt_payload = jwt.decode(headers['jwt'], 'abcd', algorithm='HS256') assert jwt_payload['path'] == '/test_bucket/test_key' bucket = conn.get_bucket('test_bucket', validate=False) key = bucket.get_key('test_key', validate=False) assert key.generate_url(0).split( '?')[0] == 'https://s3.test.com/test_bucket/test_key'
def test_path_key(self): conn = create_connection(host='s3.store.com', host_suffix='/test', auth_headers=self.auth_headers) path = conn.get_path(path='/bucket/key') assert path == '/test/bucket/key', path