def test_return_none_if_object_found(self):
with self.app.app_context():
bit_store = BitStore('test_pub', 'test_package')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
read_me_key = bit_store.build_s3_key('test.md')
s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='')
self.assertEqual(bit_store.get_s3_object(read_me_key + "testing"),
None)
self.assertEqual(bit_store.get_s3_object(None), None)
def finalize_metadata(publisher, package):
"""
DPR metadata finalize operation.
This API is responsible for getting data from S3 and push it to RDS.
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
responses:
200:
description: Data transfer complete
400:
description: JWT is invalid
401:
description: Invalid Header for JWT
403:
description: User name and publisher not matched
404:
description: User not found
500:
description: Internal Server Error
"""
try:
user = _request_ctx_stack.top.current_user
user_id = user['user']
user = User.query.filter_by(id=user_id).first()
if user is not None:
if user.name == publisher:
bit_store = BitStore(publisher, package)
body = bit_store.get_metadata_body()
if body is not None:
readme = bit_store.get_s3_object(
bit_store.get_readme_object_key())
MetaDataDB.create_or_update(name=package,
publisher_name=publisher,
descriptor=body,
readme=readme)
return jsonify({"status": "OK"}), 200
raise Exception("Failed to get data from s3")
return handle_error('NOT_PERMITTED',
'user name and publisher not matched', 403)
return handle_error('USER_NOT_FOUND', 'user not found', 404)
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def get_resource(publisher, package, resource):
"""
DPR resource get operation.
This API is responsible for getting resource from S3.
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name - to retrieve the data package metadata
- in: path
name: resource
type: string
required: true
description: resource index or name
responses:
200:
description: Get Data package for one key
schema:
id: get_data_package
properties:
data:
type: string
description: The resource
500:
description: Internal Server Error
"""
try:
path = request.path
metadata = BitStore(publisher, package)
if path.endswith('csv'):
resource_key = metadata.build_s3_key(resource + '.csv')
data = metadata.get_s3_object(resource_key)
def generate():
for row in data.splitlines():
yield row + '\n'
return Response(generate()), 200
else:
resource_key = metadata.build_s3_key(resource + '.csv')
data = metadata.get_s3_object(resource_key)
data = csv.DictReader(data.splitlines())
# taking first and adding at the end to avoid last comma
first_row = next(data)
def generate():
yield '['
for row in data:
yield json.dumps(row) + ','
yield json.dumps(first_row) + ']'
return Response(generate(), content_type='application/json'), 200
except Exception as e:
return handle_error('GENERIC_ERROR', e.message, 500)