def wrapped(*args, **kwargs):
entity_str, action_str = action.split("::")
user_id, instance = None, None
jwt_status, user_info = get_user_from_jwt(request,
app.config['API_KEY'])
if jwt_status:
user_id = user_info['user']
if entity_str == 'Package':
publisher_name, package_name = kwargs['publisher'], kwargs[
'package']
instance = Package.get_package(publisher_name, package_name)
elif entity_str == 'Publisher':
publisher_name = kwargs['publisher']
instance = Publisher.query.filter_by(name=publisher_name).one()
else:
return handle_error(
"INVALID_ENTITY",
"{e} is not a valid one".format(e=entity_str), 401)
status = ia(user_id, instance, action)
if not status:
return handle_error("NOT_ALLOWED",
"The operation is not allowed", 403)
return f(*args, **kwargs)
def get_user_from_jwt(req, api_key):
jwt_helper = JWT(api_key)
auth = req.headers.get('Authorization', None)
if not auth:
return False, handle_error('authorization_header_missing',
'Authorization header is expected', 401)
parts = auth.split()
if parts[0].lower() != 'bearer':
return False, handle_error('invalid_header',
'Authorization header must start with Bearer',
401)
elif len(parts) == 1:
return False, handle_error('invalid_header', 'Token not found', 401)
elif len(parts) > 2:
return False, handle_error(
'invalid_header',
'Authorization header must\ be Bearer + \s + token',
401)
token = parts[1]
try:
return True, jwt_helper.decode(token)
except Exception as e:
return False, handle_error('jwt_error', e.message, 400)
def decorated(*args, **kwargs):
jwt_helper = JWTHelper(app.config['API_KEY'])
auth = request.headers.get('Authorization', None)
if not auth:
return handle_error('authorization_header_missing',
'Authorization header is expected', 401)
parts = auth.split()
if parts[0].lower() != 'bearer':
return handle_error('invalid_header',
'Authorization header must start with Bearer',
401)
elif len(parts) == 1:
return handle_error('invalid_header', 'Token not found', 401)
elif len(parts) > 2:
return handle_error(
'invalid_header',
'Authorization header must\ be Bearer + \s + token',
401)
token = parts[1]
try:
payload = jwt_helper.decode(token)
except Exception as e:
return handle_error('jwt_error', e.message, 400)
_request_ctx_stack.top.current_user = user = payload
return f(*args, **kwargs)
def tag_data_package(publisher, package):
"""
DPR metadata put operation.
This API is responsible for tagging data package
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
- in: body
name: version
type: string
required: true
description: version value
responses:
400:
description: JWT is invalid or req body is not valid
401:
description: Invalid Header for JWT
403:
description: User not allowed for operation
404:
description: User not found
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: put_package_success
properties:
status:
type: string
description: Status of the operation
default: OK
"""
try:
data = request.get_json()
if 'version' not in data:
return handle_error('ATTRIBUTE_MISSING', 'version not found', 400)
bitstore = BitStore(publisher, package)
status_db = MetaDataDB.create_or_update_version(publisher, package, data['version'])
status_bitstore = bitstore.copy_to_new_version(data['version'])
if status_db is False or status_bitstore is False:
raise Exception("failed to tag data package")
return jsonify({"status": "OK"}), 200
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def get_s3_signed_url():
"""
This API is responsible for generate signed url to post data to S3
---
tags:
- auth
parameters:
- in: body
name: publisher
type: string
required: true
description: publisher name
- in: body
name: package
type: string
required: true
description: package name
- in: body
name: path
type: string
required: true
description: relative path of the resources
responses:
200:
description: Success
schema:
id: get_signed_url
properties:
key:
type: string
description: signed url for post data to S3
400:
description: Publisher or package can not be empty
500:
description: Internal Server Error
"""
try:
data = request.get_json()
publisher = data.get('publisher', None)
package = data.get('package', None)
path = data.get('path', None)
md5 = data.get('md5', None)
if publisher is None or package is None:
return handle_error('INVALID_INPUT',
'publisher or package can not be empty', 400)
if md5 is None:
return handle_error('INVALID_INPUT', 'md5 hash can not be empty',
400)
if path == 'datapackage.json':
return handle_error(
'INVALID_INPUT',
'datapackage.json should not publish with this api', 400)
metadata = BitStore(publisher=publisher, package=package)
url = metadata.generate_pre_signed_post_object(path, md5)
return jsonify({'data': url}), 200
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def get_publisher_profile(name):
"""
DPR metadata put operation.
This API is responsible for getting publisher profile
---
tags:
- profile
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
responses:
404:
description: Publisher not found
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: get_package_success
properties:
data:
type: object
description: data of publisher profile
properties:
description:
type: string
title:
type: string
name:
type: string
joined:
type: string
contact:
type: object
properties:
phone:
type: string
email:
type: string
country:
type: string
status:
type: string
default: SUCCESS
"""
try:
info = Publisher.get_publisher_info(name)
if info is None:
return handle_error("NOT_FOUND", "publisher not found", 404)
return jsonify(dict(data=info, status="SUCCESS"))
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def finalize_metadata(publisher, package):
"""
DPR metadata finalize operation.
This API is responsible for getting data from S3 and push it to RDS.
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
responses:
200:
description: Data transfer complete
400:
description: JWT is invalid
401:
description: Invalid Header for JWT
403:
description: User name and publisher not matched
404:
description: User not found
500:
description: Internal Server Error
"""
try:
user = _request_ctx_stack.top.current_user
user_id = user['user']
user = User.query.filter_by(id=user_id).first()
if user is not None:
if user.name == publisher:
bit_store = BitStore(publisher, package)
body = bit_store.get_metadata_body()
if body is not None:
readme = bit_store.get_s3_object(
bit_store.get_readme_object_key())
MetaDataDB.create_or_update(name=package,
publisher_name=publisher,
descriptor=body,
readme=readme)
return jsonify({"status": "OK"}), 200
raise Exception("Failed to get data from s3")
return handle_error('NOT_PERMITTED',
'user name and publisher not matched', 403)
return handle_error('USER_NOT_FOUND', 'user not found', 404)
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def get_metadata(publisher, package):
"""
DPR meta-data get operation.
This API is responsible for getting datapackage.json from S3.
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: false
description: package name - to retrieve the data package metadata
responses:
200:
description: Get Data package for one key
schema:
id: get_data_package
properties:
data:
type: map
description: The datapackage.json
500:
description: Internal Server Error
404:
description: No metadata found for the package
"""
try:
data = MetaDataDB.query.join(Publisher).\
filter(Publisher.name == publisher, MetaDataDB.name == package).\
first()
if data is None:
return handle_error('DATA_NOT_FOUND',
'No metadata found for the package',
404)
metadata = {
'id': data.id,
'name': data.name,
'publisher': data.publisher.name,
'readme': data.readme or '',
'descriptor': json.loads(data.descriptor)
}
return jsonify(metadata), 200
except Exception as e:
return handle_error('GENERIC_ERROR', e.message, 500)
def get_all_metadata_names_for_publisher(publisher):
"""
DPR meta-data get operation.
This API is responsible for getting All keys for the publisher
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
responses:
200:
description: Get Data package for one key
schema:
id: get_data_package
properties:
data:
type: array
items:
type: string
description: All data package names for the publisher
500:
description: Internal Server Error
404:
description: No metadata found for the package
"""
try:
metadata = MetaDataDB.query.join(Publisher).\
with_entities(MetaDataDB.name).\
filter(Publisher.name == publisher).all()
if len(metadata) is 0:
return handle_error('DATA_NOT_FOUND',
'No metadata found for the package',
404)
keys = []
for d in metadata:
keys.append(d[0])
return jsonify({'data': metadata}), 200
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def undelete_data_package(publisher, package):
"""
DPR data package un-delete operation.
This API is responsible for un-mark the mark for delete of data package
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
- in: header
name: Authorization
type: string
required: true
description: >
Jwt token in format of "bearer {token}.
The token can be generated from /api/auth/token"
responses:
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: put_package_success
properties:
status:
type: string
default: OK
"""
try:
bitstore = BitStore(publisher=publisher, package=package)
status_acl = bitstore.change_acl('public-read')
status_db = Package.change_status(publisher, package,
PackageStateEnum.active)
if status_acl and status_db:
return jsonify({"status": "OK"}), 200
if not status_acl:
raise Exception('Failed to change acl')
if not status_db:
raise Exception('Failed to change status')
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def callback_handling():
"""
This ia callback api when we redirect the api to Auth0 or any external
Auth provider.
---
tags:
- auth
response:
500:
description: Internal Server Error
200:
description: Updated Db with user
schema:
id: auth_callback
properties:
status:
type: string
description: Status of the operation
token:
type: string
description: The jwt
user:
type: map
description: Returns back email, nickname,
picture, name
"""
try:
code = request.args.get('code')
user_info = get_user_info_with_code(code, request.base_url)
user_id = user_info['user_id']
jwt_helper = JWTHelper(app.config['API_KEY'], user_id)
user = User().create_or_update_user_from_callback(user_info)
return render_template("dashboard.html",
user=user,
title='Dashboard',
encoded_token=jwt_helper.encode(),
zappa_env=get_zappa_prefix(),
s3_cdn=get_s3_cdn_prefix()), 200
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def purge_data_package(publisher, package):
"""
DPR data package hard delete operation.
This API is responsible for deletion of data package
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
responses:
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: put_package_success
properties:
status:
type: string
default: OK
"""
try:
bitstore = BitStore(publisher=publisher, package=package)
status_acl = bitstore.delete_data_package()
status_db = MetaDataDB.delete_data_package(publisher, package)
if status_acl and status_db:
return jsonify({"status": "OK"}), 200
if not status_acl:
raise Exception('Failed to delete from s3')
if not status_db:
raise Exception('Failed to delete from db')
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def callback_handling():
"""
This ia callback api when we redirect the api to Auth0 or any external
Auth provider.
---
tags:
- auth
response:
500:
description: Internal Server Error
200:
description: Updated Db with user
schema:
id: auth_callback
properties:
status:
type: string
description: Status of the operation
token:
type: string
description: The jwt
user:
type: map
description: Returns back email, nickname,
picture, name
"""
try:
code = request.args.get('code')
auth0 = Auth0()
user_info = auth0.get_user_info_with_code(code, request.base_url)
user = User().create_or_update_user_from_callback(user_info)
jwt_helper = JWT(app.config['API_KEY'], user.id)
return jsonify(
dict(token=jwt_helper.encode(),
email=user.email,
username=user.name,
secret=user.secret))
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def search_packages():
"""
DPR data package search operation.
This API is responsible for searching data package
---
tags:
- search
parameters:
- in: query
name: q
type: string
required: true
description: search query string e.g. q=query publisher=pub
responses:
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: search_package_success
properties:
total_count:
type: integer
description: Total datapackage count
items:
type: list
properties:
type: object
"""
try:
q = request.args.get('q')
result = DataPackageQuery(query_string=q).get_data()
return jsonify(dict(items=result, total_count=len(result)))
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def get_jwt():
"""
This API is responsible for returning JWT token
---
tags:
- auth
parameters:
- in: body
name: email
type: string
required: false
description: user email id
- in: body
name: username
type: string
required: false
description: user name
- in: body
name: password
type: string
required: true
description: user password
responses:
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: put_package_success
properties:
token:
type: string
description: jwt token
400:
description: Bad input data
404:
description: User not found
403:
description: Secret key do not match
"""
try:
data = request.get_json()
user_name = data.get('username', None)
email = data.get('email', None)
secret = data.get('secret', None)
verify = False
user_id = None
if user_name is None and email is None:
return handle_error('INVALID_INPUT',
'User name or email both can not be empty',
400)
if secret is None:
return handle_error('INVALID_INPUT', 'secret can not be empty',
400)
elif user_name is not None:
user = User.query.filter_by(name=user_name).first()
if user is None:
return handle_error('USER_NOT_FOUND', 'user does not exists',
404)
if secret == user.secret:
verify = True
user_id = user.id
elif email is not None:
user = User.query.filter_by(email=email).first()
if user is None:
return handle_error('USER_NOT_FOUND', 'user does not exists',
404)
if secret == user.secret:
verify = True
user_id = user.id
if verify:
return jsonify(
{'token': JWTHelper(app.config['API_KEY'],
user_id).encode()}), 200
else:
return handle_error('SECRET_ERROR', 'Secret key do not match', 403)
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def save_metadata(publisher, package):
"""
DPR metadata put operation.
This API is responsible for pushing datapackage.json to S3.
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
responses:
400:
description: JWT is invalid or req body is not valid
401:
description: Invalid Header for JWT
403:
description: User name and publisher not matched
404:
description: User not found
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: put_package_success
properties:
status:
type: string
description: Status of the operation
default: OK
"""
try:
user = _request_ctx_stack.top.current_user
user_id = user['user']
user = User.query.filter_by(id=user_id).first()
if user is not None:
if user.name == publisher:
metadata = BitStore(publisher=publisher,
package=package,
body=request.data)
is_valid = metadata.validate()
if not is_valid:
return handle_error('INVALID_DATA',
'Missing required field in metadata',
400)
metadata.save()
return jsonify({"status": "OK"}), 200
return handle_error('NOT_PERMITTED',
'user name and publisher not matched', 403)
return handle_error('USER_NOT_FOUND', 'user not found', 404)
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def get_resource(publisher, package, resource):
"""
DPR resource get operation.
This API is responsible for getting resource from S3.
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name - to retrieve the data package metadata
- in: path
name: resource
type: string
required: true
description: resource index or name
responses:
200:
description: Get Data package for one key
schema:
id: get_data_package
properties:
data:
type: string
description: The resource
500:
description: Internal Server Error
"""
try:
path = request.path
metadata = BitStore(publisher, package)
if path.endswith('csv'):
resource_key = metadata.build_s3_key(resource + '.csv')
data = metadata.get_s3_object(resource_key)
def generate():
for row in data.splitlines():
yield row + '\n'
return Response(generate()), 200
else:
resource_key = metadata.build_s3_key(resource + '.csv')
data = metadata.get_s3_object(resource_key)
data = csv.DictReader(data.splitlines())
# taking first and adding at the end to avoid last comma
first_row = next(data)
def generate():
yield '['
for row in data:
yield json.dumps(row) + ','
yield json.dumps(first_row) + ']'
return Response(generate(), content_type='application/json'), 200
except Exception as e:
return handle_error('GENERIC_ERROR', e.message, 500)
