def get_jwt_token():
data = request.get_json()
user_name = data.get('username', None)
email = data.get('email', None)
secret = data.get('secret', None)
verify = False
user_id = None
if user_name is None and email is None:
raise InvalidUsage('User name or email both can not be empty', 400)
if secret is None:
raise InvalidUsage('Secret can not be empty', 400)
elif user_name is not None:
try:
user = models.User.query.filter_by(name=user_name).one()
except NoResultFound as e:
app.logger.error(e)
raise InvalidUsage('user does not exists', 404)
if secret == user.secret:
verify = True
user_id = user.id
elif email is not None:
try:
user = models.User.query.filter_by(email=email).one()
except NoResultFound as e:
app.logger.error(e)
raise InvalidUsage('user does not exists', 404)
if secret == user.secret:
verify = True
user_id = user.id
if verify:
return JWT(app.config['JWT_SEED'], user_id).encode()
else:
raise InvalidUsage('Secret key do not match', 403)
def decode(self, token):
try:
return jwt.decode(token,
self.secret,
algorithm=self.algorithm,
issuer=self.issuer)
except jwt.ExpiredSignature:
raise InvalidUsage("Token is expired")
except jwt.DecodeError:
raise InvalidUsage('Token signature is invalid')
except Exception:
raise Exception('Unable to parse authentication token.')
def get_user_from_jwt(req, api_key):
jwt_helper = JWT(api_key)
token = req.headers.get('Authorization', None)
if token is None:
token = req.headers.get('Auth-Token', None)
if token is None:
token = req.values.get('jwt')
if not token:
raise InvalidUsage('Authorization header is expected', 401)
try:
return True, jwt_helper.decode(token)
except Exception as e:
raise InvalidUsage(e.message, 400)
def generate_signed_url():
user_id = None
jwt_status, user_info = get_user_from_jwt(request, app.config['JWT_SEED'])
if jwt_status:
user_id = user_info['user']
data = request.get_json()
metadata, filedata = data['metadata'], data['filedata']
publisher, package_name = metadata['owner'], metadata['name']
res_payload = {'filedata': {}}
if Package.exists(publisher, package_name):
status = check_is_authorized('Package::Update', publisher,
package_name, user_id)
else:
status = check_is_authorized('Package::Create', publisher,
package_name, user_id)
if not status:
raise InvalidUsage('Not authorized to upload data', 400)
for relative_path in filedata.keys():
response = FileData(package_name=package_name,
publisher=publisher,
relative_path=relative_path,
props=filedata[relative_path])
res_payload['filedata'][
relative_path] = response.build_file_information()
return res_payload
def generate_signed_url(user_id, data):
'''
@param data: data is dictionary of metadata (package owner and name) and
filedata (info for all the failes for package)
@param user_id: uniq id for user
'''
metadata, filedata = data['metadata'], data['filedata']
publisher, package_name = metadata['owner'], metadata['name']
res_payload = {'filedata': {}}
if Package.exists(publisher, package_name):
status = check_is_authorized('Package::Update', publisher, package_name, user_id)
else:
status = check_is_authorized('Package::Create', publisher, package_name, user_id)
if not status:
raise InvalidUsage('Not authorized to upload data', 400)
for relative_path in filedata.keys():
response = FileData(package_name=package_name,
publisher=publisher,
relative_path=relative_path,
props=filedata[relative_path])
res_payload['filedata'][relative_path] = response.build_file_information()
return res_payload
def finalize_publish(cls, user_id, datapackage_url):
'''
Gets the datapackage.json and README from S3 and imports into database.
Returns status "queued" if ok, else - None
'''
publisher, package, version = BitStore.extract_information_from_s3_url(
datapackage_url)
if Package.exists(publisher, package):
status = check_is_authorized('Package::Update', publisher, package,
user_id)
else:
status = check_is_authorized('Package::Create', publisher, package,
user_id)
if not status:
raise InvalidUsage('Not authorized to upload data', 400)
bit_store = BitStore(publisher, package)
b = bit_store.get_metadata_body()
body = json.loads(b)
bit_store.change_acl('public-read')
readme = bit_store.get_s3_object(bit_store.get_readme_object_key())
Package.create_or_update(name=package,
publisher_name=publisher,
descriptor=body,
readme=readme)
return "queued"
def get_top_ten_similar(movie_id):
# return top 10 movies' movie_ids
if movie_id.isdigit():
try:
model2 = load_pickle('ml_models/model2.pickle')
ret = {
'status': 'success',
'top-ten':
model2.get_recommendations(int(movie_id)).index.tolist()
}
return jsonify(ret)
except Exception as e:
raise InvalidUsage('Have error when get top similars: ' + str(e),
status_code=502)
else:
raise InvalidUsage('Need movie id', status_code=400)
def get_jwt_token(secret, username=None, email=None):
if username is not None:
user = models.User.query.filter_by(name=username, secret=secret).first()
else:
user = models.User.query.filter_by(email=email, secret=secret).first()
if not user:
raise InvalidUsage('Invalid secret for user', 403)
else:
return JWT(app.config['JWT_SEED'], user.id).encode()
def publisher_dashboard(publisher):
datapackage_list = logic.search.DataPackageQuery(
query_string="* publisher:{publisher}".format(
publisher=publisher)).get_data()
publisher = logic.Publisher.get(publisher)
if not publisher:
raise InvalidUsage('Not Found', 404)
return render_template("publisher.html",
publisher=publisher,
datapackage_list=datapackage_list), 200
def wrapped(*args, **kwargs):
user_id, instance = None, None
jwt_status, user_info = get_user_from_jwt(request,
app.config['JWT_SEED'])
if jwt_status:
user_id = user_info['user']
status = check_is_authorized(action, kwargs['publisher'],
kwargs['package'], user_id)
if not status:
raise InvalidUsage("The operation is not allowed", 403)
return f(*args, **kwargs)
def get_publisher_profile(name):
"""
DPR metadata put operation.
This API is responsible for getting publisher profile
---
tags:
- profile
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
responses:
404:
description: Publisher not found
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: get_package_success
properties:
data:
type: object
description: data of publisher profile
properties:
description:
type: string
title:
type: string
name:
type: string
joined:
type: string
contact:
type: object
properties:
phone:
type: string
email:
type: string
country:
type: string
status:
type: string
default: SUCCESS
"""
info = logic.Publisher.get(name)
if not info:
raise InvalidUsage('Not Found', 404)
return jsonify(dict(data=info, status="SUCCESS"))
def load_user(self, data):
email = data.get('email')
emails = data.get('emails')
if email:
return data
if not emails or not len(emails):
raise InvalidUsage('Email Not Found', 404)
for email in emails:
if email.get('primary') == 'true':
data['email'] = email.get('email')
return data
def check_is_authorized(action, publisher, package=None, user_id=None):
entity_str, action_str = action.split("::")
if entity_str == 'Package':
publisher_name, package_name = publisher, package
instance = Package.get_by_publisher(publisher_name, package_name)
elif entity_str == 'Publisher':
publisher_name = publisher
instance = Publisher.query.filter_by(name=publisher_name).one()
else:
raise InvalidUsage("{e} is not a valid one".format(e=entity_str), 401)
return is_authorize(user_id, instance, action)
def datapackage_show(publisher, package):
"""
Loads datapackage page for given owner
"""
datapackage = logic.Package.get(publisher, package)
if not datapackage:
raise InvalidUsage("Page Not Found", 404)
return render_template("dataset.html",
dataset=datapackage.get('descriptor'),
datapackageUrl=datapackage.get('datapackag_url'),
showDataApi=True,
dataViews=datapackage.get('views'),
readmeShort=datapackage.get('short_readme'),
readme_long=datapackage.get('readme')), 200
def undelete_data_package(publisher, package):
"""
DPR data package un-delete operation.
This API is responsible for un-mark the mark for delete of data package
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
- in: header
name: Authorization
type: string
required: true
description: JWT Token
responses:
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: put_package_success
properties:
status:
type: string
default: OK
"""
bitstore = BitStore(publisher=publisher, package=package)
status_db = logic.Package.change_status(publisher, package,
models.PackageStateEnum.active)
try:
status_acl = bitstore.change_acl('public-read')
except Exception as e:
## TODO roll back changes in db
raise InvalidUsage(e.message, 500)
if status_acl and status_db:
return jsonify({"status": "OK"}), 200
def delete_data_package(publisher, package):
"""
DPR Data Package Soft Delete
Marks Data Package as private
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
- in: header
name: Authorization
type: string
required: true
description: JWT Token
responses:
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: put_package_success
properties:
status:
type: string
default: OK
"""
bitstore = BitStore(publisher=publisher, package=package)
status_db = logic.Package.change_status(publisher, package,
models.PackageStateEnum.deleted)
try:
status_acl = bitstore.change_acl('private')
except Exception as e:
## TODO roll back changes in db
raise InvalidUsage(e.message, 500)
if status_acl and status_db:
return jsonify({"status": "OK"}), 200
def get_authorized_user_info():
'''
Return user info (with guaranteed Email) if authorized
'''
github = app.config['github']
resp = github.authorized_response()
if resp is None or resp.get('access_token') is None:
raise InvalidUsage('Access Denied', 400)
session['github_token'] = (resp['access_token'], '')
user_info = github.get('user').data
emails = github.get('user/emails').data
user_info['emails'] = emails
user_info_schema = UserInfoSchema()
user_info = user_info_schema.load(user_info).data
return user_info
def finalize_publish():
"""
Data Package finalize operation.
Gets the datapackage.json and README for Data Package and imports into database.
---
tags:
- package
parameters:
- in: body
name: url
type: map
required: true
description: URL to bitstore (S3) for Data Package.
- in: header
name: Authorization
type: string
required: true
description: JWT Token
responses:
200:
description: Data Uploaded
400:
description: Un-Authorized
401:
description: Invalid Header For JWT
500:
description: Internal Server Error
"""
data = request.get_json()
datapackage_url = data['datapackage']
jwt_status, user_info = get_user_from_jwt(request, app.config['JWT_SEED'])
user_id = None
if jwt_status:
user_id = user_info['user']
status = logic.Package.finalize_publish(user_id, datapackage_url)
if status:
return jsonify({"status": status}), 200
raise InvalidUsage("Failed to get data from s3")
def render_map(data, file_type, force=False):
if not has_app_context():
from app import app
app.app_context().push()
if ('grid' not in data or 'features' not in data):
raise InvalidUsage('invalid data')
map_id = data['id']
version = get_version(data)
file_info = get_file_info(map_id, version, file_type)
# if map already is rendered, do nothing
if not force and file_exists(file_info):
return
static_dir = current_app.static_folder
map_dir = os.path.join(static_dir, file_info['dir'])
if not os.path.exists(map_dir):
os.makedirs(map_dir)
# render map as in-memory file
m = MapRenderer(data)
f = m.render(file_info['mimetype'])
# write file atomically (otherwise we end up in serving incomplete files)
path = os.path.join(static_dir, file_info['path'])
with NamedTemporaryFile(dir=os.path.dirname(path), delete=False) as tmp_f:
tmp_f.write(f.read())
os.replace(tmp_f.name, path)
# update latest symlink to this
symlink_name = 'LATEST' + file_info['suffix']
path_latest = os.path.join(static_dir, file_info['dir'], symlink_name)
try:
os.symlink(path, path_latest)
except FileExistsError:
os.unlink(path_latest)
os.symlink(path, path_latest)
def _build_sql_query(self, query, query_filters):
sql_query = Package.query.join(Package.publisher)
sa_filters = []
for f in query_filters:
filter_class, filter_term = f.split(":")
if filter_class == 'publisher':
sa_filters.append(Publisher.name == filter_term)
else:
raise InvalidUsage("not supported any other filter right now")
if len(sa_filters) > 0:
sql_query = sql_query.filter(or_(*sa_filters))
if query != '*' or not query.strip():
sql_query = sql_query.join(Package.tags)\
.filter(PackageTag.descriptor.op('->>')('title')
.cast(sqlalchemy.TEXT)
.ilike("%{q}%".format(q=query)),
PackageTag.tag == 'latest',
Package.status == PackageStateEnum.active)
return sql_query
def get_metadata(publisher, package):
"""
DPR meta-data get operation.
This API is responsible for getting datapackage.json from S3.
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: false
description: package name - to retrieve the data package metadata
responses:
200:
description: Get Data package for one key
schema:
id: get_data_package
properties:
data:
type: map
description: The datapackage.json
500:
description: Internal Server Error
404:
description: No metadata found for the package
"""
metadata = logic.Package.get(publisher, package)
if metadata is None:
raise InvalidUsage('No metadata found for the package', 404)
return jsonify(metadata), 200
def tag_data_package(publisher, package):
"""
DPR metadata put operation.
This API is responsible for tagging data package
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
- in: body
name: version
type: string
required: true
description: version value
- in: header
name: Authorization
type: string
required: true
description: JWT Token
responses:
400:
description: JWT is invalid or req body is not valid
401:
description: Invalid Header for JWT
403:
description: User not allowed for operation
404:
description: User not found
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: put_package_success
properties:
status:
type: string
description: Status of the operation
default: OK
"""
data = request.get_json()
if 'version' not in data:
raise InvalidUsage('version not found', 400)
bitstore = BitStore(publisher, package)
status_db = logic.Package.create_or_update_tag(publisher, package,
data['version'])
try:
status_bitstore = bitstore.copy_to_new_version(data['version'])
except Exception as e:
## TODO roll back changes in db
raise InvalidUsage(e.message, 500)
return jsonify({"status": "OK"}), 200
