def _get_table_metadata(*, table_key: str, index: int,
source: str) -> Dict[str, Any]:
results_dict = {
'tableData': {},
'msg': '',
}
try:
table_endpoint = _get_table_endpoint()
url = '{0}/{1}'.format(table_endpoint, table_key)
response = request_metadata(url=url)
except ValueError as e:
# envoy client BadResponse is a subclass of ValueError
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
results_dict['status_code'] = getattr(e, 'code',
HTTPStatus.INTERNAL_SERVER_ERROR)
logging.exception(message)
return results_dict
status_code = response.status_code
results_dict['status_code'] = status_code
if status_code != HTTPStatus.OK:
message = 'Encountered error: Metadata request failed'
results_dict['msg'] = message
logging.error(message)
return results_dict
try:
table_data_raw: dict = response.json()
# Ideally the response should include 'key' to begin with
table_data_raw['key'] = table_key
results_dict['tableData'] = marshall_table_full(table_data_raw)
results_dict['msg'] = 'Success'
return results_dict
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
# explicitly raise the exception which will trigger 500 api response
results_dict['status_code'] = getattr(e, 'code',
HTTPStatus.INTERNAL_SERVER_ERROR)
return results_dict
def _get_related_dashboards_metadata(*, url: str) -> Dict[str, Any]:
results_dict = {
'dashboards': [],
'msg': '',
}
try:
response = request_metadata(url=url)
except ValueError as e:
# envoy client BadResponse is a subclass of ValueError
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
results_dict['status_code'] = getattr(e, 'code',
HTTPStatus.INTERNAL_SERVER_ERROR)
logging.exception(message)
return results_dict
status_code = response.status_code
results_dict['status_code'] = status_code
if status_code != HTTPStatus.OK:
message = f'Encountered {status_code} Error: Related dashboard metadata request failed'
results_dict['msg'] = message
logging.error(message)
return results_dict
try:
dashboard_data_raw = response.json().get('dashboards', [])
return {
'dashboards': [
marshall_dashboard_partial(dashboard)
for dashboard in dashboard_data_raw
],
'msg':
'Success',
'status_code':
status_code
}
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
# explicitly raise the exception which will trigger 500 api response
results_dict['status_code'] = getattr(e, 'code',
HTTPStatus.INTERNAL_SERVER_ERROR)
return results_dict
def get_feature_generation_code() -> Response:
"""
Call metadata service to fetch feature generation code
:return:
"""
try:
feature_key = get_query_param(request.args, 'key')
endpoint = _get_feature_endpoint()
url = f'{endpoint}/{feature_key}/generation_code'
response = request_metadata(url=url, method=request.method)
payload = response.json()
return make_response(jsonify(payload), 200)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def update_bookmark() -> Response:
"""
Call metadata service to PUT or DELETE a bookmark
Params
:param type: Resource type for the bookmarked item. e.g. 'table'
:param key: Resource key for the bookmarked item.
:return:
"""
@action_logging
def _log_update_bookmark(*, resource_key: str, resource_type: str,
method: str) -> None:
pass # pragma: no cover
try:
if app.config['AUTH_USER_METHOD']:
user = app.config['AUTH_USER_METHOD'](app)
else:
raise Exception('AUTH_USER_METHOD is not configured')
args = request.get_json()
resource_type = get_query_param(args, 'type')
resource_key = get_query_param(args, 'key')
url = '{0}{1}/{2}/follow/{3}/{4}'.format(
app.config['METADATASERVICE_BASE'], USER_ENDPOINT, user.user_id,
resource_type, resource_key)
_log_update_bookmark(resource_key=resource_key,
resource_type=resource_type,
method=request.method)
response = request_metadata(url=url, method=request.method)
status_code = response.status_code
return make_response(
jsonify({
'msg': 'success',
'response': response.json()
}), status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
return make_response(jsonify({'msg': message}),
HTTPStatus.INTERNAL_SERVER_ERROR)
def put_column_description() -> Response:
@action_logging
def _log_put_column_description(*, table_key: str, column_name: str,
description: str, source: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
table_key = get_query_param(args, 'key')
table_endpoint = _get_table_endpoint()
column_name = get_query_param(args, 'column_name')
description = get_query_param(args, 'description')
src = get_query_param(args, 'source')
table_uri = TableUri.from_uri(table_key)
if not is_table_editable(table_uri.schema, table_uri.table):
return make_response('', HTTPStatus.FORBIDDEN)
url = '{0}/{1}/column/{2}/description'.format(table_endpoint,
table_key, column_name)
_log_put_column_description(table_key=table_key,
column_name=column_name,
description=description,
source=src)
response = request_metadata(url=url,
method='PUT',
data=json.dumps(
{'description': description}))
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
else:
message = 'Update column description failed'
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_bookmark() -> Response:
"""
Call metadata service to fetch a specified user's bookmarks.
If no 'user_id' is specified, it will fetch the logged-in user's bookmarks
:param user_id: (optional) the user whose bookmarks are fetched.
:return: a JSON object with an array of bookmarks under 'bookmarks' key
"""
try:
user_id = request.args.get('user_id')
if user_id is None:
if app.config['AUTH_USER_METHOD']:
user_id = app.config['AUTH_USER_METHOD'](app).user_id
else:
raise Exception('AUTH_USER_METHOD is not configured')
url = '{0}{1}/{2}/follow/'.format(app.config['METADATASERVICE_BASE'],
USER_ENDPOINT, user_id)
response = request_metadata(url=url, method=request.method)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
tables = response.json().get('table')
table_bookmarks = [
marshall_table_partial(table) for table in tables
]
else:
message = f'Encountered error: failed to get bookmark for user_id: {user_id}'
logging.error(message)
table_bookmarks = []
return make_response(
jsonify({
'msg': message,
'bookmarks': table_bookmarks
}), status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
return make_response(jsonify({'msg': message}),
HTTPStatus.INTERNAL_SERVER_ERROR)
def popular_tables() -> Response:
"""
call the metadata service endpoint to get the current popular tables
:return: a json output containing an array of popular table metadata as 'popular_tables'
Schema Defined Here:
https://github.com/lyft/amundsenmetadatalibrary/blob/master/metadata_service/api/popular_tables.py
"""
try:
if app.config['AUTH_USER_METHOD'] and app.config[
'POPULAR_TABLE_PERSONALIZATION']:
user_id = app.config['AUTH_USER_METHOD'](app).user_id
else:
user_id = ''
service_base = app.config['METADATASERVICE_BASE']
count = app.config['POPULAR_TABLE_COUNT']
url = f'{service_base}{POPULAR_TABLES_ENDPOINT}/{user_id}?limit={count}'
response = request_metadata(url=url)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
response_list = response.json().get('popular_tables')
popular_tables = [
marshall_table_partial(result) for result in response_list
]
else:
message = 'Encountered error: Request to metadata service failed with status code ' + str(
status_code)
logging.error(message)
popular_tables = [{}]
payload = jsonify({'results': popular_tables, 'msg': message})
return make_response(payload, status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
payload = jsonify({'results': [{}], 'msg': message})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_user_own() -> Response:
"""
Calls metadata service to GET owned resources
:return: a JSON object with an array of owned resources
"""
try:
user_id = get_query_param(request.args, 'user_id')
url = '{0}{1}/{2}/own/'.format(app.config['METADATASERVICE_BASE'],
USER_ENDPOINT,
user_id)
response = request_metadata(url=url, method=request.method)
status_code = response.status_code
owned_tables_raw = response.json().get('table')
owned_tables = [marshall_table_partial(table) for table in owned_tables_raw]
return make_response(jsonify({'msg': 'success', 'own': owned_tables}), status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
return make_response(jsonify({'msg': message}), HTTPStatus.INTERNAL_SERVER_ERROR)
def get_table_description() -> Response:
try:
table_endpoint = _get_table_endpoint()
table_key = get_query_param(request.args, 'key')
url = '{0}/{1}/description'.format(table_endpoint, table_key)
response = request_metadata(url=url)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
description = response.json().get('description')
else:
message = 'Get table description failed'
description = None
payload = jsonify({'description': description, 'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({'description': None, 'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_table_lineage() -> Response:
"""
Call metadata service to fetch table lineage for a given table
:return:
"""
try:
table_endpoint = _get_table_endpoint()
table_key = get_query_param(request.args, 'key')
url = f'{table_endpoint}/{table_key}/lineage'
response = request_metadata(url=url, method=request.method)
json = response.json()
downstream = [marshall_lineage_table(table) for table in json.get('downstream_entities')]
upstream = [marshall_lineage_table(table) for table in json.get('upstream_entities')]
payload = {
'downstream_entities': downstream,
'upstream_entities': upstream,
}
return make_response(jsonify(payload), 200)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def put_column_description() -> Response:
@action_logging
def _log_put_column_description(*, table_key: str, column_name: str,
description: str, source: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
table_key = get_table_key(args)
table_endpoint = _get_table_endpoint()
column_name = get_query_param(args, 'column_name')
description = get_query_param(args, 'description')
description = ' '.join(description.split())
src = get_query_param(args, 'source')
url = '{0}/{1}/column/{2}/description/{3}'.format(
table_endpoint, table_key, column_name, description)
_log_put_column_description(table_key=table_key,
column_name=column_name,
description=description,
source=src)
response = request_metadata(url=url, method='PUT')
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
else:
message = 'Update column description failed'
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def update_feature_owner() -> Response:
try:
args = request.get_json()
feature_key = get_query_param(args, 'key')
owner = get_query_param(args, 'owner')
endpoint = _get_feature_endpoint()
url = '{0}/{1}/owner/{2}'.format(endpoint, feature_key, owner)
method = request.method
response = request_metadata(url=url, method=method)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Updated owner'
else:
message = 'There was a problem updating owner {0}'.format(owner)
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def _authorize_access(self, user_id: str) -> None:
"""
Get Mode user ID via metadata service. Note that metadata service needs to be at least v2.5.2 and
Databuilder should also have ingested Mode user.
https://github.com/lyft/amundsendatabuilder#modedashboarduserextractor
:param user_id:
:return:
:raise: PermissionError when user is not allowed to access the dashboard
"""
metadata_svc_url = '{0}{1}/{2}'.format(
app.config['METADATASERVICE_BASE'], USER_ENDPOINT, user_id)
response = request_metadata(url=metadata_svc_url)
response.raise_for_status()
user = load_user(response.json())
if user.is_active and user.other_key_values and user.other_key_values.get(
'mode_user_id'):
return
raise PermissionError(
'User {} is not authorized to preview Mode Dashboard'.format(
user_id))
def _get_table_metadata(*, table_key: str, index: int,
source: str) -> Dict[str, Any]:
def _map_user_object_to_schema(u: Dict) -> Dict:
return dump_user(load_user(u))
results_dict = {
'tableData': {},
'msg': '',
}
try:
table_endpoint = _get_table_endpoint()
url = '{0}/{1}'.format(table_endpoint, table_key)
response = request_metadata(url=url)
except ValueError as e:
# envoy client BadResponse is a subclass of ValueError
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
results_dict['status_code'] = getattr(e, 'code',
HTTPStatus.INTERNAL_SERVER_ERROR)
logging.exception(message)
return results_dict
status_code = response.status_code
results_dict['status_code'] = status_code
if status_code != HTTPStatus.OK:
message = 'Encountered error: Metadata request failed'
results_dict['msg'] = message
logging.error(message)
return results_dict
try:
# Filter and parse the response dictionary from the metadata service
params = [
'columns',
'cluster',
'database',
'owners',
'is_view',
'schema',
'table_description',
'table_name',
'table_readers',
'table_writer',
'tags',
'watermarks',
'source',
]
results = {key: response.json().get(key, None) for key in params}
results['key'] = table_key
is_editable = results['schema'] not in app.config['UNEDITABLE_SCHEMAS']
results['is_editable'] = is_editable
# In the list of owners, sanitize each entry
results['owners'] = [
_map_user_object_to_schema(owner) for owner in results['owners']
]
# In the list of reader_objects, sanitize the reader value on each entry
readers = results['table_readers']
for reader_object in readers:
reader_object['reader'] = _map_user_object_to_schema(
reader_object['reader'])
# If order is provided, we sort the column based on the pre-defined order
if app.config['COLUMN_STAT_ORDER']:
columns = results['columns']
for col in columns:
# the stat_type isn't defined in COLUMN_STAT_ORDER, we just use the max index for sorting
col['stats'].sort(
key=lambda x: app.config['COLUMN_STAT_ORDER'].get(
x['stat_type'], len(app.config['COLUMN_STAT_ORDER'])))
col['is_editable'] = is_editable
# Temp code to make 'partition_key' and 'partition_value' part of the table
results['partition'] = _get_partition_data(results['watermarks'])
results_dict['tableData'] = results
results_dict['msg'] = 'Success'
return results_dict
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
# explicitly raise the exception which will trigger 500 api response
results_dict['status_code'] = getattr(e, 'code',
HTTPStatus.INTERNAL_SERVER_ERROR)
return results_dict