def put_table_description() -> Response:
@action_logging
def _log_put_table_description(*, table_key: str, description: str,
source: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
table_endpoint = _get_table_endpoint()
table_key = get_query_param(args, 'key')
description = get_query_param(args, 'description')
description = ' '.join(description.split())
src = get_query_param(args, 'source')
url = '{0}/{1}/description/{2}'.format(table_endpoint, table_key,
description)
_log_put_table_description(table_key=table_key,
description=description,
source=src)
response = request_metadata(url=url, method='PUT')
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
else:
message = 'Update table description failed'
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def search_table() -> Response:
"""
Parse the request arguments and call the helper method to execute a table search
:return: a Response created with the results from the helper method
"""
try:
request_json = request.get_json()
search_term = get_query_param(
request_json, 'term', '"term" parameter expected in request data')
page_index = get_query_param(
request_json, 'pageIndex',
'"pageIndex" parameter expected in request data')
search_type = request_json.get('searchType')
transformed_filters = transform_filters(
filters=request_json.get('filters', {}))
results_dict = _search_table(filters=transformed_filters,
search_term=search_term,
page_index=page_index,
search_type=search_type)
return make_response(
jsonify(results_dict),
results_dict.get('status_code', HTTPStatus.INTERNAL_SERVER_ERROR))
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
return make_response(jsonify(results_dict),
HTTPStatus.INTERNAL_SERVER_ERROR)
def get_feature_lineage() -> Response:
"""
Call metadata service to fetch table lineage for a given feature
:return:
"""
try:
feature_key = get_query_param(request.args, 'key')
depth = get_query_param(request.args, 'depth')
direction = get_query_param(request.args, 'direction')
endpoint = _get_feature_endpoint()
url = f'{endpoint}/{feature_key}/lineage?depth={depth}&direction={direction}'
response = request_metadata(url=url, method=request.method)
json = response.json()
downstream = [
marshall_lineage_table(table)
for table in json.get('downstream_entities')
]
upstream = [
marshall_lineage_table(table)
for table in json.get('upstream_entities')
]
payload = {
'downstream_entities': downstream,
'upstream_entities': upstream,
}
return make_response(jsonify(payload), 200)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def search_dashboard() -> Response:
"""
Parse the request arguments and call the helper method to execute a dashboard search
:return: a Response created with the results from the helper method
"""
try:
search_term = get_query_param(request.args, 'query',
'Endpoint takes a "query" parameter')
page_index = get_query_param(
request.args, 'page_index',
'Endpoint takes a "page_index" parameter')
search_type = request.args.get('search_type')
results_dict = _search_dashboard(search_term=search_term,
page_index=int(page_index),
search_type=search_type)
return make_response(
jsonify(results_dict),
results_dict.get('status_code', HTTPStatus.INTERNAL_SERVER_ERROR))
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
return make_response(jsonify(results_dict),
HTTPStatus.INTERNAL_SERVER_ERROR)
def update_table_owner() -> Response:
@action_logging
def _log_update_table_owner(*, table_key: str, method: str,
owner: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
table_key = get_query_param(args, 'key')
owner = get_query_param(args, 'owner')
table_endpoint = _get_table_endpoint()
url = '{0}/{1}/owner/{2}'.format(table_endpoint, table_key, owner)
method = request.method
_log_update_table_owner(table_key=table_key,
method=method,
owner=owner)
response = request_metadata(url=url, method=method)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Updated owner'
else:
message = 'There was a problem updating owner {0}'.format(owner)
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def _get_table_key(args: Dict) -> str:
db = get_query_param(args, 'db')
cluster = get_query_param(args, 'cluster')
schema = get_query_param(args, 'schema')
table = get_query_param(args, 'table')
table_key = '{db}://{cluster}.{schema}/{table}'.format(**locals())
return table_key
def search() -> Response:
"""
Parse the request arguments and call the helper method to execute a search for specified resources
:return: a Response created with the results from the helper method
"""
results_dict = {}
try:
request_json = request.get_json()
search_term = get_query_param(
request_json, 'searchTerm',
'"searchTerm" parameter expected in request data')
page_index = get_query_param(
request_json, 'pageIndex',
'"pageIndex" parameter expected in request data')
results_per_page = get_query_param(
request_json, 'resultsPerPage',
'"resultsPerPage" parameter expected in request data')
search_type = request_json.get('searchType')
resources = request_json.get('resources', [])
filters = request_json.get('filters', {})
results_dict = _search_resources(
search_term=search_term,
resources=resources,
page_index=int(page_index),
results_per_page=int(results_per_page),
filters=filters,
search_type=search_type)
return make_response(jsonify(results_dict),
results_dict.get('status_code', HTTPStatus.OK))
except Exception as e:
message = 'Encountered exception: ' + str(e)
LOGGER.exception(message)
return make_response(jsonify(results_dict),
HTTPStatus.INTERNAL_SERVER_ERROR)
def get_column_description() -> Response:
try:
table_endpoint = _get_table_endpoint()
table_key = get_query_param(request.args, 'key')
column_name = get_query_param(request.args, 'column_name')
url = '{0}/{1}/column/{2}/description'.format(table_endpoint,
table_key, column_name)
response = request_metadata(url=url)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
description = response.json().get('description')
else:
message = 'Get column description failed'
description = None
payload = jsonify({'description': description, 'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({
'description': None,
'msg': 'Encountered exception: ' + str(e)
})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def update_table_tags() -> Response:
@action_logging
def _log_update_table_tags(*, table_key: str, method: str, tag: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
method = request.method
table_key = get_query_param(args, 'key')
tag = get_query_param(args, 'tag')
_log_update_table_tags(table_key=table_key, method=method, tag=tag)
metadata_status_code = _update_metadata_tag(table_key=table_key, method=method, tag=tag)
search_status_code = _update_search_tag(table_key=table_key, method=method, tag=tag)
http_status_code = HTTPStatus.OK
if metadata_status_code == HTTPStatus.OK and search_status_code == HTTPStatus.OK:
message = 'Success'
else:
message = f'Encountered error: {method} table tag failed'
logging.error(message)
http_status_code = HTTPStatus.INTERNAL_SERVER_ERROR
payload = jsonify({'msg': message})
return make_response(payload, http_status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
payload = jsonify({'msg': message})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def update_table_tags() -> Response:
@action_logging
def _log_update_table_tags(*, table_key: str, method: str,
tag: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
method = request.method
table_endpoint = _get_table_endpoint()
table_key = get_query_param(args, 'key')
tag = get_query_param(args, 'tag')
url = '{0}/{1}/tag/{2}'.format(table_endpoint, table_key, tag)
_log_update_table_tags(table_key=table_key, method=method, tag=tag)
response = request_metadata(url=url, method=method)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
else:
message = 'Encountered error: {0} tag failed'.format(method)
logging.error(message)
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
payload = jsonify({'msg': message})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def put_table_description() -> Response:
@action_logging
def _log_put_table_description(*, table_key: str, description: str, source: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
table_endpoint = _get_table_endpoint()
table_key = get_query_param(args, 'key')
description = get_query_param(args, 'description')
src = get_query_param(args, 'source')
table_uri = TableUri.from_uri(table_key)
if not is_table_editable(table_uri.schema, table_uri.table):
return make_response('', HTTPStatus.FORBIDDEN)
url = '{0}/{1}/description'.format(table_endpoint, table_key)
_log_put_table_description(table_key=table_key, description=description, source=src)
response = request_metadata(url=url, method='PUT', data=json.dumps({'description': description}))
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
else:
message = 'Update table description failed'
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def put_feature_description() -> Response:
try:
args = request.get_json()
feature_key = get_query_param(args, 'key')
description = get_query_param(args, 'description')
endpoint = _get_feature_endpoint()
url = '{0}/{1}/description'.format(endpoint, feature_key)
response = request_metadata(url=url,
method='PUT',
data=json.dumps(
{'description': description}))
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
else:
message = 'Update feature description failed'
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_column_lineage() -> Response:
"""
Call metadata service to fetch table lineage for a given table
:return:
"""
try:
table_endpoint = _get_table_endpoint()
table_key = get_query_param(request.args, 'key')
column_name = get_query_param(request.args, 'column_name')
url = f'{table_endpoint}/{table_key}/column/{column_name}/lineage'
response = request_metadata(url=url, method=request.method)
json = response.json()
downstream = [
marshall_lineage_table(table)
for table in json.get('downstream_entities')
]
upstream = [
marshall_lineage_table(table)
for table in json.get('upstream_entities')
]
payload = {
'downstream_entities': downstream,
'upstream_entities': upstream,
}
return make_response(jsonify(payload), 200)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_table_metadata() -> Response:
"""
call the metadata service endpoint and return matching results
:return: a json output containing a table metdata object as 'tableData'
Schema Defined Here: https://github.com/lyft/amundsenmetadatalibrary/blob/master/metadata_service/api/table.py
TODO: Define type for this
TODO: Define an interface for envoy_client
"""
try:
table_key = get_query_param(request.args, 'key')
list_item_index = get_query_param(request.args, 'index')
list_item_source = get_query_param(request.args, 'source')
results_dict = _get_table_metadata(table_key=table_key,
index=list_item_index,
source=list_item_source)
return make_response(
jsonify(results_dict),
results_dict.get('status_code', HTTPStatus.INTERNAL_SERVER_ERROR))
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
return make_response(jsonify({
'tableData': {},
'msg': message
}), HTTPStatus.INTERNAL_SERVER_ERROR)
def search_table_query_string() -> Response:
"""
TODO (ttannis): Update this docstring after amundsensearch documentation is merged
Calls the search service to execute a search. The request data is transformed
to the json payload defined [link]
"""
request_json = request.get_json()
search_term = get_query_param(request_json, 'term', '"term" parameter expected in request data')
page_index = get_query_param(request_json, 'pageIndex', '"pageIndex" parameter expected in request data')
filters = request_json.get('filters', {})
# Default results
tables = {
'page_index': int(page_index),
'results': [],
'total_results': 0,
}
results_dict = {
'search_term': search_term,
'msg': '',
'tables': tables,
}
try:
query_json = generate_query_json(filters=filters, page_index=page_index, search_term=search_term)
except Exception as e:
message = 'Encountered exception generating query json: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
return make_response(jsonify(results_dict), HTTPStatus.INTERNAL_SERVER_ERROR)
try:
# TODO (ttannis): Change actual endpoint name after amundsensearch PR is merged
url = app.config['SEARCHSERVICE_BASE'] + '/search_table'
response = request_search(url=url,
headers={'Content-Type': 'application/json'},
method='POST',
data=json.dumps(query_json))
status_code = response.status_code
if status_code == HTTPStatus.OK:
results_dict['msg'] = 'Success'
results = response.json().get('results')
tables['results'] = [map_table_result(result) for result in results]
tables['total_results'] = response.json().get('total_results')
else:
message = 'Encountered error: Search request failed'
results_dict['msg'] = message
logging.error(message)
results_dict['status_code'] = status_code
return make_response(jsonify(results_dict), status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
return make_response(jsonify(results_dict), HTTPStatus.INTERNAL_SERVER_ERROR)
def search_table() -> Response:
search_term = get_query_param(request.args, 'query', 'Endpoint takes a "query" parameter')
page_index = get_query_param(request.args, 'page_index', 'Endpoint takes a "page_index" parameter')
error_response = _validate_search_term(search_term=search_term, page_index=int(page_index))
if error_response is not None:
return error_response
results_dict = _search_table(search_term=search_term, page_index=page_index)
return make_response(jsonify(results_dict), results_dict.get('status_code', HTTPStatus.INTERNAL_SERVER_ERROR))
def search_user() -> Response:
search_term = get_query_param(request.args, 'query',
'Endpoint takes a "query" parameter')
page_index = get_query_param(request.args, 'page_index',
'Endpoint takes a "page_index" parameter')
results_dict = _search_user(search_term=search_term, page_index=page_index)
return make_response(
jsonify(results_dict),
results_dict.get('status_code', HTTPStatus.INTERNAL_SERVER_ERROR))
def update_table_owner() -> Response:
try:
args = request.get_json()
table_key = get_query_param(args, 'key')
owner = get_query_param(args, 'owner')
payload = jsonify(_update_table_owner(table_key=table_key, method=request.method, owner=owner))
return make_response(payload, HTTPStatus.OK)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_table_key(args: Dict) -> str:
"""
Extracts the 'key' for a table resource
:param args: Dict which includes 'db', 'cluster', 'schema', and 'table'
:return: the table key
"""
db = get_query_param(args, 'db')
cluster = get_query_param(args, 'cluster')
schema = get_query_param(args, 'schema')
table = get_query_param(args, 'table')
table_key = '{db}://{cluster}.{schema}/{table}'.format(**locals())
return table_key
def update_table_tags() -> Response:
@action_logging
def _log_update_table_tags(*, table_key: str, method: str,
tag: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
method = request.method
table_key = get_query_param(args, 'key')
tag = get_query_param(args, 'tag')
_log_update_table_tags(table_key=table_key, method=method, tag=tag)
metadata_status_code = _update_metadata_tag(table_key=table_key,
method=method,
tag=tag)
search_method = method if method == 'DELETE' else 'POST'
update_request = UpdateDocumentRequest(
resource_key=table_key,
resource_type=ResourceType.Table.name.lower(),
field='tag',
value=tag,
operation='add')
request_json = json.dumps(
UpdateDocumentRequestSchema().dump(update_request))
search_status_code = execute_search_document_request(
request_json=request_json, method=search_method)
http_status_code = HTTPStatus.OK
if metadata_status_code == HTTPStatus.OK and search_status_code == HTTPStatus.OK:
message = 'Success'
else:
message = f'Encountered error: {method} table tag failed'
logging.error(message)
http_status_code = HTTPStatus.INTERNAL_SERVER_ERROR
payload = jsonify({'msg': message})
return make_response(payload, http_status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
payload = jsonify({'msg': message})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def put_column_description() -> Response:
@action_logging
def _log_put_column_description(*, table_key: str, column_name: str,
description: str, source: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
table_key = _get_table_key(args)
table_endpoint = _get_table_endpoint()
column_name = get_query_param(args, 'column_name')
description = get_query_param(args, 'description')
description = ' '.join(description.split())
src = get_query_param(args, 'source')
url = '{0}/{1}/column/{2}/description/{3}'.format(
table_endpoint, table_key, column_name, description)
_log_put_column_description(table_key=table_key,
column_name=column_name,
description=description,
source=src)
# TODO: Create an abstraction for this logic that is reused many times
envoy_client = app.config['METADATASERVICE_REQUEST_CLIENT']
if envoy_client is not None:
envoy_headers = app.config['METADATASERVICE_REQUEST_HEADERS']
response = envoy_client.put(url,
headers=envoy_headers,
raw_response=True)
else:
with requests.Session() as s:
response = s.put(url, timeout=REQUEST_SESSION_TIMEOUT)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
else:
message = 'Update column description failed'
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_user() -> Response:
@action_logging
def _log_get_user(*, user_id: str) -> None:
pass # pragma: no cover
try:
user_id = get_query_param(request.args, 'user_id')
url = '{0}{1}/{2}'.format(app.config['METADATASERVICE_BASE'],
USER_ENDPOINT, user_id)
_log_get_user(user_id=user_id)
response = request_metadata(url=url)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
else:
message = 'Encountered error: failed to fetch user with user_id: {0}'.format(
user_id)
logging.error(message)
payload = {
'msg': message,
'user': dump_user(load_user(response.json())),
}
return make_response(jsonify(payload), status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
payload = jsonify({'msg': message})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_column_description() -> Response:
try:
table_endpoint = _get_table_endpoint()
table_key = _get_table_key(request.args)
column_name = get_query_param(request.args, 'column_name')
url = '{0}/{1}/column/{2}/description'.format(table_endpoint, table_key, column_name)
# TODO: Create an abstraction for this logic that is reused many times
envoy_client = app.config['METADATASERVICE_REQUEST_CLIENT']
if envoy_client is not None:
envoy_headers = app.config['METADATASERVICE_REQUEST_HEADERS']
response = envoy_client.get(url, headers=envoy_headers, raw_response=True)
else:
with requests.Session() as s:
response = s.get(url, timeout=REQUEST_SESSION_TIMEOUT)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
description = response.json().get('description')
else:
message = 'Get column description failed'
description = None
payload = jsonify({'description': description, 'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({'description': None, 'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_user_own() -> Response:
"""
Calls metadata service to GET owned resources
:return: a JSON object with an array of owned resources
"""
try:
user_id = get_query_param(request.args, 'user_id')
url = '{0}{1}/{2}/own/'.format(app.config['METADATASERVICE_BASE'],
USER_ENDPOINT, user_id)
response = request_metadata(url=url, method=request.method)
status_code = response.status_code
owned_tables_raw = response.json().get('table')
owned_tables = [
marshall_table_partial(table) for table in owned_tables_raw
]
return make_response(jsonify({
'msg': 'success',
'own': owned_tables
}), status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
return make_response(jsonify({'msg': message}),
HTTPStatus.INTERNAL_SERVER_ERROR)
def get_dashboard_metadata() -> Response:
"""
Call metadata service endpoint to fetch specified dashboard metadata
:return:
"""
@action_logging
def _get_dashboard_metadata(*, uri: str, index: int, source: str) -> None:
pass # pragma: no cover
try:
uri = get_query_param(request.args, 'uri')
index = request.args.get('index', None)
source = request.args.get('source', None)
_get_dashboard_metadata(uri=uri, index=index, source=source)
url = f'{app.config["METADATASERVICE_BASE"]}{DASHBOARD_ENDPOINT}/{uri}'
response = request_metadata(url=url, method=request.method)
dashboard = marshall_dashboard_full(response.json())
status_code = response.status_code
return make_response(
jsonify({
'msg': 'success',
'dashboard': dashboard
}), status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
return make_response(jsonify({
'dashboard': {},
'msg': message
}), HTTPStatus.INTERNAL_SERVER_ERROR)
def get_column_description() -> Response:
try:
table_endpoint = _get_table_endpoint()
table_key = _get_table_key(request.args)
column_name = get_query_param(request.args, 'column_name')
url = '{0}/{1}/column/{2}/description'.format(table_endpoint,
table_key, column_name)
response = request_wrapper(
method='GET',
url=url,
client=app.config['METADATASERVICE_REQUEST_CLIENT'],
headers=app.config['METADATASERVICE_REQUEST_HEADERS'],
timeout_sec=REQUEST_SESSION_TIMEOUT)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
description = response.json().get('description')
else:
message = 'Get column description failed'
description = None
payload = jsonify({'description': description, 'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({
'description': None,
'msg': 'Encountered exception: ' + str(e)
})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def put_column_description() -> Response:
@action_logging
def _log_put_column_description(*, table_key: str, column_name: str,
description: str, source: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
table_key = _get_table_key(args)
table_endpoint = _get_table_endpoint()
column_name = get_query_param(args, 'column_name')
description = get_query_param(args, 'description')
description = ' '.join(description.split())
src = get_query_param(args, 'source')
url = '{0}/{1}/column/{2}/description/{3}'.format(
table_endpoint, table_key, column_name, description)
_log_put_column_description(table_key=table_key,
column_name=column_name,
description=description,
source=src)
response = request_wrapper(
method='PUT',
url=url,
client=app.config['METADATASERVICE_REQUEST_CLIENT'],
headers=app.config['METADATASERVICE_REQUEST_HEADERS'],
timeout_sec=REQUEST_SESSION_TIMEOUT)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
else:
message = 'Update column description failed'
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def update_bookmark() -> Response:
"""
Call metadata service to PUT or DELETE a bookmark
Params
:param type: Resource type for the bookmarked item. e.g. 'table'
:param key: Resource key for the bookmarked item.
:return:
"""
@action_logging
def _log_update_bookmark(*, resource_key: str, resource_type: str,
method: str) -> None:
pass # pragma: no cover
try:
if app.config['AUTH_USER_METHOD']:
user = app.config['AUTH_USER_METHOD'](app)
else:
raise Exception('AUTH_USER_METHOD is not configured')
args = request.get_json()
resource_type = get_query_param(args, 'type')
resource_key = get_query_param(args, 'key')
url = '{0}{1}/{2}/follow/{3}/{4}'.format(
app.config['METADATASERVICE_BASE'], USER_ENDPOINT, user.user_id,
resource_type, resource_key)
_log_update_bookmark(resource_key=resource_key,
resource_type=resource_type,
method=request.method)
response = request_metadata(url=url, method=request.method)
status_code = response.status_code
return make_response(
jsonify({
'msg': 'success',
'response': response.json()
}), status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
return make_response(jsonify({'msg': message}),
HTTPStatus.INTERNAL_SERVER_ERROR)
def popular_resources() -> Response:
"""
call the metadata service endpoint to get the current popular tables, dashboards etc.
this takes a required query parameter "types", that is a comma separated string of requested resource types
:return: a json output containing an array of popular table metadata as 'popular_tables'
Schema Defined Here:
https://github.com/lyft/amundsenmetadatalibrary/blob/master/metadata_service/api/popular_tables.py
"""
try:
if app.config['AUTH_USER_METHOD'] and app.config[
'POPULAR_RESOURCES_PERSONALIZATION']:
user_id = app.config['AUTH_USER_METHOD'](app).user_id
else:
user_id = ''
resource_types = get_query_param(request.args, 'types')
service_base = app.config['METADATASERVICE_BASE']
count = app.config['POPULAR_RESOURCES_COUNT']
url = f'{service_base}{POPULAR_RESOURCES_ENDPOINT}/{user_id}?limit={count}&types={resource_types}'
response = request_metadata(url=url)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
json_response = response.json()
tables = json_response.get(ResourceType.Table.name, [])
popular_tables = [
marshall_table_partial(result) for result in tables
]
dashboards = json_response.get(ResourceType.Dashboard.name, [])
popular_dashboards = [
marshall_dashboard_partial(dashboard)
for dashboard in dashboards
]
else:
message = 'Encountered error: Request to metadata service failed with status code ' + str(
status_code)
logging.error(message)
popular_tables = []
popular_dashboards = []
all_popular_resources = {
to_label(resource_type=ResourceType.Table): popular_tables,
to_label(resource_type=ResourceType.Dashboard): popular_dashboards
}
payload = jsonify({'results': all_popular_resources, 'msg': message})
return make_response(payload, status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
payload = jsonify({'results': [{}], 'msg': message})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def update_table_tags() -> Response:
@action_logging
def _log_update_table_tags(*, table_key: str, method: str,
tag: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
method = request.method
table_endpoint = _get_table_endpoint()
table_key = _get_table_key(args)
tag = get_query_param(args, 'tag')
url = '{0}/{1}/tag/{2}'.format(table_endpoint, table_key, tag)
_log_update_table_tags(table_key=table_key, method=method, tag=tag)
# TODO: Create an abstraction for this logic that is reused many times
envoy_client = app.config['METADATASERVICE_REQUEST_CLIENT']
if envoy_client is not None:
envoy_headers = app.config['METADATASERVICE_REQUEST_HEADERS']
if method == 'PUT':
response = envoy_client.put(url,
headers=envoy_headers,
raw_response=True)
else:
response = envoy_client.delete(url,
headers=envoy_headers,
raw_response=True)
else:
with requests.Session() as s:
if method == 'PUT':
response = s.put(url, timeout=REQUEST_SESSION_TIMEOUT)
else:
response = s.delete(url, timeout=REQUEST_SESSION_TIMEOUT)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
else:
message = 'Encountered error: {0} tag failed'.format(method)
logging.error(message)
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
payload = jsonify({'msg': message})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
