def _update_table_owner(*, table_key: str, method: str, owner: str) -> Dict[str, str]:
try:
table_endpoint = _get_table_endpoint()
url = '{0}/{1}/owner/{2}'.format(table_endpoint, table_key, owner)
request_wrapper(method=method,
url=url,
client=app.config['METADATASERVICE_REQUEST_CLIENT'],
headers=app.config['METADATASERVICE_REQUEST_HEADERS'],
timeout_sec=REQUEST_SESSION_TIMEOUT)
# TODO: Figure out a way to get this payload from flask.jsonify which wraps with app's response_class
return {'msg': 'Updated owner'}
except Exception as e:
return {'msg': 'Encountered exception: ' + str(e)}
def get_tags() -> Response:
"""
call the metadata service endpoint to get the list of all tags from neo4j
:return: a json output containing the list of all tags, as 'tags'
Schema Defined Here: https://github.com/lyft/amundsenmetadatalibrary/blob/master/metadata_service/api/tag.py
"""
try:
url = app.config['METADATASERVICE_BASE'] + TAGS_ENDPOINT
response = request_wrapper(
method='GET',
url=url,
client=app.config['METADATASERVICE_REQUEST_CLIENT'],
headers=app.config['METADATASERVICE_REQUEST_HEADERS'],
timeout_sec=REQUEST_SESSION_TIMEOUT)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
tags = response.json().get('tag_usages')
else:
message = 'Encountered error: Tags Unavailable'
logging.error(message)
tags = []
payload = jsonify({'tags': tags, 'msg': message})
return make_response(payload, status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
payload = jsonify({'tags': [], 'msg': message})
logging.exception(message)
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_column_description() -> Response:
try:
table_endpoint = _get_table_endpoint()
table_key = _get_table_key(request.args)
column_name = get_query_param(request.args, 'column_name')
url = '{0}/{1}/column/{2}/description'.format(table_endpoint,
table_key, column_name)
response = request_wrapper(
method='GET',
url=url,
client=app.config['METADATASERVICE_REQUEST_CLIENT'],
headers=app.config['METADATASERVICE_REQUEST_HEADERS'],
timeout_sec=REQUEST_SESSION_TIMEOUT)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
description = response.json().get('description')
else:
message = 'Get column description failed'
description = None
payload = jsonify({'description': description, 'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({
'description': None,
'msg': 'Encountered exception: ' + str(e)
})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_last_indexed() -> Response:
"""
call the metadata service endpoint to get the last indexed timestamp of neo4j
:return: a json output containing the last indexed timestamp, in unix epoch time, as 'timestamp'
Schema Defined Here: https://github.com/lyft/amundsenmetadatalibrary/blob/master/metadata_service/api/system.py
"""
try:
url = app.config['METADATASERVICE_BASE'] + LAST_INDEXED_ENDPOINT
response = request_wrapper(
method='GET',
url=url,
client=app.config['METADATASERVICE_REQUEST_CLIENT'],
headers=app.config['METADATASERVICE_REQUEST_HEADERS'],
timeout_sec=REQUEST_SESSION_TIMEOUT)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
timestamp = response.json().get('neo4j_latest_timestamp')
else:
message = 'Timestamp Unavailable'
timestamp = None
payload = jsonify({'timestamp': timestamp, 'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({
'timestamp': None,
'msg': 'Encountered exception: ' + str(e)
})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def _send_metadata_get_request(url: str) -> Response:
return request_wrapper(
method='GET',
url=url,
client=app.config['METADATASERVICE_REQUEST_CLIENT'],
headers=app.config['METADATASERVICE_REQUEST_HEADERS'],
timeout_sec=REQUEST_SESSION_TIMEOUT)
def popular_tables() -> Response:
"""
call the metadata service endpoint to get the current popular tables
:return: a json output containing an array of popular table metadata as 'popular_tables'
Schema Defined Here:
https://github.com/lyft/amundsenmetadatalibrary/blob/master/metadata_service/api/popular_tables.py
"""
def _map_results(result: Dict) -> Dict:
table_name = result.get('table_name')
schema_name = result.get('schema')
cluster = result.get('cluster')
db = result.get('database')
return {
'name': table_name,
'schema_name': schema_name,
'cluster': cluster,
'database': db,
'description': result.get('table_description'),
'key': '{0}://{1}.{2}/{3}'.format(db, cluster, schema_name,
table_name),
'type': 'table',
}
try:
url = app.config['METADATASERVICE_BASE'] + POPULAR_TABLES_ENDPOINT
response = request_wrapper(
method='GET',
url=url,
client=app.config['METADATASERVICE_REQUEST_CLIENT'],
headers=app.config['METADATASERVICE_REQUEST_HEADERS'],
timeout_sec=REQUEST_SESSION_TIMEOUT)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
response_list = response.json().get('popular_tables')
top4 = response_list[0:min(len(response_list), app.
config['POPULAR_TABLE_COUNT'])]
popular_tables = [_map_results(result) for result in top4]
else:
message = 'Encountered error: Request to metadata service failed with status code ' + str(
status_code)
logging.error(message)
popular_tables = [{}]
payload = jsonify({'results': popular_tables, 'msg': message})
return make_response(payload, status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
payload = jsonify({'results': [{}], 'msg': message})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def put_column_description() -> Response:
@action_logging
def _log_put_column_description(*, table_key: str, column_name: str,
description: str, source: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
table_key = _get_table_key(args)
table_endpoint = _get_table_endpoint()
column_name = get_query_param(args, 'column_name')
description = get_query_param(args, 'description')
description = ' '.join(description.split())
src = get_query_param(args, 'source')
url = '{0}/{1}/column/{2}/description/{3}'.format(
table_endpoint, table_key, column_name, description)
_log_put_column_description(table_key=table_key,
column_name=column_name,
description=description,
source=src)
response = request_wrapper(
method='PUT',
url=url,
client=app.config['METADATASERVICE_REQUEST_CLIENT'],
headers=app.config['METADATASERVICE_REQUEST_HEADERS'],
timeout_sec=REQUEST_SESSION_TIMEOUT)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
else:
message = 'Update column description failed'
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
payload = jsonify({'msg': 'Encountered exception: ' + str(e)})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def update_table_tags() -> Response:
@action_logging
def _log_update_table_tags(*, table_key: str, method: str,
tag: str) -> None:
pass # pragma: no cover
try:
args = request.get_json()
method = request.method
table_endpoint = _get_table_endpoint()
table_key = _get_table_key(args)
tag = get_query_param(args, 'tag')
url = '{0}/{1}/tag/{2}'.format(table_endpoint, table_key, tag)
_log_update_table_tags(table_key=table_key, method=method, tag=tag)
response = request_wrapper(
method=method,
url=url,
client=app.config['METADATASERVICE_REQUEST_CLIENT'],
headers=app.config['METADATASERVICE_REQUEST_HEADERS'],
timeout_sec=REQUEST_SESSION_TIMEOUT)
status_code = response.status_code
if status_code == HTTPStatus.OK:
message = 'Success'
else:
message = 'Encountered error: {0} tag failed'.format(method)
logging.error(message)
payload = jsonify({'msg': message})
return make_response(payload, status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
payload = jsonify({'msg': message})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def _search_table(*, search_term: str, page_index: int) -> Dict[str, Any]:
"""
call the search service endpoint and return matching results
:return: a json output containing search results array as 'results'
Schema Defined Here: https://github.com/lyft/
amundsensearchlibrary/blob/master/search_service/api/search.py
TODO: Define an interface for envoy_client
"""
def _map_table_result(result: Dict) -> Dict:
return {
'type': 'table',
'key': result.get('key', None),
'name': result.get('name', None),
'cluster': result.get('cluster', None),
'description': result.get('description', None),
'database': result.get('database', None),
'schema_name': result.get('schema_name', None),
'last_updated_epoch': result.get('last_updated_epoch', None),
}
tables = {
'page_index': int(page_index),
'results': [],
'total_results': 0,
}
results_dict = {
'search_term': search_term,
'msg': '',
'tables': tables,
}
try:
if ':' in search_term:
url = _create_url_with_field(search_term=search_term,
page_index=page_index)
else:
url = '{0}?query_term={1}&page_index={2}'.format(
app.config['SEARCHSERVICE_BASE'] + SEARCH_ENDPOINT,
search_term, page_index)
response = request_wrapper(
method='GET',
url=url,
client=app.config['SEARCHSERVICE_REQUEST_CLIENT'],
headers=app.config['SEARCHSERVICE_REQUEST_HEADERS'],
timeout_sec=REQUEST_SESSION_TIMEOUT)
status_code = response.status_code
if status_code == HTTPStatus.OK:
results_dict['msg'] = 'Success'
results = response.json().get('results')
tables['results'] = [
_map_table_result(result) for result in results
]
tables['total_results'] = response.json().get('total_results')
else:
message = 'Encountered error: Search request failed'
results_dict['msg'] = message
logging.error(message)
results_dict['status_code'] = status_code
return results_dict
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
return results_dict