def _update_table_owner(*, table_key: str, method: str, owner: str) -> Dict[str, str]: try: table_endpoint = _get_table_endpoint() url = '{0}/{1}/owner/{2}'.format(table_endpoint, table_key, owner) request_wrapper(method=method, url=url, client=app.config['METADATASERVICE_REQUEST_CLIENT'], headers=app.config['METADATASERVICE_REQUEST_HEADERS'], timeout_sec=REQUEST_SESSION_TIMEOUT) # TODO: Figure out a way to get this payload from flask.jsonify which wraps with app's response_class return {'msg': 'Updated owner'} except Exception as e: return {'msg': 'Encountered exception: ' + str(e)}
def get_tags() -> Response: """ call the metadata service endpoint to get the list of all tags from neo4j :return: a json output containing the list of all tags, as 'tags' Schema Defined Here: https://github.com/lyft/amundsenmetadatalibrary/blob/master/metadata_service/api/tag.py """ try: url = app.config['METADATASERVICE_BASE'] + TAGS_ENDPOINT response = request_wrapper( method='GET', url=url, client=app.config['METADATASERVICE_REQUEST_CLIENT'], headers=app.config['METADATASERVICE_REQUEST_HEADERS'], timeout_sec=REQUEST_SESSION_TIMEOUT) status_code = response.status_code if status_code == HTTPStatus.OK: message = 'Success' tags = response.json().get('tag_usages') else: message = 'Encountered error: Tags Unavailable' logging.error(message) tags = [] payload = jsonify({'tags': tags, 'msg': message}) return make_response(payload, status_code) except Exception as e: message = 'Encountered exception: ' + str(e) payload = jsonify({'tags': [], 'msg': message}) logging.exception(message) return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_column_description() -> Response: try: table_endpoint = _get_table_endpoint() table_key = _get_table_key(request.args) column_name = get_query_param(request.args, 'column_name') url = '{0}/{1}/column/{2}/description'.format(table_endpoint, table_key, column_name) response = request_wrapper( method='GET', url=url, client=app.config['METADATASERVICE_REQUEST_CLIENT'], headers=app.config['METADATASERVICE_REQUEST_HEADERS'], timeout_sec=REQUEST_SESSION_TIMEOUT) status_code = response.status_code if status_code == HTTPStatus.OK: message = 'Success' description = response.json().get('description') else: message = 'Get column description failed' description = None payload = jsonify({'description': description, 'msg': message}) return make_response(payload, status_code) except Exception as e: payload = jsonify({ 'description': None, 'msg': 'Encountered exception: ' + str(e) }) return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_last_indexed() -> Response: """ call the metadata service endpoint to get the last indexed timestamp of neo4j :return: a json output containing the last indexed timestamp, in unix epoch time, as 'timestamp' Schema Defined Here: https://github.com/lyft/amundsenmetadatalibrary/blob/master/metadata_service/api/system.py """ try: url = app.config['METADATASERVICE_BASE'] + LAST_INDEXED_ENDPOINT response = request_wrapper( method='GET', url=url, client=app.config['METADATASERVICE_REQUEST_CLIENT'], headers=app.config['METADATASERVICE_REQUEST_HEADERS'], timeout_sec=REQUEST_SESSION_TIMEOUT) status_code = response.status_code if status_code == HTTPStatus.OK: message = 'Success' timestamp = response.json().get('neo4j_latest_timestamp') else: message = 'Timestamp Unavailable' timestamp = None payload = jsonify({'timestamp': timestamp, 'msg': message}) return make_response(payload, status_code) except Exception as e: payload = jsonify({ 'timestamp': None, 'msg': 'Encountered exception: ' + str(e) }) return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def _send_metadata_get_request(url: str) -> Response: return request_wrapper( method='GET', url=url, client=app.config['METADATASERVICE_REQUEST_CLIENT'], headers=app.config['METADATASERVICE_REQUEST_HEADERS'], timeout_sec=REQUEST_SESSION_TIMEOUT)
def popular_tables() -> Response: """ call the metadata service endpoint to get the current popular tables :return: a json output containing an array of popular table metadata as 'popular_tables' Schema Defined Here: https://github.com/lyft/amundsenmetadatalibrary/blob/master/metadata_service/api/popular_tables.py """ def _map_results(result: Dict) -> Dict: table_name = result.get('table_name') schema_name = result.get('schema') cluster = result.get('cluster') db = result.get('database') return { 'name': table_name, 'schema_name': schema_name, 'cluster': cluster, 'database': db, 'description': result.get('table_description'), 'key': '{0}://{1}.{2}/{3}'.format(db, cluster, schema_name, table_name), 'type': 'table', } try: url = app.config['METADATASERVICE_BASE'] + POPULAR_TABLES_ENDPOINT response = request_wrapper( method='GET', url=url, client=app.config['METADATASERVICE_REQUEST_CLIENT'], headers=app.config['METADATASERVICE_REQUEST_HEADERS'], timeout_sec=REQUEST_SESSION_TIMEOUT) status_code = response.status_code if status_code == HTTPStatus.OK: message = 'Success' response_list = response.json().get('popular_tables') top4 = response_list[0:min(len(response_list), app. config['POPULAR_TABLE_COUNT'])] popular_tables = [_map_results(result) for result in top4] else: message = 'Encountered error: Request to metadata service failed with status code ' + str( status_code) logging.error(message) popular_tables = [{}] payload = jsonify({'results': popular_tables, 'msg': message}) return make_response(payload, status_code) except Exception as e: message = 'Encountered exception: ' + str(e) logging.exception(message) payload = jsonify({'results': [{}], 'msg': message}) return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def put_column_description() -> Response: @action_logging def _log_put_column_description(*, table_key: str, column_name: str, description: str, source: str) -> None: pass # pragma: no cover try: args = request.get_json() table_key = _get_table_key(args) table_endpoint = _get_table_endpoint() column_name = get_query_param(args, 'column_name') description = get_query_param(args, 'description') description = ' '.join(description.split()) src = get_query_param(args, 'source') url = '{0}/{1}/column/{2}/description/{3}'.format( table_endpoint, table_key, column_name, description) _log_put_column_description(table_key=table_key, column_name=column_name, description=description, source=src) response = request_wrapper( method='PUT', url=url, client=app.config['METADATASERVICE_REQUEST_CLIENT'], headers=app.config['METADATASERVICE_REQUEST_HEADERS'], timeout_sec=REQUEST_SESSION_TIMEOUT) status_code = response.status_code if status_code == HTTPStatus.OK: message = 'Success' else: message = 'Update column description failed' payload = jsonify({'msg': message}) return make_response(payload, status_code) except Exception as e: payload = jsonify({'msg': 'Encountered exception: ' + str(e)}) return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def update_table_tags() -> Response: @action_logging def _log_update_table_tags(*, table_key: str, method: str, tag: str) -> None: pass # pragma: no cover try: args = request.get_json() method = request.method table_endpoint = _get_table_endpoint() table_key = _get_table_key(args) tag = get_query_param(args, 'tag') url = '{0}/{1}/tag/{2}'.format(table_endpoint, table_key, tag) _log_update_table_tags(table_key=table_key, method=method, tag=tag) response = request_wrapper( method=method, url=url, client=app.config['METADATASERVICE_REQUEST_CLIENT'], headers=app.config['METADATASERVICE_REQUEST_HEADERS'], timeout_sec=REQUEST_SESSION_TIMEOUT) status_code = response.status_code if status_code == HTTPStatus.OK: message = 'Success' else: message = 'Encountered error: {0} tag failed'.format(method) logging.error(message) payload = jsonify({'msg': message}) return make_response(payload, status_code) except Exception as e: message = 'Encountered exception: ' + str(e) logging.exception(message) payload = jsonify({'msg': message}) return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def _search_table(*, search_term: str, page_index: int) -> Dict[str, Any]: """ call the search service endpoint and return matching results :return: a json output containing search results array as 'results' Schema Defined Here: https://github.com/lyft/ amundsensearchlibrary/blob/master/search_service/api/search.py TODO: Define an interface for envoy_client """ def _map_table_result(result: Dict) -> Dict: return { 'type': 'table', 'key': result.get('key', None), 'name': result.get('name', None), 'cluster': result.get('cluster', None), 'description': result.get('description', None), 'database': result.get('database', None), 'schema_name': result.get('schema_name', None), 'last_updated_epoch': result.get('last_updated_epoch', None), } tables = { 'page_index': int(page_index), 'results': [], 'total_results': 0, } results_dict = { 'search_term': search_term, 'msg': '', 'tables': tables, } try: if ':' in search_term: url = _create_url_with_field(search_term=search_term, page_index=page_index) else: url = '{0}?query_term={1}&page_index={2}'.format( app.config['SEARCHSERVICE_BASE'] + SEARCH_ENDPOINT, search_term, page_index) response = request_wrapper( method='GET', url=url, client=app.config['SEARCHSERVICE_REQUEST_CLIENT'], headers=app.config['SEARCHSERVICE_REQUEST_HEADERS'], timeout_sec=REQUEST_SESSION_TIMEOUT) status_code = response.status_code if status_code == HTTPStatus.OK: results_dict['msg'] = 'Success' results = response.json().get('results') tables['results'] = [ _map_table_result(result) for result in results ] tables['total_results'] = response.json().get('total_results') else: message = 'Encountered error: Search request failed' results_dict['msg'] = message logging.error(message) results_dict['status_code'] = status_code return results_dict except Exception as e: message = 'Encountered exception: ' + str(e) results_dict['msg'] = message logging.exception(message) return results_dict