def _search_table(*, search_term: str, page_index: int, filters: Dict, search_type: str) -> Dict[str, Any]: """ Call the search service endpoint and return matching results Search service logic defined here: https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/table.py :return: a json output containing search results array as 'results' """ # Default results tables = { 'page_index': int(page_index), 'results': [], 'total_results': 0, } results_dict = { 'search_term': search_term, 'msg': '', 'tables': tables, } try: if has_filters(filters=filters): query_json = generate_query_json(filters=filters, page_index=page_index, search_term=search_term) url_base = app.config[ 'SEARCHSERVICE_BASE'] + SEARCH_TABLE_FILTER_ENDPOINT response = request_search( url=url_base, headers={'Content-Type': 'application/json'}, method='POST', data=json.dumps(query_json)) else: url_base = app.config['SEARCHSERVICE_BASE'] + SEARCH_TABLE_ENDPOINT url = f'{url_base}?query_term={search_term}&page_index={page_index}' response = request_search(url=url) status_code = response.status_code if status_code == HTTPStatus.OK: results_dict['msg'] = 'Success' results = response.json().get('results') tables['results'] = [ map_table_result(result) for result in results ] tables['total_results'] = response.json().get('total_results') else: message = 'Encountered error: Search request failed' results_dict['msg'] = message logging.error(message) results_dict['status_code'] = status_code return results_dict except Exception as e: message = 'Encountered exception: ' + str(e) results_dict['msg'] = message logging.exception(message) return results_dict
def search_table_query_string() -> Response: """ TODO (ttannis): Update this docstring after amundsensearch documentation is merged Calls the search service to execute a search. The request data is transformed to the json payload defined [link] """ request_json = request.get_json() search_term = get_query_param(request_json, 'term', '"term" parameter expected in request data') page_index = get_query_param(request_json, 'pageIndex', '"pageIndex" parameter expected in request data') filters = request_json.get('filters', {}) # Default results tables = { 'page_index': int(page_index), 'results': [], 'total_results': 0, } results_dict = { 'search_term': search_term, 'msg': '', 'tables': tables, } try: query_json = generate_query_json(filters=filters, page_index=page_index, search_term=search_term) except Exception as e: message = 'Encountered exception generating query json: ' + str(e) results_dict['msg'] = message logging.exception(message) return make_response(jsonify(results_dict), HTTPStatus.INTERNAL_SERVER_ERROR) try: # TODO (ttannis): Change actual endpoint name after amundsensearch PR is merged url = app.config['SEARCHSERVICE_BASE'] + '/search_table' response = request_search(url=url, headers={'Content-Type': 'application/json'}, method='POST', data=json.dumps(query_json)) status_code = response.status_code if status_code == HTTPStatus.OK: results_dict['msg'] = 'Success' results = response.json().get('results') tables['results'] = [map_table_result(result) for result in results] tables['total_results'] = response.json().get('total_results') else: message = 'Encountered error: Search request failed' results_dict['msg'] = message logging.error(message) results_dict['status_code'] = status_code return make_response(jsonify(results_dict), status_code) except Exception as e: message = 'Encountered exception: ' + str(e) results_dict['msg'] = message logging.exception(message) return make_response(jsonify(results_dict), HTTPStatus.INTERNAL_SERVER_ERROR)
def _search_user(*, search_term: str, page_index: int) -> Dict[str, Any]: """ call the search service endpoint and return matching results :return: a json output containing search results array as 'results' Schema Defined Here: https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/user.py TODO: Define an interface for envoy_client """ def _map_user_result(result: Dict) -> Dict: user_result = dump_user(load_user(result)) user_result['type'] = 'user' return user_result users = { 'page_index': int(page_index), 'results': [], 'total_results': 0, } results_dict = { 'search_term': search_term, 'msg': 'Success', 'status_code': HTTPStatus.OK, 'users': users, } try: url = '{0}?query_term={1}&page_index={2}'.format(app.config['SEARCHSERVICE_BASE'] + SEARCH_USER_ENDPOINT, search_term, page_index) response = request_search(url=url) status_code = response.status_code if status_code == HTTPStatus.OK: results_dict['msg'] = 'Success' results = response.json().get('results') users['results'] = [_map_user_result(result) for result in results] users['total_results'] = response.json().get('total_results') else: message = 'Encountered error: Search request failed' results_dict['msg'] = message logging.error(message) results_dict['status_code'] = status_code return results_dict except Exception as e: message = 'Encountered exception: ' + str(e) results_dict['msg'] = message logging.exception(message) return results_dict
def _search_user(*, search_term: str, page_index: int, search_type: str) -> Dict[str, Any]: """ Call the search service endpoint and return matching results Search service logic defined here: https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/user.py :return: a json output containing search results array as 'results' """ def _map_user_result(result: Dict) -> Dict: user_result = dump_user(load_user(result)) user_result['type'] = 'user' return user_result users = { 'page_index': page_index, 'results': [], 'total_results': 0, } results_dict = { 'search_term': search_term, 'msg': 'Success', 'status_code': HTTPStatus.OK, 'users': users, } try: url_base = app.config['SEARCHSERVICE_BASE'] + SEARCH_USER_ENDPOINT url = f'{url_base}?query_term={search_term}&page_index={page_index}' response = request_search(url=url) status_code = response.status_code if status_code == HTTPStatus.OK: results_dict['msg'] = 'Success' results = response.json().get('results') users['results'] = [_map_user_result(result) for result in results] users['total_results'] = response.json().get('total_results') else: message = 'Encountered error: Search request failed' results_dict['msg'] = message logging.error(message) results_dict['status_code'] = status_code return results_dict except Exception as e: message = 'Encountered exception: ' + str(e) results_dict['msg'] = message results_dict['status_code'] = HTTPStatus.INTERNAL_SERVER_ERROR logging.exception(message) return results_dict
def _search_table(*, search_term: str, page_index: int) -> Dict[str, Any]: """ call the search service endpoint and return matching results :return: a json output containing search results array as 'results' Schema Defined Here: https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/search.py TODO: Define an interface for envoy_client """ tables = { 'page_index': int(page_index), 'results': [], 'total_results': 0, } results_dict = { 'search_term': search_term, 'msg': '', 'tables': tables, } try: if ':' in search_term: url = _create_url_with_field(search_term=search_term, page_index=page_index) else: url = '{0}?query_term={1}&page_index={2}'.format(app.config['SEARCHSERVICE_BASE'] + SEARCH_ENDPOINT, search_term, page_index) response = request_search(url=url) status_code = response.status_code if status_code == HTTPStatus.OK: results_dict['msg'] = 'Success' results = response.json().get('results') tables['results'] = [map_table_result(result) for result in results] tables['total_results'] = response.json().get('total_results') else: message = 'Encountered error: Search request failed' results_dict['msg'] = message logging.error(message) results_dict['status_code'] = status_code return results_dict except Exception as e: message = 'Encountered exception: ' + str(e) results_dict['msg'] = message logging.exception(message) return results_dict
def _search_dashboard(*, search_term: str, page_index: int, search_type: str) -> Dict[str, Any]: """ Call the search service endpoint and return matching results Search service logic defined here: https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/dashboard.py :return: a json output containing search results array as 'results' """ # Default results dashboards = { 'page_index': page_index, 'results': [], 'total_results': 0, } results_dict = { 'search_term': search_term, 'msg': '', 'dashboards': dashboards, } try: url_base = app.config['SEARCHSERVICE_BASE'] + SEARCH_DASHBOARD_ENDPOINT url = f'{url_base}?query_term={search_term}&page_index={page_index}' response = request_search(url=url) status_code = response.status_code if status_code == HTTPStatus.OK: results_dict['msg'] = 'Success' results = response.json().get('results') dashboards['results'] = [ marshall_dashboard_partial(result) for result in results ] dashboards['total_results'] = response.json().get('total_results') else: message = 'Encountered error: Search request failed' results_dict['msg'] = message logging.error(message) results_dict['status_code'] = status_code return results_dict except Exception as e: message = 'Encountered exception: ' + str(e) results_dict['msg'] = message logging.exception(message) return results_dict
def execute_search_document_request(request_json: str, method: str) -> int: search_service_base = app.config['SEARCHSERVICE_BASE'] search_document_url = f'{search_service_base}/v2/document' update_response = request_search( url=search_document_url, method=method, headers={'Content-Type': 'application/json'}, data=request_json, ) status_code = update_response.status_code if status_code != HTTPStatus.OK: LOGGER.info( f'Failed to execute {method} for {request_json} in searchservice, status code: {status_code}' ) LOGGER.info(update_response.text) return status_code
def _search_resources(*, search_term: str, resources: List[str], page_index: int, results_per_page: int, filters: Dict, search_type: str) -> Dict[str, Any]: """ Call the search service endpoint and return matching results :return: a json output containing search results array as 'results' """ default_results = { 'page_index': int(page_index), 'results': [], 'total_results': 0, } results_dict = { 'search_term': search_term, 'msg': '', 'table': default_results, 'dashboard': default_results, 'feature': default_results, 'user': default_results, } try: transformed_filters = _transform_filters(filters=filters, resources=resources) query_request = generate_query_request( filters=transformed_filters, resources=resources, page_index=page_index, results_per_page=results_per_page, search_term=search_term) request_json = json.dumps(SearchRequestSchema().dump(query_request)) url_base = app.config['SEARCHSERVICE_BASE'] + SEARCH_ENDPOINT response = request_search(url=url_base, headers={'Content-Type': 'application/json'}, method='POST', data=request_json) status_code = response.status_code if status_code == HTTPStatus.OK: search_response = SearchResponseSchema().loads( json.dumps(response.json())) results_dict['msg'] = search_response.msg results = search_response.results for resource in results.keys(): results_dict[resource] = { 'page_index': int(page_index), 'results': [ RESOURCE_TO_MAPPING[resource](result) for result in results[resource]['results'] ], 'total_results': results[resource]['total_results'], } else: message = 'Encountered error: Search request failed' results_dict['msg'] = message results_dict['status_code'] = status_code return results_dict except Exception as e: message = f'Encountered exception: {str(e)}' results_dict['msg'] = message LOGGER.exception(message) return results_dict
def _update_search_tag(table_key: str, method: str, tag: str) -> int: """ call the search service endpoint to get whole table information uniquely identified by table_key update tags list, call search service endpoint again to write back the updated field TODO: we should update dashboard tag in the future :param table_key: table key e.g. 'database://cluster.schema/table' :param method: PUT or DELETE :param tag: tag name to be put/delete :return: HTTP status code """ searchservice_base = app.config['SEARCHSERVICE_BASE'] searchservice_get_table_url = f'{searchservice_base}/search_table' # searchservice currently doesn't allow colon or / inside filters, thus can't get item based on key # table key e.g: 'database://cluster.schema/table' table_uri = TableUri.from_uri(table_key) request_param_map = { 'search_request': { 'type': 'AND', 'filters': { 'database': [table_uri.database], 'schema': [table_uri.schema], 'table': [table_uri.table], 'cluster': [table_uri.cluster] } }, 'query_term': '' } get_table_response = request_search(url=searchservice_get_table_url, method='POST', json=request_param_map) get_status_code = get_table_response.status_code if get_status_code != HTTPStatus.OK: LOGGER.info( f'Fail to get table info from serviceservice, http status code: {get_status_code}' ) LOGGER.debug(get_table_response.text) return get_status_code raw_data_map = json.loads(get_table_response.text) # key is unique, thus (database, cluster, schema, table) should uniquely identify the table if len(raw_data_map['results']) > 1: LOGGER.error(f'Error! Duplicate table key: {table_key}') table = raw_data_map['results'][0] old_tags_list = table['tags'] new_tags_list = [item for item in old_tags_list if item['tag_name'] != tag] if method != 'DELETE': new_tags_list.append({'tag_name': tag}) table['tags'] = new_tags_list # remove None values pruned_table = {k: v for k, v in table.items() if v is not None} post_param_map = {"data": pruned_table} searchservice_update_url = f'{searchservice_base}/document_table' update_table_response = request_search(url=searchservice_update_url, method='PUT', json=post_param_map) update_status_code = update_table_response.status_code if update_status_code != HTTPStatus.OK: LOGGER.info( f'Fail to update table info in searchservice, http status code: {update_status_code}' ) LOGGER.debug(update_table_response.text) return update_table_response.status_code return HTTPStatus.OK