def _search_table(*, search_term: str, page_index: int, filters: Dict,
search_type: str) -> Dict[str, Any]:
"""
Call the search service endpoint and return matching results
Search service logic defined here:
https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/table.py
:return: a json output containing search results array as 'results'
"""
# Default results
tables = {
'page_index': int(page_index),
'results': [],
'total_results': 0,
}
results_dict = {
'search_term': search_term,
'msg': '',
'tables': tables,
}
try:
if has_filters(filters=filters):
query_json = generate_query_json(filters=filters,
page_index=page_index,
search_term=search_term)
url_base = app.config[
'SEARCHSERVICE_BASE'] + SEARCH_TABLE_FILTER_ENDPOINT
response = request_search(
url=url_base,
headers={'Content-Type': 'application/json'},
method='POST',
data=json.dumps(query_json))
else:
url_base = app.config['SEARCHSERVICE_BASE'] + SEARCH_TABLE_ENDPOINT
url = f'{url_base}?query_term={search_term}&page_index={page_index}'
response = request_search(url=url)
status_code = response.status_code
if status_code == HTTPStatus.OK:
results_dict['msg'] = 'Success'
results = response.json().get('results')
tables['results'] = [
map_table_result(result) for result in results
]
tables['total_results'] = response.json().get('total_results')
else:
message = 'Encountered error: Search request failed'
results_dict['msg'] = message
logging.error(message)
results_dict['status_code'] = status_code
return results_dict
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
return results_dict
def search_table_query_string() -> Response:
"""
TODO (ttannis): Update this docstring after amundsensearch documentation is merged
Calls the search service to execute a search. The request data is transformed
to the json payload defined [link]
"""
request_json = request.get_json()
search_term = get_query_param(request_json, 'term', '"term" parameter expected in request data')
page_index = get_query_param(request_json, 'pageIndex', '"pageIndex" parameter expected in request data')
filters = request_json.get('filters', {})
# Default results
tables = {
'page_index': int(page_index),
'results': [],
'total_results': 0,
}
results_dict = {
'search_term': search_term,
'msg': '',
'tables': tables,
}
try:
query_json = generate_query_json(filters=filters, page_index=page_index, search_term=search_term)
except Exception as e:
message = 'Encountered exception generating query json: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
return make_response(jsonify(results_dict), HTTPStatus.INTERNAL_SERVER_ERROR)
try:
# TODO (ttannis): Change actual endpoint name after amundsensearch PR is merged
url = app.config['SEARCHSERVICE_BASE'] + '/search_table'
response = request_search(url=url,
headers={'Content-Type': 'application/json'},
method='POST',
data=json.dumps(query_json))
status_code = response.status_code
if status_code == HTTPStatus.OK:
results_dict['msg'] = 'Success'
results = response.json().get('results')
tables['results'] = [map_table_result(result) for result in results]
tables['total_results'] = response.json().get('total_results')
else:
message = 'Encountered error: Search request failed'
results_dict['msg'] = message
logging.error(message)
results_dict['status_code'] = status_code
return make_response(jsonify(results_dict), status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
return make_response(jsonify(results_dict), HTTPStatus.INTERNAL_SERVER_ERROR)
def _search_user(*, search_term: str, page_index: int) -> Dict[str, Any]:
"""
call the search service endpoint and return matching results
:return: a json output containing search results array as 'results'
Schema Defined Here:
https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/user.py
TODO: Define an interface for envoy_client
"""
def _map_user_result(result: Dict) -> Dict:
user_result = dump_user(load_user(result))
user_result['type'] = 'user'
return user_result
users = {
'page_index': int(page_index),
'results': [],
'total_results': 0,
}
results_dict = {
'search_term': search_term,
'msg': 'Success',
'status_code': HTTPStatus.OK,
'users': users,
}
try:
url = '{0}?query_term={1}&page_index={2}'.format(app.config['SEARCHSERVICE_BASE'] + SEARCH_USER_ENDPOINT,
search_term,
page_index)
response = request_search(url=url)
status_code = response.status_code
if status_code == HTTPStatus.OK:
results_dict['msg'] = 'Success'
results = response.json().get('results')
users['results'] = [_map_user_result(result) for result in results]
users['total_results'] = response.json().get('total_results')
else:
message = 'Encountered error: Search request failed'
results_dict['msg'] = message
logging.error(message)
results_dict['status_code'] = status_code
return results_dict
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
return results_dict
def _search_user(*, search_term: str, page_index: int,
search_type: str) -> Dict[str, Any]:
"""
Call the search service endpoint and return matching results
Search service logic defined here:
https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/user.py
:return: a json output containing search results array as 'results'
"""
def _map_user_result(result: Dict) -> Dict:
user_result = dump_user(load_user(result))
user_result['type'] = 'user'
return user_result
users = {
'page_index': page_index,
'results': [],
'total_results': 0,
}
results_dict = {
'search_term': search_term,
'msg': 'Success',
'status_code': HTTPStatus.OK,
'users': users,
}
try:
url_base = app.config['SEARCHSERVICE_BASE'] + SEARCH_USER_ENDPOINT
url = f'{url_base}?query_term={search_term}&page_index={page_index}'
response = request_search(url=url)
status_code = response.status_code
if status_code == HTTPStatus.OK:
results_dict['msg'] = 'Success'
results = response.json().get('results')
users['results'] = [_map_user_result(result) for result in results]
users['total_results'] = response.json().get('total_results')
else:
message = 'Encountered error: Search request failed'
results_dict['msg'] = message
logging.error(message)
results_dict['status_code'] = status_code
return results_dict
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
results_dict['status_code'] = HTTPStatus.INTERNAL_SERVER_ERROR
logging.exception(message)
return results_dict
def _search_table(*, search_term: str, page_index: int) -> Dict[str, Any]:
"""
call the search service endpoint and return matching results
:return: a json output containing search results array as 'results'
Schema Defined Here:
https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/search.py
TODO: Define an interface for envoy_client
"""
tables = {
'page_index': int(page_index),
'results': [],
'total_results': 0,
}
results_dict = {
'search_term': search_term,
'msg': '',
'tables': tables,
}
try:
if ':' in search_term:
url = _create_url_with_field(search_term=search_term,
page_index=page_index)
else:
url = '{0}?query_term={1}&page_index={2}'.format(app.config['SEARCHSERVICE_BASE'] + SEARCH_ENDPOINT,
search_term,
page_index)
response = request_search(url=url)
status_code = response.status_code
if status_code == HTTPStatus.OK:
results_dict['msg'] = 'Success'
results = response.json().get('results')
tables['results'] = [map_table_result(result) for result in results]
tables['total_results'] = response.json().get('total_results')
else:
message = 'Encountered error: Search request failed'
results_dict['msg'] = message
logging.error(message)
results_dict['status_code'] = status_code
return results_dict
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
return results_dict
def _search_dashboard(*, search_term: str, page_index: int,
search_type: str) -> Dict[str, Any]:
"""
Call the search service endpoint and return matching results
Search service logic defined here:
https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/dashboard.py
:return: a json output containing search results array as 'results'
"""
# Default results
dashboards = {
'page_index': page_index,
'results': [],
'total_results': 0,
}
results_dict = {
'search_term': search_term,
'msg': '',
'dashboards': dashboards,
}
try:
url_base = app.config['SEARCHSERVICE_BASE'] + SEARCH_DASHBOARD_ENDPOINT
url = f'{url_base}?query_term={search_term}&page_index={page_index}'
response = request_search(url=url)
status_code = response.status_code
if status_code == HTTPStatus.OK:
results_dict['msg'] = 'Success'
results = response.json().get('results')
dashboards['results'] = [
marshall_dashboard_partial(result) for result in results
]
dashboards['total_results'] = response.json().get('total_results')
else:
message = 'Encountered error: Search request failed'
results_dict['msg'] = message
logging.error(message)
results_dict['status_code'] = status_code
return results_dict
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
return results_dict
def execute_search_document_request(request_json: str, method: str) -> int:
search_service_base = app.config['SEARCHSERVICE_BASE']
search_document_url = f'{search_service_base}/v2/document'
update_response = request_search(
url=search_document_url,
method=method,
headers={'Content-Type': 'application/json'},
data=request_json,
)
status_code = update_response.status_code
if status_code != HTTPStatus.OK:
LOGGER.info(
f'Failed to execute {method} for {request_json} in searchservice, status code: {status_code}'
)
LOGGER.info(update_response.text)
return status_code
def _search_resources(*, search_term: str, resources: List[str],
page_index: int, results_per_page: int, filters: Dict,
search_type: str) -> Dict[str, Any]:
"""
Call the search service endpoint and return matching results
:return: a json output containing search results array as 'results'
"""
default_results = {
'page_index': int(page_index),
'results': [],
'total_results': 0,
}
results_dict = {
'search_term': search_term,
'msg': '',
'table': default_results,
'dashboard': default_results,
'feature': default_results,
'user': default_results,
}
try:
transformed_filters = _transform_filters(filters=filters,
resources=resources)
query_request = generate_query_request(
filters=transformed_filters,
resources=resources,
page_index=page_index,
results_per_page=results_per_page,
search_term=search_term)
request_json = json.dumps(SearchRequestSchema().dump(query_request))
url_base = app.config['SEARCHSERVICE_BASE'] + SEARCH_ENDPOINT
response = request_search(url=url_base,
headers={'Content-Type': 'application/json'},
method='POST',
data=request_json)
status_code = response.status_code
if status_code == HTTPStatus.OK:
search_response = SearchResponseSchema().loads(
json.dumps(response.json()))
results_dict['msg'] = search_response.msg
results = search_response.results
for resource in results.keys():
results_dict[resource] = {
'page_index':
int(page_index),
'results': [
RESOURCE_TO_MAPPING[resource](result)
for result in results[resource]['results']
],
'total_results':
results[resource]['total_results'],
}
else:
message = 'Encountered error: Search request failed'
results_dict['msg'] = message
results_dict['status_code'] = status_code
return results_dict
except Exception as e:
message = f'Encountered exception: {str(e)}'
results_dict['msg'] = message
LOGGER.exception(message)
return results_dict
def _update_search_tag(table_key: str, method: str, tag: str) -> int:
"""
call the search service endpoint to get whole table information uniquely identified by table_key
update tags list, call search service endpoint again to write back the updated field
TODO: we should update dashboard tag in the future
:param table_key: table key e.g. 'database://cluster.schema/table'
:param method: PUT or DELETE
:param tag: tag name to be put/delete
:return: HTTP status code
"""
searchservice_base = app.config['SEARCHSERVICE_BASE']
searchservice_get_table_url = f'{searchservice_base}/search_table'
# searchservice currently doesn't allow colon or / inside filters, thus can't get item based on key
# table key e.g: 'database://cluster.schema/table'
table_uri = TableUri.from_uri(table_key)
request_param_map = {
'search_request': {
'type': 'AND',
'filters': {
'database': [table_uri.database],
'schema': [table_uri.schema],
'table': [table_uri.table],
'cluster': [table_uri.cluster]
}
},
'query_term': ''
}
get_table_response = request_search(url=searchservice_get_table_url,
method='POST',
json=request_param_map)
get_status_code = get_table_response.status_code
if get_status_code != HTTPStatus.OK:
LOGGER.info(
f'Fail to get table info from serviceservice, http status code: {get_status_code}'
)
LOGGER.debug(get_table_response.text)
return get_status_code
raw_data_map = json.loads(get_table_response.text)
# key is unique, thus (database, cluster, schema, table) should uniquely identify the table
if len(raw_data_map['results']) > 1:
LOGGER.error(f'Error! Duplicate table key: {table_key}')
table = raw_data_map['results'][0]
old_tags_list = table['tags']
new_tags_list = [item for item in old_tags_list if item['tag_name'] != tag]
if method != 'DELETE':
new_tags_list.append({'tag_name': tag})
table['tags'] = new_tags_list
# remove None values
pruned_table = {k: v for k, v in table.items() if v is not None}
post_param_map = {"data": pruned_table}
searchservice_update_url = f'{searchservice_base}/document_table'
update_table_response = request_search(url=searchservice_update_url,
method='PUT',
json=post_param_map)
update_status_code = update_table_response.status_code
if update_status_code != HTTPStatus.OK:
LOGGER.info(
f'Fail to update table info in searchservice, http status code: {update_status_code}'
)
LOGGER.debug(update_table_response.text)
return update_table_response.status_code
return HTTPStatus.OK
