def get_preview_data(self, params: Dict, optionalHeaders: Dict = None) -> FlaskResponse:
"""
Returns a FlaskResponse object, where the response data represents a json object
with the preview data accessible on 'preview_data' key. The preview data should
match amundsen_application.models.preview_data.PreviewDataSchema
"""
try:
# Clone headers so that it does not mutate instance's state
headers = dict(self.headers)
# Merge optionalHeaders into headers
if optionalHeaders is not None:
headers.update(optionalHeaders)
# Request preview data
response = self.post_to_sql_json(params=params, headers=headers)
# Verify and return the results
response_dict = response.json()
columns = [ColumnItem(c['name'], c['type']) for c in response_dict['columns']]
preview_data = PreviewData(columns, response_dict['data'])
try:
data = PreviewDataSchema().dump(preview_data)
PreviewDataSchema().load(data) # for validation only
payload = jsonify({'preview_data': data})
return make_response(payload, response.status_code)
except ValidationError as err:
logging.error("PreviewDataSchema serialization error " + str(err.messages))
return make_response(jsonify({'preview_data': {}}), HTTPStatus.INTERNAL_SERVER_ERROR)
except Exception:
return make_response(jsonify({'preview_data': {}}), HTTPStatus.INTERNAL_SERVER_ERROR)
def get_preview_data(self,
params: Dict,
optionalHeaders: Dict = None) -> Response:
"""Preview data from Dremio source
"""
database = params.get('database')
if database != 'DREMIO':
logging.info('Skipping table preview for non-Dremio table')
return make_response(jsonify({'preview_data': {}}), HTTPStatus.OK)
try:
# Format base SQL_STATEMENT with request table and schema
schema = '"{}"'.format(params['schema'].replace('.', '"."'))
table = params['tableName']
sql = DremioPreviewClient.SQL_STATEMENT.format(schema=schema,
table=table)
client = flight.FlightClient(self.url, **self.connection_args)
client.authenticate(
_DremioAuthHandler(self.username, self.password))
flight_descriptor = flight.FlightDescriptor.for_command(sql)
flight_info = client.get_flight_info(flight_descriptor)
reader = client.do_get(flight_info.endpoints[0].ticket)
result = reader.read_all()
names = result.schema.names
types = result.schema.types
columns = map(lambda x: x.to_pylist(), result.columns)
rows = [dict(zip(names, row)) for row in zip(*columns)]
column_items = [ColumnItem(n, t) for n, t in zip(names, types)]
preview_data = PreviewData(column_items, rows)
try:
data = PreviewDataSchema().dump(preview_data)
PreviewDataSchema().load(data) # for validation only
payload = jsonify({'preview_data': data})
return make_response(payload, HTTPStatus.OK)
except ValidationError as err:
logging.error(
f'Error(s) occurred while building preview data: {err.messages}'
)
payload = jsonify({'preview_data': {}})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
except Exception as e:
logging.error(f'Encountered exception: {e}')
payload = jsonify({'preview_data': {}})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_preview_data(self,
params: Dict,
optionalHeaders: Dict = None) -> FlaskResponse:
"""
Returns a FlaskResponse object, where the response data represents a json object
with the preview data accessible on 'preview_data' key. The preview data should
match amundsen_application.models.preview_data.PreviewDataSchema
"""
LOGGER.debug('Retrieving preview data from Redash with params: %s',
params)
try:
query_id = self.get_redash_query_id(params)
if query_id is None:
raise RedashQueryTemplateDoesNotExistForResource(
'Could not find query for params: %s', params)
# Build headers to use the Query API key or User API key
self._build_headers(params)
query_params = self.build_redash_query_params(params)
query_results, cached_result = self._start_redash_query(
query_id=query_id, query_params=query_params)
# Redash attempts to use internal caching. The format of the response
# changes based on whether or not a cached response is returned
if not cached_result:
query_result_id = self._wait_for_query_finish(
job_id=query_results['job']['id'])
query_results = self._get_query_results(
query_result_id=query_result_id)
columns = [
ColumnItem(c['name'], c['type'])
for c in query_results['query_result']['data']['columns']
]
preview_data = PreviewData(
columns, query_results['query_result']['data']['rows'])
data = PreviewDataSchema().dump(preview_data)
PreviewDataSchema().load(data) # for validation only
payload = jsonify({'preview_data': data})
return make_response(payload, HTTPStatus.OK)
except Exception as e:
LOGGER.error('ERROR getting Redash preview: %s', e)
return make_response(jsonify({'preview_data': {}}),
HTTPStatus.INTERNAL_SERVER_ERROR)
def get_preview_data(self, params: Dict, optionalHeaders: Dict = None) -> FlaskResponse:
"""
Returns a FlaskResponse object, where the response data represents a json object
with the preview data accessible on 'preview_data' key. The preview data should
match amundsen_application.models.preview_data.PreviewDataSchema
"""
try:
# Clone headers so that it does not mutate instance's state
headers = dict(self.headers)
# Merge optionalHeaders into headers
if optionalHeaders is not None:
headers.update(optionalHeaders)
# Request preview data
response = self.post_to_sql_json(params=params, headers=headers)
# Verify and return the results
response_dict = response.json()
logging.info(f"Response:{response_dict}")
# Modified portion
for column in response_dict['columns']:
if(column['type'] == None):
column['type'] = 'STRING'
columns = [ColumnItem(c['name'], c['type']) for c in response_dict['columns']]
preview_data = PreviewData(columns, response_dict['data'])
data = PreviewDataSchema().dump(preview_data)[0]
errors = PreviewDataSchema().load(data)[1]
logging.info(f"Errors:{errors}")
if not errors:
payload = jsonify({'preview_data': data})
return make_response(payload, response.status_code)
else:
return make_response(jsonify({'preview_data': {}}), HTTPStatus.INTERNAL_SERVER_ERROR)
except Exception as e:
return make_response(jsonify({'preview_data': {}}), HTTPStatus.INTERNAL_SERVER_ERROR)
def get_table_preview() -> Response:
global PREVIEW_CLIENT_INSTANCE
global PREVIEW_CLIENT_CLASS
try:
if PREVIEW_CLIENT_INSTANCE is None:
if PREVIEW_CLIENT_CLASS is not None:
PREVIEW_CLIENT_INSTANCE = PREVIEW_CLIENT_CLASS()
logging.warn(
'Setting preview_client via entry_point is DEPRECATED and '
'will be removed in a future version')
elif (app.config['PREVIEW_CLIENT_ENABLED']
and app.config['PREVIEW_CLIENT'] is not None):
PREVIEW_CLIENT_CLASS = import_string(
app.config['PREVIEW_CLIENT'])
PREVIEW_CLIENT_INSTANCE = PREVIEW_CLIENT_CLASS()
else:
payload = jsonify({
'previewData': {},
'msg':
'A client for the preview feature must be configured'
})
return make_response(payload, HTTPStatus.NOT_IMPLEMENTED)
response = PREVIEW_CLIENT_INSTANCE.get_preview_data(
params=request.get_json())
status_code = response.status_code
preview_data = json.loads(response.data).get('preview_data')
if status_code == HTTPStatus.OK:
# validate the returned table preview data
data, errors = PreviewDataSchema().load(preview_data)
if not errors:
payload = jsonify({'previewData': data, 'msg': 'Success'})
else:
logging.error('Preview data dump returned errors: ' +
str(errors))
raise Exception(
'The preview client did not return a valid PreviewData object'
)
else:
message = 'Encountered error: Preview client request failed with code ' + str(
status_code)
logging.error(message)
# only necessary to pass the error text
payload = jsonify({
'previewData': {
'error_text': preview_data.get('error_text', '')
},
'msg': message
})
return make_response(payload, status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
payload = jsonify({'previewData': {}, 'msg': message})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
def get_preview_data(self,
params: Dict,
optionalHeaders: Dict = None) -> FlaskResponse:
try:
preview_data = self.get_s3_preview_data(params=params)
try:
data = PreviewDataSchema().dump(preview_data)
PreviewDataSchema().load(data) # for validation only
payload = jsonify({'preview_data': data})
return make_response(payload, HTTPStatus.OK)
except ValidationError as err:
logging.error("PreviewDataSchema serialization error " +
str(err.messages))
return make_response(jsonify({'preview_data': {}}),
HTTPStatus.INTERNAL_SERVER_ERROR)
except Exception as err:
logging.error("error getting s3 preview data " + str(err))
return make_response(jsonify({'preview_data': {}}),
HTTPStatus.INTERNAL_SERVER_ERROR)
def get_preview_data(self,
params: Dict,
optionalHeaders: Dict = None) -> Response:
if self.previewable_projects and params[
"cluster"] not in self.previewable_projects:
return make_response(jsonify({"preview_data": {}}),
HTTPStatus.FORBIDDEN)
preview_data = self._bq_list_rows(
params["cluster"],
params["schema"],
params["tableName"],
)
data = PreviewDataSchema().dump(preview_data)[0]
errors = PreviewDataSchema().load(data)[1]
payload = jsonify({"preview_data": data})
if not errors:
payload = jsonify({"preview_data": data})
return make_response(payload, HTTPStatus.OK)
return make_response(jsonify({"preview_data": {}}),
HTTPStatus.INTERNAL_SERVER_ERROR)
def get_preview_data(self,
params: Dict,
optionalHeaders: Dict = None) -> Response:
if self.previewable_projects and params[
"cluster"] not in self.previewable_projects:
return make_response(jsonify({"preview_data": {}}),
HTTPStatus.FORBIDDEN)
preview_data = self._bq_list_rows(
params["cluster"],
params["schema"],
params["tableName"],
)
try:
data = PreviewDataSchema().dump(preview_data)
PreviewDataSchema().load(data) # for validation only
payload = jsonify({"preview_data": data})
return make_response(payload, HTTPStatus.OK)
except ValidationError as err:
logging.error("PreviewDataSchema serialization error + " +
str(err.messages))
return make_response(jsonify({"preview_data": {}}),
HTTPStatus.INTERNAL_SERVER_ERROR)
def get_table_preview() -> Response:
# TODO: Want to further separate this file into more blueprints, perhaps a Blueprint class is the way to go
global PREVIEW_CLIENT_INSTANCE
try:
if PREVIEW_CLIENT_INSTANCE is None and PREVIEW_CLIENT_CLASS is not None:
PREVIEW_CLIENT_INSTANCE = PREVIEW_CLIENT_CLASS()
if PREVIEW_CLIENT_INSTANCE is None:
payload = jsonify({
'previewData': {},
'msg':
'A client for the preview feature must be configured'
})
return make_response(payload, HTTPStatus.NOT_IMPLEMENTED)
# request table preview data
response = PREVIEW_CLIENT_INSTANCE.get_preview_data(
params=request.get_json())
status_code = response.status_code
preview_data = json.loads(response.data).get('preview_data')
if status_code == HTTPStatus.OK:
# validate the returned table preview data
data, errors = PreviewDataSchema().load(preview_data)
if not errors:
payload = jsonify({'previewData': data, 'msg': 'Success'})
else:
logging.error('Preview data dump returned errors: ' +
str(errors))
raise Exception(
'The preview client did not return a valid PreviewData object'
)
else:
message = 'Encountered error: Preview client request failed with code ' + str(
status_code)
logging.error(message)
# only necessary to pass the error text
payload = jsonify({
'previewData': {
'error_text': preview_data.get('error_text', '')
},
'msg': message
})
return make_response(payload, status_code)
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
payload = jsonify({'previewData': {}, 'msg': message})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
