def _execute_notebook(request, notebook, snippet):
response = {'status': -1}
result = None
history = None
historify = (notebook['type'] != 'notebook'
or snippet.get('wasBatchExecuted')
) and not notebook.get('skipHistorify')
try:
try:
session = notebook.get('sessions') and notebook['sessions'][
0] # Session reference for snippet execution without persisting it
if historify:
history = _historify(notebook, request.user)
notebook = Notebook(document=history).get_data()
interpreter = get_api(request, snippet)
if snippet.get('interface') == 'sqlalchemy':
interpreter.options['session'] = session
response['handle'] = interpreter.execute(notebook, snippet)
# Retrieve and remove the result from the handle
if response['handle'].get('sync'):
result = response['handle'].pop('result')
finally:
if historify:
_snippet = [
s for s in notebook['snippets'] if s['id'] == snippet['id']
][0]
if 'handle' in response: # No failure
_snippet['result']['handle'] = response['handle']
_snippet['result']['statements_count'] = response[
'handle'].get('statements_count', 1)
_snippet['result']['statement_id'] = response[
'handle'].get('statement_id', 0)
_snippet['result']['handle']['statement'] = response[
'handle'].get('statement', snippet['statement']).strip(
) # For non HS2, as non multi query yet
else:
_snippet['status'] = 'failed'
if history: # If _historify failed, history will be None
history.update_data(notebook)
history.save()
response['history_id'] = history.id
response['history_uuid'] = history.uuid
if notebook[
'isSaved']: # Keep track of history of saved queries
response[
'history_parent_uuid'] = history.dependencies.filter(
type__startswith='query-').latest(
'last_modified').uuid
except QueryError, ex: # We inject the history information from _historify() to the failed queries
if response.get('history_id'):
ex.extra['history_id'] = response['history_id']
if response.get('history_uuid'):
ex.extra['history_uuid'] = response['history_uuid']
if response.get('history_parent_uuid'):
ex.extra['history_parent_uuid'] = response['history_parent_uuid']
raise ex
def export_result(request):
response = {'status': -1, 'message': _('Success')}
# Passed by check_document_access_permission but unused by APIs
notebook = json.loads(request.POST.get('notebook', '{}'))
snippet = json.loads(request.POST.get('snippet', '{}'))
data_format = json.loads(request.POST.get('format', '"hdfs-file"'))
destination = urllib.unquote(
json.loads(request.POST.get('destination', '""')))
overwrite = json.loads(request.POST.get('overwrite', 'false'))
is_embedded = json.loads(request.POST.get('is_embedded', 'false'))
start_time = json.loads(request.POST.get('start_time', '-1'))
api = get_api(request, snippet)
if data_format == 'hdfs-file': # Blocking operation, like downloading
if request.fs.isdir(destination):
if notebook.get('name'):
destination += '/%(name)s.csv' % notebook
else:
destination += '/%(type)s-%(id)s.csv' % notebook
if overwrite and request.fs.exists(destination):
request.fs.do_as_user(request.user.username, request.fs.rmtree,
destination)
response['watch_url'] = api.export_data_as_hdfs_file(
snippet, destination, overwrite)
response['status'] = 0
request.audit = {
'operation':
'EXPORT',
'operationText':
'User %s exported to HDFS destination: %s' %
(request.user.username, destination),
'allowed':
True
}
elif data_format == 'hive-table':
if is_embedded:
sql, success_url = api.export_data_as_table(
notebook, snippet, destination)
task = make_notebook(name=_('Export %s query to table %s') %
(snippet['type'], destination),
description=_('Query %s to %s') %
(_get_snippet_name(notebook), success_url),
editor_type=snippet['type'],
statement=sql,
status='ready',
database=snippet['database'],
on_success_url=success_url,
last_executed=start_time,
is_task=True)
response = task.execute(request)
else:
notebook_id = notebook['id'] or request.GET.get(
'editor', request.GET.get('notebook'))
response['watch_url'] = reverse(
'notebook:execute_and_watch'
) + '?action=save_as_table¬ebook=' + str(
notebook_id) + '&snippet=0&destination=' + destination
response['status'] = 0
request.audit = {
'operation':
'EXPORT',
'operationText':
'User %s exported to Hive table: %s' %
(request.user.username, destination),
'allowed':
True
}
elif data_format == 'hdfs-directory':
if is_embedded:
sql, success_url = api.export_large_data_to_hdfs(
notebook, snippet, destination)
task = make_notebook(name=_('Export %s query to directory') %
snippet['type'],
description=_('Query %s to %s') %
(_get_snippet_name(notebook), success_url),
editor_type=snippet['type'],
statement=sql,
status='ready-execute',
database=snippet['database'],
on_success_url=success_url,
last_executed=start_time,
is_task=True)
response = task.execute(request)
else:
notebook_id = notebook['id'] or request.GET.get(
'editor', request.GET.get('notebook'))
response['watch_url'] = reverse(
'notebook:execute_and_watch'
) + '?action=insert_as_query¬ebook=' + str(
notebook_id) + '&snippet=0&destination=' + destination
response['status'] = 0
request.audit = {
'operation':
'EXPORT',
'operationText':
'User %s exported to HDFS directory: %s' %
(request.user.username, destination),
'allowed':
True
}
elif data_format in ('search-index', 'dashboard'):
# Open the result in the Dashboard via a SQL sub-query or the Import wizard (quick vs scalable)
if is_embedded:
notebook_id = notebook['id'] or request.GET.get(
'editor', request.GET.get('notebook'))
if data_format == 'dashboard':
engine = notebook['type'].replace('query-', '')
response['watch_url'] = reverse(
'dashboard:browse', kwargs={
'name': notebook_id
}) + '?source=query&engine=%(engine)s' % {
'engine': engine
}
response['status'] = 0
else:
sample = get_api(request,
snippet).fetch_result(notebook,
snippet,
rows=4,
start_over=True)
for col in sample['meta']:
col['type'] = HiveFormat.FIELD_TYPE_TRANSLATE.get(
col['type'], 'string')
response['status'] = 0
response['id'] = notebook_id
response['name'] = _get_snippet_name(notebook)
response['source_type'] = 'query'
response['target_type'] = 'index'
response['target_path'] = destination
response['sample'] = list(sample['data'])
response['columns'] = [
Field(col['name'], col['type']).to_dict()
for col in sample['meta']
]
else:
notebook_id = notebook['id'] or request.GET.get(
'editor', request.GET.get('notebook'))
response['watch_url'] = reverse(
'notebook:execute_and_watch'
) + '?action=index_query¬ebook=' + str(
notebook_id) + '&snippet=0&destination=' + destination
response['status'] = 0
if response.get('status') != 0:
response['message'] = _('Exporting result failed.')
return JsonResponse(response)
try:
response['result'].append(
get_api(request, session).close_session(session))
except QueryExpired:
pass
except Exception, e:
LOG.exception('Error closing session %s' % str(e))
for snippet in [
_s for _s in notebook['snippets']
if _s['type'] in ('hive', 'impala')
]:
try:
if snippet['status'] != 'running':
response['result'].append(
get_api(request,
snippet).close_statement(notebook, snippet))
else:
LOG.info('Not closing SQL snippet as still running.')
except QueryExpired:
pass
except Exception, e:
LOG.exception('Error closing statement %s' % str(e))
response['status'] = 0
response['message'] = _('Notebook closed successfully')
return JsonResponse(response)
@require_POST
@check_document_access_permission()