def _execute_notebook(request, notebook, snippet): response = {'status': -1} result = None history = None historify = (notebook['type'] != 'notebook' or snippet.get('wasBatchExecuted') ) and not notebook.get('skipHistorify') try: try: session = notebook.get('sessions') and notebook['sessions'][ 0] # Session reference for snippet execution without persisting it if historify: history = _historify(notebook, request.user) notebook = Notebook(document=history).get_data() interpreter = get_api(request, snippet) if snippet.get('interface') == 'sqlalchemy': interpreter.options['session'] = session response['handle'] = interpreter.execute(notebook, snippet) # Retrieve and remove the result from the handle if response['handle'].get('sync'): result = response['handle'].pop('result') finally: if historify: _snippet = [ s for s in notebook['snippets'] if s['id'] == snippet['id'] ][0] if 'handle' in response: # No failure _snippet['result']['handle'] = response['handle'] _snippet['result']['statements_count'] = response[ 'handle'].get('statements_count', 1) _snippet['result']['statement_id'] = response[ 'handle'].get('statement_id', 0) _snippet['result']['handle']['statement'] = response[ 'handle'].get('statement', snippet['statement']).strip( ) # For non HS2, as non multi query yet else: _snippet['status'] = 'failed' if history: # If _historify failed, history will be None history.update_data(notebook) history.save() response['history_id'] = history.id response['history_uuid'] = history.uuid if notebook[ 'isSaved']: # Keep track of history of saved queries response[ 'history_parent_uuid'] = history.dependencies.filter( type__startswith='query-').latest( 'last_modified').uuid except QueryError, ex: # We inject the history information from _historify() to the failed queries if response.get('history_id'): ex.extra['history_id'] = response['history_id'] if response.get('history_uuid'): ex.extra['history_uuid'] = response['history_uuid'] if response.get('history_parent_uuid'): ex.extra['history_parent_uuid'] = response['history_parent_uuid'] raise ex
def export_result(request): response = {'status': -1, 'message': _('Success')} # Passed by check_document_access_permission but unused by APIs notebook = json.loads(request.POST.get('notebook', '{}')) snippet = json.loads(request.POST.get('snippet', '{}')) data_format = json.loads(request.POST.get('format', '"hdfs-file"')) destination = urllib.unquote( json.loads(request.POST.get('destination', '""'))) overwrite = json.loads(request.POST.get('overwrite', 'false')) is_embedded = json.loads(request.POST.get('is_embedded', 'false')) start_time = json.loads(request.POST.get('start_time', '-1')) api = get_api(request, snippet) if data_format == 'hdfs-file': # Blocking operation, like downloading if request.fs.isdir(destination): if notebook.get('name'): destination += '/%(name)s.csv' % notebook else: destination += '/%(type)s-%(id)s.csv' % notebook if overwrite and request.fs.exists(destination): request.fs.do_as_user(request.user.username, request.fs.rmtree, destination) response['watch_url'] = api.export_data_as_hdfs_file( snippet, destination, overwrite) response['status'] = 0 request.audit = { 'operation': 'EXPORT', 'operationText': 'User %s exported to HDFS destination: %s' % (request.user.username, destination), 'allowed': True } elif data_format == 'hive-table': if is_embedded: sql, success_url = api.export_data_as_table( notebook, snippet, destination) task = make_notebook(name=_('Export %s query to table %s') % (snippet['type'], destination), description=_('Query %s to %s') % (_get_snippet_name(notebook), success_url), editor_type=snippet['type'], statement=sql, status='ready', database=snippet['database'], on_success_url=success_url, last_executed=start_time, is_task=True) response = task.execute(request) else: notebook_id = notebook['id'] or request.GET.get( 'editor', request.GET.get('notebook')) response['watch_url'] = reverse( 'notebook:execute_and_watch' ) + '?action=save_as_table¬ebook=' + str( notebook_id) + '&snippet=0&destination=' + destination response['status'] = 0 request.audit = { 'operation': 'EXPORT', 'operationText': 'User %s exported to Hive table: %s' % (request.user.username, destination), 'allowed': True } elif data_format == 'hdfs-directory': if is_embedded: sql, success_url = api.export_large_data_to_hdfs( notebook, snippet, destination) task = make_notebook(name=_('Export %s query to directory') % snippet['type'], description=_('Query %s to %s') % (_get_snippet_name(notebook), success_url), editor_type=snippet['type'], statement=sql, status='ready-execute', database=snippet['database'], on_success_url=success_url, last_executed=start_time, is_task=True) response = task.execute(request) else: notebook_id = notebook['id'] or request.GET.get( 'editor', request.GET.get('notebook')) response['watch_url'] = reverse( 'notebook:execute_and_watch' ) + '?action=insert_as_query¬ebook=' + str( notebook_id) + '&snippet=0&destination=' + destination response['status'] = 0 request.audit = { 'operation': 'EXPORT', 'operationText': 'User %s exported to HDFS directory: %s' % (request.user.username, destination), 'allowed': True } elif data_format in ('search-index', 'dashboard'): # Open the result in the Dashboard via a SQL sub-query or the Import wizard (quick vs scalable) if is_embedded: notebook_id = notebook['id'] or request.GET.get( 'editor', request.GET.get('notebook')) if data_format == 'dashboard': engine = notebook['type'].replace('query-', '') response['watch_url'] = reverse( 'dashboard:browse', kwargs={ 'name': notebook_id }) + '?source=query&engine=%(engine)s' % { 'engine': engine } response['status'] = 0 else: sample = get_api(request, snippet).fetch_result(notebook, snippet, rows=4, start_over=True) for col in sample['meta']: col['type'] = HiveFormat.FIELD_TYPE_TRANSLATE.get( col['type'], 'string') response['status'] = 0 response['id'] = notebook_id response['name'] = _get_snippet_name(notebook) response['source_type'] = 'query' response['target_type'] = 'index' response['target_path'] = destination response['sample'] = list(sample['data']) response['columns'] = [ Field(col['name'], col['type']).to_dict() for col in sample['meta'] ] else: notebook_id = notebook['id'] or request.GET.get( 'editor', request.GET.get('notebook')) response['watch_url'] = reverse( 'notebook:execute_and_watch' ) + '?action=index_query¬ebook=' + str( notebook_id) + '&snippet=0&destination=' + destination response['status'] = 0 if response.get('status') != 0: response['message'] = _('Exporting result failed.') return JsonResponse(response)
try: response['result'].append( get_api(request, session).close_session(session)) except QueryExpired: pass except Exception, e: LOG.exception('Error closing session %s' % str(e)) for snippet in [ _s for _s in notebook['snippets'] if _s['type'] in ('hive', 'impala') ]: try: if snippet['status'] != 'running': response['result'].append( get_api(request, snippet).close_statement(notebook, snippet)) else: LOG.info('Not closing SQL snippet as still running.') except QueryExpired: pass except Exception, e: LOG.exception('Error closing statement %s' % str(e)) response['status'] = 0 response['message'] = _('Notebook closed successfully') return JsonResponse(response) @require_POST @check_document_access_permission()