def fetch_results(request, id, first_row=0): """ Returns the results of the QueryHistory with the given id. The query results MUST be ready. If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just spits out a warning if first_row doesn't match the servers conception. Multiple readers will produce a confusing interaction here, and that's known. """ first_row = long(first_row) results = type('Result', (object,), { 'rows': 0, 'columns': [], 'has_more': False, 'start_row': 0, }) fetch_error = False error_message = '' query_history = authorized_get_query_history(request, id, must_exist=True) query_server = query_history.get_query_server_config() design = SQLdesign.loads(query_history.design.data) db = dbms.get(request.user, query_server) try: database = design.query.get('database', 'default') db.use(database) datatable = db.execute_and_wait(design) results = db.client.create_result(datatable) status = 0 except Exception, e: fetch_error = True error_message = str(e) status = -1
def error_handler(view_fn): def decorator(request, *args, **kwargs): try: return view_fn(request, *args, **kwargs) except Http404, e: raise e except Exception, e: if not hasattr(e, 'message') or not e.message: message = str(e) else: message = force_unicode(e.message, strings_only=True, errors='replace') if 'Invalid OperationHandle' in message and 'id' in kwargs: # Expired state. query_history = authorized_get_query_history(request, kwargs['id'], must_exist=False) if query_history: query_history.set_to_expired() query_history.save() response = { 'status': -1, 'message': message, } if 'database is locked' in message or 'Invalid query handle' in message: response['status'] = 2 # Frontend will not display this type of error return HttpResponse(json.dumps(response), mimetype="application/json", status=200)
def error_handler(view_fn): def decorator(request, *args, **kwargs): try: return view_fn(request, *args, **kwargs) except Http404, e: raise e except Exception, e: LOG.exception('error in %s' % view_fn) if not hasattr(e, 'message') or not e.message: message = str(e) else: message = force_unicode(e.message, strings_only=True, errors='replace') if 'Invalid OperationHandle' in message and 'id' in kwargs: # Expired state. query_history = authorized_get_query_history(request, kwargs['id'], must_exist=False) if query_history: query_history.set_to_expired() query_history.save() response = { 'status': -1, 'message': message, } if re.search('database is locked|Invalid query handle|not JSON serializable', message, re.IGNORECASE): response['status'] = 2 # Frontend will not display this type of error LOG.warn('error_handler silencing the exception: %s' % e) return JsonResponse(response)
def save_results_hdfs_file(request, query_history_id): """ Save the results of a query to an HDFS file. Do not rerun the query. """ response = {'status': 0, 'message': ''} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method != 'POST': response['message'] = _('A POST request is required.') else: if not query_history.is_success(): response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state} response['status'] = -1 return JsonResponse(response) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsFileForm({ 'target_file': request.POST.get('path'), 'overwrite': request.POST.get('overwrite', False), }) if form.is_valid(): target_file = form.cleaned_data['target_file'] overwrite = form.cleaned_data['overwrite'] try: handle, state = _get_query_handle_and_state(query_history) except Exception, ex: response['message'] = _('Cannot find query handle and state: %s') % str(query_history) response['status'] = -2 return JsonResponse(response) try: if overwrite and request.fs.exists(target_file): if request.fs.isfile(target_file): request.fs.do_as_user(request.user.username, request.fs.rmtree, target_file) else: raise PopupException(_("The target path is a directory")) upload(target_file, handle, request.user, db, request.fs) response['type'] = 'hdfs-file' response['id'] = query_history.id response['query'] = query_history.query response['path'] = target_file response['success_url'] = '/filebrowser/view=%s' % target_file response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) except Exception, ex: error_msg, log = expand_exception(ex, db) response['message'] = _('The result could not be saved: %s.') % error_msg response['status'] = -3
def fetch_query_history(request, query_history_id): response = {'status': 0, 'message': ''} if request.method != 'GET': response['message'] = _('A GET request is required.') query = authorized_get_query_history(request, query_history_id, must_exist=True) response['query_history'] = query_history_to_dict(request, query) return JsonResponse(response)
def fetch_query_history(request, query_history_id): response = {'status': 0, 'message': ''} if request.method != 'GET': response['message'] = _('A GET request is required.') query = authorized_get_query_history(request, query_history_id, must_exist=True) response['query_history'] = query_history_to_dict(request, query) return HttpResponse(json.dumps(response), mimetype="application/json")
def watch_query_refresh_json(request, id): query_history = authorized_get_query_history(request, id, must_exist=True) db = dbms.get(request.user, query_history.get_query_server_config()) if not request.POST.get('next'): # We need this as multi query would fail as current query is closed handle, state = _get_query_handle_and_state(query_history) query_history.save_state(state) # Go to next statement if asked to continue or when a statement with no dataset finished. try: if request.POST.get('next') or (not query_history.is_finished() and query_history.is_success() and not query_history.has_results): close_operation(request, id) query_history = db.execute_next_statement(query_history, request.POST.get('query-query')) handle, state = _get_query_handle_and_state(query_history) except QueryServerException as ex: raise ex except Exception as ex: LOG.exception(ex) handle, state = _get_query_handle_and_state(query_history) try: start_over = request.POST.get('log-start-over') == 'true' log = db.get_log(handle, start_over=start_over) except Exception as ex: log = str(ex) jobs = parse_out_jobs(log) job_urls = massage_job_urls_for_json(jobs) result = { 'status': -1, 'log': log, 'jobs': jobs, 'jobUrls': job_urls, 'isSuccess': query_history.is_success(), 'isFailure': query_history.is_failure(), 'id': id, 'statement': query_history.get_current_statement(), 'watch_url': reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}), 'oldLogsApi': USE_GET_LOG_API.get() } # Run time error if query_history.is_failure(): res = db.get_operation_status(handle) if query_history.is_canceled(res): result['status'] = 0 elif hasattr(res, 'errorMessage') and res.errorMessage: result['message'] = res.errorMessage else: result['message'] = _('Bad status for request %s:\n%s') % (id, res) else: result['status'] = 0 return JsonResponse(result)
def save_results_hdfs_directory(request, query_history_id): """ Save the results of a query to an HDFS directory. Rerun the query. """ response = {'status': 0, 'message': ''} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method != 'POST': response['message'] = _('A POST request is required.') else: if not query_history.is_success(): response['message'] = _( 'This query is %(state)s. Results unavailable.') % { 'state': state } response['status'] = -1 return JsonResponse(response) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsDirectoryForm( {'target_dir': request.POST.get('path')}, fs=request.fs) if form.is_valid(): target_dir = request.POST.get('path') try: response['type'] = 'hdfs-dir' response['id'] = query_history.id response['query'] = query_history.query response['path'] = target_dir response['success_url'] = '/filebrowser/view=%s' % target_dir query_history = db.insert_query_into_directory( query_history, target_dir) response['watch_url'] = reverse( get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) except Exception as ex: error_msg, log = expand_exception(ex, db) response['message'] = _( 'The result could not be saved: %s.') % error_msg response['status'] = -3 else: response['status'] = 1 response['errors'] = form.errors return JsonResponse(response)
def save_results_hive_table(request, query_history_id): """ Save the results of a query to a hive table. Rerun the query. """ response = {'status': 0, 'message': ''} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method != 'POST': response['message'] = _('A POST request is required.') else: if not query_history.is_success(): response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state} response['status'] = -1 return JsonResponse(response) db = dbms.get(request.user, query_history.get_query_server_config()) database = query_history.design.get_design().query.get('database', 'default') form = beeswax.forms.SaveResultsTableForm({ 'target_table': request.POST.get('table') }, db=db, database=database) if form.is_valid(): try: handle, state = _get_query_handle_and_state(query_history) result_meta = db.get_results_metadata(handle) except Exception as ex: response['message'] = _('Cannot find query handle and state: %s') % str(query_history) response['status'] = -2 return JsonResponse(response) try: query_history = db.create_table_as_a_select(request, query_history, form.target_database, form.cleaned_data['target_table'], result_meta) response['id'] = query_history.id response['query'] = query_history.query response['type'] = 'hive-table' response['path'] = form.cleaned_data['target_table'] response['success_url'] = reverse('metastore:describe_table', kwargs={'database': form.target_database, 'table': form.cleaned_data['target_table']}) response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) except Exception as ex: error_msg, log = expand_exception(ex, db) response['message'] = _('The result could not be saved: %s.') % error_msg response['status'] = -3 else: response['status'] = 1 response['message'] = '\n'.join(list(form.errors.values())[0]) return JsonResponse(response)
def cancel_query(request, query_history_id): response = {'status': -1, 'message': ''} if request.method != 'POST': response['message'] = _('A POST request is required.') else: try: query_history = authorized_get_query_history(request, query_history_id, must_exist=True) db = dbms.get(request.user, query_history.get_query_server_config()) db.cancel_operation(query_history.get_handle()) query_history.set_to_expired() response['status'] = 0 except Exception, e: response['message'] = unicode(e)
def watch_query_refresh_json(request, id): query_history = authorized_get_query_history(request, id, must_exist=True) db = dbms.get(request.user, query_history.get_query_server_config()) if not request.POST.get('next'): # We need this as multi query would fail as current query is closed handle, state = _get_query_handle_and_state(query_history) query_history.save_state(state) # Go to next statement if asked to continue or when a statement with no dataset finished. try: if request.POST.get('next') or (not query_history.is_finished() and query_history.is_success() and not query_history.has_results): query_history = db.execute_next_statement(query_history, request.POST.get('query-query')) handle, state = _get_query_handle_and_state(query_history) except QueryServerException, ex: raise ex
def save_results_hive_table(request, query_history_id): """ Save the results of a query to a hive table. Rerun the query. """ response = {'status': 0, 'message': ''} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method != 'POST': response['message'] = _('A POST request is required.') else: if not query_history.is_success(): response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state} response['status'] = -1 return JsonResponse(response) db = dbms.get(request.user, query_history.get_query_server_config()) database = query_history.design.get_design().query.get('database', 'default') form = beeswax.forms.SaveResultsTableForm({ 'target_table': request.POST.get('table') }, db=db, database=database) if form.is_valid(): try: handle, state = _get_query_handle_and_state(query_history) result_meta = db.get_results_metadata(handle) except Exception, ex: response['message'] = _('Cannot find query handle and state: %s') % str(query_history) response['status'] = -2 return JsonResponse(response) try: query_history = db.create_table_as_a_select(request, query_history, form.target_database, form.cleaned_data['target_table'], result_meta) response['id'] = query_history.id response['query'] = query_history.query response['type'] = 'hive-table' response['path'] = form.cleaned_data['target_table'] response['success_url'] = reverse('metastore:describe_table', kwargs={'database': form.target_database, 'table': form.cleaned_data['target_table']}) response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) except Exception, ex: error_msg, log = expand_exception(ex, db) response['message'] = _('The result could not be saved: %s.') % error_msg response['status'] = -3
def close_operation(request, query_history_id): response = { 'status': -1, 'message': '' } if request.method != 'POST': response['message'] = _('A POST request is required.') else: query_history = authorized_get_query_history(request, query_history_id, must_exist=True) db = dbms.get(query_history.owner, query_history.get_query_server_config()) handle = query_history.get_handle() db.close_operation(handle) query_history.set_to_expired() query_history.save() response['status'] = 0 return JsonResponse(response)
def get_exec_summary(request, query_history_id): query_server = dbms.get_query_server_config() db = beeswax_dbms.get(request.user, query_server=query_server) response = {'status': -1} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) if query_history is None: response['message'] = _('get_exec_summary requires a valid query_history_id') else: session = Session.objects.get_session(request.user, query_server['server_name']) operation_handle = query_history.get_handle().get_rpc_handle() session_handle = session.get_handle() summary = db.get_exec_summary(operation_handle, session_handle) response['status'] = 0 response['summary'] = summary return JsonResponse(response)
def get_runtime_profile(request, query_history_id): query_server = dbms.get_query_server_config() db = beeswax_dbms.get(request.user, query_server=query_server) response = {'status': -1} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) if query_history is None: response['message'] = _('get_runtime_profile requires a valid query_history_id') else: session = Session.objects.get_session(request.user, query_server['server_name']) operation_handle = query_history.get_handle().get_rpc_handle() session_handle = session.get_handle() profile = db.get_runtime_profile(operation_handle, session_handle) response['status'] = 0 response['profile'] = profile return JsonResponse(response)
def fetch_results(request, id, first_row=0): """ Returns the results of the QueryHistory with the given id. The query results MUST be ready. If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just spits out a warning if first_row doesn't match the servers conception. Multiple readers will produce a confusing interaction here, and that's known. """ first_row = int(first_row) results = type('Result', (object,), { 'rows': 0, 'columns': [], 'has_more': False, 'start_row': 0, }) fetch_error = False error_message = '' query_history = authorized_get_query_history(request, id, must_exist=True) query_server = query_history.get_query_server_config() design = SQLdesign.loads(query_history.design.data) db = dbms.get(request.user, query_server) try: database = design.query.get('database', 'default') db.use(database) datatable = db.execute_and_wait(design) results = db.client.create_result(datatable) status = 0 except Exception as e: fetch_error = True error_message = str(e) status = -1 response = { 'status': status, 'message': fetch_error and error_message or '', 'results': results_to_dict(results) } return JsonResponse(response)
def save_results_hdfs_directory(request, query_history_id): """ Save the results of a query to an HDFS directory. Rerun the query. """ response = {'status': 0, 'message': ''} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method != 'POST': response['message'] = _('A POST request is required.') else: if not query_history.is_success(): response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state} response['status'] = -1 return JsonResponse(response) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsDirectoryForm({ 'target_dir': request.POST.get('path') }, fs=request.fs) if form.is_valid(): target_dir = request.POST.get('path') try: response['type'] = 'hdfs-dir' response['id'] = query_history.id response['query'] = query_history.query response['path'] = target_dir response['success_url'] = '/filebrowser/view=%s' % target_dir query_history = db.insert_query_into_directory(query_history, target_dir) response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) except Exception, ex: error_msg, log = expand_exception(ex, db) response['message'] = _('The result could not be saved: %s.') % error_msg response['status'] = -3 else:
def error_handler(view_fn): def decorator(request, *args, **kwargs): try: return view_fn(request, *args, **kwargs) except Http404, e: raise e except Exception, e: if not hasattr(e, 'message') or not e.message: message = str(e) else: message = force_unicode(e.message, strings_only=True, errors='replace') if 'Invalid OperationHandle' in message and 'id' in kwargs: # Expired state. query_history = authorized_get_query_history( request, kwargs['id'], must_exist=False) if query_history: query_history.set_to_expired() query_history.save() response = { 'status': -1, 'message': message, } if re.search( 'database is locked|Invalid query handle|not JSON serializable', message, re.IGNORECASE): response[ 'status'] = 2 # Frontend will not display this type of error LOG.warn('error_handler silencing the exception: %s' % e) return HttpResponse(json.dumps(response), mimetype="application/json", status=200)
def save_results(request, query_history_id): """ Save the results of a query to an HDFS directory or Hive table. """ response = {'status': 0, 'message': ''} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method != 'POST': response['message'] = _('A POST request is required.') else: if not query_history.is_success(): response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state} response['status'] = -1 return HttpResponse(json.dumps(response), mimetype="application/json") # massage data to work with old forms data = {} if request.POST.get('type') == 'hive-table': data['save_target'] = 'to a new table' data['target_table'] = request.POST.get('path', None) elif request.POST.get('type') == 'hdfs': data['save_target'] = 'to HDFS directory' data['target_dir'] = request.POST.get('path', None) else: data['save_target'] = None data['target_table'] = request.POST.get('path', None) data['target_dir'] = request.POST.get('path', None) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsForm(data, db=db, fs=request.fs) if form.is_valid(): try: handle, state = _get_query_handle_and_state(query_history) result_meta = db.get_results_metadata(handle) except Exception, ex: response['message'] = _('Cannot find query: %s') % {'state': state} response['status'] = -2 return HttpResponse(json.dumps(response), mimetype="application/json") try: if form.cleaned_data['save_target'] == form.SAVE_TYPE_DIR: target_dir = form.cleaned_data['target_dir'] query_history = db.insert_query_into_directory(query_history, target_dir) response['type'] = 'hdfs' response['id'] = query_history.id response['query'] = query_history.query response['path'] = target_dir response['success_url'] = '/filebrowser/view%s' % target_dir response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL: query_history = db.create_table_as_a_select(request, query_history, form.cleaned_data['target_table'], result_meta) response['id'] = query_history.id response['query'] = query_history.query response['type'] = 'hive-table' response['path'] = form.cleaned_data['target_table'] response['success_url'] = reverse('metastore:describe_table', kwargs={'database': 'default', 'table': form.cleaned_data['target_table']}) response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) except Exception, ex: error_msg, log = expand_exception(ex, db) response['message'] = _('The result could not be saved: %s.') % error_msg response['status'] = -3
def save_results_hdfs_file(request, query_history_id): """ Save the results of a query to an HDFS file. Do not rerun the query. """ response = {'status': 0, 'message': ''} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method != 'POST': response['message'] = _('A POST request is required.') else: if not query_history.is_success(): response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state} response['status'] = -1 return JsonResponse(response) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsFileForm({ 'target_file': request.POST.get('path'), 'overwrite': request.POST.get('overwrite', False), }) if form.is_valid(): target_file = form.cleaned_data['target_file'] overwrite = form.cleaned_data['overwrite'] try: handle, state = _get_query_handle_and_state(query_history) except Exception as ex: response['message'] = _('Cannot find query handle and state: %s') % str(query_history) response['status'] = -2 return JsonResponse(response) try: if overwrite and request.fs.exists(target_file): if request.fs.isfile(target_file): request.fs.do_as_user(request.user.username, request.fs.rmtree, target_file) else: raise PopupException(_("The target path is a directory")) upload(target_file, handle, request.user, db, request.fs) response['type'] = 'hdfs-file' response['id'] = query_history.id response['query'] = query_history.query response['path'] = target_file response['success_url'] = '/filebrowser/view=%s' % target_file response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) except Exception as ex: error_msg, log = expand_exception(ex, db) response['message'] = _('The result could not be saved: %s.') % error_msg response['status'] = -3 else: response['status'] = 1 response['errors'] = form.errors return JsonResponse(response)