def view_results(request, id, first_row=0, last_result_len=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the watch_query view.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See watch_query().)
"""
# Coerce types; manage arguments
id = int(id)
first_row = long(first_row)
start_over = (first_row == 0)
query_history = authorized_get_history(request, id, must_exist=True)
handle = QueryHandle(id=query_history.server_id, log_context=query_history.log_context)
context = _parse_query_context(request.GET.get('context'))
# Retrieve query results
try:
results = db_utils.db_client(query_history.get_query_server()).fetch(handle, start_over, -1)
assert results.ready, _('Trying to display result that is not yet ready. Query id %(id)s') % {'id': id}
# We display the "Download" button only when we know
# that there are results:
downloadable = (first_row > 0 or len(results.data) > 0)
fetch_error = False
except BeeswaxException, ex:
fetch_error = True
error_message, log = expand_exception(ex)
def fetch_results(request, id, first_row=0):
"""
Returns the results of the QueryHistory with the given id.
The query results MUST be ready.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
"""
first_row = long(first_row)
results = type('Result', (object,), {
'rows': 0,
'columns': [],
'has_more': False,
'start_row': 0,
})
fetch_error = False
error_message = ''
query_history = authorized_get_history(request, id, must_exist=True)
query_server = query_history.get_query_server_config()
design = SQLdesign.loads(query_history.design.data)
db = dbms.get(request.user, query_server)
try:
database = design.query.get('database', 'default')
db.use(database)
datatable = db.execute_and_wait(design)
results = db.client.create_result(datatable)
status = 0
except Exception, e:
fetch_error = True
error_message = str(e)
status = -1
def watch_query(request, id):
"""
Wait for the query to finish and (by default) displays the results of query id.
It understands the optional GET params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
context
A string of "name:data" that describes the context
that generated this query result. It may be:
- "table":"<table_name>"
- "design":<design_id>
All other GET params will be passed to on_success_url (if present).
"""
# Coerce types; manage arguments
id = int(id)
query_history = authorized_get_history(request, id, must_exist=True)
# GET param: context.
context_param = request.GET.get('context', '')
# GET param: on_success_url. Default to view_results
results_url = urlresolvers.reverse(view_results, kwargs=dict(id=str(id), first_row=0, last_result_len=0))
on_success_url = request.GET.get('on_success_url')
if not on_success_url:
on_success_url = results_url
# Get the server_id
server_id, state = _get_server_id_and_state(query_history)
query_history.save_state(state)
# Query finished?
if state == QueryHistory.STATE.expired:
raise PopupException(_("The result of this query has expired."))
elif state == QueryHistory.STATE.available:
return format_preserving_redirect(request, on_success_url, request.GET)
elif state == QueryHistory.STATE.failed:
# When we fetch, Beeswax server will throw us a BeeswaxException, which has the
# log we want to display.
return format_preserving_redirect(request, results_url, request.GET)
# Still running
log = db_utils.db_client(query_history.get_query_server()).get_log(server_id)
# Keep waiting
# - Translate context into something more meaningful (type, data)
context = _parse_query_context(context_param)
return render('watch_wait.mako', request, {
'query': query_history,
'fwd_params': request.GET.urlencode(),
'log': log,
'hadoop_jobs': _parse_out_hadoop_jobs(log),
'query_context': context,
})
def cancel_query(request, query_id):
response = {'status': -1, 'message': ''}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
else:
try:
query_history = authorized_get_history(request, query_id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server_config())
db.cancel_operation(query_history.get_handle())
_get_query_handle_and_state(query_history)
response['status'] = 0
except Exception, e:
response['message'] = unicode(e)
def watch_query_refresh_json(request, id):
query_history = authorized_get_history(request, id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server_config())
if not request.POST.get('next'): # We need this as multi query would fail as current query is closed
handle, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
# Go to next statement if asked to continue or when a statement with no dataset finished.
try:
if request.POST.get('next') or (not query_history.is_finished() and query_history.is_success() and not query_history.has_results):
query_history = db.execute_next_statement(query_history)
handle, state = _get_query_handle_and_state(query_history)
except Exception, ex:
LOG.exception(ex)
handle, state = _get_query_handle_and_state(query_history)
def close_operation(request, query_id):
response = {
'status': -1,
'message': ''
}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
else:
try:
query_history = authorized_get_history(request, query_id, must_exist=True)
db = dbms.get(query_history.owner, query_history.get_query_server_config())
handle = query_history.get_handle()
db.close_operation(handle)
query_history.set_to_expired()
query_history.save()
response['status'] = 0
except Exception, e:
response['message'] = unicode(e)
def view_results(request, id, first_row=0, last_result_len=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the watch_query view.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See watch_query().)
"""
# Coerce types; manage arguments
id = int(id)
first_row = long(first_row)
start_over = (first_row == 0)
query_history = authorized_get_history(request, id, must_exist=True)
handle = QueryHandle(id=query_history.server_id,
log_context=query_history.log_context)
context = _parse_query_context(request.GET.get('context'))
# Retrieve query results
try:
results = db_utils.db_client(query_history.get_query_server()).fetch(
handle, start_over, -1)
assert results.ready, _(
'Trying to display result that is not yet ready. Query id %(id)s'
) % {
'id': id
}
# We display the "Download" button only when we know
# that there are results:
downloadable = (first_row > 0 or len(results.data) > 0)
fetch_error = False
except BeeswaxException, ex:
fetch_error = True
error_message, log = expand_exception(ex)
def watch_query_refresh_json(request, id):
query_history = authorized_get_history(request, id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server_config())
handle, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
# Show popup message if error, should be better in error tab instead and merged into the result response below
if query_history.is_failure():
res = db.get_operation_status(handle)
if hasattr(res, 'errorMessage') and res.errorMessage:
message = res.errorMessage
else:
message = ''
raise QueryServerException(Exception('Bad status for request %s:\n%s' % (id, res)), message=message)
# Multi query if more statements
try:
if not query_history.is_finished() and query_history.is_success() and not query_history.has_results:
db.execute_next_statement(query_history)
handle, state = _get_query_handle_and_state(query_history)
except Exception, ex:
LOG.exception(ex)
handle, state = _get_query_handle_and_state(query_history)
def save_results(request, query_id):
"""
Save the results of a query to an HDFS directory or Hive table.
"""
response = {'status': 0, 'message': ''}
query_history = authorized_get_history(request, query_id, must_exist=True)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method != 'POST':
response['message'] = _('A POST request is required.')
else:
if not query_history.is_success():
response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state}
response['status'] = -1
return HttpResponse(json.dumps(response), mimetype="application/json")
# massage data to work with old forms
data = {}
if request.POST.get('type') == 'hive-table':
data['save_target'] = 'to a new table'
data['target_table'] = request.POST.get('path', None)
elif request.POST.get('type') == 'hdfs':
data['save_target'] = 'to HDFS directory'
data['target_dir'] = request.POST.get('path', None)
else:
data['save_target'] = None
data['target_table'] = request.POST.get('path', None)
data['target_dir'] = request.POST.get('path', None)
db = dbms.get(request.user, query_history.get_query_server_config())
form = beeswax.forms.SaveResultsForm(data, db=db, fs=request.fs)
if form.is_valid():
try:
handle, state = _get_query_handle_and_state(query_history)
result_meta = db.get_results_metadata(handle)
except Exception, ex:
response['message'] = _('Cannot find query: %s') % {'state': state}
response['status'] = -2
return HttpResponse(json.dumps(response), mimetype="application/json")
try:
if form.cleaned_data['save_target'] == form.SAVE_TYPE_DIR:
target_dir = form.cleaned_data['target_dir']
query_history = db.insert_query_into_directory(query_history, target_dir)
response['type'] = 'hdfs'
response['id'] = query_history.id
response['query'] = query_history.query
response['path'] = target_dir
response['watch_url'] = reverse(get_app_name(request) + ':watch_query_refresh_json', kwargs={'id': query_history.id})
elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL:
db.create_table_as_a_select(request, query_history, form.cleaned_data['target_table'], result_meta)
response['type'] = 'hive-table'
response['path'] = form.cleaned_data['target_table']
except Exception, ex:
error_msg, log = expand_exception(ex, db)
response['message'] = _('The result could not be saved: %s.') % error_msg
response['status'] = -3
def view_results(request, id, first_row=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the watch_query view.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See watch_query().)
"""
first_row = long(first_row)
start_over = (first_row == 0)
results = None
data = None
fetch_error = False
error_message = ''
log = ''
app_name = get_app_name(request)
query_history = authorized_get_history(request, id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server_config())
handle, state = _get_query_handle_and_state(query_history)
context_param = request.GET.get('context', '')
query_context = _parse_query_context(context_param)
# To remove in Hue 2.3
download = request.GET.get('download', '')
# Update the status as expired should not be accessible
expired = state == QueryHistory.STATE.expired
if expired:
state = QueryHistory.STATE.expired
query_history.save_state(state)
# Retrieve query results
try:
if not download:
results = db.fetch(handle, start_over, 100)
data = list(results.rows()) # Materialize results
# We display the "Download" button only when we know that there are results:
downloadable = first_row > 0 or data
else:
downloadable = True
data = []
results = type('Result', (object,), {
'rows': 0,
'columns': [],
'has_more': False,
'start_row': 0, })
log = db.get_log(handle)
except Exception as ex:
fetch_error = True
error_message, log = expand_exception(ex, db)
# Handle errors
error = fetch_error or results is None or expired
context = {
'error': error,
'error_message': error_message,
'has_more': True,
'query': query_history,
'results': data,
'expected_first_row': first_row,
'log': log,
'hadoop_jobs': _parse_out_hadoop_jobs(log)[0],
'query_context': query_context,
'can_save': False,
'context_param': context_param,
'expired': expired,
'app_name': app_name,
'download': download,
}
if not error:
download_urls = {}
if downloadable:
for format in common.DL_FORMATS:
download_urls[format] = urlresolvers.reverse('beeswax' + ':download',
kwargs=dict(id=str(id), format=format))
save_form = SaveResultsForm()
results.start_row = first_row
context.update({
'results': data,
'has_more': results.has_more,
'next_row': results.start_row + len(data),
'start_row': results.start_row,
'expected_first_row': first_row,
'columns': results.columns,
'download_urls': download_urls,
'save_form': save_form,
'can_save': query_history.owner == request.user and not download,
})
return render('watch_results.mako', request, context)
def watch_query(request, id, download_format=None):
"""
Wait for the query to finish and (by default) displays the results of query id.
It understands the optional GET params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
context
A string of "name:data" that describes the context
that generated this query result. It may be:
- "table":"<table_name>"
- "design":<design_id>
All other GET params will be passed to on_success_url (if present).
"""
# Coerce types; manage arguments
query_history = authorized_get_history(request, id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server_config())
# GET param: context.
context_param = request.GET.get('context', '')
# GET param: on_success_url. Default to view_results
if request.session.get('dl_status', False)==False and download_format in common.DL_FORMATS:
results_url = urlresolvers.reverse(get_app_name(request) + ':execute_query')
else:
results_url = urlresolvers.reverse(get_app_name(request) + ':view_results', kwargs={'id': id, 'first_row': 0})
if request.GET.get('download', ''):
results_url += '?download=true'
on_success_url = request.GET.get('on_success_url')
if not on_success_url:
on_success_url = results_url
# Go to next statement if asked to continue or when a statement with no dataset finished.
if request.method == 'POST' or (
not query_history.is_finished() and query_history.is_success() and not query_history.has_results):
try:
query_history = db.execute_next_statement(query_history)
except Exception:
pass
# Check query state
handle, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
if query_history.is_failure():
# When we fetch, Beeswax server will throw us a BeeswaxException, which has the
# log we want to display.
return format_preserving_redirect(request, results_url, request.GET)
elif query_history.is_finished() or (query_history.is_success() and query_history.has_results):
if request.session.get('dl_status', False): # BUG-20020
on_success_url = urlresolvers.reverse(get_app_name(request) + ':download', kwargs=dict(id=str(id), format=download_format))
_clean_session(request)
return format_preserving_redirect(request, on_success_url, request.GET)
# Still running
log = db.get_log(handle)
# Keep waiting
# - Translate context into something more meaningful (type, data)
query_context = _parse_query_context(context_param)
return render('watch_wait.mako', request, {
'query': query_history,
'fwd_params': request.GET.urlencode(),
'log': log,
'hadoop_jobs': _parse_out_hadoop_jobs(log)[0],
'query_context': query_context,
'download_format': download_format, ## ExpV
})
def watch_query(request, id):
"""
Wait for the query to finish and (by default) displays the results of query id.
It understands the optional GET params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
context
A string of "name:data" that describes the context
that generated this query result. It may be:
- "table":"<table_name>"
- "design":<design_id>
All other GET params will be passed to on_success_url (if present).
"""
# Coerce types; manage arguments
id = int(id)
query_history = authorized_get_history(request, id, must_exist=True)
# GET param: context.
context_param = request.GET.get('context', '')
# GET param: on_success_url. Default to view_results
results_url = urlresolvers.reverse(view_results,
kwargs=dict(id=str(id),
first_row=0,
last_result_len=0))
on_success_url = request.GET.get('on_success_url')
if not on_success_url:
on_success_url = results_url
# Get the server_id
server_id, state = _get_server_id_and_state(query_history)
query_history.save_state(state)
# Query finished?
if state == QueryHistory.STATE.expired:
raise PopupException(_("The result of this query has expired."))
elif state == QueryHistory.STATE.available:
return format_preserving_redirect(request, on_success_url, request.GET)
elif state == QueryHistory.STATE.failed:
# When we fetch, Beeswax server will throw us a BeeswaxException, which has the
# log we want to display.
return format_preserving_redirect(request, results_url, request.GET)
# Still running
log = db_utils.db_client(
query_history.get_query_server()).get_log(server_id)
# Keep waiting
# - Translate context into something more meaningful (type, data)
context = _parse_query_context(context_param)
return render(
'watch_wait.mako', request, {
'query': query_history,
'fwd_params': request.GET.urlencode(),
'log': log,
'hadoop_jobs': _parse_out_hadoop_jobs(log),
'query_context': context,
})
def view_results(request, id, first_row=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the watch_query view.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See watch_query().)
"""
first_row = long(first_row)
start_over = (first_row == 0)
results = None
data = None
fetch_error = False
error_message = ''
log = ''
app_name = get_app_name(request)
query_history = authorized_get_history(request, id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server_config())
handle, state = _get_query_handle_and_state(query_history)
context_param = request.GET.get('context', '')
query_context = _parse_query_context(context_param)
# To remove in Hue 2.3
download = request.GET.get('download', '')
# Update the status as expired should not be accessible
expired = state == QueryHistory.STATE.expired
if expired:
state = QueryHistory.STATE.expired
query_history.save_state(state)
# Retrieve query results
try:
if not download:
results = db.fetch(handle, start_over, 100)
data = list(results.rows()) # Materialize results
# We display the "Download" button only when we know that there are results:
downloadable = first_row > 0 or data
else:
downloadable = True
data = []
results = type('Result', (object, ), {
'rows': 0,
'columns': [],
'has_more': False,
'start_row': 0,
})
log = db.get_log(handle)
except Exception as ex:
fetch_error = True
error_message, log = expand_exception(ex, db)
# Handle errors
error = fetch_error or results is None or expired
context = {
'error': error,
'error_message': error_message,
'has_more': True,
'query': query_history,
'results': data,
'expected_first_row': first_row,
'log': log,
'hadoop_jobs': _parse_out_hadoop_jobs(log)[0],
'query_context': query_context,
'can_save': False,
'context_param': context_param,
'expired': expired,
'app_name': app_name,
'download': download,
}
if not error:
download_urls = {}
if downloadable:
for format in common.DL_FORMATS:
download_urls[format] = urlresolvers.reverse(
'beeswax' + ':download',
kwargs=dict(id=str(id), format=format))
save_form = SaveResultsForm()
results.start_row = first_row
context.update({
'results':
data,
'has_more':
results.has_more,
'next_row':
results.start_row + len(data),
'start_row':
results.start_row,
'expected_first_row':
first_row,
'columns':
results.columns,
'download_urls':
download_urls,
'save_form':
save_form,
'can_save':
query_history.owner == request.user and not download,
})
return render('watch_results.mako', request, context)
def watch_query(request, id, download_format=None):
"""
Wait for the query to finish and (by default) displays the results of query id.
It understands the optional GET params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
context
A string of "name:data" that describes the context
that generated this query result. It may be:
- "table":"<table_name>"
- "design":<design_id>
All other GET params will be passed to on_success_url (if present).
"""
# Coerce types; manage arguments
query_history = authorized_get_history(request, id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server_config())
# GET param: context.
context_param = request.GET.get('context', '')
# GET param: on_success_url. Default to view_results
if request.session.get(
'dl_status',
False) == False and download_format in common.DL_FORMATS:
results_url = urlresolvers.reverse(
get_app_name(request) + ':execute_query')
else:
results_url = urlresolvers.reverse(get_app_name(request) +
':view_results',
kwargs={
'id': id,
'first_row': 0
})
if request.GET.get('download', ''):
results_url += '?download=true'
on_success_url = request.GET.get('on_success_url')
if not on_success_url:
on_success_url = results_url
# Go to next statement if asked to continue or when a statement with no dataset finished.
if request.method == 'POST' or (not query_history.is_finished()
and query_history.is_success()
and not query_history.has_results):
try:
query_history = db.execute_next_statement(query_history)
except Exception:
pass
# Check query state
handle, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
if query_history.is_failure():
# When we fetch, Beeswax server will throw us a BeeswaxException, which has the
# log we want to display.
return format_preserving_redirect(request, results_url, request.GET)
elif query_history.is_finished() or (query_history.is_success()
and query_history.has_results):
if request.session.get('dl_status', False): # BUG-20020
on_success_url = urlresolvers.reverse(
get_app_name(request) + ':download',
kwargs=dict(id=str(id), format=download_format))
_clean_session(request)
return format_preserving_redirect(request, on_success_url, request.GET)
# Still running
log = db.get_log(handle)
# Keep waiting
# - Translate context into something more meaningful (type, data)
query_context = _parse_query_context(context_param)
return render(
'watch_wait.mako',
request,
{
'query': query_history,
'fwd_params': request.GET.urlencode(),
'log': log,
'hadoop_jobs': _parse_out_hadoop_jobs(log)[0],
'query_context': query_context,
'download_format': download_format, ## ExpV
})