def drop_partition(request, database, table):
source_type = request.POST.get('source_type', 'hive')
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
if request.method == 'POST':
partition_specs = request.POST.getlist('partition_selection')
partition_specs = [spec for spec in partition_specs]
try:
if request.GET.get("format", "html") == "json":
last_executed = json.loads(request.POST.get('start_time'), '-1')
sql = db.drop_partitions(database, table, partition_specs, design=None, generate_ddl_only=True)
job = make_notebook(
name=_('Drop partition %s') % ', '.join(partition_specs)[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=None,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed
)
return JsonResponse(job.execute(request))
else:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_partitions(database, table, partition_specs, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + \
reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(partition)s. Error: %(error)s") % {'partition': '\n'.join(partition_specs), 'error': error_message}
raise PopupException(error, title=_("DB Error"), detail=log)
def drop_table(request, database):
db = dbms.get(request.user)
if request.method == 'POST':
try:
tables = request.POST.getlist('table_selection')
tables_objects = [db.get_table(database, table) for table in tables]
skip_trash = request.POST.get('skip_trash') == 'on'
if request.POST.get('is_embeddable'):
sql = db.drop_tables(database, tables_objects, design=None, skip_trash=skip_trash, generate_ddl_only=True)
job = make_notebook(
name='Execute and watch',
editor_type='hive',
statement=sql.strip(),
status='ready',
database=database,
on_success_url='assist.db.refresh',
is_task=True
)
return JsonResponse(job.execute(request))
else:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_tables(database, tables_objects, design, skip_trash=skip_trash)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:show_tables', kwargs={'database': database})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def execute_query(request, design_id=None):
"""
View function for executing an arbitrary query.
It understands the optional GET/POST params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
"""
authorized_get_design(request, design_id)
error_message = None
form = QueryForm()
action = request.path
log = None
app_name = get_app_name(request)
query_type = SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
on_success_url = request.REQUEST.get('on_success_url')
databases = []
query_server = get_query_server_config(app_name)
db = dbms.get(request.user, query_server)
try:
databases = get_db_choices(request)
except Exception, ex:
error_message, log = expand_exception(ex, db)
def drop_database(request):
db = dbms.get(request.user)
if request.method == 'POST':
databases = request.POST.getlist('database_selection')
try:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
if request.POST.get('is_embeddable'):
sql = db.drop_databases(databases, design, generate_ddl_only=True)
job = make_notebook(
name='Execute and watch',
editor_type='hive',
statement=sql.strip(),
status='ready',
database=None,
on_success_url='assist.db.refresh',
is_task=True
)
return JsonResponse(job.execute(request))
else:
query_history = db.drop_databases(databases, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:databases')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {'databases': ','.join(databases), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def view_results(request, id, first_row=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the execute_query view.
If the result set has has_result_set=False, display an empty result.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See execute_query().)
"""
first_row = long(first_row)
start_over = (first_row == 0)
results = type('Result', (object,), {
'rows': 0,
'columns': [],
'has_more': False,
'start_row': 0,
})
data = []
fetch_error = False
error_message = ''
log = ''
columns = []
app_name = get_app_name(request)
query_history = authorized_get_query_history(request, id, must_exist=True)
query_server = query_history.get_query_server_config()
db = dbms.get(request.user, query_server)
handle, state = _get_query_handle_and_state(query_history)
context_param = request.GET.get('context', '')
query_context = parse_query_context(context_param)
# Update the status as expired should not be accessible
expired = state == models.QueryHistory.STATE.expired
# Retrieve query results or use empty result if no result set
try:
if query_server['server_name'] == 'impala' and not handle.has_result_set:
downloadable = False
else:
results = db.fetch(handle, start_over, 100)
# Materialize and HTML escape results
data = escape_rows(results.rows())
# We display the "Download" button only when we know that there are results:
downloadable = first_row > 0 or data
log = db.get_log(handle)
columns = results.data_table.cols()
except Exception, ex:
LOG.exception('error fetching results')
fetch_error = True
error_message, log = expand_exception(ex, db, handle)
def view_results(request, id, first_row=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the watch_query view.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See watch_query().)
"""
first_row = long(first_row)
start_over = (first_row == 0)
results = None
data = None
fetch_error = False
error_message = ''
log = ''
app_name = get_app_name(request)
query_history = authorized_get_history(request, id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server_config())
handle, state = _get_query_handle_and_state(query_history)
context_param = request.GET.get('context', '')
query_context = _parse_query_context(context_param)
# To remove in Hue 2.3
download = request.GET.get('download', '')
# Update the status as expired should not be accessible
# Impala does not support startover for now
expired = state == models.QueryHistory.STATE.expired
if expired or app_name == 'impala':
state = models.QueryHistory.STATE.expired
query_history.save_state(state)
# Retrieve query results
try:
if not download:
results = db.fetch(handle, start_over, 100)
data = list(results.rows()) # Materialize results
# We display the "Download" button only when we know that there are results:
downloadable = first_row > 0 or data
else:
downloadable = True
data = []
results = type('Result', (object,), {
'rows': 0,
'columns': [],
'has_more': False,
'start_row': 0, })
log = db.get_log(handle)
except Exception, ex:
fetch_error = True
error_message, log = expand_exception(ex, db)
def execute_query(request, design_id=None):
"""
View function for executing an arbitrary query.
It understands the optional GET/POST params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
"""
authorized_get_design(request, design_id)
error_message = None
form = QueryForm()
action = request.path
log = None
design = safe_get_design(request, models.SavedQuery.HQL, design_id)
on_success_url = request.REQUEST.get('on_success_url')
if request.method == 'POST':
form.bind(request.POST)
to_explain = request.POST.has_key('button-explain')
to_submit = request.POST.has_key('button-submit')
# Always validate the saveform, which will tell us whether it needs explicit saving
if form.is_valid():
to_save = form.saveform.cleaned_data['save']
to_saveas = form.saveform.cleaned_data['saveas']
if to_saveas and not design.is_auto:
# Save As only affects a previously saved query
design = design.clone()
if to_submit or to_save or to_saveas or to_explain:
explicit_save = to_save or to_saveas
design = save_design(request, form, models.SavedQuery.HQL, design, explicit_save)
action = urlresolvers.reverse(execute_query, kwargs=dict(design_id=design.id))
if to_explain or to_submit:
query_str = form.query.cleaned_data["query"]
query_server = get_query_server(form.query_servers.cleaned_data["server"])
# (Optional) Parameterization.
parameterization = get_parameterization(request, query_str, form, design, to_explain)
if parameterization:
return parameterization
try:
query = HQLdesign(form)
if to_explain:
return explain_directly(request, query, design, query_server)
else:
notify = form.query.cleaned_data.get('email_notify', False)
return execute_directly(request, query, query_server, design, on_success_url=on_success_url, notify=notify)
except BeeswaxException, ex:
print ex.errorCode
print ex.SQLState
db = dbms.get(request.user, query_server)
error_message, log = expand_exception(ex, db)
def view_results(request, id, first_row=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the watch_query view.
If the result set has has_result_set=False, display an empty result.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See watch_query().)
"""
first_row = long(first_row)
start_over = first_row == 0
results = type("Result", (object,), {"rows": 0, "columns": [], "has_more": False, "start_row": 0})
data = []
fetch_error = False
error_message = ""
log = ""
app_name = get_app_name(request)
query_history = authorized_get_history(request, id, must_exist=True)
query_server = query_history.get_query_server_config()
db = dbms.get(request.user, query_server)
handle, state = _get_query_handle_and_state(query_history)
context_param = request.GET.get("context", "")
query_context = _parse_query_context(context_param)
# To remove in Hue 2.3
download = request.GET.get("download", "")
# Update the status as expired should not be accessible
# Impala does not support startover for now
expired = state == models.QueryHistory.STATE.expired
if expired or app_name == "impala":
state = models.QueryHistory.STATE.expired
query_history.save_state(state)
# Retrieve query results or use empty result if no result set
try:
if query_server["server_name"] == "impala" and not handle.has_result_set:
downloadable = False
elif not download:
results = db.fetch(handle, start_over, 100)
data = list(results.rows()) # Materialize results
# We display the "Download" button only when we know that there are results:
downloadable = first_row > 0 or data
log = db.get_log(handle)
else:
downloadable = True
except Exception, ex:
fetch_error = True
error_message, log = expand_exception(ex, db, handle)
def _run_parameterized_query(request, design_id, explain):
"""
Given a design and arguments to parameterize that design, runs the query.
- explain is a boolean to determine whether to run as an explain or as an
execute.
This is an extra "step" in the flow from execute_query.
"""
design = authorized_get_design(request, design_id, must_exist=True)
# Reconstitute the form
design_obj = beeswax.design.HQLdesign.loads(design.data)
query_form = QueryForm()
params = design_obj.get_query_dict()
params.update(request.POST)
databases = _get_db_choices(request)
query_form.bind(params)
query_form.query.fields["database"].choices = databases # Could not do it in the form
if not query_form.is_valid():
raise PopupException(_("Query form is invalid: %s") % query_form.errors)
query_str = query_form.query.cleaned_data["query"]
app_name = get_app_name(request)
query_server = get_query_server_config(app_name)
query_type = SavedQuery.TYPES_MAPPING[app_name]
parameterization_form_cls = make_parameterization_form(query_str)
if not parameterization_form_cls:
raise PopupException(_("Query is not parameterizable."))
parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization")
if parameterization_form.is_valid():
real_query = substitute_variables(query_str, parameterization_form.cleaned_data)
query = HQLdesign(query_form, query_type=query_type)
query._data_dict["query"]["query"] = real_query
try:
if explain:
return explain_directly(request, query, design, query_server)
else:
return execute_directly(request, query, query_server, design)
except Exception, ex:
db = dbms.get(request.user, query_server)
error_message, log = expand_exception(ex, db)
return render(
"execute.mako",
request,
{
"action": reverse(get_app_name(request) + ":execute_query"),
"design": design,
"error_message": error_message,
"form": query_form,
"log": log,
"autocomplete_base_url": reverse(get_app_name(request) + ":autocomplete", kwargs={}),
},
)
def save_results_hdfs_file(request, query_history_id):
"""
Save the results of a query to an HDFS file.
Do not rerun the query.
"""
response = {'status': 0, 'message': ''}
query_history = authorized_get_query_history(request, query_history_id, must_exist=True)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method != 'POST':
response['message'] = _('A POST request is required.')
else:
if not query_history.is_success():
response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state}
response['status'] = -1
return JsonResponse(response)
db = dbms.get(request.user, query_history.get_query_server_config())
form = beeswax.forms.SaveResultsFileForm({
'target_file': request.POST.get('path'),
'overwrite': request.POST.get('overwrite', False),
})
if form.is_valid():
target_file = form.cleaned_data['target_file']
overwrite = form.cleaned_data['overwrite']
try:
handle, state = _get_query_handle_and_state(query_history)
except Exception, ex:
response['message'] = _('Cannot find query handle and state: %s') % str(query_history)
response['status'] = -2
return JsonResponse(response)
try:
if overwrite and request.fs.exists(target_file):
if request.fs.isfile(target_file):
request.fs.do_as_user(request.user.username, request.fs.rmtree, target_file)
else:
raise PopupException(_("The target path is a directory"))
upload(target_file, handle, request.user, db, request.fs)
response['type'] = 'hdfs-file'
response['id'] = query_history.id
response['query'] = query_history.query
response['path'] = target_file
response['success_url'] = '/filebrowser/view=%s' % target_file
response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id})
except Exception, ex:
error_msg, log = expand_exception(ex, db)
response['message'] = _('The result could not be saved: %s.') % error_msg
response['status'] = -3
def describe_table(request, database, table):
db = dbms.get(request.user)
error_message = ''
table_data = ''
table = db.get_table(database, table)
try:
table_data = db.get_sample(database, table)
except BeeswaxException, ex:
error_message, logs = expand_exception(ex, db)
def execute(request, design_id=None):
response = {'status': -1, 'message': ''}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
app_name = get_app_name(request)
query_server = get_query_server_config(app_name)
query_type = beeswax.models.SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
try:
query_form = get_query_form(request)
if query_form.is_valid():
query_str = query_form.query.cleaned_data["query"]
explain = request.GET.get('explain', 'false').lower() == 'true'
design = save_design(request, query_form, query_type, design, False)
if query_form.query.cleaned_data['is_parameterized']:
# Parameterized query
parameterization_form_cls = make_parameterization_form(query_str)
if parameterization_form_cls:
parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization")
if parameterization_form.is_valid():
parameters = parameterization_form.cleaned_data
real_query = substitute_variables(query_str, parameters)
query = HQLdesign(query_form, query_type=query_type)
query._data_dict['query']['query'] = real_query
try:
if explain:
return explain_directly(request, query, design, query_server)
else:
return execute_directly(request, query, design, query_server, parameters=parameters)
except Exception, ex:
db = dbms.get(request.user, query_server)
error_message, log = expand_exception(ex, db)
response['message'] = error_message
return JsonResponse(response)
else:
response['errors'] = parameterization_form.errors
return JsonResponse(response)
# Non-parameterized query
query = HQLdesign(query_form, query_type=query_type)
if request.GET.get('explain', 'false').lower() == 'true':
return explain_directly(request, query, design, query_server)
else:
return execute_directly(request, query, design, query_server)
else:
def describe_table(request, database, table):
db = dbms.get(request.user)
error_message = ''
table_data = ''
table = db.get_table(database, table)
partitions = None
if table.partition_keys:
partitions = db.get_partitions(database, table, max_parts=None)
try:
table_data = db.get_sample(database, table)
except Exception, ex:
error_message, logs = dbms.expand_exception(ex, db)
def get_sample_data(request, database, table):
db = dbms.get(request.user)
response = {'status': -1, 'error_message': ''}
try:
table_obj = db.get_table(database, table)
sample_data = db.get_sample(database, table_obj)
response = {
'status': 0,
'headers': sample_data and sample_data.cols(),
'rows': sample_data and list(sample_data.rows())
}
except Exception, ex:
error_message, logs = dbms.expand_exception(ex, db)
response['error_message'] = error_message
def drop_table(request, table):
db = dbms.get(request.user)
table = db.get_table('default', table)
if request.method == 'POST':
try:
query_history = db.drop_table(table)
url = reverse(watch_query, args=[query_history.id]) + '?on_success_url=' + urlresolvers.reverse(show_tables)
return redirect(url)
except BeeswaxException, ex:
error_message, log = expand_exception(ex, db)
error = _("Failed to remove %(table)s. Error: %(error)s") % {'table': table.name, 'error': error_message}
raise PopupException(error, title=_("Beeswax Error"), detail=log)
def save_results_hive_table(request, query_history_id):
"""
Save the results of a query to a hive table.
Rerun the query.
"""
response = {'status': 0, 'message': ''}
query_history = authorized_get_query_history(request, query_history_id, must_exist=True)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method != 'POST':
response['message'] = _('A POST request is required.')
else:
if not query_history.is_success():
response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state}
response['status'] = -1
return JsonResponse(response)
db = dbms.get(request.user, query_history.get_query_server_config())
database = query_history.design.get_design().query.get('database', 'default')
form = beeswax.forms.SaveResultsTableForm({
'target_table': request.POST.get('table')
}, db=db, database=database)
if form.is_valid():
try:
handle, state = _get_query_handle_and_state(query_history)
result_meta = db.get_results_metadata(handle)
except Exception, ex:
response['message'] = _('Cannot find query handle and state: %s') % str(query_history)
response['status'] = -2
return JsonResponse(response)
try:
query_history = db.create_table_as_a_select(request, query_history, form.target_database, form.cleaned_data['target_table'], result_meta)
response['id'] = query_history.id
response['query'] = query_history.query
response['type'] = 'hive-table'
response['path'] = form.cleaned_data['target_table']
response['success_url'] = reverse('metastore:describe_table', kwargs={'database': form.target_database, 'table': form.cleaned_data['target_table']})
response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id})
except Exception, ex:
error_msg, log = expand_exception(ex, db)
response['message'] = _('The result could not be saved: %s.') % error_msg
response['status'] = -3
def save_results_hive_table(request, query_history_id):
"""
Save the results of a query to a hive table.
Rerun the query.
"""
response = {'status': 0, 'message': ''}
query_history = authorized_get_query_history(request, query_history_id, must_exist=True)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method != 'POST':
response['message'] = _('A POST request is required.')
else:
if not query_history.is_success():
response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state}
response['status'] = -1
return JsonResponse(response)
db = dbms.get(request.user, query_history.get_query_server_config())
database = query_history.design.get_design().query.get('database', 'default')
form = beeswax.forms.SaveResultsTableForm({
'target_table': request.POST.get('table')
}, db=db, database=database)
if form.is_valid():
try:
handle, state = _get_query_handle_and_state(query_history)
result_meta = db.get_results_metadata(handle)
except Exception, ex:
response['message'] = _('Cannot find query handle and state: %s') % str(query_history)
response['status'] = -2
return JsonResponse(response)
try:
query_history = db.create_table_as_a_select(request, query_history, form.target_database, form.cleaned_data['target_table'], result_meta)
response['id'] = query_history.id
response['query'] = query_history.query
response['type'] = 'hive-table'
response['path'] = form.cleaned_data['target_table']
response['success_url'] = reverse('metastore:describe_table', kwargs={'database': form.target_database, 'table': form.cleaned_data['target_table']})
response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id})
except Exception, ex:
error_msg, log = expand_exception(ex, db)
response['message'] = _('The result could not be saved: %s.') % error_msg
response['status'] = -3
def drop_table(request, database):
db = dbms.get(request.user)
if request.method == 'POST':
tables = request.POST.getlist('table_selection')
tables_objects = [db.get_table(database, table) for table in tables]
try:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user)
query_history = db.drop_tables(database, tables_objects, design)
url = reverse('beeswax:watch_query', args=[query_history.id]) + '?on_success_url=' + reverse('catalog:show_tables')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message}
raise PopupException(error, title=_("Beeswax Error"), detail=log)
def get_indexes(request, database, table):
query_server = dbms.get_query_server_config(get_app_name(request))
db = dbms.get(request.user, query_server)
response = {'status': -1, 'error_message': ''}
try:
indexes = db.get_indexes(database, table)
if indexes:
response['status'] = 0
response['headers'] = indexes.cols()
response['rows'] = escape_rows(indexes.rows(), nulls_only=True)
else:
response['error_message'] = _('Index data took too long to be generated')
except Exception, ex:
error_message, logs = dbms.expand_exception(ex, db)
response['error_message'] = error_message
def drop_partition(request, database, table):
db = dbms.get(request.user)
if request.method == 'POST':
partition_specs = request.POST.getlist('partition_selection')
partition_specs = [spec for spec in partition_specs]
try:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_partitions(database, table, partition_specs, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + \
reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(partition)s. Error: %(error)s") % {'partition': '\n'.join(partition_specs), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_database(request):
db = dbms.get(request.user)
if request.method == 'POST':
databases = request.POST.getlist('database_selection')
try:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_databases(databases, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:databases')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {'databases': ','.join(databases), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_table(request, database):
db = dbms.get(request.user)
if request.method == 'POST':
tables = request.POST.getlist('table_selection')
tables_objects = [db.get_table(database, table) for table in tables]
try:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user)
query_history = db.drop_tables(database, tables_objects, design)
url = reverse('beeswax:watch_query', args=[query_history.id]) + '?on_success_url=' + reverse('metastore:show_tables')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def get_sample_data(request, database, table):
db = dbms.get(request.user)
response = {'status': -1, 'error_message': ''}
try:
table_obj = db.get_table(database, table)
sample_data = db.get_sample(database, table_obj)
if sample_data:
response['status'] = 0
response['headers'] = sample_data.cols()
response['rows'] = escape_rows(sample_data.rows(), nulls_only=True)
else:
response['error_message'] = _('Sample data took too long to be generated')
except Exception, ex:
error_message, logs = dbms.expand_exception(ex, db)
response['error_message'] = error_message
def drop_database(request):
db = dbms.get(request.user)
if request.method == 'POST':
databases = request.POST.getlist('database_selection')
try:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_databases(databases, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:databases')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {'databases': ','.join(databases), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_partition(request, database, table):
db = dbms.get(request.user)
if request.method == 'POST':
partition_specs = request.POST.getlist('partition_selection')
partition_specs = [spec for spec in partition_specs]
try:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_partitions(database, table, partition_specs, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + \
reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(partition)s. Error: %(error)s") % {'partition': '\n'.join(partition_specs), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def _run_parameterized_query(request, design_id, explain):
"""
Given a design and arguments to parameterize that design, runs the query.
- explain is a boolean to determine whether to run as an explain or as an
execute.
This is an extra "step" in the flow from execute_query.
"""
design = authorized_get_design(request, design_id, must_exist=True)
# Reconstitute the form
design_obj = beeswax.design.HQLdesign.loads(design.data)
query_form = QueryForm()
params = design_obj.get_query_dict()
params.update(request.POST)
query_form.bind(params)
assert query_form.is_valid()
query_str = query_form.query.cleaned_data["query"]
query_server = get_query_server_config(get_app_name(request))
parameterization_form_cls = make_parameterization_form(query_str)
if not parameterization_form_cls:
raise PopupException(_("Query is not parameterizable."))
parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization")
if parameterization_form.is_valid():
real_query = substitute_variables(query_str, parameterization_form.cleaned_data)
query = HQLdesign(query_form)
query._data_dict['query']['query'] = real_query
try:
if explain:
return explain_directly(request, query, design, query_server)
else:
return execute_directly(request, query, query_server, design)
except Exception, ex:
db = dbms.get(request.user, query_server)
error_message, log = expand_exception(ex, db)
return render('execute.mako', request, {
'action': reverse(get_app_name(request) + ':execute_query'),
'design': design,
'error_message': error_message,
'form': query_form,
'log': log,
'autocomplete_base_url': reverse(get_app_name(request) + ':autocomplete', kwargs={}),
})
def save_results(request, id):
"""
Save the results of a query to an HDFS directory or Hive table.
"""
query_history = authorized_get_history(request, id, must_exist=True)
app_name = get_app_name(request)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method == "POST":
if not query_history.is_success():
msg = _("This query is %(state)s. Results unavailable.") % {"state": state}
raise PopupException(msg)
db = dbms.get(request.user, query_history.get_query_server_config())
form = beeswax.forms.SaveResultsForm(request.POST, db=db, fs=request.fs)
if request.POST.get("cancel"):
return format_preserving_redirect(request, "/%s/watch/%s" % (app_name, id))
if form.is_valid():
try:
handle, state = _get_query_handle_and_state(query_history)
result_meta = db.get_results_metadata(handle)
except Exception, ex:
raise PopupException(_("Cannot find query: %s") % ex)
try:
if form.cleaned_data["save_target"] == form.SAVE_TYPE_DIR:
target_dir = form.cleaned_data["target_dir"]
query_history = db.insert_query_into_directory(query_history, target_dir)
redirected = redirect(
reverse("beeswax:watch_query", args=[query_history.id])
+ "?on_success_url="
+ reverse("filebrowser.views.view", kwargs={"path": target_dir})
)
elif form.cleaned_data["save_target"] == form.SAVE_TYPE_TBL:
redirected = db.create_table_as_a_select(
request, query_history, form.cleaned_data["target_table"], result_meta
)
except Exception, ex:
error_msg, log = expand_exception(ex, db)
raise PopupException(_("The result could not be saved: %s.") % log, detail=ex)
return redirected
def drop_database(request):
source_type = request.POST.get('source_type', 'hive')
db = _get_db(user=request.user, source_type=source_type)
if request.method == 'POST':
databases = request.POST.getlist('database_selection')
try:
if request.POST.get('is_embeddable'):
design = SavedQuery.create_empty(
app_name=source_type
if source_type != 'hive' else 'beeswax',
owner=request.user,
data=hql_query('').dumps())
last_executed = json.loads(request.POST.get('start_time'),
'-1')
sql = db.drop_databases(databases,
design,
generate_ddl_only=True)
job = make_notebook(name=_('Drop database %s') %
', '.join(databases)[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=None,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed)
return JsonResponse(job.execute(request))
else:
design = SavedQuery.create_empty(app_name='beeswax',
owner=request.user,
data=hql_query('').dumps())
query_history = db.drop_databases(databases, design)
url = reverse(
'beeswax:watch_query_history',
kwargs={
'query_history_id': query_history.id
}) + '?on_success_url=' + reverse('metastore:databases')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {
'databases': ','.join(databases),
'error': error_message
}
raise PopupException(error, title=_("DB Error"), detail=log)
def drop_table(request, database):
db = dbms.get(request.user)
if request.method == 'POST':
try:
tables = request.POST.getlist('table_selection')
tables_objects = [
db.get_table(database, table) for table in tables
]
skip_trash = request.POST.get('skip_trash') == 'on'
if request.POST.get('is_embeddable'):
sql = db.drop_tables(database,
tables_objects,
design=None,
skip_trash=skip_trash,
generate_ddl_only=True)
job = make_notebook(name='Execute and watch',
editor_type='hive',
statement=sql.strip(),
status='ready',
database=database,
on_success_url='assist.db.refresh',
is_task=True)
return JsonResponse(job.execute(request))
else:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax',
owner=request.user,
data=hql_query('').dumps())
query_history = db.drop_tables(database,
tables_objects,
design,
skip_trash=skip_trash)
url = reverse('beeswax:watch_query_history',
kwargs={'query_history_id': query_history.id
}) + '?on_success_url=' + reverse(
'metastore:show_tables',
kwargs={'database': database})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {
'tables': ','.join(tables),
'error': error_message
}
raise PopupException(error, title=_("Hive Error"), detail=log)
def _run_parameterized_query(request, design_id, explain):
"""
Given a design and arguments to parameterize that design, runs the query.
- explain is a boolean to determine whether to run as an explain or as an
execute.
This is an extra "step" in the flow from execute_query.
"""
design = authorized_get_design(request, design_id, must_exist=True)
# Reconstitute the form
design_obj = beeswax.design.HQLdesign.loads(design.data)
query_form = QueryForm()
params = design_obj.get_query_dict()
params.update(request.POST)
query_form.bind(params)
assert query_form.is_valid()
query_str = query_form.query.cleaned_data["query"]
query_server = get_query_server_config(get_app_name(request))
parameterization_form_cls = make_parameterization_form(query_str)
if not parameterization_form_cls:
raise PopupException(_("Query is not parameterizable."))
parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization")
if parameterization_form.is_valid():
real_query = substitute_variables(query_str, parameterization_form.cleaned_data)
query = HQLdesign(query_form)
query._data_dict['query']['query'] = real_query
try:
if explain:
return explain_directly(request, query, design, query_server)
else:
return execute_directly(request, query, query_server, design)
except Exception, ex:
db = dbms.get(request.user, query_server)
error_message, log = expand_exception(ex, db)
return render('execute.mako', request, {
'action': reverse(get_app_name(request) + ':execute_query'),
'design': design,
'error_message': error_message,
'form': query_form,
'log': log,
'autocomplete': json.dumps({}),
})
def drop_table(request, database):
source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive'))
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
if request.method == 'POST':
try:
tables = request.POST.getlist('table_selection')
tables_objects = [db.get_table(database, table) for table in tables]
skip_trash = request.POST.get('skip_trash') == 'on'
cluster = json.loads(request.POST.get('cluster', '{}'))
namespace = json.loads(request.POST.get('namespace', '{}'))
if request.POST.get('is_embeddable'):
last_executed = json.loads(request.POST.get('start_time', '-1'))
sql = db.drop_tables(database, tables_objects, design=None, skip_trash=skip_trash, generate_ddl_only=True)
job = make_notebook(
name=_('Drop table %s') % ', '.join([table.name for table in tables_objects])[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=database,
namespace=namespace,
compute=cluster,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed
)
return JsonResponse(job.execute(request))
else:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_tables(database, tables_objects, design, skip_trash=skip_trash)
url = reverse(
'beeswax:watch_query_history',
kwargs={'query_history_id': query_history.id}
) + '?on_success_url=' + reverse('metastore:show_tables', kwargs={'database': database})
return redirect(url)
except Exception as ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message}
raise PopupException(error, title=_("DB Error"), detail=log)
else:
title = _("Do you really want to delete the table(s)?")
return render('confirm.mako', request, {'url': request.path, 'title': title})
def save_results_hdfs_directory(request, query_history_id):
"""
Save the results of a query to an HDFS directory.
Rerun the query.
"""
response = {'status': 0, 'message': ''}
query_history = authorized_get_query_history(request, query_history_id, must_exist=True)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method != 'POST':
response['message'] = _('A POST request is required.')
else:
if not query_history.is_success():
response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state}
response['status'] = -1
return JsonResponse(response)
db = dbms.get(request.user, query_history.get_query_server_config())
form = beeswax.forms.SaveResultsDirectoryForm({
'target_dir': request.POST.get('path')
}, fs=request.fs)
if form.is_valid():
target_dir = request.POST.get('path')
try:
response['type'] = 'hdfs-dir'
response['id'] = query_history.id
response['query'] = query_history.query
response['path'] = target_dir
response['success_url'] = '/filebrowser/view=%s' % target_dir
query_history = db.insert_query_into_directory(query_history, target_dir)
response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id})
except Exception as ex:
error_msg, log = expand_exception(ex, db)
response['message'] = _('The result could not be saved: %s.') % error_msg
response['status'] = -3
else:
response['status'] = 1
response['errors'] = form.errors
return JsonResponse(response)
def drop_partition(request, database, table):
source_type = request.POST.get('source_type', 'hive')
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
if request.method == 'POST':
partition_specs = request.POST.getlist('partition_selection')
partition_specs = [spec for spec in partition_specs]
try:
if request.GET.get("format", "html") == "json":
last_executed = json.loads(request.POST.get('start_time'),
'-1')
sql = db.drop_partitions(database,
table,
partition_specs,
design=None,
generate_ddl_only=True)
job = make_notebook(name=_('Drop partition %s') %
', '.join(partition_specs)[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=None,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed)
return JsonResponse(job.execute(request))
else:
design = SavedQuery.create_empty(app_name='beeswax',
owner=request.user,
data=hql_query('').dumps())
query_history = db.drop_partitions(database, table,
partition_specs, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + \
reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(partition)s. Error: %(error)s") % {
'partition': '\n'.join(partition_specs),
'error': error_message
}
raise PopupException(error, title=_("DB Error"), detail=log)
def save_results(request, id):
"""
DEPRECATED. Need to get rid of watch_wait dependency first.
Save the results of a query to an HDFS directory or Hive table.
"""
query_history = authorized_get_history(request, id, must_exist=True)
app_name = get_app_name(request)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method == 'POST':
if not query_history.is_success():
msg = _('This query is %(state)s. Results unavailable.') % {'state': state}
raise PopupException(msg)
db = dbms.get(request.user, query_history.get_query_server_config())
form = beeswax.forms.SaveResultsForm(request.POST, db=db, fs=request.fs)
if request.POST.get('cancel'):
return format_preserving_redirect(request, '/%s/watch/%s' % (app_name, id))
if form.is_valid():
try:
handle, state = _get_query_handle_and_state(query_history)
result_meta = db.get_results_metadata(handle)
except Exception, ex:
raise PopupException(_('Cannot find query: %s') % ex)
try:
if form.cleaned_data['save_target'] == form.SAVE_TYPE_DIR:
target_dir = form.cleaned_data['target_dir']
query_history = db.insert_query_into_directory(query_history, target_dir)
redirected = redirect(reverse('beeswax:watch_query', args=[query_history.id]) \
+ '?on_success_url=' + reverse('filebrowser.views.view', kwargs={'path': target_dir}))
elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL:
redirected = db.create_table_as_a_select(request, query_history, form.cleaned_data['target_table'], result_meta)
except Exception, ex:
error_msg, log = expand_exception(ex, db)
raise PopupException(_('The result could not be saved: %s.') % log, detail=ex)
return redirected
def save_results(request, id):
"""
Save the results of a query to an HDFS directory or Hive table.
"""
query_history = authorized_get_history(request, id, must_exist=True)
app_name = get_app_name(request)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method == 'POST':
if not query_history.is_success():
msg = _('This query is %(state)s. Results unavailable.') % {'state': state}
raise PopupException(msg)
db = dbms.get(request.user, query_history.get_query_server_config())
form = beeswax.forms.SaveResultsForm(request.POST, db=db, fs=request.fs)
if request.POST.get('cancel'):
return format_preserving_redirect(request, '/%s/watch/%s' % (app_name, id))
if form.is_valid():
try:
handle, state = _get_query_handle_and_state(query_history)
result_meta = db.get_results_metadata(handle)
except Exception, ex:
raise PopupException(_('Cannot find query: %s') % ex)
try:
if form.cleaned_data['save_target'] == form.SAVE_TYPE_DIR:
target_dir = form.cleaned_data['target_dir']
query_history = db.insert_query_into_directory(query_history, target_dir)
redirected = redirect(reverse('beeswax:watch_query', args=[query_history.id]) \
+ '?on_success_url=' + reverse('filebrowser.views.view', kwargs={'path': target_dir}))
elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL:
redirected = db.create_table_as_a_select(request, query_history, form.cleaned_data['target_table'], result_meta)
except Exception, ex:
error_msg, log = expand_exception(ex, db)
raise PopupException(_('The result could not be saved: %s.') % log, detail=ex)
return redirected
def save_results_hdfs_directory(request, query_history_id):
"""
Save the results of a query to an HDFS directory.
Rerun the query.
"""
response = {'status': 0, 'message': ''}
query_history = authorized_get_query_history(request, query_history_id, must_exist=True)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method != 'POST':
response['message'] = _('A POST request is required.')
else:
if not query_history.is_success():
response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state}
response['status'] = -1
return JsonResponse(response)
db = dbms.get(request.user, query_history.get_query_server_config())
form = beeswax.forms.SaveResultsDirectoryForm({
'target_dir': request.POST.get('path')
}, fs=request.fs)
if form.is_valid():
target_dir = request.POST.get('path')
try:
response['type'] = 'hdfs-dir'
response['id'] = query_history.id
response['query'] = query_history.query
response['path'] = target_dir
response['success_url'] = '/filebrowser/view=%s' % target_dir
query_history = db.insert_query_into_directory(query_history, target_dir)
response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id})
except Exception, ex:
error_msg, log = expand_exception(ex, db)
response['message'] = _('The result could not be saved: %s.') % error_msg
response['status'] = -3
else:
def drop_table(request, database):
source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive'))
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
if request.method == 'POST':
try:
tables = request.POST.getlist('table_selection')
tables_objects = [db.get_table(database, table) for table in tables]
skip_trash = request.POST.get('skip_trash') == 'on'
cluster = json.loads(request.POST.get('cluster', '{}'))
namespace = request.POST.get('namespace')
if request.POST.get('is_embeddable'):
last_executed = json.loads(request.POST.get('start_time'), '-1')
sql = db.drop_tables(database, tables_objects, design=None, skip_trash=skip_trash, generate_ddl_only=True)
job = make_notebook(
name=_('Drop table %s') % ', '.join([table.name for table in tables_objects])[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=database,
namespace=namespace,
compute=cluster,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed
)
return JsonResponse(job.execute(request))
else:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_tables(database, tables_objects, design, skip_trash=skip_trash)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:show_tables', kwargs={'database': database})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message}
raise PopupException(error, title=_("DB Error"), detail=log)
def drop_partition(request, database, table):
db = dbms.get(request.user)
if request.method == "POST":
partition_specs = request.POST.getlist("partition_selection")
partition_specs = [spec for spec in partition_specs]
try:
design = SavedQuery.create_empty(app_name="beeswax", owner=request.user, data=hql_query("").dumps())
query_history = db.drop_partitions(database, table, partition_specs, design)
url = (
reverse("beeswax:watch_query_history", kwargs={"query_history_id": query_history.id})
+ "?on_success_url="
+ reverse("metastore:describe_partitions", kwargs={"database": database, "table": table})
)
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(partition)s. Error: %(error)s") % {
"partition": "\n".join(partition_specs),
"error": error_message,
}
raise PopupException(error, title=_("Hive Error"), detail=log)
def view_results(request, id, first_row=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the watch_query view.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See watch_query().)
"""
first_row = long(first_row)
start_over = (first_row == 0)
results = None
data = None
fetch_error = False
error_message = ''
log = ''
query_history = authorized_get_history(request, id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server())
handle, state = _get_query_handle_and_state(query_history)
context_param = request.GET.get('context', '')
query_context = _parse_query_context(context_param)
# Retrieve query results
try:
results = db.fetch(handle, start_over, 100)
data = list(results.rows()) # Materialize results
# We display the "Download" button only when we know that there are results:
downloadable = first_row > 0 or data
log = db.get_log(handle)
except BeeswaxException, ex:
fetch_error = True
error_message, log = expand_exception(ex, db)
def drop_database(request):
source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive'))
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
if request.method == 'POST':
databases = request.POST.getlist('database_selection')
try:
if request.POST.get('is_embeddable'):
design = SavedQuery.create_empty(app_name=source_type if source_type != 'hive' else 'beeswax', owner=request.user, data=hql_query('').dumps())
last_executed = json.loads(request.POST.get('start_time'), '-1')
cluster = json.loads(request.POST.get('cluster', '{}'))
namespace = request.POST.get('namespace')
sql = db.drop_databases(databases, design, generate_ddl_only=True)
job = make_notebook(
name=_('Drop database %s') % ', '.join(databases)[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=None,
namespace=namespace,
compute=cluster,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed
)
return JsonResponse(job.execute(request))
else:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_databases(databases, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:databases')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {'databases': ','.join(databases), 'error': error_message}
raise PopupException(error, title=_("DB Error"), detail=log)
def drop_database(request):
db = dbms.get(request.user)
if request.method == "POST":
databases = request.POST.getlist("database_selection")
try:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name="beeswax", owner=request.user, data=hql_query("").dumps())
query_history = db.drop_databases(databases, design)
url = (
reverse("beeswax:watch_query_history", kwargs={"query_history_id": query_history.id})
+ "?on_success_url="
+ reverse("metastore:databases")
)
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {
"databases": ",".join(databases),
"error": error_message,
}
raise PopupException(error, title=_("Hive Error"), detail=log)
try:
table = db.get_table(database, table)
except Exception, e:
if hasattr(e, 'message') and e.message:
raise PopupException(_("Hive Error"), detail=e.message)
else:
raise PopupException(_("Hive Error"), detail=e)
partitions = None
if app_name != 'impala' and table.partition_keys:
partitions = db.get_partitions(database, table, max_parts=None)
try:
table_data = db.get_sample(database, table)
except Exception, ex:
error_message, logs = dbms.expand_exception(ex, db)
renderable = "describe_table.mako"
if request.REQUEST.get("sample", "false") == "true":
renderable = "sample.mako"
return render(
renderable, request, {
'breadcrumbs': [
{
'name':
database,
'url':
reverse('metastore:show_tables',
kwargs={'database': database})
},
def save_results(request, id):
"""
Save the results of a query to an HDFS directory
"""
query_history = authorized_get_history(request, id, must_exist=True)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method == 'POST':
# Make sure the result is available.
# Note that we may still hit errors during the actual save
if not query_history.is_success():
if query_history.is_failure():
msg = _('This query has %(state)s. Results unavailable.') % {'state': state}
else:
msg = _('The result of this query is not available yet.')
raise PopupException(msg)
db = dbms.get(request.user, query_history.get_query_server_config())
form = beeswax.forms.SaveResultsForm(request.POST, db=db)
# Cancel goes back to results
if request.POST.get('cancel'):
return format_preserving_redirect(request, '/beeswax/watch/%s' % (id,))
if form.is_valid():
# Do save
# 1. Get the results metadata
assert request.POST.get('save')
try:
handle, state = _get_query_handle_and_state(query_history)
result_meta = db.get_results_metadata(handle)
except Exception, ex:
LOG.exception(ex)
raise PopupException(_('Cannot find query.'))
if result_meta.table_dir:
result_meta.table_dir = request.fs.urlsplit(result_meta.table_dir)[2]
# 2. Check for partitioned tables
if result_meta.table_dir is None:
raise PopupException(_('Saving results from a partitioned table is not supported. You may copy from the HDFS location manually.'))
# 3. Actual saving of results
try:
if form.cleaned_data['save_target'] == form.SAVE_TYPE_DIR:
# To dir
if result_meta.in_tablename:
raise PopupException(_('Saving results from a query with no MapReduce jobs is not supported. '
'You may copy manually from the HDFS location %(path)s.') % {'path': result_meta.table_dir})
target_dir = form.cleaned_data['target_dir']
request.fs.rename_star(result_meta.table_dir, target_dir)
LOG.debug("Moved results from %s to %s" % (result_meta.table_dir, target_dir))
query_history.save_state(models.QueryHistory.STATE.expired)
return redirect(reverse('filebrowser.views.view', kwargs={'path': target_dir}))
elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL:
# To new table
try:
return _save_results_ctas(request, query_history, form.cleaned_data['target_table'], result_meta)
except Exception, bex:
LOG.exception(bex)
error_msg, log = expand_exception(bex, db)
except WebHdfsException, ex:
raise PopupException(_('The table could not be saved.'), detail=ex)
except IOError, ex:
LOG.exception(ex)
error_msg = str(ex)
def execute_query(request, design_id=None):
"""
View function for executing an arbitrary query.
It understands the optional GET/POST params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
"""
authorized_get_design(request, design_id)
error_message = None
form = QueryForm()
action = request.path
log = None
app_name = get_app_name(request)
query_type = SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
on_success_url = request.REQUEST.get('on_success_url')
query_server = get_query_server_config(app_name)
db = dbms.get(request.user, query_server)
databases = _get_db_choices(request)
if request.method == 'POST':
form.bind(request.POST)
form.query.fields['database'].choices = databases # Could not do it in the form
to_explain = request.POST.has_key('button-explain')
to_submit = request.POST.has_key('button-submit')
# Always validate the saveform, which will tell us whether it needs explicit saving
if form.is_valid():
to_save = form.saveform.cleaned_data['save']
to_saveas = form.saveform.cleaned_data['saveas']
if to_saveas and not design.is_auto:
# Save As only affects a previously saved query
design = design.clone()
if to_submit or to_save or to_saveas or to_explain:
explicit_save = to_save or to_saveas
design = save_design(request, form, query_type, design, explicit_save)
action = reverse(app_name + ':execute_query', kwargs=dict(design_id=design.id))
if to_explain or to_submit:
query_str = form.query.cleaned_data["query"]
# (Optional) Parameterization.
parameterization = get_parameterization(request, query_str, form, design, to_explain)
if parameterization:
return parameterization
try:
query = HQLdesign(form, query_type=query_type)
if to_explain:
return explain_directly(request, query, design, query_server)
else:
download = request.POST.has_key('download')
return execute_directly(request, query, query_server, design, on_success_url=on_success_url, download=download)
except Exception, ex:
error_message, log = expand_exception(ex, db)
def save_results_hdfs_file(request, query_history_id):
"""
Save the results of a query to an HDFS file.
Do not rerun the query.
"""
response = {'status': 0, 'message': ''}
query_history = authorized_get_query_history(request, query_history_id, must_exist=True)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method != 'POST':
response['message'] = _('A POST request is required.')
else:
if not query_history.is_success():
response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state}
response['status'] = -1
return JsonResponse(response)
db = dbms.get(request.user, query_history.get_query_server_config())
form = beeswax.forms.SaveResultsFileForm({
'target_file': request.POST.get('path'),
'overwrite': request.POST.get('overwrite', False),
})
if form.is_valid():
target_file = form.cleaned_data['target_file']
overwrite = form.cleaned_data['overwrite']
try:
handle, state = _get_query_handle_and_state(query_history)
except Exception as ex:
response['message'] = _('Cannot find query handle and state: %s') % str(query_history)
response['status'] = -2
return JsonResponse(response)
try:
if overwrite and request.fs.exists(target_file):
if request.fs.isfile(target_file):
request.fs.do_as_user(request.user.username, request.fs.rmtree, target_file)
else:
raise PopupException(_("The target path is a directory"))
upload(target_file, handle, request.user, db, request.fs)
response['type'] = 'hdfs-file'
response['id'] = query_history.id
response['query'] = query_history.query
response['path'] = target_file
response['success_url'] = '/filebrowser/view=%s' % target_file
response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id})
except Exception as ex:
error_msg, log = expand_exception(ex, db)
response['message'] = _('The result could not be saved: %s.') % error_msg
response['status'] = -3
else:
response['status'] = 1
response['errors'] = form.errors
return JsonResponse(response)
def execute(request, design_id=None):
response = {'status': -1, 'message': ''}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
app_name = get_app_name(request)
query_server = get_query_server_config(app_name)
query_type = beeswax.models.SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
try:
query_form = get_query_form(request)
if query_form.is_valid():
query_str = query_form.query.cleaned_data["query"]
explain = request.GET.get('explain', 'false').lower() == 'true'
design = save_design(request, query_form, query_type, design, False)
if query_form.query.cleaned_data['is_parameterized']:
# Parameterized query
parameterization_form_cls = make_parameterization_form(query_str)
if parameterization_form_cls:
parameterization_form = parameterization_form_cls(request.POST.get('query-query', ''), prefix="parameterization")
if parameterization_form.is_valid():
parameters = parameterization_form.cleaned_data
real_query = substitute_variables(query_str, parameters)
query = HQLdesign(query_form, query_type=query_type)
query._data_dict['query']['query'] = real_query
try:
if explain:
return explain_directly(request, query_server, query)
else:
return execute_directly(request, query, design, query_server, parameters=parameters)
except Exception as ex:
db = dbms.get(request.user, query_server)
error_message, log = expand_exception(ex, db)
response['message'] = error_message
return JsonResponse(response)
else:
response['errors'] = parameterization_form.errors
return JsonResponse(response)
# Non-parameterized query
query = HQLdesign(query_form, query_type=query_type)
if request.GET.get('explain', 'false').lower() == 'true':
return explain_directly(request, query_server, query)
else:
return execute_directly(request, query, design, query_server)
else:
response['message'] = _('There was an error with your query.')
response['errors'] = {
'query': [query_form.query.errors],
'settings': query_form.settings.errors,
'file_resources': query_form.file_resources.errors,
'functions': query_form.functions.errors,
}
except RuntimeError as e:
response['message']= str(e)
return JsonResponse(response)
query_str = form.query.cleaned_data["query"]
# (Optional) Parameterization.
parameterization = get_parameterization(request, query_str, form, design, to_explain)
if parameterization:
return parameterization
try:
query = HQLdesign(form, query_type=query_type)
if to_explain:
return explain_directly(request, query, design, query_server)
else:
download = request.POST.has_key('download')
return execute_directly(request, query, query_server, design, on_success_url=on_success_url, download=download)
except Exception, ex:
error_message, log = expand_exception(ex, db)
else:
if design.id is not None:
data = HQLdesign.loads(design.data).get_query_dict()
form.bind(data)
form.saveform.set_data(design.name, design.desc)
else:
# New design
form.bind()
form.query.fields['database'].choices = databases # Could not do it in the form
if not databases:
request.error(_('No databases are available. Permissions could be missing.'))
return render('execute.mako', request, {
'action': action,
def view_results(request, id, first_row=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the watch_query view.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See watch_query().)
"""
first_row = long(first_row)
start_over = (first_row == 0)
results = None
data = None
fetch_error = False
error_message = ''
log = ''
app_name = get_app_name(request)
query_history = authorized_get_history(request, id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server_config())
handle, state = _get_query_handle_and_state(query_history)
context_param = request.GET.get('context', '')
query_context = _parse_query_context(context_param)
# To remove in Hue 2.3
download = request.GET.get('download', '')
# Update the status as expired should not be accessible
expired = state == QueryHistory.STATE.expired
if expired:
state = QueryHistory.STATE.expired
query_history.save_state(state)
# Retrieve query results
try:
if not download:
results = db.fetch(handle, start_over, 100)
data = list(results.rows()) # Materialize results
# We display the "Download" button only when we know that there are results:
downloadable = first_row > 0 or data
else:
downloadable = True
data = []
results = type('Result', (object, ), {
'rows': 0,
'columns': [],
'has_more': False,
'start_row': 0,
})
log = db.get_log(handle)
except Exception as ex:
fetch_error = True
error_message, log = expand_exception(ex, db)
# Handle errors
error = fetch_error or results is None or expired
context = {
'error': error,
'error_message': error_message,
'has_more': True,
'query': query_history,
'results': data,
'expected_first_row': first_row,
'log': log,
'hadoop_jobs': _parse_out_hadoop_jobs(log)[0],
'query_context': query_context,
'can_save': False,
'context_param': context_param,
'expired': expired,
'app_name': app_name,
'download': download,
}
if not error:
download_urls = {}
if downloadable:
for format in common.DL_FORMATS:
download_urls[format] = urlresolvers.reverse(
'beeswax' + ':download',
kwargs=dict(id=str(id), format=format))
save_form = SaveResultsForm()
results.start_row = first_row
context.update({
'results':
data,
'has_more':
results.has_more,
'next_row':
results.start_row + len(data),
'start_row':
results.start_row,
'expected_first_row':
first_row,
'columns':
results.columns,
'download_urls':
download_urls,
'save_form':
save_form,
'can_save':
query_history.owner == request.user and not download,
})
return render('watch_results.mako', request, context)
def view_results(request, id, first_row=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the execute_query view.
If the result set has has_result_set=False, display an empty result.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See execute_query().)
"""
first_row = long(first_row)
start_over = (first_row == 0)
results = type('Result', (object,), {
'rows': 0,
'columns': [],
'has_more': False,
'start_row': 0,
})
data = []
fetch_error = False
error_message = ''
log = ''
columns = []
app_name = get_app_name(request)
query_history = authorized_get_query_history(request, id, must_exist=True)
query_server = query_history.get_query_server_config()
db = dbms.get(request.user, query_server)
handle, state = _get_query_handle_and_state(query_history)
context_param = request.GET.get('context', '')
query_context = parse_query_context(context_param)
# Update the status as expired should not be accessible
expired = state == models.QueryHistory.STATE.expired
# Retrieve query results or use empty result if no result set
try:
if query_server['server_name'] == 'impala' and not handle.has_result_set:
downloadable = False
else:
results = db.fetch(handle, start_over, 100)
data = []
# Materialize and HTML escape results
# TODO: use Number + list comprehension
for row in results.rows():
escaped_row = []
for field in row:
if isinstance(field, (int, long, float, complex, bool)):
if math.isnan(field) or math.isinf(field):
escaped_field = json.dumps(field)
else:
escaped_field = field
elif field is None:
escaped_field = 'NULL'
else:
field = smart_unicode(field, errors='replace') # Prevent error when getting back non utf8 like charset=iso-8859-1
escaped_field = escape(field).replace(' ', ' ')
escaped_row.append(escaped_field)
data.append(escaped_row)
# We display the "Download" button only when we know that there are results:
downloadable = first_row > 0 or data
log = db.get_log(handle)
columns = results.data_table.cols()
except Exception, ex:
fetch_error = True
error_message, log = expand_exception(ex, db, handle)
def view_results(request, id, first_row=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the execute_query view.
If the result set has has_result_set=False, display an empty result.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See execute_query().)
"""
first_row = long(first_row)
start_over = (first_row == 0)
results = type('Result', (object, ), {
'rows': 0,
'columns': [],
'has_more': False,
'start_row': 0,
})
data = []
fetch_error = False
error_message = ''
log = ''
columns = []
app_name = get_app_name(request)
query_history = authorized_get_query_history(request, id, must_exist=True)
query_server = query_history.get_query_server_config()
db = dbms.get(request.user, query_server)
handle, state = _get_query_handle_and_state(query_history)
context_param = request.GET.get('context', '')
query_context = parse_query_context(context_param)
# Update the status as expired should not be accessible
expired = state == models.QueryHistory.STATE.expired
# Retrieve query results or use empty result if no result set
try:
if query_server[
'server_name'] == 'impala' and not handle.has_result_set:
downloadable = False
else:
results = db.fetch(handle, start_over, 100)
# Materialize and HTML escape results
data = escape_rows(results.rows())
# We display the "Download" button only when we know that there are results:
downloadable = first_row > 0 or data
log = db.get_log(handle)
columns = results.data_table.cols()
except Exception, ex:
LOG.exception('error fetching results')
fetch_error = True
error_message, log = expand_exception(ex, db, handle)
def view_results(request, id, first_row=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the execute_query view.
If the result set has has_result_set=False, display an empty result.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See execute_query().)
"""
first_row = int(first_row)
start_over = (first_row == 0)
results = type('Result', (object, ), {
'rows': 0,
'columns': [],
'has_more': False,
'start_row': 0,
})
data = []
fetch_error = False
error_message = ''
log = ''
columns = []
app_name = get_app_name(request)
query_history = authorized_get_query_history(request, id, must_exist=True)
query_server = query_history.get_query_server_config()
db = dbms.get(request.user, query_server)
handle, state = _get_query_handle_and_state(query_history)
context_param = request.GET.get('context', '')
query_context = parse_query_context(context_param)
# Update the status as expired should not be accessible
expired = state == models.QueryHistory.STATE.expired
# Retrieve query results or use empty result if no result set
try:
if query_server[
'server_name'] == 'impala' and not handle.has_result_set:
downloadable = False
else:
results = db.fetch(handle, start_over, 100)
# Materialize and HTML escape results
data = escape_rows(results.rows())
# We display the "Download" button only when we know that there are results:
downloadable = first_row > 0 or data
log = db.get_log(handle)
columns = results.data_table.cols()
except Exception as ex:
LOG.exception('error fetching results')
fetch_error = True
error_message, log = expand_exception(ex, db, handle)
# Handle errors
error = fetch_error or results is None or expired
context = {
'error': error,
'message': error_message,
'query': query_history,
'results': data,
'columns': columns,
'expected_first_row': first_row,
'log': log,
'hadoop_jobs': app_name != 'impala' and parse_out_jobs(log),
'query_context': query_context,
'can_save': False,
'context_param': context_param,
'expired': expired,
'app_name': app_name,
'next_json_set': None,
'is_finished': query_history.is_finished()
}
if not error:
download_urls = {}
if downloadable:
for format in common.DL_FORMATS:
download_urls[format] = reverse(app_name + ':download',
kwargs=dict(id=str(id),
format=format))
results.start_row = first_row
context.update({
'id':
id,
'results':
data,
'has_more':
results.has_more,
'next_row':
results.start_row + len(data),
'start_row':
results.start_row,
'expected_first_row':
first_row,
'columns':
columns,
'download_urls':
download_urls,
'can_save':
query_history.owner == request.user,
'next_json_set':
reverse(get_app_name(request) + ':view_results',
kwargs={
'id': str(id),
'first_row': results.start_row + len(data)
}) + ('?context=' + context_param or '') + '&format=json'
})
context['columns'] = massage_columns_for_json(columns)
if 'save_form' in context:
del context['save_form']
if 'query' in context:
del context['query']
return JsonResponse(context)
def execute_query(request, design_id=None):
"""
View function for executing an arbitrary query.
It understands the optional GET/POST params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
"""
authorized_get_design(request, design_id)
request.session['start_time'] = time.time(
) # FIXME: add job id to not intersect simultaneous jobs
error_message = None
form = QueryForm()
action = request.path
log = None
app_name = get_app_name(request)
query_type = SavedQuery.TYPES_MAPPING['beeswax']
design = safe_get_design(request, query_type, design_id)
on_success_url = request.REQUEST.get('on_success_url')
query_server = get_query_server_config(app_name)
db = dbms.get(request.user, query_server)
databases = _get_db_choices(request)
if request.method == 'POST':
form.bind(request.POST)
form.query.fields[
'database'].choices = databases # Could not do it in the form
to_explain = request.POST.has_key('button-explain')
to_submit = request.POST.has_key('button-submit')
# Always validate the saveform, which will tell us whether it needs explicit saving
if form.is_valid():
to_save = form.saveform.cleaned_data['save']
to_saveas = form.saveform.cleaned_data['saveas']
if to_save or to_saveas:
if 'beeswax-autosave' in request.session:
del request.session['beeswax-autosave']
if to_saveas and not design.is_auto:
# Save As only affects a previously saved query
design = design.clone()
if to_submit or to_save or to_saveas or to_explain:
explicit_save = to_save or to_saveas
design = save_design(request, form, query_type, design,
explicit_save)
action = urlresolvers.reverse(app_name + ':execute_query',
kwargs=dict(design_id=design.id))
if to_explain or to_submit:
query_str = form.query.cleaned_data["query"]
if conf.CHECK_PARTITION_CLAUSE_IN_QUERY.get():
query_str = _strip_trailing_semicolon(query_str)
# check query. if a select query on partitioned table without partition keys,
# intercept it and raise a PopupException.
_check_partition_clause_in_query(
form.query.cleaned_data.get('database', None),
query_str, db)
# (Optional) Parameterization.
parameterization = get_parameterization(
request, query_str, form, design, to_explain)
if parameterization:
return parameterization
try:
query = HQLdesign(form, query_type=query_type)
if to_explain:
return explain_directly(request, query, design,
query_server)
else:
download = request.POST.has_key('download')
download_format = form.query.cleaned_data.get(
'download_format', None)
if not download_format: download_format = None
if download_format in common.DL_FORMATS:
request.session['dl_status'] = True
return execute_directly(
request,
query,
query_server,
design,
on_success_url=on_success_url,
download_format=download_format,
download=download)
except QueryServerException, ex:
error_message, log = expand_exception(ex, db)
