def confirm_query(request, query, on_success_url=None):
"""
Used by other forms to confirm a query before it's executed.
The form is the same as execute_query below.
query - The HQL about to be executed
on_success_url - The page to go to upon successful execution
"""
mform = QueryForm()
mform.bind()
mform.query.initial = dict(query=query)
return render(
"execute.mako",
request,
{
"form": mform,
"action": reverse(get_app_name(request) + ":execute_query"),
"error_message": None,
"design": None,
"on_success_url": on_success_url,
"design": None,
"autocomplete_base_url": reverse(get_app_name(request) + ":autocomplete", kwargs={}),
},
)
def get_query_dict(self):
"""get_query_dict() -> QueryDict"""
# We construct the mform to use its structure and prefix. We don't actually bind
# data to the forms.
from beeswax.forms import QueryForm
mform = QueryForm()
mform.bind()
res = django.http.QueryDict('', mutable=True)
res.update(
denormalize_form_dict(self._data_dict['query'], mform.query,
HQLdesign._QUERY_ATTRS))
res.update(
denormalize_formset_dict(self._data_dict['settings'],
mform.settings,
HQLdesign._SETTINGS_ATTRS))
res.update(
denormalize_formset_dict(self._data_dict['file_resources'],
mform.file_resources,
HQLdesign._FILE_RES_ATTRS))
res.update(
denormalize_formset_dict(self._data_dict['functions'],
mform.functions,
HQLdesign._FUNCTIONS_ATTRS))
return res
def get_query_form(request):
try:
try:
# Get database choices
query_server = dbms.get_query_server_config(get_app_name(request))
db = dbms.get(request.user, query_server)
databases = [(database, database)
for database in db.get_databases()]
except StructuredThriftTransportException as e:
# If Thrift exception was due to failed authentication, raise corresponding message
if 'TSocket read 0 bytes' in str(
e) or 'Error validating the login' in str(e):
raise PopupException(_(
'Failed to authenticate to query server, check authentication configurations.'
),
detail=e)
else:
raise e
except Exception as e:
raise PopupException(_(
'Unable to access databases, Query Server or Metastore may be down.'
),
detail=e)
if not databases:
raise RuntimeError(
_("No databases are available. Permissions could be missing."))
query_form = QueryForm()
query_form.bind(request.POST)
query_form.query.fields[
'database'].choices = databases # Could not do it in the form
return query_form
def execute_query(request, design_id=None):
"""
View function for executing an arbitrary query.
It understands the optional GET/POST params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
"""
authorized_get_design(request, design_id)
error_message = None
form = QueryForm()
action = request.path
log = None
design = safe_get_design(request, models.SavedQuery.HQL, design_id)
on_success_url = request.REQUEST.get('on_success_url')
if request.method == 'POST':
form.bind(request.POST)
to_explain = request.POST.has_key('button-explain')
to_submit = request.POST.has_key('button-submit')
# Always validate the saveform, which will tell us whether it needs explicit saving
if form.is_valid():
to_save = form.saveform.cleaned_data['save']
to_saveas = form.saveform.cleaned_data['saveas']
if to_saveas and not design.is_auto:
# Save As only affects a previously saved query
design = design.clone()
if to_submit or to_save or to_saveas or to_explain:
explicit_save = to_save or to_saveas
design = save_design(request, form, models.SavedQuery.HQL, design, explicit_save)
action = urlresolvers.reverse(execute_query, kwargs=dict(design_id=design.id))
if to_explain or to_submit:
query_str = form.query.cleaned_data["query"]
query_server = get_query_server(form.query_servers.cleaned_data["server"])
# (Optional) Parameterization.
parameterization = get_parameterization(request, query_str, form, design, to_explain)
if parameterization:
return parameterization
try:
query = HQLdesign(form)
if to_explain:
return explain_directly(request, query, design, query_server)
else:
notify = form.query.cleaned_data.get('email_notify', False)
return execute_directly(request, query, query_server, design, on_success_url=on_success_url, notify=notify)
except BeeswaxException, ex:
print ex.errorCode
print ex.SQLState
db = dbms.get(request.user, query_server)
error_message, log = expand_exception(ex, db)
def _run_parameterized_query(request, design_id, explain):
"""
Given a design and arguments to parameterize that design, runs the query.
- explain is a boolean to determine whether to run as an explain or as an
execute.
This is an extra "step" in the flow from execute_query.
"""
design = authorized_get_design(request, design_id, must_exist=True)
# Reconstitute the form
design_obj = beeswax.design.HQLdesign.loads(design.data)
query_form = QueryForm()
params = design_obj.get_query_dict()
params.update(request.POST)
databases = _get_db_choices(request)
query_form.bind(params)
query_form.query.fields["database"].choices = databases # Could not do it in the form
if not query_form.is_valid():
raise PopupException(_("Query form is invalid: %s") % query_form.errors)
query_str = query_form.query.cleaned_data["query"]
app_name = get_app_name(request)
query_server = get_query_server_config(app_name)
query_type = SavedQuery.TYPES_MAPPING[app_name]
parameterization_form_cls = make_parameterization_form(query_str)
if not parameterization_form_cls:
raise PopupException(_("Query is not parameterizable."))
parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization")
if parameterization_form.is_valid():
real_query = substitute_variables(query_str, parameterization_form.cleaned_data)
query = HQLdesign(query_form, query_type=query_type)
query._data_dict["query"]["query"] = real_query
try:
if explain:
return explain_directly(request, query, design, query_server)
else:
return execute_directly(request, query, query_server, design)
except Exception, ex:
db = dbms.get(request.user, query_server)
error_message, log = expand_exception(ex, db)
return render(
"execute.mako",
request,
{
"action": reverse(get_app_name(request) + ":execute_query"),
"design": design,
"error_message": error_message,
"form": query_form,
"log": log,
"autocomplete_base_url": reverse(get_app_name(request) + ":autocomplete", kwargs={}),
},
)
def get_query_dict(self):
# We construct the form to use its structure and prefix. We don't actually bind data to the forms.
from beeswax.forms import QueryForm
mform = QueryForm()
mform.bind()
res = django.http.QueryDict('', mutable=True)
res.update(denormalize_form_dict(self._data_dict['query'], mform.query, SparkDesign._QUERY_ATTRS))
return res
def get_query_dict(self):
# We construct the mform to use its structure and prefix. We don't actually bind data to the forms.
from beeswax.forms import QueryForm
mform = QueryForm()
mform.bind()
res = django.http.QueryDict('', mutable=True)
res.update(denormalize_form_dict(
self._data_dict['query'], mform.query, SQLdesign._QUERY_ATTRS))
return res
def _run_parameterized_query(request, design_id, explain):
"""
Given a design and arguments to parameterize that design, runs the query.
- explain is a boolean to determine whether to run as an explain or as an
execute.
This is an extra "step" in the flow from execute_query.
"""
design = authorized_get_design(request, design_id, must_exist=True)
# Reconstitute the form
design_obj = beeswax.design.HQLdesign.loads(design.data)
query_form = QueryForm()
params = design_obj.get_query_dict()
params.update(request.POST)
databases = _get_db_choices(request)
query_form.bind(params)
query_form.query.fields['database'].choices = databases # Could not do it in the form
if not query_form.is_valid():
raise PopupException(_("Query form is invalid: %s") % query_form.errors)
query_str = query_form.query.cleaned_data["query"]
app_name = get_app_name(request)
query_server = get_query_server_config(app_name)
query_type = SavedQuery.TYPES_MAPPING[app_name]
parameterization_form_cls = make_parameterization_form(query_str)
if not parameterization_form_cls:
raise PopupException(_("Query is not parameterizable."))
parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization")
if parameterization_form.is_valid():
real_query = substitute_variables(query_str, parameterization_form.cleaned_data)
query = HQLdesign(query_form, query_type=query_type)
query._data_dict['query']['query'] = real_query
try:
if explain:
return explain_directly(request, query, design, query_server)
else:
return execute_directly(request, query, query_server, design)
except Exception, ex:
db = dbms.get(request.user, query_server)
error_message, log = expand_exception(ex, db)
return render('execute.mako', request, {
'action': reverse(get_app_name(request) + ':execute_query'),
'design': design,
'error_message': error_message,
'form': query_form,
'log': log,
'autocomplete_base_url': reverse(get_app_name(request) + ':autocomplete', kwargs={}),
})
def get_query_form(request):
# Get database choices
query_server = dbms.get_query_server_config(get_app_name(request))
db = dbms.get(request.user, query_server)
databases = [(database, database) for database in db.get_databases()]
if not databases:
raise RuntimeError(_("No databases are available. Permissions could be missing."))
query_form = QueryForm()
query_form.bind(request.POST)
query_form.query.fields['database'].choices = databases # Could not do it in the form
return query_form
def get_query_form(request):
# Get database choices
query_server = dbms.get_query_server_config(get_app_name(request))
db = dbms.get(request.user, query_server)
databases = [(database, database) for database in db.get_databases()]
if not databases:
raise RuntimeError(_("No databases are available. Permissions could be missing."))
query_form = QueryForm()
query_form.bind(request.POST)
query_form.query.fields['database'].choices = databases # Could not do it in the form
return query_form
def get_query_dict(self):
# We construct the mform to use its structure and prefix. We don't actually bind data to the forms.
from beeswax.forms import QueryForm
mform = QueryForm()
mform.bind()
res = django.http.QueryDict("", mutable=True)
res.update(denormalize_form_dict(self._data_dict["query"], mform.query, HQLdesign._QUERY_ATTRS))
res.update(denormalize_formset_dict(self._data_dict["settings"], mform.settings, HQLdesign._SETTINGS_ATTRS))
res.update(
denormalize_formset_dict(self._data_dict["file_resources"], mform.file_resources, HQLdesign._FILE_RES_ATTRS)
)
res.update(denormalize_formset_dict(self._data_dict["functions"], mform.functions, HQLdesign._FUNCTIONS_ATTRS))
return res
def execute_query(request, design_id=None):
"""
View function for executing an arbitrary query.
It understands the optional GET/POST params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
"""
authorized_get_design(request, design_id)
error_message = None
form = QueryForm()
action = request.path
log = None
app_name = get_app_name(request)
query_type = SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
on_success_url = request.REQUEST.get('on_success_url')
databases = []
query_server = get_query_server_config(app_name)
db = dbms.get(request.user, query_server)
try:
databases = get_db_choices(request)
except Exception, ex:
error_message, log = expand_exception(ex, db)
def _run_parameterized_query(request, design_id, explain):
"""
Given a design and arguments to parameterize that design, runs the query.
- explain is a boolean to determine whether to run as an explain or as an
execute.
This is an extra "step" in the flow from execute_query.
"""
design = authorized_get_design(request, design_id, must_exist=True)
# Reconstitute the form
design_obj = beeswax.design.HQLdesign.loads(design.data)
query_form = QueryForm()
params = design_obj.get_query_dict()
params.update(request.POST)
query_form.bind(params)
assert query_form.is_valid()
query_str = query_form.query.cleaned_data["query"]
query_server = get_query_server_config(get_app_name(request))
parameterization_form_cls = make_parameterization_form(query_str)
if not parameterization_form_cls:
raise PopupException(_("Query is not parameterizable."))
parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization")
if parameterization_form.is_valid():
real_query = substitute_variables(query_str, parameterization_form.cleaned_data)
query = HQLdesign(query_form)
query._data_dict['query']['query'] = real_query
try:
if explain:
return explain_directly(request, query, design, query_server)
else:
return execute_directly(request, query, query_server, design)
except Exception, ex:
db = dbms.get(request.user, query_server)
error_message, log = expand_exception(ex, db)
return render('execute.mako', request, {
'action': reverse(get_app_name(request) + ':execute_query'),
'design': design,
'error_message': error_message,
'form': query_form,
'log': log,
'autocomplete': json.dumps({}),
})
def confirm_query(request, query, on_success_url=None):
"""
Used by other forms to confirm a query before it's executed.
The form is the same as execute_query below.
query - The HQL about to be executed
on_success_url - The page to go to upon successful execution
"""
mform = QueryForm()
mform.bind()
mform.query.initial = dict(query=query)
return render('execute.mako', request, {
'form': mform,
'action': urlresolvers.reverse(execute_query),
'error_message': None,
'design': None,
'on_success_url': on_success_url,
})
def confirm_query(request, query, on_success_url=None):
"""
Used by other forms to confirm a query before it's executed.
The form is the same as execute_query below.
query - The HQL about to be executed
on_success_url - The page to go to upon successful execution
"""
mform = QueryForm()
mform.bind()
mform.query.initial = dict(query=query)
return render('execute.mako', request, {
'form': mform,
'action': reverse(get_app_name(request) + ':execute_query'),
'error_message': None,
'design': None,
'on_success_url': on_success_url,
'design': None,
})
query_server = dbms.get_query_server_config(get_app_name(request))
db = dbms.get(request.user, query_server)
databases = [(database, database) for database in db.get_databases()]
except StructuredThriftTransportException, e:
# If Thrift exception was due to failed authentication, raise corresponding message
if 'TSocket read 0 bytes' in str(e) or 'Error validating the login' in str(e):
raise PopupException(_('Failed to authenticate to query server, check authentication configurations.'), detail=e)
else:
raise e
except Exception, e:
raise PopupException(_('Unable to access databases, Query Server or Metastore may be down.'), detail=e)
if not databases:
raise RuntimeError(_("No databases are available. Permissions could be missing."))
query_form = QueryForm()
query_form.bind(request.POST)
query_form.query.fields['database'].choices = databases # Could not do it in the form
return query_form
@error_handler
def analyze_table(request, database, table, columns=None):
app_name = get_app_name(request)
query_server = get_query_server_config(app_name)
db = dbms.get(request.user, query_server)
response = {'status': -1, 'message': '', 'redirect': ''}
if request.method == "POST":
def execute_query(request, design_id=None):
"""
View function for executing an arbitrary query.
It understands the optional GET/POST params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
"""
authorized_get_design(request, design_id)
error_message = None
form = QueryForm()
action = request.path
log = None
app_name = get_app_name(request)
query_type = SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
on_success_url = request.REQUEST.get('on_success_url')
query_server = get_query_server_config(app_name)
db = dbms.get(request.user, query_server)
databases = _get_db_choices(request)
if request.method == 'POST':
form.bind(request.POST)
form.query.fields['database'].choices = databases # Could not do it in the form
to_explain = request.POST.has_key('button-explain')
to_submit = request.POST.has_key('button-submit')
# Always validate the saveform, which will tell us whether it needs explicit saving
if form.is_valid():
to_save = form.saveform.cleaned_data['save']
to_saveas = form.saveform.cleaned_data['saveas']
if to_saveas and not design.is_auto:
# Save As only affects a previously saved query
design = design.clone()
if to_submit or to_save or to_saveas or to_explain:
explicit_save = to_save or to_saveas
design = save_design(request, form, query_type, design, explicit_save)
action = reverse(app_name + ':execute_query', kwargs=dict(design_id=design.id))
if to_explain or to_submit:
query_str = form.query.cleaned_data["query"]
# (Optional) Parameterization.
parameterization = get_parameterization(request, query_str, form, design, to_explain)
if parameterization:
return parameterization
try:
query = HQLdesign(form, query_type=query_type)
if to_explain:
return explain_directly(request, query, design, query_server)
else:
download = request.POST.has_key('download')
return execute_directly(request, query, query_server, design, on_success_url=on_success_url, download=download)
except Exception, ex:
error_message, log = expand_exception(ex, db)
def execute_query(request, design_id=None):
"""
View function for executing an arbitrary query.
It understands the optional GET/POST params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
"""
authorized_get_design(request, design_id)
request.session['start_time'] = time.time() # FIXME: add job id to not intersect simultaneous jobs
error_message = None
form = QueryForm()
action = request.path
log = None
app_name = get_app_name(request)
query_type = SavedQuery.TYPES_MAPPING['beeswax']
design = safe_get_design(request, query_type, design_id)
on_success_url = request.REQUEST.get('on_success_url')
query_server = get_query_server_config(app_name)
db = dbms.get(request.user, query_server)
databases = _get_db_choices(request)
if request.method == 'POST':
form.bind(request.POST)
form.query.fields['database'].choices = databases # Could not do it in the form
to_explain = request.POST.has_key('button-explain')
to_submit = request.POST.has_key('button-submit')
# Always validate the saveform, which will tell us whether it needs explicit saving
if form.is_valid():
to_save = form.saveform.cleaned_data['save']
to_saveas = form.saveform.cleaned_data['saveas']
if to_save or to_saveas:
if 'beeswax-autosave' in request.session:
del request.session['beeswax-autosave']
if to_saveas and not design.is_auto:
# Save As only affects a previously saved query
design = design.clone()
if to_submit or to_save or to_saveas or to_explain:
explicit_save = to_save or to_saveas
design = save_design(request, form, query_type, design, explicit_save)
action = urlresolvers.reverse(app_name + ':execute_query', kwargs=dict(design_id=design.id))
if to_explain or to_submit:
query_str = form.query.cleaned_data["query"]
if conf.CHECK_PARTITION_CLAUSE_IN_QUERY.get():
query_str = _strip_trailing_semicolon(query_str)
# check query. if a select query on partitioned table without partition keys,
# intercept it and raise a PopupException.
_check_partition_clause_in_query(form.query.cleaned_data.get('database', None), query_str, db)
# (Optional) Parameterization.
parameterization = get_parameterization(request, query_str, form, design, to_explain)
if parameterization:
return parameterization
try:
query = HQLdesign(form, query_type=query_type)
if to_explain:
return explain_directly(request, query, design, query_server)
else:
download = request.POST.has_key('download')
download_format = form.query.cleaned_data.get('download_format', None)
if not download_format: download_format = None
if download_format in common.DL_FORMATS:
request.session['dl_status'] = True
return execute_directly(request, query, query_server, design, on_success_url=on_success_url, download_format=download_format, download=download)
except QueryServerException, ex:
error_message, log = expand_exception(ex, db)
def execute_query(request, design_id=None):
"""
View function for executing an arbitrary query.
It understands the optional GET/POST params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
"""
authorized_get_design(request, design_id)
error_message = None
form = QueryForm()
action = request.path
log = None
app_name = get_app_name(request)
query_type = SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
on_success_url = request.REQUEST.get('on_success_url')
query_server = get_query_server_config(app_name)
db = dbms.get(request.user, query_server)
databases = _get_db_choices(request)
if request.method == 'POST':
form.bind(request.POST)
form.query.fields['database'].choices = databases # Could not do it in the form
to_explain = request.POST.has_key('button-explain')
to_submit = request.POST.has_key('button-submit')
# Always validate the saveform, which will tell us whether it needs explicit saving
if form.is_valid():
to_save = form.saveform.cleaned_data['save']
to_saveas = form.saveform.cleaned_data['saveas']
if to_saveas and not design.is_auto:
# Save As only affects a previously saved query
design = design.clone()
if to_submit or to_save or to_saveas or to_explain:
explicit_save = to_save or to_saveas
design = save_design(request, form, query_type, design, explicit_save)
action = reverse(app_name + ':execute_query', kwargs=dict(design_id=design.id))
if to_explain or to_submit:
query_str = form.query.cleaned_data["query"]
# (Optional) Parameterization.
parameterization = get_parameterization(request, query_str, form, design, to_explain)
if parameterization:
return parameterization
try:
query = HQLdesign(form, query_type=query_type)
if to_explain:
return explain_directly(request, query, design, query_server)
else:
download = request.POST.has_key('download')
return execute_directly(request, query, query_server, design, on_success_url=on_success_url, download=download)
except Exception, ex:
error_message, log = expand_exception(ex, db)
query_server = dbms.get_query_server_config(get_app_name(request))
db = dbms.get(request.user, query_server)
databases = [(database, database) for database in db.get_databases()]
except StructuredThriftTransportException, e:
# If Thrift exception was due to failed authentication, raise corresponding message
if 'TSocket read 0 bytes' in str(e) or 'Error validating the login' in str(e):
raise PopupException(_('Failed to authenticate to query server, check authentication configurations.'), detail=e)
else:
raise e
except Exception, e:
raise PopupException(_('Unable to access databases, Query Server or Metastore may be down.'), detail=e)
if not databases:
raise RuntimeError(_("No databases are available. Permissions could be missing."))
query_form = QueryForm()
query_form.bind(request.POST)
query_form.query.fields['database'].choices = databases # Could not do it in the form
return query_form
"""
Utils
"""
def _extract_nested_type(parse_tree, nested_path):
nested_tokens = nested_path.strip('/').split('/')
subtree = parse_tree
for token in nested_tokens:
def execute_query(request, design_id=None):
"""
View function for executing an arbitrary query.
It understands the optional GET/POST params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
"""
authorized_get_design(request, design_id)
request.session['start_time'] = time.time(
) # FIXME: add job id to not intersect simultaneous jobs
error_message = None
form = QueryForm()
action = request.path
log = None
app_name = get_app_name(request)
query_type = SavedQuery.TYPES_MAPPING['beeswax']
design = safe_get_design(request, query_type, design_id)
on_success_url = request.REQUEST.get('on_success_url')
query_server = get_query_server_config(app_name)
db = dbms.get(request.user, query_server)
databases = _get_db_choices(request)
if request.method == 'POST':
form.bind(request.POST)
form.query.fields[
'database'].choices = databases # Could not do it in the form
to_explain = request.POST.has_key('button-explain')
to_submit = request.POST.has_key('button-submit')
# Always validate the saveform, which will tell us whether it needs explicit saving
if form.is_valid():
to_save = form.saveform.cleaned_data['save']
to_saveas = form.saveform.cleaned_data['saveas']
if to_save or to_saveas:
if 'beeswax-autosave' in request.session:
del request.session['beeswax-autosave']
if to_saveas and not design.is_auto:
# Save As only affects a previously saved query
design = design.clone()
if to_submit or to_save or to_saveas or to_explain:
explicit_save = to_save or to_saveas
design = save_design(request, form, query_type, design,
explicit_save)
action = urlresolvers.reverse(app_name + ':execute_query',
kwargs=dict(design_id=design.id))
if to_explain or to_submit:
query_str = form.query.cleaned_data["query"]
if conf.CHECK_PARTITION_CLAUSE_IN_QUERY.get():
query_str = _strip_trailing_semicolon(query_str)
# check query. if a select query on partitioned table without partition keys,
# intercept it and raise a PopupException.
_check_partition_clause_in_query(
form.query.cleaned_data.get('database', None),
query_str, db)
# (Optional) Parameterization.
parameterization = get_parameterization(
request, query_str, form, design, to_explain)
if parameterization:
return parameterization
try:
query = HQLdesign(form, query_type=query_type)
if to_explain:
return explain_directly(request, query, design,
query_server)
else:
download = request.POST.has_key('download')
download_format = form.query.cleaned_data.get(
'download_format', None)
if not download_format: download_format = None
if download_format in common.DL_FORMATS:
request.session['dl_status'] = True
return execute_directly(
request,
query,
query_server,
design,
on_success_url=on_success_url,
download_format=download_format,
download=download)
except QueryServerException, ex:
error_message, log = expand_exception(ex, db)
