def execute_query(request, design_id=None, table=None):
authorized_get_design(request, design_id)
error_message, log = None, None
form = query_form()
action = request.path
design = safe_get_design(request, SavedQuery.HQL, design_id)
on_success_url = request.REQUEST.get('on_success_url')
for _ in range(1):
if request.method == 'POST':
form.bind(request.POST)
to_explain = request.POST.has_key('button-explain')
to_submit = request.POST.has_key('button-submit')
# Always validate the saveform, which will tell us whether it needs explicit saving
if not form.is_valid():
break
to_save = form.saveform.cleaned_data['save']
to_saveas = form.saveform.cleaned_data['saveas']
if to_saveas and not design.is_auto:
# Save As only affects a previously saved query
design = design.clone()
if to_submit or to_save or to_saveas or to_explain:
explicit_save = to_save or to_saveas
design = save_design(request, form, SavedQuery.HQL, design, explicit_save)
action = urlresolvers.reverse(execute_query, kwargs=dict(design_id=design.id))
# We're not going to process the form. Simply re-render it.
if not to_explain and not to_submit:
break
query_str = _strip_trailing_semicolon(form.query.cleaned_data["query"])
query_server = db_utils.get_query_server(form.query_servers.cleaned_data["server"])
# (Optional) Parameterization.
parameterization = get_parameterization(request, query_str, form, design, to_explain)
if parameterization:
return parameterization
query_msg = make_beeswax_query(request, query_str, form)
try:
if to_explain:
return expl_d(request, query_str, query_msg, design, query_server)
else:
notify = form.query.cleaned_data.get('email_notify', False)
return e_d(request, query_msg, design=design,
on_success_url=on_success_url,
notify=notify)
except BeeswaxException, ex:
error_message, log = expand_exception(ex)
# Fall through to re-render the execute form.
else:
# GET request
if design.id is not None:
data = HQLdesign.loads(design.data).get_query_dict()
form.bind(data)
form.saveform.set_data(design.name, design.desc)
else:
# New design
form.bind()
def configuration(request):
if request.method == 'POST':
server_form = QueryServerForm(request.POST)
if server_form.is_valid():
query_server = db_utils.get_query_server(server_form.cleaned_data["server"])
config_values = db_utils.db_client(query_server).get_default_configuration(
bool(request.REQUEST.get("include_hadoop", False)))
else:
server_form = QueryServerForm()
config_values = {}
return render("configuration.mako", request, {'config_values': config_values,
'server_form': server_form})
def _run_parameterized_query(request, design_id, explain):
"""
Given a design and arguments to parameterize that design, runs the query.
- explain is a boolean to determine whether to run as an explain or as an
execute.
This is an extra "step" in the flow from execute_query.
"""
design = authorized_get_design(request, design_id, must_exist=True)
# Reconstitute the form
design_obj = beeswax.design.HQLdesign.loads(design.data)
query_form = beeswax.forms.query_form()
params = design_obj.get_query_dict()
params.update(request.POST)
query_form.bind(params)
assert query_form.is_valid()
query_str = _strip_trailing_semicolon(query_form.query.cleaned_data["query"])
query_server = db_utils.get_query_server(query_form.query_servers.cleaned_data["server"])
parameterization_form_cls = make_parameterization_form(query_str)
if not parameterization_form_cls:
raise PopupException(_("Query is not parameterizable."))
parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization")
if parameterization_form.is_valid():
real_query = substitute_variables(query_str, parameterization_form.cleaned_data)
query_msg = make_beeswax_query(request, real_query, query_form)
try:
if explain:
return explain_directly(request, query_str, query_msg, design, query_server)
else:
return execute_directly(request, query_msg, query_server, design)
except BeeswaxException, ex:
error_message, log = expand_exception(ex, query_server)
return render('execute.mako', request, {
'action': urlresolvers.reverse(execute_query),
'design': design,
'error_message': error_message,
'form': query_form,
'log': log,
})
def execute_query(request, design_id=None, table=None):
authorized_get_design(request, design_id)
error_message, log = None, None
form = query_form()
action = request.path
design = safe_get_design(request, SavedQuery.HQL, design_id)
on_success_url = request.REQUEST.get('on_success_url')
for _ in range(1):
if request.method == 'POST':
form.bind(request.POST)
to_explain = request.POST.has_key('button-explain')
to_submit = request.POST.has_key('button-submit')
# Always validate the saveform, which will tell us whether it needs explicit saving
if not form.is_valid():
break
to_save = form.saveform.cleaned_data['save']
to_saveas = form.saveform.cleaned_data['saveas']
if to_saveas and not design.is_auto:
# Save As only affects a previously saved query
design = design.clone()
if to_submit or to_save or to_saveas or to_explain:
explicit_save = to_save or to_saveas
design = save_design(request, form, SavedQuery.HQL, design,
explicit_save)
action = urlresolvers.reverse(execute_query,
kwargs=dict(design_id=design.id))
# We're not going to process the form. Simply re-render it.
if not to_explain and not to_submit:
break
query_str = _strip_trailing_semicolon(
form.query.cleaned_data["query"])
query_server = db_utils.get_query_server(
form.query_servers.cleaned_data["server"])
# (Optional) Parameterization.
parameterization = get_parameterization(request, query_str, form,
design, to_explain)
if parameterization:
return parameterization
query_msg = make_beeswax_query(request, query_str, form)
try:
if to_explain:
return expl_d(request, query_str, query_msg, design,
query_server)
else:
notify = form.query.cleaned_data.get('email_notify', False)
return e_d(request,
query_msg,
design=design,
on_success_url=on_success_url,
notify=notify)
except BeeswaxException, ex:
error_message, log = expand_exception(ex)
# Fall through to re-render the execute form.
else:
# GET request
if design.id is not None:
data = HQLdesign.loads(design.data).get_query_dict()
form.bind(data)
form.saveform.set_data(design.name, design.desc)
else:
# New design
form.bind()
def get_shared_beeswax_server():
# Make it happens only once
global _SHARED_BEESWAX_SERVER
global _SHARED_BEESWAX_SERVER_CLOSER
if _SHARED_BEESWAX_SERVER is None:
# Copy hive-default.xml.template from BEESWAX_HIVE_CONF_DIR before it is set to
# /my/bogus/path
default_xml = file(beeswax.conf.BEESWAX_HIVE_CONF_DIR.get() +
"/hive-default.xml.template").read()
finish = (
beeswax.conf.QUERY_SERVERS['default'].SERVER_HOST.set_for_testing(
"localhost"),
beeswax.conf.QUERY_SERVERS['default'].SERVER_PORT.set_for_testing(
BEESWAXD_TEST_PORT),
beeswax.conf.QUERY_SERVERS['default'].SUPPORT_DDL.set_for_testing(
True),
beeswax.conf.BEESWAX_META_SERVER_HOST.set_for_testing("localhost"),
beeswax.conf.BEESWAX_META_SERVER_PORT.set_for_testing(
BEESWAXD_TEST_PORT + 1),
# Use a bogus path to avoid loading the normal hive-site.xml
beeswax.conf.BEESWAX_HIVE_CONF_DIR.set_for_testing('/my/bogus/path'
))
cluster = pseudo_hdfs4.shared_cluster()
# Copy hive-default.xml into the mini_cluster's conf dir, which happens to be
# in the cluster's tmpdir. This tmpdir is determined during the mini_cluster
# startup, during which BEESWAX_HIVE_CONF_DIR needs to be set to
# /my/bogus/path. Hence the step of writing to memory.
# hive-default.xml will get picked up by the beeswax_server during startup
file(cluster._tmpdir + "/conf/hive-default.xml",
'w').write(default_xml)
global _SHARED_BEESWAX_SERVER_PROCESS
if _SHARED_BEESWAX_SERVER_PROCESS is None:
p = _start_server(cluster)
_SHARED_BEESWAX_SERVER_PROCESS = p
def kill():
LOG.info("Killing beeswax server (pid %d)." % p.pid)
os.kill(p.pid, 9)
p.wait()
atexit.register(kill)
# Wait for server to come up, by repeatedly trying.
start = time.time()
started = False
sleep = 0.001
while not started and time.time() - start < 20.0:
try:
query_server = get_query_server(support_ddl=True)
client = beeswax.db_utils.db_client(query_server)
meta_client = beeswax.db_utils.meta_client()
client.echo("echo")
if meta_client.getStatus() == fb303.ttypes.fb_status.ALIVE:
started = True
break
time.sleep(sleep)
sleep *= 2
except:
time.sleep(sleep)
sleep *= 2
pass
if not started:
raise Exception("Beeswax server took too long to come up.")
# Make sure /tmp is 0777
cluster.fs.setuser(cluster.superuser)
if not cluster.fs.isdir('/tmp'):
cluster.fs.mkdir('/tmp', 0777)
else:
cluster.fs.chmod('/tmp', 0777)
cluster.fs.chmod(cluster._tmpdir, 0777)
cluster.fs.chmod(cluster._tmpdir + '/hadoop_tmp_dir/mapred', 0777)
def s():
for f in finish:
f()
cluster.stop()
_SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER = cluster, s
return _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER
def execute_query(request, design_id=None):
"""
View function for executing an arbitrary query.
It understands the optional GET/POST params:
on_success_url
If given, it will be displayed when the query is successfully finished.
Otherwise, it will display the view query results page by default.
"""
authorized_get_design(request, design_id)
error_message = None
form = beeswax.forms.query_form()
action = request.path
log = None
design = safe_get_design(request, models.SavedQuery.HQL, design_id)
on_success_url = request.REQUEST.get('on_success_url')
# Use a loop structure to allow the use of 'break' to get out
for _ in range(1):
if request.method == 'POST':
form.bind(request.POST)
to_explain = request.POST.has_key('button-explain')
to_submit = request.POST.has_key('button-submit')
# Always validate the saveform, which will tell us whether it needs explicit saving
if not form.is_valid():
break
to_save = form.saveform.cleaned_data['save']
to_saveas = form.saveform.cleaned_data['saveas']
if to_saveas and not design.is_auto:
# Save As only affects a previously saved query
design = design.clone()
if to_submit or to_save or to_saveas or to_explain:
explicit_save = to_save or to_saveas
design = save_design(request, form, models.SavedQuery.HQL, design, explicit_save)
action = urlresolvers.reverse(execute_query, kwargs=dict(design_id=design.id))
# We're not going to process the form. Simply re-render it.
if not to_explain and not to_submit:
break
query_str = _strip_trailing_semicolon(form.query.cleaned_data["query"])
query_server = db_utils.get_query_server(form.query_servers.cleaned_data["server"])
# (Optional) Parameterization.
parameterization = get_parameterization(request, query_str, form, design, to_explain)
if parameterization:
return parameterization
query_msg = make_beeswax_query(request, query_str, form)
try:
if to_explain:
return explain_directly(request, query_str, query_msg, design, query_server)
else:
notify = form.query.cleaned_data.get('email_notify', False)
return execute_directly(request, query_msg, query_server, design,
on_success_url=on_success_url,
notify=notify)
except BeeswaxException, ex:
error_message, log = expand_exception(ex, query_server)
# Fall through to re-render the execute form.
else:
# GET request
if design.id is not None:
data = beeswax.design.HQLdesign.loads(design.data).get_query_dict()
form.bind(data)
form.saveform.set_data(design.name, design.desc)
else:
# New design
form.bind()
def get_shared_beeswax_server():
# Make it happens only once
global _SHARED_BEESWAX_SERVER
global _SHARED_BEESWAX_SERVER_CLOSER
if _SHARED_BEESWAX_SERVER is None:
# Copy hive-default.xml.template from BEESWAX_HIVE_CONF_DIR before it is set to
# /my/bogus/path
default_xml = file(beeswax.conf.BEESWAX_HIVE_CONF_DIR.get()+"/hive-default.xml.template").read()
finish = (
beeswax.conf.QUERY_SERVERS['default'].SERVER_HOST.set_for_testing("localhost"),
beeswax.conf.QUERY_SERVERS['default'].SERVER_PORT.set_for_testing(BEESWAXD_TEST_PORT),
beeswax.conf.QUERY_SERVERS['default'].SUPPORT_DDL.set_for_testing(True),
beeswax.conf.BEESWAX_META_SERVER_HOST.set_for_testing("localhost"),
beeswax.conf.BEESWAX_META_SERVER_PORT.set_for_testing(BEESWAXD_TEST_PORT + 1),
# Use a bogus path to avoid loading the normal hive-site.xml
beeswax.conf.BEESWAX_HIVE_CONF_DIR.set_for_testing('/my/bogus/path')
)
cluster = pseudo_hdfs4.shared_cluster()
# Copy hive-default.xml into the mini_cluster's conf dir, which happens to be
# in the cluster's tmpdir. This tmpdir is determined during the mini_cluster
# startup, during which BEESWAX_HIVE_CONF_DIR needs to be set to
# /my/bogus/path. Hence the step of writing to memory.
# hive-default.xml will get picked up by the beeswax_server during startup
file(cluster._tmpdir+"/conf/hive-default.xml", 'w').write(default_xml)
global _SHARED_BEESWAX_SERVER_PROCESS
if _SHARED_BEESWAX_SERVER_PROCESS is None:
p = _start_server(cluster)
_SHARED_BEESWAX_SERVER_PROCESS = p
def kill():
LOG.info("Killing beeswax server (pid %d)." % p.pid)
os.kill(p.pid, 9)
p.wait()
atexit.register(kill)
# Wait for server to come up, by repeatedly trying.
start = time.time()
started = False
sleep = 0.001
while not started and time.time() - start < 20.0:
try:
query_server = get_query_server(support_ddl=True)
client = beeswax.db_utils.db_client(query_server)
meta_client = beeswax.db_utils.meta_client()
client.echo("echo")
if meta_client.getStatus() == fb303.ttypes.fb_status.ALIVE:
started = True
break
time.sleep(sleep)
sleep *= 2
except:
time.sleep(sleep)
sleep *= 2
pass
if not started:
raise Exception("Beeswax server took too long to come up.")
# Make sure /tmp is 0777
cluster.fs.setuser(cluster.superuser)
if not cluster.fs.isdir('/tmp'):
cluster.fs.mkdir('/tmp', 0777)
else:
cluster.fs.chmod('/tmp', 0777)
cluster.fs.chmod(cluster._tmpdir, 0777)
cluster.fs.chmod(cluster._tmpdir + '/hadoop_tmp_dir/mapred', 0777)
def s():
for f in finish:
f()
cluster.stop()
_SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER = cluster, s
return _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER
