def execute_query(request, design_id=None, table=None): authorized_get_design(request, design_id) error_message, log = None, None form = query_form() action = request.path design = safe_get_design(request, SavedQuery.HQL, design_id) on_success_url = request.REQUEST.get('on_success_url') for _ in range(1): if request.method == 'POST': form.bind(request.POST) to_explain = request.POST.has_key('button-explain') to_submit = request.POST.has_key('button-submit') # Always validate the saveform, which will tell us whether it needs explicit saving if not form.is_valid(): break to_save = form.saveform.cleaned_data['save'] to_saveas = form.saveform.cleaned_data['saveas'] if to_saveas and not design.is_auto: # Save As only affects a previously saved query design = design.clone() if to_submit or to_save or to_saveas or to_explain: explicit_save = to_save or to_saveas design = save_design(request, form, SavedQuery.HQL, design, explicit_save) action = urlresolvers.reverse(execute_query, kwargs=dict(design_id=design.id)) # We're not going to process the form. Simply re-render it. if not to_explain and not to_submit: break query_str = _strip_trailing_semicolon(form.query.cleaned_data["query"]) query_server = db_utils.get_query_server(form.query_servers.cleaned_data["server"]) # (Optional) Parameterization. parameterization = get_parameterization(request, query_str, form, design, to_explain) if parameterization: return parameterization query_msg = make_beeswax_query(request, query_str, form) try: if to_explain: return expl_d(request, query_str, query_msg, design, query_server) else: notify = form.query.cleaned_data.get('email_notify', False) return e_d(request, query_msg, design=design, on_success_url=on_success_url, notify=notify) except BeeswaxException, ex: error_message, log = expand_exception(ex) # Fall through to re-render the execute form. else: # GET request if design.id is not None: data = HQLdesign.loads(design.data).get_query_dict() form.bind(data) form.saveform.set_data(design.name, design.desc) else: # New design form.bind()
def configuration(request): if request.method == 'POST': server_form = QueryServerForm(request.POST) if server_form.is_valid(): query_server = db_utils.get_query_server(server_form.cleaned_data["server"]) config_values = db_utils.db_client(query_server).get_default_configuration( bool(request.REQUEST.get("include_hadoop", False))) else: server_form = QueryServerForm() config_values = {} return render("configuration.mako", request, {'config_values': config_values, 'server_form': server_form})
def _run_parameterized_query(request, design_id, explain): """ Given a design and arguments to parameterize that design, runs the query. - explain is a boolean to determine whether to run as an explain or as an execute. This is an extra "step" in the flow from execute_query. """ design = authorized_get_design(request, design_id, must_exist=True) # Reconstitute the form design_obj = beeswax.design.HQLdesign.loads(design.data) query_form = beeswax.forms.query_form() params = design_obj.get_query_dict() params.update(request.POST) query_form.bind(params) assert query_form.is_valid() query_str = _strip_trailing_semicolon(query_form.query.cleaned_data["query"]) query_server = db_utils.get_query_server(query_form.query_servers.cleaned_data["server"]) parameterization_form_cls = make_parameterization_form(query_str) if not parameterization_form_cls: raise PopupException(_("Query is not parameterizable.")) parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization") if parameterization_form.is_valid(): real_query = substitute_variables(query_str, parameterization_form.cleaned_data) query_msg = make_beeswax_query(request, real_query, query_form) try: if explain: return explain_directly(request, query_str, query_msg, design, query_server) else: return execute_directly(request, query_msg, query_server, design) except BeeswaxException, ex: error_message, log = expand_exception(ex, query_server) return render('execute.mako', request, { 'action': urlresolvers.reverse(execute_query), 'design': design, 'error_message': error_message, 'form': query_form, 'log': log, })
def execute_query(request, design_id=None, table=None): authorized_get_design(request, design_id) error_message, log = None, None form = query_form() action = request.path design = safe_get_design(request, SavedQuery.HQL, design_id) on_success_url = request.REQUEST.get('on_success_url') for _ in range(1): if request.method == 'POST': form.bind(request.POST) to_explain = request.POST.has_key('button-explain') to_submit = request.POST.has_key('button-submit') # Always validate the saveform, which will tell us whether it needs explicit saving if not form.is_valid(): break to_save = form.saveform.cleaned_data['save'] to_saveas = form.saveform.cleaned_data['saveas'] if to_saveas and not design.is_auto: # Save As only affects a previously saved query design = design.clone() if to_submit or to_save or to_saveas or to_explain: explicit_save = to_save or to_saveas design = save_design(request, form, SavedQuery.HQL, design, explicit_save) action = urlresolvers.reverse(execute_query, kwargs=dict(design_id=design.id)) # We're not going to process the form. Simply re-render it. if not to_explain and not to_submit: break query_str = _strip_trailing_semicolon( form.query.cleaned_data["query"]) query_server = db_utils.get_query_server( form.query_servers.cleaned_data["server"]) # (Optional) Parameterization. parameterization = get_parameterization(request, query_str, form, design, to_explain) if parameterization: return parameterization query_msg = make_beeswax_query(request, query_str, form) try: if to_explain: return expl_d(request, query_str, query_msg, design, query_server) else: notify = form.query.cleaned_data.get('email_notify', False) return e_d(request, query_msg, design=design, on_success_url=on_success_url, notify=notify) except BeeswaxException, ex: error_message, log = expand_exception(ex) # Fall through to re-render the execute form. else: # GET request if design.id is not None: data = HQLdesign.loads(design.data).get_query_dict() form.bind(data) form.saveform.set_data(design.name, design.desc) else: # New design form.bind()
def get_shared_beeswax_server(): # Make it happens only once global _SHARED_BEESWAX_SERVER global _SHARED_BEESWAX_SERVER_CLOSER if _SHARED_BEESWAX_SERVER is None: # Copy hive-default.xml.template from BEESWAX_HIVE_CONF_DIR before it is set to # /my/bogus/path default_xml = file(beeswax.conf.BEESWAX_HIVE_CONF_DIR.get() + "/hive-default.xml.template").read() finish = ( beeswax.conf.QUERY_SERVERS['default'].SERVER_HOST.set_for_testing( "localhost"), beeswax.conf.QUERY_SERVERS['default'].SERVER_PORT.set_for_testing( BEESWAXD_TEST_PORT), beeswax.conf.QUERY_SERVERS['default'].SUPPORT_DDL.set_for_testing( True), beeswax.conf.BEESWAX_META_SERVER_HOST.set_for_testing("localhost"), beeswax.conf.BEESWAX_META_SERVER_PORT.set_for_testing( BEESWAXD_TEST_PORT + 1), # Use a bogus path to avoid loading the normal hive-site.xml beeswax.conf.BEESWAX_HIVE_CONF_DIR.set_for_testing('/my/bogus/path' )) cluster = pseudo_hdfs4.shared_cluster() # Copy hive-default.xml into the mini_cluster's conf dir, which happens to be # in the cluster's tmpdir. This tmpdir is determined during the mini_cluster # startup, during which BEESWAX_HIVE_CONF_DIR needs to be set to # /my/bogus/path. Hence the step of writing to memory. # hive-default.xml will get picked up by the beeswax_server during startup file(cluster._tmpdir + "/conf/hive-default.xml", 'w').write(default_xml) global _SHARED_BEESWAX_SERVER_PROCESS if _SHARED_BEESWAX_SERVER_PROCESS is None: p = _start_server(cluster) _SHARED_BEESWAX_SERVER_PROCESS = p def kill(): LOG.info("Killing beeswax server (pid %d)." % p.pid) os.kill(p.pid, 9) p.wait() atexit.register(kill) # Wait for server to come up, by repeatedly trying. start = time.time() started = False sleep = 0.001 while not started and time.time() - start < 20.0: try: query_server = get_query_server(support_ddl=True) client = beeswax.db_utils.db_client(query_server) meta_client = beeswax.db_utils.meta_client() client.echo("echo") if meta_client.getStatus() == fb303.ttypes.fb_status.ALIVE: started = True break time.sleep(sleep) sleep *= 2 except: time.sleep(sleep) sleep *= 2 pass if not started: raise Exception("Beeswax server took too long to come up.") # Make sure /tmp is 0777 cluster.fs.setuser(cluster.superuser) if not cluster.fs.isdir('/tmp'): cluster.fs.mkdir('/tmp', 0777) else: cluster.fs.chmod('/tmp', 0777) cluster.fs.chmod(cluster._tmpdir, 0777) cluster.fs.chmod(cluster._tmpdir + '/hadoop_tmp_dir/mapred', 0777) def s(): for f in finish: f() cluster.stop() _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER = cluster, s return _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER
def execute_query(request, design_id=None): """ View function for executing an arbitrary query. It understands the optional GET/POST params: on_success_url If given, it will be displayed when the query is successfully finished. Otherwise, it will display the view query results page by default. """ authorized_get_design(request, design_id) error_message = None form = beeswax.forms.query_form() action = request.path log = None design = safe_get_design(request, models.SavedQuery.HQL, design_id) on_success_url = request.REQUEST.get('on_success_url') # Use a loop structure to allow the use of 'break' to get out for _ in range(1): if request.method == 'POST': form.bind(request.POST) to_explain = request.POST.has_key('button-explain') to_submit = request.POST.has_key('button-submit') # Always validate the saveform, which will tell us whether it needs explicit saving if not form.is_valid(): break to_save = form.saveform.cleaned_data['save'] to_saveas = form.saveform.cleaned_data['saveas'] if to_saveas and not design.is_auto: # Save As only affects a previously saved query design = design.clone() if to_submit or to_save or to_saveas or to_explain: explicit_save = to_save or to_saveas design = save_design(request, form, models.SavedQuery.HQL, design, explicit_save) action = urlresolvers.reverse(execute_query, kwargs=dict(design_id=design.id)) # We're not going to process the form. Simply re-render it. if not to_explain and not to_submit: break query_str = _strip_trailing_semicolon(form.query.cleaned_data["query"]) query_server = db_utils.get_query_server(form.query_servers.cleaned_data["server"]) # (Optional) Parameterization. parameterization = get_parameterization(request, query_str, form, design, to_explain) if parameterization: return parameterization query_msg = make_beeswax_query(request, query_str, form) try: if to_explain: return explain_directly(request, query_str, query_msg, design, query_server) else: notify = form.query.cleaned_data.get('email_notify', False) return execute_directly(request, query_msg, query_server, design, on_success_url=on_success_url, notify=notify) except BeeswaxException, ex: error_message, log = expand_exception(ex, query_server) # Fall through to re-render the execute form. else: # GET request if design.id is not None: data = beeswax.design.HQLdesign.loads(design.data).get_query_dict() form.bind(data) form.saveform.set_data(design.name, design.desc) else: # New design form.bind()
def get_shared_beeswax_server(): # Make it happens only once global _SHARED_BEESWAX_SERVER global _SHARED_BEESWAX_SERVER_CLOSER if _SHARED_BEESWAX_SERVER is None: # Copy hive-default.xml.template from BEESWAX_HIVE_CONF_DIR before it is set to # /my/bogus/path default_xml = file(beeswax.conf.BEESWAX_HIVE_CONF_DIR.get()+"/hive-default.xml.template").read() finish = ( beeswax.conf.QUERY_SERVERS['default'].SERVER_HOST.set_for_testing("localhost"), beeswax.conf.QUERY_SERVERS['default'].SERVER_PORT.set_for_testing(BEESWAXD_TEST_PORT), beeswax.conf.QUERY_SERVERS['default'].SUPPORT_DDL.set_for_testing(True), beeswax.conf.BEESWAX_META_SERVER_HOST.set_for_testing("localhost"), beeswax.conf.BEESWAX_META_SERVER_PORT.set_for_testing(BEESWAXD_TEST_PORT + 1), # Use a bogus path to avoid loading the normal hive-site.xml beeswax.conf.BEESWAX_HIVE_CONF_DIR.set_for_testing('/my/bogus/path') ) cluster = pseudo_hdfs4.shared_cluster() # Copy hive-default.xml into the mini_cluster's conf dir, which happens to be # in the cluster's tmpdir. This tmpdir is determined during the mini_cluster # startup, during which BEESWAX_HIVE_CONF_DIR needs to be set to # /my/bogus/path. Hence the step of writing to memory. # hive-default.xml will get picked up by the beeswax_server during startup file(cluster._tmpdir+"/conf/hive-default.xml", 'w').write(default_xml) global _SHARED_BEESWAX_SERVER_PROCESS if _SHARED_BEESWAX_SERVER_PROCESS is None: p = _start_server(cluster) _SHARED_BEESWAX_SERVER_PROCESS = p def kill(): LOG.info("Killing beeswax server (pid %d)." % p.pid) os.kill(p.pid, 9) p.wait() atexit.register(kill) # Wait for server to come up, by repeatedly trying. start = time.time() started = False sleep = 0.001 while not started and time.time() - start < 20.0: try: query_server = get_query_server(support_ddl=True) client = beeswax.db_utils.db_client(query_server) meta_client = beeswax.db_utils.meta_client() client.echo("echo") if meta_client.getStatus() == fb303.ttypes.fb_status.ALIVE: started = True break time.sleep(sleep) sleep *= 2 except: time.sleep(sleep) sleep *= 2 pass if not started: raise Exception("Beeswax server took too long to come up.") # Make sure /tmp is 0777 cluster.fs.setuser(cluster.superuser) if not cluster.fs.isdir('/tmp'): cluster.fs.mkdir('/tmp', 0777) else: cluster.fs.chmod('/tmp', 0777) cluster.fs.chmod(cluster._tmpdir, 0777) cluster.fs.chmod(cluster._tmpdir + '/hadoop_tmp_dir/mapred', 0777) def s(): for f in finish: f() cluster.stop() _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER = cluster, s return _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER