예제 #1
0
def execute_query(request, design_id=None, table=None):
  authorized_get_design(request, design_id)
  error_message, log = None, None
  form = query_form()
  action = request.path
  design = safe_get_design(request, SavedQuery.HQL, design_id)
  on_success_url = request.REQUEST.get('on_success_url')

  for _ in range(1):
    if request.method == 'POST':
      form.bind(request.POST)
      to_explain = request.POST.has_key('button-explain')
      to_submit = request.POST.has_key('button-submit')
      # Always validate the saveform, which will tell us whether it needs explicit saving
      if not form.is_valid():
        break
      to_save = form.saveform.cleaned_data['save']
      to_saveas = form.saveform.cleaned_data['saveas']
      if to_saveas and not design.is_auto:
        # Save As only affects a previously saved query
        design = design.clone()
      if to_submit or to_save or to_saveas or to_explain:
        explicit_save = to_save or to_saveas
        design = save_design(request, form, SavedQuery.HQL, design, explicit_save)
        action = urlresolvers.reverse(execute_query, kwargs=dict(design_id=design.id))

      # We're not going to process the form. Simply re-render it.
      if not to_explain and not to_submit:
        break

      query_str = _strip_trailing_semicolon(form.query.cleaned_data["query"])
      query_server = db_utils.get_query_server(form.query_servers.cleaned_data["server"])
      # (Optional) Parameterization.
      parameterization = get_parameterization(request, query_str, form, design, to_explain)
      if parameterization:
        return parameterization

      query_msg = make_beeswax_query(request, query_str, form)
      try:
        if to_explain:
          return expl_d(request, query_str, query_msg, design, query_server)
        else:
          notify = form.query.cleaned_data.get('email_notify', False)
          return e_d(request, query_msg, design=design,
            on_success_url=on_success_url,
            notify=notify)
      except BeeswaxException, ex:
        error_message, log = expand_exception(ex)
        # Fall through to re-render the execute form.
    else:
      # GET request
      if design.id is not None:
        data = HQLdesign.loads(design.data).get_query_dict()
        form.bind(data)
        form.saveform.set_data(design.name, design.desc)
      else:
        # New design
        form.bind()
예제 #2
0
def configuration(request):
  if request.method == 'POST':
    server_form = QueryServerForm(request.POST)
    if server_form.is_valid():
      query_server = db_utils.get_query_server(server_form.cleaned_data["server"])
      config_values = db_utils.db_client(query_server).get_default_configuration(
                          bool(request.REQUEST.get("include_hadoop", False)))
  else:
    server_form = QueryServerForm()
    config_values = {}

  return render("configuration.mako", request, {'config_values': config_values,
                                                'server_form': server_form})
예제 #3
0
def _run_parameterized_query(request, design_id, explain):
  """
  Given a design and arguments to parameterize that design, runs the query.
  - explain is a boolean to determine whether to run as an explain or as an
  execute.

  This is an extra "step" in the flow from execute_query.
  """
  design = authorized_get_design(request, design_id, must_exist=True)

  # Reconstitute the form
  design_obj = beeswax.design.HQLdesign.loads(design.data)
  query_form = beeswax.forms.query_form()
  params = design_obj.get_query_dict()
  params.update(request.POST)
  query_form.bind(params)
  assert query_form.is_valid()

  query_str = _strip_trailing_semicolon(query_form.query.cleaned_data["query"])
  query_server = db_utils.get_query_server(query_form.query_servers.cleaned_data["server"])

  parameterization_form_cls = make_parameterization_form(query_str)
  if not parameterization_form_cls:
    raise PopupException(_("Query is not parameterizable."))

  parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization")
  if parameterization_form.is_valid():
    real_query = substitute_variables(query_str, parameterization_form.cleaned_data)
    query_msg = make_beeswax_query(request, real_query, query_form)
    try:
      if explain:
        return explain_directly(request, query_str, query_msg, design, query_server)
      else:
        return execute_directly(request, query_msg, query_server, design)
    except BeeswaxException, ex:
      error_message, log = expand_exception(ex, query_server)
      return render('execute.mako', request, {
        'action': urlresolvers.reverse(execute_query),
        'design': design,
        'error_message': error_message,
        'form': query_form,
        'log': log,
      })
예제 #4
0
def execute_query(request, design_id=None, table=None):
    authorized_get_design(request, design_id)
    error_message, log = None, None
    form = query_form()
    action = request.path
    design = safe_get_design(request, SavedQuery.HQL, design_id)
    on_success_url = request.REQUEST.get('on_success_url')

    for _ in range(1):
        if request.method == 'POST':
            form.bind(request.POST)
            to_explain = request.POST.has_key('button-explain')
            to_submit = request.POST.has_key('button-submit')
            # Always validate the saveform, which will tell us whether it needs explicit saving
            if not form.is_valid():
                break
            to_save = form.saveform.cleaned_data['save']
            to_saveas = form.saveform.cleaned_data['saveas']
            if to_saveas and not design.is_auto:
                # Save As only affects a previously saved query
                design = design.clone()
            if to_submit or to_save or to_saveas or to_explain:
                explicit_save = to_save or to_saveas
                design = save_design(request, form, SavedQuery.HQL, design,
                                     explicit_save)
                action = urlresolvers.reverse(execute_query,
                                              kwargs=dict(design_id=design.id))

            # We're not going to process the form. Simply re-render it.
            if not to_explain and not to_submit:
                break

            query_str = _strip_trailing_semicolon(
                form.query.cleaned_data["query"])
            query_server = db_utils.get_query_server(
                form.query_servers.cleaned_data["server"])
            # (Optional) Parameterization.
            parameterization = get_parameterization(request, query_str, form,
                                                    design, to_explain)
            if parameterization:
                return parameterization

            query_msg = make_beeswax_query(request, query_str, form)
            try:
                if to_explain:
                    return expl_d(request, query_str, query_msg, design,
                                  query_server)
                else:
                    notify = form.query.cleaned_data.get('email_notify', False)
                    return e_d(request,
                               query_msg,
                               design=design,
                               on_success_url=on_success_url,
                               notify=notify)
            except BeeswaxException, ex:
                error_message, log = expand_exception(ex)
                # Fall through to re-render the execute form.
        else:
            # GET request
            if design.id is not None:
                data = HQLdesign.loads(design.data).get_query_dict()
                form.bind(data)
                form.saveform.set_data(design.name, design.desc)
            else:
                # New design
                form.bind()
예제 #5
0
def get_shared_beeswax_server():
    # Make it happens only once
    global _SHARED_BEESWAX_SERVER
    global _SHARED_BEESWAX_SERVER_CLOSER
    if _SHARED_BEESWAX_SERVER is None:
        # Copy hive-default.xml.template from BEESWAX_HIVE_CONF_DIR before it is set to
        # /my/bogus/path
        default_xml = file(beeswax.conf.BEESWAX_HIVE_CONF_DIR.get() +
                           "/hive-default.xml.template").read()

        finish = (
            beeswax.conf.QUERY_SERVERS['default'].SERVER_HOST.set_for_testing(
                "localhost"),
            beeswax.conf.QUERY_SERVERS['default'].SERVER_PORT.set_for_testing(
                BEESWAXD_TEST_PORT),
            beeswax.conf.QUERY_SERVERS['default'].SUPPORT_DDL.set_for_testing(
                True),
            beeswax.conf.BEESWAX_META_SERVER_HOST.set_for_testing("localhost"),
            beeswax.conf.BEESWAX_META_SERVER_PORT.set_for_testing(
                BEESWAXD_TEST_PORT + 1),
            # Use a bogus path to avoid loading the normal hive-site.xml
            beeswax.conf.BEESWAX_HIVE_CONF_DIR.set_for_testing('/my/bogus/path'
                                                               ))

        cluster = pseudo_hdfs4.shared_cluster()

        # Copy hive-default.xml into the mini_cluster's conf dir, which happens to be
        # in the cluster's tmpdir. This tmpdir is determined during the mini_cluster
        # startup, during which BEESWAX_HIVE_CONF_DIR needs to be set to
        # /my/bogus/path. Hence the step of writing to memory.
        # hive-default.xml will get picked up by the beeswax_server during startup
        file(cluster._tmpdir + "/conf/hive-default.xml",
             'w').write(default_xml)

        global _SHARED_BEESWAX_SERVER_PROCESS
        if _SHARED_BEESWAX_SERVER_PROCESS is None:
            p = _start_server(cluster)
            _SHARED_BEESWAX_SERVER_PROCESS = p

            def kill():
                LOG.info("Killing beeswax server (pid %d)." % p.pid)
                os.kill(p.pid, 9)
                p.wait()

            atexit.register(kill)
            # Wait for server to come up, by repeatedly trying.
            start = time.time()
            started = False
            sleep = 0.001
            while not started and time.time() - start < 20.0:
                try:
                    query_server = get_query_server(support_ddl=True)
                    client = beeswax.db_utils.db_client(query_server)
                    meta_client = beeswax.db_utils.meta_client()

                    client.echo("echo")
                    if meta_client.getStatus() == fb303.ttypes.fb_status.ALIVE:
                        started = True
                        break
                    time.sleep(sleep)
                    sleep *= 2
                except:
                    time.sleep(sleep)
                    sleep *= 2
                    pass
            if not started:
                raise Exception("Beeswax server took too long to come up.")

            # Make sure /tmp is 0777
            cluster.fs.setuser(cluster.superuser)
            if not cluster.fs.isdir('/tmp'):
                cluster.fs.mkdir('/tmp', 0777)
            else:
                cluster.fs.chmod('/tmp', 0777)

            cluster.fs.chmod(cluster._tmpdir, 0777)
            cluster.fs.chmod(cluster._tmpdir + '/hadoop_tmp_dir/mapred', 0777)

        def s():
            for f in finish:
                f()
            cluster.stop()

        _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER = cluster, s

    return _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER
예제 #6
0
def execute_query(request, design_id=None):
  """
  View function for executing an arbitrary query.
  It understands the optional GET/POST params:

    on_success_url
      If given, it will be displayed when the query is successfully finished.
      Otherwise, it will display the view query results page by default.
  """
  authorized_get_design(request, design_id)

  error_message = None
  form = beeswax.forms.query_form()
  action = request.path
  log = None
  design = safe_get_design(request, models.SavedQuery.HQL, design_id)
  on_success_url = request.REQUEST.get('on_success_url')

  # Use a loop structure to allow the use of 'break' to get out
  for _ in range(1):
    if request.method == 'POST':
      form.bind(request.POST)

      to_explain = request.POST.has_key('button-explain')
      to_submit = request.POST.has_key('button-submit')
      # Always validate the saveform, which will tell us whether it needs explicit saving
      if not form.is_valid():
        break
      to_save = form.saveform.cleaned_data['save']
      to_saveas = form.saveform.cleaned_data['saveas']
      if to_saveas and not design.is_auto:
        # Save As only affects a previously saved query
        design = design.clone()
      if to_submit or to_save or to_saveas or to_explain:
        explicit_save = to_save or to_saveas
        design = save_design(request, form, models.SavedQuery.HQL, design, explicit_save)
        action = urlresolvers.reverse(execute_query, kwargs=dict(design_id=design.id))

      # We're not going to process the form. Simply re-render it.
      if not to_explain and not to_submit:
        break

      query_str = _strip_trailing_semicolon(form.query.cleaned_data["query"])
      query_server = db_utils.get_query_server(form.query_servers.cleaned_data["server"])

      # (Optional) Parameterization.
      parameterization = get_parameterization(request, query_str, form, design, to_explain)
      if parameterization:
        return parameterization

      query_msg = make_beeswax_query(request, query_str, form)
      try:
        if to_explain:
          return explain_directly(request, query_str, query_msg, design, query_server)
        else:
          notify = form.query.cleaned_data.get('email_notify', False)
          return execute_directly(request, query_msg, query_server, design,
                                  on_success_url=on_success_url,
                                  notify=notify)
      except BeeswaxException, ex:
        error_message, log = expand_exception(ex, query_server)
        # Fall through to re-render the execute form.
    else:
      # GET request
      if design.id is not None:
        data = beeswax.design.HQLdesign.loads(design.data).get_query_dict()
        form.bind(data)
        form.saveform.set_data(design.name, design.desc)
      else:
        # New design
        form.bind()
예제 #7
0
def get_shared_beeswax_server():
  # Make it happens only once
  global _SHARED_BEESWAX_SERVER
  global _SHARED_BEESWAX_SERVER_CLOSER
  if _SHARED_BEESWAX_SERVER is None:
    # Copy hive-default.xml.template from BEESWAX_HIVE_CONF_DIR before it is set to
    # /my/bogus/path
    default_xml = file(beeswax.conf.BEESWAX_HIVE_CONF_DIR.get()+"/hive-default.xml.template").read()

    finish = (
      beeswax.conf.QUERY_SERVERS['default'].SERVER_HOST.set_for_testing("localhost"),
      beeswax.conf.QUERY_SERVERS['default'].SERVER_PORT.set_for_testing(BEESWAXD_TEST_PORT),
      beeswax.conf.QUERY_SERVERS['default'].SUPPORT_DDL.set_for_testing(True),
      beeswax.conf.BEESWAX_META_SERVER_HOST.set_for_testing("localhost"),
      beeswax.conf.BEESWAX_META_SERVER_PORT.set_for_testing(BEESWAXD_TEST_PORT + 1),
      # Use a bogus path to avoid loading the normal hive-site.xml
      beeswax.conf.BEESWAX_HIVE_CONF_DIR.set_for_testing('/my/bogus/path')
    )

    cluster = pseudo_hdfs4.shared_cluster()

    # Copy hive-default.xml into the mini_cluster's conf dir, which happens to be
    # in the cluster's tmpdir. This tmpdir is determined during the mini_cluster
    # startup, during which BEESWAX_HIVE_CONF_DIR needs to be set to
    # /my/bogus/path. Hence the step of writing to memory.
    # hive-default.xml will get picked up by the beeswax_server during startup
    file(cluster._tmpdir+"/conf/hive-default.xml", 'w').write(default_xml)

    global _SHARED_BEESWAX_SERVER_PROCESS
    if _SHARED_BEESWAX_SERVER_PROCESS is None:
      p = _start_server(cluster)
      _SHARED_BEESWAX_SERVER_PROCESS = p
      def kill():
        LOG.info("Killing beeswax server (pid %d)." % p.pid)
        os.kill(p.pid, 9)
        p.wait()
      atexit.register(kill)
      # Wait for server to come up, by repeatedly trying.
      start = time.time()
      started = False
      sleep = 0.001
      while not started and time.time() - start < 20.0:
        try:
          query_server = get_query_server(support_ddl=True)
          client = beeswax.db_utils.db_client(query_server)
          meta_client = beeswax.db_utils.meta_client()

          client.echo("echo")
          if meta_client.getStatus() == fb303.ttypes.fb_status.ALIVE:
            started = True
            break
          time.sleep(sleep)
          sleep *= 2
        except:
          time.sleep(sleep)
          sleep *= 2
          pass
      if not started:
        raise Exception("Beeswax server took too long to come up.")

      # Make sure /tmp is 0777
      cluster.fs.setuser(cluster.superuser)
      if not cluster.fs.isdir('/tmp'):
        cluster.fs.mkdir('/tmp', 0777)
      else:
        cluster.fs.chmod('/tmp', 0777)

      cluster.fs.chmod(cluster._tmpdir, 0777)
      cluster.fs.chmod(cluster._tmpdir + '/hadoop_tmp_dir/mapred', 0777)

    def s():
      for f in finish:
        f()
      cluster.stop()

    _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER = cluster, s

  return _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER