Ejemplo n.º 1
0
def load_after_create(request, database):
    """
  Automatically load data into a newly created table.

  We get here from the create's on_success_url, and expect to find
  ``table`` and ``path`` from the parameters.
  """
    tablename = request.REQUEST.get("table")
    path = request.REQUEST.get("path")
    is_remove_header = request.REQUEST.get("removeHeader").lower() == "on" and not path.endswith("gz")

    if not tablename or not path:
        msg = _("Internal error: Missing needed parameter to load data into table.")
        LOG.error(msg)
        raise PopupException(msg)

    if is_remove_header:
        remove_header(request.fs, path)

    LOG.debug("Auto loading data from %s into table %s" % (path, tablename))
    hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (path, database, tablename)
    query = hql_query(hql)

    on_success_url = reverse("metastore:describe_table", kwargs={"database": database, "table": tablename})

    return execute_directly(request, query, on_success_url=on_success_url)
Ejemplo n.º 2
0
def load_after_create(request, database):
    """
  Automatically load data into a newly created table.

  We get here from the create's on_success_url, and expect to find
  ``table`` and ``path`` from the parameters.
  """
    tablename = request.REQUEST.get('table')
    path = request.REQUEST.get('path')
    is_remove_header = request.REQUEST.get(
        'removeHeader').lower() == 'on' and not path.endswith('gz')

    if not tablename or not path:
        msg = _(
            'Internal error: Missing needed parameter to load data into table.'
        )
        LOG.error(msg)
        raise PopupException(msg)

    if is_remove_header:
        remove_header(request.fs, path)

    LOG.debug("Auto loading data from %s into table %s" % (path, tablename))
    hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (path, database,
                                                        tablename)
    query = hql_query(hql)

    on_success_url = reverse('metastore:describe_table',
                             kwargs={
                                 'database': database,
                                 'table': tablename
                             })

    return execute_directly(request, query, on_success_url=on_success_url)
Ejemplo n.º 3
0
def load_after_create(request, database):
    """
  Automatically load data into a newly created table.

  We get here from the create's on_success_url, and expect to find
  ``table`` and ``path`` from the parameters.
  """
    tablename = request.REQUEST.get('table')
    path = request.REQUEST.get('path')
    is_remove_header = request.REQUEST.get(
        'removeHeader').lower() == 'on' and not path.endswith('gz')

    if not tablename or not path:
        msg = _(
            'Internal error: Missing needed parameter to load data into table.'
        )
        LOG.error(msg)
        raise PopupException(msg)

    if is_remove_header:
        try:
            remove_header(request.fs, path)
        except Exception, e:
            raise PopupException(
                _("The headers of the file could not be removed."), detail=e)
Ejemplo n.º 4
0
def load_after_create(request, database):
  """
  Automatically load data into a newly created table.

  We get here from the create's on_success_url, and expect to find
  ``table`` and ``path`` from the parameters.
  """
  tablename = request.REQUEST.get('table')
  path = request.REQUEST.get('path')
  is_remove_header = request.REQUEST.get('removeHeader').lower() == 'on' and not path.endswith('gz')

  if not tablename or not path:
    msg = _('Internal error: Missing needed parameter to load data into table.')
    LOG.error(msg)
    raise PopupException(msg)

  if is_remove_header:
    try:
      remove_header(request.fs, path)
    except Exception, e:
      raise PopupException(_("The headers of the file could not be removed."), detail=e)
Ejemplo n.º 5
0
  def test_remove_header(self):
    fs = self.cluster.fs

    path = "/tmp/test_remove_header.txt"
    data_header = "destination\trank"
    data_body = """thailand\t10
costarica\t?
curacao\t?"""
    data = data_header + '\n' + data_body

    f = fs.open(path, "w")
    f.write("hello")
    f.close()

    encoding = i18n.get_site_encoding()
    do_overwrite_save(fs, path, data.encode(encoding))

    assert_not_equal(data_body, fs.open(path).read())

    remove_header(fs, path)

    assert_equals(data_body, fs.open(path).read())
Ejemplo n.º 6
0
    def test_remove_header(self):
        fs = self.cluster.fs

        path = "/tmp/test_remove_header.txt"
        data_header = "destination\trank"
        data_body = """thailand\t10
costarica\t?
curacao\t?"""
        data = data_header + '\n' + data_body

        f = fs.open(path, "w")
        f.write("hello")
        f.close()

        encoding = i18n.get_site_encoding()
        do_overwrite_save(fs, path, data.encode(encoding))

        assert_not_equal(data_body, fs.open(path).read())

        remove_header(fs, path)

        assert_equals(data_body, fs.open(path).read())