def load_after_create(request, database):
"""
Automatically load data into a newly created table.
We get here from the create's on_success_url, and expect to find
``table`` and ``path`` from the parameters.
"""
tablename = request.REQUEST.get("table")
path = request.REQUEST.get("path")
is_remove_header = request.REQUEST.get("removeHeader").lower() == "on" and not path.endswith("gz")
if not tablename or not path:
msg = _("Internal error: Missing needed parameter to load data into table.")
LOG.error(msg)
raise PopupException(msg)
if is_remove_header:
remove_header(request.fs, path)
LOG.debug("Auto loading data from %s into table %s" % (path, tablename))
hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (path, database, tablename)
query = hql_query(hql)
on_success_url = reverse("metastore:describe_table", kwargs={"database": database, "table": tablename})
return execute_directly(request, query, on_success_url=on_success_url)
def load_after_create(request, database):
"""
Automatically load data into a newly created table.
We get here from the create's on_success_url, and expect to find
``table`` and ``path`` from the parameters.
"""
tablename = request.REQUEST.get('table')
path = request.REQUEST.get('path')
is_remove_header = request.REQUEST.get(
'removeHeader').lower() == 'on' and not path.endswith('gz')
if not tablename or not path:
msg = _(
'Internal error: Missing needed parameter to load data into table.'
)
LOG.error(msg)
raise PopupException(msg)
if is_remove_header:
remove_header(request.fs, path)
LOG.debug("Auto loading data from %s into table %s" % (path, tablename))
hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (path, database,
tablename)
query = hql_query(hql)
on_success_url = reverse('metastore:describe_table',
kwargs={
'database': database,
'table': tablename
})
return execute_directly(request, query, on_success_url=on_success_url)
def load_after_create(request, database):
"""
Automatically load data into a newly created table.
We get here from the create's on_success_url, and expect to find
``table`` and ``path`` from the parameters.
"""
tablename = request.REQUEST.get('table')
path = request.REQUEST.get('path')
is_remove_header = request.REQUEST.get(
'removeHeader').lower() == 'on' and not path.endswith('gz')
if not tablename or not path:
msg = _(
'Internal error: Missing needed parameter to load data into table.'
)
LOG.error(msg)
raise PopupException(msg)
if is_remove_header:
try:
remove_header(request.fs, path)
except Exception, e:
raise PopupException(
_("The headers of the file could not be removed."), detail=e)
def load_after_create(request, database):
"""
Automatically load data into a newly created table.
We get here from the create's on_success_url, and expect to find
``table`` and ``path`` from the parameters.
"""
tablename = request.REQUEST.get('table')
path = request.REQUEST.get('path')
is_remove_header = request.REQUEST.get('removeHeader').lower() == 'on' and not path.endswith('gz')
if not tablename or not path:
msg = _('Internal error: Missing needed parameter to load data into table.')
LOG.error(msg)
raise PopupException(msg)
if is_remove_header:
try:
remove_header(request.fs, path)
except Exception, e:
raise PopupException(_("The headers of the file could not be removed."), detail=e)
def test_remove_header(self):
fs = self.cluster.fs
path = "/tmp/test_remove_header.txt"
data_header = "destination\trank"
data_body = """thailand\t10
costarica\t?
curacao\t?"""
data = data_header + '\n' + data_body
f = fs.open(path, "w")
f.write("hello")
f.close()
encoding = i18n.get_site_encoding()
do_overwrite_save(fs, path, data.encode(encoding))
assert_not_equal(data_body, fs.open(path).read())
remove_header(fs, path)
assert_equals(data_body, fs.open(path).read())
def test_remove_header(self):
fs = self.cluster.fs
path = "/tmp/test_remove_header.txt"
data_header = "destination\trank"
data_body = """thailand\t10
costarica\t?
curacao\t?"""
data = data_header + '\n' + data_body
f = fs.open(path, "w")
f.write("hello")
f.close()
encoding = i18n.get_site_encoding()
do_overwrite_save(fs, path, data.encode(encoding))
assert_not_equal(data_body, fs.open(path).read())
remove_header(fs, path)
assert_equals(data_body, fs.open(path).read())
