def _save_results_ctas(request, query_history, target_table, result_meta):
"""
Handle saving results as a new table. Returns HTTP response.
May raise BeeswaxException, IOError.
"""
query_server = query_history.get_query_server() # Query server requires DDL support
# Case 1: The results are straight from an existing table
if result_meta.in_tablename:
hql = 'CREATE TABLE `%s` AS SELECT * FROM %s' % (target_table, result_meta.in_tablename)
query_msg = make_beeswax_query(request, hql)
# Display the CTAS running. Could take a long time.
return execute_directly(request, query_msg, query_server, on_success_url=urlresolvers.reverse(show_tables))
# Case 2: The results are in some temporary location
# 1. Create table
cols = ''
schema = result_meta.schema
for i, field in enumerate(schema.fieldSchemas):
if i != 0:
cols += ',\n'
cols += '`%s` %s' % (field.name, field.type)
# The representation of the delimiter is messy.
# It came from Java as a string, which might has been converted from an integer.
# So it could be "1" (^A), or "10" (\n), or "," (a comma literally).
delim = result_meta.delim
if not delim.isdigit():
delim = str(ord(delim))
hql = '''
CREATE TABLE `%s` (
%s
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\%s'
STORED AS TextFile
''' % (target_table, cols, delim.zfill(3))
query_msg = make_beeswax_query(request, hql)
db_utils.execute_and_wait(request.user, query_msg, query_server)
try:
# 2. Move the results into the table's storage
table_obj = db_utils.meta_client().get_table("default", target_table)
table_loc = request.fs.urlsplit(table_obj.sd.location)[2]
request.fs.rename_star(result_meta.table_dir, table_loc)
LOG.debug("Moved results from %s to %s" % (result_meta.table_dir, table_loc))
messages.info(request, _('Saved query results as new table %(table)s') % {'table': target_table})
query_history.save_state(models.QueryHistory.STATE.expired)
except Exception, ex:
LOG.error('Error moving data into storage of table %s. Will drop table.' % (target_table,))
query_msg = make_beeswax_query(request, 'DROP TABLE `%s`' % (target_table,))
try:
db_utils.execute_directly(request.user, query_msg, query_server) # Don't wait for results
except Exception, double_trouble:
LOG.exception('Failed to drop table "%s" as well: %s' % (target_table, double_trouble))
def describe_table(request, table):
table_obj = db_utils.meta_client().get_table("default", table)
sample_results = None
is_view = table_obj.tableType == 'VIRTUAL_VIEW'
# Don't show samples if it's a view (HUE-526).
if not is_view:
# Show the first few rows
hql = "SELECT * FROM `%s` %s" % (table, _get_browse_limit_clause(table_obj))
query_msg = make_beeswax_query(request, hql)
try:
sample_results = db_utils.execute_and_wait(request.user, query_msg, timeout_sec=5.0)
except:
# Gracefully degrade if we're unable to load the results.
logging.exception("Failed to read table '%s'" % table)
sample_results = None
hdfs_link = location_to_url(request, table_obj.sd.location)
load_form = beeswax.forms.LoadDataForm(table_obj)
return render("describe_table.mako", request, dict(
table=table_obj,
table_name=table,
top_rows=sample_results and list(parse_results(sample_results.data)) or None,
hdfs_link=hdfs_link,
load_form=load_form,
is_view=is_view
))
def load(self, django_user):
"""
Load data into table. Raise InstallException on failure.
"""
LOAD_HQL = \
"""
LOAD DATA local INPATH
'%(filename)s' OVERWRITE INTO TABLE %(tablename)s
"""
LOG.info('Loading data into table "%s"' % (self.name, ))
hql = LOAD_HQL % dict(tablename=self.name,
filename=self._contents_file)
query_msg = _make_query_msg(hql)
try:
results = db_utils.execute_and_wait(django_user, query_msg)
if not results:
msg = _(
'Error loading table %(table)s: Operation timeout.') % {
'table': self.name
}
LOG.error(msg)
raise InstallException(msg)
except BeeswaxException, ex:
msg = _('Error loading table %(table)s: %(error)s') % {
'table': self.name,
'error': ex
}
LOG.error(msg)
raise InstallException(msg)
def create(self, django_user):
"""
Create in Hive. Raise InstallException on failure.
"""
# Create table
LOG.info('Creating table "%s"' % (self.name, ))
try:
# Already exists?
tables = db_utils.meta_client().get_table("default", self.name)
msg = _('Table "%(table)s" already exists.') % {'table': self.name}
LOG.error(msg)
raise InstallException(msg)
except hive_metastore.ttypes.NoSuchObjectException:
query_msg = _make_query_msg(self.hql)
try:
results = db_utils.execute_and_wait(django_user, query_msg)
if not results:
msg = _(
'Error creating table %(table)s: Operation timeout.'
) % {
'table': self.name
}
LOG.error(msg)
raise InstallException(msg)
except BeeswaxException, ex:
msg = _('Error creating table %(table)s: %(error)s') % {
'table': self.name,
'error': ex
}
LOG.error(msg)
raise InstallException(msg)
def describe_table(request, table):
table_obj = db_utils.meta_client().get_table("default", table)
# Show the first few rows
hql = "SELECT * FROM `%s`" % (table,)
query_msg = make_beeswax_query(request, hql)
try:
results = db_utils.execute_and_wait(request.user, query_msg, timeout_sec=5.0)
except:
# Gracefully degrade if we're unable to load the results.
logging.exception("Failed to read table '%s'" % table)
results = None
hdfs_link = location_to_url(request, table_obj.sd.location)
load_form = beeswax.forms.LoadDataForm(table_obj)
return render("describe_table.mako", request, dict(
table=table_obj,
table_name=table,
top_rows=results and list(parse_results(results.data)) or None,
hdfs_link=hdfs_link,
load_form=load_form
))
def load(self, django_user):
"""
Load data into table.
"""
LOAD_HQL = \
"""
LOAD DATA local INPATH
'%(filename)s' OVERWRITE INTO TABLE %(tablename)s
"""
LOG.info('Loading data into table "%s"' % (self.name,))
hql = LOAD_HQL % dict(tablename=self.name, filename=self._contents_file)
query_msg = _make_query_msg(hql)
try:
results = db_utils.execute_and_wait(django_user, query_msg)
if not results:
LOG.error('Error loading table %s: Operation timeout' % (self.name,))
return False
except BeeswaxException, ex:
LOG.error('Error loading table %s: %s' % (self.name, ex))
return False
def create(self, django_user):
"""
Create in Hive. Returns True/False.
"""
# Create table
LOG.info('Creating table "%s"' % (self.name,))
try:
# Already exists?
tables = db_utils.meta_client().get_table("default", self.name)
LOG.error('Table "%s" already exists' % (self.name,))
return False
except hive_metastore.ttypes.NoSuchObjectException:
query_msg = _make_query_msg(self.hql)
try:
results = db_utils.execute_and_wait(django_user, query_msg)
if not results:
LOG.error('Error creating table %s: Operation timeout' % (self.name,))
return False
except BeeswaxException, ex:
LOG.error('Error creating table %s: %s' % (self.name, ex))
return False
return True
def load(self, django_user):
"""
Load data into table. Raise InstallException on failure.
"""
LOAD_HQL = \
"""
LOAD DATA local INPATH
'%(filename)s' OVERWRITE INTO TABLE %(tablename)s
"""
LOG.info('Loading data into table "%s"' % (self.name,))
hql = LOAD_HQL % dict(tablename=self.name, filename=self._contents_file)
query_msg = _make_query_msg(hql)
try:
results = db_utils.execute_and_wait(django_user, query_msg)
if not results:
msg = _('Error loading table %(table)s: Operation timeout.') % {'table': self.name}
LOG.error(msg)
raise InstallException(msg)
except BeeswaxException, ex:
msg = _('Error loading table %(table)s: %(error)s') % {'table': self.name, 'error': ex}
LOG.error(msg)
raise InstallException(msg)
def create(self, django_user):
"""
Create in Hive. Raise InstallException on failure.
"""
# Create table
LOG.info('Creating table "%s"' % (self.name,))
try:
# Already exists?
tables = db_utils.meta_client().get_table("default", self.name)
msg = _('Table "%(table)s" already exists.') % {'table': self.name}
LOG.error(msg)
raise InstallException(msg)
except hive_metastore.ttypes.NoSuchObjectException:
query_msg = _make_query_msg(self.hql)
try:
results = db_utils.execute_and_wait(django_user, query_msg)
if not results:
msg = _('Error creating table %(table)s: Operation timeout.') % {'table': self.name}
LOG.error(msg)
raise InstallException(msg)
except BeeswaxException, ex:
msg = _('Error creating table %(table)s: %(error)s') % {'table': self.name, 'error': ex}
LOG.error(msg)
raise InstallException(msg)
