def _clean_tablename(name):
try:
db_utils.meta_client().get_table("default", name)
raise forms.ValidationError(
_('Table "%(name)s" already exists') % {'name': name})
except hive_metastore.ttypes.NoSuchObjectException:
return name
def describe_partitions(request, table):
table_obj = db_utils.meta_client().get_table("default", table)
if len(table_obj.partitionKeys) == 0:
raise PopupException("Table '%s' is not partitioned." % table)
partitions = db_utils.meta_client().get_partitions("default", table, max_parts=-1)
return render("describe_partitions.mako", request,
dict(table=table_obj, partitions=partitions, request=request))
def clean_target_table(self):
tbl = self.cleaned_data.get('target_table')
if tbl:
try:
db_utils.meta_client().get_table("default", tbl)
raise forms.ValidationError('Table already exists')
except hive_metastore.ttypes.NoSuchObjectException:
pass
return tbl
def clean_target_table(self):
tbl = self.cleaned_data.get('target_table')
if tbl:
try:
db_utils.meta_client().get_table("default", tbl)
raise forms.ValidationError(_t('Table already exists'))
except hive_metastore.ttypes.NoSuchObjectException:
pass
return tbl
def drop_table(request, table):
table_obj = db_utils.meta_client().get_table("default", table)
is_view = table_obj.tableType == 'VIRTUAL_VIEW'
if request.method == 'GET':
# It may be possible to determine whether the table is
# external by looking at db_utils.meta_client().get_table("default", table).tableType,
# but this was introduced in Hive 0.5, and therefore may not be available
# with older metastores.
if is_view:
title = _("Do you really want to drop the view '%(table)s'?") % {'table': table}
else:
title = _("This may delete the underlying data as well as the metadata. Drop table '%(table)s'?") % {'table': table}
return render('confirm.html', request, dict(url=request.path, title=title))
elif request.method == 'POST':
if is_view:
hql = "DROP VIEW `%s`" % (table,)
else:
hql = "DROP TABLE `%s`" % (table,)
query_msg = make_beeswax_query(request, hql)
try:
return execute_directly(request,
query_msg,
on_success_url=urlresolvers.reverse(show_tables))
except BeeswaxException, ex:
# Note that this state is difficult to get to.
error_message, log = expand_exception(ex)
error = _("Failed to remove %(table)s. Error: %(error)s") % {'table': table, 'error': error_message}
raise PopupException(error, title=_("Beeswax Error"), detail=log)
def describe_table(request, table):
table_obj = db_utils.meta_client().get_table("default", table)
sample_results = None
is_view = table_obj.tableType == 'VIRTUAL_VIEW'
# Don't show samples if it's a view (HUE-526).
if not is_view:
# Show the first few rows
hql = "SELECT * FROM `%s` %s" % (table, _get_browse_limit_clause(table_obj))
query_msg = make_beeswax_query(request, hql)
try:
sample_results = db_utils.execute_and_wait(request.user, query_msg, timeout_sec=5.0)
except:
# Gracefully degrade if we're unable to load the results.
logging.exception("Failed to read table '%s'" % table)
sample_results = None
hdfs_link = location_to_url(request, table_obj.sd.location)
load_form = beeswax.forms.LoadDataForm(table_obj)
return render("describe_table.mako", request, dict(
table=table_obj,
table_name=table,
top_rows=sample_results and list(parse_results(sample_results.data)) or None,
hdfs_link=hdfs_link,
load_form=load_form,
is_view=is_view
))
def create(self, django_user):
"""
Create in Hive. Raise InstallException on failure.
"""
# Create table
LOG.info('Creating table "%s"' % (self.name, ))
try:
# Already exists?
tables = db_utils.meta_client().get_table("default", self.name)
msg = _('Table "%(table)s" already exists.') % {'table': self.name}
LOG.error(msg)
raise InstallException(msg)
except hive_metastore.ttypes.NoSuchObjectException:
query_msg = _make_query_msg(self.hql)
try:
results = db_utils.execute_and_wait(django_user, query_msg)
if not results:
msg = _(
'Error creating table %(table)s: Operation timeout.'
) % {
'table': self.name
}
LOG.error(msg)
raise InstallException(msg)
except BeeswaxException, ex:
msg = _('Error creating table %(table)s: %(error)s') % {
'table': self.name,
'error': ex
}
LOG.error(msg)
raise InstallException(msg)
def index(request):
tables = db_utils.meta_client().get_tables("default", ".*")
if not tables:
examples_installed = beeswax.models.MetaInstall.get().installed_example
return render("index.mako", request, {'examples_installed': examples_installed})
else:
return execute_query(request)
def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(table=beeswax.forms.CreateTableForm,
columns=beeswax.forms.ColumnTypeFormSet,
partitions=beeswax.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid():
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [f.cleaned_data for f in form.partitions.forms]
proposed_query = django_mako.render_to_string(
"create_table_statement.mako", {
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
})
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
tablename = form.table.cleaned_data['name']
on_success_url = urlresolvers.reverse(describe_table,
kwargs={'table': tablename})
return confirm_query(request, proposed_query, on_success_url)
else:
form.bind()
return render(
"create_table_manually.mako", request,
dict(action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(db_utils.meta_client().get_tables("default", ".*"))
> 0))
def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(
table=beeswax.forms.CreateTableForm,
columns=beeswax.forms.ColumnTypeFormSet,
partitions=beeswax.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid():
columns = [ f.cleaned_data for f in form.columns.forms ]
partition_columns = [ f.cleaned_data for f in form.partitions.forms ]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
tablename = form.table.cleaned_data['name']
on_success_url = urlresolvers.reverse(describe_table, kwargs={'table': tablename})
return confirm_query(request, proposed_query, on_success_url)
else:
form.bind()
return render("create_table_manually.mako", request, dict(
action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(db_utils.meta_client().get_tables("default", ".*")) > 0
))
def load_table(request, table):
"""
Loads data into a table.
"""
table_obj = db_utils.meta_client().get_table("default", table)
if request.method == "POST":
form = beeswax.forms.LoadDataForm(table_obj, request.POST)
if form.is_valid():
# TODO(philip/todd): When PathField might refer to non-HDFS,
# we need a pathfield.is_local function.
hql = "LOAD DATA INPATH"
hql += " '%s'" % form.cleaned_data['path']
if form.cleaned_data['overwrite']:
hql += " OVERWRITE"
hql += " INTO TABLE "
hql += "`%s`" % (table,)
if len(form.partition_columns) > 0:
hql += " PARTITION ("
vals = []
for key, column_name in form.partition_columns.iteritems():
vals.append("%s='%s'" % (column_name, form.cleaned_data[key]))
hql += ", ".join(vals)
hql += ")"
on_success_url = urlresolvers.reverse(describe_table, kwargs={'table': table})
return confirm_query(request, hql, on_success_url)
else:
form = beeswax.forms.LoadDataForm(table_obj)
return render("load_table.mako", request, dict(form=form, table=table, action=request.get_full_path()))
def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(
table=beeswax.forms.CreateTableForm,
columns=beeswax.forms.ColumnTypeFormSet,
partitions=beeswax.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid() and 'createTable' in request.POST:
columns = [ f.cleaned_data for f in form.columns.forms ]
partition_columns = [ f.cleaned_data for f in form.partitions.forms ]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
table_name = form.table.cleaned_data['name']
return _submit_create_and_load(request, proposed_query, table_name, None, False)
else:
form.bind()
return render("create_table_manually.mako", request, dict(
action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(db_utils.meta_client().get_tables("default", ".*")) > 0
))
def get_columns(self):
"""
get_columns() -> List of (cached) column names
May raise ttypes.NoSuchObjectException if the table name is bogus.
"""
if self.columns is None:
ttable = db_utils.meta_client().get_table("default", self.name)
self.columns = [ c.name for c in ttable.sd.cols ]
return self.columns
def get_columns(self):
"""
get_columns() -> List of (cached) column names
May raise ttypes.NoSuchObjectException if the table name is bogus.
"""
if self.columns is None:
ttable = db_utils.meta_client().get_table("default", self.name)
self.columns = [c.name for c in ttable.sd.cols]
return self.columns
def _save_results_ctas(request, query_history, target_table, result_meta):
"""
Handle saving results as a new table. Returns HTTP response.
May raise BeeswaxException, IOError.
"""
query_server = query_history.get_query_server() # Query server requires DDL support
# Case 1: The results are straight from an existing table
if result_meta.in_tablename:
hql = 'CREATE TABLE `%s` AS SELECT * FROM %s' % (target_table, result_meta.in_tablename)
query_msg = make_beeswax_query(request, hql)
# Display the CTAS running. Could take a long time.
return execute_directly(request, query_msg, query_server, on_success_url=urlresolvers.reverse(show_tables))
# Case 2: The results are in some temporary location
# 1. Create table
cols = ''
schema = result_meta.schema
for i, field in enumerate(schema.fieldSchemas):
if i != 0:
cols += ',\n'
cols += '`%s` %s' % (field.name, field.type)
# The representation of the delimiter is messy.
# It came from Java as a string, which might has been converted from an integer.
# So it could be "1" (^A), or "10" (\n), or "," (a comma literally).
delim = result_meta.delim
if not delim.isdigit():
delim = str(ord(delim))
hql = '''
CREATE TABLE `%s` (
%s
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\%s'
STORED AS TextFile
''' % (target_table, cols, delim.zfill(3))
query_msg = make_beeswax_query(request, hql)
db_utils.execute_and_wait(request.user, query_msg, query_server)
try:
# 2. Move the results into the table's storage
table_obj = db_utils.meta_client().get_table("default", target_table)
table_loc = request.fs.urlsplit(table_obj.sd.location)[2]
request.fs.rename_star(result_meta.table_dir, table_loc)
LOG.debug("Moved results from %s to %s" % (result_meta.table_dir, table_loc))
messages.info(request, _('Saved query results as new table %(table)s') % {'table': target_table})
query_history.save_state(models.QueryHistory.STATE.expired)
except Exception, ex:
LOG.error('Error moving data into storage of table %s. Will drop table.' % (target_table,))
query_msg = make_beeswax_query(request, 'DROP TABLE `%s`' % (target_table,))
try:
db_utils.execute_directly(request.user, query_msg, query_server) # Don't wait for results
except Exception, double_trouble:
LOG.exception('Failed to drop table "%s" as well: %s' % (target_table, double_trouble))
def read_table(request, table):
"""View function for select * from table"""
table_obj = db_utils.meta_client().get_table("default", table)
hql = "SELECT * FROM `%s` %s" % (table, _get_browse_limit_clause(table_obj))
query_msg = make_beeswax_query(request, hql)
try:
return execute_directly(request, query_msg, tablename=table)
except BeeswaxException, e:
# Note that this state is difficult to get to.
error_message, log = expand_exception(e)
error = _("Failed to read table. Error: %(error)s") % {'error': error_message}
raise PopupException(error, title=_("Beeswax Error"), detail=log)
def read_table(request, table):
"""View function for select * from table"""
table_obj = db_utils.meta_client().get_table("default", table)
hql = "SELECT * FROM `%s` %s" % (table,
_get_browse_limit_clause(table_obj))
query_msg = make_beeswax_query(request, hql)
try:
return execute_directly(request, query_msg, tablename=table)
except BeeswaxException, e:
# Note that this state is difficult to get to.
error_message, log = expand_exception(e)
error = _("Failed to read table. Error: %(error)s") % {
'error': error_message
}
raise PopupException(error, title=_("Beeswax Error"), detail=log)
def describe_table(request, table):
table_obj = db_utils.meta_client().get_table("default", table)
# Show the first few rows
hql = "SELECT * FROM `%s`" % (table,)
query_msg = make_beeswax_query(request, hql)
try:
results = db_utils.execute_and_wait(request.user, query_msg, timeout_sec=5.0)
except:
# Gracefully degrade if we're unable to load the results.
logging.exception("Failed to read table '%s'" % table)
results = None
hdfs_link = location_to_url(request, table_obj.sd.location)
load_form = beeswax.forms.LoadDataForm(table_obj)
return render("describe_table.mako", request, dict(
table=table_obj,
table_name=table,
top_rows=results and list(parse_results(results.data)) or None,
hdfs_link=hdfs_link,
load_form=load_form
))
def create(self, django_user):
"""
Create in Hive. Returns True/False.
"""
# Create table
LOG.info('Creating table "%s"' % (self.name,))
try:
# Already exists?
tables = db_utils.meta_client().get_table("default", self.name)
LOG.error('Table "%s" already exists' % (self.name,))
return False
except hive_metastore.ttypes.NoSuchObjectException:
query_msg = _make_query_msg(self.hql)
try:
results = db_utils.execute_and_wait(django_user, query_msg)
if not results:
LOG.error('Error creating table %s: Operation timeout' % (self.name,))
return False
except BeeswaxException, ex:
LOG.error('Error creating table %s: %s' % (self.name, ex))
return False
return True
def create(self, django_user):
"""
Create in Hive. Raise InstallException on failure.
"""
# Create table
LOG.info('Creating table "%s"' % (self.name,))
try:
# Already exists?
tables = db_utils.meta_client().get_table("default", self.name)
msg = _('Table "%(table)s" already exists.') % {'table': self.name}
LOG.error(msg)
raise InstallException(msg)
except hive_metastore.ttypes.NoSuchObjectException:
query_msg = _make_query_msg(self.hql)
try:
results = db_utils.execute_and_wait(django_user, query_msg)
if not results:
msg = _('Error creating table %(table)s: Operation timeout.') % {'table': self.name}
LOG.error(msg)
raise InstallException(msg)
except BeeswaxException, ex:
msg = _('Error creating table %(table)s: %(error)s') % {'table': self.name, 'error': ex}
LOG.error(msg)
raise InstallException(msg)
def __init__(self, *args, **kwargs):
table_names = db_utils.meta_client().get_tables("default", ".*")
kwargs['choices'] = to_choices([''] + table_names)
forms.ChoiceField.__init__(self, *args, **kwargs)
def _clean_tablename(name):
try:
db_utils.meta_client().get_table("default", name)
raise forms.ValidationError(_t('Table "%(name)s" already exists') % {'name': name})
except hive_metastore.ttypes.NoSuchObjectException:
return name
def hive_view(request, table=None):
tables = db_utils.meta_client().get_tables("default", ".*")
if not tables:
return render("show_tables.mako", request, dict(tables=[],))
else:
return execute_query(request, table=table)
def show_tables(request):
tables = db_utils.meta_client().get_tables("default", ".*")
examples_installed = beeswax.models.MetaInstall.get().installed_example
return render("show_tables.mako", request, dict(tables=tables, examples_installed=examples_installed))
def index(request):
tables = db_utils.meta_client().get_tables("default", ".*")
if not tables:
return render("index.mako", request, {})
else:
return execute_query(request)
def __init__(self, *args, **kwargs):
table_names = db_utils.meta_client().get_tables("default", ".*")
kwargs['choices'] = to_choices([''] + table_names)
forms.ChoiceField.__init__(self, *args, **kwargs)
def hive_view(request, table=None):
tables = db_utils.meta_client().get_tables("default", ".*")
if not tables:
return render("show_tables.mako", request, dict(tables=[], ))
else:
return execute_query(request, table=table)