def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(
table=hcatalog.forms.CreateTableForm,
columns=hcatalog.forms.ColumnTypeFormSet,
partitions=hcatalog.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid() and 'createTable' in request.POST:
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [f.cleaned_data for f in form.partitions.forms]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
tablename = form.table.cleaned_data['name']
tables = []
try:
hcat_client().create_table("default", proposed_query)
tables = hcat_client().get_tables()
except Exception, ex:
raise PopupException('Error on creating table', title="Error on creating table", detail=str(ex))
return render("show_tables.mako", request, dict(tables=tables,))
def describe_table(request, table):
try:
table_desc_extended = hcat_client().describe_table_extended(table)
is_table_partitioned = table_desc_extended['partitioned']
partitions = []
partitionColumns = []
if is_table_partitioned:
partitions = hcat_client().get_partitions(table)
partitionColumns = table_desc_extended['partitionColumns']
table_obj = {
'tableName': table,
'columns': table_desc_extended['columns'],
'partitionKeys': partitionColumns,
'partitions': partitions
}
except Exception:
import traceback
error = traceback.format_exc()
raise PopupException('Error getting table description',
title="Error getting table description",
detail=error)
load_form = hcatalog.forms.LoadDataForm(table_obj)
return render(
"describe_table.mako", request,
dict(
table=table_obj,
table_name=table,
load_form=load_form,
is_table_partitioned=is_table_partitioned,
))
def do_load_table(request, create_hql):
script_file = "/tmp/hive_%s.hcat" % datetime.now().strftime("%s")
file = open(script_file, 'w+')
file.write(create_hql)
file.close()
hcat_client().hive_cli(file=script_file)
if os.path.exists(script_file):
os.remove(script_file)
def do_load_table(request, create_hql):
script_file = "/tmp/hive_%s.hcat" % datetime.now().strftime("%s")
file = open(script_file, 'w+')
file.write(create_hql)
file.close()
hcat_client().hive_cli(file=script_file)
if os.path.exists(script_file):
os.remove(script_file)
def clean_target_table(self):
tbl = self.cleaned_data.get('target_table')
if tbl:
try:
hcat_client().get_table("default", tbl)
raise forms.ValidationError('Table already exists')
except hive_metastore.ttypes.NoSuchObjectException:
pass
return tbl
def _submit_create_and_load(request, create_hql, table_name, path, do_load):
"""
Submit the table creation, and setup the load to happen (if ``do_load``).
"""
tables = []
try:
hcat_client().create_table("default", create_hql)
tables = hcat_client().get_tables()
except Exception, ex:
raise PopupException('Error on creating and loading table',
title="Error on creating and loading table", detail=str(ex))
def drop_table(request, table):
if request.method == 'GET':
title = "This may delete the underlying data as well as the metadata. Drop table '%s'?" % table
return render('confirm.html', request, dict(url=request.path, title=title))
elif request.method == 'POST':
tables = []
try:
hcat_client().drop_table(table)
except Exception, ex:
raise PopupException('Drop table', title="Drop table", detail=str(ex))
on_success_url = urlresolvers.reverse(show_tables)
result = {'on_success_url': on_success_url}
return HttpResponse(json.dumps(result))
def drop_partition(request, table):
if request.method == 'GET':
title = "Do you really want to drop this partition?"
return render('confirm.html', request, dict(url=request.path, title=title))
elif request.method == 'POST':
try:
partition_name = request.POST.get('partition_name')
hcat_client().drop_partition(table, partition_name)
except Exception, ex:
raise PopupException('Drop partition', title="Drop partition", detail=str(ex))
on_success_url = urlresolvers.reverse(describe_table, kwargs=dict(table=table))
result = {'on_success_url': on_success_url}
return HttpResponse(json.dumps(result))
def drop_table(request, table):
if request.method == 'GET':
title = "This may delete the underlying data as well as the metadata. Drop table '%s'?" % table
return render('confirm.html', request,
dict(url=request.path, title=title))
elif request.method == 'POST':
tables = []
try:
hcat_client().drop_table(table)
except Exception, ex:
raise PopupException('Drop table',
title="Drop table",
detail=str(ex))
on_success_url = urlresolvers.reverse(show_tables)
result = {'on_success_url': on_success_url}
return HttpResponse(json.dumps(result))
def drop_partition(request, table):
if request.method == 'GET':
title = "Do you really want to drop this partition?"
return render('confirm.html', request,
dict(url=request.path, title=title))
elif request.method == 'POST':
try:
partition_name = request.POST.get('partition_name')
hcat_client().drop_partition(table, partition_name)
except Exception, ex:
raise PopupException('Drop partition',
title="Drop partition",
detail=str(ex))
on_success_url = urlresolvers.reverse(describe_table,
kwargs=dict(table=table))
result = {'on_success_url': on_success_url}
return HttpResponse(json.dumps(result))
def load_table(request, table):
"""
Loads data into a table.
"""
try:
table_desc_extended = hcat_client().describe_table_extended(table)
is_table_partitioned = table_desc_extended['partitioned']
partitionColumns = []
if is_table_partitioned:
partitionColumns = table_desc_extended['partitionColumns']
table_obj = {
'tableName': table,
'columns': table_desc_extended['columns'],
'partitionKeys': partitionColumns
}
except Exception:
import traceback
error = traceback.format_exc()
raise PopupException('Error getting table description',
title="Error getting table description",
detail=error)
if request.method == "POST":
form = hcatalog.forms.LoadDataForm(table_obj, request.POST)
hql = ''
if form.is_valid():
hql += "LOAD DATA INPATH"
hql += " '%s'" % form.cleaned_data['path']
if form.cleaned_data['overwrite']:
hql += " OVERWRITE"
hql += " INTO TABLE "
hql += "`%s`" % (table, )
if len(form.partition_columns) > 0:
hql += " PARTITION ("
vals = []
for key, column_name in form.partition_columns.iteritems():
vals.append("%s='%s'" %
(column_name, form.cleaned_data[key]))
hql += ", ".join(vals)
hql += ")"
hql += ";"
try:
do_load_table(request, hql)
except Exception:
import traceback
error = traceback.format_exc()
raise PopupException('Error loading data into the table',
title="Error loading data into the table",
detail=error)
on_success_url = urlresolvers.reverse(describe_table,
kwargs=dict(table=table))
result = {'on_success_url': on_success_url}
return HttpResponse(json.dumps(result))
else:
form = hcatalog.forms.LoadDataForm(table_obj)
return render(
"load_table.mako", request,
dict(form=form, table=table, action=request.get_full_path()))
def get_tables(request):
"""Returns a table list"""
tables = []
describe_table_urls = []
read_table_urls = []
try:
tables = hcat_client().get_tables()
except Exception, ex:
result = {'tables': tables, 'exception': str(ex)}
return HttpResponse(json.dumps(result))
def get_tables(request):
"""Returns a table list"""
tables = []
describe_table_urls = []
read_table_urls = []
try:
tables = hcat_client().get_tables()
except Exception, ex:
result = {'tables': tables, 'exception': str(ex)}
return HttpResponse(json.dumps(result))
def browse_partition(request, table):
if request.method == 'POST':
try:
partition_name = request.POST.get('partition_name')
location = hcat_client().get_partition_location(table, partition_name)
#location = hcat_client().describe_partition(table, partition_name)
url = location_to_url(request, location)
result = {'url': url}
return HttpResponse(json.dumps(result))
except Exception, ex:
raise PopupException('Browse partition', title="Browse partition", detail=str(ex))
def describe_table(request, table):
try:
table_desc_extended = hcat_client().describe_table_extended(table)
is_table_partitioned = table_desc_extended['partitioned']
partitions = []
partitionColumns = []
if is_table_partitioned:
partitions = hcat_client().get_partitions(table)
partitionColumns = table_desc_extended['partitionColumns']
table_obj = {'tableName': table, 'columns': table_desc_extended['columns'], 'partitionKeys': partitionColumns, 'partitions': partitions}
except Exception:
import traceback
error = traceback.format_exc()
raise PopupException('Error getting table description', title="Error getting table description", detail=error)
load_form = hcatalog.forms.LoadDataForm(table_obj)
return render("describe_table.mako", request, dict(
table=table_obj,
table_name=table,
load_form=load_form,
is_table_partitioned=is_table_partitioned,
))
def browse_partition(request, table):
if request.method == 'POST':
try:
partition_name = request.POST.get('partition_name')
location = hcat_client().get_partition_location(
table, partition_name)
#location = hcat_client().describe_partition(table, partition_name)
url = location_to_url(request, location)
result = {'url': url}
return HttpResponse(json.dumps(result))
except Exception, ex:
raise PopupException('Browse partition',
title="Browse partition",
detail=str(ex))
def load_table(request, table):
"""
Loads data into a table.
"""
try:
table_desc_extended = hcat_client().describe_table_extended(table)
is_table_partitioned = table_desc_extended['partitioned']
partitionColumns = []
if is_table_partitioned:
partitionColumns = table_desc_extended['partitionColumns']
table_obj = {'tableName': table, 'columns': table_desc_extended['columns'], 'partitionKeys': partitionColumns}
except Exception:
import traceback
error = traceback.format_exc()
raise PopupException('Error getting table description', title="Error getting table description", detail=error)
if request.method == "POST":
form = hcatalog.forms.LoadDataForm(table_obj, request.POST)
hql = ''
if form.is_valid():
hql += "LOAD DATA INPATH"
hql += " '%s'" % form.cleaned_data['path']
if form.cleaned_data['overwrite']:
hql += " OVERWRITE"
hql += " INTO TABLE "
hql += "`%s`" % (table,)
if len(form.partition_columns) > 0:
hql += " PARTITION ("
vals = []
for key, column_name in form.partition_columns.iteritems():
vals.append("%s='%s'" % (column_name, form.cleaned_data[key]))
hql += ", ".join(vals)
hql += ")"
hql += ";"
try:
do_load_table(request, hql)
except Exception:
import traceback
error = traceback.format_exc()
raise PopupException('Error loading data into the table', title="Error loading data into the table", detail=error)
on_success_url = urlresolvers.reverse(describe_table, kwargs=dict(table=table))
result = {'on_success_url': on_success_url}
return HttpResponse(json.dumps(result))
else:
form = hcatalog.forms.LoadDataForm(table_obj)
return render("load_table.mako", request, dict(form=form, table=table, action=request.get_full_path()))
def _clean_tablename(name):
tables = []
try:
tables = hcat_client().get_tables()
except Exception, ex:
raise forms.ValidationError('Error occurred: %s' % (str(ex),))
