def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(
table=hcatalog.forms.CreateTableForm,
columns=hcatalog.forms.ColumnTypeFormSet,
partitions=hcatalog.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid() and 'createTable' in request.POST:
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [f.cleaned_data for f in form.partitions.forms]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
tablename = form.table.cleaned_data['name']
tables = []
try:
hcat_client().create_table("default", proposed_query)
tables = hcat_client().get_tables()
except Exception, ex:
raise PopupException('Error on creating table', title="Error on creating table", detail=str(ex))
return render("show_tables.mako", request, dict(tables=tables,))
def __init__(self, *args, **kwargs):
MultiForm.__init__(self,
mgmt=UnionManagementForm,
bool=ReportConditionBoolForm,
conds=ReportConditionFormset,
*args,
**kwargs)
def design_form_by_instance(design_obj, data=None): action_obj = design_obj.get_root_action() cls = _node_type_TO_FORM_CLS[action_obj.node_type] instances = dict(wf=design_obj, action=action_obj) res = MultiForm(wf=WorkflowForm, action=cls) res.bind(data=data, instances=instances) return res
def design_form_by_instance(design_obj, data=None):
action_obj = design_obj.get_root_action()
cls = _node_type_TO_FORM_CLS[action_obj.node_type]
instances = dict(wf=design_obj, action=action_obj)
res = MultiForm(wf=WorkflowForm, action=cls)
res.bind(data=data, instances=instances)
return res
def query_form():
"""Generates a multi form object for queries."""
return MultiForm(
query=HQLForm,
settings=SettingFormSet,
file_resources=FileResourceFormSet,
functions=FunctionFormSet,
saveform=SaveForm)
def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(table=beeswax.forms.CreateTableForm,
columns=beeswax.forms.ColumnTypeFormSet,
partitions=beeswax.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid():
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [f.cleaned_data for f in form.partitions.forms]
proposed_query = django_mako.render_to_string(
"create_table_statement.mako", {
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
})
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
tablename = form.table.cleaned_data['name']
on_success_url = urlresolvers.reverse(describe_table,
kwargs={'table': tablename})
return confirm_query(request, proposed_query, on_success_url)
else:
form.bind()
return render(
"create_table_manually.mako", request,
dict(action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(db_utils.meta_client().get_tables("default", ".*"))
> 0))
def create_table(request, database='default'):
"""Create a table by specifying its attributes manually"""
db = dbms.get(request.user)
dbs = db.get_databases()
databases = [{
'name':
db,
'url':
reverse('beeswax:create_table', kwargs={'database': db})
} for db in dbs]
form = MultiForm(table=CreateTableForm,
columns=ColumnTypeFormSet,
partitions=PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
form.table.db = db # curry is invalid
form.table.database = database
if request.POST.get('create'):
if form.is_valid():
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [
f.cleaned_data for f in form.partitions.forms
]
proposed_query = django_mako.render_to_string(
"create_table_statement.mako", {
'databases': databases,
'database': database,
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
})
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
table_name = form.table.cleaned_data['name']
return _submit_create_and_load(request,
proposed_query,
table_name,
None,
False,
database=database)
else:
form.bind()
apps_list = _get_apps(request.user, '')
return render(
"create_table_manually.mako", request, {
'apps': apps_list,
'action': "#",
'databases': databases,
'table_form': form.table,
'columns_form': form.columns,
'partitions_form': form.partitions,
'has_tables': len(dbms.get(request.user).get_tables()) > 0,
'database': database,
})
def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(
table=beeswax.forms.CreateTableForm,
columns=beeswax.forms.ColumnTypeFormSet,
partitions=beeswax.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid() and 'createTable' in request.POST:
columns = [ f.cleaned_data for f in form.columns.forms ]
partition_columns = [ f.cleaned_data for f in form.partitions.forms ]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
table_name = form.table.cleaned_data['name']
return _submit_create_and_load(request, proposed_query, table_name, None, False)
else:
form.bind()
return render("create_table_manually.mako", request, dict(
action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(db_utils.meta_client().get_tables("default", ".*")) > 0
))
def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(
table=beeswax.forms.CreateTableForm,
columns=beeswax.forms.ColumnTypeFormSet,
partitions=beeswax.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid():
columns = [ f.cleaned_data for f in form.columns.forms ]
partition_columns = [ f.cleaned_data for f in form.partitions.forms ]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
tablename = form.table.cleaned_data['name']
on_success_url = urlresolvers.reverse(describe_table, kwargs={'table': tablename})
return confirm_query(request, proposed_query, on_success_url)
else:
form.bind()
return render("create_table_manually.mako", request, dict(
action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(db_utils.meta_client().get_tables("default", ".*")) > 0
))
def create_table(request, database=None):
"""Create a table by specifying its attributes manually"""
if database is None:
database = _get_last_database(request, database)
form = MultiForm(
table=hcatalog.forms.CreateTableForm,
columns=hcatalog.forms.ColumnTypeFormSet,
partitions=hcatalog.forms.PartitionTypeFormSet)
db = dbms.get(request.user)
databases = db.get_databases()
db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases)
error = None
if request.method == "POST":
form.bind(request.POST)
form.table.table_list = _get_table_list(request)
if form.is_valid() and 'createTable' in request.POST:
try:
columns = [f.cleaned_data for f in form.columns.forms]
column_names = [col["column_name"] for col in columns]
isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable(column_names)
if not isTableValid:
raise Exception(tableValidErrMsg)
partition_columns = [f.cleaned_data for f in form.partitions.forms]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
})
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
tablename = form.table.cleaned_data['name']
hcat_cli = HCatClient(request.user.username)
hcat_cli.create_table(database, tablename, proposed_query)
databases = hcat_cli.get_databases(like="*")
db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases)
return render("show_tables.mako", request, {
'database': database,
'db_form': db_form,
})
except Exception as ex:
error = ex.message
else:
form.bind()
return render("create_table_manually.mako", request, dict(
database=database,
db_form=db_form,
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
error=error,
))
def create_table(request, database="default"):
"""Create a table by specifying its attributes manually"""
db = dbms.get(request.user)
dbs = db.get_databases()
databases = [{"name": db, "url": reverse("beeswax:create_table", kwargs={"database": db})} for db in dbs]
form = MultiForm(table=CreateTableForm, columns=ColumnTypeFormSet, partitions=PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
form.table.db = db # curry is invalid
form.table.database = database
if request.POST.get("create"):
if form.is_valid():
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [f.cleaned_data for f in form.partitions.forms]
proposed_query = django_mako.render_to_string(
"create_table_statement.mako",
{
"databases": databases,
"database": database,
"table": form.table.cleaned_data,
"columns": columns,
"partition_columns": partition_columns,
},
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode("utf-8")
table_name = form.table.cleaned_data["name"]
return _submit_create_and_load(request, proposed_query, table_name, None, False, database=database)
else:
form.bind()
return render(
"create_table_manually.mako",
request,
{
"action": "#",
"databases": databases,
"table_form": form.table,
"columns_form": form.columns,
"partitions_form": form.partitions,
"has_tables": len(dbms.get(request.user).get_tables()) > 0,
"database": database,
},
)
def create_table(request, database='default'):
"""Create a table by specifying its attributes manually"""
db = dbms.get(request.user)
dbs = db.get_databases()
databases = [{'name':db, 'url':reverse('beeswax:create_table', kwargs={'database': db})} for db in dbs]
form = MultiForm(
table=CreateTableForm,
columns=ColumnTypeFormSet,
partitions=PartitionTypeFormSet
)
if request.method == "POST":
form.bind(request.POST)
form.table.db = db # curry is invalid
form.table.database = database
if request.POST.get('create'):
if form.is_valid():
columns = [ f.cleaned_data for f in form.columns.forms ]
partition_columns = [ f.cleaned_data for f in form.partitions.forms ]
proposed_query = django_mako.render_to_string("create_table_statement.mako", {
'databases': databases,
'database': database,
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
table_name = form.table.cleaned_data['name']
return _submit_create_and_load(request, proposed_query, table_name, None, False, database=database)
else:
form.bind()
return render("create_table_manually.mako", request, {
'action': "#",
'databases': databases,
'table_form': form.table,
'columns_form': form.columns,
'partitions_form': form.partitions,
'has_tables': len(dbms.get(request.user).get_tables()) > 0,
'database': database,
})
def create_table(request):
"""Create a table by specifying its attributes manually"""
db = dbms.get(request.user)
form = MultiForm(table=CreateTableForm,
columns=ColumnTypeFormSet,
partitions=PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
form.table.db = db # curry is invalid
if request.POST.get('create'):
if form.is_valid():
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [
f.cleaned_data for f in form.partitions.forms
]
proposed_query = django_mako.render_to_string(
"create_table_statement.mako", {
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
})
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
table_name = form.table.cleaned_data['name']
return _submit_create_and_load(request, proposed_query,
table_name, None, False)
else:
form.bind()
return render(
"create_table_manually.mako", request,
dict(action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(dbms.get(request.user).get_tables()) > 0))
def create_table(request):
"""Create a table by specifying its attributes manually"""
db = dbms.get(request.user)
form = MultiForm(
table=CreateTableForm,
columns=ColumnTypeFormSet,
partitions=PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
form.table.db = db # curry is invalid
if request.POST.get('create'):
if form.is_valid():
columns = [ f.cleaned_data for f in form.columns.forms ]
partition_columns = [ f.cleaned_data for f in form.partitions.forms ]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
table_name = form.table.cleaned_data['name']
return _submit_create_and_load(request, proposed_query, table_name, None, False)
else:
form.bind()
return render("create_table_manually.mako", request, dict(
action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(dbms.get(request.user).get_tables()) > 0
))
def design_form_by_type(action_type):
cls = _ACTION_TYPE_TO_FORM_CLS[action_type]
return MultiForm(wf=WorkflowDesignForm, action=cls)
def report_form():
"""report_form() -> A MultiForm object for report generator"""
return MultiForm(columns=ReportColumnFormset,
union=UnionMultiForm,
saveform=beeswax.forms.SaveForm)
def create_from_file(request, database=None):
"""Create a table by import from file"""
if database is None:
database = _get_last_database(request)
form = MultiForm(table=hcatalog.forms.CreateTableFromFileForm, )
db = dbms.get(request.user)
databases = db.get_databases()
db_form = hcatalog.forms.DbForm(initial={'database': database},
databases=databases)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid():
parser_options = {}
# table options
table_name = form.table.cleaned_data['name']
replace_delimiter_with = form.table.cleaned_data[
'replace_delimiter_with']
parser_options['replace_delimiter_with'] = replace_delimiter_with
# common options
parser_options['path'] = form.table.cleaned_data['path']
file_type = request.POST.get('file_type',
hcatalog.forms.IMPORT_FILE_TYPE_NONE)
parser_options['file_type'] = file_type
parser_options['preview_start_idx'] = 0
parser_options['preview_end_idx'] = 0
# csv/tsv options
parser_options['do_import_data'] = form.table.cleaned_data[
'import_data']
parser_options['encoding'] = form.table.cleaned_data['encoding']
parser_options['autodetect_delimiter'] = form.table.cleaned_data[
'autodetect_delimiter']
parser_options['read_column_headers'] = form.table.cleaned_data[
'read_column_headers']
parser_options['apply_excel_dialect'] = True
parser_options['ignore_whitespaces'] = form.table.cleaned_data[
'ignore_whitespaces']
parser_options['ignore_tabs'] = form.table.cleaned_data[
'ignore_tabs']
parser_options['single_line_comment'] = form.table.cleaned_data[
'single_line_comment']
parser_options['java_style_comments'] = form.table.cleaned_data[
'java_style_comments']
# xls/xlsx options
parser_options['xls_sheet'] = form.table.cleaned_data['xls_sheet']
parser_options['xls_cell_range'] = form.table.cleaned_data[
'xls_cell_range']
parser_options[
'xls_read_column_headers'] = form.table.cleaned_data[
'xls_read_column_headers']
parser_options['delimiter'] = form.table.cleaned_data['delimiter']
if parser_options['autodetect_delimiter']:
parser_options['delimiters'] = DELIMITERS
else:
parser_options['delimiters'] = (parser_options['delimiter'], )
if parser_options['xls_read_column_headers'] and parser_options[
'preview_start_idx'] == 0:
parser_options['preview_start_idx'] = 1
is_preview_action = 'submitPreviewAction' in request.POST
is_preview_next = 'submitPreviewNext' in request.POST
is_preview_beginning = 'submitPreviewBeginning' in request.POST
if is_preview_action or is_preview_next or is_preview_beginning: # preview action
preview_results = {}
preview_table_resp = ''
fields_list, n_cols, col_names = ([], 0, [])
# validate input parameters
if file_type == hcatalog.forms.IMPORT_FILE_TYPE_NONE:
preview_results['error'] = unicode(
'Cannot define file type.')
return HttpResponse(json.dumps(preview_results))
if is_preview_next and 'preview_start_idx' in request.POST and 'preview_end_idx' in request.POST:
parser_options['preview_start_idx'] = int(
request.POST.get('preview_end_idx')) + 1
parser_options['preview_end_idx'] = int(
request.POST.get(
'preview_end_idx')) + IMPORT_PEEK_NLINES
else:
parser_options['preview_start_idx'] = 0
parser_options['preview_end_idx'] = IMPORT_PEEK_NLINES - 1
if parser_options['xls_read_column_headers']:
parser_options['preview_start_idx'] += 1
parser_options['preview_end_idx'] += 1
try:
parser_options = _on_table_preview(
request.fs,
[processor.TYPE
for processor in FILE_PROCESSORS], parser_options)
row_start_index = parser_options['preview_start_idx']
if parser_options['xls_read_column_headers']:
row_start_index -= 1
preview_table_resp = django_mako.render_to_string(
"file_import_preview_table.mako",
dict(fields_list=parser_options['fields_list'],
column_formset=parser_options['col_formset'],
row_start_index=row_start_index))
isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable(
parser_options['col_names'])
except Exception as ex:
preview_results['error'] = escapejs(ex.message)
else:
preview_results['results'] = preview_table_resp
if not isTableValid:
preview_results['error'] = escapejs(tableValidErrMsg)
options = {}
if file_type == hcatalog.forms.IMPORT_FILE_TYPE_TEXT:
options['delimiter_0'] = parser_options['delimiter_0']
options['delimiter_1'] = parser_options['delimiter_1']
options['file_processor_type'] = parser_options[
'file_processor_type']
elif file_type == hcatalog.forms.IMPORT_FILE_TYPE_SPREADSHEET:
options['xls_sheet'] = parser_options['xls_sheet']
options['xls_sheet_list'] = parser_options[
'xls_sheet_list']
options['preview_start_idx'] = parser_options[
'preview_start_idx']
options['preview_end_idx'] = parser_options[
'preview_end_idx']
options['preview_has_more'] = parser_options[
'preview_has_more']
preview_results['options'] = options
return HttpResponse(
json.dumps(preview_results, cls=JSONEncoderForHTML))
else: # create table action
# validate input parameters
if file_type == hcatalog.forms.IMPORT_FILE_TYPE_NONE:
err_msg = unicode('Cannot define file type.')
return render(
"create_table_from_file.mako", request,
dict(action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=err_msg))
# getting back file_processor_type from preview
parser_options['file_processor_type'] = request.POST.get(
'file_processor_type', None)
user_def_columns = []
user_def_column_names = []
column_count = 0
while 'cols-%d-column_name' % column_count in request.POST \
and 'cols-%d-column_type' % column_count in request.POST:
user_def_column_names.append(
request.POST.get('cols-%d-column_name' % column_count))
user_def_columns.append(
dict(
column_name=request.POST.get(
'cols-%d-column_name' % column_count),
column_type=request.POST.get(
'cols-%d-column_type' % column_count),
))
column_count += 1
try:
isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable(
user_def_column_names)
if not isTableValid:
return render(
"create_table_from_file.mako", request,
dict(action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=escapejs(tableValidErrMsg)))
LOG.debug('Creating table by hcatalog')
proposed_query = django_mako.render_to_string(
"create_table_statement.mako", {
'table':
dict(name=table_name,
comment=form.table.cleaned_data['comment'],
row_format='Delimited',
field_terminator=replace_delimiter_with),
'columns':
user_def_columns,
'partition_columns': []
})
proposed_query = proposed_query.decode('utf-8')
hcat_cli = HCatClient(request.user.username)
hcat_cli.create_table(database, table_name, proposed_query)
do_import_data = parser_options.get('do_import_data', True)
LOG.debug('Data processing stage')
if do_import_data:
parser_options = _on_table_create(
request.fs, parser_options)
path = parser_options.get('results_path', None)
if not path or not request.fs.exists(path):
msg = 'Missing needed result file to load data into table'
LOG.error(msg)
raise Exception(msg)
if not table_name:
msg = 'Internal error: Missing needed parameter to load data into table'
LOG.error(msg)
raise Exception(msg)
LOG.info("Auto loading data from %s into table %s%s" %
(path, database, table_name))
hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (
path, database, table_name)
job_id = hcat_cli.do_hive_query(execute=hql)
on_success_url = urlresolvers.reverse(
get_app_name(request) + ':index')
return render(
"create_table_from_file.mako", request,
dict(
action="#",
job_id=job_id,
on_success_url=on_success_url,
database=database,
db_form=db_form,
table_form=form.table,
error=None,
))
# clean up tmp dir
tmp_dir = parser_options.get('tmp_dir', None)
if tmp_dir and request.fs.exists:
request.fs.rmtree(tmp_dir)
databases = hcat_cli.get_databases(like="*")
db_form = hcatalog.forms.DbForm(
initial={'database': database}, databases=databases)
return render("show_tables.mako", request, {
'database': database,
'db_form': db_form,
})
except Exception as ex:
return render(
"create_table_from_file.mako", request,
dict(action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=escapejs(ex.message)))
return render(
"create_table_from_file.mako", request,
dict(
action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=
"User form is not valid. Please check all input parameters.",
))
else:
form.bind()
return render(
"create_table_from_file.mako", request,
dict(
action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=None,
))
def create_from_file(request, database=None):
"""Create a table by import from file"""
if database is None:
database = _get_last_database(request)
form = MultiForm(
table=hcatalog.forms.CreateTableFromFileForm,
)
db = dbms.get(request.user)
databases = db.get_databases()
db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid():
parser_options = {}
# table options
table_name = form.table.cleaned_data['name']
replace_delimiter_with = form.table.cleaned_data['replace_delimiter_with']
parser_options['replace_delimiter_with'] = replace_delimiter_with
# common options
parser_options['path'] = form.table.cleaned_data['path']
file_type = request.POST.get('file_type', hcatalog.forms.IMPORT_FILE_TYPE_NONE)
parser_options['file_type'] = file_type
parser_options['preview_start_idx'] = 0
parser_options['preview_end_idx'] = 0
# csv/tsv options
parser_options['do_import_data'] = form.table.cleaned_data['import_data']
parser_options['encoding'] = form.table.cleaned_data['encoding']
parser_options['autodetect_delimiter'] = form.table.cleaned_data['autodetect_delimiter']
parser_options['read_column_headers'] = form.table.cleaned_data['read_column_headers']
parser_options['apply_excel_dialect'] = True
parser_options['ignore_whitespaces'] = form.table.cleaned_data['ignore_whitespaces']
parser_options['ignore_tabs'] = form.table.cleaned_data['ignore_tabs']
parser_options['single_line_comment'] = form.table.cleaned_data['single_line_comment']
parser_options['java_style_comments'] = form.table.cleaned_data['java_style_comments']
# xls/xlsx options
parser_options['xls_sheet'] = form.table.cleaned_data['xls_sheet']
parser_options['xls_cell_range'] = form.table.cleaned_data['xls_cell_range']
parser_options['xls_read_column_headers'] = form.table.cleaned_data['xls_read_column_headers']
parser_options['delimiter'] = form.table.cleaned_data['delimiter']
if parser_options['autodetect_delimiter']:
parser_options['delimiters'] = DELIMITERS
else:
parser_options['delimiters'] = (parser_options['delimiter'],)
if parser_options['xls_read_column_headers'] and parser_options['preview_start_idx'] == 0:
parser_options['preview_start_idx'] = 1
is_preview_action = 'submitPreviewAction' in request.POST
is_preview_next = 'submitPreviewNext' in request.POST
is_preview_beginning = 'submitPreviewBeginning' in request.POST
if is_preview_action or is_preview_next or is_preview_beginning: # preview action
preview_results = {}
preview_table_resp = ''
fields_list, n_cols, col_names = ([], 0, [])
# validate input parameters
if file_type == hcatalog.forms.IMPORT_FILE_TYPE_NONE:
preview_results['error'] = unicode('Cannot define file type.')
return HttpResponse(json.dumps(preview_results))
if is_preview_next and 'preview_start_idx' in request.POST and 'preview_end_idx' in request.POST:
parser_options['preview_start_idx'] = int(request.POST.get('preview_end_idx')) + 1
parser_options['preview_end_idx'] = int(request.POST.get('preview_end_idx')) + IMPORT_PEEK_NLINES
else:
parser_options['preview_start_idx'] = 0
parser_options['preview_end_idx'] = IMPORT_PEEK_NLINES - 1
if parser_options['xls_read_column_headers']:
parser_options['preview_start_idx'] += 1
parser_options['preview_end_idx'] += 1
try:
parser_options = _on_table_preview(request.fs, [processor.TYPE for processor in FILE_PROCESSORS],
parser_options)
row_start_index = parser_options['preview_start_idx']
if parser_options['xls_read_column_headers']:
row_start_index -= 1
preview_table_resp = django_mako.render_to_string("file_import_preview_table.mako", dict(
fields_list=parser_options['fields_list'],
column_formset=parser_options['col_formset'],
row_start_index=row_start_index
))
isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable(parser_options['col_names'])
except Exception as ex:
preview_results['error'] = escapejs(ex.message)
else:
preview_results['results'] = preview_table_resp
if not isTableValid:
preview_results['error'] = escapejs(tableValidErrMsg)
options = {}
if file_type == hcatalog.forms.IMPORT_FILE_TYPE_TEXT:
options['delimiter_0'] = parser_options['delimiter_0']
options['delimiter_1'] = parser_options['delimiter_1']
options['file_processor_type'] = parser_options['file_processor_type']
elif file_type == hcatalog.forms.IMPORT_FILE_TYPE_SPREADSHEET:
options['xls_sheet'] = parser_options['xls_sheet']
options['xls_sheet_list'] = parser_options['xls_sheet_list']
options['preview_start_idx'] = parser_options['preview_start_idx']
options['preview_end_idx'] = parser_options['preview_end_idx']
options['preview_has_more'] = parser_options['preview_has_more']
preview_results['options'] = options
return HttpResponse(json.dumps(preview_results, cls=JSONEncoderForHTML))
else: # create table action
# validate input parameters
if file_type == hcatalog.forms.IMPORT_FILE_TYPE_NONE:
err_msg = unicode('Cannot define file type.')
return render("create_table_from_file.mako", request, dict(
action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=err_msg)
)
# getting back file_processor_type from preview
parser_options['file_processor_type'] = request.POST.get('file_processor_type', None)
user_def_columns = []
user_def_column_names = []
column_count = 0
while 'cols-%d-column_name' % column_count in request.POST \
and 'cols-%d-column_type' % column_count in request.POST:
user_def_column_names.append(request.POST.get('cols-%d-column_name' % column_count))
user_def_columns.append(dict(column_name=request.POST.get('cols-%d-column_name' % column_count),
column_type=request.POST.get('cols-%d-column_type' % column_count), ))
column_count += 1
try:
isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable(user_def_column_names)
if not isTableValid:
return render("create_table_from_file.mako", request, dict(
action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=escapejs(tableValidErrMsg)
))
LOG.debug('Creating table by hcatalog')
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': dict(name=table_name,
comment=form.table.cleaned_data[
'comment'],
row_format='Delimited',
field_terminator=replace_delimiter_with),
'columns': user_def_columns,
'partition_columns': []
})
proposed_query = proposed_query.decode('utf-8')
hcat_cli = HCatClient(request.user.username)
hcat_cli.create_table(database, table_name, proposed_query)
do_import_data = parser_options.get('do_import_data', True)
LOG.debug('Data processing stage')
if do_import_data:
parser_options = _on_table_create(request.fs, parser_options)
path = parser_options.get('results_path', None)
if not path or not request.fs.exists(path):
msg = 'Missing needed result file to load data into table'
LOG.error(msg)
raise Exception(msg)
if not table_name:
msg = 'Internal error: Missing needed parameter to load data into table'
LOG.error(msg)
raise Exception(msg)
LOG.info("Auto loading data from %s into table %s%s" % (path, database, table_name))
hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (path, database, table_name)
job_id = hcat_cli.do_hive_query(execute=hql)
on_success_url = urlresolvers.reverse(get_app_name(request) + ':index')
return render("create_table_from_file.mako", request, dict(
action="#",
job_id=job_id,
on_success_url=on_success_url,
database=database,
db_form=db_form,
table_form=form.table,
error=None,
))
# clean up tmp dir
tmp_dir = parser_options.get('tmp_dir', None)
if tmp_dir and request.fs.exists:
request.fs.rmtree(tmp_dir)
databases = hcat_cli.get_databases(like="*")
db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases)
return render("show_tables.mako", request, {
'database': database,
'db_form': db_form,
})
except Exception as ex:
return render("create_table_from_file.mako", request, dict(
action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=escapejs(ex.message)
))
return render("create_table_from_file.mako", request, dict(
action="#",
database=database,
db_form=db_form,
table_form=form.table,
error="User form is not valid. Please check all input parameters.",
))
else:
form.bind()
return render("create_table_from_file.mako", request, dict(
action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=None,
))
def __init__(self, *args, **kwargs):
MultiForm.__init__(self,
mgmt=UnionManagementForm,
bool=ReportConditionBoolForm,
conds=ReportConditionFormset,
*args, **kwargs)
