def create_table(request): """Create a table by specifying its attributes manually""" form = MultiForm( table=beeswax.forms.CreateTableForm, columns=beeswax.forms.ColumnTypeFormSet, partitions=beeswax.forms.PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) if form.is_valid(): columns = [ f.cleaned_data for f in form.columns.forms ] partition_columns = [ f.cleaned_data for f in form.partitions.forms ] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns } ) tablename = form.table.cleaned_data['name'] on_success_url = urlresolvers.reverse(describe_table, kwargs={'table': tablename}) return confirm_query(request, proposed_query, on_success_url) else: form.bind() return render("create_table_manually.mako", request, dict( action="#", table_form=form.table, columns_form=form.columns, partitions_form=form.partitions, has_tables=len(db_utils.meta_client().get_tables("default", ".*")) > 0 ))
def create_table(request): """Create a table by specifying its attributes manually""" form = MultiForm( table=hcatalog.forms.CreateTableForm, columns=hcatalog.forms.ColumnTypeFormSet, partitions=hcatalog.forms.PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) if form.is_valid() and 'createTable' in request.POST: columns = [f.cleaned_data for f in form.columns.forms] partition_columns = [f.cleaned_data for f in form.partitions.forms] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns } ) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') tablename = form.table.cleaned_data['name'] tables = [] try: hcat_client().create_table("default", proposed_query) tables = hcat_client().get_tables() except Exception, ex: raise PopupException('Error on creating table', title="Error on creating table", detail=str(ex)) return render("show_tables.mako", request, dict(tables=tables,))
def create_table(request): """Create a table by specifying its attributes manually""" form = MultiForm( table=beeswax.forms.CreateTableForm, columns=beeswax.forms.ColumnTypeFormSet, partitions=beeswax.forms.PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) if form.is_valid() and 'createTable' in request.POST: columns = [ f.cleaned_data for f in form.columns.forms ] partition_columns = [ f.cleaned_data for f in form.partitions.forms ] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns } ) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') table_name = form.table.cleaned_data['name'] return _submit_create_and_load(request, proposed_query, table_name, None, False) else: form.bind() return render("create_table_manually.mako", request, dict( action="#", table_form=form.table, columns_form=form.columns, partitions_form=form.partitions, has_tables=len(db_utils.meta_client().get_tables("default", ".*")) > 0 ))
def design_form_by_instance(design_obj, data=None): action_obj = design_obj.get_root_action() cls = _node_type_TO_FORM_CLS[action_obj.node_type] instances = dict(wf=design_obj, action=action_obj) res = MultiForm(wf=WorkflowForm, action=cls) res.bind(data=data, instances=instances) return res
def create_table(request, database=None): """Create a table by specifying its attributes manually""" if database is None: database = _get_last_database(request, database) form = MultiForm( table=hcatalog.forms.CreateTableForm, columns=hcatalog.forms.ColumnTypeFormSet, partitions=hcatalog.forms.PartitionTypeFormSet) db = dbms.get(request.user) databases = db.get_databases() db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases) error = None if request.method == "POST": form.bind(request.POST) form.table.table_list = _get_table_list(request) if form.is_valid() and 'createTable' in request.POST: try: columns = [f.cleaned_data for f in form.columns.forms] column_names = [col["column_name"] for col in columns] isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable(column_names) if not isTableValid: raise Exception(tableValidErrMsg) partition_columns = [f.cleaned_data for f in form.partitions.forms] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns }) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') tablename = form.table.cleaned_data['name'] hcat_cli = HCatClient(request.user.username) hcat_cli.create_table(database, tablename, proposed_query) databases = hcat_cli.get_databases(like="*") db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases) return render("show_tables.mako", request, { 'database': database, 'db_form': db_form, }) except Exception as ex: error = ex.message else: form.bind() return render("create_table_manually.mako", request, dict( database=database, db_form=db_form, table_form=form.table, columns_form=form.columns, partitions_form=form.partitions, error=error, ))
def create_table(request, database="default"): """Create a table by specifying its attributes manually""" db = dbms.get(request.user) dbs = db.get_databases() databases = [{"name": db, "url": reverse("beeswax:create_table", kwargs={"database": db})} for db in dbs] form = MultiForm(table=CreateTableForm, columns=ColumnTypeFormSet, partitions=PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) form.table.db = db # curry is invalid form.table.database = database if request.POST.get("create"): if form.is_valid(): columns = [f.cleaned_data for f in form.columns.forms] partition_columns = [f.cleaned_data for f in form.partitions.forms] proposed_query = django_mako.render_to_string( "create_table_statement.mako", { "databases": databases, "database": database, "table": form.table.cleaned_data, "columns": columns, "partition_columns": partition_columns, }, ) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode("utf-8") table_name = form.table.cleaned_data["name"] return _submit_create_and_load(request, proposed_query, table_name, None, False, database=database) else: form.bind() return render( "create_table_manually.mako", request, { "action": "#", "databases": databases, "table_form": form.table, "columns_form": form.columns, "partitions_form": form.partitions, "has_tables": len(dbms.get(request.user).get_tables()) > 0, "database": database, }, )
def create_table(request, database='default'): """Create a table by specifying its attributes manually""" db = dbms.get(request.user) dbs = db.get_databases() databases = [{'name':db, 'url':reverse('beeswax:create_table', kwargs={'database': db})} for db in dbs] form = MultiForm( table=CreateTableForm, columns=ColumnTypeFormSet, partitions=PartitionTypeFormSet ) if request.method == "POST": form.bind(request.POST) form.table.db = db # curry is invalid form.table.database = database if request.POST.get('create'): if form.is_valid(): columns = [ f.cleaned_data for f in form.columns.forms ] partition_columns = [ f.cleaned_data for f in form.partitions.forms ] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'databases': databases, 'database': database, 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns } ) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') table_name = form.table.cleaned_data['name'] return _submit_create_and_load(request, proposed_query, table_name, None, False, database=database) else: form.bind() apps_list = _get_apps(request.user, '') return render("create_table_manually.mako", request, { 'apps': apps_list, 'action': "#", 'databases': databases, 'table_form': form.table, 'columns_form': form.columns, 'partitions_form': form.partitions, 'has_tables': len(dbms.get(request.user).get_tables()) > 0, 'database': database, })
def create_table(request, database='default'): """Create a table by specifying its attributes manually""" db = dbms.get(request.user) dbs = db.get_databases() databases = [{'name':db, 'url':reverse('beeswax:create_table', kwargs={'database': db})} for db in dbs] form = MultiForm( table=CreateTableForm, columns=ColumnTypeFormSet, partitions=PartitionTypeFormSet ) if request.method == "POST": form.bind(request.POST) form.table.db = db # curry is invalid form.table.database = database if request.POST.get('create'): if form.is_valid(): columns = [ f.cleaned_data for f in form.columns.forms ] partition_columns = [ f.cleaned_data for f in form.partitions.forms ] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'databases': databases, 'database': database, 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns } ) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') table_name = form.table.cleaned_data['name'] return _submit_create_and_load(request, proposed_query, table_name, None, False, database=database) else: form.bind() return render("create_table_manually.mako", request, { 'action': "#", 'databases': databases, 'table_form': form.table, 'columns_form': form.columns, 'partitions_form': form.partitions, 'has_tables': len(dbms.get(request.user).get_tables()) > 0, 'database': database, })
def create_table(request): """Create a table by specifying its attributes manually""" db = dbms.get(request.user) form = MultiForm(table=CreateTableForm, columns=ColumnTypeFormSet, partitions=PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) form.table.db = db # curry is invalid if request.POST.get('create'): if form.is_valid(): columns = [f.cleaned_data for f in form.columns.forms] partition_columns = [ f.cleaned_data for f in form.partitions.forms ] proposed_query = django_mako.render_to_string( "create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns }) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') table_name = form.table.cleaned_data['name'] return _submit_create_and_load(request, proposed_query, table_name, None, False) else: form.bind() return render( "create_table_manually.mako", request, dict(action="#", table_form=form.table, columns_form=form.columns, partitions_form=form.partitions, has_tables=len(dbms.get(request.user).get_tables()) > 0))
def create_table(request): """Create a table by specifying its attributes manually""" db = dbms.get(request.user) form = MultiForm( table=CreateTableForm, columns=ColumnTypeFormSet, partitions=PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) form.table.db = db # curry is invalid if request.POST.get('create'): if form.is_valid(): columns = [ f.cleaned_data for f in form.columns.forms ] partition_columns = [ f.cleaned_data for f in form.partitions.forms ] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns } ) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') table_name = form.table.cleaned_data['name'] return _submit_create_and_load(request, proposed_query, table_name, None, False) else: form.bind() return render("create_table_manually.mako", request, dict( action="#", table_form=form.table, columns_form=form.columns, partitions_form=form.partitions, has_tables=len(dbms.get(request.user).get_tables()) > 0 ))
def create_from_file(request, database=None): """Create a table by import from file""" if database is None: database = _get_last_database(request) form = MultiForm(table=hcatalog.forms.CreateTableFromFileForm, ) db = dbms.get(request.user) databases = db.get_databases() db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases) if request.method == "POST": form.bind(request.POST) if form.is_valid(): parser_options = {} # table options table_name = form.table.cleaned_data['name'] replace_delimiter_with = form.table.cleaned_data[ 'replace_delimiter_with'] parser_options['replace_delimiter_with'] = replace_delimiter_with # common options parser_options['path'] = form.table.cleaned_data['path'] file_type = request.POST.get('file_type', hcatalog.forms.IMPORT_FILE_TYPE_NONE) parser_options['file_type'] = file_type parser_options['preview_start_idx'] = 0 parser_options['preview_end_idx'] = 0 # csv/tsv options parser_options['do_import_data'] = form.table.cleaned_data[ 'import_data'] parser_options['encoding'] = form.table.cleaned_data['encoding'] parser_options['autodetect_delimiter'] = form.table.cleaned_data[ 'autodetect_delimiter'] parser_options['read_column_headers'] = form.table.cleaned_data[ 'read_column_headers'] parser_options['apply_excel_dialect'] = True parser_options['ignore_whitespaces'] = form.table.cleaned_data[ 'ignore_whitespaces'] parser_options['ignore_tabs'] = form.table.cleaned_data[ 'ignore_tabs'] parser_options['single_line_comment'] = form.table.cleaned_data[ 'single_line_comment'] parser_options['java_style_comments'] = form.table.cleaned_data[ 'java_style_comments'] # xls/xlsx options parser_options['xls_sheet'] = form.table.cleaned_data['xls_sheet'] parser_options['xls_cell_range'] = form.table.cleaned_data[ 'xls_cell_range'] parser_options[ 'xls_read_column_headers'] = form.table.cleaned_data[ 'xls_read_column_headers'] parser_options['delimiter'] = form.table.cleaned_data['delimiter'] if parser_options['autodetect_delimiter']: parser_options['delimiters'] = DELIMITERS else: parser_options['delimiters'] = (parser_options['delimiter'], ) if parser_options['xls_read_column_headers'] and parser_options[ 'preview_start_idx'] == 0: parser_options['preview_start_idx'] = 1 is_preview_action = 'submitPreviewAction' in request.POST is_preview_next = 'submitPreviewNext' in request.POST is_preview_beginning = 'submitPreviewBeginning' in request.POST if is_preview_action or is_preview_next or is_preview_beginning: # preview action preview_results = {} preview_table_resp = '' fields_list, n_cols, col_names = ([], 0, []) # validate input parameters if file_type == hcatalog.forms.IMPORT_FILE_TYPE_NONE: preview_results['error'] = unicode( 'Cannot define file type.') return HttpResponse(json.dumps(preview_results)) if is_preview_next and 'preview_start_idx' in request.POST and 'preview_end_idx' in request.POST: parser_options['preview_start_idx'] = int( request.POST.get('preview_end_idx')) + 1 parser_options['preview_end_idx'] = int( request.POST.get( 'preview_end_idx')) + IMPORT_PEEK_NLINES else: parser_options['preview_start_idx'] = 0 parser_options['preview_end_idx'] = IMPORT_PEEK_NLINES - 1 if parser_options['xls_read_column_headers']: parser_options['preview_start_idx'] += 1 parser_options['preview_end_idx'] += 1 try: parser_options = _on_table_preview( request.fs, [processor.TYPE for processor in FILE_PROCESSORS], parser_options) row_start_index = parser_options['preview_start_idx'] if parser_options['xls_read_column_headers']: row_start_index -= 1 preview_table_resp = django_mako.render_to_string( "file_import_preview_table.mako", dict(fields_list=parser_options['fields_list'], column_formset=parser_options['col_formset'], row_start_index=row_start_index)) isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable( parser_options['col_names']) except Exception as ex: preview_results['error'] = escapejs(ex.message) else: preview_results['results'] = preview_table_resp if not isTableValid: preview_results['error'] = escapejs(tableValidErrMsg) options = {} if file_type == hcatalog.forms.IMPORT_FILE_TYPE_TEXT: options['delimiter_0'] = parser_options['delimiter_0'] options['delimiter_1'] = parser_options['delimiter_1'] options['file_processor_type'] = parser_options[ 'file_processor_type'] elif file_type == hcatalog.forms.IMPORT_FILE_TYPE_SPREADSHEET: options['xls_sheet'] = parser_options['xls_sheet'] options['xls_sheet_list'] = parser_options[ 'xls_sheet_list'] options['preview_start_idx'] = parser_options[ 'preview_start_idx'] options['preview_end_idx'] = parser_options[ 'preview_end_idx'] options['preview_has_more'] = parser_options[ 'preview_has_more'] preview_results['options'] = options return HttpResponse( json.dumps(preview_results, cls=JSONEncoderForHTML)) else: # create table action # validate input parameters if file_type == hcatalog.forms.IMPORT_FILE_TYPE_NONE: err_msg = unicode('Cannot define file type.') return render( "create_table_from_file.mako", request, dict(action="#", database=database, db_form=db_form, table_form=form.table, error=err_msg)) # getting back file_processor_type from preview parser_options['file_processor_type'] = request.POST.get( 'file_processor_type', None) user_def_columns = [] user_def_column_names = [] column_count = 0 while 'cols-%d-column_name' % column_count in request.POST \ and 'cols-%d-column_type' % column_count in request.POST: user_def_column_names.append( request.POST.get('cols-%d-column_name' % column_count)) user_def_columns.append( dict( column_name=request.POST.get( 'cols-%d-column_name' % column_count), column_type=request.POST.get( 'cols-%d-column_type' % column_count), )) column_count += 1 try: isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable( user_def_column_names) if not isTableValid: return render( "create_table_from_file.mako", request, dict(action="#", database=database, db_form=db_form, table_form=form.table, error=escapejs(tableValidErrMsg))) LOG.debug('Creating table by hcatalog') proposed_query = django_mako.render_to_string( "create_table_statement.mako", { 'table': dict(name=table_name, comment=form.table.cleaned_data['comment'], row_format='Delimited', field_terminator=replace_delimiter_with), 'columns': user_def_columns, 'partition_columns': [] }) proposed_query = proposed_query.decode('utf-8') hcat_cli = HCatClient(request.user.username) hcat_cli.create_table(database, table_name, proposed_query) do_import_data = parser_options.get('do_import_data', True) LOG.debug('Data processing stage') if do_import_data: parser_options = _on_table_create( request.fs, parser_options) path = parser_options.get('results_path', None) if not path or not request.fs.exists(path): msg = 'Missing needed result file to load data into table' LOG.error(msg) raise Exception(msg) if not table_name: msg = 'Internal error: Missing needed parameter to load data into table' LOG.error(msg) raise Exception(msg) LOG.info("Auto loading data from %s into table %s%s" % (path, database, table_name)) hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % ( path, database, table_name) job_id = hcat_cli.do_hive_query(execute=hql) on_success_url = urlresolvers.reverse( get_app_name(request) + ':index') return render( "create_table_from_file.mako", request, dict( action="#", job_id=job_id, on_success_url=on_success_url, database=database, db_form=db_form, table_form=form.table, error=None, )) # clean up tmp dir tmp_dir = parser_options.get('tmp_dir', None) if tmp_dir and request.fs.exists: request.fs.rmtree(tmp_dir) databases = hcat_cli.get_databases(like="*") db_form = hcatalog.forms.DbForm( initial={'database': database}, databases=databases) return render("show_tables.mako", request, { 'database': database, 'db_form': db_form, }) except Exception as ex: return render( "create_table_from_file.mako", request, dict(action="#", database=database, db_form=db_form, table_form=form.table, error=escapejs(ex.message))) return render( "create_table_from_file.mako", request, dict( action="#", database=database, db_form=db_form, table_form=form.table, error= "User form is not valid. Please check all input parameters.", )) else: form.bind() return render( "create_table_from_file.mako", request, dict( action="#", database=database, db_form=db_form, table_form=form.table, error=None, ))
def create_from_file(request, database=None): """Create a table by import from file""" if database is None: database = _get_last_database(request) form = MultiForm( table=hcatalog.forms.CreateTableFromFileForm, ) db = dbms.get(request.user) databases = db.get_databases() db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases) if request.method == "POST": form.bind(request.POST) if form.is_valid(): parser_options = {} # table options table_name = form.table.cleaned_data['name'] replace_delimiter_with = form.table.cleaned_data['replace_delimiter_with'] parser_options['replace_delimiter_with'] = replace_delimiter_with # common options parser_options['path'] = form.table.cleaned_data['path'] file_type = request.POST.get('file_type', hcatalog.forms.IMPORT_FILE_TYPE_NONE) parser_options['file_type'] = file_type parser_options['preview_start_idx'] = 0 parser_options['preview_end_idx'] = 0 # csv/tsv options parser_options['do_import_data'] = form.table.cleaned_data['import_data'] parser_options['encoding'] = form.table.cleaned_data['encoding'] parser_options['autodetect_delimiter'] = form.table.cleaned_data['autodetect_delimiter'] parser_options['read_column_headers'] = form.table.cleaned_data['read_column_headers'] parser_options['apply_excel_dialect'] = True parser_options['ignore_whitespaces'] = form.table.cleaned_data['ignore_whitespaces'] parser_options['ignore_tabs'] = form.table.cleaned_data['ignore_tabs'] parser_options['single_line_comment'] = form.table.cleaned_data['single_line_comment'] parser_options['java_style_comments'] = form.table.cleaned_data['java_style_comments'] # xls/xlsx options parser_options['xls_sheet'] = form.table.cleaned_data['xls_sheet'] parser_options['xls_cell_range'] = form.table.cleaned_data['xls_cell_range'] parser_options['xls_read_column_headers'] = form.table.cleaned_data['xls_read_column_headers'] parser_options['delimiter'] = form.table.cleaned_data['delimiter'] if parser_options['autodetect_delimiter']: parser_options['delimiters'] = DELIMITERS else: parser_options['delimiters'] = (parser_options['delimiter'],) if parser_options['xls_read_column_headers'] and parser_options['preview_start_idx'] == 0: parser_options['preview_start_idx'] = 1 is_preview_action = 'submitPreviewAction' in request.POST is_preview_next = 'submitPreviewNext' in request.POST is_preview_beginning = 'submitPreviewBeginning' in request.POST if is_preview_action or is_preview_next or is_preview_beginning: # preview action preview_results = {} preview_table_resp = '' fields_list, n_cols, col_names = ([], 0, []) # validate input parameters if file_type == hcatalog.forms.IMPORT_FILE_TYPE_NONE: preview_results['error'] = unicode('Cannot define file type.') return HttpResponse(json.dumps(preview_results)) if is_preview_next and 'preview_start_idx' in request.POST and 'preview_end_idx' in request.POST: parser_options['preview_start_idx'] = int(request.POST.get('preview_end_idx')) + 1 parser_options['preview_end_idx'] = int(request.POST.get('preview_end_idx')) + IMPORT_PEEK_NLINES else: parser_options['preview_start_idx'] = 0 parser_options['preview_end_idx'] = IMPORT_PEEK_NLINES - 1 if parser_options['xls_read_column_headers']: parser_options['preview_start_idx'] += 1 parser_options['preview_end_idx'] += 1 try: parser_options = _on_table_preview(request.fs, [processor.TYPE for processor in FILE_PROCESSORS], parser_options) row_start_index = parser_options['preview_start_idx'] if parser_options['xls_read_column_headers']: row_start_index -= 1 preview_table_resp = django_mako.render_to_string("file_import_preview_table.mako", dict( fields_list=parser_options['fields_list'], column_formset=parser_options['col_formset'], row_start_index=row_start_index )) isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable(parser_options['col_names']) except Exception as ex: preview_results['error'] = escapejs(ex.message) else: preview_results['results'] = preview_table_resp if not isTableValid: preview_results['error'] = escapejs(tableValidErrMsg) options = {} if file_type == hcatalog.forms.IMPORT_FILE_TYPE_TEXT: options['delimiter_0'] = parser_options['delimiter_0'] options['delimiter_1'] = parser_options['delimiter_1'] options['file_processor_type'] = parser_options['file_processor_type'] elif file_type == hcatalog.forms.IMPORT_FILE_TYPE_SPREADSHEET: options['xls_sheet'] = parser_options['xls_sheet'] options['xls_sheet_list'] = parser_options['xls_sheet_list'] options['preview_start_idx'] = parser_options['preview_start_idx'] options['preview_end_idx'] = parser_options['preview_end_idx'] options['preview_has_more'] = parser_options['preview_has_more'] preview_results['options'] = options return HttpResponse(json.dumps(preview_results, cls=JSONEncoderForHTML)) else: # create table action # validate input parameters if file_type == hcatalog.forms.IMPORT_FILE_TYPE_NONE: err_msg = unicode('Cannot define file type.') return render("create_table_from_file.mako", request, dict( action="#", database=database, db_form=db_form, table_form=form.table, error=err_msg) ) # getting back file_processor_type from preview parser_options['file_processor_type'] = request.POST.get('file_processor_type', None) user_def_columns = [] user_def_column_names = [] column_count = 0 while 'cols-%d-column_name' % column_count in request.POST \ and 'cols-%d-column_type' % column_count in request.POST: user_def_column_names.append(request.POST.get('cols-%d-column_name' % column_count)) user_def_columns.append(dict(column_name=request.POST.get('cols-%d-column_name' % column_count), column_type=request.POST.get('cols-%d-column_type' % column_count), )) column_count += 1 try: isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable(user_def_column_names) if not isTableValid: return render("create_table_from_file.mako", request, dict( action="#", database=database, db_form=db_form, table_form=form.table, error=escapejs(tableValidErrMsg) )) LOG.debug('Creating table by hcatalog') proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'table': dict(name=table_name, comment=form.table.cleaned_data[ 'comment'], row_format='Delimited', field_terminator=replace_delimiter_with), 'columns': user_def_columns, 'partition_columns': [] }) proposed_query = proposed_query.decode('utf-8') hcat_cli = HCatClient(request.user.username) hcat_cli.create_table(database, table_name, proposed_query) do_import_data = parser_options.get('do_import_data', True) LOG.debug('Data processing stage') if do_import_data: parser_options = _on_table_create(request.fs, parser_options) path = parser_options.get('results_path', None) if not path or not request.fs.exists(path): msg = 'Missing needed result file to load data into table' LOG.error(msg) raise Exception(msg) if not table_name: msg = 'Internal error: Missing needed parameter to load data into table' LOG.error(msg) raise Exception(msg) LOG.info("Auto loading data from %s into table %s%s" % (path, database, table_name)) hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (path, database, table_name) job_id = hcat_cli.do_hive_query(execute=hql) on_success_url = urlresolvers.reverse(get_app_name(request) + ':index') return render("create_table_from_file.mako", request, dict( action="#", job_id=job_id, on_success_url=on_success_url, database=database, db_form=db_form, table_form=form.table, error=None, )) # clean up tmp dir tmp_dir = parser_options.get('tmp_dir', None) if tmp_dir and request.fs.exists: request.fs.rmtree(tmp_dir) databases = hcat_cli.get_databases(like="*") db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases) return render("show_tables.mako", request, { 'database': database, 'db_form': db_form, }) except Exception as ex: return render("create_table_from_file.mako", request, dict( action="#", database=database, db_form=db_form, table_form=form.table, error=escapejs(ex.message) )) return render("create_table_from_file.mako", request, dict( action="#", database=database, db_form=db_form, table_form=form.table, error="User form is not valid. Please check all input parameters.", )) else: form.bind() return render("create_table_from_file.mako", request, dict( action="#", database=database, db_form=db_form, table_form=form.table, error=None, ))