def make_beeswax_query(request, hql, query_form=None): """ make_beeswax_query(request, hql, query_type, query_form=None) -> BeeswaxService.Query object It sets the various configuration (file resources, fuctions, etc) as well. """ query_msg = BeeswaxService.Query(query=hql, configuration=[]) # Configure running user and group. query_msg.hadoop_user = request.user.username query_msg.hadoop_groups = request.user.get_groups() if query_form is not None: for f in query_form.settings.forms: query_msg.configuration.append(django_mako.render_to_string( "hql_set.mako", f.cleaned_data)) for f in query_form.file_resources.forms: type = f.cleaned_data["type"] # Perhaps we should have fully-qualified URIs here already? path = request.fs.uri + f.cleaned_data["path"] query_msg.configuration.append( django_mako.render_to_string("hql_resource.mako", dict(type=type, path=path))) for f in query_form.functions.forms: query_msg.configuration.append( django_mako.render_to_string("hql_function.mako", f.cleaned_data)) return query_msg
def get_configuration_statements(self): configuration = [] for f in self.file_resources: configuration.append(render_to_string("hql_resource.mako", dict(type=f['type'], path=f['path']))) for f in self.functions: configuration.append(render_to_string("hql_function.mako", f)) return configuration
def get_configuration_statements(self): configuration = [] for f in self.file_resources: configuration.append( render_to_string("hql_resource.mako", dict(type=f['type'], path=f['path']))) for f in self.functions: configuration.append(render_to_string("hql_function.mako", f)) return configuration
def get_configuration(self): configuration = [] for f in self.settings: configuration.append(render_to_string("hql_set.mako", f)) for f in self.file_resources: configuration.append(render_to_string("hql_resource.mako", dict(type=f["type"], path=f["path"]))) for f in self.functions: configuration.append(render_to_string("hql_function.mako", f)) return configuration
def get_configuration_statements(self): configuration = [] for f in self.file_resources: if not urlparse.urlsplit(f['path']).scheme: scheme = get_hdfs().fs_defaultfs else: scheme = '' configuration.append(render_to_string("hql_resource.mako", dict(type=f['type'], path=f['path'], scheme=scheme))) for f in self.functions: configuration.append(render_to_string("hql_function.mako", f)) return configuration
def create_table(request): """Create a table by specifying its attributes manually""" form = MultiForm( table=hcatalog.forms.CreateTableForm, columns=hcatalog.forms.ColumnTypeFormSet, partitions=hcatalog.forms.PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) if form.is_valid() and 'createTable' in request.POST: columns = [f.cleaned_data for f in form.columns.forms] partition_columns = [f.cleaned_data for f in form.partitions.forms] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns } ) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') tablename = form.table.cleaned_data['name'] tables = [] try: hcat_client().create_table("default", proposed_query) tables = hcat_client().get_tables() except Exception, ex: raise PopupException('Error on creating table', title="Error on creating table", detail=str(ex)) return render("show_tables.mako", request, dict(tables=tables,))
def _create_database(request, source, destination, start_time): database = destination['name'] comment = destination['description'] use_default_location = destination['useDefaultLocation'] external_path = destination['nonDefaultLocation'] sql = django_mako.render_to_string( "gen/create_database_statement.mako", { 'database': { 'name': database, 'comment': comment, 'use_default_location': use_default_location, 'external_location': external_path, 'properties': [], } }) editor_type = destination['apiHelperType'] on_success_url = reverse('metastore:show_tables', kwargs={'database': database }) + "?source_type=" + source.get( 'sourceType', 'hive') notebook = make_notebook(name=_('Creating database %(name)s') % destination, editor_type=editor_type, statement=sql, status='ready', on_success_url=on_success_url, last_executed=start_time, is_task=True) return notebook.execute(request, batch=False)
def create_table(user=None, query_server=None, table=None): if not user: user = install_sample_user() if not query_server: query_server = get_query_server_config('beeswax') if not table: base_dir = hive_site.get_hive_hook_proto_base_directory() if not base_dir: msg = _( 'Error creating table query_data hive.hook.proto.base-directory is not configured' ) LOG.error(msg) return False table = {'name': 'query_data', 'external_location': base_dir} server = dbms.get(user, query_server) for query in ["create_table_query_data.mako", "msck.mako"]: proposed_query = django_mako.render_to_string(query, {'table': table}) query = hql_query(proposed_query) try: handle = server.execute_and_wait(query) if not handle: LOG.error(_('Error executing %s: Operation timeout.' % query)) return False server.close(handle) except Exception as ex: LOG.error( _('Error executing %(query)s: %(error)s.') % { 'query': query, 'error': ex }) return False LOG.info(_('Table query_data has been created successfully')) return True
def create_table(request): """Create a table by specifying its attributes manually""" form = MultiForm( table=beeswax.forms.CreateTableForm, columns=beeswax.forms.ColumnTypeFormSet, partitions=beeswax.forms.PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) if form.is_valid() and 'createTable' in request.POST: columns = [ f.cleaned_data for f in form.columns.forms ] partition_columns = [ f.cleaned_data for f in form.partitions.forms ] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns } ) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') table_name = form.table.cleaned_data['name'] return _submit_create_and_load(request, proposed_query, table_name, None, False) else: form.bind() return render("create_table_manually.mako", request, dict( action="#", table_form=form.table, columns_form=form.columns, partitions_form=form.partitions, has_tables=len(db_utils.meta_client().get_tables("default", ".*")) > 0 ))
def commonheader(title, section, user, padding="90px", skip_topbar=False): """ Returns the rendered common header """ current_app = None other_apps = [] if user.is_authenticated(): apps = appmanager.get_apps(user) apps_list = appmanager.get_apps_dict(user) for app in apps: if app.display_name not in [ 'beeswax', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser', 'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark', 'indexer', 'security', 'notebook'] and app.menu_index != -1: other_apps.append(app) if section == app.display_name: current_app = app else: apps_list = [] return django_mako.render_to_string("common_header.mako", { 'current_app': current_app, 'apps': apps_list, 'other_apps': other_apps, 'title': title, 'section': section, 'padding': padding, 'user': user, 'skip_topbar': skip_topbar, 'leaflet': { 'layer': desktop.conf.LEAFLET_TILE_LAYER.get(), 'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get() }, 'is_demo': desktop.conf.DEMO_ENABLED.get(), 'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get() })
def create_table(request): """Create a table by specifying its attributes manually""" form = MultiForm( table=beeswax.forms.CreateTableForm, columns=beeswax.forms.ColumnTypeFormSet, partitions=beeswax.forms.PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) if form.is_valid(): columns = [ f.cleaned_data for f in form.columns.forms ] partition_columns = [ f.cleaned_data for f in form.partitions.forms ] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns } ) tablename = form.table.cleaned_data['name'] on_success_url = urlresolvers.reverse(describe_table, kwargs={'table': tablename}) return confirm_query(request, proposed_query, on_success_url) else: form.bind() return render("create_table_manually.mako", request, dict( action="#", table_form=form.table, columns_form=form.columns, partitions_form=form.partitions, has_tables=len(db_utils.meta_client().get_tables("default", ".*")) > 0 ))
def show_tables(request, database=None): if database is None: database = _get_last_database(request, database) if request.method == 'POST': resp = {} try: tables = _get_table_list(request, database) table_list_rendered = django_mako.render_to_string("table_list.mako", dict( app_name=get_app_name(request), database=database, tables=tables, )) except Exception as ex: resp['error'] = escapejs(ex.message) else: resp['table_list_rendered'] = table_list_rendered resp['tables'] = tables return HttpResponse(json.dumps(resp)) db = dbms.get(request.user) databases = db.get_databases() db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases) response = render("show_tables.mako", request, { 'database': database, 'db_form': db_form, }) response.set_cookie("hueHcatalogLastDatabase", database, expires=90) return response
def create_database(request, source, destination): database = destination['name'] comment = destination['description'] use_default_location = destination['useDefaultLocation'] external_path = destination['nonDefaultLocation'] sql = django_mako.render_to_string( "gen/create_database_statement.mako", { 'database': { 'name': database, 'comment': comment, 'use_default_location': use_default_location, 'external_location': external_path, 'properties': [], } }) editor_type = 'hive' on_success_url = reverse('metastore:show_tables', kwargs={'database': database}) try: notebook = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready', on_success_url=on_success_url) return notebook.execute(request, batch=False) except Exception, e: raise PopupException(_('The table could not be created.'), detail=e.message)
def _get_query_history_latest(request_user=None, query_id=None, start_date=None, start_time=None, status=None, limit=25, force_refresh=False): proposed_query = django_mako.render_to_string( "select_table_query_data_latest.mako", { 'table': { 'name': 'query_data', 'request_user': request_user, 'query_id': query_id, 'start_date': start_date, 'start_time': start_time, 'status': status, 'limit': limit, 'force_refresh': force_refresh } }) data = _execute_query(proposed_query, limit) for row in data['data']: if row[1]: row[1] = json.loads(row[1]) if row[5]: row[5] = json.loads(row[5]) if row[8]: row[8] = json.loads(row[8]) return data
def _get_query_history_from(request_user=None, start_date=None, start_time=None, status=None, query_id=None, limit=25): proposed_query = django_mako.render_to_string( "select_table_query_data_from.mako", { 'table': { 'name': 'query_data', 'request_user': request_user, 'start_date': start_date, 'start_time': start_time, 'query_id': query_id, 'status': status, 'limit': limit } }) data = _execute_query(proposed_query, limit) for row in data['data']: if row[1]: row[1] = [row[1]] if row[5]: row[5] = json.loads(row[5]) if row[8]: row[8] = [row[8]] return data
def create_table(request): """Create a table by specifying its attributes manually""" form = MultiForm(table=beeswax.forms.CreateTableForm, columns=beeswax.forms.ColumnTypeFormSet, partitions=beeswax.forms.PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) if form.is_valid(): columns = [f.cleaned_data for f in form.columns.forms] partition_columns = [f.cleaned_data for f in form.partitions.forms] proposed_query = django_mako.render_to_string( "create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns }) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') tablename = form.table.cleaned_data['name'] on_success_url = urlresolvers.reverse(describe_table, kwargs={'table': tablename}) return confirm_query(request, proposed_query, on_success_url) else: form.bind() return render( "create_table_manually.mako", request, dict(action="#", table_form=form.table, columns_form=form.columns, partitions_form=form.partitions, has_tables=len(db_utils.meta_client().get_tables("default", ".*")) > 0))
def commonheader(title, section, user, padding="90px"): """ Returns the rendered common header """ current_app = None other_apps = [] if user.is_authenticated(): apps = appmanager.get_apps(user) apps_list = appmanager.get_apps_dict(user) for app in apps: if app.display_name not in [ 'beeswax', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser', 'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark']: other_apps.append(app) if section == app.display_name: current_app = app else: apps_list = [] return django_mako.render_to_string("common_header.mako", { 'current_app': current_app, 'apps': apps_list, 'other_apps': other_apps, 'title': title, 'section': section, 'padding': padding, 'user': user, 'is_demo': desktop.conf.DEMO_ENABLED.get() })
def create_database(request, source, destination): database = destination['name'] comment = destination['description'] use_default_location = destination['useDefaultLocation'] external_path = destination['nonDefaultLocation'] sql = django_mako.render_to_string("gen/create_database_statement.mako", { 'database': { 'name': database, 'comment': comment, 'use_default_location': use_default_location, 'external_location': external_path, 'properties': [], } } ) editor_type = 'hive' on_success_url = reverse('metastore:show_tables', kwargs={'database': database}) try: notebook = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready', on_success_url=on_success_url) return notebook.execute(request, batch=False) except Exception, e: raise PopupException(_('The table could not be created.'), detail=e.message)
def commonheader(title, section, user, padding="90px"): """ Returns the rendered common header """ current_app = None other_apps = [] if user.is_authenticated(): apps = appmanager.get_apps(user) apps_list = appmanager.get_apps_dict(user) for app in apps: if app.display_name not in [ 'beeswax', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser', 'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark']: other_apps.append(app) if section == app.display_name: current_app = app else: apps_list = [] return django_mako.render_to_string("common_header.mako", { 'current_app': current_app, 'apps': apps_list, 'other_apps': other_apps, 'title': title, 'section': section, 'padding': padding, 'user': user })
def show_tables(request, database=None): if database is None: database = _get_last_database(request, database) if request.method == 'POST': resp = {} try: tables = _get_table_list(request, database) table_list_rendered = django_mako.render_to_string( "table_list.mako", dict( app_name=get_app_name(request), database=database, tables=tables, )) except Exception as ex: resp['error'] = escapejs(ex.message) else: resp['table_list_rendered'] = table_list_rendered resp['tables'] = tables return HttpResponse(json.dumps(resp)) db = dbms.get(request.user) databases = db.get_databases() db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases) response = render("show_tables.mako", request, { 'database': database, 'db_form': db_form, }) response.set_cookie("hueHcatalogLastDatabase", database, expires=90) return response
def commonheader(title, section, user, request=None, padding="90px", skip_topbar=False, skip_idle_timeout=False, is_mobile=False): """ Returns the rendered common header """ current_app, other_apps, apps_list = _get_apps(user, section) template = 'common_header.mako' if is_mobile: template = 'common_header_m.mako' return django_mako.render_to_string(template, { 'current_app': current_app, 'apps': apps_list, 'other_apps': other_apps, 'title': title, 'section': section, 'padding': padding, 'user': user, 'request': request, 'skip_topbar': skip_topbar, 'skip_idle_timeout': skip_idle_timeout, 'leaflet': { 'layer': desktop.conf.LEAFLET_TILE_LAYER.get(), 'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get(), 'map_options': json.dumps(desktop.conf.LEAFLET_MAP_OPTIONS.get()), 'layer_options': json.dumps(desktop.conf.LEAFLET_TILE_LAYER_OPTIONS.get()), }, 'is_demo': desktop.conf.DEMO_ENABLED.get(), 'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(), 'is_s3_enabled': fsmanager.is_enabled('s3a') and fsmanager.has_access('s3a', request.user), 'is_adls_enabled': fsmanager.is_enabled('adl') and fsmanager.has_access('adl', request.user), 'banner_message': get_banner_message(request) })
def get_configuration_statements(self): configuration = [] for f in self.file_resources: if not urlparse.urlsplit(f['path']).scheme: scheme = get_hdfs().fs_defaultfs else: scheme = '' configuration.append( render_to_string( "hql_resource.mako", dict(type=f['type'], path=f['path'], scheme=scheme))) for f in self.functions: configuration.append(render_to_string("hql_function.mako", f)) return configuration
def create_table(request, database='default'): """Create a table by specifying its attributes manually""" db = dbms.get(request.user) dbs = db.get_databases() databases = [{ 'name': db, 'url': reverse('beeswax:create_table', kwargs={'database': db}) } for db in dbs] form = MultiForm(table=CreateTableForm, columns=ColumnTypeFormSet, partitions=PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) form.table.db = db # curry is invalid form.table.database = database if request.POST.get('create'): if form.is_valid(): columns = [f.cleaned_data for f in form.columns.forms] partition_columns = [ f.cleaned_data for f in form.partitions.forms ] proposed_query = django_mako.render_to_string( "create_table_statement.mako", { 'databases': databases, 'database': database, 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns }) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') table_name = form.table.cleaned_data['name'] return _submit_create_and_load(request, proposed_query, table_name, None, False, database=database) else: form.bind() apps_list = _get_apps(request.user, '') return render( "create_table_manually.mako", request, { 'apps': apps_list, 'action': "#", 'databases': databases, 'table_form': form.table, 'columns_form': form.columns, 'partitions_form': form.partitions, 'has_tables': len(dbms.get(request.user).get_tables()) > 0, 'database': database, })
def commonheader(title, section, padding="60px"): """ Returns the rendered common header """ apps_list = sorted(appmanager.DESKTOP_APPS[:], key=lambda app: app.menu_index) return django_mako.render_to_string( "common_header.mako", dict(apps=apps_list, title=title, section=section, padding=padding) )
def render_to_string(template, *args, **kwargs): """Wrapper around django.template.loader.render_to_string which supports different template libraries.""" template_lib = _get_template_lib(template, kwargs) if template_lib == DJANGO: return django_render_to_string(template, *args, **kwargs) elif template_lib == MAKO: return django_mako.render_to_string(template, *args, **kwargs) else: raise Exception("Bad template lib: %s" % template_lib)
def create_table(request, database=None): """Create a table by specifying its attributes manually""" if database is None: database = _get_last_database(request, database) form = MultiForm( table=hcatalog.forms.CreateTableForm, columns=hcatalog.forms.ColumnTypeFormSet, partitions=hcatalog.forms.PartitionTypeFormSet) db = dbms.get(request.user) databases = db.get_databases() db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases) error = None if request.method == "POST": form.bind(request.POST) form.table.table_list = _get_table_list(request) if form.is_valid() and 'createTable' in request.POST: try: columns = [f.cleaned_data for f in form.columns.forms] column_names = [col["column_name"] for col in columns] isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable(column_names) if not isTableValid: raise Exception(tableValidErrMsg) partition_columns = [f.cleaned_data for f in form.partitions.forms] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns }) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') tablename = form.table.cleaned_data['name'] hcat_cli = HCatClient(request.user.username) hcat_cli.create_table(database, tablename, proposed_query) databases = hcat_cli.get_databases(like="*") db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases) return render("show_tables.mako", request, { 'database': database, 'db_form': db_form, }) except Exception as ex: error = ex.message else: form.bind() return render("create_table_manually.mako", request, dict( database=database, db_form=db_form, table_form=form.table, columns_form=form.columns, partitions_form=form.partitions, error=error, ))
def commonheader(title, section, user, padding="60px"): """ Returns the rendered common header """ apps_list = sorted(appmanager.get_apps(user), key=lambda app: app.menu_index) return django_mako.render_to_string("common_header.mako", dict( apps=apps_list, title=title, section=section, padding=padding, user=user ))
def commonheader(title, section, user, padding="90px"): """ Returns the rendered common header """ current_app = None other_apps = [] if user.is_authenticated(): apps = appmanager.get_apps(user) apps_list = appmanager.get_apps_dict(user) for app in apps: if app.display_name not in [ "beeswax", "impala", "pig", "jobsub", "jobbrowser", "metastore", "hbase", "sqoop", "oozie", "filebrowser", "useradmin", "search", "help", "about", "zookeeper", "proxy", "rdbms", "spark", "indexer", "security", ]: other_apps.append(app) if section == app.display_name: current_app = app else: apps_list = [] return django_mako.render_to_string( "common_header.mako", { "current_app": current_app, "apps": apps_list, "other_apps": other_apps, "title": title, "section": section, "padding": padding, "user": user, "is_demo": desktop.conf.DEMO_ENABLED.get(), }, )
def commonheader(title, section, user, padding="60px"): """ Returns the rendered common header """ apps = appmanager.get_apps(user) apps_list = sorted(apps, key=lambda app: app.menu_index) return django_mako.render_to_string("common_header.mako", { 'apps': apps_list, 'title': title, 'section': section, 'padding': padding, 'user': user })
def commonheader(title, section, user, padding="60px"): """ Returns the rendered common header """ apps = appmanager.get_apps(user) apps_list = sorted(apps, key=lambda app: app.menu_index) return django_mako.render_to_string("common_header.mako", dict( apps=apps_list, title=title, section=section, padding=padding, user=user ))
def commonfooter(messages=None): """ Returns the rendered common footer """ if messages is None: messages = {} hue_settings = Settings.get_settings() return django_mako.render_to_string("common_footer.mako", { 'messages': messages, 'version': settings.HUE_DESKTOP_VERSION, 'collect_usage': desktop.conf.COLLECT_USAGE.get(), 'tours_and_tutorials': hue_settings.tours_and_tutorials })
def commonfooter(messages=None): """ Returns the rendered common footer """ if messages is None: messages = {} hue_settings = Settings.get_settings() return django_mako.render_to_string("common_footer.mako", { 'messages': messages, 'version': settings.HUE_DESKTOP_VERSION, 'collect_usage': collect_usage(), 'tours_and_tutorials': hue_settings.tours_and_tutorials })
def show_databases(request): if request.method == 'POST': resp = {} try: databases = HCatClient(request.user.username).get_databases(like="*") databases_list_rendered = django_mako.render_to_string("database_list.mako", dict( app_name=get_app_name(request), databases=databases)) except Exception as ex: resp['error'] = escapejs(ex.message) else: resp['database_list_rendered'] = databases_list_rendered resp['databases'] = databases return HttpResponse(json.dumps(resp)) return render("show_databases.mako", request, {})
def _generate_workflow_xml(self, namenode): """Return a string that is the workflow.xml of this workflow""" action_type = self._design_obj.root_action.action_type data = { 'design': self._design_obj, 'nameNode': namenode, } if action_type == models.OozieStreamingAction.ACTION_TYPE: tmpl = "workflow-streaming.xml.mako" elif action_type == models.OozieMapreduceAction.ACTION_TYPE: tmpl = "workflow-mapreduce.xml.mako" elif action_type == models.OozieJavaAction.ACTION_TYPE: tmpl = "workflow-java.xml.mako" return django_mako.render_to_string(tmpl, data)
def commonfooter(request, messages=None): """ Returns the rendered common footer """ if messages is None: messages = {} hue_settings = Settings.get_settings() return django_mako.render_to_string("common_footer.mako", { 'request': request, 'messages': messages, 'version': hue_version(), 'collect_usage': collect_usage(), 'tours_and_tutorials': hue_settings.tours_and_tutorials })
def show_databases(request): if request.method == 'POST': resp = {} try: databases = HCatClient( request.user.username).get_databases(like="*") databases_list_rendered = django_mako.render_to_string( "database_list.mako", dict(app_name=get_app_name(request), databases=databases)) except Exception as ex: resp['error'] = escapejs(ex.message) else: resp['database_list_rendered'] = databases_list_rendered resp['databases'] = databases return HttpResponse(json.dumps(resp)) return render("show_databases.mako", request, {})
def create_table(request, database="default"): """Create a table by specifying its attributes manually""" db = dbms.get(request.user) dbs = db.get_databases() databases = [{"name": db, "url": reverse("beeswax:create_table", kwargs={"database": db})} for db in dbs] form = MultiForm(table=CreateTableForm, columns=ColumnTypeFormSet, partitions=PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) form.table.db = db # curry is invalid form.table.database = database if request.POST.get("create"): if form.is_valid(): columns = [f.cleaned_data for f in form.columns.forms] partition_columns = [f.cleaned_data for f in form.partitions.forms] proposed_query = django_mako.render_to_string( "create_table_statement.mako", { "databases": databases, "database": database, "table": form.table.cleaned_data, "columns": columns, "partition_columns": partition_columns, }, ) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode("utf-8") table_name = form.table.cleaned_data["name"] return _submit_create_and_load(request, proposed_query, table_name, None, False, database=database) else: form.bind() return render( "create_table_manually.mako", request, { "action": "#", "databases": databases, "table_form": form.table, "columns_form": form.columns, "partitions_form": form.partitions, "has_tables": len(dbms.get(request.user).get_tables()) > 0, "database": database, }, )
def create_table(request, database='default'): """Create a table by specifying its attributes manually""" db = dbms.get(request.user) dbs = db.get_databases() databases = [{'name':db, 'url':reverse('beeswax:create_table', kwargs={'database': db})} for db in dbs] form = MultiForm( table=CreateTableForm, columns=ColumnTypeFormSet, partitions=PartitionTypeFormSet ) if request.method == "POST": form.bind(request.POST) form.table.db = db # curry is invalid form.table.database = database if request.POST.get('create'): if form.is_valid(): columns = [ f.cleaned_data for f in form.columns.forms ] partition_columns = [ f.cleaned_data for f in form.partitions.forms ] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'databases': databases, 'database': database, 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns } ) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') table_name = form.table.cleaned_data['name'] return _submit_create_and_load(request, proposed_query, table_name, None, False, database=database) else: form.bind() return render("create_table_manually.mako", request, { 'action': "#", 'databases': databases, 'table_form': form.table, 'columns_form': form.columns, 'partitions_form': form.partitions, 'has_tables': len(dbms.get(request.user).get_tables()) > 0, 'database': database, })
def commonheader(title, section, user, padding="60px"): """ Returns the rendered common header """ if user.is_authenticated(): apps = appmanager.get_apps(user) apps_list = sorted(apps, key=lambda app: app.menu_index) else: apps_list = [] return django_mako.render_to_string( "common_header.mako", { 'apps': apps_list, 'title': title, 'section': section, 'padding': padding, 'user': user })
def commonfooter(messages=None): """ Returns the rendered common footer """ if messages is None: messages = {} hue_settings = Settings.get_settings() return django_mako.render_to_string( "common_footer.mako", { "messages": messages, "version": settings.HUE_DESKTOP_VERSION, "collect_usage": collect_usage(), "tours_and_tutorials": hue_settings.tours_and_tutorials, }, )
def commonfooter(request, messages=None, is_mobile=False): """ Returns the rendered common footer """ if messages is None: messages = {} hue_settings = Settings.get_settings() template = 'common_footer.mako' if is_mobile: template = 'common_footer_m.mako' return django_mako.render_to_string(template, { 'request': request, 'messages': messages, 'version': hue_version(), 'collect_usage': collect_usage(), })
def create_database(request): if request.method == "POST": data = request.POST.copy() data.setdefault("use_default_location", False) form = CreateDatabaseForm(data) if form.is_valid(): proposed_query = django_mako.render_to_string("create_database_statement.mako", { 'database': form.cleaned_data, }) query = hql_query(proposed_query) return execute_directly(request, query, on_success_url=reverse('metastore:databases')) else: form = CreateDatabaseForm() return render("create_database.mako", request, { 'database_form': form, })
def commonfooter(request, messages=None, is_mobile=False): """ Returns the rendered common footer """ if messages is None: messages = {} hue_settings = Settings.get_settings() template = 'common_footer.mako' if is_mobile: template = 'common_footer_m.mako' return django_mako.render_to_string(template, { 'request': request, 'messages': messages, 'version': hue_version(), 'collect_usage': collect_usage(), 'tours_and_tutorials': hue_settings.tours_and_tutorials })
def create_database(request): if request.method == "POST": data = request.POST.copy() data.setdefault("use_default_location", False) form = CreateDatabaseForm(data) if form.is_valid(): proposed_query = django_mako.render_to_string("create_database_statement.mako", { 'database': form.cleaned_data, }) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') query = hql_query(proposed_query) return execute_directly(request, query, on_success_url=reverse('metastore:databases')) else: form = CreateDatabaseForm() return render("create_database.mako", request, { 'database_form': form, })
def create_database(request): db = dbms.get(request.user) if request.method == "POST": data = request.POST.copy() data.setdefault("use_default_location", False) form = CreateDatabaseForm(data) if form.is_valid(): proposed_query = django_mako.render_to_string("create_database_statement.mako", { 'database': form.cleaned_data, }) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') query = hql_query(proposed_query) return execute_directly(request, query, on_success_url=reverse('catalog:show_databases')) else: form = CreateDatabaseForm() return render("create_database.mako", request, { 'database_form': form, })
def create_table(request): """Create a table by specifying its attributes manually""" db = dbms.get(request.user) form = MultiForm(table=CreateTableForm, columns=ColumnTypeFormSet, partitions=PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) form.table.db = db # curry is invalid if request.POST.get('create'): if form.is_valid(): columns = [f.cleaned_data for f in form.columns.forms] partition_columns = [ f.cleaned_data for f in form.partitions.forms ] proposed_query = django_mako.render_to_string( "create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns }) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') table_name = form.table.cleaned_data['name'] return _submit_create_and_load(request, proposed_query, table_name, None, False) else: form.bind() return render( "create_table_manually.mako", request, dict(action="#", table_form=form.table, columns_form=form.columns, partitions_form=form.partitions, has_tables=len(dbms.get(request.user).get_tables()) > 0))
def create_table(request): """Create a table by specifying its attributes manually""" db = dbms.get(request.user) form = MultiForm( table=CreateTableForm, columns=ColumnTypeFormSet, partitions=PartitionTypeFormSet) if request.method == "POST": form.bind(request.POST) form.table.db = db # curry is invalid if request.POST.get('create'): if form.is_valid(): columns = [ f.cleaned_data for f in form.columns.forms ] partition_columns = [ f.cleaned_data for f in form.partitions.forms ] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'table': form.table.cleaned_data, 'columns': columns, 'partition_columns': partition_columns } ) # Mako outputs bytestring in utf8 proposed_query = proposed_query.decode('utf-8') table_name = form.table.cleaned_data['name'] return _submit_create_and_load(request, proposed_query, table_name, None, False) else: form.bind() return render("create_table_manually.mako", request, dict( action="#", table_form=form.table, columns_form=form.columns, partitions_form=form.partitions, has_tables=len(dbms.get(request.user).get_tables()) > 0 ))
def create_table_from_a_file(self, source, destination, start_time=-1, file_encoding=None): if '.' in destination['name']: database, table_name = destination['name'].split('.', 1) else: database = 'default' table_name = destination['name'] final_table_name = table_name table_format = destination['tableFormat'] source_type = source['sourceType'] columns = destination['columns'] partition_columns = destination['partitionColumns'] kudu_partition_columns = destination['kuduPartitionColumns'] comment = destination['description'] source_path = urllib_unquote(source['path']) load_data = destination['importData'] external = not destination['useDefaultLocation'] external_path = urllib_unquote(destination['nonDefaultLocation']) editor_type = destination['sourceType'] is_transactional = destination['isTransactional'] default_transactional_type = 'insert_only' if destination[ 'isInsertOnly'] else 'default' skip_header = destination['hasHeader'] primary_keys = destination['primaryKeys'] if destination['useCustomDelimiters']: field_delimiter = destination['customFieldDelimiter'] collection_delimiter = destination[ 'customCollectionDelimiter'] or None map_delimiter = destination['customMapDelimiter'] or None else: field_delimiter = ',' collection_delimiter = r'\002' map_delimiter = r'\003' regexp_delimiter = destination['customRegexp'] file_format = 'TextFile' row_format = 'Delimited' serde_name = '' serde_properties = '' extra_create_properties = '' sql = '' if source['inputFormat'] == 'manual': load_data = False source['format'] = {'quoteChar': '"', 'fieldSeparator': ','} if table_format == 'json': row_format = 'serde' serde_name = 'org.apache.hive.hcatalog.data.JsonSerDe' elif table_format == 'regexp': row_format = 'serde' serde_name = 'org.apache.hadoop.hive.serde2.RegexSerDe' serde_properties = '"input.regex" = "%s"' % regexp_delimiter elif table_format == 'csv': if source['format']['quoteChar'] == '"': source['format']['quoteChar'] = '\\"' row_format = 'serde' serde_name = 'org.apache.hadoop.hive.serde2.OpenCSVSerde' serde_properties = '''"separatorChar" = "%(fieldSeparator)s", "quoteChar" = "%(quoteChar)s", "escapeChar" = "\\\\" ''' % source['format'] use_temp_table = table_format in ('parquet', 'orc', 'kudu') or is_transactional if use_temp_table: # We'll be using a temp table to load data if load_data: table_name, final_table_name = 'hue__tmp_%s' % table_name, table_name sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;\n' % { 'database': database, 'table_name': table_name } else: # Manual row_format = '' file_format = table_format skip_header = False if table_format == 'kudu': columns = [ col for col in columns if col['name'] in primary_keys ] + [ col for col in columns if col['name'] not in primary_keys ] if table_format == 'kudu': collection_delimiter = None map_delimiter = None if external or (load_data and table_format in ( 'parquet', 'orc', 'kudu')): # We'll use location to load data if not self.fs.isdir(external_path): # File selected external_path, external_file_name = Hdfs.split(external_path) if len(self.fs.listdir(external_path)) > 1: # If dir not just the file, create data dir and move file there. Make sure it's unique. external_path = external_path + '/%s%s_table' % ( external_file_name, str(uuid.uuid4())) self.fs.mkdir(external_path) self.fs.rename(source_path, external_path) elif load_data: # We'll use load data command parent_path = self.fs.parent_path(source_path) stats = self.fs.stats(parent_path) split = urlparse(source_path) # Only for HDFS, import data and non-external table if split.scheme in ('', 'hdfs') and oct(stats["mode"])[-1] != '7': user_scratch_dir = self.fs.get_home_dir( ) + '/.scratchdir/%s' % str( uuid.uuid4()) # Make sure it's unique. self.fs.do_as_user(self.user, self.fs.mkdir, user_scratch_dir, 0o0777) self.fs.do_as_user(self.user, self.fs.rename, source['path'], user_scratch_dir) source_path = user_scratch_dir + '/' + source['path'].split( '/')[-1] if external_path.lower().startswith( "abfs"): #this is to check if its using an ABFS path external_path = abfspath(external_path) tbl_properties = OrderedDict() if skip_header: tbl_properties['skip.header.line.count'] = '1' # The temp table is not transactional, but final table can be if is_transactional. # tbl_properties that don't exist in previous versions can safely be added without error. tbl_properties['transactional'] = 'false' sql += django_mako.render_to_string( "gen/create_table_statement.mako", { 'table': { 'name': table_name, 'comment': comment, 'row_format': row_format, 'field_terminator': field_delimiter, 'collection_terminator': collection_delimiter if source_type == 'hive' else None, 'map_key_terminator': map_delimiter if source_type == 'hive' else None, 'serde_name': serde_name, 'serde_properties': serde_properties, 'file_format': file_format, 'external': external or load_data and table_format in ('parquet', 'orc', 'kudu'), 'path': external_path, 'primary_keys': primary_keys if table_format == 'kudu' and not load_data else [], 'tbl_properties': tbl_properties }, 'columns': columns, 'partition_columns': partition_columns, 'kudu_partition_columns': kudu_partition_columns, 'database': database }) if file_encoding and file_encoding != 'ASCII' and file_encoding != 'utf-8' and not use_temp_table: sql += '\n\nALTER TABLE `%(database)s`.`%(final_table_name)s` ' \ 'SET serdeproperties ("serialization.encoding"="%(file_encoding)s");' % { 'database': database, 'final_table_name': final_table_name, 'file_encoding': file_encoding } if table_format in ('text', 'json', 'csv', 'regexp') and not external and load_data: form_data = { 'path': source_path, 'overwrite': False, 'partition_columns': [(partition['name'], partition['partitionValue']) for partition in partition_columns], } query_server_config = dbms.get_query_server_config( name=source_type) db = dbms.get(self.user, query_server=query_server_config) sql += "\n\n%s;" % db.load_data( database, table_name, form_data, None, generate_ddl_only=True) if load_data and use_temp_table: file_format = 'TextFile' if table_format == 'text' else table_format if table_format == 'kudu': columns_list = [ '`%s`' % col for col in primary_keys + [ col['name'] for col in destination['columns'] if col['name'] not in primary_keys and col['keep'] ] ] extra_create_properties = """PRIMARY KEY (%(primary_keys)s) PARTITION BY HASH PARTITIONS 16 STORED AS %(file_format)s TBLPROPERTIES( 'kudu.num_tablet_replicas' = '1' )""" % { 'file_format': file_format, 'primary_keys': ', '.join(primary_keys) } else: columns_list = ['*'] extra_create_properties = 'STORED AS %(file_format)s' % { 'file_format': file_format } if is_transactional: extra_create_properties += '\nTBLPROPERTIES("transactional"="true", "transactional_properties"="%s")' % \ default_transactional_type sql += '''\n\nCREATE TABLE `%(database)s`.`%(final_table_name)s`%(comment)s %(extra_create_properties)s AS SELECT %(columns_list)s FROM `%(database)s`.`%(table_name)s`;''' % { 'database': database, 'final_table_name': final_table_name, 'table_name': table_name, 'extra_create_properties': extra_create_properties, 'columns_list': ', '.join(columns_list), 'comment': ' COMMENT "%s"' % comment if comment else '' } sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;\n' % { 'database': database, 'table_name': table_name } if file_encoding and file_encoding != 'ASCII' and file_encoding != 'utf-8': sql += '\n\nALTER TABLE `%(database)s`.`%(final_table_name)s` ' \ 'SET serdeproperties ("serialization.encoding"="%(file_encoding)s");' % { 'database': database, 'final_table_name': final_table_name, 'file_encoding': file_encoding } on_success_url = reverse('metastore:describe_table', kwargs={ 'database': database, 'table': final_table_name }) + '?source_type=' + source_type return make_notebook(name=_('Creating table %(database)s.%(table)s') % { 'database': database, 'table': final_table_name }, editor_type=editor_type, statement=sql.strip(), status='ready', database=database, on_success_url=on_success_url, last_executed=start_time, is_task=True)
if request.GET.get('format') == 'json': # For Editor return JsonResponse({ 'coordinator': coordinator.get_data_for_json(), 'credentials': credentials.credentials.keys(), 'workflows': workflows, 'doc_uuid': doc.uuid if doc else '', 'can_edit': doc is None or doc.doc.get().is_editable(request.user), 'layout': django_mako.render_to_string( 'editor2/common_scheduler.mako', {'coordinator_json': coordinator.to_json_for_html()}) }) else: return render( 'editor2/coordinator_editor.mako', request, { 'coordinator_json': coordinator.to_json_for_html(), 'credentials_json': json.dumps(credentials.credentials.keys(), cls=JSONEncoderForHTML), 'workflows_json': json.dumps(workflows, cls=JSONEncoderForHTML), 'doc_uuid': doc.uuid if doc else '', 'can_edit_json':
raise PopupException(_('You don\'t have access to the workflow of this coordinator.')) if USE_NEW_EDITOR.get(): # In Hue 4, merge with above workflows = [dict([('uuid', d.uuid), ('name', d.name)]) for d in Document2.objects.documents(request.user, include_managed=False).search_documents(types=['oozie-workflow2'])] can_edit = doc is None or (doc.can_write(request.user) if USE_NEW_EDITOR.get() else doc.doc.get().is_editable(request.user)) if request.GET.get('format') == 'json': # For Editor return JsonResponse({ 'coordinator': coordinator.get_data_for_json(), 'credentials': credentials.credentials.keys(), 'workflows': workflows, 'doc_uuid': doc.uuid if doc else '', 'is_embeddable': request.GET.get('is_embeddable', False), 'can_edit': can_edit, 'layout': django_mako.render_to_string('editor2/common_scheduler.mako', {'coordinator_json': coordinator.to_json_for_html()}) }) else: return render('editor2/coordinator_editor.mako', request, { 'coordinator_json': coordinator.to_json_for_html(), 'credentials_json': json.dumps(credentials.credentials.keys(), cls=JSONEncoderForHTML), 'workflows_json': json.dumps(workflows, cls=JSONEncoderForHTML), 'doc_uuid': doc.uuid if doc else '', 'is_embeddable': request.GET.get('is_embeddable', False), 'can_edit_json': json.dumps(can_edit) }) @check_editor_access_permission def new_coordinator(request): return edit_coordinator(request)
def commonshare2(): return django_mako.render_to_string("common_share2.mako", {})
def import_wizard(request, database='default'): """ Help users define table and based on a file they want to import to Hive. Limitations: - Rows are delimited (no serde). - No detection for map and array types. - No detection for the presence of column header in the first row. - No partition table. - Does not work with binary data. """ encoding = i18n.get_site_encoding() app_name = get_app_name(request) db = dbms.get(request.user) dbs = db.get_databases() databases = [{'name':db, 'url':reverse('beeswax:import_wizard', kwargs={'database': db})} for db in dbs] if request.method == 'POST': # # General processing logic: # - We have 3 steps. Each requires the previous. # * Step 1 : Table name and file location # * Step 2a : Display sample with auto chosen delim # * Step 2b : Display sample with user chosen delim (if user chooses one) # * Step 3 : Display sample, and define columns # - Each step is represented by a different form. The form of an earlier step # should be present when submitting to a later step. # - To preserve the data from the earlier steps, we send the forms back as # hidden fields. This way, when users revisit a previous step, the data would # be there as well. # delim_is_auto = False fields_list, n_cols = [[]], 0 s3_col_formset = None s1_file_form = CreateByImportFileForm(request.POST, db=db) if s1_file_form.is_valid(): do_s2_auto_delim = request.POST.get('submit_file') # Step 1 -> 2 do_s2_user_delim = request.POST.get('submit_preview') # Step 2 -> 2 do_s3_column_def = request.POST.get('submit_delim') # Step 2 -> 3 do_hive_create = request.POST.get('submit_create') # Step 3 -> execute cancel_s2_user_delim = request.POST.get('cancel_delim') # Step 2 -> 1 cancel_s3_column_def = request.POST.get('cancel_create') # Step 3 -> 2 # Exactly one of these should be True if len(filter(None, (do_s2_auto_delim, do_s2_user_delim, do_s3_column_def, do_hive_create, cancel_s2_user_delim, cancel_s3_column_def))) != 1: raise PopupException(_('Invalid form submission')) if not do_s2_auto_delim: # We should have a valid delim form s2_delim_form = CreateByImportDelimForm(request.POST) if not s2_delim_form.is_valid(): # Go back to picking delimiter do_s2_user_delim, do_s3_column_def, do_hive_create = True, False, False if do_hive_create: # We should have a valid columns formset s3_col_formset = ColumnTypeFormSet(prefix='cols', data=request.POST) if not s3_col_formset.is_valid(): # Go back to define columns do_s3_column_def, do_hive_create = True, False # # Go to step 2: We've just picked the file. Preview it. # if do_s2_auto_delim: delim_is_auto = True fields_list, n_cols, s2_delim_form = _delim_preview(request.fs, s1_file_form, encoding, [reader.TYPE for reader in FILE_READERS], DELIMITERS) if (do_s2_user_delim or do_s3_column_def or cancel_s3_column_def) and s2_delim_form.is_valid(): # Delimit based on input fields_list, n_cols, s2_delim_form = _delim_preview(request.fs, s1_file_form, encoding, (s2_delim_form.cleaned_data['file_type'],), (s2_delim_form.cleaned_data['delimiter'],)) if do_s2_auto_delim or do_s2_user_delim or cancel_s3_column_def: return render('choose_delimiter.mako', request, { 'action': reverse(app_name + ':import_wizard', kwargs={'database': database}), 'delim_readable': DELIMITER_READABLE.get(s2_delim_form['delimiter'].data[0], s2_delim_form['delimiter'].data[1]), 'initial': delim_is_auto, 'file_form': s1_file_form, 'delim_form': s2_delim_form, 'fields_list': fields_list, 'delimiter_choices': TERMINATOR_CHOICES, 'n_cols': n_cols, 'database': database, 'databases': databases }) # # Go to step 3: Define column. # if do_s3_column_def: if s3_col_formset is None: columns = [] for i in range(n_cols): columns.append({ 'column_name': 'col_%s' % (i,), 'column_type': 'string', }) s3_col_formset = ColumnTypeFormSet(prefix='cols', initial=columns) try: fields_list_for_json = list(fields_list) if fields_list_for_json: fields_list_for_json[0] = map(lambda a: re.sub('[^\w]', '', a), fields_list_for_json[0]) # Cleaning headers return render('define_columns.mako', request, { 'action': reverse(app_name + ':import_wizard', kwargs={'database': database}), 'file_form': s1_file_form, 'delim_form': s2_delim_form, 'column_formset': s3_col_formset, 'fields_list': fields_list, 'fields_list_json': json.dumps(fields_list_for_json), 'n_cols': n_cols, 'database': database, 'databases': databases }) except Exception, e: raise PopupException(_("The selected delimiter is creating an un-even number of columns. Please make sure you don't have empty columns."), detail=e) # # Final: Execute # if do_hive_create: delim = s2_delim_form.cleaned_data['delimiter'] table_name = s1_file_form.cleaned_data['name'] proposed_query = django_mako.render_to_string("create_table_statement.mako", { 'table': { 'name': table_name, 'comment': s1_file_form.cleaned_data['comment'], 'row_format': 'Delimited', 'field_terminator': delim }, 'columns': [ f.cleaned_data for f in s3_col_formset.forms ], 'partition_columns': [], 'database': database, 'databases': databases } ) do_load_data = s1_file_form.cleaned_data.get('do_import') path = s1_file_form.cleaned_data['path'] try: return _submit_create_and_load(request, proposed_query, table_name, path, do_load_data, database=database) except QueryServerException, e: raise PopupException(_('The table could not be created.'), detail=e.message)
def commonimportexport(request): return django_mako.render_to_string("common_import_export.mako", {'request': request})
def create_from_file(request, database=None): """Create a table by import from file""" if database is None: database = _get_last_database(request) form = MultiForm(table=hcatalog.forms.CreateTableFromFileForm, ) db = dbms.get(request.user) databases = db.get_databases() db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases) if request.method == "POST": form.bind(request.POST) if form.is_valid(): parser_options = {} # table options table_name = form.table.cleaned_data['name'] replace_delimiter_with = form.table.cleaned_data[ 'replace_delimiter_with'] parser_options['replace_delimiter_with'] = replace_delimiter_with # common options parser_options['path'] = form.table.cleaned_data['path'] file_type = request.POST.get('file_type', hcatalog.forms.IMPORT_FILE_TYPE_NONE) parser_options['file_type'] = file_type parser_options['preview_start_idx'] = 0 parser_options['preview_end_idx'] = 0 # csv/tsv options parser_options['do_import_data'] = form.table.cleaned_data[ 'import_data'] parser_options['encoding'] = form.table.cleaned_data['encoding'] parser_options['autodetect_delimiter'] = form.table.cleaned_data[ 'autodetect_delimiter'] parser_options['read_column_headers'] = form.table.cleaned_data[ 'read_column_headers'] parser_options['apply_excel_dialect'] = True parser_options['ignore_whitespaces'] = form.table.cleaned_data[ 'ignore_whitespaces'] parser_options['ignore_tabs'] = form.table.cleaned_data[ 'ignore_tabs'] parser_options['single_line_comment'] = form.table.cleaned_data[ 'single_line_comment'] parser_options['java_style_comments'] = form.table.cleaned_data[ 'java_style_comments'] # xls/xlsx options parser_options['xls_sheet'] = form.table.cleaned_data['xls_sheet'] parser_options['xls_cell_range'] = form.table.cleaned_data[ 'xls_cell_range'] parser_options[ 'xls_read_column_headers'] = form.table.cleaned_data[ 'xls_read_column_headers'] parser_options['delimiter'] = form.table.cleaned_data['delimiter'] if parser_options['autodetect_delimiter']: parser_options['delimiters'] = DELIMITERS else: parser_options['delimiters'] = (parser_options['delimiter'], ) if parser_options['xls_read_column_headers'] and parser_options[ 'preview_start_idx'] == 0: parser_options['preview_start_idx'] = 1 is_preview_action = 'submitPreviewAction' in request.POST is_preview_next = 'submitPreviewNext' in request.POST is_preview_beginning = 'submitPreviewBeginning' in request.POST if is_preview_action or is_preview_next or is_preview_beginning: # preview action preview_results = {} preview_table_resp = '' fields_list, n_cols, col_names = ([], 0, []) # validate input parameters if file_type == hcatalog.forms.IMPORT_FILE_TYPE_NONE: preview_results['error'] = unicode( 'Cannot define file type.') return HttpResponse(json.dumps(preview_results)) if is_preview_next and 'preview_start_idx' in request.POST and 'preview_end_idx' in request.POST: parser_options['preview_start_idx'] = int( request.POST.get('preview_end_idx')) + 1 parser_options['preview_end_idx'] = int( request.POST.get( 'preview_end_idx')) + IMPORT_PEEK_NLINES else: parser_options['preview_start_idx'] = 0 parser_options['preview_end_idx'] = IMPORT_PEEK_NLINES - 1 if parser_options['xls_read_column_headers']: parser_options['preview_start_idx'] += 1 parser_options['preview_end_idx'] += 1 try: parser_options = _on_table_preview( request.fs, [processor.TYPE for processor in FILE_PROCESSORS], parser_options) row_start_index = parser_options['preview_start_idx'] if parser_options['xls_read_column_headers']: row_start_index -= 1 preview_table_resp = django_mako.render_to_string( "file_import_preview_table.mako", dict(fields_list=parser_options['fields_list'], column_formset=parser_options['col_formset'], row_start_index=row_start_index)) isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable( parser_options['col_names']) except Exception as ex: preview_results['error'] = escapejs(ex.message) else: preview_results['results'] = preview_table_resp if not isTableValid: preview_results['error'] = escapejs(tableValidErrMsg) options = {} if file_type == hcatalog.forms.IMPORT_FILE_TYPE_TEXT: options['delimiter_0'] = parser_options['delimiter_0'] options['delimiter_1'] = parser_options['delimiter_1'] options['file_processor_type'] = parser_options[ 'file_processor_type'] elif file_type == hcatalog.forms.IMPORT_FILE_TYPE_SPREADSHEET: options['xls_sheet'] = parser_options['xls_sheet'] options['xls_sheet_list'] = parser_options[ 'xls_sheet_list'] options['preview_start_idx'] = parser_options[ 'preview_start_idx'] options['preview_end_idx'] = parser_options[ 'preview_end_idx'] options['preview_has_more'] = parser_options[ 'preview_has_more'] preview_results['options'] = options return HttpResponse( json.dumps(preview_results, cls=JSONEncoderForHTML)) else: # create table action # validate input parameters if file_type == hcatalog.forms.IMPORT_FILE_TYPE_NONE: err_msg = unicode('Cannot define file type.') return render( "create_table_from_file.mako", request, dict(action="#", database=database, db_form=db_form, table_form=form.table, error=err_msg)) # getting back file_processor_type from preview parser_options['file_processor_type'] = request.POST.get( 'file_processor_type', None) user_def_columns = [] user_def_column_names = [] column_count = 0 while 'cols-%d-column_name' % column_count in request.POST \ and 'cols-%d-column_type' % column_count in request.POST: user_def_column_names.append( request.POST.get('cols-%d-column_name' % column_count)) user_def_columns.append( dict( column_name=request.POST.get( 'cols-%d-column_name' % column_count), column_type=request.POST.get( 'cols-%d-column_type' % column_count), )) column_count += 1 try: isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable( user_def_column_names) if not isTableValid: return render( "create_table_from_file.mako", request, dict(action="#", database=database, db_form=db_form, table_form=form.table, error=escapejs(tableValidErrMsg))) LOG.debug('Creating table by hcatalog') proposed_query = django_mako.render_to_string( "create_table_statement.mako", { 'table': dict(name=table_name, comment=form.table.cleaned_data['comment'], row_format='Delimited', field_terminator=replace_delimiter_with), 'columns': user_def_columns, 'partition_columns': [] }) proposed_query = proposed_query.decode('utf-8') hcat_cli = HCatClient(request.user.username) hcat_cli.create_table(database, table_name, proposed_query) do_import_data = parser_options.get('do_import_data', True) LOG.debug('Data processing stage') if do_import_data: parser_options = _on_table_create( request.fs, parser_options) path = parser_options.get('results_path', None) if not path or not request.fs.exists(path): msg = 'Missing needed result file to load data into table' LOG.error(msg) raise Exception(msg) if not table_name: msg = 'Internal error: Missing needed parameter to load data into table' LOG.error(msg) raise Exception(msg) LOG.info("Auto loading data from %s into table %s%s" % (path, database, table_name)) hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % ( path, database, table_name) job_id = hcat_cli.do_hive_query(execute=hql) on_success_url = urlresolvers.reverse( get_app_name(request) + ':index') return render( "create_table_from_file.mako", request, dict( action="#", job_id=job_id, on_success_url=on_success_url, database=database, db_form=db_form, table_form=form.table, error=None, )) # clean up tmp dir tmp_dir = parser_options.get('tmp_dir', None) if tmp_dir and request.fs.exists: request.fs.rmtree(tmp_dir) databases = hcat_cli.get_databases(like="*") db_form = hcatalog.forms.DbForm( initial={'database': database}, databases=databases) return render("show_tables.mako", request, { 'database': database, 'db_form': db_form, }) except Exception as ex: return render( "create_table_from_file.mako", request, dict(action="#", database=database, db_form=db_form, table_form=form.table, error=escapejs(ex.message))) return render( "create_table_from_file.mako", request, dict( action="#", database=database, db_form=db_form, table_form=form.table, error= "User form is not valid. Please check all input parameters.", )) else: form.bind() return render( "create_table_from_file.mako", request, dict( action="#", database=database, db_form=db_form, table_form=form.table, error=None, ))