def make_beeswax_query(request, hql, query_form=None):
"""
make_beeswax_query(request, hql, query_type, query_form=None) -> BeeswaxService.Query object
It sets the various configuration (file resources, fuctions, etc) as well.
"""
query_msg = BeeswaxService.Query(query=hql, configuration=[])
# Configure running user and group.
query_msg.hadoop_user = request.user.username
query_msg.hadoop_groups = request.user.get_groups()
if query_form is not None:
for f in query_form.settings.forms:
query_msg.configuration.append(django_mako.render_to_string(
"hql_set.mako", f.cleaned_data))
for f in query_form.file_resources.forms:
type = f.cleaned_data["type"]
# Perhaps we should have fully-qualified URIs here already?
path = request.fs.uri + f.cleaned_data["path"]
query_msg.configuration.append(
django_mako.render_to_string("hql_resource.mako", dict(type=type, path=path)))
for f in query_form.functions.forms:
query_msg.configuration.append(
django_mako.render_to_string("hql_function.mako", f.cleaned_data))
return query_msg
def get_configuration_statements(self):
configuration = []
for f in self.file_resources:
configuration.append(render_to_string("hql_resource.mako", dict(type=f['type'], path=f['path'])))
for f in self.functions:
configuration.append(render_to_string("hql_function.mako", f))
return configuration
def get_configuration_statements(self):
configuration = []
for f in self.file_resources:
configuration.append(
render_to_string("hql_resource.mako",
dict(type=f['type'], path=f['path'])))
for f in self.functions:
configuration.append(render_to_string("hql_function.mako", f))
return configuration
def get_configuration(self):
configuration = []
for f in self.settings:
configuration.append(render_to_string("hql_set.mako", f))
for f in self.file_resources:
configuration.append(render_to_string("hql_resource.mako", dict(type=f["type"], path=f["path"])))
for f in self.functions:
configuration.append(render_to_string("hql_function.mako", f))
return configuration
def get_configuration_statements(self):
configuration = []
for f in self.file_resources:
if not urlparse.urlsplit(f['path']).scheme:
scheme = get_hdfs().fs_defaultfs
else:
scheme = ''
configuration.append(render_to_string("hql_resource.mako", dict(type=f['type'], path=f['path'], scheme=scheme)))
for f in self.functions:
configuration.append(render_to_string("hql_function.mako", f))
return configuration
def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(
table=hcatalog.forms.CreateTableForm,
columns=hcatalog.forms.ColumnTypeFormSet,
partitions=hcatalog.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid() and 'createTable' in request.POST:
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [f.cleaned_data for f in form.partitions.forms]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
tablename = form.table.cleaned_data['name']
tables = []
try:
hcat_client().create_table("default", proposed_query)
tables = hcat_client().get_tables()
except Exception, ex:
raise PopupException('Error on creating table', title="Error on creating table", detail=str(ex))
return render("show_tables.mako", request, dict(tables=tables,))
def _create_database(request, source, destination, start_time):
database = destination['name']
comment = destination['description']
use_default_location = destination['useDefaultLocation']
external_path = destination['nonDefaultLocation']
sql = django_mako.render_to_string(
"gen/create_database_statement.mako", {
'database': {
'name': database,
'comment': comment,
'use_default_location': use_default_location,
'external_location': external_path,
'properties': [],
}
})
editor_type = destination['apiHelperType']
on_success_url = reverse('metastore:show_tables',
kwargs={'database': database
}) + "?source_type=" + source.get(
'sourceType', 'hive')
notebook = make_notebook(name=_('Creating database %(name)s') %
destination,
editor_type=editor_type,
statement=sql,
status='ready',
on_success_url=on_success_url,
last_executed=start_time,
is_task=True)
return notebook.execute(request, batch=False)
def create_table(user=None, query_server=None, table=None):
if not user:
user = install_sample_user()
if not query_server:
query_server = get_query_server_config('beeswax')
if not table:
base_dir = hive_site.get_hive_hook_proto_base_directory()
if not base_dir:
msg = _(
'Error creating table query_data hive.hook.proto.base-directory is not configured'
)
LOG.error(msg)
return False
table = {'name': 'query_data', 'external_location': base_dir}
server = dbms.get(user, query_server)
for query in ["create_table_query_data.mako", "msck.mako"]:
proposed_query = django_mako.render_to_string(query, {'table': table})
query = hql_query(proposed_query)
try:
handle = server.execute_and_wait(query)
if not handle:
LOG.error(_('Error executing %s: Operation timeout.' % query))
return False
server.close(handle)
except Exception as ex:
LOG.error(
_('Error executing %(query)s: %(error)s.') % {
'query': query,
'error': ex
})
return False
LOG.info(_('Table query_data has been created successfully'))
return True
def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(
table=beeswax.forms.CreateTableForm,
columns=beeswax.forms.ColumnTypeFormSet,
partitions=beeswax.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid() and 'createTable' in request.POST:
columns = [ f.cleaned_data for f in form.columns.forms ]
partition_columns = [ f.cleaned_data for f in form.partitions.forms ]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
table_name = form.table.cleaned_data['name']
return _submit_create_and_load(request, proposed_query, table_name, None, False)
else:
form.bind()
return render("create_table_manually.mako", request, dict(
action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(db_utils.meta_client().get_tables("default", ".*")) > 0
))
def commonheader(title, section, user, padding="90px", skip_topbar=False):
"""
Returns the rendered common header
"""
current_app = None
other_apps = []
if user.is_authenticated():
apps = appmanager.get_apps(user)
apps_list = appmanager.get_apps_dict(user)
for app in apps:
if app.display_name not in [
'beeswax', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser',
'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark', 'indexer', 'security', 'notebook'] and app.menu_index != -1:
other_apps.append(app)
if section == app.display_name:
current_app = app
else:
apps_list = []
return django_mako.render_to_string("common_header.mako", {
'current_app': current_app,
'apps': apps_list,
'other_apps': other_apps,
'title': title,
'section': section,
'padding': padding,
'user': user,
'skip_topbar': skip_topbar,
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get()
},
'is_demo': desktop.conf.DEMO_ENABLED.get(),
'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get()
})
def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(
table=beeswax.forms.CreateTableForm,
columns=beeswax.forms.ColumnTypeFormSet,
partitions=beeswax.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid():
columns = [ f.cleaned_data for f in form.columns.forms ]
partition_columns = [ f.cleaned_data for f in form.partitions.forms ]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
tablename = form.table.cleaned_data['name']
on_success_url = urlresolvers.reverse(describe_table, kwargs={'table': tablename})
return confirm_query(request, proposed_query, on_success_url)
else:
form.bind()
return render("create_table_manually.mako", request, dict(
action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(db_utils.meta_client().get_tables("default", ".*")) > 0
))
def show_tables(request, database=None):
if database is None:
database = _get_last_database(request, database)
if request.method == 'POST':
resp = {}
try:
tables = _get_table_list(request, database)
table_list_rendered = django_mako.render_to_string("table_list.mako", dict(
app_name=get_app_name(request),
database=database,
tables=tables,
))
except Exception as ex:
resp['error'] = escapejs(ex.message)
else:
resp['table_list_rendered'] = table_list_rendered
resp['tables'] = tables
return HttpResponse(json.dumps(resp))
db = dbms.get(request.user)
databases = db.get_databases()
db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases)
response = render("show_tables.mako", request, {
'database': database,
'db_form': db_form,
})
response.set_cookie("hueHcatalogLastDatabase", database, expires=90)
return response
def create_database(request, source, destination):
database = destination['name']
comment = destination['description']
use_default_location = destination['useDefaultLocation']
external_path = destination['nonDefaultLocation']
sql = django_mako.render_to_string(
"gen/create_database_statement.mako", {
'database': {
'name': database,
'comment': comment,
'use_default_location': use_default_location,
'external_location': external_path,
'properties': [],
}
})
editor_type = 'hive'
on_success_url = reverse('metastore:show_tables',
kwargs={'database': database})
try:
notebook = make_notebook(name='Execute and watch',
editor_type=editor_type,
statement=sql,
status='ready',
on_success_url=on_success_url)
return notebook.execute(request, batch=False)
except Exception, e:
raise PopupException(_('The table could not be created.'),
detail=e.message)
def _get_query_history_latest(request_user=None,
query_id=None,
start_date=None,
start_time=None,
status=None,
limit=25,
force_refresh=False):
proposed_query = django_mako.render_to_string(
"select_table_query_data_latest.mako", {
'table': {
'name': 'query_data',
'request_user': request_user,
'query_id': query_id,
'start_date': start_date,
'start_time': start_time,
'status': status,
'limit': limit,
'force_refresh': force_refresh
}
})
data = _execute_query(proposed_query, limit)
for row in data['data']:
if row[1]:
row[1] = json.loads(row[1])
if row[5]:
row[5] = json.loads(row[5])
if row[8]:
row[8] = json.loads(row[8])
return data
def _get_query_history_from(request_user=None,
start_date=None,
start_time=None,
status=None,
query_id=None,
limit=25):
proposed_query = django_mako.render_to_string(
"select_table_query_data_from.mako", {
'table': {
'name': 'query_data',
'request_user': request_user,
'start_date': start_date,
'start_time': start_time,
'query_id': query_id,
'status': status,
'limit': limit
}
})
data = _execute_query(proposed_query, limit)
for row in data['data']:
if row[1]:
row[1] = [row[1]]
if row[5]:
row[5] = json.loads(row[5])
if row[8]:
row[8] = [row[8]]
return data
def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(table=beeswax.forms.CreateTableForm,
columns=beeswax.forms.ColumnTypeFormSet,
partitions=beeswax.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid():
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [f.cleaned_data for f in form.partitions.forms]
proposed_query = django_mako.render_to_string(
"create_table_statement.mako", {
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
})
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
tablename = form.table.cleaned_data['name']
on_success_url = urlresolvers.reverse(describe_table,
kwargs={'table': tablename})
return confirm_query(request, proposed_query, on_success_url)
else:
form.bind()
return render(
"create_table_manually.mako", request,
dict(action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(db_utils.meta_client().get_tables("default", ".*"))
> 0))
def commonheader(title, section, user, padding="90px"):
"""
Returns the rendered common header
"""
current_app = None
other_apps = []
if user.is_authenticated():
apps = appmanager.get_apps(user)
apps_list = appmanager.get_apps_dict(user)
for app in apps:
if app.display_name not in [
'beeswax', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser',
'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark']:
other_apps.append(app)
if section == app.display_name:
current_app = app
else:
apps_list = []
return django_mako.render_to_string("common_header.mako", {
'current_app': current_app,
'apps': apps_list,
'other_apps': other_apps,
'title': title,
'section': section,
'padding': padding,
'user': user,
'is_demo': desktop.conf.DEMO_ENABLED.get()
})
def create_database(request, source, destination):
database = destination['name']
comment = destination['description']
use_default_location = destination['useDefaultLocation']
external_path = destination['nonDefaultLocation']
sql = django_mako.render_to_string("gen/create_database_statement.mako", {
'database': {
'name': database,
'comment': comment,
'use_default_location': use_default_location,
'external_location': external_path,
'properties': [],
}
}
)
editor_type = 'hive'
on_success_url = reverse('metastore:show_tables', kwargs={'database': database})
try:
notebook = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready', on_success_url=on_success_url)
return notebook.execute(request, batch=False)
except Exception, e:
raise PopupException(_('The table could not be created.'), detail=e.message)
def commonheader(title, section, user, padding="90px"):
"""
Returns the rendered common header
"""
current_app = None
other_apps = []
if user.is_authenticated():
apps = appmanager.get_apps(user)
apps_list = appmanager.get_apps_dict(user)
for app in apps:
if app.display_name not in [
'beeswax', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser',
'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark']:
other_apps.append(app)
if section == app.display_name:
current_app = app
else:
apps_list = []
return django_mako.render_to_string("common_header.mako", {
'current_app': current_app,
'apps': apps_list,
'other_apps': other_apps,
'title': title,
'section': section,
'padding': padding,
'user': user
})
def show_tables(request, database=None):
if database is None:
database = _get_last_database(request, database)
if request.method == 'POST':
resp = {}
try:
tables = _get_table_list(request, database)
table_list_rendered = django_mako.render_to_string(
"table_list.mako",
dict(
app_name=get_app_name(request),
database=database,
tables=tables,
))
except Exception as ex:
resp['error'] = escapejs(ex.message)
else:
resp['table_list_rendered'] = table_list_rendered
resp['tables'] = tables
return HttpResponse(json.dumps(resp))
db = dbms.get(request.user)
databases = db.get_databases()
db_form = hcatalog.forms.DbForm(initial={'database': database},
databases=databases)
response = render("show_tables.mako", request, {
'database': database,
'db_form': db_form,
})
response.set_cookie("hueHcatalogLastDatabase", database, expires=90)
return response
def commonheader(title, section, user, request=None, padding="90px", skip_topbar=False, skip_idle_timeout=False, is_mobile=False):
"""
Returns the rendered common header
"""
current_app, other_apps, apps_list = _get_apps(user, section)
template = 'common_header.mako'
if is_mobile:
template = 'common_header_m.mako'
return django_mako.render_to_string(template, {
'current_app': current_app,
'apps': apps_list,
'other_apps': other_apps,
'title': title,
'section': section,
'padding': padding,
'user': user,
'request': request,
'skip_topbar': skip_topbar,
'skip_idle_timeout': skip_idle_timeout,
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get(),
'map_options': json.dumps(desktop.conf.LEAFLET_MAP_OPTIONS.get()),
'layer_options': json.dumps(desktop.conf.LEAFLET_TILE_LAYER_OPTIONS.get()),
},
'is_demo': desktop.conf.DEMO_ENABLED.get(),
'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(),
'is_s3_enabled': fsmanager.is_enabled('s3a') and fsmanager.has_access('s3a', request.user),
'is_adls_enabled': fsmanager.is_enabled('adl') and fsmanager.has_access('adl', request.user),
'banner_message': get_banner_message(request)
})
def get_configuration_statements(self):
configuration = []
for f in self.file_resources:
if not urlparse.urlsplit(f['path']).scheme:
scheme = get_hdfs().fs_defaultfs
else:
scheme = ''
configuration.append(
render_to_string(
"hql_resource.mako",
dict(type=f['type'], path=f['path'], scheme=scheme)))
for f in self.functions:
configuration.append(render_to_string("hql_function.mako", f))
return configuration
def create_table(request, database='default'):
"""Create a table by specifying its attributes manually"""
db = dbms.get(request.user)
dbs = db.get_databases()
databases = [{
'name':
db,
'url':
reverse('beeswax:create_table', kwargs={'database': db})
} for db in dbs]
form = MultiForm(table=CreateTableForm,
columns=ColumnTypeFormSet,
partitions=PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
form.table.db = db # curry is invalid
form.table.database = database
if request.POST.get('create'):
if form.is_valid():
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [
f.cleaned_data for f in form.partitions.forms
]
proposed_query = django_mako.render_to_string(
"create_table_statement.mako", {
'databases': databases,
'database': database,
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
})
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
table_name = form.table.cleaned_data['name']
return _submit_create_and_load(request,
proposed_query,
table_name,
None,
False,
database=database)
else:
form.bind()
apps_list = _get_apps(request.user, '')
return render(
"create_table_manually.mako", request, {
'apps': apps_list,
'action': "#",
'databases': databases,
'table_form': form.table,
'columns_form': form.columns,
'partitions_form': form.partitions,
'has_tables': len(dbms.get(request.user).get_tables()) > 0,
'database': database,
})
def commonheader(title, section, padding="60px"):
"""
Returns the rendered common header
"""
apps_list = sorted(appmanager.DESKTOP_APPS[:], key=lambda app: app.menu_index)
return django_mako.render_to_string(
"common_header.mako", dict(apps=apps_list, title=title, section=section, padding=padding)
)
def render_to_string(template, *args, **kwargs):
"""Wrapper around django.template.loader.render_to_string which supports
different template libraries."""
template_lib = _get_template_lib(template, kwargs)
if template_lib == DJANGO:
return django_render_to_string(template, *args, **kwargs)
elif template_lib == MAKO:
return django_mako.render_to_string(template, *args, **kwargs)
else:
raise Exception("Bad template lib: %s" % template_lib)
def render_to_string(template, *args, **kwargs):
"""Wrapper around django.template.loader.render_to_string which supports
different template libraries."""
template_lib = _get_template_lib(template, kwargs)
if template_lib == DJANGO:
return django_render_to_string(template, *args, **kwargs)
elif template_lib == MAKO:
return django_mako.render_to_string(template, *args, **kwargs)
else:
raise Exception("Bad template lib: %s" % template_lib)
def create_table(request, database=None):
"""Create a table by specifying its attributes manually"""
if database is None:
database = _get_last_database(request, database)
form = MultiForm(
table=hcatalog.forms.CreateTableForm,
columns=hcatalog.forms.ColumnTypeFormSet,
partitions=hcatalog.forms.PartitionTypeFormSet)
db = dbms.get(request.user)
databases = db.get_databases()
db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases)
error = None
if request.method == "POST":
form.bind(request.POST)
form.table.table_list = _get_table_list(request)
if form.is_valid() and 'createTable' in request.POST:
try:
columns = [f.cleaned_data for f in form.columns.forms]
column_names = [col["column_name"] for col in columns]
isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable(column_names)
if not isTableValid:
raise Exception(tableValidErrMsg)
partition_columns = [f.cleaned_data for f in form.partitions.forms]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
})
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
tablename = form.table.cleaned_data['name']
hcat_cli = HCatClient(request.user.username)
hcat_cli.create_table(database, tablename, proposed_query)
databases = hcat_cli.get_databases(like="*")
db_form = hcatalog.forms.DbForm(initial={'database': database}, databases=databases)
return render("show_tables.mako", request, {
'database': database,
'db_form': db_form,
})
except Exception as ex:
error = ex.message
else:
form.bind()
return render("create_table_manually.mako", request, dict(
database=database,
db_form=db_form,
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
error=error,
))
def commonheader(title, section, user, padding="60px"):
"""
Returns the rendered common header
"""
apps_list = sorted(appmanager.get_apps(user), key=lambda app: app.menu_index)
return django_mako.render_to_string("common_header.mako", dict(
apps=apps_list,
title=title,
section=section,
padding=padding,
user=user
))
def commonheader(title, section, user, padding="90px"):
"""
Returns the rendered common header
"""
current_app = None
other_apps = []
if user.is_authenticated():
apps = appmanager.get_apps(user)
apps_list = appmanager.get_apps_dict(user)
for app in apps:
if app.display_name not in [
"beeswax",
"impala",
"pig",
"jobsub",
"jobbrowser",
"metastore",
"hbase",
"sqoop",
"oozie",
"filebrowser",
"useradmin",
"search",
"help",
"about",
"zookeeper",
"proxy",
"rdbms",
"spark",
"indexer",
"security",
]:
other_apps.append(app)
if section == app.display_name:
current_app = app
else:
apps_list = []
return django_mako.render_to_string(
"common_header.mako",
{
"current_app": current_app,
"apps": apps_list,
"other_apps": other_apps,
"title": title,
"section": section,
"padding": padding,
"user": user,
"is_demo": desktop.conf.DEMO_ENABLED.get(),
},
)
def commonheader(title, section, user, padding="60px"):
"""
Returns the rendered common header
"""
apps = appmanager.get_apps(user)
apps_list = sorted(apps, key=lambda app: app.menu_index)
return django_mako.render_to_string("common_header.mako", {
'apps': apps_list,
'title': title,
'section': section,
'padding': padding,
'user': user
})
def commonheader(title, section, user, padding="60px"):
"""
Returns the rendered common header
"""
apps = appmanager.get_apps(user)
apps_list = sorted(apps, key=lambda app: app.menu_index)
return django_mako.render_to_string("common_header.mako", dict(
apps=apps_list,
title=title,
section=section,
padding=padding,
user=user
))
def commonfooter(messages=None):
"""
Returns the rendered common footer
"""
if messages is None:
messages = {}
hue_settings = Settings.get_settings()
return django_mako.render_to_string("common_footer.mako", {
'messages': messages,
'version': settings.HUE_DESKTOP_VERSION,
'collect_usage': desktop.conf.COLLECT_USAGE.get(),
'tours_and_tutorials': hue_settings.tours_and_tutorials
})
def commonfooter(messages=None):
"""
Returns the rendered common footer
"""
if messages is None:
messages = {}
hue_settings = Settings.get_settings()
return django_mako.render_to_string("common_footer.mako", {
'messages': messages,
'version': settings.HUE_DESKTOP_VERSION,
'collect_usage': collect_usage(),
'tours_and_tutorials': hue_settings.tours_and_tutorials
})
def show_databases(request):
if request.method == 'POST':
resp = {}
try:
databases = HCatClient(request.user.username).get_databases(like="*")
databases_list_rendered = django_mako.render_to_string("database_list.mako", dict(
app_name=get_app_name(request),
databases=databases))
except Exception as ex:
resp['error'] = escapejs(ex.message)
else:
resp['database_list_rendered'] = databases_list_rendered
resp['databases'] = databases
return HttpResponse(json.dumps(resp))
return render("show_databases.mako", request, {})
def _generate_workflow_xml(self, namenode):
"""Return a string that is the workflow.xml of this workflow"""
action_type = self._design_obj.root_action.action_type
data = {
'design': self._design_obj,
'nameNode': namenode,
}
if action_type == models.OozieStreamingAction.ACTION_TYPE:
tmpl = "workflow-streaming.xml.mako"
elif action_type == models.OozieMapreduceAction.ACTION_TYPE:
tmpl = "workflow-mapreduce.xml.mako"
elif action_type == models.OozieJavaAction.ACTION_TYPE:
tmpl = "workflow-java.xml.mako"
return django_mako.render_to_string(tmpl, data)
def _generate_workflow_xml(self, namenode):
"""Return a string that is the workflow.xml of this workflow"""
action_type = self._design_obj.root_action.action_type
data = {
'design': self._design_obj,
'nameNode': namenode,
}
if action_type == models.OozieStreamingAction.ACTION_TYPE:
tmpl = "workflow-streaming.xml.mako"
elif action_type == models.OozieMapreduceAction.ACTION_TYPE:
tmpl = "workflow-mapreduce.xml.mako"
elif action_type == models.OozieJavaAction.ACTION_TYPE:
tmpl = "workflow-java.xml.mako"
return django_mako.render_to_string(tmpl, data)
def commonfooter(request, messages=None):
"""
Returns the rendered common footer
"""
if messages is None:
messages = {}
hue_settings = Settings.get_settings()
return django_mako.render_to_string("common_footer.mako", {
'request': request,
'messages': messages,
'version': hue_version(),
'collect_usage': collect_usage(),
'tours_and_tutorials': hue_settings.tours_and_tutorials
})
def commonfooter(request, messages=None):
"""
Returns the rendered common footer
"""
if messages is None:
messages = {}
hue_settings = Settings.get_settings()
return django_mako.render_to_string("common_footer.mako", {
'request': request,
'messages': messages,
'version': hue_version(),
'collect_usage': collect_usage(),
'tours_and_tutorials': hue_settings.tours_and_tutorials
})
def show_databases(request):
if request.method == 'POST':
resp = {}
try:
databases = HCatClient(
request.user.username).get_databases(like="*")
databases_list_rendered = django_mako.render_to_string(
"database_list.mako",
dict(app_name=get_app_name(request), databases=databases))
except Exception as ex:
resp['error'] = escapejs(ex.message)
else:
resp['database_list_rendered'] = databases_list_rendered
resp['databases'] = databases
return HttpResponse(json.dumps(resp))
return render("show_databases.mako", request, {})
def create_table(request, database="default"):
"""Create a table by specifying its attributes manually"""
db = dbms.get(request.user)
dbs = db.get_databases()
databases = [{"name": db, "url": reverse("beeswax:create_table", kwargs={"database": db})} for db in dbs]
form = MultiForm(table=CreateTableForm, columns=ColumnTypeFormSet, partitions=PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
form.table.db = db # curry is invalid
form.table.database = database
if request.POST.get("create"):
if form.is_valid():
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [f.cleaned_data for f in form.partitions.forms]
proposed_query = django_mako.render_to_string(
"create_table_statement.mako",
{
"databases": databases,
"database": database,
"table": form.table.cleaned_data,
"columns": columns,
"partition_columns": partition_columns,
},
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode("utf-8")
table_name = form.table.cleaned_data["name"]
return _submit_create_and_load(request, proposed_query, table_name, None, False, database=database)
else:
form.bind()
return render(
"create_table_manually.mako",
request,
{
"action": "#",
"databases": databases,
"table_form": form.table,
"columns_form": form.columns,
"partitions_form": form.partitions,
"has_tables": len(dbms.get(request.user).get_tables()) > 0,
"database": database,
},
)
def create_table(request, database='default'):
"""Create a table by specifying its attributes manually"""
db = dbms.get(request.user)
dbs = db.get_databases()
databases = [{'name':db, 'url':reverse('beeswax:create_table', kwargs={'database': db})} for db in dbs]
form = MultiForm(
table=CreateTableForm,
columns=ColumnTypeFormSet,
partitions=PartitionTypeFormSet
)
if request.method == "POST":
form.bind(request.POST)
form.table.db = db # curry is invalid
form.table.database = database
if request.POST.get('create'):
if form.is_valid():
columns = [ f.cleaned_data for f in form.columns.forms ]
partition_columns = [ f.cleaned_data for f in form.partitions.forms ]
proposed_query = django_mako.render_to_string("create_table_statement.mako", {
'databases': databases,
'database': database,
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
table_name = form.table.cleaned_data['name']
return _submit_create_and_load(request, proposed_query, table_name, None, False, database=database)
else:
form.bind()
return render("create_table_manually.mako", request, {
'action': "#",
'databases': databases,
'table_form': form.table,
'columns_form': form.columns,
'partitions_form': form.partitions,
'has_tables': len(dbms.get(request.user).get_tables()) > 0,
'database': database,
})
def commonheader(title, section, user, padding="60px"):
"""
Returns the rendered common header
"""
if user.is_authenticated():
apps = appmanager.get_apps(user)
apps_list = sorted(apps, key=lambda app: app.menu_index)
else:
apps_list = []
return django_mako.render_to_string(
"common_header.mako", {
'apps': apps_list,
'title': title,
'section': section,
'padding': padding,
'user': user
})
def commonfooter(messages=None):
"""
Returns the rendered common footer
"""
if messages is None:
messages = {}
hue_settings = Settings.get_settings()
return django_mako.render_to_string(
"common_footer.mako",
{
"messages": messages,
"version": settings.HUE_DESKTOP_VERSION,
"collect_usage": collect_usage(),
"tours_and_tutorials": hue_settings.tours_and_tutorials,
},
)
def commonfooter(request, messages=None, is_mobile=False):
"""
Returns the rendered common footer
"""
if messages is None:
messages = {}
hue_settings = Settings.get_settings()
template = 'common_footer.mako'
if is_mobile:
template = 'common_footer_m.mako'
return django_mako.render_to_string(template, {
'request': request,
'messages': messages,
'version': hue_version(),
'collect_usage': collect_usage(),
})
def create_database(request):
if request.method == "POST":
data = request.POST.copy()
data.setdefault("use_default_location", False)
form = CreateDatabaseForm(data)
if form.is_valid():
proposed_query = django_mako.render_to_string("create_database_statement.mako", {
'database': form.cleaned_data,
})
query = hql_query(proposed_query)
return execute_directly(request, query, on_success_url=reverse('metastore:databases'))
else:
form = CreateDatabaseForm()
return render("create_database.mako", request, {
'database_form': form,
})
def commonfooter(request, messages=None, is_mobile=False):
"""
Returns the rendered common footer
"""
if messages is None:
messages = {}
hue_settings = Settings.get_settings()
template = 'common_footer.mako'
if is_mobile:
template = 'common_footer_m.mako'
return django_mako.render_to_string(template, {
'request': request,
'messages': messages,
'version': hue_version(),
'collect_usage': collect_usage(),
'tours_and_tutorials': hue_settings.tours_and_tutorials
})
def create_database(request):
if request.method == "POST":
data = request.POST.copy()
data.setdefault("use_default_location", False)
form = CreateDatabaseForm(data)
if form.is_valid():
proposed_query = django_mako.render_to_string("create_database_statement.mako", {
'database': form.cleaned_data,
})
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
query = hql_query(proposed_query)
return execute_directly(request, query, on_success_url=reverse('metastore:databases'))
else:
form = CreateDatabaseForm()
return render("create_database.mako", request, {
'database_form': form,
})
def create_database(request):
db = dbms.get(request.user)
if request.method == "POST":
data = request.POST.copy()
data.setdefault("use_default_location", False)
form = CreateDatabaseForm(data)
if form.is_valid():
proposed_query = django_mako.render_to_string("create_database_statement.mako", {
'database': form.cleaned_data,
})
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
query = hql_query(proposed_query)
return execute_directly(request, query, on_success_url=reverse('catalog:show_databases'))
else:
form = CreateDatabaseForm()
return render("create_database.mako", request, {
'database_form': form,
})
def create_table(request):
"""Create a table by specifying its attributes manually"""
db = dbms.get(request.user)
form = MultiForm(table=CreateTableForm,
columns=ColumnTypeFormSet,
partitions=PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
form.table.db = db # curry is invalid
if request.POST.get('create'):
if form.is_valid():
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [
f.cleaned_data for f in form.partitions.forms
]
proposed_query = django_mako.render_to_string(
"create_table_statement.mako", {
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
})
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
table_name = form.table.cleaned_data['name']
return _submit_create_and_load(request, proposed_query,
table_name, None, False)
else:
form.bind()
return render(
"create_table_manually.mako", request,
dict(action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(dbms.get(request.user).get_tables()) > 0))
def create_table(request):
"""Create a table by specifying its attributes manually"""
db = dbms.get(request.user)
form = MultiForm(
table=CreateTableForm,
columns=ColumnTypeFormSet,
partitions=PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
form.table.db = db # curry is invalid
if request.POST.get('create'):
if form.is_valid():
columns = [ f.cleaned_data for f in form.columns.forms ]
partition_columns = [ f.cleaned_data for f in form.partitions.forms ]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
table_name = form.table.cleaned_data['name']
return _submit_create_and_load(request, proposed_query, table_name, None, False)
else:
form.bind()
return render("create_table_manually.mako", request, dict(
action="#",
table_form=form.table,
columns_form=form.columns,
partitions_form=form.partitions,
has_tables=len(dbms.get(request.user).get_tables()) > 0
))
def create_table_from_a_file(self,
source,
destination,
start_time=-1,
file_encoding=None):
if '.' in destination['name']:
database, table_name = destination['name'].split('.', 1)
else:
database = 'default'
table_name = destination['name']
final_table_name = table_name
table_format = destination['tableFormat']
source_type = source['sourceType']
columns = destination['columns']
partition_columns = destination['partitionColumns']
kudu_partition_columns = destination['kuduPartitionColumns']
comment = destination['description']
source_path = urllib_unquote(source['path'])
load_data = destination['importData']
external = not destination['useDefaultLocation']
external_path = urllib_unquote(destination['nonDefaultLocation'])
editor_type = destination['sourceType']
is_transactional = destination['isTransactional']
default_transactional_type = 'insert_only' if destination[
'isInsertOnly'] else 'default'
skip_header = destination['hasHeader']
primary_keys = destination['primaryKeys']
if destination['useCustomDelimiters']:
field_delimiter = destination['customFieldDelimiter']
collection_delimiter = destination[
'customCollectionDelimiter'] or None
map_delimiter = destination['customMapDelimiter'] or None
else:
field_delimiter = ','
collection_delimiter = r'\002'
map_delimiter = r'\003'
regexp_delimiter = destination['customRegexp']
file_format = 'TextFile'
row_format = 'Delimited'
serde_name = ''
serde_properties = ''
extra_create_properties = ''
sql = ''
if source['inputFormat'] == 'manual':
load_data = False
source['format'] = {'quoteChar': '"', 'fieldSeparator': ','}
if table_format == 'json':
row_format = 'serde'
serde_name = 'org.apache.hive.hcatalog.data.JsonSerDe'
elif table_format == 'regexp':
row_format = 'serde'
serde_name = 'org.apache.hadoop.hive.serde2.RegexSerDe'
serde_properties = '"input.regex" = "%s"' % regexp_delimiter
elif table_format == 'csv':
if source['format']['quoteChar'] == '"':
source['format']['quoteChar'] = '\\"'
row_format = 'serde'
serde_name = 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
serde_properties = '''"separatorChar" = "%(fieldSeparator)s",
"quoteChar" = "%(quoteChar)s",
"escapeChar" = "\\\\"
''' % source['format']
use_temp_table = table_format in ('parquet', 'orc',
'kudu') or is_transactional
if use_temp_table: # We'll be using a temp table to load data
if load_data:
table_name, final_table_name = 'hue__tmp_%s' % table_name, table_name
sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;\n' % {
'database': database,
'table_name': table_name
}
else: # Manual
row_format = ''
file_format = table_format
skip_header = False
if table_format == 'kudu':
columns = [
col for col in columns if col['name'] in primary_keys
] + [
col
for col in columns if col['name'] not in primary_keys
]
if table_format == 'kudu':
collection_delimiter = None
map_delimiter = None
if external or (load_data and table_format in (
'parquet', 'orc', 'kudu')): # We'll use location to load data
if not self.fs.isdir(external_path): # File selected
external_path, external_file_name = Hdfs.split(external_path)
if len(self.fs.listdir(external_path)) > 1:
# If dir not just the file, create data dir and move file there. Make sure it's unique.
external_path = external_path + '/%s%s_table' % (
external_file_name, str(uuid.uuid4()))
self.fs.mkdir(external_path)
self.fs.rename(source_path, external_path)
elif load_data: # We'll use load data command
parent_path = self.fs.parent_path(source_path)
stats = self.fs.stats(parent_path)
split = urlparse(source_path)
# Only for HDFS, import data and non-external table
if split.scheme in ('', 'hdfs') and oct(stats["mode"])[-1] != '7':
user_scratch_dir = self.fs.get_home_dir(
) + '/.scratchdir/%s' % str(
uuid.uuid4()) # Make sure it's unique.
self.fs.do_as_user(self.user, self.fs.mkdir, user_scratch_dir,
0o0777)
self.fs.do_as_user(self.user, self.fs.rename, source['path'],
user_scratch_dir)
source_path = user_scratch_dir + '/' + source['path'].split(
'/')[-1]
if external_path.lower().startswith(
"abfs"): #this is to check if its using an ABFS path
external_path = abfspath(external_path)
tbl_properties = OrderedDict()
if skip_header:
tbl_properties['skip.header.line.count'] = '1'
# The temp table is not transactional, but final table can be if is_transactional.
# tbl_properties that don't exist in previous versions can safely be added without error.
tbl_properties['transactional'] = 'false'
sql += django_mako.render_to_string(
"gen/create_table_statement.mako", {
'table': {
'name':
table_name,
'comment':
comment,
'row_format':
row_format,
'field_terminator':
field_delimiter,
'collection_terminator':
collection_delimiter if source_type == 'hive' else None,
'map_key_terminator':
map_delimiter if source_type == 'hive' else None,
'serde_name':
serde_name,
'serde_properties':
serde_properties,
'file_format':
file_format,
'external':
external or load_data
and table_format in ('parquet', 'orc', 'kudu'),
'path':
external_path,
'primary_keys':
primary_keys
if table_format == 'kudu' and not load_data else [],
'tbl_properties':
tbl_properties
},
'columns': columns,
'partition_columns': partition_columns,
'kudu_partition_columns': kudu_partition_columns,
'database': database
})
if file_encoding and file_encoding != 'ASCII' and file_encoding != 'utf-8' and not use_temp_table:
sql += '\n\nALTER TABLE `%(database)s`.`%(final_table_name)s` ' \
'SET serdeproperties ("serialization.encoding"="%(file_encoding)s");' % {
'database': database,
'final_table_name': final_table_name,
'file_encoding': file_encoding
}
if table_format in ('text', 'json', 'csv',
'regexp') and not external and load_data:
form_data = {
'path':
source_path,
'overwrite':
False,
'partition_columns':
[(partition['name'], partition['partitionValue'])
for partition in partition_columns],
}
query_server_config = dbms.get_query_server_config(
name=source_type)
db = dbms.get(self.user, query_server=query_server_config)
sql += "\n\n%s;" % db.load_data(
database, table_name, form_data, None, generate_ddl_only=True)
if load_data and use_temp_table:
file_format = 'TextFile' if table_format == 'text' else table_format
if table_format == 'kudu':
columns_list = [
'`%s`' % col for col in primary_keys + [
col['name'] for col in destination['columns']
if col['name'] not in primary_keys and col['keep']
]
]
extra_create_properties = """PRIMARY KEY (%(primary_keys)s)
PARTITION BY HASH PARTITIONS 16
STORED AS %(file_format)s
TBLPROPERTIES(
'kudu.num_tablet_replicas' = '1'
)""" % {
'file_format': file_format,
'primary_keys': ', '.join(primary_keys)
}
else:
columns_list = ['*']
extra_create_properties = 'STORED AS %(file_format)s' % {
'file_format': file_format
}
if is_transactional:
extra_create_properties += '\nTBLPROPERTIES("transactional"="true", "transactional_properties"="%s")' % \
default_transactional_type
sql += '''\n\nCREATE TABLE `%(database)s`.`%(final_table_name)s`%(comment)s
%(extra_create_properties)s
AS SELECT %(columns_list)s
FROM `%(database)s`.`%(table_name)s`;''' % {
'database': database,
'final_table_name': final_table_name,
'table_name': table_name,
'extra_create_properties': extra_create_properties,
'columns_list': ', '.join(columns_list),
'comment': ' COMMENT "%s"' % comment if comment else ''
}
sql += '\n\nDROP TABLE IF EXISTS `%(database)s`.`%(table_name)s`;\n' % {
'database': database,
'table_name': table_name
}
if file_encoding and file_encoding != 'ASCII' and file_encoding != 'utf-8':
sql += '\n\nALTER TABLE `%(database)s`.`%(final_table_name)s` ' \
'SET serdeproperties ("serialization.encoding"="%(file_encoding)s");' % {
'database': database,
'final_table_name': final_table_name,
'file_encoding': file_encoding
}
on_success_url = reverse('metastore:describe_table',
kwargs={
'database': database,
'table': final_table_name
}) + '?source_type=' + source_type
return make_notebook(name=_('Creating table %(database)s.%(table)s') %
{
'database': database,
'table': final_table_name
},
editor_type=editor_type,
statement=sql.strip(),
status='ready',
database=database,
on_success_url=on_success_url,
last_executed=start_time,
is_task=True)
if request.GET.get('format') == 'json': # For Editor
return JsonResponse({
'coordinator':
coordinator.get_data_for_json(),
'credentials':
credentials.credentials.keys(),
'workflows':
workflows,
'doc_uuid':
doc.uuid if doc else '',
'can_edit':
doc is None or doc.doc.get().is_editable(request.user),
'layout':
django_mako.render_to_string(
'editor2/common_scheduler.mako',
{'coordinator_json': coordinator.to_json_for_html()})
})
else:
return render(
'editor2/coordinator_editor.mako', request, {
'coordinator_json':
coordinator.to_json_for_html(),
'credentials_json':
json.dumps(credentials.credentials.keys(),
cls=JSONEncoderForHTML),
'workflows_json':
json.dumps(workflows, cls=JSONEncoderForHTML),
'doc_uuid':
doc.uuid if doc else '',
'can_edit_json':
raise PopupException(_('You don\'t have access to the workflow of this coordinator.'))
if USE_NEW_EDITOR.get(): # In Hue 4, merge with above
workflows = [dict([('uuid', d.uuid), ('name', d.name)])
for d in Document2.objects.documents(request.user, include_managed=False).search_documents(types=['oozie-workflow2'])]
can_edit = doc is None or (doc.can_write(request.user) if USE_NEW_EDITOR.get() else doc.doc.get().is_editable(request.user))
if request.GET.get('format') == 'json': # For Editor
return JsonResponse({
'coordinator': coordinator.get_data_for_json(),
'credentials': credentials.credentials.keys(),
'workflows': workflows,
'doc_uuid': doc.uuid if doc else '',
'is_embeddable': request.GET.get('is_embeddable', False),
'can_edit': can_edit,
'layout': django_mako.render_to_string('editor2/common_scheduler.mako', {'coordinator_json': coordinator.to_json_for_html()})
})
else:
return render('editor2/coordinator_editor.mako', request, {
'coordinator_json': coordinator.to_json_for_html(),
'credentials_json': json.dumps(credentials.credentials.keys(), cls=JSONEncoderForHTML),
'workflows_json': json.dumps(workflows, cls=JSONEncoderForHTML),
'doc_uuid': doc.uuid if doc else '',
'is_embeddable': request.GET.get('is_embeddable', False),
'can_edit_json': json.dumps(can_edit)
})
@check_editor_access_permission
def new_coordinator(request):
return edit_coordinator(request)
def commonshare2():
return django_mako.render_to_string("common_share2.mako", {})
def import_wizard(request, database='default'):
"""
Help users define table and based on a file they want to import to Hive.
Limitations:
- Rows are delimited (no serde).
- No detection for map and array types.
- No detection for the presence of column header in the first row.
- No partition table.
- Does not work with binary data.
"""
encoding = i18n.get_site_encoding()
app_name = get_app_name(request)
db = dbms.get(request.user)
dbs = db.get_databases()
databases = [{'name':db, 'url':reverse('beeswax:import_wizard', kwargs={'database': db})} for db in dbs]
if request.method == 'POST':
#
# General processing logic:
# - We have 3 steps. Each requires the previous.
# * Step 1 : Table name and file location
# * Step 2a : Display sample with auto chosen delim
# * Step 2b : Display sample with user chosen delim (if user chooses one)
# * Step 3 : Display sample, and define columns
# - Each step is represented by a different form. The form of an earlier step
# should be present when submitting to a later step.
# - To preserve the data from the earlier steps, we send the forms back as
# hidden fields. This way, when users revisit a previous step, the data would
# be there as well.
#
delim_is_auto = False
fields_list, n_cols = [[]], 0
s3_col_formset = None
s1_file_form = CreateByImportFileForm(request.POST, db=db)
if s1_file_form.is_valid():
do_s2_auto_delim = request.POST.get('submit_file') # Step 1 -> 2
do_s2_user_delim = request.POST.get('submit_preview') # Step 2 -> 2
do_s3_column_def = request.POST.get('submit_delim') # Step 2 -> 3
do_hive_create = request.POST.get('submit_create') # Step 3 -> execute
cancel_s2_user_delim = request.POST.get('cancel_delim') # Step 2 -> 1
cancel_s3_column_def = request.POST.get('cancel_create') # Step 3 -> 2
# Exactly one of these should be True
if len(filter(None, (do_s2_auto_delim, do_s2_user_delim, do_s3_column_def, do_hive_create, cancel_s2_user_delim, cancel_s3_column_def))) != 1:
raise PopupException(_('Invalid form submission'))
if not do_s2_auto_delim:
# We should have a valid delim form
s2_delim_form = CreateByImportDelimForm(request.POST)
if not s2_delim_form.is_valid():
# Go back to picking delimiter
do_s2_user_delim, do_s3_column_def, do_hive_create = True, False, False
if do_hive_create:
# We should have a valid columns formset
s3_col_formset = ColumnTypeFormSet(prefix='cols', data=request.POST)
if not s3_col_formset.is_valid():
# Go back to define columns
do_s3_column_def, do_hive_create = True, False
#
# Go to step 2: We've just picked the file. Preview it.
#
if do_s2_auto_delim:
delim_is_auto = True
fields_list, n_cols, s2_delim_form = _delim_preview(request.fs, s1_file_form, encoding, [reader.TYPE for reader in FILE_READERS], DELIMITERS)
if (do_s2_user_delim or do_s3_column_def or cancel_s3_column_def) and s2_delim_form.is_valid():
# Delimit based on input
fields_list, n_cols, s2_delim_form = _delim_preview(request.fs, s1_file_form, encoding, (s2_delim_form.cleaned_data['file_type'],),
(s2_delim_form.cleaned_data['delimiter'],))
if do_s2_auto_delim or do_s2_user_delim or cancel_s3_column_def:
return render('choose_delimiter.mako', request, {
'action': reverse(app_name + ':import_wizard', kwargs={'database': database}),
'delim_readable': DELIMITER_READABLE.get(s2_delim_form['delimiter'].data[0], s2_delim_form['delimiter'].data[1]),
'initial': delim_is_auto,
'file_form': s1_file_form,
'delim_form': s2_delim_form,
'fields_list': fields_list,
'delimiter_choices': TERMINATOR_CHOICES,
'n_cols': n_cols,
'database': database,
'databases': databases
})
#
# Go to step 3: Define column.
#
if do_s3_column_def:
if s3_col_formset is None:
columns = []
for i in range(n_cols):
columns.append({
'column_name': 'col_%s' % (i,),
'column_type': 'string',
})
s3_col_formset = ColumnTypeFormSet(prefix='cols', initial=columns)
try:
fields_list_for_json = list(fields_list)
if fields_list_for_json:
fields_list_for_json[0] = map(lambda a: re.sub('[^\w]', '', a), fields_list_for_json[0]) # Cleaning headers
return render('define_columns.mako', request, {
'action': reverse(app_name + ':import_wizard', kwargs={'database': database}),
'file_form': s1_file_form,
'delim_form': s2_delim_form,
'column_formset': s3_col_formset,
'fields_list': fields_list,
'fields_list_json': json.dumps(fields_list_for_json),
'n_cols': n_cols,
'database': database,
'databases': databases
})
except Exception, e:
raise PopupException(_("The selected delimiter is creating an un-even number of columns. Please make sure you don't have empty columns."), detail=e)
#
# Final: Execute
#
if do_hive_create:
delim = s2_delim_form.cleaned_data['delimiter']
table_name = s1_file_form.cleaned_data['name']
proposed_query = django_mako.render_to_string("create_table_statement.mako", {
'table': {
'name': table_name,
'comment': s1_file_form.cleaned_data['comment'],
'row_format': 'Delimited',
'field_terminator': delim
},
'columns': [ f.cleaned_data for f in s3_col_formset.forms ],
'partition_columns': [],
'database': database,
'databases': databases
}
)
do_load_data = s1_file_form.cleaned_data.get('do_import')
path = s1_file_form.cleaned_data['path']
try:
return _submit_create_and_load(request, proposed_query, table_name, path, do_load_data, database=database)
except QueryServerException, e:
raise PopupException(_('The table could not be created.'), detail=e.message)
def commonimportexport(request):
return django_mako.render_to_string("common_import_export.mako",
{'request': request})
def create_from_file(request, database=None):
"""Create a table by import from file"""
if database is None:
database = _get_last_database(request)
form = MultiForm(table=hcatalog.forms.CreateTableFromFileForm, )
db = dbms.get(request.user)
databases = db.get_databases()
db_form = hcatalog.forms.DbForm(initial={'database': database},
databases=databases)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid():
parser_options = {}
# table options
table_name = form.table.cleaned_data['name']
replace_delimiter_with = form.table.cleaned_data[
'replace_delimiter_with']
parser_options['replace_delimiter_with'] = replace_delimiter_with
# common options
parser_options['path'] = form.table.cleaned_data['path']
file_type = request.POST.get('file_type',
hcatalog.forms.IMPORT_FILE_TYPE_NONE)
parser_options['file_type'] = file_type
parser_options['preview_start_idx'] = 0
parser_options['preview_end_idx'] = 0
# csv/tsv options
parser_options['do_import_data'] = form.table.cleaned_data[
'import_data']
parser_options['encoding'] = form.table.cleaned_data['encoding']
parser_options['autodetect_delimiter'] = form.table.cleaned_data[
'autodetect_delimiter']
parser_options['read_column_headers'] = form.table.cleaned_data[
'read_column_headers']
parser_options['apply_excel_dialect'] = True
parser_options['ignore_whitespaces'] = form.table.cleaned_data[
'ignore_whitespaces']
parser_options['ignore_tabs'] = form.table.cleaned_data[
'ignore_tabs']
parser_options['single_line_comment'] = form.table.cleaned_data[
'single_line_comment']
parser_options['java_style_comments'] = form.table.cleaned_data[
'java_style_comments']
# xls/xlsx options
parser_options['xls_sheet'] = form.table.cleaned_data['xls_sheet']
parser_options['xls_cell_range'] = form.table.cleaned_data[
'xls_cell_range']
parser_options[
'xls_read_column_headers'] = form.table.cleaned_data[
'xls_read_column_headers']
parser_options['delimiter'] = form.table.cleaned_data['delimiter']
if parser_options['autodetect_delimiter']:
parser_options['delimiters'] = DELIMITERS
else:
parser_options['delimiters'] = (parser_options['delimiter'], )
if parser_options['xls_read_column_headers'] and parser_options[
'preview_start_idx'] == 0:
parser_options['preview_start_idx'] = 1
is_preview_action = 'submitPreviewAction' in request.POST
is_preview_next = 'submitPreviewNext' in request.POST
is_preview_beginning = 'submitPreviewBeginning' in request.POST
if is_preview_action or is_preview_next or is_preview_beginning: # preview action
preview_results = {}
preview_table_resp = ''
fields_list, n_cols, col_names = ([], 0, [])
# validate input parameters
if file_type == hcatalog.forms.IMPORT_FILE_TYPE_NONE:
preview_results['error'] = unicode(
'Cannot define file type.')
return HttpResponse(json.dumps(preview_results))
if is_preview_next and 'preview_start_idx' in request.POST and 'preview_end_idx' in request.POST:
parser_options['preview_start_idx'] = int(
request.POST.get('preview_end_idx')) + 1
parser_options['preview_end_idx'] = int(
request.POST.get(
'preview_end_idx')) + IMPORT_PEEK_NLINES
else:
parser_options['preview_start_idx'] = 0
parser_options['preview_end_idx'] = IMPORT_PEEK_NLINES - 1
if parser_options['xls_read_column_headers']:
parser_options['preview_start_idx'] += 1
parser_options['preview_end_idx'] += 1
try:
parser_options = _on_table_preview(
request.fs,
[processor.TYPE
for processor in FILE_PROCESSORS], parser_options)
row_start_index = parser_options['preview_start_idx']
if parser_options['xls_read_column_headers']:
row_start_index -= 1
preview_table_resp = django_mako.render_to_string(
"file_import_preview_table.mako",
dict(fields_list=parser_options['fields_list'],
column_formset=parser_options['col_formset'],
row_start_index=row_start_index))
isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable(
parser_options['col_names'])
except Exception as ex:
preview_results['error'] = escapejs(ex.message)
else:
preview_results['results'] = preview_table_resp
if not isTableValid:
preview_results['error'] = escapejs(tableValidErrMsg)
options = {}
if file_type == hcatalog.forms.IMPORT_FILE_TYPE_TEXT:
options['delimiter_0'] = parser_options['delimiter_0']
options['delimiter_1'] = parser_options['delimiter_1']
options['file_processor_type'] = parser_options[
'file_processor_type']
elif file_type == hcatalog.forms.IMPORT_FILE_TYPE_SPREADSHEET:
options['xls_sheet'] = parser_options['xls_sheet']
options['xls_sheet_list'] = parser_options[
'xls_sheet_list']
options['preview_start_idx'] = parser_options[
'preview_start_idx']
options['preview_end_idx'] = parser_options[
'preview_end_idx']
options['preview_has_more'] = parser_options[
'preview_has_more']
preview_results['options'] = options
return HttpResponse(
json.dumps(preview_results, cls=JSONEncoderForHTML))
else: # create table action
# validate input parameters
if file_type == hcatalog.forms.IMPORT_FILE_TYPE_NONE:
err_msg = unicode('Cannot define file type.')
return render(
"create_table_from_file.mako", request,
dict(action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=err_msg))
# getting back file_processor_type from preview
parser_options['file_processor_type'] = request.POST.get(
'file_processor_type', None)
user_def_columns = []
user_def_column_names = []
column_count = 0
while 'cols-%d-column_name' % column_count in request.POST \
and 'cols-%d-column_type' % column_count in request.POST:
user_def_column_names.append(
request.POST.get('cols-%d-column_name' % column_count))
user_def_columns.append(
dict(
column_name=request.POST.get(
'cols-%d-column_name' % column_count),
column_type=request.POST.get(
'cols-%d-column_type' % column_count),
))
column_count += 1
try:
isTableValid, tableValidErrMsg = hcatalog.common.validateHiveTable(
user_def_column_names)
if not isTableValid:
return render(
"create_table_from_file.mako", request,
dict(action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=escapejs(tableValidErrMsg)))
LOG.debug('Creating table by hcatalog')
proposed_query = django_mako.render_to_string(
"create_table_statement.mako", {
'table':
dict(name=table_name,
comment=form.table.cleaned_data['comment'],
row_format='Delimited',
field_terminator=replace_delimiter_with),
'columns':
user_def_columns,
'partition_columns': []
})
proposed_query = proposed_query.decode('utf-8')
hcat_cli = HCatClient(request.user.username)
hcat_cli.create_table(database, table_name, proposed_query)
do_import_data = parser_options.get('do_import_data', True)
LOG.debug('Data processing stage')
if do_import_data:
parser_options = _on_table_create(
request.fs, parser_options)
path = parser_options.get('results_path', None)
if not path or not request.fs.exists(path):
msg = 'Missing needed result file to load data into table'
LOG.error(msg)
raise Exception(msg)
if not table_name:
msg = 'Internal error: Missing needed parameter to load data into table'
LOG.error(msg)
raise Exception(msg)
LOG.info("Auto loading data from %s into table %s%s" %
(path, database, table_name))
hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (
path, database, table_name)
job_id = hcat_cli.do_hive_query(execute=hql)
on_success_url = urlresolvers.reverse(
get_app_name(request) + ':index')
return render(
"create_table_from_file.mako", request,
dict(
action="#",
job_id=job_id,
on_success_url=on_success_url,
database=database,
db_form=db_form,
table_form=form.table,
error=None,
))
# clean up tmp dir
tmp_dir = parser_options.get('tmp_dir', None)
if tmp_dir and request.fs.exists:
request.fs.rmtree(tmp_dir)
databases = hcat_cli.get_databases(like="*")
db_form = hcatalog.forms.DbForm(
initial={'database': database}, databases=databases)
return render("show_tables.mako", request, {
'database': database,
'db_form': db_form,
})
except Exception as ex:
return render(
"create_table_from_file.mako", request,
dict(action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=escapejs(ex.message)))
return render(
"create_table_from_file.mako", request,
dict(
action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=
"User form is not valid. Please check all input parameters.",
))
else:
form.bind()
return render(
"create_table_from_file.mako", request,
dict(
action="#",
database=database,
db_form=db_form,
table_form=form.table,
error=None,
))
