def notebook(request, is_embeddable=False):
if not SHOW_NOTEBOOKS.get() or not request.user.has_hue_permission(
action="access", app='notebook'):
return serve_403_error(request)
notebook_id = request.GET.get('notebook', request.GET.get('editor'))
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception('Spark is not enabled')
return render(
'notebook.mako', request, {
'editor_id':
notebook_id or None,
'notebooks_json':
'{}',
'is_embeddable':
request.GET.get('is_embeddable', False),
'options_json':
json.dumps({
'languages': get_ordered_interpreters(request.user),
'session_properties': SparkApi.get_properties(),
'is_optimizer_enabled': has_optimizer(),
'is_wa_enabled': has_workload_analytics(),
'is_navigator_enabled': has_catalog(request.user),
'editor_type': 'notebook'
}),
'is_yarn_mode':
is_yarn_mode,
})
def editor(request):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
if editor_id: # Open existing saved editor document
editor = Notebook(document=Document2.objects.get(id=editor_id))
editor_type = editor.get_data()['type'].rsplit('-', 1)[-1]
editor = upgrade_session_properties(request, notebook=editor)
else: # Create new editor
editor = Notebook()
data = editor.get_data()
data['name'] = ''
data['type'] = 'query-%s' % editor_type # TODO: Add handling for non-SQL types
editor.data = json.dumps(data)
return render('editor.mako', request, {
'notebooks_json': json.dumps([editor.get_data()]),
'options_json': json.dumps({
'languages': [{"name": "%s SQL" % editor_type.title(), "type": editor_type}],
'mode': 'editor',
'is_optimizer_enabled': has_optimizer(),
}),
'editor_type': editor_type,
})
def editor(request, is_mobile=False, is_embeddable=False):
editor_id = request.GET.get("editor")
editor_type = request.GET.get("type", "hive")
if editor_id: # Open existing saved editor document
document = Document2.objects.get(id=editor_id)
editor_type = document.type.rsplit("-", 1)[-1]
template = "editor.mako"
if is_mobile:
template = "editor_m.mako"
if is_embeddable:
template = "editor_embeddable.mako"
return render(
template,
request,
{
"editor_id": editor_id or None,
"notebooks_json": "{}",
"editor_type": editor_type,
"options_json": json.dumps(
{
"languages": get_ordered_interpreters(request.user),
"mode": "editor",
"is_optimizer_enabled": has_optimizer(),
"is_navigator_enabled": has_navigator(request.user),
"editor_type": editor_type,
"mobile": is_mobile,
}
),
},
)
def editor(request, is_mobile=False, is_embeddable=False):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
if editor_type == 'notebook' or request.GET.get('notebook'):
return notebook(request)
if editor_id: # Open existing saved editor document
document = Document2.objects.get(id=editor_id)
editor_type = document.type.rsplit('-', 1)[-1]
template = 'editor.mako'
if is_mobile:
template = 'editor_m.mako'
return render(template, request, {
'editor_id': editor_id or None,
'notebooks_json': '{}',
'is_embeddable': request.GET.get('is_embeddable', False),
'editor_type': editor_type,
'options_json': json.dumps({
'languages': get_ordered_interpreters(request.user),
'mode': 'editor',
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'editor_type': editor_type,
'mobile': is_mobile
})
})
def editor(request, is_mobile=False, is_embeddable=False):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
if editor_type == 'notebook' or request.GET.get('notebook'):
return notebook(request)
if editor_id: # Open existing saved editor document
editor_type = _get_editor_type(editor_id)
template = 'editor.mako'
if is_mobile:
template = 'editor_m.mako'
return render(
template, request, {
'editor_id':
editor_id or None,
'notebooks_json':
'{}',
'is_embeddable':
request.GET.get('is_embeddable', False),
'editor_type':
editor_type,
'options_json':
json.dumps({
'languages': get_ordered_interpreters(request.user),
'mode': 'editor',
'is_optimizer_enabled': has_optimizer(),
'is_wa_enabled': has_workload_analytics(),
'is_navigator_enabled': has_catalog(request.user),
'editor_type': editor_type,
'mobile': is_mobile
})
})
def notebook(request, is_embeddable=False):
notebook_id = request.GET.get("notebook")
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception("Spark is not enabled")
template = "notebook.mako"
if is_embeddable:
template = "notebook_embeddable.mako"
return render(
template,
request,
{
"editor_id": notebook_id or None,
"notebooks_json": "{}",
"options_json": json.dumps(
{
"languages": get_ordered_interpreters(request.user),
"session_properties": SparkApi.get_properties(),
"is_optimizer_enabled": has_optimizer(),
"is_navigator_enabled": has_navigator(request.user),
"editor_type": "notebook",
}
),
"is_yarn_mode": is_yarn_mode,
},
)
def editor(request):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
if editor_id: # Open existing saved editor document
document = Document2.objects.get(id=editor_id)
editor_type = document.type.rsplit('-', 1)[-1]
return render(
'editor.mako', request, {
'editor_id':
editor_id or None,
'notebooks_json':
'{}',
'options_json':
json.dumps({
'languages': [{
"name": "%s SQL" % editor_type.title(),
"type": editor_type
}],
'mode':
'editor',
'is_optimizer_enabled':
has_optimizer(),
'is_navigator_enabled':
has_navigator(),
'editor_type':
editor_type
})
})
def notebook(request):
notebook_id = request.GET.get('notebook')
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception('Spark is not enabled')
return render(
'notebook.mako', request, {
'editor_id':
notebook_id or None,
'notebooks_json':
'{}',
'options_json':
json.dumps({
'languages': get_interpreters(request.user),
'session_properties': SparkApi.get_properties(),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(),
'editor_type': 'notebook'
}),
'is_yarn_mode':
is_yarn_mode,
})
def editor(request):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
if editor_id:
editor = Notebook(document=Document2.objects.get(id=editor_id))
editor_type = editor.get_data()['type'].rsplit('-', 1)[-1]
else:
editor = Notebook()
data = editor.get_data()
data['name'] = ''
data[
'type'] = 'query-%s' % editor_type # TODO: Add handling for non-SQL types
editor.data = json.dumps(data)
return render(
'editor.mako', request, {
'notebooks_json':
json.dumps([editor.get_data()]),
'options_json':
json.dumps({
'languages': [{
"name": "%s SQL" % editor_type.title(),
"type": editor_type
}],
'mode':
'editor',
'is_optimizer_enabled':
has_optimizer(),
}),
'editor_type':
editor_type,
})
def notebook(request):
notebook_id = request.GET.get('notebook')
if notebook_id:
notebook = Notebook(document=Document2.objects.get(id=notebook_id))
else:
notebook = Notebook()
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception('Spark is not enabled')
return render(
'notebook.mako', request, {
'notebooks_json':
json.dumps([notebook.get_data()]),
'options_json':
json.dumps({
'languages': get_interpreters(request.user),
'session_properties': SparkApi.PROPERTIES,
'is_optimizer_enabled': has_optimizer(),
}),
'is_yarn_mode':
is_yarn_mode,
})
def show_tables(request, database=None):
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, cluster=cluster)
if database is None:
database = 'default' # Assume always 'default'
if request.GET.get("format", "html") == "json":
try:
databases = db.get_databases()
if database not in databases:
database = 'default'
if request.method == 'POST':
db_form = DbForm(request.POST, databases=databases)
if db_form.is_valid():
database = db_form.cleaned_data['database']
else:
db_form = DbForm(initial={'database': database},
databases=databases)
search_filter = request.GET.get('filter', '')
tables = db.get_tables_meta(
database=database,
table_names=search_filter) # SparkSql returns []
table_names = [table['name'] for table in tables]
except Exception as e:
raise PopupException(_(
'Failed to retrieve tables for database: %s' % database),
detail=e)
resp = JsonResponse({
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
apps_list = _get_apps(request.user, '')
resp = render(
"metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
return resp
def describe_partitions(request, database, table):
db = _get_db(user=request.user)
table_obj = db.get_table(database, table)
if not table_obj.partition_keys:
raise PopupException(_("Table '%(table)s' is not partitioned.") % {'table': table})
reverse_sort = request.GET.get("sort", "desc").lower() == "desc"
if request.method == "POST":
partition_filters = {}
for part in table_obj.partition_keys:
if request.GET.get(part.name):
partition_filters[part.name] = request.GET.get(part.name)
partition_spec = ','.join(["%s='%s'" % (k, v) for k, v in partition_filters.items()])
else:
partition_spec = ''
try:
partitions = db.get_partitions(database, table_obj, partition_spec, reverse_sort=reverse_sort)
except:
LOG.exception('Table partitions could not be retrieved')
partitions = []
massaged_partitions = [_massage_partition(database, table_obj, partition) for partition in partitions]
if request.method == "POST" or request.GET.get('format', 'html') == 'json':
return JsonResponse({
'partition_keys_json': [partition.name for partition in table_obj.partition_keys],
'partition_values_json': massaged_partitions,
})
else:
return render("metastore.mako", request, {
'breadcrumbs': [{
'name': database,
'url': reverse('metastore:show_tables', kwargs={'database': database})
}, {
'name': table,
'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
},{
'name': 'partitions',
'url': reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
},
],
'database': database,
'table': table_obj,
'partitions': partitions,
'partition_keys_json': json.dumps([partition.name for partition in table_obj.partition_keys]),
'partition_values_json': json.dumps(massaged_partitions),
'request': request,
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def describe_table(request, database, table):
app_name = get_app_name(request)
cluster = json.loads(request.POST.get('cluster', '{}'))
source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
try:
table = db.get_table(database, table)
except Exception as e:
LOG.exception("Describe table error")
raise PopupException(_("DB Error"), detail=e.message if hasattr(e, 'message') and e.message else e)
if request.POST.get("format", "html") == "json":
return JsonResponse({
'status': 0,
'name': table.name,
'partition_keys': [{'name': part.name, 'type': part.type} for part in table.partition_keys],
'cols': [{'name': col.name, 'type': col.type, 'comment': col.comment} for col in table.cols],
'path_location': table.path_location,
'hdfs_link': table.hdfs_link,
'comment': table.comment,
'is_view': table.is_view,
'properties': table.properties,
'details': table.details,
'stats': table.stats
})
else: # Render HTML
renderable = "metastore.mako"
apps_list = _get_apps(request.user, '')
partitions = None
if app_name != 'impala' and table.partition_keys:
try:
partitions = [_massage_partition(database, table, partition) for partition in db.get_partitions(database, table)]
except:
LOG.exception('Table partitions could not be retrieved')
return render(renderable, request, {
'apps': apps_list,
'breadcrumbs': [{
'name': database,
'url': reverse('metastore:show_tables', kwargs={'database': database})
}, {
'name': str(table.name),
'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table.name})
},
],
'table': table,
'partitions': partitions,
'database': database,
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def describe_partitions(request, database, table):
db = dbms.get(request.user)
table_obj = db.get_table(database, table)
if not table_obj.partition_keys:
raise PopupException(_("Table '%(table)s' is not partitioned.") % {'table': table})
reverse_sort = request.REQUEST.get("sort", "desc").lower() == "desc"
if request.method == "POST":
partition_filters = {}
for part in table_obj.partition_keys:
if request.REQUEST.get(part.name):
partition_filters[part.name] = request.REQUEST.get(part.name)
partition_spec = ','.join(["%s='%s'" % (k, v) for k, v in partition_filters.items()])
else:
partition_spec = ''
try:
partitions = db.get_partitions(database, table_obj, partition_spec, reverse_sort=reverse_sort)
except:
LOG.exception('Table partitions could not be retrieved')
partitions = []
massaged_partitions = [_massage_partition(database, table_obj, partition) for partition in partitions]
if request.method == "POST" or request.GET.get('format', 'html') == 'json':
return JsonResponse({
'partition_keys_json': [partition.name for partition in table_obj.partition_keys],
'partition_values_json': massaged_partitions,
})
else:
return render("describe_partitions.mako", request, {
'breadcrumbs': [{
'name': database,
'url': reverse('metastore:show_tables', kwargs={'database': database})
}, {
'name': table,
'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
},{
'name': 'partitions',
'url': reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
},
],
'database': database,
'table': table_obj,
'partitions': partitions,
'partition_keys_json': json.dumps([partition.name for partition in table_obj.partition_keys]),
'partition_values_json': json.dumps(massaged_partitions),
'request': request,
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
})
def setup_class(cls):
if not has_optimizer():
raise SkipTest
cls.client = make_logged_in_client(username='******', is_superuser=False)
cls.user = User.objects.get(username='******')
cls.user = rewrite_user(cls.user)
add_to_group('test')
grant_access("test", "test", "metadata")
grant_access("test", "test", "optimizer")
cls.api = OptimizerApi()
def databases(request):
search_filter = request.GET.get('filter', '')
db = dbms.get(request.user)
databases = db.get_databases(search_filter)
return render("metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
})
def editor(request, is_mobile=False, is_embeddable=False):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
gist_id = request.GET.get('gist')
if editor_type == 'notebook' or request.GET.get('notebook'):
return notebook(request)
if editor_type == 'gist':
gist_doc = _get_gist_document(uuid=gist_id)
editor_type = gist_doc.extra
if EXAMPLES.AUTO_OPEN.get() and not editor_id:
sample_query = _get_dialect_example(dialect=editor_type)
if sample_query:
editor_id = sample_query.id
if editor_id and not gist_id: # Open existing saved editor document
editor_type = _get_editor_type(editor_id)
template = 'editor.mako'
if ENABLE_NOTEBOOK_2.get():
template = 'editor2.mako'
elif is_mobile:
template = 'editor_m.mako'
return render(
template, request, {
'editor_id':
editor_id or None,
'notebooks_json':
'{}',
'is_embeddable':
request.GET.get('is_embeddable', False),
'editor_type':
editor_type,
'options_json':
json.dumps({
'languages': get_ordered_interpreters(request.user),
'mode': 'editor',
'is_optimizer_enabled': has_optimizer(),
'is_wa_enabled': has_workload_analytics(),
'is_navigator_enabled': has_catalog(request.user),
'editor_type': editor_type,
'mobile': is_mobile
})
})
def databases(request):
search_filter = request.GET.get('filter', '')
db = dbms.get(request.user)
databases = db.get_databases(search_filter)
return render(
"metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
})
def editor(request):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
if editor_id: # Open existing saved editor document
document = Document2.objects.get(id=editor_id)
editor_type = document.type.rsplit('-', 1)[-1]
return render('editor.mako', request, {
'editor_id': editor_id or None,
'notebooks_json': '{}',
'options_json': json.dumps({
'languages': [{"name": "%s SQL" % editor_type.title(), "type": editor_type}],
'mode': 'editor',
'is_optimizer_enabled': has_optimizer(),
'editor_type': editor_type
})
})
def databases(request):
search_filter = request.GET.get('filter', '')
db = _get_db(user=request.user)
databases = db.get_databases(search_filter)
return render("metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def notebook(request):
notebook_id = request.GET.get('notebook')
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception('Spark is not enabled')
return render('notebook.mako', request, {
'editor_id': notebook_id or None,
'notebooks_json': '{}',
'options_json': json.dumps({
'languages': get_interpreters(request.user),
'session_properties': SparkApi.get_properties(),
'is_optimizer_enabled': has_optimizer(),
}),
'is_yarn_mode': is_yarn_mode,
})
def databases(request):
search_filter = request.GET.get('filter', '')
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, cluster=cluster)
databases = db.get_databases(search_filter)
apps_list = _get_apps(request.user, '')
return render("metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def databases(request):
search_filter = request.GET.get('filter', '')
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, cluster=cluster)
databases = db.get_databases(search_filter)
apps_list = _get_apps(request.user, '')
return render("metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def notebook(request):
notebook_id = request.GET.get('notebook')
if notebook_id:
notebook = Notebook(document=Document2.objects.get(id=notebook_id))
else:
notebook = Notebook()
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception('Spark is not enabled')
return render('notebook.mako', request, {
'notebooks_json': json.dumps([notebook.get_data()]),
'options_json': json.dumps({
'languages': get_interpreters(request.user),
'session_properties': SparkApi.get_properties(),
'is_optimizer_enabled': has_optimizer(),
}),
'is_yarn_mode': is_yarn_mode,
})
resp = JsonResponse({
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
resp = render(
"metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
'is_embeddable': request.REQUEST.get('is_embeddable', False),
})
return resp
def get_table_metadata(request, database, table):
db = dbms.get(request.user)
response = {'status': -1, 'data': ''}
try:
table_metadata = db.get_table(database, table)
response['status'] = 0
raise PopupException(_('Failed to retrieve tables for database: %s' % database), detail=e)
resp = JsonResponse({
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
resp = render("metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
'is_embeddable': request.REQUEST.get('is_embeddable', False),
})
return resp
def get_table_metadata(request, database, table):
db = dbms.get(request.user)
response = {'status': -1, 'data': ''}
try:
table_metadata = db.get_table(database, table)
response['status'] = 0