def search_entities_interactive(request):
sources = json.loads(request.POST.get('sources')) or []
if 'documents' in sources:
search_text = json.loads(request.POST.get('query_s', ''))
limit = int(request.POST.get('limit', 25))
entities = _search(user=request.user, search_text=search_text, limit=limit)
response = {
'results': [{
'hue_name': _highlight(search_text, escape(e.name)),
'hue_description': _highlight(search_text, escape(e.description)),
'link': e.get_absolute_url(),
'doc_type': escape(e.type),
'type': 'HUE',
'uuid': e.uuid,
'parentUuid': e.parent_directory.uuid,
'originalName': escape(e.name)
} for e in entities['documents']
],
'count': len(entities['documents']),
'status': 0
}
return JsonResponse(response)
else:
if has_navigator(request.user):
return metadata_search_entities_interactive(request)
else:
return JsonResponse({'status': 1, 'message': _('Navigator not enabled')})
def editor(request):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
if editor_id: # Open existing saved editor document
document = Document2.objects.get(id=editor_id)
editor_type = document.type.rsplit('-', 1)[-1]
return render(
'editor.mako', request, {
'editor_id':
editor_id or None,
'notebooks_json':
'{}',
'options_json':
json.dumps({
'languages': [{
"name": "%s SQL" % editor_type.title(),
"type": editor_type
}],
'mode':
'editor',
'is_optimizer_enabled':
has_optimizer(),
'is_navigator_enabled':
has_navigator(),
'editor_type':
editor_type
})
})
def editor(request, is_mobile=False, is_embeddable=False):
editor_id = request.GET.get("editor")
editor_type = request.GET.get("type", "hive")
if editor_id: # Open existing saved editor document
document = Document2.objects.get(id=editor_id)
editor_type = document.type.rsplit("-", 1)[-1]
template = "editor.mako"
if is_mobile:
template = "editor_m.mako"
if is_embeddable:
template = "editor_embeddable.mako"
return render(
template,
request,
{
"editor_id": editor_id or None,
"notebooks_json": "{}",
"editor_type": editor_type,
"options_json": json.dumps(
{
"languages": get_ordered_interpreters(request.user),
"mode": "editor",
"is_optimizer_enabled": has_optimizer(),
"is_navigator_enabled": has_navigator(request.user),
"editor_type": editor_type,
"mobile": is_mobile,
}
),
},
)
def notebook(request, is_embeddable=False):
if not SHOW_NOTEBOOKS.get() or not request.user.has_hue_permission(
action="access", app='notebook'):
return serve_403_error(request)
notebook_id = request.GET.get('notebook', request.GET.get('editor'))
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception('Spark is not enabled')
return render(
'notebook.mako', request, {
'editor_id':
notebook_id or None,
'notebooks_json':
'{}',
'is_embeddable':
request.GET.get('is_embeddable', False),
'options_json':
json.dumps({
'languages': get_ordered_interpreters(request.user),
'session_properties': SparkApi.get_properties(),
'is_optimizer_enabled': has_optimizer(),
'is_wa_enabled': has_workload_analytics(),
'is_navigator_enabled': has_navigator(request.user),
'editor_type': 'notebook'
}),
'is_yarn_mode':
is_yarn_mode,
})
def notebook(request, is_embeddable=False):
notebook_id = request.GET.get("notebook")
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception("Spark is not enabled")
template = "notebook.mako"
if is_embeddable:
template = "notebook_embeddable.mako"
return render(
template,
request,
{
"editor_id": notebook_id or None,
"notebooks_json": "{}",
"options_json": json.dumps(
{
"languages": get_ordered_interpreters(request.user),
"session_properties": SparkApi.get_properties(),
"is_optimizer_enabled": has_optimizer(),
"is_navigator_enabled": has_navigator(request.user),
"editor_type": "notebook",
}
),
"is_yarn_mode": is_yarn_mode,
},
)
def editor(request, is_mobile=False, is_embeddable=False):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
if editor_type == 'notebook' or request.GET.get('notebook'):
return notebook(request)
if editor_id: # Open existing saved editor document
document = Document2.objects.get(id=editor_id)
editor_type = document.type.rsplit('-', 1)[-1]
template = 'editor.mako'
if is_mobile:
template = 'editor_m.mako'
return render(template, request, {
'editor_id': editor_id or None,
'notebooks_json': '{}',
'is_embeddable': request.GET.get('is_embeddable', False),
'editor_type': editor_type,
'options_json': json.dumps({
'languages': get_ordered_interpreters(request.user),
'mode': 'editor',
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'editor_type': editor_type,
'mobile': is_mobile
})
})
def notebook(request):
notebook_id = request.GET.get('notebook')
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception('Spark is not enabled')
return render(
'notebook.mako', request, {
'editor_id':
notebook_id or None,
'notebooks_json':
'{}',
'options_json':
json.dumps({
'languages': get_interpreters(request.user),
'session_properties': SparkApi.get_properties(),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(),
'editor_type': 'notebook'
}),
'is_yarn_mode':
is_yarn_mode,
})
def search_entities_interactive(request):
sources = json.loads(request.POST.get('sources')) or []
if 'documents' in sources:
search_text = json.loads(request.POST.get('query_s', ''))
limit = int(request.POST.get('limit', 25))
entities = _search(user=request.user,
search_text=search_text,
limit=limit)
response = {
'results': [{
'hue_name': e.name,
'hue_description': e.description,
'type': 'HUE',
'originalName': e.name
} for e in entities['documents']],
'count':
len(entities['documents']),
'status':
0
}
return JsonResponse(response)
else:
if has_navigator(request.user) and False:
return metadata_search_entities_interactive(request)
else:
return JsonResponse({
'status': 1,
'message': _('Navigator not enabled')
})
def search_entities(request):
sources = json.loads(request.POST.get('sources')) or []
if 'documents' in sources:
search_text = json.loads(request.POST.get('query_s', ''))
entities = _search(user=request.user, search_text=search_text)
response = {
'entities': [{
'hue_name': e.name,
'hue_description': e.description,
'type': 'HUE',
'originalName': e.name,
'link': '/home?uuid=%s' % e.uuid
} for e in entities['documents']],
'count':
len(entities['documents']),
'status':
0
}
return JsonResponse(response)
else:
if has_navigator(request.user):
return metadata_search_entities(request)
else:
return JsonResponse({
'status': 1,
'message': _('Navigator not enabled')
})
def decorator(*args, **kwargs):
try:
if has_navigator(args[0].user):
return view_fn(*args, **kwargs)
else:
raise MetadataApiException('Navigator API is not configured.')
except Http404, e:
raise e
def decorator(*args, **kwargs):
try:
if has_navigator():
return view_fn(*args, **kwargs)
else:
raise MetadataApiException('Navigator API is not configured.')
except Http404, e:
raise e
def describe_partitions(request, database, table):
db = _get_db(user=request.user)
table_obj = db.get_table(database, table)
if not table_obj.partition_keys:
raise PopupException(_("Table '%(table)s' is not partitioned.") % {'table': table})
reverse_sort = request.GET.get("sort", "desc").lower() == "desc"
if request.method == "POST":
partition_filters = {}
for part in table_obj.partition_keys:
if request.GET.get(part.name):
partition_filters[part.name] = request.GET.get(part.name)
partition_spec = ','.join(["%s='%s'" % (k, v) for k, v in partition_filters.items()])
else:
partition_spec = ''
try:
partitions = db.get_partitions(database, table_obj, partition_spec, reverse_sort=reverse_sort)
except:
LOG.exception('Table partitions could not be retrieved')
partitions = []
massaged_partitions = [_massage_partition(database, table_obj, partition) for partition in partitions]
if request.method == "POST" or request.GET.get('format', 'html') == 'json':
return JsonResponse({
'partition_keys_json': [partition.name for partition in table_obj.partition_keys],
'partition_values_json': massaged_partitions,
})
else:
return render("metastore.mako", request, {
'breadcrumbs': [{
'name': database,
'url': reverse('metastore:show_tables', kwargs={'database': database})
}, {
'name': table,
'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
},{
'name': 'partitions',
'url': reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
},
],
'database': database,
'table': table_obj,
'partitions': partitions,
'partition_keys_json': json.dumps([partition.name for partition in table_obj.partition_keys]),
'partition_values_json': json.dumps(massaged_partitions),
'request': request,
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def setup_class(cls):
cls.client = make_logged_in_client(username='******', is_superuser=False)
cls.user = User.objects.get(username='******')
cls.user = rewrite_user(cls.user)
add_to_group('test')
grant_access("test", "test", "metadata")
if not is_live_cluster() or not has_navigator(cls.user):
raise SkipTest
def setup_class(cls):
cls.client = make_logged_in_client(username='******', is_superuser=False)
cls.user = User.objects.get(username='******')
cls.user = rewrite_user(cls.user)
add_to_group('test')
grant_access("test", "test", "metadata")
if not is_live_cluster() or not has_navigator(cls.user):
raise SkipTest
def describe_partitions(request, database, table):
db = dbms.get(request.user)
table_obj = db.get_table(database, table)
if not table_obj.partition_keys:
raise PopupException(_("Table '%(table)s' is not partitioned.") % {'table': table})
reverse_sort = request.REQUEST.get("sort", "desc").lower() == "desc"
if request.method == "POST":
partition_filters = {}
for part in table_obj.partition_keys:
if request.REQUEST.get(part.name):
partition_filters[part.name] = request.REQUEST.get(part.name)
partition_spec = ','.join(["%s='%s'" % (k, v) for k, v in partition_filters.items()])
else:
partition_spec = ''
try:
partitions = db.get_partitions(database, table_obj, partition_spec, reverse_sort=reverse_sort)
except:
LOG.exception('Table partitions could not be retrieved')
partitions = []
massaged_partitions = [_massage_partition(database, table_obj, partition) for partition in partitions]
if request.method == "POST" or request.GET.get('format', 'html') == 'json':
return JsonResponse({
'partition_keys_json': [partition.name for partition in table_obj.partition_keys],
'partition_values_json': massaged_partitions,
})
else:
return render("describe_partitions.mako", request, {
'breadcrumbs': [{
'name': database,
'url': reverse('metastore:show_tables', kwargs={'database': database})
}, {
'name': table,
'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
},{
'name': 'partitions',
'url': reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
},
],
'database': database,
'table': table_obj,
'partitions': partitions,
'partition_keys_json': json.dumps([partition.name for partition in table_obj.partition_keys]),
'partition_values_json': json.dumps(massaged_partitions),
'request': request,
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
})
def decorator(*args, **kwargs):
status = 500
response = {'message': ''}
try:
if has_navigator(args[0].user):
return view_fn(*args, **kwargs)
else:
raise MetadataApiException('Navigator API is not configured.')
except Http404, e:
raise e
def decorator(*args, **kwargs):
status = 500
response = {'message': ''}
try:
if has_navigator(args[0].user): # TODO
return view_fn(*args, **kwargs)
else:
raise CatalogApiException('Navigator API is not configured.')
except CatalogApiException, e:
try:
response['message'] = json.loads(e.message)
except Exception:
response['message'] = force_unicode(e.message)
def databases(request):
search_filter = request.GET.get('filter', '')
db = dbms.get(request.user)
databases = db.get_databases(search_filter)
return render("metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
})
def databases(request):
search_filter = request.GET.get('filter', '')
db = dbms.get(request.user)
databases = db.get_databases(search_filter)
return render("metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
})
def databases(request):
search_filter = request.GET.get('filter', '')
db = _get_db(user=request.user)
databases = db.get_databases(search_filter)
return render("metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def editor(request):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
if editor_id: # Open existing saved editor document
document = Document2.objects.get(id=editor_id)
editor_type = document.type.rsplit('-', 1)[-1]
return render('editor.mako', request, {
'editor_id': editor_id or None,
'notebooks_json': '{}',
'options_json': json.dumps({
'languages': [{"name": "%s SQL" % editor_type.title(), "type": editor_type}],
'mode': 'editor',
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(),
'editor_type': editor_type
})
})
def notebook(request):
notebook_id = request.GET.get('notebook')
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception('Spark is not enabled')
return render('notebook.mako', request, {
'editor_id': notebook_id or None,
'notebooks_json': '{}',
'options_json': json.dumps({
'languages': get_interpreters(request.user),
'session_properties': SparkApi.get_properties(),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(),
'editor_type': 'notebook'
}),
'is_yarn_mode': is_yarn_mode,
})
def setup_class(cls):
if not is_live_cluster() or not has_navigator(): # Until v1
raise SkipTest
resp = JsonResponse({
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
resp = render(
"metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
'is_embeddable': request.REQUEST.get('is_embeddable', False),
})
return resp
def get_table_metadata(request, database, table):
db = dbms.get(request.user)
response = {'status': -1, 'data': ''}
try:
table_metadata = db.get_table(database, table)
response['status'] = 0
response['data'] = {
resp = JsonResponse({
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
resp = render("metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
})
return resp
def get_table_metadata(request, database, table):
db = dbms.get(request.user)
response = {'status': -1, 'data': ''}
try:
table_metadata = db.get_table(database, table)
response['status'] = 0
response['data'] = {
'comment': table_metadata.comment,
resp = JsonResponse({
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
resp = render(
"metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
})
return resp
def get_table_metadata(request, database, table):
db = dbms.get(request.user)
response = {'status': -1, 'data': ''}
try:
table_metadata = db.get_table(database, table)
response['status'] = 0
response['data'] = {
'comment': table_metadata.comment,
resp = JsonResponse({
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
resp = render("metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
'is_embeddable': request.REQUEST.get('is_embeddable', False),
})
return resp
def get_table_metadata(request, database, table):
db = dbms.get(request.user)
response = {'status': -1, 'data': ''}
try:
table_metadata = db.get_table(database, table)
response['status'] = 0
response['data'] = {
