def notebook(request, is_embeddable=False):
if not SHOW_NOTEBOOKS.get() or not request.user.has_hue_permission(
action="access", app='notebook'):
return serve_403_error(request)
notebook_id = request.GET.get('notebook', request.GET.get('editor'))
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception('Spark is not enabled')
return render(
'notebook.mako', request, {
'editor_id':
notebook_id or None,
'notebooks_json':
'{}',
'is_embeddable':
request.GET.get('is_embeddable', False),
'options_json':
json.dumps({
'languages': get_ordered_interpreters(request.user),
'session_properties': SparkApi.get_properties(),
'is_optimizer_enabled': has_optimizer(),
'is_wa_enabled': has_workload_analytics(),
'is_navigator_enabled': has_catalog(request.user),
'editor_type': 'notebook'
}),
'is_yarn_mode':
is_yarn_mode,
})
def decorator(*args, **kwargs):
status = 500
response = {
'message': ''
}
try:
if has_catalog(args[0].user):
return view_fn(*args, **kwargs)
else:
raise MetadataApiException('Catalog API is not configured.')
except Http404 as e:
raise e
except CatalogEntityDoesNotExistException as e:
response['message'] = e.message
status = 404
except CatalogAuthException as e:
response['message'] = force_unicode(e.message)
status = 403
except CatalogApiException as e:
try:
response['message'] = json.loads(e.message)
except Exception:
response['message'] = force_unicode(e.message)
except Exception as e:
message = force_unicode(e)
response['message'] = message
LOG.exception(message)
return JsonResponse(response, status=status)
def search_entities_interactive(request):
sources = json.loads(request.POST.get('sources')) or []
if 'documents' in sources:
search_text = json.loads(request.POST.get('query_s', ''))
limit = int(request.POST.get('limit', 25))
entities = _search(user=request.user, search_text=search_text, limit=limit)
response = {
'results': [{
'hue_name': _highlight(search_text, escape(e.name)),
'hue_description': _highlight(search_text, escape(e.description)),
'link': e.get_absolute_url(),
'doc_type': escape(e.type),
'type': 'HUE',
'uuid': e.uuid,
'parentUuid': e.parent_directory.uuid,
'originalName': escape(e.name)
} for e in entities['documents']
],
'count': len(entities['documents']),
'status': 0
}
return JsonResponse(response)
else:
if has_catalog(request.user):
return metadata_search_entities_interactive(request)
else:
return JsonResponse({'status': 1, 'message': _('Navigator not enabled')})
def editor(request, is_mobile=False, is_embeddable=False):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
gist_id = request.GET.get('gist')
if editor_type == 'notebook' or request.GET.get('notebook'):
return notebook(request)
if editor_type == 'gist':
gist_doc = _get_gist_document(uuid=gist_id)
editor_type = gist_doc.extra
if editor_id and not gist_id: # Open existing saved editor document
editor_type = _get_editor_type(editor_id)
template = 'editor.mako'
if is_mobile:
template = 'editor_m.mako'
return render(template, request, {
'editor_id': editor_id or None,
'notebooks_json': '{}',
'is_embeddable': request.GET.get('is_embeddable', False),
'editor_type': editor_type,
'options_json': json.dumps({
'languages': get_ordered_interpreters(request.user),
'mode': 'editor',
'is_optimizer_enabled': has_optimizer(),
'is_wa_enabled': has_workload_analytics(),
'is_navigator_enabled': has_catalog(request.user),
'editor_type': editor_type,
'mobile': is_mobile
})
})
def show_tables(request, database=None):
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, cluster=cluster)
if database is None:
database = 'default' # Assume always 'default'
if request.GET.get("format", "html") == "json":
try:
databases = db.get_databases()
if database not in databases:
database = 'default'
if request.method == 'POST':
db_form = DbForm(request.POST, databases=databases)
if db_form.is_valid():
database = db_form.cleaned_data['database']
else:
db_form = DbForm(initial={'database': database},
databases=databases)
search_filter = request.GET.get('filter', '')
tables = db.get_tables_meta(
database=database,
table_names=search_filter) # SparkSql returns []
table_names = [table['name'] for table in tables]
except Exception as e:
raise PopupException(_(
'Failed to retrieve tables for database: %s' % database),
detail=e)
resp = JsonResponse({
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
apps_list = _get_apps(request.user, '')
resp = render(
"metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
return resp
def describe_table(request, database, table):
app_name = get_app_name(request)
cluster = json.loads(request.POST.get('cluster', '{}'))
source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
try:
table = db.get_table(database, table)
except Exception as e:
LOG.exception("Describe table error")
raise PopupException(_("DB Error"), detail=e.message if hasattr(e, 'message') and e.message else e)
if request.POST.get("format", "html") == "json":
return JsonResponse({
'status': 0,
'name': table.name,
'partition_keys': [{'name': part.name, 'type': part.type} for part in table.partition_keys],
'cols': [{'name': col.name, 'type': col.type, 'comment': col.comment} for col in table.cols],
'path_location': table.path_location,
'hdfs_link': table.hdfs_link,
'comment': table.comment,
'is_view': table.is_view,
'properties': table.properties,
'details': table.details,
'stats': table.stats
})
else: # Render HTML
renderable = "metastore.mako"
apps_list = _get_apps(request.user, '')
partitions = None
if app_name != 'impala' and table.partition_keys:
try:
partitions = [_massage_partition(database, table, partition) for partition in db.get_partitions(database, table)]
except:
LOG.exception('Table partitions could not be retrieved')
return render(renderable, request, {
'apps': apps_list,
'breadcrumbs': [{
'name': database,
'url': reverse('metastore:show_tables', kwargs={'database': database})
}, {
'name': str(table.name),
'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table.name})
},
],
'table': table,
'partitions': partitions,
'database': database,
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def setup_class(cls):
cls.client = make_logged_in_client(username='******', is_superuser=False)
cls.user = User.objects.get(username='******')
cls.user = rewrite_user(cls.user)
add_to_group('test')
grant_access("test", "test", "metadata")
if not is_live_cluster() or not has_catalog(cls.user):
raise SkipTest
cls.api = AtlasApi(cls.user)
def decorator(*args, **kwargs):
status = 500
response = {'message': ''}
try:
if has_catalog(args[0].user):
return view_fn(*args, **kwargs)
else:
raise MetadataApiException('Catalog API is not configured.')
except Http404, e:
raise e
def decorator(*args, **kwargs):
status = 500
response = {'message': ''}
try:
if has_catalog(args[0].user): # TODO
return view_fn(*args, **kwargs)
else:
raise CatalogApiException('Navigator API is not configured.')
except CatalogApiException, e:
try:
response['message'] = json.loads(e.message)
except Exception:
response['message'] = force_unicode(e.message)
def search_entities(request):
sources = json.loads(request.POST.get('sources')) or []
if 'documents' in sources:
search_text = json.loads(request.POST.get('query_s', ''))
entities = _search(user=request.user, search_text=search_text)
response = {
'entities': [{
'hue_name':
_highlight(search_text, escape(e.name)),
'hue_description':
_highlight(search_text, escape(e.description)),
'type':
'HUE',
'last_modified':
e.last_modified,
'owner':
escape(e.owner),
'doc_type':
escape(e.type),
'originalName':
escape(e.name),
'link':
e.get_absolute_url()
} for e in entities['documents']],
'count':
len(entities['documents']),
'status':
0
}
return JsonResponse(response)
else:
if has_catalog(request.user):
return metadata_search_entities(request)
else:
return JsonResponse({
'status': 1,
'message': _('Navigator not enabled')
})
def databases(request):
search_filter = request.GET.get('filter', '')
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, cluster=cluster)
databases = db.get_databases(search_filter)
apps_list = _get_apps(request.user, '')
return render("metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def databases(request):
search_filter = request.GET.get('filter', '')
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, cluster=cluster)
databases = db.get_databases(search_filter)
apps_list = _get_apps(request.user, '')
return render("metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def editor(request, is_mobile=False, is_embeddable=False):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
if editor_type == 'notebook' or request.GET.get('notebook'):
return notebook(request)
if editor_id: # Open existing saved editor document
document = Document2.objects.get(id=editor_id)
editor_type = document.type.rsplit('-', 1)[-1]
template = 'editor.mako'
if is_mobile:
template = 'editor_m.mako'
return render(
template, request, {
'editor_id':
editor_id or None,
'notebooks_json':
'{}',
'is_embeddable':
request.GET.get('is_embeddable', False),
'editor_type':
editor_type,
'options_json':
json.dumps({
'languages': get_ordered_interpreters(request.user),
'mode': 'editor',
'is_optimizer_enabled': has_optimizer(),
'is_wa_enabled': has_workload_analytics(),
'is_navigator_enabled': has_catalog(request.user),
'editor_type': editor_type,
'mobile': is_mobile
})
})
def describe_partitions(request, database, table):
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, cluster=cluster)
table_obj = db.get_table(database, table)
if not table_obj.partition_keys:
raise PopupException(_("Table '%(table)s' is not partitioned.") % {'table': table})
reverse_sort = request.GET.get("sort", "desc").lower() == "desc"
if request.method == "POST":
partition_filters = {}
for part in table_obj.partition_keys:
if request.GET.get(part.name):
partition_filters[part.name] = request.GET.get(part.name)
partition_spec = ','.join(["%s='%s'" % (k, v) for k, v in list(partition_filters.items())])
else:
partition_spec = ''
try:
partitions = db.get_partitions(database, table_obj, partition_spec, reverse_sort=reverse_sort)
except:
LOG.exception('Table partitions could not be retrieved')
partitions = []
massaged_partitions = [_massage_partition(database, table_obj, partition) for partition in partitions]
if request.method == "POST" or request.GET.get('format', 'html') == 'json':
return JsonResponse({
'partition_keys_json': [partition.name for partition in table_obj.partition_keys],
'partition_values_json': massaged_partitions,
})
else:
apps_list = _get_apps(request.user, '')
return render("metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [{
'name': database,
'url': reverse('metastore:show_tables', kwargs={'database': database})
}, {
'name': table,
'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
}, {
'name': 'partitions',
'url': reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
},
],
'database': database,
'table': table_obj,
'partitions': partitions,
'partition_keys_json': json.dumps([partition.name for partition in table_obj.partition_keys]),
'partition_values_json': json.dumps(massaged_partitions),
'request': request,
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
apps_list = _get_apps(request.user, '')
resp = render("metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
return resp
def get_table_metadata(request, database, table):
cluster = json.loads(request.POST.get('cluster', '{}'))
source_type = request.POST.get('source_type')
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
response = {'status': -1, 'data': ''}
def describe_partitions(request, database, table):
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, cluster=cluster)
table_obj = db.get_table(database, table)
if not table_obj.partition_keys:
raise PopupException(_("Table '%(table)s' is not partitioned.") % {'table': table})
reverse_sort = request.GET.get("sort", "desc").lower() == "desc"
if request.method == "POST":
partition_filters = {}
for part in table_obj.partition_keys:
if request.GET.get(part.name):
partition_filters[part.name] = request.GET.get(part.name)
partition_spec = ','.join(["%s='%s'" % (k, v) for k, v in partition_filters.items()])
else:
partition_spec = ''
try:
partitions = db.get_partitions(database, table_obj, partition_spec, reverse_sort=reverse_sort)
except:
LOG.exception('Table partitions could not be retrieved')
partitions = []
massaged_partitions = [_massage_partition(database, table_obj, partition) for partition in partitions]
if request.method == "POST" or request.GET.get('format', 'html') == 'json':
return JsonResponse({
'partition_keys_json': [partition.name for partition in table_obj.partition_keys],
'partition_values_json': massaged_partitions,
})
else:
apps_list = _get_apps(request.user, '')
return render("metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [{
'name': database,
'url': reverse('metastore:show_tables', kwargs={'database': database})
}, {
'name': table,
'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
},{
'name': 'partitions',
'url': reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
},
],
'database': database,
'table': table_obj,
'partitions': partitions,
'partition_keys_json': json.dumps([partition.name for partition in table_obj.partition_keys]),
'partition_values_json': json.dumps(massaged_partitions),
'request': request,
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
apps_list = _get_apps(request.user, '')
resp = render("metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
return resp
def get_table_metadata(request, database, table):
cluster = json.loads(request.POST.get('cluster', '{}'))
source_type = request.POST.get('source_type')
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
response = {'status': -1, 'data': ''}
