def show_tables(request, database=None):
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, cluster=cluster)
if database is None:
database = 'default' # Assume always 'default'
if request.GET.get("format", "html") == "json":
try:
databases = db.get_databases()
if database not in databases:
database = 'default'
if request.method == 'POST':
db_form = DbForm(request.POST, databases=databases)
if db_form.is_valid():
database = db_form.cleaned_data['database']
else:
db_form = DbForm(initial={'database': database},
databases=databases)
search_filter = request.GET.get('filter', '')
tables = db.get_tables_meta(
database=database,
table_names=search_filter) # SparkSql returns []
table_names = [table['name'] for table in tables]
except Exception as e:
raise PopupException(_(
'Failed to retrieve tables for database: %s' % database),
detail=e)
resp = JsonResponse({
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
apps_list = _get_apps(request.user, '')
resp = render(
"metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
return resp
def describe_partitions(request, database, table):
db = _get_db(user=request.user)
table_obj = db.get_table(database, table)
if not table_obj.partition_keys:
raise PopupException(_("Table '%(table)s' is not partitioned.") % {'table': table})
reverse_sort = request.GET.get("sort", "desc").lower() == "desc"
if request.method == "POST":
partition_filters = {}
for part in table_obj.partition_keys:
if request.GET.get(part.name):
partition_filters[part.name] = request.GET.get(part.name)
partition_spec = ','.join(["%s='%s'" % (k, v) for k, v in partition_filters.items()])
else:
partition_spec = ''
try:
partitions = db.get_partitions(database, table_obj, partition_spec, reverse_sort=reverse_sort)
except:
LOG.exception('Table partitions could not be retrieved')
partitions = []
massaged_partitions = [_massage_partition(database, table_obj, partition) for partition in partitions]
if request.method == "POST" or request.GET.get('format', 'html') == 'json':
return JsonResponse({
'partition_keys_json': [partition.name for partition in table_obj.partition_keys],
'partition_values_json': massaged_partitions,
})
else:
return render("metastore.mako", request, {
'breadcrumbs': [{
'name': database,
'url': reverse('metastore:show_tables', kwargs={'database': database})
}, {
'name': table,
'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
},{
'name': 'partitions',
'url': reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
},
],
'database': database,
'table': table_obj,
'partitions': partitions,
'partition_keys_json': json.dumps([partition.name for partition in table_obj.partition_keys]),
'partition_values_json': json.dumps(massaged_partitions),
'request': request,
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def describe_table(request, database, table):
app_name = get_app_name(request)
cluster = json.loads(request.POST.get('cluster', '{}'))
source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
try:
table = db.get_table(database, table)
except Exception as e:
LOG.exception("Describe table error")
raise PopupException(_("DB Error"), detail=e.message if hasattr(e, 'message') and e.message else e)
if request.POST.get("format", "html") == "json":
return JsonResponse({
'status': 0,
'name': table.name,
'partition_keys': [{'name': part.name, 'type': part.type} for part in table.partition_keys],
'cols': [{'name': col.name, 'type': col.type, 'comment': col.comment} for col in table.cols],
'path_location': table.path_location,
'hdfs_link': table.hdfs_link,
'comment': table.comment,
'is_view': table.is_view,
'properties': table.properties,
'details': table.details,
'stats': table.stats
})
else: # Render HTML
renderable = "metastore.mako"
apps_list = _get_apps(request.user, '')
partitions = None
if app_name != 'impala' and table.partition_keys:
try:
partitions = [_massage_partition(database, table, partition) for partition in db.get_partitions(database, table)]
except:
LOG.exception('Table partitions could not be retrieved')
return render(renderable, request, {
'apps': apps_list,
'breadcrumbs': [{
'name': database,
'url': reverse('metastore:show_tables', kwargs={'database': database})
}, {
'name': str(table.name),
'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table.name})
},
],
'table': table,
'partitions': partitions,
'database': database,
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def __init__(self, user, api_url=None, auth_key=None, auth_key_secret=None, tenant_id=None):
self.user = user
self._api_url = (api_url or get_optimizer_url()).strip('/')
self._auth_key = auth_key if auth_key else OPTIMIZER.AUTH_KEY_ID.get()
self._auth_key_secret = auth_key_secret if auth_key_secret else (OPTIMIZER.AUTH_KEY_SECRET.get() and OPTIMIZER.AUTH_KEY_SECRET.get().replace('\\n', '\n'))
self._api = ApiLib("navopt", self._api_url, self._auth_key, self._auth_key_secret)
self._tenant_id = tenant_id if tenant_id else _get_tenant_id(self) # Aka "workload"
def __init__(self, user, api_url=None, auth_key=None, auth_key_secret=None, tenant_id='hue'):
self.user = user
self._tenant_id = tenant_id
self._api_url = (api_url or get_optimizer_url()).strip('/')
self._client = HttpClient(self._api_url, logger=LOG)
self._root = Resource(self._client)
self._api = MockApiLib()
def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None):
self._api_url = (api_url or get_optimizer_url()).strip('/')
self._email = OPTIMIZER.EMAIL.get()
self._email_password = OPTIMIZER.EMAIL_PASSWORD.get()
self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get()
self._product_auth_secret = product_auth_secret if product_auth_secret else (OPTIMIZER.PRODUCT_AUTH_SECRET.get() and OPTIMIZER.PRODUCT_AUTH_SECRET.get().replace('\\n', '\n'))
self._api = ApiLib("navopt", urlparse(self._api_url).hostname, self._product_secret, self._product_auth_secret)
self._product_name = product_name if product_name else (OPTIMIZER.PRODUCT_NAME.get() or self.get_tenant()['tenant']) # Aka "workload"
def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None):
self._api_url = (api_url or get_optimizer_url()).strip('/')
self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get()
self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get()
self._product_auth_secret = product_auth_secret if product_auth_secret else OPTIMIZER.PRODUCT_AUTH_SECRET.get()
self._email = OPTIMIZER.EMAIL.get()
self._email_password = OPTIMIZER.EMAIL_PASSWORD.get()
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = resource.Resource(self._client)
self._token = None
def __init__(self,
user,
api_url=None,
auth_key=None,
auth_key_secret=None,
tenant_id=None):
self.user = user
self._api_url = (api_url or get_optimizer_url()).strip('/')
self._auth_key = auth_key if auth_key else OPTIMIZER.AUTH_KEY_ID.get()
self._auth_key_secret = auth_key_secret if auth_key_secret else (
OPTIMIZER.AUTH_KEY_SECRET.get()
and OPTIMIZER.AUTH_KEY_SECRET.get().replace('\\n', '\n'))
self._api = None
self._tenant_id = tenant_id
def databases(request):
search_filter = request.GET.get('filter', '')
db = dbms.get(request.user)
databases = db.get_databases(search_filter)
return render("metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
})
def databases(request):
search_filter = request.GET.get('filter', '')
db = dbms.get(request.user)
databases = db.get_databases(search_filter)
return render(
"metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
})
def databases(request):
search_filter = request.GET.get('filter', '')
db = _get_db(user=request.user)
databases = db.get_databases(search_filter)
return render("metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def __init__(self,
api_url=None,
product_name=None,
product_secret=None,
ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(),
product_auth_secret=None):
self._api_url = (api_url or get_optimizer_url()).strip('/')
self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get(
)
self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get(
)
self._product_auth_secret = product_auth_secret if product_auth_secret else OPTIMIZER.PRODUCT_AUTH_SECRET.get(
)
self._email = OPTIMIZER.EMAIL.get()
self._email_password = OPTIMIZER.EMAIL_PASSWORD.get()
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = resource.Resource(self._client)
self._token = None
def databases(request):
search_filter = request.GET.get('filter', '')
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, cluster=cluster)
databases = db.get_databases(search_filter)
apps_list = _get_apps(request.user, '')
return render("metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def databases(request):
search_filter = request.GET.get('filter', '')
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, cluster=cluster)
databases = db.get_databases(search_filter)
apps_list = _get_apps(request.user, '')
return render("metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [],
'database': None,
'databases': databases,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
def is_optimizer_enabled():
return get_optimizer_url() and OPTIMIZER.PRODUCT_NAME.get()
def is_optimizer_enabled(): return get_optimizer_url() and OPTIMIZER.PRODUCT_NAME.get()
resp = JsonResponse({
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
resp = render("metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
'is_embeddable': request.REQUEST.get('is_embeddable', False),
})
return resp
def get_table_metadata(request, database, table):
db = dbms.get(request.user)
response = {'status': -1, 'data': ''}
try:
table_metadata = db.get_table(database, table)
response['status'] = 0
response['data'] = {
'comment': table_metadata.comment,
def describe_partitions(request, database, table):
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, cluster=cluster)
table_obj = db.get_table(database, table)
if not table_obj.partition_keys:
raise PopupException(_("Table '%(table)s' is not partitioned.") % {'table': table})
reverse_sort = request.GET.get("sort", "desc").lower() == "desc"
if request.method == "POST":
partition_filters = {}
for part in table_obj.partition_keys:
if request.GET.get(part.name):
partition_filters[part.name] = request.GET.get(part.name)
partition_spec = ','.join(["%s='%s'" % (k, v) for k, v in partition_filters.items()])
else:
partition_spec = ''
try:
partitions = db.get_partitions(database, table_obj, partition_spec, reverse_sort=reverse_sort)
except:
LOG.exception('Table partitions could not be retrieved')
partitions = []
massaged_partitions = [_massage_partition(database, table_obj, partition) for partition in partitions]
if request.method == "POST" or request.GET.get('format', 'html') == 'json':
return JsonResponse({
'partition_keys_json': [partition.name for partition in table_obj.partition_keys],
'partition_values_json': massaged_partitions,
})
else:
apps_list = _get_apps(request.user, '')
return render("metastore.mako", request, {
'apps': apps_list,
'breadcrumbs': [{
'name': database,
'url': reverse('metastore:show_tables', kwargs={'database': database})
}, {
'name': table,
'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table})
},{
'name': 'partitions',
'url': reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
},
],
'database': database,
'table': table_obj,
'partitions': partitions,
'partition_keys_json': json.dumps([partition.name for partition in table_obj.partition_keys]),
'partition_values_json': json.dumps(massaged_partitions),
'request': request,
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_catalog(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_catalog_url(),
'is_embeddable': request.GET.get('is_embeddable', False),
'source_type': _get_servername(db),
})
'status': 0,
'database_meta': db.get_database(database),
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
resp = render(
"metastore.mako", request, {
'breadcrumbs': [],
'database': None,
'partitions': [],
'has_write_access': has_write_access(request.user),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'optimizer_url': get_optimizer_url(),
'navigator_url': get_navigator_url(),
'is_embeddable': request.REQUEST.get('is_embeddable', False),
})
return resp
def get_table_metadata(request, database, table):
db = dbms.get(request.user)
response = {'status': -1, 'data': ''}
try:
table_metadata = db.get_table(database, table)
response['status'] = 0
response['data'] = {
'comment': table_metadata.comment,
