def show_tables(request, database=None): cluster = json.loads(request.POST.get('cluster', '{}')) db = _get_db(user=request.user, cluster=cluster) if database is None: database = 'default' # Assume always 'default' if request.GET.get("format", "html") == "json": try: databases = db.get_databases() if database not in databases: database = 'default' if request.method == 'POST': db_form = DbForm(request.POST, databases=databases) if db_form.is_valid(): database = db_form.cleaned_data['database'] else: db_form = DbForm(initial={'database': database}, databases=databases) search_filter = request.GET.get('filter', '') tables = db.get_tables_meta( database=database, table_names=search_filter) # SparkSql returns [] table_names = [table['name'] for table in tables] except Exception as e: raise PopupException(_( 'Failed to retrieve tables for database: %s' % database), detail=e) resp = JsonResponse({ 'status': 0, 'database_meta': db.get_database(database), 'tables': tables, 'table_names': table_names, 'search_filter': search_filter }) else: apps_list = _get_apps(request.user, '') resp = render( "metastore.mako", request, { 'apps': apps_list, 'breadcrumbs': [], 'database': None, 'partitions': [], 'has_write_access': has_write_access(request.user), 'is_optimizer_enabled': has_optimizer(), 'is_navigator_enabled': has_catalog(request.user), 'optimizer_url': get_optimizer_url(), 'navigator_url': get_catalog_url(), 'is_embeddable': request.GET.get('is_embeddable', False), 'source_type': _get_servername(db), }) return resp
def describe_partitions(request, database, table): db = _get_db(user=request.user) table_obj = db.get_table(database, table) if not table_obj.partition_keys: raise PopupException(_("Table '%(table)s' is not partitioned.") % {'table': table}) reverse_sort = request.GET.get("sort", "desc").lower() == "desc" if request.method == "POST": partition_filters = {} for part in table_obj.partition_keys: if request.GET.get(part.name): partition_filters[part.name] = request.GET.get(part.name) partition_spec = ','.join(["%s='%s'" % (k, v) for k, v in partition_filters.items()]) else: partition_spec = '' try: partitions = db.get_partitions(database, table_obj, partition_spec, reverse_sort=reverse_sort) except: LOG.exception('Table partitions could not be retrieved') partitions = [] massaged_partitions = [_massage_partition(database, table_obj, partition) for partition in partitions] if request.method == "POST" or request.GET.get('format', 'html') == 'json': return JsonResponse({ 'partition_keys_json': [partition.name for partition in table_obj.partition_keys], 'partition_values_json': massaged_partitions, }) else: return render("metastore.mako", request, { 'breadcrumbs': [{ 'name': database, 'url': reverse('metastore:show_tables', kwargs={'database': database}) }, { 'name': table, 'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table}) },{ 'name': 'partitions', 'url': reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table}) }, ], 'database': database, 'table': table_obj, 'partitions': partitions, 'partition_keys_json': json.dumps([partition.name for partition in table_obj.partition_keys]), 'partition_values_json': json.dumps(massaged_partitions), 'request': request, 'has_write_access': has_write_access(request.user), 'is_optimizer_enabled': has_optimizer(), 'is_navigator_enabled': has_navigator(request.user), 'optimizer_url': get_optimizer_url(), 'navigator_url': get_navigator_url(), 'is_embeddable': request.GET.get('is_embeddable', False), 'source_type': _get_servername(db), })
def describe_table(request, database, table): app_name = get_app_name(request) cluster = json.loads(request.POST.get('cluster', '{}')) source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive')) db = _get_db(user=request.user, source_type=source_type, cluster=cluster) try: table = db.get_table(database, table) except Exception as e: LOG.exception("Describe table error") raise PopupException(_("DB Error"), detail=e.message if hasattr(e, 'message') and e.message else e) if request.POST.get("format", "html") == "json": return JsonResponse({ 'status': 0, 'name': table.name, 'partition_keys': [{'name': part.name, 'type': part.type} for part in table.partition_keys], 'cols': [{'name': col.name, 'type': col.type, 'comment': col.comment} for col in table.cols], 'path_location': table.path_location, 'hdfs_link': table.hdfs_link, 'comment': table.comment, 'is_view': table.is_view, 'properties': table.properties, 'details': table.details, 'stats': table.stats }) else: # Render HTML renderable = "metastore.mako" apps_list = _get_apps(request.user, '') partitions = None if app_name != 'impala' and table.partition_keys: try: partitions = [_massage_partition(database, table, partition) for partition in db.get_partitions(database, table)] except: LOG.exception('Table partitions could not be retrieved') return render(renderable, request, { 'apps': apps_list, 'breadcrumbs': [{ 'name': database, 'url': reverse('metastore:show_tables', kwargs={'database': database}) }, { 'name': str(table.name), 'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table.name}) }, ], 'table': table, 'partitions': partitions, 'database': database, 'has_write_access': has_write_access(request.user), 'is_optimizer_enabled': has_optimizer(), 'is_navigator_enabled': has_catalog(request.user), 'optimizer_url': get_optimizer_url(), 'navigator_url': get_catalog_url(), 'is_embeddable': request.GET.get('is_embeddable', False), 'source_type': _get_servername(db), })
def __init__(self, user, api_url=None, auth_key=None, auth_key_secret=None, tenant_id=None): self.user = user self._api_url = (api_url or get_optimizer_url()).strip('/') self._auth_key = auth_key if auth_key else OPTIMIZER.AUTH_KEY_ID.get() self._auth_key_secret = auth_key_secret if auth_key_secret else (OPTIMIZER.AUTH_KEY_SECRET.get() and OPTIMIZER.AUTH_KEY_SECRET.get().replace('\\n', '\n')) self._api = ApiLib("navopt", self._api_url, self._auth_key, self._auth_key_secret) self._tenant_id = tenant_id if tenant_id else _get_tenant_id(self) # Aka "workload"
def __init__(self, user, api_url=None, auth_key=None, auth_key_secret=None, tenant_id='hue'): self.user = user self._tenant_id = tenant_id self._api_url = (api_url or get_optimizer_url()).strip('/') self._client = HttpClient(self._api_url, logger=LOG) self._root = Resource(self._client) self._api = MockApiLib()
def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None): self._api_url = (api_url or get_optimizer_url()).strip('/') self._email = OPTIMIZER.EMAIL.get() self._email_password = OPTIMIZER.EMAIL_PASSWORD.get() self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get() self._product_auth_secret = product_auth_secret if product_auth_secret else (OPTIMIZER.PRODUCT_AUTH_SECRET.get() and OPTIMIZER.PRODUCT_AUTH_SECRET.get().replace('\\n', '\n')) self._api = ApiLib("navopt", urlparse(self._api_url).hostname, self._product_secret, self._product_auth_secret) self._product_name = product_name if product_name else (OPTIMIZER.PRODUCT_NAME.get() or self.get_tenant()['tenant']) # Aka "workload"
def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None): self._api_url = (api_url or get_optimizer_url()).strip('/') self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get() self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get() self._product_auth_secret = product_auth_secret if product_auth_secret else OPTIMIZER.PRODUCT_AUTH_SECRET.get() self._email = OPTIMIZER.EMAIL.get() self._email_password = OPTIMIZER.EMAIL_PASSWORD.get() self._client = HttpClient(self._api_url, logger=LOG) self._client.set_verify(ssl_cert_ca_verify) self._root = resource.Resource(self._client) self._token = None
def __init__(self, user, api_url=None, auth_key=None, auth_key_secret=None, tenant_id=None): self.user = user self._api_url = (api_url or get_optimizer_url()).strip('/') self._auth_key = auth_key if auth_key else OPTIMIZER.AUTH_KEY_ID.get() self._auth_key_secret = auth_key_secret if auth_key_secret else ( OPTIMIZER.AUTH_KEY_SECRET.get() and OPTIMIZER.AUTH_KEY_SECRET.get().replace('\\n', '\n')) self._api = None self._tenant_id = tenant_id
def databases(request): search_filter = request.GET.get('filter', '') db = dbms.get(request.user) databases = db.get_databases(search_filter) return render("metastore.mako", request, { 'breadcrumbs': [], 'database': None, 'databases': databases, 'partitions': [], 'has_write_access': has_write_access(request.user), 'is_optimizer_enabled': has_optimizer(), 'optimizer_url': get_optimizer_url(), 'navigator_url': get_navigator_url(), })
def databases(request): search_filter = request.GET.get('filter', '') db = dbms.get(request.user) databases = db.get_databases(search_filter) return render( "metastore.mako", request, { 'breadcrumbs': [], 'database': None, 'databases': databases, 'partitions': [], 'has_write_access': has_write_access(request.user), 'is_optimizer_enabled': has_optimizer(), 'optimizer_url': get_optimizer_url(), 'navigator_url': get_navigator_url(), })
def databases(request): search_filter = request.GET.get('filter', '') db = _get_db(user=request.user) databases = db.get_databases(search_filter) return render("metastore.mako", request, { 'breadcrumbs': [], 'database': None, 'databases': databases, 'partitions': [], 'has_write_access': has_write_access(request.user), 'is_optimizer_enabled': has_optimizer(), 'is_navigator_enabled': has_navigator(request.user), 'optimizer_url': get_optimizer_url(), 'navigator_url': get_navigator_url(), 'is_embeddable': request.GET.get('is_embeddable', False), 'source_type': _get_servername(db), })
def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None): self._api_url = (api_url or get_optimizer_url()).strip('/') self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get( ) self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get( ) self._product_auth_secret = product_auth_secret if product_auth_secret else OPTIMIZER.PRODUCT_AUTH_SECRET.get( ) self._email = OPTIMIZER.EMAIL.get() self._email_password = OPTIMIZER.EMAIL_PASSWORD.get() self._client = HttpClient(self._api_url, logger=LOG) self._client.set_verify(ssl_cert_ca_verify) self._root = resource.Resource(self._client) self._token = None
def databases(request): search_filter = request.GET.get('filter', '') cluster = json.loads(request.POST.get('cluster', '{}')) db = _get_db(user=request.user, cluster=cluster) databases = db.get_databases(search_filter) apps_list = _get_apps(request.user, '') return render("metastore.mako", request, { 'apps': apps_list, 'breadcrumbs': [], 'database': None, 'databases': databases, 'partitions': [], 'has_write_access': has_write_access(request.user), 'is_optimizer_enabled': has_optimizer(), 'is_navigator_enabled': has_catalog(request.user), 'optimizer_url': get_optimizer_url(), 'navigator_url': get_catalog_url(), 'is_embeddable': request.GET.get('is_embeddable', False), 'source_type': _get_servername(db), })
def is_optimizer_enabled(): return get_optimizer_url() and OPTIMIZER.PRODUCT_NAME.get()
resp = JsonResponse({ 'status': 0, 'database_meta': db.get_database(database), 'tables': tables, 'table_names': table_names, 'search_filter': search_filter }) else: resp = render("metastore.mako", request, { 'breadcrumbs': [], 'database': None, 'partitions': [], 'has_write_access': has_write_access(request.user), 'is_optimizer_enabled': has_optimizer(), 'is_navigator_enabled': has_navigator(request.user), 'optimizer_url': get_optimizer_url(), 'navigator_url': get_navigator_url(), 'is_embeddable': request.REQUEST.get('is_embeddable', False), }) return resp def get_table_metadata(request, database, table): db = dbms.get(request.user) response = {'status': -1, 'data': ''} try: table_metadata = db.get_table(database, table) response['status'] = 0 response['data'] = { 'comment': table_metadata.comment,
def describe_partitions(request, database, table): cluster = json.loads(request.POST.get('cluster', '{}')) db = _get_db(user=request.user, cluster=cluster) table_obj = db.get_table(database, table) if not table_obj.partition_keys: raise PopupException(_("Table '%(table)s' is not partitioned.") % {'table': table}) reverse_sort = request.GET.get("sort", "desc").lower() == "desc" if request.method == "POST": partition_filters = {} for part in table_obj.partition_keys: if request.GET.get(part.name): partition_filters[part.name] = request.GET.get(part.name) partition_spec = ','.join(["%s='%s'" % (k, v) for k, v in partition_filters.items()]) else: partition_spec = '' try: partitions = db.get_partitions(database, table_obj, partition_spec, reverse_sort=reverse_sort) except: LOG.exception('Table partitions could not be retrieved') partitions = [] massaged_partitions = [_massage_partition(database, table_obj, partition) for partition in partitions] if request.method == "POST" or request.GET.get('format', 'html') == 'json': return JsonResponse({ 'partition_keys_json': [partition.name for partition in table_obj.partition_keys], 'partition_values_json': massaged_partitions, }) else: apps_list = _get_apps(request.user, '') return render("metastore.mako", request, { 'apps': apps_list, 'breadcrumbs': [{ 'name': database, 'url': reverse('metastore:show_tables', kwargs={'database': database}) }, { 'name': table, 'url': reverse('metastore:describe_table', kwargs={'database': database, 'table': table}) },{ 'name': 'partitions', 'url': reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table}) }, ], 'database': database, 'table': table_obj, 'partitions': partitions, 'partition_keys_json': json.dumps([partition.name for partition in table_obj.partition_keys]), 'partition_values_json': json.dumps(massaged_partitions), 'request': request, 'has_write_access': has_write_access(request.user), 'is_optimizer_enabled': has_optimizer(), 'is_navigator_enabled': has_catalog(request.user), 'optimizer_url': get_optimizer_url(), 'navigator_url': get_catalog_url(), 'is_embeddable': request.GET.get('is_embeddable', False), 'source_type': _get_servername(db), })
'status': 0, 'database_meta': db.get_database(database), 'tables': tables, 'table_names': table_names, 'search_filter': search_filter }) else: resp = render( "metastore.mako", request, { 'breadcrumbs': [], 'database': None, 'partitions': [], 'has_write_access': has_write_access(request.user), 'is_optimizer_enabled': has_optimizer(), 'is_navigator_enabled': has_navigator(request.user), 'optimizer_url': get_optimizer_url(), 'navigator_url': get_navigator_url(), 'is_embeddable': request.REQUEST.get('is_embeddable', False), }) return resp def get_table_metadata(request, database, table): db = dbms.get(request.user) response = {'status': -1, 'data': ''} try: table_metadata = db.get_table(database, table) response['status'] = 0 response['data'] = { 'comment': table_metadata.comment,