def get_namespace(request): api = NavigatorApi(request.user) namespace = request.POST.get('namespace') namespace = api.get_namespace(namespace) return JsonResponse(namespace)
def search_entities(request): """ For displaying results. """ api = NavigatorApi(request.user) query_s = json.loads(request.POST.get('query_s', '')) query_s = smart_str(query_s) offset = request.POST.get('offset', 0) limit = int(request.POST.get('limit', 100)) sources = json.loads(request.POST.get('sources')) or [] query_s = query_s.strip() or '*' entities = api.search_entities(query_s, limit=limit, offset=offset, sources=sources) _augment_highlighting(query_s, entities) response = { 'entities': entities, 'count': len(entities), 'offset': offset, 'limit': limit, 'query_s': query_s, 'status': 0 } return JsonResponse(response)
def search_entities_interactive(request): api = NavigatorApi() query_s = request.POST.get('query_s', '') prefix = request.POST.get('prefix') offset = request.POST.get('offset', 0) limit = request.POST.get('limit', 25) field_facets = json.loads(request.POST.get('field_facets') or '[]') sources = json.loads(request.POST.get('sources') or '[]') fqs = [] fq_type = [] if 'hive' in sources or 'impala' in sources: fq_type.append('TABLE') fq_type.append('VIEW') # 'DATABASE', 'PARTITION', 'FIELD' elif 'hdfs' in sources: fq_type.append('FILE') fq_type.append('DIRECTORY') fqs += ['{!tag=type} %s' % ' OR '.join(['type:%s' % fq for fq in fq_type])] response = api.search_entities_interactive(query=query_s, limit=limit, offset=offset, facetFields=field_facets, facetPrefix=prefix, facetRanges=None, filterQueries=fqs, firstClassEntitiesOnly=None) response['status'] = 0 return JsonResponse(response)
def get_lineage(request): response = {'status': -1, 'inputs': [], 'source_query': '', 'target_queries': [], 'targets': []} api = NavigatorApi() entity_id = request.REQUEST.get('id') if not entity_id: raise MetadataApiException("get_lineage requires an 'id' parameter") lineage = api.get_lineage(entity_id) entity_name = api.get_entity(entity_id)['originalName'].upper() response['id'] = entity_id # TODO: This is a cheat way to do to this for demo using filtering but we should really traverse relationships parent_operation = next((entity for entity in lineage['entities'] if entity.get('outputs', []) == [entity_name]), None) if parent_operation: response['inputs'] = [input.lower() for input in parent_operation['inputs']] response['source_query'] = parent_operation.get('queryText', '') children = [entity for entity in lineage['entities'] if entity.get('inputs') is not None and entity_name in entity.get('inputs')] if children is not None: response['target_queries'] = [child['queryText'] for child in children if child.get('queryText') is not None] outputs = [child['outputs'] for child in children if child.get('outputs') is not None] response['targets'] = [target.lower() for output in outputs for target in output] response['status'] = 0 return JsonResponse(response)
def search_entities(request): """ For displaying results. """ api = NavigatorApi(request.user) query_s = json.loads(request.POST.get('query_s', '')) query_s = smart_str(query_s) offset = request.POST.get('offset', 0) limit = int(request.POST.get('limit', 100)) raw_query = request.POST.get('raw_query', False) sources = json.loads(request.POST.get('sources') or '[]') if sources and not has_navigator_file_search(request.user): sources = ['sql'] query_s = query_s.strip() or '*' entities = api.search_entities(query_s, limit=limit, offset=offset, raw_query=raw_query, sources=sources) if not raw_query: _augment_highlighting(query_s, entities) response = { 'entities': entities, 'count': len(entities), 'offset': offset, 'limit': limit, 'query_s': query_s, 'status': 0 } return JsonResponse(response)
def update_properties(request): api = NavigatorApi(request.user) entity_id = json.loads(request.POST.get('id', '""')) properties = json.loads(request.POST.get('properties', '{}')) # Entity properties modified_custom_metadata = json.loads(request.POST.get('modifiedCustomMetadata', '{}')) # Aka "Custom Metadata" deleted_custom_metadata_keys = json.loads(request.POST.get('deletedCustomMetadataKeys', '[]')) is_allowed = request.user.has_hue_permission(action='write', app='metadata') request.audit = { 'allowed': is_allowed, 'operation': 'NAVIGATOR_UPDATE_PROPERTIES', 'operationText': 'Updating custom metadata %s, deleted custom metadata keys %s and properties %s of entity %s' % (modified_custom_metadata, deleted_custom_metadata_keys, properties, entity_id) } if not entity_id: # TODO: raise HueApiException(message="Missing required parameter 'id' for update_properties", source="Hue") # source so the user knows which service that failed right away, in UI: "[source] responded with error: [message]" raise Exception("Missing required parameter 'id' for the Hue update_properties API.") if not is_allowed: # TODO: HueAuthException? raise Exception("The user does not have proper Hue permissions to update Navigator properties.") return JsonResponse(api.update_properties(entity_id, properties, modified_custom_metadata, deleted_custom_metadata_keys))
def update_properties(request): response = {'status': -1} api = NavigatorApi(request.user) entity_id = json.loads(request.POST.get('id', '""')) properties = json.loads(request.POST.get('properties', '{}')) # Entity properties metadata = json.loads(request.POST.get('metadata', '{}')) # Aka "Custom Metadata" is_allowed = request.user.has_hue_permission(action='write', app='metadata') request.audit = { 'allowed': is_allowed, 'operation': 'NAVIGATOR_UPDATE_PROPERTIES', 'operationText': 'Updating metadata %s properties %s of entity %s' % (metadata, properties, entity_id) } if not entity_id or (not properties and not metadata) or not is_allowed: response['error'] = _( "update_properties requires an 'id' parameter and 'properties' or 'metadata' parameters that are non-empty dicts" ) else: response['entity'] = api.update_properties(entity_id, properties, metadata) response['status'] = 0 return JsonResponse(response)
def delete_tags(request): api = NavigatorApi(request.user) entity_id = json.loads(request.POST.get('id', '""')) tags = json.loads(request.POST.get('tags', '[]')) is_allowed = request.user.has_hue_permission(action='write', app='metadata') request.audit = { 'allowed': is_allowed, 'operation': 'NAVIGATOR_DELETE_TAG', 'operationText': 'Removing tags %s to entity %s' % (tags, entity_id) } if not is_allowed: raise Exception( "The user does not have proper Hue permissions to delete Navigator tags." ) if not entity_id: raise Exception( "Missing required parameter 'id' for the Hue delete_tags API.") if not tags: raise Exception( "Missing required parameter 'tags' for the Hue delete_tags API.") return JsonResponse(api.delete_tags(entity_id, tags))
def delete_tags(request): response = {'status': -1} api = NavigatorApi(request.user) entity_id = json.loads(request.POST.get('id', '')) tags = json.loads(request.POST.get('tags', [])) is_allowed = request.user.has_hue_permission(action='write', app='metadata') request.audit = { 'allowed': is_allowed, 'operation': 'NAVIGATOR_DELETE_TAG', 'operationText': 'Removing tags %s to entity %s' % (tags, entity_id) } if not entity_id or not tags or not isinstance(tags, list) or not is_allowed: response['error'] = _( "add_tags requires an 'id' parameter and 'tags' parameter that is a non-empty list of tags" ) else: response['entity'] = api.delete_tags(entity_id, tags) response['status'] = 0 return JsonResponse(response)
def search_entities(request): """ For displaying results. """ api = NavigatorApi(request.user) query_s = json.loads(request.POST.get('query_s', '')) query_s = smart_str(query_s) offset = request.POST.get('offset', 0) limit = request.POST.get('limit', 100) sources = json.loads(request.POST.get('sources')) or [] query_s = query_s.strip() or '*' entities = api.search_entities(query_s, limit=limit, offset=offset, sources=sources) _augment_highlighting(query_s, entities) response = { 'entities': entities, 'count': len(entities), 'offset': offset, 'limit': limit, 'query_s': query_s, 'status': 0 } return JsonResponse(response)
def update_properties(request): response = {'status': -1} api = NavigatorApi(request.user) entity_id = json.loads(request.POST.get('id', '')) properties = json.loads(request.POST.get('properties', {})) is_allowed = request.user.has_hue_permission(action='write', app='metadata') request.audit = { 'allowed': is_allowed, 'operation': 'NAVIGATOR_UPDATE_PROPERTIES', 'operationText': 'Updating property %s of entity %s' % (properties, entity_id) } if not entity_id or not properties or not isinstance( properties, dict) or not is_allowed: response['error'] = _( "update_properties requires an 'id' parameter and 'properties' parameter that is a non-empty dict" ) else: response['entity'] = api.update_properties(entity_id, properties) response['status'] = 0 return JsonResponse(response)
def delete_properties(request): response = {'status': -1} api = NavigatorApi(request.user) entity_id = json.loads(request.POST.get('id', '')) keys = json.loads(request.POST.get('keys', [])) is_allowed = request.user.has_hue_permission(action='write', app='metadata') request.audit = { 'allowed': is_allowed, 'operation': 'NAVIGATOR_DELETE_PROPERTIES', 'operationText': 'Deleting property %s of entity %s' % (keys, entity_id) } if not entity_id or not keys or not isinstance(keys, list): response['error'] = _( "update_properties requires an 'id' parameter and 'keys' parameter that is a non-empty list" ) else: response['entity'] = api.delete_properties(entity_id, keys) response['status'] = 0 return JsonResponse(response)
def search_entities(request): api = NavigatorApi() query_s = request.POST.get('query_s', '') offset = request.POST.get('offset', 0) limit = request.POST.get('limit', 100) sources = json.loads(request.POST.get('sources')) or [] query_s = query_s.strip() or '*' entities = api.search_entities(query_s, limit=limit, offset=offset, sources=sources) response = { 'entities': entities, 'count': len(entities), 'offset': offset, 'limit': limit, 'query_s': query_s, 'status': 0 } return JsonResponse(response)
def test_search_entities(self): api = NavigatorApi(self.user) api._root = MockedRoot() assert_equal( '((originalName:*cases*)OR(originalDescription:*cases*)OR(name:*cases*)OR(description:*cases*)OR(tags:*cases*)) AND (*) AND ((type:TABLE)OR(type:VIEW))', api.search_entities(query_s='cases', sources=['hive'])[0][1]) assert_equal( '* AND ((type:FIELD))', api.search_entities(query_s='type:FIELD', sources=['hive'])[0][1])
def suggest(request): response = {'status': -1} api = NavigatorApi(request.user) prefix = request.POST.get('prefix') suggest = api.suggest(prefix) response['suggest'] = suggest response['status'] = 0 return JsonResponse(response)
def test_search_entities(self): api = NavigatorApi(self.user) api._root = MockedRoot() assert_equal( '((originalName:*cases*)OR(originalDescription:*cases*)OR(name:*cases*)OR(description:*cases*)OR(tags:*cases*)) AND (*) AND ((type:TABLE)OR(type:VIEW))', api.search_entities(query_s='cases', sources=['hive'])[0][1] ) assert_equal( '* AND ((type:FIELD))', api.search_entities(query_s='type:FIELD', sources=['hive'])[0][1] )
def map_namespace_property(request): """ { namespace: "huecatalog", name: "relatedEntities" }""" api = NavigatorApi(request.user) clazz = request.POST.get('class') properties = json.loads(request.POST.get('properties', '[]')) namespace = api.map_namespace_property(clazz=clazz, properties=properties) return JsonResponse(namespace)
def update_properties(request): response = {'status': -1} api = NavigatorApi() entity_id = json.loads(request.POST.get('id', '')) properties = json.loads(request.POST.get('properties', {})) if not entity_id or not properties or not isinstance(properties, dict): response['error'] = _("update_properties requires an 'id' parameter and 'properties' parameter that is a non-empty dict") else: response['entity'] = api.update_properties(entity_id, properties) response['status'] = 0 return JsonResponse(response)
def delete_properties(request): response = {'status': -1} api = NavigatorApi() entity_id = json.loads(request.POST.get('id', '')) keys = json.loads(request.POST.get('keys', [])) if not entity_id or not keys or not isinstance(keys, list): response['error'] = _("update_properties requires an 'id' parameter and 'keys' parameter that is a non-empty list") else: response['entity'] = api.delete_properties(entity_id, keys) response['status'] = 0 return JsonResponse(response)
def delete_tags(request): response = {'status': -1} api = NavigatorApi() entity_id = json.loads(request.POST.get('id', '')) tags = json.loads(request.POST.get('tags', [])) if not entity_id or not tags or not isinstance(tags, list): response['error'] = _("add_tags requires an 'id' parameter and 'tags' parameter that is a non-empty list of tags") else: response['entity'] = api.delete_tags(entity_id, tags) response['status'] = 0 return JsonResponse(response)
def create_namespace(request): api = NavigatorApi(request.user) namespace = request.POST.get('namespace') description = request.POST.get('description') request.audit = { 'allowed': request.user.has_hue_permission(action='write', app='metadata'), 'operation': 'NAVIGATOR_CREATE_NAMESPACE', 'operationText': 'Creating namespace %s' % namespace } namespace = api.create_namespace(namespace=namespace, description=description) return JsonResponse(namespace)
def get_entity(request): response = {'status': -1} api = NavigatorApi() entity_id = request.REQUEST.get('id') if not entity_id: raise MetadataApiException("get_entity requires an 'id' parameter") entity = api.get_entity(entity_id) response['entity'] = entity response['status'] = 0 return JsonResponse(response)
def list_tags(request): api = NavigatorApi(request.user) prefix = request.POST.get('prefix') offset = request.POST.get('offset', 0) limit = request.POST.get('limit', 25) data = api.search_entities_interactive(facetFields=['tags'], facetPrefix=prefix, limit=limit, offset=offset) response = { 'tags': data['facets']['tags'], 'status': 0 } return JsonResponse(response)
def get_entity(request): response = {'status': -1} api = NavigatorApi(request.user) entity_id = request.REQUEST.get('id') if not entity_id: raise MetadataApiException("get_entity requires an 'id' parameter") entity = api.get_entity(entity_id) response['entity'] = entity response['status'] = 0 return JsonResponse(response)
def list_tags(request): api = NavigatorApi(request.user) prefix = request.POST.get('prefix') offset = request.POST.get('offset', 0) limit = request.POST.get('limit', 25) data = api.search_entities_interactive(facetFields=['tags'], facetPrefix=prefix, limit=limit, offset=offset) response = {'tags': data['facets']['tags'], 'status': 0} return JsonResponse(response)
def setup_class(cls): cls.client = make_logged_in_client(username='******', is_superuser=False) cls.user = User.objects.get(username='******') cls.user = rewrite_user(cls.user) add_to_group('test') grant_access("test", "test", "metadata") cls.api = NavigatorApi(cls.user) cls.api._root = MockedRoot()
def setup_class(cls): cls.client = make_logged_in_client(username='******', is_superuser=False) cls.user = User.objects.get(username='******') cls.user = rewrite_user(cls.user) add_to_group('test') grant_access("test", "test", "metadata") if not is_live_cluster() or not has_navigator(cls.user): raise SkipTest cls.api = NavigatorApi(cls.user)
def create_namespace_property(request): """ { "name" : "relatedEntities", "displayName" : "Related objects", "creator" : "admin", "description" : "My desc",, "multiValued" : true, "maxLength" : 50, "pattern" : ".*", "enumValues" : null, "type" : "TEXT", "createdDate" : "2018-04-02T22:36:19.001Z" }""" api = NavigatorApi(request.user) namespace = request.POST.get('namespace') properties = json.loads(request.POST.get('properties', '{}')) namespace = api.create_namespace_property(namespace, properties) return JsonResponse(namespace)
def search_entities(request): response = {'status': -1} api = NavigatorApi() query_s = request.GET.get('query_s') offset = request.GET.get('offset', 0) limit = request.GET.get('limit', 100) query_s = query_s.strip() entities = api.search_entities(query_s) response = { 'entities': entities, 'count': len(entities), 'offset': offset, 'limit': limit, 'query_s': query_s, 'status': 0 } return JsonResponse(response)
def search_entities(request): api = NavigatorApi() query_s = request.POST.get('query_s') offset = request.POST.get('offset', 0) limit = request.POST.get('limit', 100) sources = json.loads(request.POST.get('sources'), []) query_s = query_s.strip() entities = api.search_entities(query_s, sources=sources) response = { 'entities': entities, 'count': len(entities), 'offset': offset, 'limit': limit, 'query_s': query_s, 'status': 0 } return JsonResponse(response)
def find_entity(request): response = {'status': -1} api = NavigatorApi(request.user) entity_type = request.GET.get('type', '') database = request.GET.get('database', '') table = request.GET.get('table', '') name = request.GET.get('name', '') path = request.GET.get('path', '') # TODO: support arbitrary optional filter params if not entity_type: raise MetadataApiException( "find_entity requires a type value, e.g. - 'database', 'table', 'file'" ) if entity_type.lower() == 'database': if not name: raise MetadataApiException('get_database requires name param') response['entity'] = api.get_database(name) elif entity_type.lower() == 'table': if not database or not name: raise MetadataApiException( 'get_table requires database and name param') response['entity'] = api.get_table(database, name) elif entity_type.lower() == 'field': if not database or not table or not name: raise MetadataApiException( 'get_field requires database, table, and name params') response['entity'] = api.get_field(database, table, name) elif entity_type.lower() == 'directory': if not path: raise MetadataApiException('get_directory requires path param') response['entity'] = api.get_directory(path) elif entity_type.lower() == 'file': if not path: raise MetadataApiException('get_file requires path param') response['entity'] = api.get_file(path) else: raise MetadataApiException("type %s is unrecognized" % entity_type) # Prevent nulls later if 'tags' in response['entity'] and not response['entity']['tags']: response['entity']['tags'] = [] response['status'] = 0 return JsonResponse(response)
def find_entity(request): response = {'status': -1} api = NavigatorApi() entity_type = request.GET.get('type', '') database = request.GET.get('database', '') table = request.GET.get('table', '') name = request.GET.get('name', '') path = request.GET.get('path', '') # TODO: support arbitrary optional filter params if not entity_type: raise MetadataApiException("find_entity requires a type value, e.g. - 'database', 'table', 'file'") if entity_type.lower() == 'database': if not name: raise MetadataApiException('get_database requires name param') response['entity'] = api.get_database(name) elif entity_type.lower() == 'table': if not database or not name: raise MetadataApiException('get_table requires database and name param') response['entity'] = api.get_table(database, name) elif entity_type.lower() == 'field': if not database or not table or not name: raise MetadataApiException('get_field requires database, table, and name params') response['entity'] = api.get_field(database, table, name) elif entity_type.lower() == 'directory': if not path: raise MetadataApiException('get_directory requires path param') response['entity'] = api.get_directory(path) elif entity_type.lower() == 'file': if not path: raise MetadataApiException('get_file requires path param') response['entity'] = api.get_file(path) else: raise MetadataApiException("type %s is unrecognized" % entity_type) # Prevent nulls later if 'tags' in response['entity'] and not response['entity']['tags']: response['entity']['tags'] = [] response['status'] = 0 return JsonResponse(response)
def search_entities_interactive(request): """ For search autocomplete. """ api = NavigatorApi(request.user) query_s = json.loads(request.POST.get('query_s', '')) prefix = request.POST.get('prefix') offset = request.POST.get('offset', 0) limit = request.POST.get('limit', 25) field_facets = json.loads(request.POST.get('field_facets') or '[]') sources = json.loads(request.POST.get('sources') or '[]') f = { "outputFormat" : { "type" : "dynamic" }, "name" : { "type" : "dynamic" }, "lastModified" : { "type" : "date" }, "sourceType" : { "type" : "dynamic" }, "parentPath" : { "type" : "dynamic" }, "deleteTime" : { "type" : "date" }, "lastAccessed" : { "type" : "date" }, "type" : { "type" : "dynamic" }, "sourceId" : { "type" : "dynamic" }, "partitionColNames" : { "type" : "dynamic" }, "serDeName" : { "type" : "dynamic" }, "created" : { "type" : "date" }, "fileSystemPath" : { "type" : "dynamic" }, "compressed" : { "type" : "bool" }, "clusteredByColNames" : { "type" : "dynamic" }, "deleted" : { "type" : "bool" }, "originalName" : { "type" : "dynamic" }, "owner" : { "type" : "dynamic" }, "extractorRunId" : { "type" : "dynamic" }, "userEntity" : { "type" : "bool" }, "sortByColNames" : { "type" : "dynamic" }, "inputFormat" : { "type" : "dynamic" }, "serDeLibName" : { "type" : "dynamic" }, "originalDescription" : { "type" : "dynamic" }, "lastModifiedBy" : { "type" : "dynamic" } } field_facets = ["tags", "type"] + f.keys() query_s = query_s.strip() + '*' last_query_term = [term for term in query_s.split()][-1] if last_query_term and last_query_term != '*': last_query_term = last_query_term.rstrip('*') (fname, fval) = last_query_term.split(':') if ':' in last_query_term else (last_query_term, '') field_facets = [f for f in field_facets if f.startswith(fname)] field_facets = field_facets[:5] response = api.search_entities_interactive( query_s=query_s, limit=limit, offset=offset, facetFields=field_facets, facetPrefix=prefix, facetRanges=None, firstClassEntitiesOnly=None, sources=sources ) if response.get('facets'): # Remove empty facets for fname, fvalues in response['facets'].items(): fvalues = sorted([(k, v) for k, v in fvalues.items() if v > 0], key=lambda n: n[1], reverse=True) response['facets'][fname] = OrderedDict(fvalues) if ':' in query_s and not response['facets'][fname]: del response['facets'][fname] _augment_highlighting(query_s, response.get('results')) response['status'] = 0 return JsonResponse(response)
def search_entities_interactive(request): """ For search autocomplete. """ api = NavigatorApi(request.user) query_s = json.loads(request.POST.get('query_s', '')) prefix = request.POST.get('prefix') offset = request.POST.get('offset', 0) limit = int(request.POST.get('limit', 25)) field_facets = json.loads(request.POST.get('field_facets') or '[]') sources = json.loads(request.POST.get('sources') or '[]') f = { "outputFormat": { "type": "dynamic" }, "name": { "type": "dynamic" }, "lastModified": { "type": "date" }, "sourceType": { "type": "dynamic" }, "parentPath": { "type": "dynamic" }, "lastAccessed": { "type": "date" }, "type": { "type": "dynamic" }, "sourceId": { "type": "dynamic" }, "partitionColNames": { "type": "dynamic" }, "serDeName": { "type": "dynamic" }, "created": { "type": "date" }, "fileSystemPath": { "type": "dynamic" }, "compressed": { "type": "bool" }, "clusteredByColNames": { "type": "dynamic" }, "originalName": { "type": "dynamic" }, "owner": { "type": "dynamic" }, "extractorRunId": { "type": "dynamic" }, "userEntity": { "type": "bool" }, "sortByColNames": { "type": "dynamic" }, "inputFormat": { "type": "dynamic" }, "serDeLibName": { "type": "dynamic" }, "originalDescription": { "type": "dynamic" }, "lastModifiedBy": { "type": "dynamic" } } field_facets = ["tags", "type"] + f.keys() query_s = query_s.strip() + '*' last_query_term = [term for term in query_s.split()][-1] if last_query_term and last_query_term != '*': last_query_term = last_query_term.rstrip('*') (fname, fval) = last_query_term.split(':') if ':' in last_query_term else ( last_query_term, '') field_facets = [f for f in field_facets if f.startswith(fname)] field_facets = field_facets[:5] response = api.search_entities_interactive(query_s=query_s, limit=limit, offset=offset, facetFields=field_facets, facetPrefix=prefix, facetRanges=None, firstClassEntitiesOnly=None, sources=sources) if response.get('facets'): # Remove empty facets for fname, fvalues in response['facets'].items(): if NAVIGATOR.APPLY_SENTRY_PERMISSIONS.get(): fvalues = [] else: fvalues = sorted([(k, v) for k, v in fvalues.items() if v > 0], key=lambda n: n[1], reverse=True) response['facets'][fname] = OrderedDict(fvalues) if ':' in query_s and not response['facets'][fname]: del response['facets'][fname] _augment_highlighting(query_s, response.get('results')) response['status'] = 0 return JsonResponse(response)
def get_model_properties_mapping(request): api = NavigatorApi(request.user) namespace = api.get_model_properties_mapping() return JsonResponse(namespace)