def get_database(self, name): # Search with Atlas API for hive database with specific name if get_catalog_search_cluster(): qualifiedNameCriteria = 'qualifiedName=\'%s@%s\'' % (name, get_catalog_search_cluster()) else: qualifiedNameCriteria = 'qualifiedName like \'%s@*\'' % name return self.fetch_single_entity('hive_db where %s' % qualifiedNameCriteria)
def get_field(self, database_name, table_name, field_name): # Search with Atlas API for hive tables with specific qualified name if get_catalog_search_cluster(): qualifiedNameCriteria = 'qualifiedName=\'%s.%s.%s@%s\'' % (database_name, table_name, field_name, get_catalog_search_cluster()) else: qualifiedNameCriteria = 'qualifiedName like \'%s.%s.%s@*\'' % (database_name, table_name, field_name) return self.fetch_single_entity('hive_column where %s' % qualifiedNameCriteria)
def search_entities(self, query_s, limit=100, offset=0, raw_query=False, **filters): try: found_entities = [] search_terms = [term for term in query_s.strip().split() ] if query_s else [] parentPath = None for term in search_terms: if 'parentPath:' in term: name, val = term.split(':') parentPath = val.strip('"').lstrip('/').replace('/', '.') if query_s == 'type:database': if get_catalog_search_cluster(): atlas_dsl_query = 'from hive_db where qualifiedName like \'*@%s\' limit %s' % ( get_catalog_search_cluster(), limit) else: atlas_dsl_query = 'from hive_db limit %s' % limit elif not parentPath: return found_entities else: atlas_type = 'hive_table' if parentPath.count( '.') == 0 else 'hive_column' if get_catalog_search_cluster(): atlas_dsl_query = 'from %s where qualifiedName like \'%s*@%s\' limit %s' % ( atlas_type, parentPath, get_catalog_search_cluster(), limit) else: atlas_dsl_query = 'from %s where qualifiedName like \'%s*\' limit %s' % ( atlas_type, parentPath, limit) if CATALOG.ENABLE_BASIC_SEARCH.get(): atlas_response = self._root.get('/v2/search/basic?query=%s' % atlas_dsl_query) else: atlas_response = self._root.get('/v2/search/dsl?query=%s' % atlas_dsl_query) # Adapt Atlas entities to Navigator structure in the results if 'entities' in atlas_response: for atlas_entity in atlas_response['entities']: found_entities.append( self.adapt_atlas_entity_to_navigator(atlas_entity)) return found_entities except RestException as e: if e.code == 401: raise raise_popup_exception( 'Hue could not authenticate to Atlas', detail=e) else: raise raise_popup_exception('Hue could not query Atlas', detail=e)
def search_entities(self, query_s, limit=100, offset=0, raw_query=False, **filters): try: found_entities = [] search_terms = [term for term in query_s.strip().split() ] if query_s else [] parentPath = None for term in search_terms: if 'parentPath:' in term: name, val = term.split(':') parentPath = val.strip('"').lstrip('/').replace('/', '.') if query_s == 'type:database': if get_catalog_search_cluster(): atlas_dsl_query = 'from hive_db where qualifiedName like \'*@%s\' limit %s' % ( get_catalog_search_cluster(), limit) else: atlas_dsl_query = 'from hive_db limit %s' % limit elif not parentPath: return found_entities else: atlas_type = 'hive_table' if parentPath.count( '.') == 0 else 'hive_column' if get_catalog_search_cluster(): atlas_dsl_query = 'from %s where qualifiedName like \'%s*@%s\' limit %s' % ( atlas_type, parentPath, get_catalog_search_cluster(), limit) else: atlas_dsl_query = 'from %s where qualifiedName like \'%s*\' limit %s' % ( atlas_type, parentPath, limit) atlas_response = self._root.get('/v2/search/dsl?query=%s' % atlas_dsl_query) # Adapt Atlas entities to Navigator structure in the results if 'entities' in atlas_response: for atlas_entity in atlas_response['entities']: found_entities.append( self.adapt_atlas_entity_to_navigator(atlas_entity)) return found_entities except RestException as e: LOG.error('Failed to search for entities with search query: %s' % atlas_dsl_query) if e.code == 401: raise CatalogAuthException(_('Failed to authenticate.')) else: raise CatalogApiException(e.message)
def search_entities_interactive(self, query_s=None, limit=100, offset=0, facetFields=None, facetPrefix=None, facetRanges=None, filterQueries=None, firstClassEntitiesOnly=None, sources=None): response = { "status": 0, "results": [], "facets": { "tags": {} } } # This takes care of the list_tags endpoint if not query_s and facetFields and 'tags' in facetFields: classification_response = self._root.get('/v2/types/typedefs?type=classification') for classification_def in classification_response['classificationDefs']: if ' ' in classification_def['name']: response['facets']['tags']['"' + classification_def['name'] + '"'] = -1 else: response['facets']['tags'][classification_def['name']] = -1 return response query_s = (query_s.strip() if query_s else '').replace('*', '') atlas_type = None classification = None owner = None # Take the first classification and type facets and ignore other as we can't search multiple in Atlas. classification_facets = self.CLASSIFICATION_RE.findall(query_s) if classification_facets: classification = classification_facets[0][0] or classification_facets[0][1] query_s = self.CLASSIFICATION_RE.sub('', query_s).strip() atlas_type = 'Asset' # Filtered below to just contain hive_db, hive_table or hive_column owner_facets = self.OWNER_RE.findall(query_s) if owner_facets: owner = owner_facets[0] query_s = self.OWNER_RE.sub('', query_s).strip() type_facets = self.TYPE_RE.findall(query_s) if type_facets: atlas_type = self.NAV_TO_ATLAS_TYPE[type_facets[0].lower()] or type_facets[0] query_s = self.TYPE_RE.sub('', query_s).strip() data = { 'attributes': None, 'classification': classification, 'entityFilters': { 'condition': 'AND', 'criterion': [{ 'condition': 'OR', 'criterion': [{ 'attributeName': 'name', 'attributeValue': query_s, 'operator': 'contains' }, { 'attributeName': 'description', 'attributeValue': query_s, 'operator': 'contains' }] }] }, 'excludeDeletedEntities': True, 'includeClassificationAttributes': True, 'includeSubClassifications': True, 'includeSubTypes': True, 'limit': limit, 'offset': 0, 'tagFilters': None, 'termName': None, 'typeName': atlas_type or 'hive_table' } if get_catalog_search_cluster(): data['entityFilters']['criterion'].append({ 'attributeName': 'qualifiedName', 'operator': 'contains', 'attributeValue': '@' + get_catalog_search_cluster() }) if owner: data['entityFilters']['criterion'].append({ 'attributeName': 'owner', 'operator': 'startsWith', 'attributeValue': owner }) try: atlas_response = self._root.post('/v2/search/basic', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) # Adapt Atlas entities to Navigator structure in the results if 'entities' in atlas_response: for atlas_entity in atlas_response['entities']: if atlas_type != 'Asset' or atlas_entity['typeName'].lower() in ['hive_db', 'hive_table', 'hive_column']: response['results'].append(self.adapt_atlas_entity_to_navigator(atlas_entity)) return response except RestException as e: LOG.error('Failed to search for entities with search query: %s' % data) if e.code == 401: raise CatalogAuthException(_('Failed to authenticate.')) else: raise CatalogApiException(e.message)