def get_database(self, name):
# Search with Atlas API for hive database with specific name
if get_catalog_search_cluster():
qualifiedNameCriteria = 'qualifiedName=\'%s@%s\'' % (name, get_catalog_search_cluster())
else:
qualifiedNameCriteria = 'qualifiedName like \'%s@*\'' % name
return self.fetch_single_entity('hive_db where %s' % qualifiedNameCriteria)
def get_field(self, database_name, table_name, field_name):
# Search with Atlas API for hive tables with specific qualified name
if get_catalog_search_cluster():
qualifiedNameCriteria = 'qualifiedName=\'%s.%s.%s@%s\'' % (database_name, table_name, field_name,
get_catalog_search_cluster())
else:
qualifiedNameCriteria = 'qualifiedName like \'%s.%s.%s@*\'' % (database_name, table_name, field_name)
return self.fetch_single_entity('hive_column where %s' % qualifiedNameCriteria)
def search_entities(self,
query_s,
limit=100,
offset=0,
raw_query=False,
**filters):
try:
found_entities = []
search_terms = [term for term in query_s.strip().split()
] if query_s else []
parentPath = None
for term in search_terms:
if 'parentPath:' in term:
name, val = term.split(':')
parentPath = val.strip('"').lstrip('/').replace('/', '.')
if query_s == 'type:database':
if get_catalog_search_cluster():
atlas_dsl_query = 'from hive_db where qualifiedName like \'*@%s\' limit %s' % (
get_catalog_search_cluster(), limit)
else:
atlas_dsl_query = 'from hive_db limit %s' % limit
elif not parentPath:
return found_entities
else:
atlas_type = 'hive_table' if parentPath.count(
'.') == 0 else 'hive_column'
if get_catalog_search_cluster():
atlas_dsl_query = 'from %s where qualifiedName like \'%s*@%s\' limit %s' % (
atlas_type, parentPath, get_catalog_search_cluster(),
limit)
else:
atlas_dsl_query = 'from %s where qualifiedName like \'%s*\' limit %s' % (
atlas_type, parentPath, limit)
if CATALOG.ENABLE_BASIC_SEARCH.get():
atlas_response = self._root.get('/v2/search/basic?query=%s' %
atlas_dsl_query)
else:
atlas_response = self._root.get('/v2/search/dsl?query=%s' %
atlas_dsl_query)
# Adapt Atlas entities to Navigator structure in the results
if 'entities' in atlas_response:
for atlas_entity in atlas_response['entities']:
found_entities.append(
self.adapt_atlas_entity_to_navigator(atlas_entity))
return found_entities
except RestException as e:
if e.code == 401:
raise raise_popup_exception(
'Hue could not authenticate to Atlas', detail=e)
else:
raise raise_popup_exception('Hue could not query Atlas',
detail=e)
def search_entities(self,
query_s,
limit=100,
offset=0,
raw_query=False,
**filters):
try:
found_entities = []
search_terms = [term for term in query_s.strip().split()
] if query_s else []
parentPath = None
for term in search_terms:
if 'parentPath:' in term:
name, val = term.split(':')
parentPath = val.strip('"').lstrip('/').replace('/', '.')
if query_s == 'type:database':
if get_catalog_search_cluster():
atlas_dsl_query = 'from hive_db where qualifiedName like \'*@%s\' limit %s' % (
get_catalog_search_cluster(), limit)
else:
atlas_dsl_query = 'from hive_db limit %s' % limit
elif not parentPath:
return found_entities
else:
atlas_type = 'hive_table' if parentPath.count(
'.') == 0 else 'hive_column'
if get_catalog_search_cluster():
atlas_dsl_query = 'from %s where qualifiedName like \'%s*@%s\' limit %s' % (
atlas_type, parentPath, get_catalog_search_cluster(),
limit)
else:
atlas_dsl_query = 'from %s where qualifiedName like \'%s*\' limit %s' % (
atlas_type, parentPath, limit)
atlas_response = self._root.get('/v2/search/dsl?query=%s' %
atlas_dsl_query)
# Adapt Atlas entities to Navigator structure in the results
if 'entities' in atlas_response:
for atlas_entity in atlas_response['entities']:
found_entities.append(
self.adapt_atlas_entity_to_navigator(atlas_entity))
return found_entities
except RestException as e:
LOG.error('Failed to search for entities with search query: %s' %
atlas_dsl_query)
if e.code == 401:
raise CatalogAuthException(_('Failed to authenticate.'))
else:
raise CatalogApiException(e.message)
def search_entities_interactive(self, query_s=None, limit=100, offset=0, facetFields=None, facetPrefix=None, facetRanges=None, filterQueries=None, firstClassEntitiesOnly=None, sources=None):
response = {
"status": 0,
"results": [],
"facets": {
"tags": {}
}
}
# This takes care of the list_tags endpoint
if not query_s and facetFields and 'tags' in facetFields:
classification_response = self._root.get('/v2/types/typedefs?type=classification')
for classification_def in classification_response['classificationDefs']:
if ' ' in classification_def['name']:
response['facets']['tags']['"' + classification_def['name'] + '"'] = -1
else:
response['facets']['tags'][classification_def['name']] = -1
return response
query_s = (query_s.strip() if query_s else '').replace('*', '')
atlas_type = None
classification = None
owner = None
# Take the first classification and type facets and ignore other as we can't search multiple in Atlas.
classification_facets = self.CLASSIFICATION_RE.findall(query_s)
if classification_facets:
classification = classification_facets[0][0] or classification_facets[0][1]
query_s = self.CLASSIFICATION_RE.sub('', query_s).strip()
atlas_type = 'Asset' # Filtered below to just contain hive_db, hive_table or hive_column
owner_facets = self.OWNER_RE.findall(query_s)
if owner_facets:
owner = owner_facets[0]
query_s = self.OWNER_RE.sub('', query_s).strip()
type_facets = self.TYPE_RE.findall(query_s)
if type_facets:
atlas_type = self.NAV_TO_ATLAS_TYPE[type_facets[0].lower()] or type_facets[0]
query_s = self.TYPE_RE.sub('', query_s).strip()
data = {
'attributes': None,
'classification': classification,
'entityFilters': {
'condition': 'AND',
'criterion': [{
'condition': 'OR',
'criterion': [{
'attributeName': 'name',
'attributeValue': query_s,
'operator': 'contains'
}, {
'attributeName': 'description',
'attributeValue': query_s,
'operator': 'contains'
}]
}]
},
'excludeDeletedEntities': True,
'includeClassificationAttributes': True,
'includeSubClassifications': True,
'includeSubTypes': True,
'limit': limit,
'offset': 0,
'tagFilters': None,
'termName': None,
'typeName': atlas_type or 'hive_table'
}
if get_catalog_search_cluster():
data['entityFilters']['criterion'].append({
'attributeName': 'qualifiedName',
'operator': 'contains',
'attributeValue': '@' + get_catalog_search_cluster()
})
if owner:
data['entityFilters']['criterion'].append({
'attributeName': 'owner',
'operator': 'startsWith',
'attributeValue': owner
})
try:
atlas_response = self._root.post('/v2/search/basic', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
# Adapt Atlas entities to Navigator structure in the results
if 'entities' in atlas_response:
for atlas_entity in atlas_response['entities']:
if atlas_type != 'Asset' or atlas_entity['typeName'].lower() in ['hive_db', 'hive_table', 'hive_column']:
response['results'].append(self.adapt_atlas_entity_to_navigator(atlas_entity))
return response
except RestException as e:
LOG.error('Failed to search for entities with search query: %s' % data)
if e.code == 401:
raise CatalogAuthException(_('Failed to authenticate.'))
else:
raise CatalogApiException(e.message)
