def update_entity(self, entity, **metadata):
"""
PUT /api/v3/entities/:id
http://cloudera.github.io/navigator/apidocs/v3/path__v3_entities_-id-.html
"""
try:
# Workarounds NAV-6187: if we don't re-send those, they would get erased.
properties = {
'name': entity['name'],
'description': entity['description'],
'properties': entity['properties'] or {},
'customProperties': entity['customProperties'] or {}
}
properties.update(metadata)
data = json.dumps(properties)
return self._root.put('entities/%(identity)s' % entity,
params=self.__params,
data=data,
contenttype=_JSON_CONTENT_TYPE,
allow_redirects=True,
clear_cookies=True)
except RestException as e:
if e.code == 401:
raise raise_popup_exception(
'Hue could not authenticate to Atlas', detail=e)
else:
raise raise_popup_exception('Failed to update entity',
detail=e)
def _execute_query(proposed_query, limit):
user = install_sample_user()
query_server = get_query_server_config('beeswax')
server = dbms.get(user, query_server)
query = hql_query(proposed_query)
try:
handle = server.execute_and_wait(query)
if not handle:
LOG.error(_('Error executing %s: Operation timeout.' % query))
return []
results = server.fetch(handle, True, limit)
rows = [row for row in results.rows()]
data = {
'data': rows,
'columns': [column.name for column in results.data_table.cols()]
}
return data
except Exception as ex:
raise_popup_exception(_('Error fetching query history.'))
finally:
try:
if server and handle:
server.close(handle)
except Exception as ex:
raise_popup_exception(_('Error fetching query history.'))
def fetch_single_entity(self, dsl_query):
'''
REQUEST: hue:8889/metadata/api/navigator/find_entity?type=database&name=default
SAMPLE response for Navigator find_entity response
{"status": 0, "entity": {
"customProperties": null,
"deleteTime": null,
"fileSystemPath": "hdfs://nightly6x-1.vpc.cloudera.com:8020/user/hive/warehouse",
"description": null,
"params": null,
"type": "DATABASE",
"internalType": "hv_database",
"sourceType": "HIVE",
"tags": [],
"deleted": false, "technicalProperties": null,
"userEntity": false,
"originalDescription": "Default Hive database",
"metaClassName": "hv_database",
"properties": {"__cloudera_internal__hueLink": "https://nightly6x-1.vpc.cloudera.com:8889/hue/metastore/tables/default"},
"identity": "23",
"firstClassParentId": null,
"name": null,
"extractorRunId": "7##1",
"sourceId": "7",
"packageName": "nav",
"parentPath": null, "originalName": "default"}}
'''
response = {"status": 0, "entity": []}
try:
if CATALOG.ENABLE_BASIC_SEARCH.get():
atlas_response = self._root.get('/v2/search/basic?query=%s' %
dsl_query,
headers=self.__headers,
params=self.__params)
else:
atlas_response = self._root.get('/v2/search/dsl?query=%s' %
dsl_query,
headers=self.__headers,
params=self.__params)
if not 'entities' in atlas_response or len(
atlas_response['entities']) < 1:
raise CatalogEntityDoesNotExistException(
'Could not find entity with query: %s' % dsl_query)
for atlas_entity in atlas_response['entities']:
response['entity'].append(
self.adapt_atlas_entity_to_navigator(atlas_entity))
return response['entity'][0]
except RestException as e:
if e.code == 401:
raise raise_popup_exception(
'Hue could not authenticate to Atlas', detail=e)
else:
raise raise_popup_exception('Hue could not query Atlas',
detail=e)
def search_entities(self,
query_s,
limit=100,
offset=0,
raw_query=False,
**filters):
try:
found_entities = []
search_terms = [term for term in query_s.strip().split()
] if query_s else []
parentPath = None
for term in search_terms:
if 'parentPath:' in term:
name, val = term.split(':')
parentPath = val.strip('"').lstrip('/').replace('/', '.')
if query_s == 'type:database':
if get_catalog_search_cluster():
atlas_dsl_query = 'from hive_db where qualifiedName like \'*@%s\' limit %s' % (
get_catalog_search_cluster(), limit)
else:
atlas_dsl_query = 'from hive_db limit %s' % limit
elif not parentPath:
return found_entities
else:
atlas_type = 'hive_table' if parentPath.count(
'.') == 0 else 'hive_column'
if get_catalog_search_cluster():
atlas_dsl_query = 'from %s where qualifiedName like \'%s*@%s\' limit %s' % (
atlas_type, parentPath, get_catalog_search_cluster(),
limit)
else:
atlas_dsl_query = 'from %s where qualifiedName like \'%s*\' limit %s' % (
atlas_type, parentPath, limit)
if CATALOG.ENABLE_BASIC_SEARCH.get():
atlas_response = self._root.get('/v2/search/basic?query=%s' %
atlas_dsl_query)
else:
atlas_response = self._root.get('/v2/search/dsl?query=%s' %
atlas_dsl_query)
# Adapt Atlas entities to Navigator structure in the results
if 'entities' in atlas_response:
for atlas_entity in atlas_response['entities']:
found_entities.append(
self.adapt_atlas_entity_to_navigator(atlas_entity))
return found_entities
except RestException as e:
if e.code == 401:
raise raise_popup_exception(
'Hue could not authenticate to Atlas', detail=e)
else:
raise raise_popup_exception('Hue could not query Atlas',
detail=e)
def get_model_properties_mapping(self):
try:
return self._root.get('models/properties/mappings')
except RestException as e:
if e.code == 401:
raise raise_popup_exception(
'Hue could not authenticate to Atlas', detail=e)
else:
raise raise_popup_exception(
'Failed to get models properties mappings', detail=e)
def get_namespace(self, namespace):
try:
return self._root.get('models/namespaces/%(namespace)s' %
{'namespace': namespace})
except RestException as e:
if e.code == 401:
raise raise_popup_exception(
'Hue could not authenticate to Atlas', detail=e)
else:
raise raise_popup_exception('Failed to get namespace',
detail=e)
def suggest(self, prefix=None):
try:
return self._root.get('interactive/suggestions?query=%s' %
(prefix or '*'))
except RestException as e:
if e.code == 401:
raise raise_popup_exception(
'Hue could not authenticate to Atlas', detail=e)
else:
raise raise_popup_exception('Failed to search for entities',
detail=e)
def create_namespace(self, namespace, description=None):
try:
data = json.dumps({'name': namespace, 'description': description})
return self._root.post('models/namespaces/',
data=data,
contenttype=_JSON_CONTENT_TYPE,
clear_cookies=True)
except RestException as e:
if e.code == 401:
raise raise_popup_exception(
'Hue could not authenticate to Atlas', detail=e)
else:
raise raise_popup_exception('Failed to create namespace',
detail=e)
def map_namespace_property(self, clazz, properties):
try:
data = json.dumps(properties)
return self._root.post(
'models/packages/nav/classes/%(class)s/properties' %
{'class': clazz},
data=data,
contenttype=_JSON_CONTENT_TYPE,
clear_cookies=True)
except RestException as e:
if e.code == 401:
raise raise_popup_exception(
'Hue could not authenticate to Atlas', detail=e)
else:
raise raise_popup_exception('Failed to map class', detail=e)
def create_namespace_property(self, namespace, properties):
try:
data = json.dumps(properties)
return self._root.post(
'models/namespaces/%(namespace)s/properties' %
{'namespace': namespace},
data=data,
contenttype=_JSON_CONTENT_TYPE,
clear_cookies=True)
except RestException as e:
if e.code == 401:
raise raise_popup_exception(
'Hue could not authenticate to Atlas', detail=e)
else:
raise raise_popup_exception('Failed to create namespace',
detail=e)
def get_lineage(self, entity_id):
"""
GET /api/v3/lineage/entityIds=:id
http://cloudera.github.io/navigator/apidocs/v3/path__v3_lineage.html
"""
try:
params = self.__params
params += (('entityIds', entity_id), )
return self._root.get('lineage',
headers=self.__headers,
params=params)
except RestException as e:
if e.code == 401:
raise raise_popup_exception(
'Hue could not authenticate to Atlas', detail=e)
else:
raise raise_popup_exception('Failed to get lineage', detail=e)
def search_entities_interactive(self,
query_s=None,
limit=100,
offset=0,
facetFields=None,
facetPrefix=None,
facetRanges=None,
filterQueries=None,
firstClassEntitiesOnly=None,
sources=None):
response = {"status": 0, "results": [], "facets": {"tags": {}}}
# This takes care of the list_tags endpoint
if not query_s and facetFields and 'tags' in facetFields:
classification_response = self._root.get(
'/v2/types/typedefs?type=classification')
for classification_def in classification_response[
'classificationDefs']:
if ' ' in classification_def['name']:
response['facets']['tags']['"' +
classification_def['name'] +
'"'] = -1
else:
response['facets']['tags'][classification_def['name']] = -1
return response
query_s = (query_s.strip() if query_s else '').replace('*', '')
atlas_type = None
classification = None
owner = None
# Take the first classification and type facets and ignore other as we can't search multiple in Atlas.
classification_facets = self.CLASSIFICATION_RE.findall(query_s)
if classification_facets:
classification = classification_facets[0][
0] or classification_facets[0][1]
query_s = self.CLASSIFICATION_RE.sub('', query_s).strip()
atlas_type = 'Asset' # Filtered below to just contain hive_db, hive_table or hive_column
owner_facets = self.OWNER_RE.findall(query_s)
if owner_facets:
owner = owner_facets[0]
query_s = self.OWNER_RE.sub('', query_s).strip()
type_facets = self.TYPE_RE.findall(query_s)
if type_facets:
atlas_type = self.NAV_TO_ATLAS_TYPE[
type_facets[0].lower()] or type_facets[0]
query_s = self.TYPE_RE.sub('', query_s).strip()
data = {
'attributes': None,
'classification': classification,
'entityFilters': {
'condition':
'AND',
'criterion': [{
'condition':
'OR',
'criterion': [{
'attributeName': 'name',
'attributeValue': query_s,
'operator': 'contains'
}, {
'attributeName': 'description',
'attributeValue': query_s,
'operator': 'contains'
}]
}]
},
'excludeDeletedEntities': True,
'includeClassificationAttributes': True,
'includeSubClassifications': True,
'includeSubTypes': True,
'limit': limit,
'offset': 0,
'tagFilters': None,
'termName': None,
'typeName': atlas_type or 'hive_table'
}
if get_catalog_search_cluster():
data['entityFilters']['criterion'].append({
'attributeName':
'qualifiedName',
'operator':
'contains',
'attributeValue':
'@' + get_catalog_search_cluster()
})
if owner:
data['entityFilters']['criterion'].append({
'attributeName': 'owner',
'operator': 'startsWith',
'attributeValue': owner
})
try:
atlas_response = self._root.post('/v2/search/basic',
data=json.dumps(data),
contenttype=_JSON_CONTENT_TYPE)
# Adapt Atlas entities to Navigator structure in the results
if 'entities' in atlas_response:
for atlas_entity in atlas_response['entities']:
if atlas_type != 'Asset' or atlas_entity['typeName'].lower(
) in ['hive_db', 'hive_table', 'hive_column']:
response['results'].append(
self.adapt_atlas_entity_to_navigator(atlas_entity))
return response
except RestException as e:
if e.code == 401:
raise raise_popup_exception(
'Hue could not authenticate to Atlas', detail=e)
else:
raise raise_popup_exception('Hue could not query Atlas',
detail=e)