def create_session(self, lang='hive', properties=None):
application = 'beeswax' if lang == 'hive' or lang == 'llap' else lang
if has_session_pool():
session = Session.objects.get_tez_session(
self.user, application, MAX_NUMBER_OF_SESSIONS.get())
elif not has_multiple_sessions():
session = Session.objects.get_session(self.user,
application=application)
else:
session = None
reuse_session = session is not None
if not reuse_session:
db = dbms.get(self.user,
query_server=get_query_server_config(
name=lang, connector=self.interpreter))
session = db.open_session(self.user)
response = {'type': lang, 'id': session.id}
if not properties:
config = None
if USE_DEFAULT_CONFIGURATION.get():
config = DefaultConfiguration.objects.get_configuration_for_user(
app=lang, user=self.user)
if config is not None:
properties = config.properties_list
else:
properties = self.get_properties(lang)
response['properties'] = properties
response['configuration'] = json.loads(session.properties)
response['reuse_session'] = reuse_session
response['session_id'] = ''
try:
decoded_guid = session.get_handle().sessionId.guid
response['session_id'] = unpack_guid(decoded_guid)
except Exception as e:
LOG.warn('Failed to decode session handle: %s' % e)
if lang == 'impala' and session:
http_addr = _get_impala_server_url(session)
response['http_addr'] = http_addr
return response
class HS2Api(Api):
@staticmethod
def get_properties(lang='hive'):
return ImpalaConfiguration.PROPERTIES if lang == 'impala' else HiveConfiguration.PROPERTIES
@query_error_handler
def create_session(self, lang='hive', properties=None):
application = 'beeswax' if lang == 'hive' else lang
session = Session.objects.get_session(self.user, application=application)
reuse_session = session is not None
if not reuse_session:
session = dbms.get(self.user, query_server=get_query_server_config(name=lang, cluster=self.cluster)).open_session(self.user)
response = {
'type': lang,
'id': session.id
}
if not properties:
config = None
if USE_DEFAULT_CONFIGURATION.get():
config = DefaultConfiguration.objects.get_configuration_for_user(app=lang, user=self.user)
if config is not None:
properties = config.properties_list
else:
properties = self.get_properties(lang)
response['properties'] = properties
response['configuration'] = json.loads(session.properties)
response['reuse_session'] = reuse_session
response['session_id'] = ''
try:
decoded_guid = session.get_handle().sessionId.guid
response['session_id'] = unpack_guid(decoded_guid)
except Exception, e:
LOG.warn('Failed to decode session handle: %s' % e)
if lang == 'impala' and session:
http_addr = _get_impala_server_url(session)
response['http_addr'] = http_addr
return response
def alanize(request):
response = {'status': -1}
cluster = json.loads(request.POST.get('cluster', '{}'))
query_id = json.loads(request.POST.get('query_id'))
application = _get_server_name(cluster)
query_server = dbms.get_query_server_config()
session = Session.objects.get_session(request.user,
query_server['server_name'])
server_url = _get_impala_server_url(session)
if query_id:
LOG.debug(
"Attempting to get Impala query profile at server_url %s for query ID: %s"
% (server_url, query_id))
doc = Document2.objects.get(id=query_id)
snippets = doc.data_dict.get('snippets', [])
secret = snippets[0]['result']['handle']['secret']
api = get_impalad_api(user=request.user, url=server_url)
impala_query_id = "%x:%x" % struct.unpack(b"QQ",
base64.decodestring(secret))
api.kill(
impala_query_id
) # There are many statistics that are not present when the query is open. Close it first.
query_profile = api.get_query_profile_encoded(impala_query_id)
profile = analyzer.analyze(analyzer.parse_data(query_profile))
result = ANALYZER.run(profile)
heatmap = {}
summary = analyzer.summary(profile)
heatmapMetrics = [
'AverageThreadTokens', 'BloomFilterBytes', 'PeakMemoryUsage',
'PerHostPeakMemUsage', 'PrepareTime', 'RowsProduced',
'TotalCpuTime', 'TotalNetworkReceiveTime', 'TotalNetworkSendTime',
'TotalStorageWaitTime', 'TotalTime'
]
for key in heatmapMetrics:
heatmap[key] = analyzer.heatmap_by_host(profile, key)
response['data'] = {
'query': {
'healthChecks': result[0]['result'],
'summary': summary,
'heatmap': heatmap,
'heatmapMetrics': heatmapMetrics
}
}
response['status'] = 0
return JsonResponse(response)
def _get_impala_result_size(self, notebook, snippet):
total_records_match = None
total_records, total_size, msg = None, None, None
query_id = self._get_impala_query_id(snippet)
session = Session.objects.get_session(self.user, application='impala')
server_url = _get_impala_server_url(session)
if query_id:
LOG.info(
"Attempting to get Impala query profile at server_url %s for query ID: %s"
% (server_url, query_id))
fragment = self._get_impala_query_profile(server_url,
query_id=query_id)
total_records_re = "Coordinator Fragment F\d\d.+?RowsReturned: \d+(?:.\d+[KMB])? \((?P<total_records>\d+)\).*?(Averaged Fragment F\d\d)"
total_records_match = re.search(total_records_re, fragment,
re.MULTILINE | re.DOTALL)
if total_records_match:
total_records = int(total_records_match.group('total_records'))
return total_records, total_size, msg
def _get_api(user): session = Session.objects.get_session(user, application='impala') server_url = _get_impala_server_url(session) return get_impalad_api(user=user, url=server_url)
def _get_api(user): session = Session.objects.get_session(user, application='impala') server_url = _get_impala_server_url(session) return get_impalad_api(user=user, url=server_url)
def __init__(self, user):
self.user = user
session = Session.objects.get_session(self.user, application='impala')
self.server_url = _get_impala_server_url(session)
def create_session(self, lang='hive', properties=None):
application = 'beeswax' if lang == 'hive' or lang == 'llap' else lang
uses_session_pool = has_session_pool()
uses_multiple_sessions = has_multiple_sessions()
if lang == 'impala':
uses_session_pool = False
uses_multiple_sessions = False
try:
if uses_session_pool:
session = Session.objects.get_tez_session(
self.user, application, MAX_NUMBER_OF_SESSIONS.get())
elif not uses_multiple_sessions:
session = Session.objects.get_session(self.user,
application=application)
else:
session = None
except Exception as e:
if 'Connection refused' in str(
e) or 'Name or service not known' in str(e):
LOG.exception(
'Connection being refused or service is not available in either session or in multiple sessions'
'- HA failover')
reset_ha()
reuse_session = session is not None
if not reuse_session:
db = dbms.get(self.user,
query_server=get_query_server_config(
name=lang, connector=self.interpreter))
try:
session = db.open_session(self.user)
except Exception as e:
if 'Connection refused' in str(
e) or 'Name or service not known' in str(e):
LOG.exception(
'Connection being refused or service is not available in reuse session - HA failover'
)
reset_ha()
response = {'type': lang, 'id': session.id}
if not properties:
config = None
if USE_DEFAULT_CONFIGURATION.get():
config = DefaultConfiguration.objects.get_configuration_for_user(
app=lang, user=self.user)
if config is not None:
properties = config.properties_list
else:
properties = self.get_properties(lang)
response['properties'] = properties
response['configuration'] = json.loads(session.properties)
response['reuse_session'] = reuse_session
response['session_id'] = ''
try:
decoded_guid = session.get_handle().sessionId.guid
response['session_id'] = unpack_guid(decoded_guid)
except Exception as e:
LOG.warning('Failed to decode session handle: %s' % e)
if lang == 'impala' and session:
http_addr = _get_impala_server_url(session)
response['http_addr'] = http_addr
return response
