def close_session(self, session):
app_name = session.get('type')
session_id = session.get('id')
source_method = session.get("sourceMethod")
if not session_id:
session = Session.objects.get_session(self.user, application=app_name)
decoded_guid = session.get_handle().sessionId.guid
session_decoded_id = unpack_guid(decoded_guid)
if source_method == "dt_logout":
LOG.debug("Closing Impala session id %s on logout for user %s" % (session_decoded_id, self.user.username))
query_server = get_query_server_config(name=app_name)
response = {'status': -1, 'message': ''}
try:
filters = {'id': session_id, 'application': query_server['server_name']}
if not is_admin(self.user):
filters['owner'] = self.user
session = Session.objects.get(**filters)
except Session.DoesNotExist:
response['message'] = _('Session does not exist or you do not have permissions to close the session.')
if session:
session = dbms.get(self.user, query_server).close_session(session)
response['status'] = 0
response['message'] = _('Session successfully closed.')
response['session'] = {'id': session_id, 'application': session.application, 'status': session.status_code}
return response
def _get_impala_query_id(self, snippet):
guid = None
if 'result' in snippet and 'handle' in snippet['result'] and 'guid' in snippet['result']['handle']:
try:
guid = unpack_guid(base64.decodestring(snippet['result']['handle']['guid']))
except Exception, e:
LOG.warn('Failed to decode operation handle guid: %s' % e)
def get_jobs(self, notebook, snippet, logs):
jobs = []
if snippet['type'] == 'hive':
engine = self._get_hive_execution_engine(notebook, snippet)
jobs_with_state = parse_out_jobs(logs, engine=engine, with_state=True)
jobs = [{
'name': job.get('job_id', ''),
'url': reverse('jobbrowser.views.single_job', kwargs={'job': job.get('job_id', '')}),
'started': job.get('started', False),
'finished': job.get('finished', False)
} for job in jobs_with_state]
elif snippet['type'] == 'impala' and ENABLE_QUERY_BROWSER.get():
query_id = unpack_guid(snippet['result']['handle']['guid'])
progress = min(self.progress(snippet, logs), 99) if snippet['status'] != 'available' and snippet['status'] != 'success' else 100
jobs = [{
'name': query_id,
'url': '/hue/jobbrowser#!id=%s' % query_id,
'started': True,
'finished': False,
'percentJob': progress
}]
return jobs
def create_session(self, lang='hive', properties=None):
application = 'beeswax' if lang == 'hive' else lang
session = Session.objects.get_session(self.user, application=application)
reuse_session = session is not None
if not reuse_session:
session = dbms.get(self.user, query_server=get_query_server_config(name=lang, cluster=self.cluster)).open_session(self.user)
response = {
'type': lang,
'id': session.id
}
if not properties:
config = None
if USE_DEFAULT_CONFIGURATION.get():
config = DefaultConfiguration.objects.get_configuration_for_user(app=lang, user=self.user)
if config is not None:
properties = config.properties_list
else:
properties = self.get_properties(lang)
response['properties'] = properties
response['configuration'] = json.loads(session.properties)
response['reuse_session'] = reuse_session
response['session_id'] = ''
try:
decoded_guid = session.get_handle().sessionId.guid
response['session_id'] = unpack_guid(decoded_guid)
except Exception, e:
LOG.warn('Failed to decode session handle: %s' % e)
def alanize(request):
response = {'status': -1}
cluster = json.loads(request.POST.get('cluster', '{}'))
query_id = json.loads(request.POST.get('query_id'))
api = _get_api(request.user, cluster=cluster)
if query_id:
LOG.debug("Attempting to get Impala query profile for query ID: %s" % (query_id))
doc = Document2.objects.get(id=query_id)
snippets = doc.data_dict.get('snippets', [])
secret = snippets[0]['result']['handle']['secret']
impala_query_id = unpack_guid(base64.decodestring(secret))
query_profile = api.get_query_profile_encoded(impala_query_id)
profile = analyzer.analyze(analyzer.parse_data(query_profile))
ANALYZER.pre_process(profile)
result = ANALYZER.run(profile)
heatmap = {}
summary = analyzer.summary(profile)
heatmapMetrics = ['AverageThreadTokens', 'BloomFilterBytes', 'PeakMemoryUsage', 'PerHostPeakMemUsage', 'PrepareTime', 'RowsProduced', 'TotalCpuTime', 'TotalNetworkReceiveTime', 'TotalNetworkSendTime', 'TotalStorageWaitTime', 'TotalTime']
for key in heatmapMetrics:
metrics = analyzer.heatmap_by_host(profile, key)
if metrics['data']:
heatmap[key] = metrics
response['data'] = { 'query': { 'healthChecks' : result[0]['result'], 'summary': summary, 'heatmap': heatmap, 'heatmapMetrics': sorted(list(heatmap.iterkeys())) } }
response['status'] = 0
return JsonResponse(response)
def alanize(request):
response = {'status': -1}
cluster = json.loads(request.POST.get('cluster', '{}'))
query_id = json.loads(request.POST.get('query_id'))
api = _get_api(request.user, cluster=cluster)
if query_id:
LOG.debug("Attempting to get Impala query profile for query ID: %s" % (query_id))
doc = Document2.objects.get(id=query_id)
snippets = doc.data_dict.get('snippets', [])
secret = snippets[0]['result']['handle']['secret']
impala_query_id = unpack_guid(base64.decodestring(secret))
query_profile = api.get_query_profile_encoded(impala_query_id)
profile = analyzer.analyze(analyzer.parse_data(query_profile))
ANALYZER.pre_process(profile)
result = ANALYZER.run(profile)
heatmap = {}
summary = analyzer.summary(profile)
heatmapMetrics = ['AverageThreadTokens', 'BloomFilterBytes', 'PeakMemoryUsage', 'PerHostPeakMemUsage', 'PrepareTime', 'RowsProduced', 'TotalCpuTime', 'TotalNetworkReceiveTime', 'TotalNetworkSendTime', 'TotalStorageWaitTime', 'TotalTime']
for key in heatmapMetrics:
metrics = analyzer.heatmap_by_host(profile, key)
if metrics['data']:
heatmap[key] = metrics
response['data'] = { 'query': { 'healthChecks' : result[0]['result'], 'summary': summary, 'heatmap': heatmap, 'heatmapMetrics': sorted(list(heatmap.keys())) } }
response['status'] = 0
return JsonResponse(response)
def create_session(self, lang='hive', properties=None):
application = 'beeswax' if lang == 'hive' or lang == 'llap' else lang
if has_session_pool():
session = Session.objects.get_tez_session(
self.user, application, MAX_NUMBER_OF_SESSIONS.get())
elif not has_multiple_sessions():
session = Session.objects.get_session(self.user,
application=application)
else:
session = None
reuse_session = session is not None
if not reuse_session:
db = dbms.get(self.user,
query_server=get_query_server_config(
name=lang, connector=self.interpreter))
session = db.open_session(self.user)
response = {'type': lang, 'id': session.id}
if not properties:
config = None
if USE_DEFAULT_CONFIGURATION.get():
config = DefaultConfiguration.objects.get_configuration_for_user(
app=lang, user=self.user)
if config is not None:
properties = config.properties_list
else:
properties = self.get_properties(lang)
response['properties'] = properties
response['configuration'] = json.loads(session.properties)
response['reuse_session'] = reuse_session
response['session_id'] = ''
try:
decoded_guid = session.get_handle().sessionId.guid
response['session_id'] = unpack_guid(decoded_guid)
except Exception as e:
LOG.warn('Failed to decode session handle: %s' % e)
if lang == 'impala' and session:
http_addr = _get_impala_server_url(session)
response['http_addr'] = http_addr
return response
def create_session(self, lang='hive', properties=None):
application = 'beeswax' if lang == 'hive' or lang == 'llap' else lang
uses_session_pool = has_session_pool()
uses_multiple_sessions = has_multiple_sessions()
if lang == 'impala':
uses_session_pool = False
uses_multiple_sessions = False
try:
if uses_session_pool:
session = Session.objects.get_tez_session(
self.user, application, MAX_NUMBER_OF_SESSIONS.get())
elif not uses_multiple_sessions:
session = Session.objects.get_session(self.user,
application=application)
else:
session = None
except Exception as e:
if 'Connection refused' in str(
e) or 'Name or service not known' in str(e):
LOG.exception(
'Connection being refused or service is not available in either session or in multiple sessions'
'- HA failover')
reset_ha()
reuse_session = session is not None
if not reuse_session:
db = dbms.get(self.user,
query_server=get_query_server_config(
name=lang, connector=self.interpreter))
try:
session = db.open_session(self.user)
except Exception as e:
if 'Connection refused' in str(
e) or 'Name or service not known' in str(e):
LOG.exception(
'Connection being refused or service is not available in reuse session - HA failover'
)
reset_ha()
response = {'type': lang, 'id': session.id}
if not properties:
config = None
if USE_DEFAULT_CONFIGURATION.get():
config = DefaultConfiguration.objects.get_configuration_for_user(
app=lang, user=self.user)
if config is not None:
properties = config.properties_list
else:
properties = self.get_properties(lang)
response['properties'] = properties
response['configuration'] = json.loads(session.properties)
response['reuse_session'] = reuse_session
response['session_id'] = ''
try:
decoded_guid = session.get_handle().sessionId.guid
response['session_id'] = unpack_guid(decoded_guid)
except Exception as e:
LOG.warning('Failed to decode session handle: %s' % e)
if lang == 'impala' and session:
http_addr = _get_impala_server_url(session)
response['http_addr'] = http_addr
return response
