def close_session(self, session): app_name = session.get('type') session_id = session.get('id') source_method = session.get("sourceMethod") if not session_id: session = Session.objects.get_session(self.user, application=app_name) decoded_guid = session.get_handle().sessionId.guid session_decoded_id = unpack_guid(decoded_guid) if source_method == "dt_logout": LOG.debug("Closing Impala session id %s on logout for user %s" % (session_decoded_id, self.user.username)) query_server = get_query_server_config(name=app_name) response = {'status': -1, 'message': ''} try: filters = {'id': session_id, 'application': query_server['server_name']} if not is_admin(self.user): filters['owner'] = self.user session = Session.objects.get(**filters) except Session.DoesNotExist: response['message'] = _('Session does not exist or you do not have permissions to close the session.') if session: session = dbms.get(self.user, query_server).close_session(session) response['status'] = 0 response['message'] = _('Session successfully closed.') response['session'] = {'id': session_id, 'application': session.application, 'status': session.status_code} return response
def _get_impala_query_id(self, snippet): guid = None if 'result' in snippet and 'handle' in snippet['result'] and 'guid' in snippet['result']['handle']: try: guid = unpack_guid(base64.decodestring(snippet['result']['handle']['guid'])) except Exception, e: LOG.warn('Failed to decode operation handle guid: %s' % e)
def get_jobs(self, notebook, snippet, logs): jobs = [] if snippet['type'] == 'hive': engine = self._get_hive_execution_engine(notebook, snippet) jobs_with_state = parse_out_jobs(logs, engine=engine, with_state=True) jobs = [{ 'name': job.get('job_id', ''), 'url': reverse('jobbrowser.views.single_job', kwargs={'job': job.get('job_id', '')}), 'started': job.get('started', False), 'finished': job.get('finished', False) } for job in jobs_with_state] elif snippet['type'] == 'impala' and ENABLE_QUERY_BROWSER.get(): query_id = unpack_guid(snippet['result']['handle']['guid']) progress = min(self.progress(snippet, logs), 99) if snippet['status'] != 'available' and snippet['status'] != 'success' else 100 jobs = [{ 'name': query_id, 'url': '/hue/jobbrowser#!id=%s' % query_id, 'started': True, 'finished': False, 'percentJob': progress }] return jobs
def create_session(self, lang='hive', properties=None): application = 'beeswax' if lang == 'hive' else lang session = Session.objects.get_session(self.user, application=application) reuse_session = session is not None if not reuse_session: session = dbms.get(self.user, query_server=get_query_server_config(name=lang, cluster=self.cluster)).open_session(self.user) response = { 'type': lang, 'id': session.id } if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user(app=lang, user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties(lang) response['properties'] = properties response['configuration'] = json.loads(session.properties) response['reuse_session'] = reuse_session response['session_id'] = '' try: decoded_guid = session.get_handle().sessionId.guid response['session_id'] = unpack_guid(decoded_guid) except Exception, e: LOG.warn('Failed to decode session handle: %s' % e)
def alanize(request): response = {'status': -1} cluster = json.loads(request.POST.get('cluster', '{}')) query_id = json.loads(request.POST.get('query_id')) api = _get_api(request.user, cluster=cluster) if query_id: LOG.debug("Attempting to get Impala query profile for query ID: %s" % (query_id)) doc = Document2.objects.get(id=query_id) snippets = doc.data_dict.get('snippets', []) secret = snippets[0]['result']['handle']['secret'] impala_query_id = unpack_guid(base64.decodestring(secret)) query_profile = api.get_query_profile_encoded(impala_query_id) profile = analyzer.analyze(analyzer.parse_data(query_profile)) ANALYZER.pre_process(profile) result = ANALYZER.run(profile) heatmap = {} summary = analyzer.summary(profile) heatmapMetrics = ['AverageThreadTokens', 'BloomFilterBytes', 'PeakMemoryUsage', 'PerHostPeakMemUsage', 'PrepareTime', 'RowsProduced', 'TotalCpuTime', 'TotalNetworkReceiveTime', 'TotalNetworkSendTime', 'TotalStorageWaitTime', 'TotalTime'] for key in heatmapMetrics: metrics = analyzer.heatmap_by_host(profile, key) if metrics['data']: heatmap[key] = metrics response['data'] = { 'query': { 'healthChecks' : result[0]['result'], 'summary': summary, 'heatmap': heatmap, 'heatmapMetrics': sorted(list(heatmap.iterkeys())) } } response['status'] = 0 return JsonResponse(response)
def alanize(request): response = {'status': -1} cluster = json.loads(request.POST.get('cluster', '{}')) query_id = json.loads(request.POST.get('query_id')) api = _get_api(request.user, cluster=cluster) if query_id: LOG.debug("Attempting to get Impala query profile for query ID: %s" % (query_id)) doc = Document2.objects.get(id=query_id) snippets = doc.data_dict.get('snippets', []) secret = snippets[0]['result']['handle']['secret'] impala_query_id = unpack_guid(base64.decodestring(secret)) query_profile = api.get_query_profile_encoded(impala_query_id) profile = analyzer.analyze(analyzer.parse_data(query_profile)) ANALYZER.pre_process(profile) result = ANALYZER.run(profile) heatmap = {} summary = analyzer.summary(profile) heatmapMetrics = ['AverageThreadTokens', 'BloomFilterBytes', 'PeakMemoryUsage', 'PerHostPeakMemUsage', 'PrepareTime', 'RowsProduced', 'TotalCpuTime', 'TotalNetworkReceiveTime', 'TotalNetworkSendTime', 'TotalStorageWaitTime', 'TotalTime'] for key in heatmapMetrics: metrics = analyzer.heatmap_by_host(profile, key) if metrics['data']: heatmap[key] = metrics response['data'] = { 'query': { 'healthChecks' : result[0]['result'], 'summary': summary, 'heatmap': heatmap, 'heatmapMetrics': sorted(list(heatmap.keys())) } } response['status'] = 0 return JsonResponse(response)
def create_session(self, lang='hive', properties=None): application = 'beeswax' if lang == 'hive' or lang == 'llap' else lang if has_session_pool(): session = Session.objects.get_tez_session( self.user, application, MAX_NUMBER_OF_SESSIONS.get()) elif not has_multiple_sessions(): session = Session.objects.get_session(self.user, application=application) else: session = None reuse_session = session is not None if not reuse_session: db = dbms.get(self.user, query_server=get_query_server_config( name=lang, connector=self.interpreter)) session = db.open_session(self.user) response = {'type': lang, 'id': session.id} if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user( app=lang, user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties(lang) response['properties'] = properties response['configuration'] = json.loads(session.properties) response['reuse_session'] = reuse_session response['session_id'] = '' try: decoded_guid = session.get_handle().sessionId.guid response['session_id'] = unpack_guid(decoded_guid) except Exception as e: LOG.warn('Failed to decode session handle: %s' % e) if lang == 'impala' and session: http_addr = _get_impala_server_url(session) response['http_addr'] = http_addr return response
def create_session(self, lang='hive', properties=None): application = 'beeswax' if lang == 'hive' or lang == 'llap' else lang uses_session_pool = has_session_pool() uses_multiple_sessions = has_multiple_sessions() if lang == 'impala': uses_session_pool = False uses_multiple_sessions = False try: if uses_session_pool: session = Session.objects.get_tez_session( self.user, application, MAX_NUMBER_OF_SESSIONS.get()) elif not uses_multiple_sessions: session = Session.objects.get_session(self.user, application=application) else: session = None except Exception as e: if 'Connection refused' in str( e) or 'Name or service not known' in str(e): LOG.exception( 'Connection being refused or service is not available in either session or in multiple sessions' '- HA failover') reset_ha() reuse_session = session is not None if not reuse_session: db = dbms.get(self.user, query_server=get_query_server_config( name=lang, connector=self.interpreter)) try: session = db.open_session(self.user) except Exception as e: if 'Connection refused' in str( e) or 'Name or service not known' in str(e): LOG.exception( 'Connection being refused or service is not available in reuse session - HA failover' ) reset_ha() response = {'type': lang, 'id': session.id} if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user( app=lang, user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties(lang) response['properties'] = properties response['configuration'] = json.loads(session.properties) response['reuse_session'] = reuse_session response['session_id'] = '' try: decoded_guid = session.get_handle().sessionId.guid response['session_id'] = unpack_guid(decoded_guid) except Exception as e: LOG.warning('Failed to decode session handle: %s' % e) if lang == 'impala' and session: http_addr = _get_impala_server_url(session) response['http_addr'] = http_addr return response