def create_session(self, lang='hive', properties=None): application = 'beeswax' if lang == 'hive' or lang == 'llap' else lang if has_session_pool(): session = Session.objects.get_tez_session( self.user, application, MAX_NUMBER_OF_SESSIONS.get()) elif not has_multiple_sessions(): session = Session.objects.get_session(self.user, application=application) else: session = None reuse_session = session is not None if not reuse_session: db = dbms.get(self.user, query_server=get_query_server_config( name=lang, connector=self.interpreter)) session = db.open_session(self.user) response = {'type': lang, 'id': session.id} if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user( app=lang, user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties(lang) response['properties'] = properties response['configuration'] = json.loads(session.properties) response['reuse_session'] = reuse_session response['session_id'] = '' try: decoded_guid = session.get_handle().sessionId.guid response['session_id'] = unpack_guid(decoded_guid) except Exception as e: LOG.warn('Failed to decode session handle: %s' % e) if lang == 'impala' and session: http_addr = _get_impala_server_url(session) response['http_addr'] = http_addr return response
class HS2Api(Api): @staticmethod def get_properties(lang='hive'): return ImpalaConfiguration.PROPERTIES if lang == 'impala' else HiveConfiguration.PROPERTIES @query_error_handler def create_session(self, lang='hive', properties=None): application = 'beeswax' if lang == 'hive' else lang session = Session.objects.get_session(self.user, application=application) reuse_session = session is not None if not reuse_session: session = dbms.get(self.user, query_server=get_query_server_config(name=lang, cluster=self.cluster)).open_session(self.user) response = { 'type': lang, 'id': session.id } if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user(app=lang, user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties(lang) response['properties'] = properties response['configuration'] = json.loads(session.properties) response['reuse_session'] = reuse_session response['session_id'] = '' try: decoded_guid = session.get_handle().sessionId.guid response['session_id'] = unpack_guid(decoded_guid) except Exception, e: LOG.warn('Failed to decode session handle: %s' % e) if lang == 'impala' and session: http_addr = _get_impala_server_url(session) response['http_addr'] = http_addr return response
def alanize(request): response = {'status': -1} cluster = json.loads(request.POST.get('cluster', '{}')) query_id = json.loads(request.POST.get('query_id')) application = _get_server_name(cluster) query_server = dbms.get_query_server_config() session = Session.objects.get_session(request.user, query_server['server_name']) server_url = _get_impala_server_url(session) if query_id: LOG.debug( "Attempting to get Impala query profile at server_url %s for query ID: %s" % (server_url, query_id)) doc = Document2.objects.get(id=query_id) snippets = doc.data_dict.get('snippets', []) secret = snippets[0]['result']['handle']['secret'] api = get_impalad_api(user=request.user, url=server_url) impala_query_id = "%x:%x" % struct.unpack(b"QQ", base64.decodestring(secret)) api.kill( impala_query_id ) # There are many statistics that are not present when the query is open. Close it first. query_profile = api.get_query_profile_encoded(impala_query_id) profile = analyzer.analyze(analyzer.parse_data(query_profile)) result = ANALYZER.run(profile) heatmap = {} summary = analyzer.summary(profile) heatmapMetrics = [ 'AverageThreadTokens', 'BloomFilterBytes', 'PeakMemoryUsage', 'PerHostPeakMemUsage', 'PrepareTime', 'RowsProduced', 'TotalCpuTime', 'TotalNetworkReceiveTime', 'TotalNetworkSendTime', 'TotalStorageWaitTime', 'TotalTime' ] for key in heatmapMetrics: heatmap[key] = analyzer.heatmap_by_host(profile, key) response['data'] = { 'query': { 'healthChecks': result[0]['result'], 'summary': summary, 'heatmap': heatmap, 'heatmapMetrics': heatmapMetrics } } response['status'] = 0 return JsonResponse(response)
def _get_impala_result_size(self, notebook, snippet): total_records_match = None total_records, total_size, msg = None, None, None query_id = self._get_impala_query_id(snippet) session = Session.objects.get_session(self.user, application='impala') server_url = _get_impala_server_url(session) if query_id: LOG.info( "Attempting to get Impala query profile at server_url %s for query ID: %s" % (server_url, query_id)) fragment = self._get_impala_query_profile(server_url, query_id=query_id) total_records_re = "Coordinator Fragment F\d\d.+?RowsReturned: \d+(?:.\d+[KMB])? \((?P<total_records>\d+)\).*?(Averaged Fragment F\d\d)" total_records_match = re.search(total_records_re, fragment, re.MULTILINE | re.DOTALL) if total_records_match: total_records = int(total_records_match.group('total_records')) return total_records, total_size, msg
def _get_api(user): session = Session.objects.get_session(user, application='impala') server_url = _get_impala_server_url(session) return get_impalad_api(user=user, url=server_url)
def __init__(self, user): self.user = user session = Session.objects.get_session(self.user, application='impala') self.server_url = _get_impala_server_url(session)
def create_session(self, lang='hive', properties=None): application = 'beeswax' if lang == 'hive' or lang == 'llap' else lang uses_session_pool = has_session_pool() uses_multiple_sessions = has_multiple_sessions() if lang == 'impala': uses_session_pool = False uses_multiple_sessions = False try: if uses_session_pool: session = Session.objects.get_tez_session( self.user, application, MAX_NUMBER_OF_SESSIONS.get()) elif not uses_multiple_sessions: session = Session.objects.get_session(self.user, application=application) else: session = None except Exception as e: if 'Connection refused' in str( e) or 'Name or service not known' in str(e): LOG.exception( 'Connection being refused or service is not available in either session or in multiple sessions' '- HA failover') reset_ha() reuse_session = session is not None if not reuse_session: db = dbms.get(self.user, query_server=get_query_server_config( name=lang, connector=self.interpreter)) try: session = db.open_session(self.user) except Exception as e: if 'Connection refused' in str( e) or 'Name or service not known' in str(e): LOG.exception( 'Connection being refused or service is not available in reuse session - HA failover' ) reset_ha() response = {'type': lang, 'id': session.id} if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user( app=lang, user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties(lang) response['properties'] = properties response['configuration'] = json.loads(session.properties) response['reuse_session'] = reuse_session response['session_id'] = '' try: decoded_guid = session.get_handle().sessionId.guid response['session_id'] = unpack_guid(decoded_guid) except Exception as e: LOG.warning('Failed to decode session handle: %s' % e) if lang == 'impala' and session: http_addr = _get_impala_server_url(session) response['http_addr'] = http_addr return response