def apps(self, filters): kwargs = {} api = get_impalad_api(user=self.user, url=self.server_url) jobs = api.get_queries(**kwargs) return { 'apps': [{ 'id': app['query_id'], 'name': app['stmt'][:100] + ('...' if len(app['stmt']) > 100 else ''), 'status': app['state'], 'apiStatus': self._api_status(app['state']), 'type': app['stmt_type'], 'user': app['effective_user'], 'queue': app['resource_pool'], 'progress': app['progress'], 'duration': 0, # app['duration'], 'submitted': app['start_time'], # Extra specific 'rows_fetched': app['rows_fetched'], 'waiting': app['waiting'], 'waiting_time': app['waiting_time'] } for app in itertools.chain(jobs['in_flight_queries'], jobs['completed_queries'])], 'total': jobs['num_in_flight_queries'] + jobs['num_executing_queries'] + jobs['num_waiting_queries'] }
def _get_api(user, cluster=None): if cluster and cluster.get('type') == 'altus-dw': server_url = 'http://impala-coordinator-%(name)s:25000' % cluster else: session = Session.objects.get_session(user, application=_get_server_name(cluster)) server_url = _get_impala_server_url(session) return get_impalad_api(user=user, url=server_url)
def apps(self, filters): kwargs = {} api = get_impalad_api(user=self.user, url=self.server_url) jobs = api.get_queries(**kwargs) filter_list = self._get_filter_list(filters) jobs_iter = itertools.chain(jobs['in_flight_queries'], jobs['completed_queries']) jobs_iter_filtered = self._n_filter(filter_list, jobs_iter) return { 'apps': [{ 'id': job['query_id'], 'name': job['stmt'][:100] + ('...' if len(job['stmt']) > 100 else ''), 'status': job['state'], 'apiStatus': self._api_status(job['state']), 'type': job['stmt_type'], 'user': job['effective_user'], 'queue': job['resource_pool'], 'progress': job['progress'], 'canWrite': job in jobs['in_flight_queries'], 'duration': self._time_in_ms_groups(re.search(r"\s*(([\d.]*)([a-z]*))(([\d.]*)([a-z]*))?(([\d.]*)([a-z]*))?", job['duration'], re.MULTILINE).groups()), 'submitted': job['start_time'], # Extra specific 'rows_fetched': job['rows_fetched'], 'waiting': job['waiting'], 'waiting_time': job['waiting_time'] } for job in jobs_iter_filtered], 'total': jobs['num_in_flight_queries'] + jobs['num_executing_queries'] + jobs['num_waiting_queries'] }
def _get_impala_query_profile(self, server_url, query_id): api = get_impalad_api(user=self.user, url=server_url) try: query_profile = api.get_query_profile(query_id) profile = query_profile.get('profile') except (RestException, ImpalaDaemonApiException), e: raise PopupException(_("Failed to get query profile from Impala Daemon server: %s") % e)
def _get_impala_query_profile(self, server_url, query_id): api = get_impalad_api(user=self.user, url=server_url) try: query_profile = api.get_query_profile(query_id) profile = query_profile.get('profile') except (RestException, ImpalaDaemonApiException), e: raise PopupException(_("Failed to get query profile from Impala Daemon server: %s") % e)
def _get_api(user, cluster=None): if cluster and cluster.get('type') == 'altus-dw': server_url = 'http://impala-coordinator-%(name)s:25000' % cluster else: # TODO: multi computes if snippet.get('compute') or snippet['type'] has computes application = cluster.get('type', 'impala') session = Session.objects.get_session(user, application=application) server_url = _get_impala_server_url(session) return get_impalad_api(user=user, url=server_url)
def app(self, appid): api = get_impalad_api(user=self.user, url=self.server_url) query = api.get_query_profile(query_id=appid) user = re.search(r"^\s*User:\s*(.*)$", query['profile'], re.MULTILINE).group(1) status = re.search(r"^\s*Query State:\s*(.*)$", query['profile'], re.MULTILINE).group(1) stmt = re.search(r"^\s*Sql Statement:\s*(.*)$", query['profile'], re.MULTILINE).group(1) partitions = re.findall(r"partitions=\s*(\d)+\s*\/\s*(\d)+", query['profile']) end_time = re.search(r"^\s*End Time:\s*(.*)$", query['profile'], re.MULTILINE).group(1) duration_1 = re.search(r"\s*Rows available:\s([\d.]*)(\w*)", query['profile'], re.MULTILINE) duration_2 = re.search(r"\s*Request finished:\s([\d.]*)(\w*)", query['profile'], re.MULTILINE) duration_3 = re.search(r"\s*Query Timeline:\s([\d.]*)(\w*)", query['profile'], re.MULTILINE) submitted = re.search(r"^\s*Start Time:\s*(.*)$", query['profile'], re.MULTILINE).group(1) progress = 0 if end_time: progress = 100 elif partitions: for partition in partitions: progress += float(partition[0]) / float(partition[1]) progress /= len(partitions) progress *= 100 duration = duration_1 or duration_2 or duration_3 if duration: duration_ms = self.time_in_ms(duration.group(1), duration.group(2)) else: duration_ms = 0 common = { 'id': appid, 'name': stmt, 'status': status, 'apiStatus': self._api_status(status), 'user': user, 'progress': progress, 'duration': duration_ms, 'submitted': submitted, 'type': 'queries' } common['properties'] = {'memory': '', 'profile': '', 'plan': ''} return common
def app(self, appid): api = get_impalad_api(user=self.user, url=self.server_url) query = api.get_query_profile(query_id=appid) if query.get('error'): return { 'status': -1, 'message': query.get('error') } user = re.search(r"^\s*User:\s?([^\n\r]*)$", query['profile'], re.MULTILINE).group(1) status = re.search(r"^\s*Query State:\s?([^\n\r]*)$$", query['profile'], re.MULTILINE).group(1) stmt = re.search(r"^\s*Sql Statement:\s?([^\n\r]*)$$", query['profile'], re.MULTILINE).group(1) partitions = re.findall(r"partitions=\s*(\d)+\s*\/\s*(\d)+", query['profile']) end_time = re.search(r"^\s*End Time:\s?([^\n\r]*)$", query['profile'], re.MULTILINE).group(1) submitted = re.search(r"^\s*Start Time:\s?([^\n\r]*)$", query['profile'], re.MULTILINE).group(1) progress = 0 if end_time: progress = 100 elif partitions: for partition in partitions: progress += float(partition[0]) / float(partition[1]) progress /= len(partitions) progress *= 100 if end_time: end_time_ms = int(time.mktime(datetime.strptime(end_time[:26], '%Y-%m-%d %H:%M:%S.%f').timetuple()))*1000 start_time_ms = int(time.mktime(datetime.strptime(submitted[:26], '%Y-%m-%d %H:%M:%S.%f').timetuple()))*1000 duration_ms = end_time_ms - start_time_ms else: duration_ms = 0 common = { 'id': appid, 'name': stmt, 'status': status, 'apiStatus': self._api_status(status), 'user': user, 'progress': progress, 'duration': duration_ms, 'submitted': submitted, 'type': 'queries' } common['properties'] = { 'memory': '', 'profile': '', 'plan': '' } return common
def _get_impala_query_profile(self, server_url, query_id): api = get_impalad_api(user=self.user, url=server_url) try: query_profile = api.get_query_profile(query_id) profile = query_profile.get('profile') except (RestException, ImpalaDaemonApiException) as e: raise PopupException(_("Failed to get query profile from Impala Daemon server: %s") % e) if not profile: raise PopupException(_("Could not find profile in query profile response from Impala Daemon Server.")) return profile
def alanize(request): response = {'status': -1} cluster = json.loads(request.POST.get('cluster', '{}')) query_id = json.loads(request.POST.get('query_id')) application = _get_server_name(cluster) query_server = dbms.get_query_server_config() session = Session.objects.get_session(request.user, query_server['server_name']) server_url = _get_impala_server_url(session) if query_id: LOG.debug( "Attempting to get Impala query profile at server_url %s for query ID: %s" % (server_url, query_id)) doc = Document2.objects.get(id=query_id) snippets = doc.data_dict.get('snippets', []) secret = snippets[0]['result']['handle']['secret'] api = get_impalad_api(user=request.user, url=server_url) impala_query_id = "%x:%x" % struct.unpack(b"QQ", base64.decodestring(secret)) api.kill( impala_query_id ) # There are many statistics that are not present when the query is open. Close it first. query_profile = api.get_query_profile_encoded(impala_query_id) profile = analyzer.analyze(analyzer.parse_data(query_profile)) result = ANALYZER.run(profile) heatmap = {} summary = analyzer.summary(profile) heatmapMetrics = [ 'AverageThreadTokens', 'BloomFilterBytes', 'PeakMemoryUsage', 'PerHostPeakMemUsage', 'PrepareTime', 'RowsProduced', 'TotalCpuTime', 'TotalNetworkReceiveTime', 'TotalNetworkSendTime', 'TotalStorageWaitTime', 'TotalTime' ] for key in heatmapMetrics: metrics = analyzer.heatmap_by_host(profile, key) if metrics['data']: heatmap[key] = metrics response['data'] = { 'query': { 'healthChecks': result[0]['result'], 'summary': summary, 'heatmap': heatmap, 'heatmapMetrics': sorted(list(heatmap.iterkeys())) } } response['status'] = 0 return JsonResponse(response)
def action(self, appid, action): message = {'message': '', 'status': 0} if action.get('action') == 'kill': api = get_impalad_api(user=self.user, url=self.server_url) for _id in appid: result = api.kill(_id) if result.get('error'): message['message'] = result.get('error') message['status'] = -1 elif result.get('contents') and message.get('status') != -1: message['message'] = result.get('contents') return message;
def app(self, appid): api = get_impalad_api(user=self.user, url=self.server_url) query = api.get_query(query_id=appid) common = { 'id': appid, 'name': query['stmt'][:100] + ('...' if len(query['stmt']) > 100 else ''), 'status': query['status'], 'apiStatus': self._api_status(query['status']), 'progress': 50, 'duration': 10 * 3600, 'submitted': 0, 'type': 'NA', } common['properties'] = { 'properties': query } return common
def _get_api(user): session = Session.objects.get_session(user, application='impala') server_url = _get_impala_server_url(session) return get_impalad_api(user=user, url=server_url)
def _get_api(user): session = Session.objects.get_session(user, application='impala') server_url = _get_impala_server_url(session) return get_impalad_api(user=user, url=server_url)
def _query_profile(self, appid): api = get_impalad_api(user=self.user, url=self.server_url) return api.get_query_profile(query_id=appid)
def _memory(self, appid, app_type, app_property, app_filters): api = get_impalad_api(user=self.user, url=self.server_url) return api.get_query_memory(query_id=appid)