def apps(self, filters):
kwargs = {}
api = get_impalad_api(user=self.user, url=self.server_url)
jobs = api.get_queries(**kwargs)
return {
'apps': [{
'id': app['query_id'],
'name': app['stmt'][:100] + ('...' if len(app['stmt']) > 100 else ''),
'status': app['state'],
'apiStatus': self._api_status(app['state']),
'type': app['stmt_type'],
'user': app['effective_user'],
'queue': app['resource_pool'],
'progress': app['progress'],
'duration': 0, # app['duration'],
'submitted': app['start_time'],
# Extra specific
'rows_fetched': app['rows_fetched'],
'waiting': app['waiting'],
'waiting_time': app['waiting_time']
} for app in itertools.chain(jobs['in_flight_queries'], jobs['completed_queries'])],
'total': jobs['num_in_flight_queries'] + jobs['num_executing_queries'] + jobs['num_waiting_queries']
}
def _get_api(user, cluster=None):
if cluster and cluster.get('type') == 'altus-dw':
server_url = 'http://impala-coordinator-%(name)s:25000' % cluster
else:
session = Session.objects.get_session(user, application=_get_server_name(cluster))
server_url = _get_impala_server_url(session)
return get_impalad_api(user=user, url=server_url)
def apps(self, filters):
kwargs = {}
api = get_impalad_api(user=self.user, url=self.server_url)
jobs = api.get_queries(**kwargs)
filter_list = self._get_filter_list(filters)
jobs_iter = itertools.chain(jobs['in_flight_queries'], jobs['completed_queries'])
jobs_iter_filtered = self._n_filter(filter_list, jobs_iter)
return {
'apps': [{
'id': job['query_id'],
'name': job['stmt'][:100] + ('...' if len(job['stmt']) > 100 else ''),
'status': job['state'],
'apiStatus': self._api_status(job['state']),
'type': job['stmt_type'],
'user': job['effective_user'],
'queue': job['resource_pool'],
'progress': job['progress'],
'canWrite': job in jobs['in_flight_queries'],
'duration': self._time_in_ms_groups(re.search(r"\s*(([\d.]*)([a-z]*))(([\d.]*)([a-z]*))?(([\d.]*)([a-z]*))?", job['duration'], re.MULTILINE).groups()),
'submitted': job['start_time'],
# Extra specific
'rows_fetched': job['rows_fetched'],
'waiting': job['waiting'],
'waiting_time': job['waiting_time']
} for job in jobs_iter_filtered],
'total': jobs['num_in_flight_queries'] + jobs['num_executing_queries'] + jobs['num_waiting_queries']
}
def _get_impala_query_profile(self, server_url, query_id):
api = get_impalad_api(user=self.user, url=server_url)
try:
query_profile = api.get_query_profile(query_id)
profile = query_profile.get('profile')
except (RestException, ImpalaDaemonApiException), e:
raise PopupException(_("Failed to get query profile from Impala Daemon server: %s") % e)
def _get_impala_query_profile(self, server_url, query_id):
api = get_impalad_api(user=self.user, url=server_url)
try:
query_profile = api.get_query_profile(query_id)
profile = query_profile.get('profile')
except (RestException, ImpalaDaemonApiException), e:
raise PopupException(_("Failed to get query profile from Impala Daemon server: %s") % e)
def _get_api(user, cluster=None):
if cluster and cluster.get('type') == 'altus-dw':
server_url = 'http://impala-coordinator-%(name)s:25000' % cluster
else:
# TODO: multi computes if snippet.get('compute') or snippet['type'] has computes
application = cluster.get('type', 'impala')
session = Session.objects.get_session(user, application=application)
server_url = _get_impala_server_url(session)
return get_impalad_api(user=user, url=server_url)
def app(self, appid):
api = get_impalad_api(user=self.user, url=self.server_url)
query = api.get_query_profile(query_id=appid)
user = re.search(r"^\s*User:\s*(.*)$", query['profile'],
re.MULTILINE).group(1)
status = re.search(r"^\s*Query State:\s*(.*)$", query['profile'],
re.MULTILINE).group(1)
stmt = re.search(r"^\s*Sql Statement:\s*(.*)$", query['profile'],
re.MULTILINE).group(1)
partitions = re.findall(r"partitions=\s*(\d)+\s*\/\s*(\d)+",
query['profile'])
end_time = re.search(r"^\s*End Time:\s*(.*)$", query['profile'],
re.MULTILINE).group(1)
duration_1 = re.search(r"\s*Rows available:\s([\d.]*)(\w*)",
query['profile'], re.MULTILINE)
duration_2 = re.search(r"\s*Request finished:\s([\d.]*)(\w*)",
query['profile'], re.MULTILINE)
duration_3 = re.search(r"\s*Query Timeline:\s([\d.]*)(\w*)",
query['profile'], re.MULTILINE)
submitted = re.search(r"^\s*Start Time:\s*(.*)$", query['profile'],
re.MULTILINE).group(1)
progress = 0
if end_time:
progress = 100
elif partitions:
for partition in partitions:
progress += float(partition[0]) / float(partition[1])
progress /= len(partitions)
progress *= 100
duration = duration_1 or duration_2 or duration_3
if duration:
duration_ms = self.time_in_ms(duration.group(1), duration.group(2))
else:
duration_ms = 0
common = {
'id': appid,
'name': stmt,
'status': status,
'apiStatus': self._api_status(status),
'user': user,
'progress': progress,
'duration': duration_ms,
'submitted': submitted,
'type': 'queries'
}
common['properties'] = {'memory': '', 'profile': '', 'plan': ''}
return common
def app(self, appid):
api = get_impalad_api(user=self.user, url=self.server_url)
query = api.get_query_profile(query_id=appid)
if query.get('error'):
return {
'status': -1,
'message': query.get('error')
}
user = re.search(r"^\s*User:\s?([^\n\r]*)$", query['profile'], re.MULTILINE).group(1)
status = re.search(r"^\s*Query State:\s?([^\n\r]*)$$", query['profile'], re.MULTILINE).group(1)
stmt = re.search(r"^\s*Sql Statement:\s?([^\n\r]*)$$", query['profile'], re.MULTILINE).group(1)
partitions = re.findall(r"partitions=\s*(\d)+\s*\/\s*(\d)+", query['profile'])
end_time = re.search(r"^\s*End Time:\s?([^\n\r]*)$", query['profile'], re.MULTILINE).group(1)
submitted = re.search(r"^\s*Start Time:\s?([^\n\r]*)$", query['profile'], re.MULTILINE).group(1)
progress = 0
if end_time:
progress = 100
elif partitions:
for partition in partitions:
progress += float(partition[0]) / float(partition[1])
progress /= len(partitions)
progress *= 100
if end_time:
end_time_ms = int(time.mktime(datetime.strptime(end_time[:26], '%Y-%m-%d %H:%M:%S.%f').timetuple()))*1000
start_time_ms = int(time.mktime(datetime.strptime(submitted[:26], '%Y-%m-%d %H:%M:%S.%f').timetuple()))*1000
duration_ms = end_time_ms - start_time_ms
else:
duration_ms = 0
common = {
'id': appid,
'name': stmt,
'status': status,
'apiStatus': self._api_status(status),
'user': user,
'progress': progress,
'duration': duration_ms,
'submitted': submitted,
'type': 'queries'
}
common['properties'] = {
'memory': '',
'profile': '',
'plan': ''
}
return common
def _get_impala_query_profile(self, server_url, query_id):
api = get_impalad_api(user=self.user, url=server_url)
try:
query_profile = api.get_query_profile(query_id)
profile = query_profile.get('profile')
except (RestException, ImpalaDaemonApiException) as e:
raise PopupException(_("Failed to get query profile from Impala Daemon server: %s") % e)
if not profile:
raise PopupException(_("Could not find profile in query profile response from Impala Daemon Server."))
return profile
def alanize(request):
response = {'status': -1}
cluster = json.loads(request.POST.get('cluster', '{}'))
query_id = json.loads(request.POST.get('query_id'))
application = _get_server_name(cluster)
query_server = dbms.get_query_server_config()
session = Session.objects.get_session(request.user,
query_server['server_name'])
server_url = _get_impala_server_url(session)
if query_id:
LOG.debug(
"Attempting to get Impala query profile at server_url %s for query ID: %s"
% (server_url, query_id))
doc = Document2.objects.get(id=query_id)
snippets = doc.data_dict.get('snippets', [])
secret = snippets[0]['result']['handle']['secret']
api = get_impalad_api(user=request.user, url=server_url)
impala_query_id = "%x:%x" % struct.unpack(b"QQ",
base64.decodestring(secret))
api.kill(
impala_query_id
) # There are many statistics that are not present when the query is open. Close it first.
query_profile = api.get_query_profile_encoded(impala_query_id)
profile = analyzer.analyze(analyzer.parse_data(query_profile))
result = ANALYZER.run(profile)
heatmap = {}
summary = analyzer.summary(profile)
heatmapMetrics = [
'AverageThreadTokens', 'BloomFilterBytes', 'PeakMemoryUsage',
'PerHostPeakMemUsage', 'PrepareTime', 'RowsProduced',
'TotalCpuTime', 'TotalNetworkReceiveTime', 'TotalNetworkSendTime',
'TotalStorageWaitTime', 'TotalTime'
]
for key in heatmapMetrics:
metrics = analyzer.heatmap_by_host(profile, key)
if metrics['data']:
heatmap[key] = metrics
response['data'] = {
'query': {
'healthChecks': result[0]['result'],
'summary': summary,
'heatmap': heatmap,
'heatmapMetrics': sorted(list(heatmap.iterkeys()))
}
}
response['status'] = 0
return JsonResponse(response)
def action(self, appid, action):
message = {'message': '', 'status': 0}
if action.get('action') == 'kill':
api = get_impalad_api(user=self.user, url=self.server_url)
for _id in appid:
result = api.kill(_id)
if result.get('error'):
message['message'] = result.get('error')
message['status'] = -1
elif result.get('contents') and message.get('status') != -1:
message['message'] = result.get('contents')
return message;
def app(self, appid):
api = get_impalad_api(user=self.user, url=self.server_url)
query = api.get_query(query_id=appid)
common = {
'id': appid,
'name': query['stmt'][:100] + ('...' if len(query['stmt']) > 100 else ''),
'status': query['status'],
'apiStatus': self._api_status(query['status']),
'progress': 50,
'duration': 10 * 3600,
'submitted': 0,
'type': 'NA',
}
common['properties'] = {
'properties': query
}
return common
def _get_api(user): session = Session.objects.get_session(user, application='impala') server_url = _get_impala_server_url(session) return get_impalad_api(user=user, url=server_url)
def _get_api(user): session = Session.objects.get_session(user, application='impala') server_url = _get_impala_server_url(session) return get_impalad_api(user=user, url=server_url)
def _query_profile(self, appid):
api = get_impalad_api(user=self.user, url=self.server_url)
return api.get_query_profile(query_id=appid)
def _memory(self, appid, app_type, app_property, app_filters):
api = get_impalad_api(user=self.user, url=self.server_url)
return api.get_query_memory(query_id=appid)
