def job_executor_logs(request, job, attempt_index=0, name='syslog', offset=LOG_OFFSET_BYTES):
response = {'status': -1}
try:
log = ''
if job.status not in ('NEW', 'SUBMITTED', 'ACCEPTED'):
log = job.history_server_api.download_executors_logs(request, job, name, offset)
response['status'] = 0
response['log'] = LinkJobLogs._make_hdfs_links(log)
except Exception, e:
response['log'] = _('Failed to retrieve executor log: %s' % e)
def job_executor_logs(request, job, attempt_index=0, name='syslog', offset=LOG_OFFSET_BYTES):
response = {'status': -1}
try:
log = ''
if job.status not in ('NEW', 'SUBMITTED', 'ACCEPTED'):
log = job.history_server_api.download_executors_logs(request, job, name, offset)
response['status'] = 0
response['log'] = LinkJobLogs._make_hdfs_links(log)
except Exception, e:
response['log'] = _('Failed to retrieve executor log: %s' % e)
urlencode=False)
debug_info = ''
try:
response = root.get(link, params=params)
log = html.fromstring(response, parser=html.HTMLParser()).xpath(
'/html/body/table/tbody/tr/td[2]')[0].text_content()
except Exception, e:
log = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
debug_info += '\nHTML Response: %s' % response
LOGGER.error(debug_info)
except:
LOGGER.exception('failed to create debug info')
response = {'log': LinkJobLogs._make_hdfs_links(log), 'debug': debug_info}
return JsonResponse(response)
@check_job_permission
def job_single_logs(request, job):
"""
Try to smartly detect the most useful task attempt (e.g. Oozie launcher, failed task) and get its MR logs.
"""
def cmp_exec_time(task1, task2):
return cmp(task1.execStartTimeMs, task2.execStartTimeMs)
task = None
failed_tasks = job.filter_tasks(task_states=('failed', ))
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
debug_info = ''
try:
response = root.get(link, params=params)
log = html.fromstring(response, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
except Exception, e:
log = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
debug_info += '\nHTML Response: %s' % response
LOGGER.error(debug_info)
except:
LOGGER.exception('failed to create debug info')
response = {'log': LinkJobLogs._make_hdfs_links(log), 'debug': debug_info}
return JsonResponse(response)
@check_job_permission
def job_single_logs(request, job):
"""
Try to smartly detect the most useful task attempt (e.g. Oozie launcher, failed task) and get its MR logs.
"""
def cmp_exec_time(task1, task2):
return cmp(task1.execStartTimeMs, task2.execStartTimeMs)
task = None
params = {}
if offset != 0:
params['start'] = offset
root = Resource(get_log_client(log_link),
urlparse.urlsplit(log_link)[2],
urlencode=False)
api_resp = None
try:
api_resp = root.get(link, params=params)
log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath(
'/html/body/table/tbody/tr/td[2]')[0].text_content()
response['status'] = 0
response['log'] = LinkJobLogs._make_hdfs_links(log)
except Exception, e:
response['log'] = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
if api_resp:
debug_info += '\nHTML Response: %s' % response
response['debug'] = debug_info
LOG.error(debug_info)
except:
LOG.exception('failed to create debug info')
return JsonResponse(response)
@check_job_permission
if log_link:
link = '/%s/' % name
params = {}
if offset != 0:
params['start'] = offset
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
api_resp = None
try:
api_resp = root.get(link, params=params)
log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
response['status'] = 0
response['log'] = LinkJobLogs._make_hdfs_links(log)
except Exception, e:
response['log'] = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
if api_resp:
debug_info += '\nHTML Response: %s' % response
response['debug'] = debug_info
LOG.error(debug_info)
except:
LOG.exception('failed to create debug info')
return JsonResponse(response)
@check_job_permission
def job_attempt_logs_json(request,
job,
attempt_index=0,
name='syslog',
offset=LOG_OFFSET_BYTES,
is_embeddable=False):
"""For async log retrieval as Yarn servers are very slow"""
log_link = None
response = {'status': -1}
try:
jt = get_api(request.user, request.jt)
app = jt.get_application(job.jobId)
if app['applicationType'] == 'MAPREDUCE':
if app['finalStatus'] in ('SUCCEEDED', 'FAILED', 'KILLED'):
attempt_index = int(attempt_index)
if not job.job_attempts['jobAttempt']:
response = {'status': 0, 'log': _('Job has no tasks')}
else:
attempt = job.job_attempts['jobAttempt'][attempt_index]
log_link = attempt['logsLink']
# Reformat log link to use YARN RM, replace node addr with node ID addr
log_link = log_link.replace(attempt['nodeHttpAddress'],
attempt['nodeId'])
elif app['state'] == 'RUNNING':
log_link = app['amContainerLogs']
elif app.get('amContainerLogs'):
log_link = app.get('amContainerLogs')
except (KeyError, RestException) as e:
raise KeyError(
_("Cannot find job attempt '%(id)s'.") % {'id': job.jobId}, e)
except Exception as e:
raise Exception(
_("Failed to get application for job %s: %s") % (job.jobId, e))
if log_link:
link = '/%s/' % name
params = {'doAs': request.user.username}
if offset != 0:
params['start'] = offset
root = Resource(get_log_client(log_link),
urllib.parse.urlsplit(log_link)[2],
urlencode=False)
api_resp = None
try:
api_resp = root.get(link, params=params)
log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath(
'/html/body/table/tbody/tr/td[2]')[0].text_content()
response['status'] = 0
response['log'] = LinkJobLogs._make_hdfs_links(log, is_embeddable)
except Exception as e:
response['log'] = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
if api_resp:
debug_info += '\nHTML Response: %s' % response
response['debug'] = debug_info
LOG.error(debug_info)
except:
LOG.exception('failed to create debug info')
return JsonResponse(response)
link = '/%s/' % name
params = {
'doAs': request.user.username
}
if offset != 0:
params['start'] = offset
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
api_resp = None
try:
api_resp = root.get(link, params=params)
log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
response['status'] = 0
response['log'] = LinkJobLogs._make_hdfs_links(log, is_embeddable)
except Exception, e:
response['log'] = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
if api_resp:
debug_info += '\nHTML Response: %s' % response
response['debug'] = debug_info
LOG.error(debug_info)
except:
LOG.exception('failed to create debug info')
return JsonResponse(response)
@check_job_permission
response = root.get(link, params=params)
log = (
html.fromstring(response, parser=html.HTMLParser())
.xpath("/html/body/table/tbody/tr/td[2]")[0]
.text_content()
)
except Exception, e:
log = _("Failed to retrieve log: %s" % e)
try:
debug_info = "\nLog Link: %s" % log_link
debug_info += "\nHTML Response: %s" % response
LOGGER.error(debug_info)
except:
LOGGER.exception("failed to create debug info")
response = {"log": LinkJobLogs._make_hdfs_links(log), "debug": debug_info}
return JsonResponse(response)
@check_job_permission
def job_single_logs(request, job):
"""
Try to smartly detect the most useful task attempt (e.g. Oozie launcher, failed task) and get its MR logs.
"""
def cmp_exec_time(task1, task2):
return cmp(task1.execStartTimeMs, task2.execStartTimeMs)
task = None
if offset != 0:
params["start"] = offset
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
api_resp = None
try:
api_resp = root.get(link, params=params)
log = (
html.fromstring(api_resp, parser=html.HTMLParser())
.xpath("/html/body/table/tbody/tr/td[2]")[0]
.text_content()
)
response["status"] = 0
response["log"] = LinkJobLogs._make_hdfs_links(log)
except Exception, e:
response["log"] = _("Failed to retrieve log: %s" % e)
try:
debug_info = "\nLog Link: %s" % log_link
if api_resp:
debug_info += "\nHTML Response: %s" % response
response["debug"] = debug_info
LOG.error(debug_info)
except:
LOG.exception("failed to create debug info")
return JsonResponse(response)
@check_job_permission
params = {
'doAs': request.user.username
}
if offset != 0:
params['start'] = offset
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
api_resp = None
try:
api_resp = root.get(link, params=params)
log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
response['status'] = 0
response['log'] = LinkJobLogs._make_hdfs_links(log, is_embeddable)
except Exception, e:
response['log'] = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
if api_resp:
debug_info += '\nHTML Response: %s' % response
response['debug'] = debug_info
LOG.error(debug_info)
except:
LOG.exception('failed to create debug info')
return JsonResponse(response)
@check_job_permission
