def get_log(request, oozie_workflow, make_links=True, log_start_pattern=None, log_end_pattern=None):
logs = {}
is_really_done = False
for action in oozie_workflow.get_working_actions():
try:
if action.externalId:
data = job_single_logs(request, **{'job': action.externalId})
if data and 'logs' in data:
action_logs = data['logs'][1]
if log_start_pattern:
re_log_start = re.compile(log_start_pattern, re.M | re.DOTALL)
if re_log_start.search(action_logs):
action_logs = re.search(re_log_start, action_logs).group(1).strip()
else:
LOG.debug('Failed to find given start log pattern in logs: %s' % log_start_pattern)
if make_links:
action_logs = LinkJobLogs._make_links(action_logs)
logs[action.name] = action_logs
if log_end_pattern:
re_log_end = re.compile(log_end_pattern)
is_really_done = re_log_end.search(action_logs) is not None
if is_really_done and not action_logs:
LOG.warn('Unable to scrape full logs, try increasing the jobbrowser log_offset configuration value.')
except Exception, e:
LOG.error('An error occurred while watching the job running: %(error)s' % {'error': e})
is_really_done = True
def get_log(request,
oozie_workflow,
make_links=True,
log_start_pattern=None,
log_end_pattern=None):
logs = {}
is_really_done = False
for action in oozie_workflow.get_working_actions():
try:
if action.externalId:
data = job_single_logs(request, **{'job': action.externalId})
if data and 'logs' in data:
action_logs = data['logs'][1]
if log_start_pattern:
re_log_start = re.compile(log_start_pattern,
re.M | re.DOTALL)
if re_log_start.search(action_logs):
action_logs = re.search(
re_log_start, action_logs).group(1).strip()
else:
LOG.debug(
'Failed to find given start log pattern in logs: %s'
% log_start_pattern)
if log_end_pattern:
re_log_end = re.compile(log_end_pattern)
is_really_done = re_log_end.search(
action_logs
) is not None or oozie_workflow.status == 'KILLED'
if is_really_done and not action_logs:
LOG.warning(
'Unable to scrape full logs, try increasing the jobbrowser log_offset configuration value.'
)
if make_links:
action_logs = LinkJobLogs._make_links(action_logs)
logs[action.name] = action_logs
except Exception:
LOG.exception('An error occurred while watching the job running')
is_really_done = True
workflow_actions = _get_workflow_actions(oozie_workflow, logs,
is_really_done)
return logs, workflow_actions, is_really_done
def get_log(self, request, oozie_workflow):
logs = {}
is_really_done = False
for action in oozie_workflow.get_working_actions():
try:
if action.externalId:
data = job_single_logs(request, **{'job': action.externalId})
if data:
matched_logs = self._match_logs(data)
logs[action.name] = LinkJobLogs._make_links(matched_logs)
is_really_done = OozieApi.RE_LOG_END.search(data['logs'][1]) is not None
except Exception, e:
LOG.error('An error happen while watching the job running: %(error)s' % {'error': e})
is_really_done = True
def get_log(self, request, oozie_workflow):
logs = {}
is_really_done = False
for action in oozie_workflow.get_working_actions():
try:
if action.externalId:
data = job_single_logs(request, **{'job': action.externalId})
if data and 'logs' in data:
matched_logs = self._match_logs(data)
if matched_logs:
logs[action.name] = LinkJobLogs._make_links(matched_logs)
is_really_done = OozieApi.RE_LOG_END.search(data['logs'][1]) is not None
if is_really_done and not matched_logs:
LOG.warn('Unable to scrape full pig logs, try increasing the jobbrowser log_offset configuration value.')
except Exception, e:
LOG.error('An error occurred while watching the job running: %(error)s' % {'error': e})
is_really_done = True
def get_log(self, request, oozie_workflow):
logs = {}
is_really_done = False
for action in oozie_workflow.get_working_actions():
try:
if action.externalId:
data = job_single_logs(request, **{"job": action.externalId})
if data and "logs" in data:
matched_logs = self._match_logs(data)
if matched_logs:
logs[action.name] = LinkJobLogs._make_links(matched_logs)
is_really_done = OozieApi.RE_LOG_END.search(data["logs"][1]) is not None
if is_really_done and not matched_logs:
LOG.warn(
"Unable to scrape full pig logs, try increasing the jobbrowser log_offset configuration value."
)
except Exception, e:
LOG.error("An error occurred while watching the job running: %(error)s" % {"error": e})
is_really_done = True
logs = [_("Failed to retrieve log. TaskTracker not ready.")] * 4
context = {
"attempt": attempt,
"taskid": taskid,
"joblnk": job_link,
"task": task,
"logs": logs,
"first_log_tab": first_log_tab,
}
if request.GET.get('format') == 'python':
return context
else:
context['logs'] = [
LinkJobLogs._make_links(log) for i, log in enumerate(logs)
]
if request.GET.get('format') == 'json':
response = {
"logs": context['logs'],
"isRunning": job.status.lower() in ('running', 'pending', 'prep')
}
return JsonResponse(response)
else:
return render("attempt_logs.mako", request, context)
@check_job_permission
def task_attempt_counters(request, job, taskid, attemptid):
"""
def test_make_log_links():
"""
Unit test for models.LinkJobLogs._make_links
"""
# FileBrowser
assert_equal(
"""<a href="/filebrowser/view=/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a> <dir>""",
LinkJobLogs._make_links(
'hdfs://localhost:8020/user/romain/tmp <dir>'))
assert_equal(
"""<a href="/filebrowser/view=/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a><dir>""",
LinkJobLogs._make_links('hdfs://localhost:8020/user/romain/tmp<dir>'))
assert_equal(
"""output: <a href="/filebrowser/view=/user/romain/tmp" target="_blank">/user/romain/tmp</a> <dir>""",
LinkJobLogs._make_links('output: /user/romain/tmp <dir>'))
assert_equal(
'Successfully read 3760 records (112648 bytes) from: "<a href="/filebrowser/view=/user/hue/pig/examples/data/midsummer.txt" target="_blank">/user/hue/pig/examples/data/midsummer.txt</a>"',
LinkJobLogs._make_links(
'Successfully read 3760 records (112648 bytes) from: "/user/hue/pig/examples/data/midsummer.txt"'
))
assert_equal(
'data,upper_case MAP_ONLY <a href="/filebrowser/view=/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>,',
LinkJobLogs._make_links(
'data,upper_case MAP_ONLY hdfs://localhost:8020/user/romain/out/fffff,'
))
assert_equal(
'MAP_ONLY <a href="/filebrowser/view=/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>\n2013',
LinkJobLogs._make_links(
'MAP_ONLY hdfs://localhost:8020/user/romain/out/fffff\n2013'))
assert_equal(
' <a href="/filebrowser/view=/jobs.tsv" target="_blank">/jobs.tsv</a> ',
LinkJobLogs._make_links(' /jobs.tsv '))
assert_equal(
'<a href="/filebrowser/view=/user/romain/job_pos_2012.tsv" target="_blank">hdfs://localhost:8020/user/romain/job_pos_2012.tsv</a>',
LinkJobLogs._make_links(
'hdfs://localhost:8020/user/romain/job_pos_2012.tsv'))
# JobBrowser
assert_equal(
"""<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
LinkJobLogs._make_links('job_201306261521_0058'))
assert_equal(
"""Hadoop Job IDs executed by Pig: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
LinkJobLogs._make_links(
'Hadoop Job IDs executed by Pig: job_201306261521_0058'))
assert_equal(
"""MapReduceLauncher - HadoopJobId: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
LinkJobLogs._make_links(
'MapReduceLauncher - HadoopJobId: job_201306261521_0058'))
assert_equal(
"""- More information at: http://localhost:50030/jobdetails.jsp?jobid=<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
LinkJobLogs._make_links(
'- More information at: http://localhost:50030/jobdetails.jsp?jobid=job_201306261521_0058'
))
assert_equal(
""" Logging error messages to: <a href="/jobbrowser/jobs/job_201307091553_0028" target="_blank">job_201307091553_0028</a>/attempt_201307091553_002""",
LinkJobLogs._make_links(
' Logging error messages to: job_201307091553_0028/attempt_201307091553_002'
))
assert_equal(
""" pig-<a href="/jobbrowser/jobs/job_201307091553_0028" target="_blank">job_201307091553_0028</a>.log""",
LinkJobLogs._make_links(' pig-job_201307091553_0028.log'))
assert_equal(
"""MapReduceLauncher - HadoopJobId: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>. Look at the UI""",
LinkJobLogs._make_links(
'MapReduceLauncher - HadoopJobId: job_201306261521_0058. Look at the UI'
))
except urllib2.URLError:
logs = [_("Failed to retrieve log. TaskTracker not ready.")] * 4
context = {
"attempt": attempt,
"taskid": taskid,
"joblnk": job_link,
"task": task,
"logs": logs,
"first_log_tab": first_log_tab,
}
if request.GET.get('format') == 'python':
return context
else:
context['logs'] = [LinkJobLogs._make_links(log) for i, log in enumerate(logs)]
if request.GET.get('format') == 'json':
response = {
"logs": context['logs'],
"isRunning": job.status.lower() in ('running', 'pending', 'prep')
}
return JsonResponse(response)
else:
return render("attempt_logs.mako", request, context)
@check_job_permission
def task_attempt_counters(request, job, taskid, attemptid):
"""
We get here from /jobs/jobid/tasks/taskid/attempts/attemptid/counters
(phew!)
def test_make_log_links():
"""
Unit test for models.LinkJobLogs._make_links
"""
# FileBrowser
assert_equal(
"""<a href="/filebrowser/view=/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a> <dir>""",
LinkJobLogs._make_links('hdfs://localhost:8020/user/romain/tmp <dir>')
)
assert_equal(
"""<a href="/filebrowser/view=/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a><dir>""",
LinkJobLogs._make_links('hdfs://localhost:8020/user/romain/tmp<dir>')
)
assert_equal(
"""output: <a href="/filebrowser/view=/user/romain/tmp" target="_blank">/user/romain/tmp</a> <dir>""",
LinkJobLogs._make_links('output: /user/romain/tmp <dir>')
)
assert_equal(
'Successfully read 3760 records (112648 bytes) from: "<a href="/filebrowser/view=/user/hue/pig/examples/data/midsummer.txt" target="_blank">/user/hue/pig/examples/data/midsummer.txt</a>"',
LinkJobLogs._make_links('Successfully read 3760 records (112648 bytes) from: "/user/hue/pig/examples/data/midsummer.txt"')
)
assert_equal(
'data,upper_case MAP_ONLY <a href="/filebrowser/view=/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>,',
LinkJobLogs._make_links('data,upper_case MAP_ONLY hdfs://localhost:8020/user/romain/out/fffff,')
)
assert_equal(
'MAP_ONLY <a href="/filebrowser/view=/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>\n2013',
LinkJobLogs._make_links('MAP_ONLY hdfs://localhost:8020/user/romain/out/fffff\n2013')
)
assert_equal(
' <a href="/filebrowser/view=/jobs.tsv" target="_blank">/jobs.tsv</a> ',
LinkJobLogs._make_links(' /jobs.tsv ')
)
assert_equal(
'<a href="/filebrowser/view=/user/romain/job_pos_2012.tsv" target="_blank">hdfs://localhost:8020/user/romain/job_pos_2012.tsv</a>',
LinkJobLogs._make_links('hdfs://localhost:8020/user/romain/job_pos_2012.tsv')
)
# JobBrowser
assert_equal(
"""<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
LinkJobLogs._make_links('job_201306261521_0058')
)
assert_equal(
"""Hadoop Job IDs executed by Pig: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
LinkJobLogs._make_links('Hadoop Job IDs executed by Pig: job_201306261521_0058')
)
assert_equal(
"""MapReduceLauncher - HadoopJobId: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
LinkJobLogs._make_links('MapReduceLauncher - HadoopJobId: job_201306261521_0058')
)
assert_equal(
"""- More information at: http://localhost:50030/jobdetails.jsp?jobid=<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
LinkJobLogs._make_links('- More information at: http://localhost:50030/jobdetails.jsp?jobid=job_201306261521_0058')
)
assert_equal(
""" Logging error messages to: job_201307091553_0028/attempt_201307091553_002""",
LinkJobLogs._make_links(' Logging error messages to: job_201307091553_0028/attempt_201307091553_002')
)
assert_equal(
""" pig-job_201307091553_0028.log""",
LinkJobLogs._make_links(' pig-job_201307091553_0028.log')
)
except urllib2.URLError:
logs = [_("Failed to retrieve log. TaskTracker not ready.")] * 4
context = {
"attempt": attempt,
"taskid": taskid,
"joblnk": job_link,
"task": task,
"logs": logs,
"first_log_tab": first_log_tab,
}
if request.GET.get('format') == 'python':
return context
else:
context['logs'] = [LinkJobLogs._make_links(log) for i, log in enumerate(logs)]
if request.GET.get('format') == 'json':
response = {
"logs": context['logs'],
"isRunning": job.status.lower() in ('running', 'pending', 'prep')
}
return JsonResponse(response)
else:
return render("attempt_logs.mako", request, context)
@check_job_permission
def task_attempt_counters(request, job, taskid, attemptid):
"""
We get here from /jobs/jobid/tasks/taskid/attempts/attemptid/counters
(phew!)
def single_task_attempt_logs(request,
job,
taskid,
attemptid,
offset=LOG_OFFSET_BYTES):
jt = get_api(request.user, request.jt)
job_link = jt.get_job_link(job.jobId)
task = job_link.get_task(taskid)
try:
attempt = task.get_attempt(attemptid)
except (KeyError, RestException) as e:
raise KeyError(
_("Cannot find attempt '%(id)s' in task") % {'id': attemptid}, e)
first_log_tab = 0
try:
# Add a diagnostic log
if hasattr(task, 'job') and hasattr(task.job, 'diagnostics'):
diagnostic_log = task.job.diagnostics
elif job_link.is_mr2:
diagnostic_log = attempt.diagnostics
else:
diagnostic_log = ", ".join(task.diagnosticMap[attempt.attemptId])
logs = [diagnostic_log]
# Add remaining logs
logs += [
section.strip() for section in attempt.get_task_log(offset=offset)
]
log_tab = [i for i, log in enumerate(logs) if log]
if log_tab:
first_log_tab = log_tab[0]
except urllib.error.URLError:
logs = [_("Failed to retrieve log. TaskTracker not ready.")] * 4
context = {
"attempt": attempt,
"taskid": taskid,
"joblnk": job_link,
"task": task,
"logs": logs,
"logs_list": attempt.get_log_list(),
"first_log_tab": first_log_tab,
}
if request.GET.get('format') == 'python':
return context
else:
context['logs'] = [
LinkJobLogs._make_links(log) for i, log in enumerate(logs)
]
if request.GET.get('format') == 'json':
response = {
"logs": context['logs'],
"logsList": context['logs_list'],
"isRunning": job.status.lower() in ('running', 'pending', 'prep')
}
return JsonResponse(response)
else:
return render("attempt_logs.mako", request, context)