def get_task_log(self, offset=0):
logs = []
attempt = self.task.job.job_attempts['jobAttempt'][-1]
log_link = attempt['logsLink']
# Get MR task logs
if self.assignedContainerId:
log_link = log_link.replace(attempt['containerId'], self.assignedContainerId)
if hasattr(self, 'nodeHttpAddress'):
log_link = log_link.replace(attempt['nodeHttpAddress'].split(':')[0], self.nodeHttpAddress.split(':')[0])
for name in ('stdout', 'stderr', 'syslog'):
link = '/%s/' % name
params = {}
if int(offset) >= 0:
params['start'] = offset
try:
log_link = re.sub('job_[^/]+', self.id, log_link)
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
response = root.get(link, params=params)
log = html.fromstring(response).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
except Exception, e:
log = _('Failed to retrieve log: %s') % e
logs.append(log)
def get_task_log(self, offset=0):
logs = []
attempt = self.task.job.job_attempts["jobAttempt"][-1]
log_link = attempt["logsLink"]
# Get MR task logs
if self.assignedContainerId:
log_link = log_link.replace(attempt["containerId"], self.assignedContainerId)
if hasattr(self, "nodeHttpAddress"):
log_link = log_link.replace(attempt["nodeHttpAddress"].split(":")[0], self.nodeHttpAddress.split(":")[0])
for name in ("stdout", "stderr", "syslog"):
link = "/%s/" % name
params = {}
if int(offset) >= 0:
params["start"] = offset
try:
log_link = re.sub("job_[^/]+", self.id, log_link)
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
response = root.get(link, params=params)
log = html.fromstring(response).xpath("/html/body/table/tbody/tr/td[2]")[0].text_content()
except Exception, e:
log = _("Failed to retrieve log: %s") % e
logs.append(log)
def test_get_log_client():
old_max_heap_size = clients.MAX_HEAP_SIZE
clients.MAX_HEAP_SIZE = 2
try:
log_link1 = "http://test1:8041/container/nonsense"
log_link2 = "http://test2:8041/container/nonsense"
log_link3 = "http://test3:8041/container/nonsense"
c1 = clients.get_log_client(log_link1)
c2 = clients.get_log_client(log_link2)
assert_not_equal(c1, c2)
assert_equal(c1, clients.get_log_client(log_link1))
clients.get_log_client(log_link3)
assert_equal(2, len(clients._log_client_heap))
base_urls = [tup[1].base_url for tup in clients._log_client_heap]
assert_true('http://test1:8041' in base_urls)
assert_true('http://test3:8041' in base_urls)
finally:
clients.MAX_HEAP_SIZE = old_max_heap_size
def get_log_list(self):
log_link, user = self.get_log_link()
if not log_link:
return []
params = {
'doAs': user
}
log_link = re.sub('job_[^/]+', str(self.id), log_link)
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
response = root.get('/', params=params)
links = html.fromstring(response, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]//a/@href')
parsed_links = map(lambda x: urlparse.urlsplit(x), links)
return map(lambda x: x and len(x) >= 2 and x[2].split('/')[-2] or '', parsed_links)
def get_task_log(self, offset=0):
logs = []
log_link, user = self.get_log_link()
if not log_link:
return ['', '', '']
for name in ('stdout', 'stderr', 'syslog'):
link = '/%s/' % name
if self.type == 'Oozie Launcher' and not self.task.job.status == 'FINISHED': # Yarn currently dumps with 500 error with doas in running state
params = {}
else:
params = {
'doAs': user
}
if int(offset) != 0:
params['start'] = offset
else:
params['start'] = 0
response = None
try:
log_link = re.sub('job_[^/]+', str(self.id), log_link)
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
response = root.get(link, params=params)
log = html.fromstring(response, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
except Exception, e:
log = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
if response:
debug_info += '\nHTML Response: %s' % response
LOG.error(debug_info)
except:
LOG.exception('failed to build debug info')
logs.append(log)
def get_task_log(self, offset=0):
logs = []
attempt = self.task.job.job_attempts['jobAttempt'][-1]
log_link = attempt['logsLink']
# Get MR task logs
# Don't hack up the urls if they've been migrated to the job history server.
for cluster in YARN_CLUSTERS.get().itervalues():
if log_link.startswith(cluster.HISTORY_SERVER_API_URL.get()):
break
else:
if self.assignedContainerId:
log_link = log_link.replace(attempt['containerId'], self.assignedContainerId)
if hasattr(self, 'nodeHttpAddress'):
log_link = log_link.replace(attempt['nodeHttpAddress'].split(':')[0], self.nodeHttpAddress.split(':')[0])
for name in ('stdout', 'stderr', 'syslog'):
link = '/%s/' % name
params = {}
if int(offset) >= 0:
params['start'] = offset
try:
log_link = re.sub('job_[^/]+', self.id, log_link)
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
response = root.get(link, params=params)
log = html.fromstring(response, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
except Exception, e:
log = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
debug_info += '\nHTML Response: %s' % response
LOGGER.error(debug_info)
except:
LOG.exception('failed to build debug info')
logs.append(log)
def job_attempt_logs_json(request, job, attempt_index=0, name='syslog', offset=0):
"""For async log retrieval as Yarn servers are very slow"""
try:
attempt_index = int(attempt_index)
attempt = job.job_attempts['jobAttempt'][attempt_index]
log_link = attempt['logsLink']
except (KeyError, RestException), e:
raise KeyError(_("Cannot find job attempt '%(id)s'.") % {'id': job.jobId}, e)
link = '/%s/' % name
params = {}
if offset and int(offset) >= 0:
params['start'] = offset
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
debug_info = ''
try:
response = root.get(link, params=params)
log = html.fromstring(response, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
except Exception, e:
log = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
debug_info += '\nHTML Response: %s' % response
LOGGER.error(debug_info)
except:
LOGGER.exception('failed to create debug info')
response = {'log': LinkJobLogs._make_hdfs_links(log), 'debug': debug_info}
def get_task_log(self, offset=0):
logs = []
attempt = self.task.job.job_attempts['jobAttempt'][-1]
log_link = attempt['logsLink']
# Generate actual task log link from logsLink url
if self.task.job.status in ('NEW', 'SUBMITTED', 'RUNNING'):
logs_path = '/node/containerlogs/'
node_url, tracking_path = log_link.split(logs_path)
container_id, user = tracking_path.strip('/').split('/')
# Replace log path tokens with actual container properties if available
if hasattr(self, 'nodeHttpAddress') and 'nodeId' in attempt:
node_url = '%s://%s' % (node_url.split('://')[0], self.nodeHttpAddress)
container_id = self.assignedContainerId if hasattr(self, 'assignedContainerId') else container_id
log_link = '%(node_url)s/%(logs_path)s/%(container)s/%(user)s' % {
'node_url': node_url,
'logs_path': logs_path.strip('/'),
'container': container_id,
'user': user
}
else: # Completed jobs
logs_path = '/jobhistory/logs/'
root_url, tracking_path = log_link.split(logs_path)
node_url, container_id, attempt_id, user = tracking_path.strip('/').split('/')
# Replace log path tokens with actual attempt properties if available
if hasattr(self, 'nodeHttpAddress') and 'nodeId' in attempt:
node_url = '%s:%s' % (self.nodeHttpAddress.split(':')[0], attempt['nodeId'].split(':')[1])
container_id = self.assignedContainerId if hasattr(self, 'assignedContainerId') else container_id
attempt_id = self.attemptId if hasattr(self, 'attemptId') else attempt_id
log_link = '%(root_url)s/%(logs_path)s/%(node)s/%(container)s/%(attempt)s/%(user)s' % {
'root_url': root_url,
'logs_path': logs_path.strip('/'),
'node': node_url,
'container': container_id,
'attempt': attempt_id,
'user': user
}
for name in ('stdout', 'stderr', 'syslog'):
link = '/%s/' % name
params = {}
if int(offset) != 0:
params['start'] = offset
response = None
try:
log_link = re.sub('job_[^/]+', self.id, log_link)
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
response = root.get(link, params=params)
log = html.fromstring(response, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
except Exception, e:
log = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
if response:
debug_info += '\nHTML Response: %s' % response
LOG.error(debug_info)
except:
LOG.exception('failed to build debug info')
logs.append(log)
def job_attempt_logs_json(request,
job,
attempt_index=0,
name='syslog',
offset=LOG_OFFSET_BYTES,
is_embeddable=False):
"""For async log retrieval as Yarn servers are very slow"""
log_link = None
response = {'status': -1}
try:
jt = get_api(request.user, request.jt)
app = jt.get_application(job.jobId)
if app['applicationType'] == 'MAPREDUCE':
if app['finalStatus'] in ('SUCCEEDED', 'FAILED', 'KILLED'):
attempt_index = int(attempt_index)
if not job.job_attempts['jobAttempt']:
response = {'status': 0, 'log': _('Job has no tasks')}
else:
attempt = job.job_attempts['jobAttempt'][attempt_index]
log_link = attempt['logsLink']
# Reformat log link to use YARN RM, replace node addr with node ID addr
log_link = log_link.replace(attempt['nodeHttpAddress'],
attempt['nodeId'])
elif app['state'] == 'RUNNING':
log_link = app['amContainerLogs']
elif app.get('amContainerLogs'):
log_link = app.get('amContainerLogs')
except (KeyError, RestException) as e:
raise KeyError(
_("Cannot find job attempt '%(id)s'.") % {'id': job.jobId}, e)
except Exception as e:
raise Exception(
_("Failed to get application for job %s: %s") % (job.jobId, e))
if log_link:
link = '/%s/' % name
params = {'doAs': request.user.username}
if offset != 0:
params['start'] = offset
root = Resource(get_log_client(log_link),
urllib.parse.urlsplit(log_link)[2],
urlencode=False)
api_resp = None
try:
api_resp = root.get(link, params=params)
log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath(
'/html/body/table/tbody/tr/td[2]')[0].text_content()
response['status'] = 0
response['log'] = LinkJobLogs._make_hdfs_links(log, is_embeddable)
except Exception as e:
response['log'] = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
if api_resp:
debug_info += '\nHTML Response: %s' % response
response['debug'] = debug_info
LOG.error(debug_info)
except:
LOG.exception('failed to create debug info')
return JsonResponse(response)
elif app['state'] == 'RUNNING':
log_link = app['amContainerLogs']
except (KeyError, RestException), e:
raise KeyError(
_("Cannot find job attempt '%(id)s'.") % {'id': job.jobId}, e)
except Exception, e:
raise Exception(
_("Failed to get application for job %s: %s") % (job.jobId, e))
if log_link:
link = '/%s/' % name
params = {'doAs': request.user.username}
if offset != 0:
params['start'] = offset
root = Resource(get_log_client(log_link),
urlparse.urlsplit(log_link)[2],
urlencode=False)
api_resp = None
try:
api_resp = root.get(link, params=params)
log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath(
'/html/body/table/tbody/tr/td[2]')[0].text_content()
response['status'] = 0
response['log'] = LinkJobLogs._make_hdfs_links(log, is_embeddable)
except Exception, e:
response['log'] = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
# Reformat log link to use YARN RM, replace node addr with node ID addr
log_link = log_link.replace(attempt['nodeHttpAddress'], attempt['nodeId'])
elif app['state'] == 'RUNNING':
log_link = app['amContainerLogs']
except (KeyError, RestException), e:
raise KeyError(_("Cannot find job attempt '%(id)s'.") % {'id': job.jobId}, e)
except Exception, e:
raise Exception(_("Failed to get application for job %s: %s") % (job.jobId, e))
if log_link:
link = '/%s/' % name
params = {}
if offset != 0:
params['start'] = offset
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
api_resp = None
try:
api_resp = root.get(link, params=params)
log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
response['status'] = 0
response['log'] = LinkJobLogs._make_hdfs_links(log)
except Exception, e:
response['log'] = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
if api_resp:
debug_info += '\nHTML Response: %s' % response
response['debug'] = debug_info
def get_task_log(self, offset=0):
logs = []
attempt = self.task.job.job_attempts['jobAttempt'][-1]
log_link = attempt['logsLink']
# Generate actual task log link from logsLink url
if self.task.job.status in ('NEW', 'SUBMITTED', 'RUNNING'):
logs_path = '/node/containerlogs/'
node_url, tracking_path = log_link.split(logs_path)
container_id, user = tracking_path.strip('/').split('/')
# Replace log path tokens with actual container properties if available
if hasattr(self, 'nodeHttpAddress') and 'nodeId' in attempt:
node_url = '%s://%s:%s' % (node_url.split('://')[0],
self.nodeHttpAddress.split(':')[0],
attempt['nodeId'].split(':')[1])
container_id = self.assignedContainerId if hasattr(
self, 'assignedContainerId') else container_id
log_link = '%(node_url)s/%(logs_path)s/%(container)s/%(user)s' % {
'node_url': node_url,
'logs_path': logs_path.strip('/'),
'container': container_id,
'user': user
}
else: # Completed jobs
logs_path = '/jobhistory/logs/'
root_url, tracking_path = log_link.split(logs_path)
node_url, container_id, attempt_id, user = tracking_path.strip(
'/').split('/')
# Replace log path tokens with actual attempt properties if available
if hasattr(self, 'nodeHttpAddress') and 'nodeId' in attempt:
node_url = '%s:%s' % (self.nodeHttpAddress.split(':')[0],
attempt['nodeId'].split(':')[1])
container_id = self.assignedContainerId if hasattr(
self, 'assignedContainerId') else container_id
attempt_id = self.attemptId if hasattr(self,
'attemptId') else attempt_id
log_link = '%(root_url)s/%(logs_path)s/%(node)s/%(container)s/%(attempt)s/%(user)s' % {
'root_url': root_url,
'logs_path': logs_path.strip('/'),
'node': node_url,
'container': container_id,
'attempt': attempt_id,
'user': user
}
for name in ('stdout', 'stderr', 'syslog'):
link = '/%s/' % name
params = {}
if int(offset) >= 0:
params['start'] = offset
try:
log_link = re.sub('job_[^/]+', self.id, log_link)
root = Resource(get_log_client(log_link),
urlparse.urlsplit(log_link)[2],
urlencode=False)
response = root.get(link, params=params)
log = html.fromstring(
response, parser=html.HTMLParser()).xpath(
'/html/body/table/tbody/tr/td[2]')[0].text_content()
except Exception, e:
log = _('Failed to retrieve log: %s' % e)
try:
debug_info = '\nLog Link: %s' % log_link
debug_info += '\nHTML Response: %s' % response
LOG.error(debug_info)
except:
LOG.exception('failed to build debug info')
logs.append(log)
def job_attempt_logs_json(request, job, attempt_index=0, name='syslog', offset=0):
"""For async log retrieval as Yarn servers are very slow"""
try:
attempt_index = int(attempt_index)
attempt = job.job_attempts['jobAttempt'][attempt_index]
log_link = attempt['logsLink']
except (KeyError, RestException), e:
raise KeyError(_("Cannot find job attempt '%(id)s'.") % {'id': job.jobId}, e)
link = '/%s/' % name
params = {}
if offset and int(offset) >= 0:
params['start'] = offset
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2])
try:
response = root.get(link, params=params)
log = html.fromstring(response).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
except Exception, e:
log = _('Failed to retrieve log: %s') % e
response = {'log': log}
return HttpResponse(json.dumps(response), mimetype="application/json")
@check_job_permission
def job_single_logs(request, job):