def job_executor_logs(request, job, attempt_index=0, name='syslog', offset=LOG_OFFSET_BYTES): response = {'status': -1} try: log = '' if job.status not in ('NEW', 'SUBMITTED', 'ACCEPTED'): log = job.history_server_api.download_executors_logs(request, job, name, offset) response['status'] = 0 response['log'] = LinkJobLogs._make_hdfs_links(log) except Exception, e: response['log'] = _('Failed to retrieve executor log: %s' % e)
urlencode=False) debug_info = '' try: response = root.get(link, params=params) log = html.fromstring(response, parser=html.HTMLParser()).xpath( '/html/body/table/tbody/tr/td[2]')[0].text_content() except Exception, e: log = _('Failed to retrieve log: %s' % e) try: debug_info = '\nLog Link: %s' % log_link debug_info += '\nHTML Response: %s' % response LOGGER.error(debug_info) except: LOGGER.exception('failed to create debug info') response = {'log': LinkJobLogs._make_hdfs_links(log), 'debug': debug_info} return JsonResponse(response) @check_job_permission def job_single_logs(request, job): """ Try to smartly detect the most useful task attempt (e.g. Oozie launcher, failed task) and get its MR logs. """ def cmp_exec_time(task1, task2): return cmp(task1.execStartTimeMs, task2.execStartTimeMs) task = None failed_tasks = job.filter_tasks(task_states=('failed', ))
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False) debug_info = '' try: response = root.get(link, params=params) log = html.fromstring(response, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content() except Exception, e: log = _('Failed to retrieve log: %s' % e) try: debug_info = '\nLog Link: %s' % log_link debug_info += '\nHTML Response: %s' % response LOGGER.error(debug_info) except: LOGGER.exception('failed to create debug info') response = {'log': LinkJobLogs._make_hdfs_links(log), 'debug': debug_info} return JsonResponse(response) @check_job_permission def job_single_logs(request, job): """ Try to smartly detect the most useful task attempt (e.g. Oozie launcher, failed task) and get its MR logs. """ def cmp_exec_time(task1, task2): return cmp(task1.execStartTimeMs, task2.execStartTimeMs) task = None
params = {} if offset != 0: params['start'] = offset root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False) api_resp = None try: api_resp = root.get(link, params=params) log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath( '/html/body/table/tbody/tr/td[2]')[0].text_content() response['status'] = 0 response['log'] = LinkJobLogs._make_hdfs_links(log) except Exception, e: response['log'] = _('Failed to retrieve log: %s' % e) try: debug_info = '\nLog Link: %s' % log_link if api_resp: debug_info += '\nHTML Response: %s' % response response['debug'] = debug_info LOG.error(debug_info) except: LOG.exception('failed to create debug info') return JsonResponse(response) @check_job_permission
if log_link: link = '/%s/' % name params = {} if offset != 0: params['start'] = offset root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False) api_resp = None try: api_resp = root.get(link, params=params) log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content() response['status'] = 0 response['log'] = LinkJobLogs._make_hdfs_links(log) except Exception, e: response['log'] = _('Failed to retrieve log: %s' % e) try: debug_info = '\nLog Link: %s' % log_link if api_resp: debug_info += '\nHTML Response: %s' % response response['debug'] = debug_info LOG.error(debug_info) except: LOG.exception('failed to create debug info') return JsonResponse(response) @check_job_permission
def job_attempt_logs_json(request, job, attempt_index=0, name='syslog', offset=LOG_OFFSET_BYTES, is_embeddable=False): """For async log retrieval as Yarn servers are very slow""" log_link = None response = {'status': -1} try: jt = get_api(request.user, request.jt) app = jt.get_application(job.jobId) if app['applicationType'] == 'MAPREDUCE': if app['finalStatus'] in ('SUCCEEDED', 'FAILED', 'KILLED'): attempt_index = int(attempt_index) if not job.job_attempts['jobAttempt']: response = {'status': 0, 'log': _('Job has no tasks')} else: attempt = job.job_attempts['jobAttempt'][attempt_index] log_link = attempt['logsLink'] # Reformat log link to use YARN RM, replace node addr with node ID addr log_link = log_link.replace(attempt['nodeHttpAddress'], attempt['nodeId']) elif app['state'] == 'RUNNING': log_link = app['amContainerLogs'] elif app.get('amContainerLogs'): log_link = app.get('amContainerLogs') except (KeyError, RestException) as e: raise KeyError( _("Cannot find job attempt '%(id)s'.") % {'id': job.jobId}, e) except Exception as e: raise Exception( _("Failed to get application for job %s: %s") % (job.jobId, e)) if log_link: link = '/%s/' % name params = {'doAs': request.user.username} if offset != 0: params['start'] = offset root = Resource(get_log_client(log_link), urllib.parse.urlsplit(log_link)[2], urlencode=False) api_resp = None try: api_resp = root.get(link, params=params) log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath( '/html/body/table/tbody/tr/td[2]')[0].text_content() response['status'] = 0 response['log'] = LinkJobLogs._make_hdfs_links(log, is_embeddable) except Exception as e: response['log'] = _('Failed to retrieve log: %s' % e) try: debug_info = '\nLog Link: %s' % log_link if api_resp: debug_info += '\nHTML Response: %s' % response response['debug'] = debug_info LOG.error(debug_info) except: LOG.exception('failed to create debug info') return JsonResponse(response)
link = '/%s/' % name params = { 'doAs': request.user.username } if offset != 0: params['start'] = offset root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False) api_resp = None try: api_resp = root.get(link, params=params) log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content() response['status'] = 0 response['log'] = LinkJobLogs._make_hdfs_links(log, is_embeddable) except Exception, e: response['log'] = _('Failed to retrieve log: %s' % e) try: debug_info = '\nLog Link: %s' % log_link if api_resp: debug_info += '\nHTML Response: %s' % response response['debug'] = debug_info LOG.error(debug_info) except: LOG.exception('failed to create debug info') return JsonResponse(response) @check_job_permission
response = root.get(link, params=params) log = ( html.fromstring(response, parser=html.HTMLParser()) .xpath("/html/body/table/tbody/tr/td[2]")[0] .text_content() ) except Exception, e: log = _("Failed to retrieve log: %s" % e) try: debug_info = "\nLog Link: %s" % log_link debug_info += "\nHTML Response: %s" % response LOGGER.error(debug_info) except: LOGGER.exception("failed to create debug info") response = {"log": LinkJobLogs._make_hdfs_links(log), "debug": debug_info} return JsonResponse(response) @check_job_permission def job_single_logs(request, job): """ Try to smartly detect the most useful task attempt (e.g. Oozie launcher, failed task) and get its MR logs. """ def cmp_exec_time(task1, task2): return cmp(task1.execStartTimeMs, task2.execStartTimeMs) task = None
if offset != 0: params["start"] = offset root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False) api_resp = None try: api_resp = root.get(link, params=params) log = ( html.fromstring(api_resp, parser=html.HTMLParser()) .xpath("/html/body/table/tbody/tr/td[2]")[0] .text_content() ) response["status"] = 0 response["log"] = LinkJobLogs._make_hdfs_links(log) except Exception, e: response["log"] = _("Failed to retrieve log: %s" % e) try: debug_info = "\nLog Link: %s" % log_link if api_resp: debug_info += "\nHTML Response: %s" % response response["debug"] = debug_info LOG.error(debug_info) except: LOG.exception("failed to create debug info") return JsonResponse(response) @check_job_permission
params = { 'doAs': request.user.username } if offset != 0: params['start'] = offset root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False) api_resp = None try: api_resp = root.get(link, params=params) log = html.fromstring(api_resp, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content() response['status'] = 0 response['log'] = LinkJobLogs._make_hdfs_links(log, is_embeddable) except Exception, e: response['log'] = _('Failed to retrieve log: %s' % e) try: debug_info = '\nLog Link: %s' % log_link if api_resp: debug_info += '\nHTML Response: %s' % response response['debug'] = debug_info LOG.error(debug_info) except: LOG.exception('failed to create debug info') return JsonResponse(response) @check_job_permission