def apps(self, filters):
filter_params = {
'user': self.user,
'username': '',
'text': '',
'state': 'all',
'states': ''
}
filter_params.update(_extract_query_params(filters))
if filters.get('states'):
filter_params['states'] = filters['states']
if 'time' in filters:
filter_params['time_value'] = int(filters['time']['time_value'])
filter_params['time_unit'] = filters['time']['time_unit']
jobs = NativeYarnApi(self.user).get_jobs(**filter_params)
apps = [massage_job_for_json(job, user=self.user) for job in jobs]
return {
'apps': [{
'id': app['id'],
'name': app['name'],
'type': app['applicationType'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'user': app['user'],
'progress': app['progress'],
'duration': app['durationMs'],
'submitted': app['startTimeMs']
} for app in apps],
'total':
None
}
def profile(self, appid, app_type, app_property, app_filters):
if app_property == 'counters':
return NativeYarnApi(self.user).get_task(jobid=self.app_id, task_id=self.task_id).get_attempt(self.attempt_id).counters
return {}
def app(self, appid):
try:
job = NativeYarnApi(self.user).get_job(jobid=appid)
except ApplicationNotRunning as e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(self.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
except JobExpired as e:
raise PopupException(_('Job %s has expired.') % appid, detail=_('Cannot be found on the History Server.'))
except Exception as e:
msg = 'Could not find job %s.'
LOG.exception(msg % appid)
raise PopupException(_(msg) % appid, detail=e)
app = massage_job_for_json(job, user=self.user)
common = {
'id': app['id'],
'name': app['name'],
'type': app['applicationType'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'user': app['user'],
'progress': app['progress'],
'duration': app['durationMs'],
'submitted': app['startTimeMs'],
'canWrite': app['canKill'],
}
if app['applicationType'] == 'MR2' or app['applicationType'] == 'MAPREDUCE':
common['type'] = 'MAPREDUCE'
if app['desiredMaps'] is None or app['finishedMaps'] is None:
app['mapsPercentComplete'] = 100
if app['desiredReduces'] is None or app['finishedReduces'] is None:
app['reducesPercentComplete'] = 100
common['properties'] = {
'maps_percent_complete': app['mapsPercentComplete'] or 0,
'reduces_percent_complete': app['reducesPercentComplete'] or 0,
'finishedMaps': app['finishedMaps'] or 0,
'finishedReduces': app['finishedReduces'] or 0,
'desiredMaps': app['desiredMaps'] or 0,
'desiredReduces': app['desiredReduces'] or 0,
'durationFormatted': app['durationFormatted'],
'startTimeFormatted': app['startTimeFormatted'],
'diagnostics': app['diagnostics'] if app['diagnostics'] else '',
'tasks': [],
'metadata': [],
'counters': []
}
elif app['applicationType'] == 'SPARK':
app['logs'] = job.logs_url if hasattr(job, 'logs_url') else ''
app['trackingUrl'] = job.trackingUrl if hasattr(job, 'trackingUrl') else ''
common['type'] = 'SPARK'
common['properties'] = {
'metadata': [{'name': name, 'value': value} for name, value in app.items() if name != "url" and name != "killUrl"],
'executors': []
}
if hasattr(job, 'metrics'):
common['metrics'] = job.metrics
elif app['applicationType'] == 'YarnV2':
common['applicationType'] = app.get('type')
common['properties'] = {
'startTime': job.startTime,
'finishTime': job.finishTime,
'elapsedTime': job.duration,
'attempts': [],
'diagnostics': job.diagnostics
}
return common
def apps(self):
return [
self._massage_task(task)
for task in NativeYarnApi(self.user).get_task(
jobid=self.app_id, task_id=self.task_id).attempts
]
def apps(self):
return [
self._massage_task(task)
for task in NativeYarnApi(self.user).get_tasks(jobid=self.app_id,
pagenum=1)
]
def logs(self, appid, app_type):
task = NativeYarnApi(self.user).get_task(jobid=self.app_id, task_id=self.task_id).get_attempt(self.attempt_id)
stdout, stderr, syslog = task.get_task_log()
return {'progress': 0, 'logs': {'default': stdout, 'stdout': stdout, 'stderr': stderr, 'syslog': syslog}}
def logs(self, appid, app_type, log_name):
task = NativeYarnApi(self.user).get_task(jobid=self.app_id, task_id=self.task_id).get_attempt(self.attempt_id)
stdout, stderr, syslog = task.get_task_log()
return {'progress': 0, 'logs': syslog if log_name == 'syslog' else stderr if log_name == 'stderr' else stdout}
def apps(self):
jobs = NativeYarnApi(self.user).get_jobs(self.user,
username=self.user.username,
state='all',
text='')
return [{'id': app.jobId, 'status': app.status} for app in jobs]