def apps(self, filters):
filter_params = {
'user': self.user,
'username': '',
'text': '',
'state': 'all',
'states': ''
}
filter_params.update(_extract_query_params(filters))
if filters.get('states'):
filter_params['states'] = filters['states']
if 'time' in filters:
filter_params['time_value'] = int(filters['time']['time_value'])
filter_params['time_unit'] = filters['time']['time_unit']
filter_params['limit'] = MAX_JOB_FETCH.get()
jobs = NativeYarnApi(self.user).get_jobs(**filter_params)
apps = [
massage_job_for_json(job, user=self.user)
for job in sorted(jobs, key=lambda job: job.jobId, reverse=True)
]
return {
'apps': [{
'id': app['id'],
'name': app['name'],
'type': app['applicationType'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'user': app['user'],
'progress': app['progress'],
'queue': app['queueName'],
'duration': app['durationMs'],
'submitted': app['startTimeMs'],
'canWrite': app['canKill'],
} for app in apps],
'total':
len(apps)
}
def apps(self, filters):
filter_params = {
'user': self.user,
'username': '',
'text': '',
'state': 'all',
'states': ''
}
filter_params.update(_extract_query_params(filters))
if filters.get('states'):
filter_params['states'] = filters['states']
if 'time' in filters:
filter_params['time_value'] = int(filters['time']['time_value'])
filter_params['time_unit'] = filters['time']['time_unit']
filter_params['limit'] = MAX_JOB_FETCH.get()
jobs = NativeYarnApi(self.user).get_jobs(**filter_params)
apps = [massage_job_for_json(job, user=self.user) for job in sorted(jobs, key=lambda job: job.jobId, reverse=True)]
return {
'apps': [{
'id': app['id'],
'name': app['name'],
'type': app['applicationType'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'user': app['user'],
'progress': app['progress'],
'queue': app['queueName'],
'duration': app['durationMs'],
'submitted': app['startTimeMs'],
'canWrite': app['canKill'],
} for app in apps],
'total': len(apps)
}
def apps(self, filters):
filter_params = {
'user': self.user,
'username': '',
'text': '',
'state': 'all',
'states': ''
}
filter_params.update(_extract_query_params(filters))
if filters.get('states'):
filter_params['states'] = filters['states']
if 'time' in filters:
filter_params['time_value'] = int(filters['time']['time_value'])
filter_params['time_unit'] = filters['time']['time_unit']
jobs = NativeYarnApi(self.user).get_jobs(**filter_params)
apps = [
massage_job_for_json(job, user=self.user) for job in reversed(jobs)
]
return {
'apps': [{
'id': app['id'],
'name': app['name'],
'type': app['applicationType'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'user': app['user'],
'progress': app['progress'],
'duration': app['durationMs'],
'submitted': app['startTimeMs']
} for app in apps],
'total':
None
}
def app(self, appid):
try:
job = NativeYarnApi(self.user).get_job(jobid=appid)
except ApplicationNotRunning as e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(self.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
except JobExpired as e:
raise PopupException(_('Job %s has expired.') % appid, detail=_('Cannot be found on the History Server.'))
except Exception as e:
msg = 'Could not find job %s.'
LOG.exception(msg % appid)
raise PopupException(_(msg) % appid, detail=e)
app = massage_job_for_json(job, user=self.user)
common = {
'id': app['id'],
'name': app['name'],
'type': app['applicationType'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'user': app['user'],
'progress': app['progress'],
'duration': app['durationMs'],
'submitted': app['startTimeMs'],
'canWrite': app['canKill'],
}
if app['applicationType'] == 'MR2' or app['applicationType'] == 'MAPREDUCE':
common['type'] = 'MAPREDUCE'
if app['desiredMaps'] is None or app['finishedMaps'] is None:
app['mapsPercentComplete'] = 100
if app['desiredReduces'] is None or app['finishedReduces'] is None:
app['reducesPercentComplete'] = 100
common['properties'] = {
'maps_percent_complete': app['mapsPercentComplete'] or 0,
'reduces_percent_complete': app['reducesPercentComplete'] or 0,
'finishedMaps': app['finishedMaps'] or 0,
'finishedReduces': app['finishedReduces'] or 0,
'desiredMaps': app['desiredMaps'] or 0,
'desiredReduces': app['desiredReduces'] or 0,
'durationFormatted': app['durationFormatted'],
'startTimeFormatted': app['startTimeFormatted'],
'diagnostics': app['diagnostics'] if app['diagnostics'] else '',
'tasks': [],
'metadata': [],
'counters': []
}
elif app['applicationType'] == 'SPARK':
app['logs'] = job.logs_url if hasattr(job, 'logs_url') else ''
app['trackingUrl'] = job.trackingUrl if hasattr(job, 'trackingUrl') else ''
common['type'] = 'SPARK'
common['properties'] = {
'metadata': [{'name': name, 'value': value} for name, value in app.items() if name != "url" and name != "killUrl"],
'executors': []
}
if hasattr(job, 'metrics'):
common['metrics'] = job.metrics
elif app['applicationType'] == 'YarnV2':
common['applicationType'] = app.get('type')
common['properties'] = {
'startTime': job.startTime,
'finishTime': job.finishTime,
'elapsedTime': job.duration,
'attempts': [],
'diagnostics': job.diagnostics
}
return common
except ApplicationNotRunning, e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(self.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
except JobExpired, e:
raise PopupException(
_('Job %s has expired.') % appid,
detail=_('Cannot be found on the History Server.'))
except Exception, e:
msg = 'Could not find job %s.'
LOG.exception(msg % appid)
raise PopupException(_(msg) % appid, detail=e)
app = massage_job_for_json(job, user=self.user)
common = {
'id': app['id'],
'name': app['name'],
'type': app['applicationType'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'user': app['user'],
'progress': app['progress'],
'duration': app['durationMs'],
'submitted': app['startTimeMs'],
'canWrite': app['canKill'],
}
if app['applicationType'] == 'MR2' or app[
job = NativeYarnApi(self.user).get_job(jobid=appid)
except ApplicationNotRunning, e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(self.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
except JobExpired, e:
raise PopupException(_('Job %s has expired.') % appid, detail=_('Cannot be found on the History Server.'))
except Exception, e:
msg = 'Could not find job %s.'
LOG.exception(msg % appid)
raise PopupException(_(msg) % appid, detail=e)
app = massage_job_for_json(job, user=self.user)
common = {
'id': app['id'],
'name': app['name'],
'type': app['applicationType'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'user': app['user'],
'progress': app['progress'],
'duration': app['durationMs'],
'submitted': app['startTimeMs'],
'canWrite': app['canKill'],
}
if app['applicationType'] == 'MR2' or app['applicationType'] == 'MAPREDUCE':
