def get_jobs(self, user, **kwargs):
state_filters = {
'running': 'UNDEFINED',
'completed': 'SUCCEEDED',
'failed': 'FAILED',
'killed': 'KILLED',
}
filters = {}
if kwargs['username']:
filters['user'] = kwargs['username']
if kwargs['state'] and kwargs['state'] != 'all':
filters['finalStatus'] = state_filters[kwargs['state']]
json = self.resource_manager_api.apps(**filters)
if type(json) == str and 'This is standby RM' in json:
raise Exception(json)
if json['apps']:
jobs = [Application(app) for app in json['apps']['app']]
else:
return []
if kwargs['text']:
text = kwargs['text'].lower()
jobs = filter(
lambda job: text in job.name.lower() or text in job.id.lower()
or text in job.user.lower() or text in job.queue.lower(), jobs)
return self.filter_jobs(user, jobs)
def get_job(self, jobid):
try:
# App id
jobid = jobid.replace('job', 'application')
job = self.resource_manager_api.app(jobid)['app']
if job['state'] == 'ACCEPTED':
raise ApplicationNotRunning(jobid, job)
elif job['state'] == 'KILLED':
return KilledYarnJob(self.resource_manager_api, job)
if job.get('applicationType') == 'SPARK':
job = SparkJob(job, self.resource_manager_api)
elif job.get('applicationType') == 'MAPREDUCE':
jobid = jobid.replace('application', 'job')
if job['state'] in ('NEW', 'SUBMITTED', 'ACCEPTED', 'RUNNING'):
json = self.mapreduce_api.job(self.user, jobid)
job = YarnJob(self.mapreduce_api, json['job'])
else:
json = self.history_server_api.job(self.user, jobid)
job = YarnJob(self.history_server_api, json['job'])
else:
job = Application(job, self.resource_manager_api)
except ApplicationNotRunning, e:
raise e
def get_job(self, jobid):
job_id = jobid.replace('application', 'job')
app_id = jobid.replace('job', 'application')
try:
app = self.resource_manager_api.app(app_id)['app']
if app['finalStatus'] in ('SUCCEEDED', 'FAILED', 'KILLED'):
if app['applicationType'] == 'SPARK':
job = SparkJob(app,
rm_api=self.resource_manager_api,
hs_api=self.spark_history_server_api)
elif app['state'] == 'KILLED':
job = KilledYarnJob(self.resource_manager_api, app)
else:
resp = self.history_server_api.job(self.user, job_id)
job = YarnJob(self.history_server_api, resp['job'])
else:
if app['state'] == 'ACCEPTED':
raise ApplicationNotRunning(app_id, app)
# The MapReduce API only returns JSON when the application is in a RUNNING state
elif app['state'] in (
'NEW', 'SUBMITTED',
'RUNNING') and app['applicationType'] == 'MAPREDUCE':
resp = self.mapreduce_api.job(self.user, job_id)
job = YarnJob(self.mapreduce_api, resp['job'])
else:
job = Application(app, self.resource_manager_api)
except RestException, e:
if e.code == 404: # Job not found in RM so attempt to find job in History Server
resp = self.history_server_api.job(self.user, job_id)
job = YarnJob(self.history_server_api, resp['job'])
else:
raise JobExpired(app_id)
def get_jobs(self, user, **kwargs):
state_filters = {'running': 'UNDEFINED', 'completed': 'SUCCEEDED', 'failed': 'FAILED', 'killed': 'KILLED',}
states_filters = {'running': 'NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUNNING', 'completed': 'FINISHED', 'failed': 'FAILED,KILLED',}
filters = {}
if kwargs['username']:
filters['user'] = kwargs['username']
if kwargs['state'] and kwargs['state'] != 'all':
filters['finalStatus'] = state_filters[kwargs['state']]
if kwargs.get('states'):
filters['states'] = ','.join([states_filters[_s] for _s in kwargs['states']])
if kwargs.get('limit'):
filters['limit'] = kwargs['limit']
if kwargs.get('time_value'):
filters['startedTimeBegin'] = self._get_started_time_begin(kwargs.get('time_value'), kwargs.get('time_unit'))
json = self.resource_manager_api.apps(**filters)
if type(json) == str and 'This is standby RM' in json:
raise Exception(json)
if json['apps']:
jobs = [Application(app) for app in json['apps']['app']]
else:
return []
if kwargs['text']:
text = kwargs['text'].lower()
jobs = filter(lambda job:
text in job.name.lower() or
text in job.id.lower() or
text in job.user.lower() or
text in job.queue.lower(), jobs)
return self.filter_jobs(user, jobs)
def get_job(self, jobid):
job_id = jobid.replace('application', 'job')
app_id = jobid.replace('job', 'application')
try:
app = self.resource_manager_api.app(app_id)['app']
if app['finalStatus'] in ('SUCCEEDED', 'FAILED', 'KILLED'):
if app['applicationType'] == 'SPARK':
job = SparkJob(app, rm_api=self.resource_manager_api, hs_api=self.spark_history_server_api)
elif app['state'] in ('KILLED', 'FAILED'):
job = KilledYarnJob(self.resource_manager_api, app)
else: # Job succeeded, attempt to fetch from JHS
job = self._get_job_from_history_server(job_id)
else:
if app['state'] == 'ACCEPTED':
raise ApplicationNotRunning(app_id, app)
# The MapReduce API only returns JSON when the application is in a RUNNING state
elif app['state'] in ('NEW', 'SUBMITTED', 'RUNNING') and app['applicationType'] == 'MAPREDUCE':
resp = self.mapreduce_api.job(self.user, job_id)
if not isinstance(resp, dict):
raise PopupException(_('Mapreduce Proxy API did not return JSON response, check if the job is running.'))
job = YarnJob(self.mapreduce_api, resp['job'])
else:
job = Application(app, self.resource_manager_api)
except RestException, e:
if e.code == 404: # Job not found in RM so attempt to find job in JHS
job = self._get_job_from_history_server(job_id)
else:
raise JobExpired(app_id)
def get_job(request, job_id):
try:
job = get_api(request.user, request.jt).get_job(jobid=job_id)
except ApplicationNotRunning, e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(request.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
def app(self, appid):
try:
job = NativeYarnApi(self.user).get_job(jobid=appid)
except ApplicationNotRunning, e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(self.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
def decorate(request, *args, **kwargs):
jobid = kwargs['job']
try:
job = get_api(request.user, request.jt).get_job(jobid=jobid)
except ApplicationNotRunning, e:
if e.job.get('state', '').lower() == 'accepted' and 'kill' in request.path:
rm_api = resource_manager_api.get_resource_manager(request.user)
job = Application(e.job, rm_api)
else:
# reverse() seems broken, using request.path but beware, it discards GET and POST info
return job_not_assigned(request, jobid, request.path)
def get_job(request, job_id):
try:
job = get_api(request.user, request.jt).get_job(jobid=job_id)
except ApplicationNotRunning, e:
if e.job.get('state',
'').lower() == 'accepted' and 'kill' in request.path:
rm_pool = resource_manager_api.get_resource_manager_pool()
rm_api = rm_pool.get(request.user.username)
job = Application(e.job, rm_api)
rm_pool.put(rm_api)
else:
# reverse() seems broken, using request.path but beware, it discards GET and POST info
return job_not_assigned(request, job_id, request.path)
def get_job(request, job_id):
try:
job = get_api(request.user, request.jt).get_job(jobid=job_id)
except ApplicationNotRunning as e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(request.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
except JobExpired as e:
raise PopupException(
_('Job %s has expired.') % job_id,
detail=_('Cannot be found on the History Server.'))
except Exception as e:
msg = 'Could not find job %s.'
LOG.exception(msg % job_id)
raise PopupException(_(msg) % job_id, detail=e)
return job
def app(self, appid):
try:
job = NativeYarnApi(self.user).get_job(jobid=appid)
except ApplicationNotRunning as e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(self.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
except JobExpired as e:
raise PopupException(_('Job %s has expired.') % appid, detail=_('Cannot be found on the History Server.'))
except Exception as e:
msg = 'Could not find job %s.'
LOG.exception(msg % appid)
raise PopupException(_(msg) % appid, detail=e)
app = massage_job_for_json(job, user=self.user)
common = {
'id': app['id'],
'name': app['name'],
'type': app['applicationType'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'user': app['user'],
'progress': app['progress'],
'duration': app['durationMs'],
'submitted': app['startTimeMs'],
'canWrite': app['canKill'],
}
if app['applicationType'] == 'MR2' or app['applicationType'] == 'MAPREDUCE':
common['type'] = 'MAPREDUCE'
if app['desiredMaps'] is None or app['finishedMaps'] is None:
app['mapsPercentComplete'] = 100
if app['desiredReduces'] is None or app['finishedReduces'] is None:
app['reducesPercentComplete'] = 100
common['properties'] = {
'maps_percent_complete': app['mapsPercentComplete'] or 0,
'reduces_percent_complete': app['reducesPercentComplete'] or 0,
'finishedMaps': app['finishedMaps'] or 0,
'finishedReduces': app['finishedReduces'] or 0,
'desiredMaps': app['desiredMaps'] or 0,
'desiredReduces': app['desiredReduces'] or 0,
'durationFormatted': app['durationFormatted'],
'startTimeFormatted': app['startTimeFormatted'],
'diagnostics': app['diagnostics'] if app['diagnostics'] else '',
'tasks': [],
'metadata': [],
'counters': []
}
elif app['applicationType'] == 'SPARK':
app['logs'] = job.logs_url if hasattr(job, 'logs_url') else ''
app['trackingUrl'] = job.trackingUrl if hasattr(job, 'trackingUrl') else ''
common['type'] = 'SPARK'
common['properties'] = {
'metadata': [{'name': name, 'value': value} for name, value in app.items() if name != "url" and name != "killUrl"],
'executors': []
}
if hasattr(job, 'metrics'):
common['metrics'] = job.metrics
elif app['applicationType'] == 'YarnV2':
common['applicationType'] = app.get('type')
common['properties'] = {
'startTime': job.startTime,
'finishTime': job.finishTime,
'elapsedTime': job.duration,
'attempts': [],
'diagnostics': job.diagnostics
}
return common
