def get_jobs(self, user, **kwargs): state_filters = { 'running': 'UNDEFINED', 'completed': 'SUCCEEDED', 'failed': 'FAILED', 'killed': 'KILLED', } filters = {} if kwargs['username']: filters['user'] = kwargs['username'] if kwargs['state'] and kwargs['state'] != 'all': filters['finalStatus'] = state_filters[kwargs['state']] json = self.resource_manager_api.apps(**filters) if type(json) == str and 'This is standby RM' in json: raise Exception(json) if json['apps']: jobs = [Application(app) for app in json['apps']['app']] else: return [] if kwargs['text']: text = kwargs['text'].lower() jobs = filter( lambda job: text in job.name.lower() or text in job.id.lower() or text in job.user.lower() or text in job.queue.lower(), jobs) return self.filter_jobs(user, jobs)
def get_job(self, jobid): try: # App id jobid = jobid.replace('job', 'application') job = self.resource_manager_api.app(jobid)['app'] if job['state'] == 'ACCEPTED': raise ApplicationNotRunning(jobid, job) elif job['state'] == 'KILLED': return KilledYarnJob(self.resource_manager_api, job) if job.get('applicationType') == 'SPARK': job = SparkJob(job, self.resource_manager_api) elif job.get('applicationType') == 'MAPREDUCE': jobid = jobid.replace('application', 'job') if job['state'] in ('NEW', 'SUBMITTED', 'ACCEPTED', 'RUNNING'): json = self.mapreduce_api.job(self.user, jobid) job = YarnJob(self.mapreduce_api, json['job']) else: json = self.history_server_api.job(self.user, jobid) job = YarnJob(self.history_server_api, json['job']) else: job = Application(job, self.resource_manager_api) except ApplicationNotRunning, e: raise e
def get_job(self, jobid): job_id = jobid.replace('application', 'job') app_id = jobid.replace('job', 'application') try: app = self.resource_manager_api.app(app_id)['app'] if app['finalStatus'] in ('SUCCEEDED', 'FAILED', 'KILLED'): if app['applicationType'] == 'SPARK': job = SparkJob(app, rm_api=self.resource_manager_api, hs_api=self.spark_history_server_api) elif app['state'] == 'KILLED': job = KilledYarnJob(self.resource_manager_api, app) else: resp = self.history_server_api.job(self.user, job_id) job = YarnJob(self.history_server_api, resp['job']) else: if app['state'] == 'ACCEPTED': raise ApplicationNotRunning(app_id, app) # The MapReduce API only returns JSON when the application is in a RUNNING state elif app['state'] in ( 'NEW', 'SUBMITTED', 'RUNNING') and app['applicationType'] == 'MAPREDUCE': resp = self.mapreduce_api.job(self.user, job_id) job = YarnJob(self.mapreduce_api, resp['job']) else: job = Application(app, self.resource_manager_api) except RestException, e: if e.code == 404: # Job not found in RM so attempt to find job in History Server resp = self.history_server_api.job(self.user, job_id) job = YarnJob(self.history_server_api, resp['job']) else: raise JobExpired(app_id)
def get_jobs(self, user, **kwargs): state_filters = {'running': 'UNDEFINED', 'completed': 'SUCCEEDED', 'failed': 'FAILED', 'killed': 'KILLED',} states_filters = {'running': 'NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUNNING', 'completed': 'FINISHED', 'failed': 'FAILED,KILLED',} filters = {} if kwargs['username']: filters['user'] = kwargs['username'] if kwargs['state'] and kwargs['state'] != 'all': filters['finalStatus'] = state_filters[kwargs['state']] if kwargs.get('states'): filters['states'] = ','.join([states_filters[_s] for _s in kwargs['states']]) if kwargs.get('limit'): filters['limit'] = kwargs['limit'] if kwargs.get('time_value'): filters['startedTimeBegin'] = self._get_started_time_begin(kwargs.get('time_value'), kwargs.get('time_unit')) json = self.resource_manager_api.apps(**filters) if type(json) == str and 'This is standby RM' in json: raise Exception(json) if json['apps']: jobs = [Application(app) for app in json['apps']['app']] else: return [] if kwargs['text']: text = kwargs['text'].lower() jobs = filter(lambda job: text in job.name.lower() or text in job.id.lower() or text in job.user.lower() or text in job.queue.lower(), jobs) return self.filter_jobs(user, jobs)
def get_job(self, jobid): job_id = jobid.replace('application', 'job') app_id = jobid.replace('job', 'application') try: app = self.resource_manager_api.app(app_id)['app'] if app['finalStatus'] in ('SUCCEEDED', 'FAILED', 'KILLED'): if app['applicationType'] == 'SPARK': job = SparkJob(app, rm_api=self.resource_manager_api, hs_api=self.spark_history_server_api) elif app['state'] in ('KILLED', 'FAILED'): job = KilledYarnJob(self.resource_manager_api, app) else: # Job succeeded, attempt to fetch from JHS job = self._get_job_from_history_server(job_id) else: if app['state'] == 'ACCEPTED': raise ApplicationNotRunning(app_id, app) # The MapReduce API only returns JSON when the application is in a RUNNING state elif app['state'] in ('NEW', 'SUBMITTED', 'RUNNING') and app['applicationType'] == 'MAPREDUCE': resp = self.mapreduce_api.job(self.user, job_id) if not isinstance(resp, dict): raise PopupException(_('Mapreduce Proxy API did not return JSON response, check if the job is running.')) job = YarnJob(self.mapreduce_api, resp['job']) else: job = Application(app, self.resource_manager_api) except RestException, e: if e.code == 404: # Job not found in RM so attempt to find job in JHS job = self._get_job_from_history_server(job_id) else: raise JobExpired(app_id)
def get_job(request, job_id): try: job = get_api(request.user, request.jt).get_job(jobid=job_id) except ApplicationNotRunning, e: if e.job.get('state', '').lower() == 'accepted': rm_api = resource_manager_api.get_resource_manager(request.user) job = Application(e.job, rm_api) else: raise e # Job has not yet been accepted by RM
def app(self, appid): try: job = NativeYarnApi(self.user).get_job(jobid=appid) except ApplicationNotRunning, e: if e.job.get('state', '').lower() == 'accepted': rm_api = resource_manager_api.get_resource_manager(self.user) job = Application(e.job, rm_api) else: raise e # Job has not yet been accepted by RM
def decorate(request, *args, **kwargs): jobid = kwargs['job'] try: job = get_api(request.user, request.jt).get_job(jobid=jobid) except ApplicationNotRunning, e: if e.job.get('state', '').lower() == 'accepted' and 'kill' in request.path: rm_api = resource_manager_api.get_resource_manager(request.user) job = Application(e.job, rm_api) else: # reverse() seems broken, using request.path but beware, it discards GET and POST info return job_not_assigned(request, jobid, request.path)
def get_job(request, job_id): try: job = get_api(request.user, request.jt).get_job(jobid=job_id) except ApplicationNotRunning, e: if e.job.get('state', '').lower() == 'accepted' and 'kill' in request.path: rm_pool = resource_manager_api.get_resource_manager_pool() rm_api = rm_pool.get(request.user.username) job = Application(e.job, rm_api) rm_pool.put(rm_api) else: # reverse() seems broken, using request.path but beware, it discards GET and POST info return job_not_assigned(request, job_id, request.path)
def get_job(request, job_id): try: job = get_api(request.user, request.jt).get_job(jobid=job_id) except ApplicationNotRunning as e: if e.job.get('state', '').lower() == 'accepted': rm_api = resource_manager_api.get_resource_manager(request.user) job = Application(e.job, rm_api) else: raise e # Job has not yet been accepted by RM except JobExpired as e: raise PopupException( _('Job %s has expired.') % job_id, detail=_('Cannot be found on the History Server.')) except Exception as e: msg = 'Could not find job %s.' LOG.exception(msg % job_id) raise PopupException(_(msg) % job_id, detail=e) return job
def app(self, appid): try: job = NativeYarnApi(self.user).get_job(jobid=appid) except ApplicationNotRunning as e: if e.job.get('state', '').lower() == 'accepted': rm_api = resource_manager_api.get_resource_manager(self.user) job = Application(e.job, rm_api) else: raise e # Job has not yet been accepted by RM except JobExpired as e: raise PopupException(_('Job %s has expired.') % appid, detail=_('Cannot be found on the History Server.')) except Exception as e: msg = 'Could not find job %s.' LOG.exception(msg % appid) raise PopupException(_(msg) % appid, detail=e) app = massage_job_for_json(job, user=self.user) common = { 'id': app['id'], 'name': app['name'], 'type': app['applicationType'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'user': app['user'], 'progress': app['progress'], 'duration': app['durationMs'], 'submitted': app['startTimeMs'], 'canWrite': app['canKill'], } if app['applicationType'] == 'MR2' or app['applicationType'] == 'MAPREDUCE': common['type'] = 'MAPREDUCE' if app['desiredMaps'] is None or app['finishedMaps'] is None: app['mapsPercentComplete'] = 100 if app['desiredReduces'] is None or app['finishedReduces'] is None: app['reducesPercentComplete'] = 100 common['properties'] = { 'maps_percent_complete': app['mapsPercentComplete'] or 0, 'reduces_percent_complete': app['reducesPercentComplete'] or 0, 'finishedMaps': app['finishedMaps'] or 0, 'finishedReduces': app['finishedReduces'] or 0, 'desiredMaps': app['desiredMaps'] or 0, 'desiredReduces': app['desiredReduces'] or 0, 'durationFormatted': app['durationFormatted'], 'startTimeFormatted': app['startTimeFormatted'], 'diagnostics': app['diagnostics'] if app['diagnostics'] else '', 'tasks': [], 'metadata': [], 'counters': [] } elif app['applicationType'] == 'SPARK': app['logs'] = job.logs_url if hasattr(job, 'logs_url') else '' app['trackingUrl'] = job.trackingUrl if hasattr(job, 'trackingUrl') else '' common['type'] = 'SPARK' common['properties'] = { 'metadata': [{'name': name, 'value': value} for name, value in app.items() if name != "url" and name != "killUrl"], 'executors': [] } if hasattr(job, 'metrics'): common['metrics'] = job.metrics elif app['applicationType'] == 'YarnV2': common['applicationType'] = app.get('type') common['properties'] = { 'startTime': job.startTime, 'finishTime': job.finishTime, 'elapsedTime': job.duration, 'attempts': [], 'diagnostics': job.diagnostics } return common