def apps(self, filters): filter_params = { 'user': self.user, 'username': '', 'text': '', 'state': 'all', 'states': '' } filter_params.update(_extract_query_params(filters)) if filters.get('states'): filter_params['states'] = filters['states'] if 'time' in filters: filter_params['time_value'] = int(filters['time']['time_value']) filter_params['time_unit'] = filters['time']['time_unit'] filter_params['limit'] = MAX_JOB_FETCH.get() jobs = NativeYarnApi(self.user).get_jobs(**filter_params) apps = [ massage_job_for_json(job, user=self.user) for job in sorted(jobs, key=lambda job: job.jobId, reverse=True) ] return { 'apps': [{ 'id': app['id'], 'name': app['name'], 'type': app['applicationType'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'user': app['user'], 'progress': app['progress'], 'queue': app['queueName'], 'duration': app['durationMs'], 'submitted': app['startTimeMs'], 'canWrite': app['canKill'], } for app in apps], 'total': len(apps) }
def apps(self, filters): filter_params = { 'user': self.user, 'username': '', 'text': '', 'state': 'all', 'states': '' } filter_params.update(_extract_query_params(filters)) if filters.get('states'): filter_params['states'] = filters['states'] if 'time' in filters: filter_params['time_value'] = int(filters['time']['time_value']) filter_params['time_unit'] = filters['time']['time_unit'] filter_params['limit'] = MAX_JOB_FETCH.get() jobs = NativeYarnApi(self.user).get_jobs(**filter_params) apps = [massage_job_for_json(job, user=self.user) for job in sorted(jobs, key=lambda job: job.jobId, reverse=True)] return { 'apps': [{ 'id': app['id'], 'name': app['name'], 'type': app['applicationType'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'user': app['user'], 'progress': app['progress'], 'queue': app['queueName'], 'duration': app['durationMs'], 'submitted': app['startTimeMs'], 'canWrite': app['canKill'], } for app in apps], 'total': len(apps) }
def apps(self, filters): filter_params = { 'user': self.user, 'username': '', 'text': '', 'state': 'all', 'states': '' } filter_params.update(_extract_query_params(filters)) if filters.get('states'): filter_params['states'] = filters['states'] if 'time' in filters: filter_params['time_value'] = int(filters['time']['time_value']) filter_params['time_unit'] = filters['time']['time_unit'] jobs = NativeYarnApi(self.user).get_jobs(**filter_params) apps = [ massage_job_for_json(job, user=self.user) for job in reversed(jobs) ] return { 'apps': [{ 'id': app['id'], 'name': app['name'], 'type': app['applicationType'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'user': app['user'], 'progress': app['progress'], 'duration': app['durationMs'], 'submitted': app['startTimeMs'] } for app in apps], 'total': None }
def app(self, appid): try: job = NativeYarnApi(self.user).get_job(jobid=appid) except ApplicationNotRunning as e: if e.job.get('state', '').lower() == 'accepted': rm_api = resource_manager_api.get_resource_manager(self.user) job = Application(e.job, rm_api) else: raise e # Job has not yet been accepted by RM except JobExpired as e: raise PopupException(_('Job %s has expired.') % appid, detail=_('Cannot be found on the History Server.')) except Exception as e: msg = 'Could not find job %s.' LOG.exception(msg % appid) raise PopupException(_(msg) % appid, detail=e) app = massage_job_for_json(job, user=self.user) common = { 'id': app['id'], 'name': app['name'], 'type': app['applicationType'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'user': app['user'], 'progress': app['progress'], 'duration': app['durationMs'], 'submitted': app['startTimeMs'], 'canWrite': app['canKill'], } if app['applicationType'] == 'MR2' or app['applicationType'] == 'MAPREDUCE': common['type'] = 'MAPREDUCE' if app['desiredMaps'] is None or app['finishedMaps'] is None: app['mapsPercentComplete'] = 100 if app['desiredReduces'] is None or app['finishedReduces'] is None: app['reducesPercentComplete'] = 100 common['properties'] = { 'maps_percent_complete': app['mapsPercentComplete'] or 0, 'reduces_percent_complete': app['reducesPercentComplete'] or 0, 'finishedMaps': app['finishedMaps'] or 0, 'finishedReduces': app['finishedReduces'] or 0, 'desiredMaps': app['desiredMaps'] or 0, 'desiredReduces': app['desiredReduces'] or 0, 'durationFormatted': app['durationFormatted'], 'startTimeFormatted': app['startTimeFormatted'], 'diagnostics': app['diagnostics'] if app['diagnostics'] else '', 'tasks': [], 'metadata': [], 'counters': [] } elif app['applicationType'] == 'SPARK': app['logs'] = job.logs_url if hasattr(job, 'logs_url') else '' app['trackingUrl'] = job.trackingUrl if hasattr(job, 'trackingUrl') else '' common['type'] = 'SPARK' common['properties'] = { 'metadata': [{'name': name, 'value': value} for name, value in app.items() if name != "url" and name != "killUrl"], 'executors': [] } if hasattr(job, 'metrics'): common['metrics'] = job.metrics elif app['applicationType'] == 'YarnV2': common['applicationType'] = app.get('type') common['properties'] = { 'startTime': job.startTime, 'finishTime': job.finishTime, 'elapsedTime': job.duration, 'attempts': [], 'diagnostics': job.diagnostics } return common
except ApplicationNotRunning, e: if e.job.get('state', '').lower() == 'accepted': rm_api = resource_manager_api.get_resource_manager(self.user) job = Application(e.job, rm_api) else: raise e # Job has not yet been accepted by RM except JobExpired, e: raise PopupException( _('Job %s has expired.') % appid, detail=_('Cannot be found on the History Server.')) except Exception, e: msg = 'Could not find job %s.' LOG.exception(msg % appid) raise PopupException(_(msg) % appid, detail=e) app = massage_job_for_json(job, user=self.user) common = { 'id': app['id'], 'name': app['name'], 'type': app['applicationType'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'user': app['user'], 'progress': app['progress'], 'duration': app['durationMs'], 'submitted': app['startTimeMs'], 'canWrite': app['canKill'], } if app['applicationType'] == 'MR2' or app[
job = NativeYarnApi(self.user).get_job(jobid=appid) except ApplicationNotRunning, e: if e.job.get('state', '').lower() == 'accepted': rm_api = resource_manager_api.get_resource_manager(self.user) job = Application(e.job, rm_api) else: raise e # Job has not yet been accepted by RM except JobExpired, e: raise PopupException(_('Job %s has expired.') % appid, detail=_('Cannot be found on the History Server.')) except Exception, e: msg = 'Could not find job %s.' LOG.exception(msg % appid) raise PopupException(_(msg) % appid, detail=e) app = massage_job_for_json(job, user=self.user) common = { 'id': app['id'], 'name': app['name'], 'type': app['applicationType'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'user': app['user'], 'progress': app['progress'], 'duration': app['durationMs'], 'submitted': app['startTimeMs'], 'canWrite': app['canKill'], } if app['applicationType'] == 'MR2' or app['applicationType'] == 'MAPREDUCE':