def _filter_oozie_jobs(user, filters, kwargs): text_filters = _extract_query_params(filters) if not has_dashboard_jobs_access(user): kwargs['filters'].append(('user', user.username)) elif 'username' in text_filters: kwargs['filters'].append(('user', text_filters['username'])) if 'time' in filters: kwargs['filters'].extend([ ('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1])) ]) if hasattr(ENABLE_OOZIE_BACKEND_FILTERING, 'get') and ENABLE_OOZIE_BACKEND_FILTERING.get( ) and text_filters.get('text'): kwargs['filters'].extend([('text', text_filters.get('text'))]) if filters['pagination']: kwargs['offset'] = filters['pagination']['offset'] kwargs['cnt'] = min( filters['pagination']['limit'], hasattr(OOZIE_JOBS_COUNT, 'get') and OOZIE_JOBS_COUNT.get()) if filters.get('states'): states_filters = { 'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'], } for _state in filters.get('states'): for _status in states_filters[_state]: kwargs['filters'].extend([('status', _status)])
def apps(self, filters): filter_params = { 'task_types': None, 'task_states': None, 'task_text': None, 'jobid': self.app_id, 'pagenum': 1 } filters = _extract_query_params(filters) if filters.get('text'): filter_params['task_text'] = filters['text'] if filters.get('states'): task_states = [] if 'completed' in filters['states']: task_states.append('succeeded') if 'running' in filters['states']: task_states.extend(['running', 'pending']) if 'failed' in filters['states']: task_states.extend(['running', 'killed']) filter_params['task_states'] = task_states if filters.get('types') and len(filters.get('types')) == 1: filter_params['task_types'] = filters['types'][0] tasks = NativeYarnApi(self.user).get_tasks(**filter_params) return { 'apps': [self._massage_task(task) for task in tasks], 'total': len(tasks) }
def apps(self, filters): filter_params = { 'user': self.user, 'username': '', 'text': '', 'state': 'all', 'states': '' } filter_params.update(_extract_query_params(filters)) if filters.get('states'): filter_params['states'] = filters['states'] if 'time' in filters: filter_params['time_value'] = int(filters['time']['time_value']) filter_params['time_unit'] = filters['time']['time_unit'] jobs = NativeYarnApi(self.user).get_jobs(**filter_params) return [{ 'id': app.jobId, 'name': app.name, 'type': app.applicationType, 'status': app.status, 'apiStatus': self._api_status(app.status), 'user': self.user.username, 'progress': app.progress, 'duration': 10 * 3600, 'submitted': 10 * 3600 } for app in jobs]
def apps(self, filters): filter_params = { 'task_types': None, 'task_states': None, 'task_text': None, 'jobid': self.app_id, 'pagenum': 1 } filters = _extract_query_params(filters) if filters.get('text'): filter_params['task_text'] = filters['text'] if filters.get('states'): filter_params['task_states'] = filters['states'] if filters.get('types') and len(filters.get('types')) == 1: filter_params['task_types'] = filters['types'][0] return { 'apps': [ self._massage_task(task) for task in NativeYarnApi(self.user).get_tasks(**filter_params) ], 'total': None }
def apps(self, filters): filter_params = { 'task_types': None, 'task_states': None, 'task_text': None, 'jobid': self.app_id, 'pagenum': 1 } filters = _extract_query_params(filters) if filters.get('text'): filter_params['task_text'] = filters['text'] if filters.get('states'): task_states = [] if 'completed' in filters['states']: task_states.append('succeeded') if 'running' in filters['states']: task_states.extend(['running', 'pending']) if 'failed' in filters['states']: task_states.extend(['running', 'killed']) filter_params['task_states'] = task_states if filters.get('types') and len(filters.get('types')) == 1: filter_params['task_types'] = filters['types'][0] tasks = NativeYarnApi(self.user).get_tasks(**filter_params) return { 'apps': [self._massage_task(task) for task in tasks], 'total': len(tasks) }
def apps(self, filters): kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []} text_filters = _extract_query_params(filters) if not has_dashboard_jobs_access(self.user): kwargs['filters'].append(('user', self.user.username)) elif 'user' in text_filters: kwargs['filters'].append(('user', text_filters['username'])) if 'time' in filters: kwargs['filters'].extend([ ('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1])) ]) if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'): kwargs['filters'].extend([('text', text_filters.get('text'))]) if filters['pagination']: kwargs['offset'] = filters['pagination']['offset'] kwargs['cnt'] = min(filters['pagination']['limit'], OOZIE_JOBS_COUNT.get()) if filters.get('states'): states_filters = { 'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'], } for _state in filters.get('states'): for _status in states_filters[_state]: kwargs['filters'].extend([('status', _status)]) oozie_api = get_oozie(self.user) wf_list = oozie_api.get_workflows(**kwargs) return { 'apps': [{ 'id': app['id'], 'name': app['appName'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'type': 'workflow', 'user': app['user'], 'progress': app['progress'], 'duration': app['durationInMillis'], 'submitted': app['startTimeInMillis'] } for app in massaged_oozie_jobs_for_json( wf_list.jobs, self.user)['jobs']], 'total': wf_list.total }
def apps(self, filters): kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []} text_filters = _extract_query_params(filters) if not has_dashboard_jobs_access(self.user): kwargs['filters'].append(('user', self.user.username)) elif 'user' in text_filters: kwargs['filters'].append(('user', text_filters['username'])) if 'time' in filters: kwargs['filters'].extend([ ('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1])) ]) if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'): kwargs['filters'].extend([('text', text_filters.get('text'))]) if filters.get('states'): states_filters = { 'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'], } for _state in filters.get('states'): for _status in states_filters[_state]: kwargs['filters'].extend([('status', _status)]) oozie_api = get_oozie(self.user) wf_list = oozie_api.get_workflows(**kwargs) return [{ 'id': app.id, 'name': app.appName, 'status': app.status, 'apiStatus': self._api_status(app.status), 'type': 'workflow', 'user': app.user, 'progress': app.get_progress(), 'duration': 10 * 3600, 'submitted': 10 * 3600 } for app in wf_list.jobs]
def apps(self, filters): filter_params = { 'user': self.user, 'username': '', 'text': '', 'state': 'all', 'states': '' } filter_params.update(_extract_query_params(filters)) if filters.get('states'): filter_params['states'] = filters['states'] if 'time' in filters: filter_params['time_value'] = int(filters['time']['time_value']) filter_params['time_unit'] = filters['time']['time_unit'] filter_params['limit'] = MAX_JOB_FETCH.get() jobs = NativeYarnApi(self.user).get_jobs(**filter_params) apps = [ massage_job_for_json(job, user=self.user) for job in sorted(jobs, key=lambda job: job.jobId, reverse=True) ] return { 'apps': [{ 'id': app['id'], 'name': app['name'], 'type': app['applicationType'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'user': app['user'], 'progress': app['progress'], 'queue': app['queueName'], 'duration': app['durationMs'], 'submitted': app['startTimeMs'], 'canWrite': app['canKill'], } for app in apps], 'total': len(apps) }
def apps(self, filters): kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []} text_filters = _extract_query_params(filters) if not has_dashboard_jobs_access(self.user): kwargs['filters'].append(('user', self.user.username)) elif 'user' in text_filters: kwargs['filters'].append(('user', text_filters['username'])) if 'time' in filters: kwargs['filters'].extend([('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1]))]) if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'): kwargs['filters'].extend([('text', text_filters.get('text'))]) if filters['pagination']: kwargs['offset'] = filters['pagination']['offset'] kwargs['cnt'] = min(filters['pagination']['limit'], OOZIE_JOBS_COUNT.get()) if filters.get('states'): states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],} for _state in filters.get('states'): for _status in states_filters[_state]: kwargs['filters'].extend([('status', _status)]) oozie_api = get_oozie(self.user) wf_list = oozie_api.get_workflows(**kwargs) return { 'apps':[{ 'id': app['id'], 'name': app['appName'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'type': 'workflow', 'user': app['user'], 'progress': app['progress'], 'duration': app['durationInMillis'], 'submitted': app['startTimeInMillis'] } for app in massaged_oozie_jobs_for_json(wf_list.jobs, self.user)['jobs']], 'total': wf_list.total }
def apps(self, filters): filter_params = { 'user': self.user, 'username': '', 'text': '', 'state': 'all', 'states': '' } filter_params.update(_extract_query_params(filters)) if filters.get('states'): filter_params['states'] = filters['states'] if 'time' in filters: filter_params['time_value'] = int(filters['time']['time_value']) filter_params['time_unit'] = filters['time']['time_unit'] filter_params['limit'] = MAX_JOB_FETCH.get() jobs = NativeYarnApi(self.user).get_jobs(**filter_params) apps = [massage_job_for_json(job, user=self.user) for job in sorted(jobs, key=lambda job: job.jobId, reverse=True)] return { 'apps': [{ 'id': app['id'], 'name': app['name'], 'type': app['applicationType'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'user': app['user'], 'progress': app['progress'], 'queue': app['queueName'], 'duration': app['durationMs'], 'submitted': app['startTimeMs'], 'canWrite': app['canKill'], } for app in apps], 'total': len(apps) }
def apps(self, filters): filter_params = { 'user': self.user, 'username': '', 'text': '', 'state': 'all', 'states': '' } filter_params.update(_extract_query_params(filters)) if filters.get('states'): filter_params['states'] = filters['states'] if 'time' in filters: filter_params['time_value'] = int(filters['time']['time_value']) filter_params['time_unit'] = filters['time']['time_unit'] jobs = NativeYarnApi(self.user).get_jobs(**filter_params) apps = [ massage_job_for_json(job, user=self.user) for job in reversed(jobs) ] return { 'apps': [{ 'id': app['id'], 'name': app['name'], 'type': app['applicationType'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'user': app['user'], 'progress': app['progress'], 'duration': app['durationMs'], 'submitted': app['startTimeMs'] } for app in apps], 'total': None }
def _filter_oozie_jobs(user, filters, kwargs): text_filters = _extract_query_params(filters) if has_oozie_installed and not has_dashboard_jobs_access(user): kwargs['filters'].append(('user', user.username)) elif 'username' in text_filters: kwargs['filters'].append(('user', text_filters['username'])) if 'time' in filters: kwargs['filters'].extend([('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1]))]) if has_oozie_installed and ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'): kwargs['filters'].extend([('text', text_filters.get('text'))]) if filters['pagination']: kwargs['offset'] = filters['pagination']['offset'] kwargs['cnt'] = min(filters['pagination']['limit'], hasattr(OOZIE_JOBS_COUNT, 'get') and OOZIE_JOBS_COUNT.get()) if filters.get('states'): states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],} for _state in filters.get('states'): for _status in states_filters[_state]: kwargs['filters'].extend([('status', _status)])