def _get_job_selector_and_task(config, job_selector_str, require_task): if not require_task: try: # try to build job selector without task return (None, JobSelector.create(job_selector_str)) except Exception: if not except_nested(TaskNeededException, get_current_exception()): raise clear_current_exception() task = gc_create_workflow(config).task return (task, JobSelector.create(job_selector_str, task=task))
def delete(self, wms, select): selector = AndJobSelector(ClassSelector(JobClass.PROCESSING), JobSelector.create(select, task=self._task)) jobs = self.jobDB.getJobs(selector) if jobs: self._log_user.warning('Cancelling the following jobs:') self.cancel(wms, jobs, interactive=True, showJobs=True)
def cancel(self, task, wms, select): selector = AndJobSelector(ClassSelector(JobClass.PROCESSING), JobSelector.create(select, task=task)) jobs = self.job_db.get_job_list(selector) if jobs: self._log.warning('Cancelling the following jobs:') self._cancel(task, wms, jobs, interactive=self._interactive_cancel, show_jobs=True)
def __init__(self, config, name, task, eventhandler): NamedPlugin.__init__(self, config, name) (self._task, self._eventhandler) = (task, eventhandler) self._log_user = logging.getLogger('user') self._log_user_time = logging.getLogger('user.time') self._njobs_limit = config.getInt('jobs', -1, onChange = None) self._njobs_inflight = config.getInt('in flight', -1, onChange = None) self._njobs_inqueue = config.getInt('in queue', -1, onChange = None) self._chunks_submit = config.getInt('chunks submit', 100, onChange = None) self._chunks_check = config.getInt('chunks check', 100, onChange = None) self._chunks_retrieve = config.getInt('chunks retrieve', 100, onChange = None) self._job_timeout = config.getTime('queue timeout', -1, onChange = None) self._job_retries = config.getInt('max retry', -1, onChange = None) selected = JobSelector.create(config.get('selected', '', onChange = None), task = self._task) self.jobDB = config.getPlugin('job database', 'JobDB', cls = JobDB, pargs = (self.getMaxJobs(self._task), selected)) self._disabled_jobs_logfile = config.getWorkPath('disabled') self._outputProcessor = config.getPlugin('output processor', 'SandboxProcessor', cls = TaskOutputProcessor, pargs = (task,)) self._do_shuffle = config.getBool('shuffle', False, onChange = None) self._continuous = config.getBool('continuous', False, onChange = None) self._reportClass = Report.getClass(config.get('abort report', 'LocationReport', onChange = None)) self._showBlocker = True
def __init__(self, config): InfoScanner.__init__(self, config) ext_config_fn = config.getPath('source config', onChange=triggerDataResync) ext_config = create_config( ext_config_fn, useDefaultFiles=True).changeView(setSections=['global']) self._extWorkDir = ext_config.getWorkPath() logging.getLogger().disabled = True self._extWorkflow = ext_config.getPlugin('workflow', 'Workflow:global', cls='Workflow', pargs=('task', )) logging.getLogger().disabled = False self._extTask = self._extWorkflow.task selector = config.get('source job selector', '', onChange=triggerDataResync) ext_job_db = ext_config.getPlugin( 'job database', 'TextFileJobDB', cls='JobDB', pkwargs={ 'jobSelector': lambda jobNum, jobObj: jobObj.state == Job.SUCCESS }, onChange=None) self._selected = sorted( ext_job_db.getJobs(JobSelector.create(selector, task=self._extTask)))
def __init__(self, config, datasource_name): InfoScanner.__init__(self, config, datasource_name) self._ext_work_dn = config.get_dn('source directory') self._ext_output_dir = os.path.join(self._ext_work_dn, 'output') if not os.path.isdir(self._ext_output_dir): raise DatasetError('Unable to find task output directory %s' % repr(self._ext_output_dir)) self._selector = JobSelector.create(config.get('source job selector', ''))
def initGC(args): if len(args) > 0: config = getConfig(args[0]) userSelector = None if len(args) != 1: userSelector = JobSelector.create(args[1]) return (config, JobDB(config, jobSelector = userSelector)) sys.stderr.write('Syntax: %s <config file> [<job id>, ...]\n\n' % sys.argv[0]) sys.exit(os.EX_USAGE)
def reset(self, wms, select): jobs = self.jobDB.getJobs(JobSelector.create(select, task = self._task)) if jobs: self._log_user.warning('Resetting the following jobs:') self._reportClass(self.jobDB, self._task, jobs).display() if utils.getUserBool('Are you sure you want to reset the state of these jobs?', False): self.cancel(wms, self.jobDB.getJobs(ClassSelector(JobClass.PROCESSING), jobs), False, False) for jobNum in jobs: self.jobDB.commit(jobNum, Job())
def initGC(args): if len(args) > 0: config = getConfig(args[0]) userSelector = None if len(args) != 1: userSelector = JobSelector.create(args[1]) return (config.getWorkPath(), config, JobDB(config, jobSelector = userSelector)) sys.stderr.write('Syntax: %s <config file> [<job id>, ...]\n\n' % sys.argv[0]) sys.exit(os.EX_USAGE)
def __init__(self, config, datasource_name): InfoScanner.__init__(self, config, datasource_name) self._ext_work_dn = config.get_dn('source directory') self._ext_output_dir = os.path.join(self._ext_work_dn, 'output') if not os.path.isdir(self._ext_output_dir): raise DatasetError('Unable to find task output directory %s' % repr(self._ext_output_dir)) self._selector = JobSelector.create( config.get('source job selector', ''))
def delete(self, task, wms, select): selector = AndJobSelector(ClassSelector(JobClass.PROCESSING), JobSelector.create(select, task=task)) jobs = self.job_db.get_job_list(selector) if jobs: self._log.warning('Cancelling the following jobs:') self.cancel(wms, jobs, interactive=self._interactive_delete, show_jobs=True)
def __init__(self, config): InfoScanner.__init__(self, config) self._extWorkDir = config.getPath('source directory', onChange=triggerDataResync) self._extOutputDir = os.path.join(self._extWorkDir, 'output') if not os.path.isdir(self._extOutputDir): raise DatasetError('Unable to find task output directory %s' % repr(self._extOutputDir)) self._selector = JobSelector.create( config.get('source job selector', '', onChange=triggerDataResync))
def __init__(self, config): from grid_control.tasks import TaskModule newVerbosity = utils.verbosity(utils.verbosity() - 3) extConfigFN = config.getPath('source config') extConfig = createConfigFactory(extConfigFN).getConfig(setSections = ['global']) self.extWorkDir = extConfig.getWorkPath() self.extTask = extConfig.getPlugin(['task', 'module'], cls = TaskModule).getInstance() selector = config.get('source job selector', '') extJobDB = JobDB(extConfig, jobSelector = lambda jobNum, jobObj: jobObj.state == Job.SUCCESS) self.selected = sorted(extJobDB.getJobs(JobSelector.create(selector, task = self.extTask))) utils.verbosity(newVerbosity + 3)
def reset(self, task, wms, select): jobnum_list = self.job_db.get_job_list(JobSelector.create(select, task=task)) if jobnum_list: self._log.warning('Resetting the following jobs:') self._abort_report.show_report(self.job_db, jobnum_list) ask_user_msg = 'Are you sure you want to reset the state of these jobs?' if self._interactive_reset or self._uii.prompt_bool(ask_user_msg, False): self._cancel(task, wms, self.job_db.get_job_list( ClassSelector(JobClass.PROCESSING), jobnum_list), interactive=False, show_jobs=False) for jobnum in jobnum_list: self.job_db.commit(jobnum, Job())
def __init__(self, config): InfoScanner.__init__(self, config) newVerbosity = utils.verbosity(utils.verbosity() - 3) extConfigFN = config.getPath('source config') extConfig = createConfig(extConfigFN).changeView(setSections = ['global']) self._extWorkDir = extConfig.getWorkPath() self._extTask = extConfig.getPlugin(['task', 'module'], cls = 'TaskModule') selector = config.get('source job selector', '') extJobDB = JobDB(extConfig, jobSelector = lambda jobNum, jobObj: jobObj.state == Job.SUCCESS) self._selected = sorted(extJobDB.getJobs(JobSelector.create(selector, task = self._extTask))) utils.verbosity(newVerbosity + 3)
def __init__(self, config): InfoScanner.__init__(self, config) ext_config_fn = config.getPath('source config') ext_config = createConfig(ext_config_fn).changeView(setSections = ['global']) self._extWorkDir = ext_config.getWorkPath() logging.getLogger('user').disabled = True self._extWorkflow = ext_config.getPlugin('workflow', 'Workflow:global', cls = 'Workflow', pargs = ('task',)) logging.getLogger('user').disabled = False self._extTask = self._extWorkflow.task selector = config.get('source job selector', '') ext_job_db = JobDB(ext_config, jobSelector = lambda jobNum, jobObj: jobObj.state == Job.SUCCESS) self._selected = sorted(ext_job_db.getJobs(JobSelector.create(selector, task = self._extTask)))
def __init__(self, config, datasource_name): InfoScanner.__init__(self, config, datasource_name) ext_config_fn = config.get_fn('source config') ext_config_raw = create_config(ext_config_fn, load_only_old_config=True) ext_config = ext_config_raw.change_view(set_sections=['global']) self._ext_work_dn = ext_config.get_work_path() logging.getLogger().disabled = True ext_workflow = ext_config.get_plugin('workflow', 'Workflow:global', cls='Workflow', pkwargs={'backend': 'NullWMS'}) logging.getLogger().disabled = False self._ext_task = ext_workflow.task job_selector = JobSelector.create(config.get('source job selector', ''), task=self._ext_task) self._selected = sorted(ext_workflow.job_manager.job_db.get_job_list(AndJobSelector( ClassSelector(JobClass.SUCCESS), job_selector)))
def reset(self, wms, select): jobs = self.jobDB.getJobs(JobSelector.create(select, task=self._task)) if jobs: self._log_user.warning('Resetting the following jobs:') self._reportClass(self.jobDB, self._task, jobs).display() if utils.getUserBool( 'Are you sure you want to reset the state of these jobs?', False): self.cancel( wms, self.jobDB.getJobs(ClassSelector(JobClass.PROCESSING), jobs), False, False) for jobNum in jobs: self.jobDB.commit(jobNum, Job())
def __init__(self, config): InfoScanner.__init__(self, config) newVerbosity = utils.verbosity(utils.verbosity() - 3) extConfigFN = config.getPath('source config') extConfig = createConfig(extConfigFN).changeView( setSections=['global']) self._extWorkDir = extConfig.getWorkPath() self._extTask = extConfig.getPlugin(['task', 'module'], cls='TaskModule') selector = config.get('source job selector', '') extJobDB = JobDB( extConfig, jobSelector=lambda jobNum, jobObj: jobObj.state == Job.SUCCESS) self._selected = sorted( extJobDB.getJobs(JobSelector.create(selector, task=self._extTask))) utils.verbosity(newVerbosity + 3)
def reset(self, task, wms, select): jobnum_list = self.job_db.get_job_list( JobSelector.create(select, task=task)) if jobnum_list: self._log.warning('Resetting the following jobs:') self._abort_report.show_report(self.job_db, jobnum_list) ask_user_msg = 'Are you sure you want to reset the state of these jobs?' if self._interactive_reset or self._uii.prompt_bool( ask_user_msg, False): self.cancel(wms, self.job_db.get_job_list( ClassSelector(JobClass.PROCESSING), jobnum_list), interactive=False, show_jobs=False) for jobnum in jobnum_list: self.job_db.commit(jobnum, Job())
def get_script_object(config_file, job_selector_str, only_success=False, require_task=False): config = gc_create_config(config_file=config_file, load_only_old_config=True) (task, job_selector) = _get_job_selector_and_task(config, job_selector_str, require_task) if only_success: job_selector = JobSelector.create_instance('AndJobSelector', ClassSelector(JobClass.SUCCESS), job_selector) new_config = gc_create_config(config_file=config_file) jobs_config = new_config.change_view(set_sections=['jobs']) job_db = jobs_config.get_plugin('job database', 'TextFileJobDB', cls='JobDB', pkwargs={'job_selector': job_selector}, on_change=None) class ScriptObject(object): def __init__(self, config, new_config, task, job_db): (self.config, self.new_config) = (config, new_config) (self.task, self.job_db) = (task, job_db) return ScriptObject(config, new_config, task, job_db)
def __init__(self, config, name, task, eventhandler): NamedPlugin.__init__(self, config, name) (self._task, self._eventhandler) = (task, eventhandler) self.jobLimit = config.getInt('jobs', -1, onChange = None) selected = JobSelector.create(config.get('selected', '', onChange = None), task = self._task) jobDBClass = config.getPlugin('jobdb', 'JobDB', cls = JobDB) self.jobDB = jobDBClass.getInstance(config, self.getMaxJobs(self._task), selected) self.disableLog = config.getWorkPath('disabled') outputProcessorClass = config.getPlugin('output processor', 'SandboxProcessor', cls = TaskOutputProcessor) self._outputProcessor = outputProcessorClass.getInstance(task) self.timeout = config.getTime('queue timeout', -1, onChange = None) self.inFlight = config.getInt('in flight', -1, onChange = None) self.inQueue = config.getInt('in queue', -1, onChange = None) self.doShuffle = config.getBool('shuffle', False, onChange = None) self.maxRetry = config.getInt('max retry', -1, onChange = None) self.continuous = config.getBool('continuous', False, onChange = None) self._reportClass = config.getPlugin('abort report', 'LocationReport', cls = Report, onChange = None)
def __init__(self, config, name, task, eventhandler): NamedPlugin.__init__(self, config, name) (self._task, self._eventhandler) = (task, eventhandler) self._log_user = logging.getLogger('user') self._log_user_time = logging.getLogger('user.time') self._njobs_limit = config.getInt('jobs', -1, onChange=None) self._njobs_inflight = config.getInt('in flight', -1, onChange=None) self._njobs_inqueue = config.getInt('in queue', -1, onChange=None) self._chunks_enabled = config.getBool('chunks enabled', True, onChange=None) self._chunks_submit = config.getInt('chunks submit', 100, onChange=None) self._chunks_check = config.getInt('chunks check', 100, onChange=None) self._chunks_retrieve = config.getInt('chunks retrieve', 100, onChange=None) self._job_timeout = config.getTime('queue timeout', -1, onChange=None) self._job_retries = config.getInt('max retry', -1, onChange=None) selected = JobSelector.create(config.get('selected', '', onChange=None), task=self._task) self.jobDB = config.getPlugin('job database', 'JobDB', cls=JobDB, pargs=(self.getMaxJobs(self._task), selected)) self._disabled_jobs_logfile = config.getWorkPath('disabled') self._outputProcessor = config.getPlugin('output processor', 'SandboxProcessor', cls=TaskOutputProcessor, pargs=(task, )) self._do_shuffle = config.getBool('shuffle', False, onChange=None) self._reportClass = Report.getClass( config.get('abort report', 'LocationReport', onChange=None)) self._showBlocker = True
def __init__(self, config, datasource_name): InfoScanner.__init__(self, config, datasource_name) ext_config_fn = config.get_fn('source config') ext_config_raw = create_config(ext_config_fn, load_only_old_config=True) ext_config = ext_config_raw.change_view(set_sections=['global']) self._ext_work_dn = ext_config.get_work_path() logging.getLogger().disabled = True ext_workflow = ext_config.get_plugin('workflow', 'Workflow:global', cls='Workflow', pkwargs={'backend': 'NullWMS'}) logging.getLogger().disabled = False self._ext_task = ext_workflow.task job_selector = JobSelector.create(config.get('source job selector', ''), task=self._ext_task) self._selected = sorted( ext_workflow.job_manager.job_db.get_job_list( AndJobSelector(ClassSelector(JobClass.SUCCESS), job_selector)))
def __init__(self, config): InfoScanner.__init__(self, config) ext_config_fn = config.getPath('source config') ext_config = createConfig(ext_config_fn).changeView( setSections=['global']) self._extWorkDir = ext_config.getWorkPath() logging.getLogger('user').disabled = True self._extWorkflow = ext_config.getPlugin('workflow', 'Workflow:global', cls='Workflow', pargs=('task', )) logging.getLogger('user').disabled = False self._extTask = self._extWorkflow.task selector = config.get('source job selector', '') ext_job_db = JobDB( ext_config, jobSelector=lambda jobNum, jobObj: jobObj.state == Job.SUCCESS) self._selected = sorted( ext_job_db.getJobs(JobSelector.create(selector, task=self._extTask)))
def __init__(self, config, name, task): NamedPlugin.__init__(self, config, name) self._local_event_handler = config.get_composited_plugin( ['local monitor', 'local event handler'], 'logmonitor', 'MultiLocalEventHandler', cls=LocalEventHandler, bind_kwargs={'tags': [self, task]}, require_plugin=False, on_change=None) self._local_event_handler = self._local_event_handler or LocalEventHandler(None, '') self._log = logging.getLogger('jobs.manager') self._njobs_limit = config.get_int('jobs', -1, on_change=None) self._njobs_inflight = config.get_int('in flight', -1, on_change=None) self._njobs_inqueue = config.get_int('in queue', -1, on_change=None) self._chunks_enabled = config.get_bool('chunks enabled', True, on_change=None) self._chunks_submit = config.get_int('chunks submit', 100, on_change=None) self._chunks_check = config.get_int('chunks check', 100, on_change=None) self._chunks_retrieve = config.get_int('chunks retrieve', 100, on_change=None) self._timeout_unknown = config.get_time('unknown timeout', -1, on_change=None) self._timeout_queue = config.get_time('queue timeout', -1, on_change=None) self._job_retries = config.get_int('max retry', -1, on_change=None) selected = JobSelector.create(config.get('selected', '', on_change=None), task=task) self.job_db = config.get_plugin('job database', 'TextFileJobDB', cls=JobDB, pargs=(self._get_max_jobs(task), selected), on_change=None) self._disabled_jobs_logfile = config.get_work_path('disabled') self._output_processor = config.get_plugin('output processor', 'SandboxProcessor', cls=TaskOutputProcessor, on_change=None) self._uii = UserInputInterface() self._interactive_cancel = config.is_interactive(['delete jobs', 'cancel jobs'], True) self._interactive_reset = config.is_interactive('reset jobs', True) self._do_shuffle = config.get_bool('shuffle', False, on_change=None) self._abort_report = config.get_plugin('abort report', 'LocationReport', cls=Report, pargs=(self.job_db, task), on_change=None) self._show_blocker = True self._callback_list = []
def __init__(self, config): InfoScanner.__init__(self, config) self._extWorkDir = config.get('source directory') self._extOutputDir = os.path.join(self._extWorkDir, 'output') self._selector = JobSelector.create(config.get('source job selector', ''))
def __init__(self, config, name, task): NamedPlugin.__init__(self, config, name) self._local_event_handler = config.get_composited_plugin( ['local monitor', 'local event handler'], 'logmonitor', 'MultiLocalEventHandler', cls=LocalEventHandler, bind_kwargs={'tags': [self, task]}, require_plugin=False, on_change=None) self._local_event_handler = self._local_event_handler or LocalEventHandler( None, '') self._log = logging.getLogger('jobs.manager') self._njobs_limit = config.get_int('jobs', -1, on_change=None) self._njobs_inflight = config.get_int('in flight', -1, on_change=None) self._njobs_inqueue = config.get_int('in queue', -1, on_change=None) self._chunks_enabled = config.get_bool('chunks enabled', True, on_change=None) self._chunks_submit = config.get_int('chunks submit', 100, on_change=None) self._chunks_check = config.get_int('chunks check', 100, on_change=None) self._chunks_retrieve = config.get_int('chunks retrieve', 100, on_change=None) self._timeout_unknown = config.get_time('unknown timeout', -1, on_change=None) self._timeout_queue = config.get_time('queue timeout', -1, on_change=None) self._job_retries = config.get_int('max retry', -1, on_change=None) selected = JobSelector.create(config.get('selected', '', on_change=None), task=task) self.job_db = config.get_plugin('job database', 'TextFileJobDB', cls=JobDB, pargs=(self._get_max_jobs(task), selected), on_change=None) self._disabled_jobs_logfile = config.get_work_path('disabled') self._output_processor = config.get_plugin('output processor', 'SandboxProcessor', cls=TaskOutputProcessor, on_change=None) self._uii = UserInputInterface() self._interactive_cancel = config.is_interactive( ['delete jobs', 'cancel jobs'], True) self._interactive_reset = config.is_interactive('reset jobs', True) self._do_shuffle = config.get_bool('shuffle', False, on_change=None) self._abort_report = config.get_plugin('abort report', 'LocationReport', cls=Report, pargs=(self.job_db, task), on_change=None) self._show_blocker = True self._callback_list = []
def __init__(self, jobDB, task, jobs = None, configString = ''): Report.__init__(self, jobDB, task, jobs, configString) self._selector = JobSelector.create(configString, task = task)
def __init__(self, jobDB, task, jobs=None, configString=''): Report.__init__(self, jobDB, task, jobs, configString) self._selector = JobSelector.create(configString, task=task)
def delete(self, wms, select): selector = AndJobSelector(ClassSelector(JobClass.PROCESSING), JobSelector.create(select, task = self._task)) jobs = self.jobDB.getJobs(selector) if jobs: self._log_user.warning('Cancelling the following jobs:') self.cancel(wms, jobs, True)
def __init__(self, config): InfoScanner.__init__(self, config) self._extWorkDir = config.getPath('source directory') self._extOutputDir = os.path.join(self._extWorkDir, 'output') self._selector = JobSelector.create( config.get('source job selector', ''))