示例#1
0
def _get_job_selector_and_task(config, job_selector_str, require_task):
	if not require_task:
		try:  # try to build job selector without task
			return (None, JobSelector.create(job_selector_str))
		except Exception:
			if not except_nested(TaskNeededException, get_current_exception()):
				raise
			clear_current_exception()
	task = gc_create_workflow(config).task
	return (task, JobSelector.create(job_selector_str, task=task))
示例#2
0
 def delete(self, wms, select):
     selector = AndJobSelector(ClassSelector(JobClass.PROCESSING),
                               JobSelector.create(select, task=self._task))
     jobs = self.jobDB.getJobs(selector)
     if jobs:
         self._log_user.warning('Cancelling the following jobs:')
         self.cancel(wms, jobs, interactive=True, showJobs=True)
示例#3
0
	def cancel(self, task, wms, select):
		selector = AndJobSelector(ClassSelector(JobClass.PROCESSING),
			JobSelector.create(select, task=task))
		jobs = self.job_db.get_job_list(selector)
		if jobs:
			self._log.warning('Cancelling the following jobs:')
			self._cancel(task, wms, jobs, interactive=self._interactive_cancel, show_jobs=True)
	def __init__(self, config, name, task, eventhandler):
		NamedPlugin.__init__(self, config, name)
		(self._task, self._eventhandler) = (task, eventhandler)
		self._log_user = logging.getLogger('user')
		self._log_user_time = logging.getLogger('user.time')

		self._njobs_limit = config.getInt('jobs', -1, onChange = None)
		self._njobs_inflight = config.getInt('in flight', -1, onChange = None)
		self._njobs_inqueue = config.getInt('in queue', -1, onChange = None)

		self._chunks_submit = config.getInt('chunks submit', 100, onChange = None)
		self._chunks_check = config.getInt('chunks check', 100, onChange = None)
		self._chunks_retrieve = config.getInt('chunks retrieve', 100, onChange = None)

		self._job_timeout = config.getTime('queue timeout', -1, onChange = None)
		self._job_retries = config.getInt('max retry', -1, onChange = None)

		selected = JobSelector.create(config.get('selected', '', onChange = None), task = self._task)
		self.jobDB = config.getPlugin('job database', 'JobDB',
			cls = JobDB, pargs = (self.getMaxJobs(self._task), selected))
		self._disabled_jobs_logfile = config.getWorkPath('disabled')
		self._outputProcessor = config.getPlugin('output processor', 'SandboxProcessor',
			cls = TaskOutputProcessor, pargs = (task,))

		self._do_shuffle = config.getBool('shuffle', False, onChange = None)
		self._continuous = config.getBool('continuous', False, onChange = None)
		self._reportClass = Report.getClass(config.get('abort report', 'LocationReport', onChange = None))
		self._showBlocker = True
示例#5
0
 def __init__(self, config):
     InfoScanner.__init__(self, config)
     ext_config_fn = config.getPath('source config',
                                    onChange=triggerDataResync)
     ext_config = create_config(
         ext_config_fn,
         useDefaultFiles=True).changeView(setSections=['global'])
     self._extWorkDir = ext_config.getWorkPath()
     logging.getLogger().disabled = True
     self._extWorkflow = ext_config.getPlugin('workflow',
                                              'Workflow:global',
                                              cls='Workflow',
                                              pargs=('task', ))
     logging.getLogger().disabled = False
     self._extTask = self._extWorkflow.task
     selector = config.get('source job selector',
                           '',
                           onChange=triggerDataResync)
     ext_job_db = ext_config.getPlugin(
         'job database',
         'TextFileJobDB',
         cls='JobDB',
         pkwargs={
             'jobSelector':
             lambda jobNum, jobObj: jobObj.state == Job.SUCCESS
         },
         onChange=None)
     self._selected = sorted(
         ext_job_db.getJobs(JobSelector.create(selector,
                                               task=self._extTask)))
	def __init__(self, config, datasource_name):
		InfoScanner.__init__(self, config, datasource_name)
		self._ext_work_dn = config.get_dn('source directory')
		self._ext_output_dir = os.path.join(self._ext_work_dn, 'output')
		if not os.path.isdir(self._ext_output_dir):
			raise DatasetError('Unable to find task output directory %s' % repr(self._ext_output_dir))
		self._selector = JobSelector.create(config.get('source job selector', ''))
示例#7
0
def initGC(args):
	if len(args) > 0:
		config = getConfig(args[0])
		userSelector = None
		if len(args) != 1:
			userSelector = JobSelector.create(args[1])
		return (config, JobDB(config, jobSelector = userSelector))
	sys.stderr.write('Syntax: %s <config file> [<job id>, ...]\n\n' % sys.argv[0])
	sys.exit(os.EX_USAGE)
	def reset(self, wms, select):
		jobs = self.jobDB.getJobs(JobSelector.create(select, task = self._task))
		if jobs:
			self._log_user.warning('Resetting the following jobs:')
			self._reportClass(self.jobDB, self._task, jobs).display()
			if utils.getUserBool('Are you sure you want to reset the state of these jobs?', False):
				self.cancel(wms, self.jobDB.getJobs(ClassSelector(JobClass.PROCESSING), jobs), False, False)
				for jobNum in jobs:
					self.jobDB.commit(jobNum, Job())
示例#9
0
def initGC(args):
	if len(args) > 0:
		config = getConfig(args[0])
		userSelector = None
		if len(args) != 1:
			userSelector = JobSelector.create(args[1])
		return (config.getWorkPath(), config, JobDB(config, jobSelector = userSelector))
	sys.stderr.write('Syntax: %s <config file> [<job id>, ...]\n\n' % sys.argv[0])
	sys.exit(os.EX_USAGE)
示例#10
0
 def __init__(self, config, datasource_name):
     InfoScanner.__init__(self, config, datasource_name)
     self._ext_work_dn = config.get_dn('source directory')
     self._ext_output_dir = os.path.join(self._ext_work_dn, 'output')
     if not os.path.isdir(self._ext_output_dir):
         raise DatasetError('Unable to find task output directory %s' %
                            repr(self._ext_output_dir))
     self._selector = JobSelector.create(
         config.get('source job selector', ''))
示例#11
0
 def delete(self, task, wms, select):
     selector = AndJobSelector(ClassSelector(JobClass.PROCESSING),
                               JobSelector.create(select, task=task))
     jobs = self.job_db.get_job_list(selector)
     if jobs:
         self._log.warning('Cancelling the following jobs:')
         self.cancel(wms,
                     jobs,
                     interactive=self._interactive_delete,
                     show_jobs=True)
示例#12
0
 def __init__(self, config):
     InfoScanner.__init__(self, config)
     self._extWorkDir = config.getPath('source directory',
                                       onChange=triggerDataResync)
     self._extOutputDir = os.path.join(self._extWorkDir, 'output')
     if not os.path.isdir(self._extOutputDir):
         raise DatasetError('Unable to find task output directory %s' %
                            repr(self._extOutputDir))
     self._selector = JobSelector.create(
         config.get('source job selector', '', onChange=triggerDataResync))
示例#13
0
	def __init__(self, config):
		from grid_control.tasks import TaskModule
		newVerbosity = utils.verbosity(utils.verbosity() - 3)
		extConfigFN = config.getPath('source config')
		extConfig = createConfigFactory(extConfigFN).getConfig(setSections = ['global'])
		self.extWorkDir = extConfig.getWorkPath()
		self.extTask = extConfig.getPlugin(['task', 'module'], cls = TaskModule).getInstance()
		selector = config.get('source job selector', '')
		extJobDB = JobDB(extConfig, jobSelector = lambda jobNum, jobObj: jobObj.state == Job.SUCCESS)
		self.selected = sorted(extJobDB.getJobs(JobSelector.create(selector, task = self.extTask)))
		utils.verbosity(newVerbosity + 3)
示例#14
0
	def reset(self, task, wms, select):
		jobnum_list = self.job_db.get_job_list(JobSelector.create(select, task=task))
		if jobnum_list:
			self._log.warning('Resetting the following jobs:')
			self._abort_report.show_report(self.job_db, jobnum_list)
			ask_user_msg = 'Are you sure you want to reset the state of these jobs?'
			if self._interactive_reset or self._uii.prompt_bool(ask_user_msg, False):
				self._cancel(task, wms, self.job_db.get_job_list(
					ClassSelector(JobClass.PROCESSING), jobnum_list), interactive=False, show_jobs=False)
				for jobnum in jobnum_list:
					self.job_db.commit(jobnum, Job())
	def __init__(self, config):
		InfoScanner.__init__(self, config)
		newVerbosity = utils.verbosity(utils.verbosity() - 3)
		extConfigFN = config.getPath('source config')
		extConfig = createConfig(extConfigFN).changeView(setSections = ['global'])
		self._extWorkDir = extConfig.getWorkPath()
		self._extTask = extConfig.getPlugin(['task', 'module'], cls = 'TaskModule')
		selector = config.get('source job selector', '')
		extJobDB = JobDB(extConfig, jobSelector = lambda jobNum, jobObj: jobObj.state == Job.SUCCESS)
		self._selected = sorted(extJobDB.getJobs(JobSelector.create(selector, task = self._extTask)))
		utils.verbosity(newVerbosity + 3)
	def __init__(self, config):
		InfoScanner.__init__(self, config)
		ext_config_fn = config.getPath('source config')
		ext_config = createConfig(ext_config_fn).changeView(setSections = ['global'])
		self._extWorkDir = ext_config.getWorkPath()
		logging.getLogger('user').disabled = True
		self._extWorkflow = ext_config.getPlugin('workflow', 'Workflow:global', cls = 'Workflow',
			pargs = ('task',))
		logging.getLogger('user').disabled = False
		self._extTask = self._extWorkflow.task
		selector = config.get('source job selector', '')
		ext_job_db = JobDB(ext_config, jobSelector = lambda jobNum, jobObj: jobObj.state == Job.SUCCESS)
		self._selected = sorted(ext_job_db.getJobs(JobSelector.create(selector, task = self._extTask)))
示例#17
0
	def __init__(self, config, datasource_name):
		InfoScanner.__init__(self, config, datasource_name)
		ext_config_fn = config.get_fn('source config')
		ext_config_raw = create_config(ext_config_fn, load_only_old_config=True)
		ext_config = ext_config_raw.change_view(set_sections=['global'])
		self._ext_work_dn = ext_config.get_work_path()
		logging.getLogger().disabled = True
		ext_workflow = ext_config.get_plugin('workflow', 'Workflow:global', cls='Workflow',
			pkwargs={'backend': 'NullWMS'})
		logging.getLogger().disabled = False
		self._ext_task = ext_workflow.task
		job_selector = JobSelector.create(config.get('source job selector', ''), task=self._ext_task)
		self._selected = sorted(ext_workflow.job_manager.job_db.get_job_list(AndJobSelector(
			ClassSelector(JobClass.SUCCESS), job_selector)))
示例#18
0
 def reset(self, wms, select):
     jobs = self.jobDB.getJobs(JobSelector.create(select, task=self._task))
     if jobs:
         self._log_user.warning('Resetting the following jobs:')
         self._reportClass(self.jobDB, self._task, jobs).display()
         if utils.getUserBool(
                 'Are you sure you want to reset the state of these jobs?',
                 False):
             self.cancel(
                 wms,
                 self.jobDB.getJobs(ClassSelector(JobClass.PROCESSING),
                                    jobs), False, False)
             for jobNum in jobs:
                 self.jobDB.commit(jobNum, Job())
 def __init__(self, config):
     InfoScanner.__init__(self, config)
     newVerbosity = utils.verbosity(utils.verbosity() - 3)
     extConfigFN = config.getPath('source config')
     extConfig = createConfig(extConfigFN).changeView(
         setSections=['global'])
     self._extWorkDir = extConfig.getWorkPath()
     self._extTask = extConfig.getPlugin(['task', 'module'],
                                         cls='TaskModule')
     selector = config.get('source job selector', '')
     extJobDB = JobDB(
         extConfig,
         jobSelector=lambda jobNum, jobObj: jobObj.state == Job.SUCCESS)
     self._selected = sorted(
         extJobDB.getJobs(JobSelector.create(selector, task=self._extTask)))
     utils.verbosity(newVerbosity + 3)
示例#20
0
 def reset(self, task, wms, select):
     jobnum_list = self.job_db.get_job_list(
         JobSelector.create(select, task=task))
     if jobnum_list:
         self._log.warning('Resetting the following jobs:')
         self._abort_report.show_report(self.job_db, jobnum_list)
         ask_user_msg = 'Are you sure you want to reset the state of these jobs?'
         if self._interactive_reset or self._uii.prompt_bool(
                 ask_user_msg, False):
             self.cancel(wms,
                         self.job_db.get_job_list(
                             ClassSelector(JobClass.PROCESSING),
                             jobnum_list),
                         interactive=False,
                         show_jobs=False)
             for jobnum in jobnum_list:
                 self.job_db.commit(jobnum, Job())
示例#21
0
def get_script_object(config_file, job_selector_str, only_success=False, require_task=False):
	config = gc_create_config(config_file=config_file, load_only_old_config=True)
	(task, job_selector) = _get_job_selector_and_task(config, job_selector_str, require_task)
	if only_success:
		job_selector = JobSelector.create_instance('AndJobSelector',
			ClassSelector(JobClass.SUCCESS), job_selector)
	new_config = gc_create_config(config_file=config_file)
	jobs_config = new_config.change_view(set_sections=['jobs'])
	job_db = jobs_config.get_plugin('job database', 'TextFileJobDB', cls='JobDB',
		pkwargs={'job_selector': job_selector}, on_change=None)

	class ScriptObject(object):
		def __init__(self, config, new_config, task, job_db):
			(self.config, self.new_config) = (config, new_config)
			(self.task, self.job_db) = (task, job_db)

	return ScriptObject(config, new_config, task, job_db)
示例#22
0
	def __init__(self, config, name, task, eventhandler):
		NamedPlugin.__init__(self, config, name)
		(self._task, self._eventhandler) = (task, eventhandler)
		self.jobLimit = config.getInt('jobs', -1, onChange = None)
		selected = JobSelector.create(config.get('selected', '', onChange = None), task = self._task)
		jobDBClass = config.getPlugin('jobdb', 'JobDB', cls = JobDB)
		self.jobDB = jobDBClass.getInstance(config, self.getMaxJobs(self._task), selected)
		self.disableLog = config.getWorkPath('disabled')
		outputProcessorClass = config.getPlugin('output processor', 'SandboxProcessor', cls = TaskOutputProcessor)
		self._outputProcessor = outputProcessorClass.getInstance(task)

		self.timeout = config.getTime('queue timeout', -1, onChange = None)
		self.inFlight = config.getInt('in flight', -1, onChange = None)
		self.inQueue = config.getInt('in queue', -1, onChange = None)
		self.doShuffle = config.getBool('shuffle', False, onChange = None)
		self.maxRetry = config.getInt('max retry', -1, onChange = None)
		self.continuous = config.getBool('continuous', False, onChange = None)
		self._reportClass = config.getPlugin('abort report', 'LocationReport', cls = Report, onChange = None)
示例#23
0
    def __init__(self, config, name, task, eventhandler):
        NamedPlugin.__init__(self, config, name)
        (self._task, self._eventhandler) = (task, eventhandler)
        self._log_user = logging.getLogger('user')
        self._log_user_time = logging.getLogger('user.time')

        self._njobs_limit = config.getInt('jobs', -1, onChange=None)
        self._njobs_inflight = config.getInt('in flight', -1, onChange=None)
        self._njobs_inqueue = config.getInt('in queue', -1, onChange=None)

        self._chunks_enabled = config.getBool('chunks enabled',
                                              True,
                                              onChange=None)
        self._chunks_submit = config.getInt('chunks submit',
                                            100,
                                            onChange=None)
        self._chunks_check = config.getInt('chunks check', 100, onChange=None)
        self._chunks_retrieve = config.getInt('chunks retrieve',
                                              100,
                                              onChange=None)

        self._job_timeout = config.getTime('queue timeout', -1, onChange=None)
        self._job_retries = config.getInt('max retry', -1, onChange=None)

        selected = JobSelector.create(config.get('selected', '',
                                                 onChange=None),
                                      task=self._task)
        self.jobDB = config.getPlugin('job database',
                                      'JobDB',
                                      cls=JobDB,
                                      pargs=(self.getMaxJobs(self._task),
                                             selected))
        self._disabled_jobs_logfile = config.getWorkPath('disabled')
        self._outputProcessor = config.getPlugin('output processor',
                                                 'SandboxProcessor',
                                                 cls=TaskOutputProcessor,
                                                 pargs=(task, ))

        self._do_shuffle = config.getBool('shuffle', False, onChange=None)
        self._reportClass = Report.getClass(
            config.get('abort report', 'LocationReport', onChange=None))
        self._showBlocker = True
示例#24
0
 def __init__(self, config, datasource_name):
     InfoScanner.__init__(self, config, datasource_name)
     ext_config_fn = config.get_fn('source config')
     ext_config_raw = create_config(ext_config_fn,
                                    load_only_old_config=True)
     ext_config = ext_config_raw.change_view(set_sections=['global'])
     self._ext_work_dn = ext_config.get_work_path()
     logging.getLogger().disabled = True
     ext_workflow = ext_config.get_plugin('workflow',
                                          'Workflow:global',
                                          cls='Workflow',
                                          pkwargs={'backend': 'NullWMS'})
     logging.getLogger().disabled = False
     self._ext_task = ext_workflow.task
     job_selector = JobSelector.create(config.get('source job selector',
                                                  ''),
                                       task=self._ext_task)
     self._selected = sorted(
         ext_workflow.job_manager.job_db.get_job_list(
             AndJobSelector(ClassSelector(JobClass.SUCCESS), job_selector)))
示例#25
0
 def __init__(self, config):
     InfoScanner.__init__(self, config)
     ext_config_fn = config.getPath('source config')
     ext_config = createConfig(ext_config_fn).changeView(
         setSections=['global'])
     self._extWorkDir = ext_config.getWorkPath()
     logging.getLogger('user').disabled = True
     self._extWorkflow = ext_config.getPlugin('workflow',
                                              'Workflow:global',
                                              cls='Workflow',
                                              pargs=('task', ))
     logging.getLogger('user').disabled = False
     self._extTask = self._extWorkflow.task
     selector = config.get('source job selector', '')
     ext_job_db = JobDB(
         ext_config,
         jobSelector=lambda jobNum, jobObj: jobObj.state == Job.SUCCESS)
     self._selected = sorted(
         ext_job_db.getJobs(JobSelector.create(selector,
                                               task=self._extTask)))
示例#26
0
	def __init__(self, config, name, task):
		NamedPlugin.__init__(self, config, name)
		self._local_event_handler = config.get_composited_plugin(
			['local monitor', 'local event handler'], 'logmonitor', 'MultiLocalEventHandler',
			cls=LocalEventHandler, bind_kwargs={'tags': [self, task]},
			require_plugin=False, on_change=None)
		self._local_event_handler = self._local_event_handler or LocalEventHandler(None, '')
		self._log = logging.getLogger('jobs.manager')

		self._njobs_limit = config.get_int('jobs', -1, on_change=None)
		self._njobs_inflight = config.get_int('in flight', -1, on_change=None)
		self._njobs_inqueue = config.get_int('in queue', -1, on_change=None)

		self._chunks_enabled = config.get_bool('chunks enabled', True, on_change=None)
		self._chunks_submit = config.get_int('chunks submit', 100, on_change=None)
		self._chunks_check = config.get_int('chunks check', 100, on_change=None)
		self._chunks_retrieve = config.get_int('chunks retrieve', 100, on_change=None)

		self._timeout_unknown = config.get_time('unknown timeout', -1, on_change=None)
		self._timeout_queue = config.get_time('queue timeout', -1, on_change=None)
		self._job_retries = config.get_int('max retry', -1, on_change=None)

		selected = JobSelector.create(config.get('selected', '', on_change=None), task=task)
		self.job_db = config.get_plugin('job database', 'TextFileJobDB',
			cls=JobDB, pargs=(self._get_max_jobs(task), selected), on_change=None)
		self._disabled_jobs_logfile = config.get_work_path('disabled')
		self._output_processor = config.get_plugin('output processor', 'SandboxProcessor',
			cls=TaskOutputProcessor, on_change=None)

		self._uii = UserInputInterface()
		self._interactive_cancel = config.is_interactive(['delete jobs', 'cancel jobs'], True)
		self._interactive_reset = config.is_interactive('reset jobs', True)
		self._do_shuffle = config.get_bool('shuffle', False, on_change=None)
		self._abort_report = config.get_plugin('abort report', 'LocationReport',
			cls=Report, pargs=(self.job_db, task), on_change=None)
		self._show_blocker = True
		self._callback_list = []
	def __init__(self, config):
		InfoScanner.__init__(self, config)
		self._extWorkDir = config.get('source directory')
		self._extOutputDir = os.path.join(self._extWorkDir, 'output')
		self._selector = JobSelector.create(config.get('source job selector', ''))
示例#28
0
    def __init__(self, config, name, task):
        NamedPlugin.__init__(self, config, name)
        self._local_event_handler = config.get_composited_plugin(
            ['local monitor', 'local event handler'],
            'logmonitor',
            'MultiLocalEventHandler',
            cls=LocalEventHandler,
            bind_kwargs={'tags': [self, task]},
            require_plugin=False,
            on_change=None)
        self._local_event_handler = self._local_event_handler or LocalEventHandler(
            None, '')
        self._log = logging.getLogger('jobs.manager')

        self._njobs_limit = config.get_int('jobs', -1, on_change=None)
        self._njobs_inflight = config.get_int('in flight', -1, on_change=None)
        self._njobs_inqueue = config.get_int('in queue', -1, on_change=None)

        self._chunks_enabled = config.get_bool('chunks enabled',
                                               True,
                                               on_change=None)
        self._chunks_submit = config.get_int('chunks submit',
                                             100,
                                             on_change=None)
        self._chunks_check = config.get_int('chunks check',
                                            100,
                                            on_change=None)
        self._chunks_retrieve = config.get_int('chunks retrieve',
                                               100,
                                               on_change=None)

        self._timeout_unknown = config.get_time('unknown timeout',
                                                -1,
                                                on_change=None)
        self._timeout_queue = config.get_time('queue timeout',
                                              -1,
                                              on_change=None)
        self._job_retries = config.get_int('max retry', -1, on_change=None)

        selected = JobSelector.create(config.get('selected',
                                                 '',
                                                 on_change=None),
                                      task=task)
        self.job_db = config.get_plugin('job database',
                                        'TextFileJobDB',
                                        cls=JobDB,
                                        pargs=(self._get_max_jobs(task),
                                               selected),
                                        on_change=None)
        self._disabled_jobs_logfile = config.get_work_path('disabled')
        self._output_processor = config.get_plugin('output processor',
                                                   'SandboxProcessor',
                                                   cls=TaskOutputProcessor,
                                                   on_change=None)

        self._uii = UserInputInterface()
        self._interactive_cancel = config.is_interactive(
            ['delete jobs', 'cancel jobs'], True)
        self._interactive_reset = config.is_interactive('reset jobs', True)
        self._do_shuffle = config.get_bool('shuffle', False, on_change=None)
        self._abort_report = config.get_plugin('abort report',
                                               'LocationReport',
                                               cls=Report,
                                               pargs=(self.job_db, task),
                                               on_change=None)
        self._show_blocker = True
        self._callback_list = []
示例#29
0
	def __init__(self, jobDB, task, jobs = None, configString = ''):
		Report.__init__(self, jobDB, task, jobs, configString)
		self._selector = JobSelector.create(configString, task = task)
示例#30
0
 def __init__(self, jobDB, task, jobs=None, configString=''):
     Report.__init__(self, jobDB, task, jobs, configString)
     self._selector = JobSelector.create(configString, task=task)
示例#31
0
	def delete(self, wms, select):
		selector = AndJobSelector(ClassSelector(JobClass.PROCESSING), JobSelector.create(select, task = self._task))
		jobs = self.jobDB.getJobs(selector)
		if jobs:
			self._log_user.warning('Cancelling the following jobs:')
			self.cancel(wms, jobs, True)
示例#32
0
 def __init__(self, config):
     InfoScanner.__init__(self, config)
     self._extWorkDir = config.getPath('source directory')
     self._extOutputDir = os.path.join(self._extWorkDir, 'output')
     self._selector = JobSelector.create(
         config.get('source job selector', ''))