def __init__(self, config, wmsName): wmsName = (wmsName or self.__class__.__name__).upper().replace('.', '_') NamedPlugin.__init__(self, config, wmsName) (self.config, self.wmsName) = (config, wmsName) self._wait_idle = config.getInt('wait idle', 60, onChange=None) self._wait_work = config.getInt('wait work', 10, onChange=None)
def __init__(self, config, name, task, eventhandler): NamedPlugin.__init__(self, config, name) (self._task, self._eventhandler) = (task, eventhandler) self._log_user = logging.getLogger('user') self._log_user_time = logging.getLogger('user.time') self._njobs_limit = config.getInt('jobs', -1, onChange = None) self._njobs_inflight = config.getInt('in flight', -1, onChange = None) self._njobs_inqueue = config.getInt('in queue', -1, onChange = None) self._chunks_submit = config.getInt('chunks submit', 100, onChange = None) self._chunks_check = config.getInt('chunks check', 100, onChange = None) self._chunks_retrieve = config.getInt('chunks retrieve', 100, onChange = None) self._job_timeout = config.getTime('queue timeout', -1, onChange = None) self._job_retries = config.getInt('max retry', -1, onChange = None) selected = JobSelector.create(config.get('selected', '', onChange = None), task = self._task) self.jobDB = config.getPlugin('job database', 'JobDB', cls = JobDB, pargs = (self.getMaxJobs(self._task), selected)) self._disabled_jobs_logfile = config.getWorkPath('disabled') self._outputProcessor = config.getPlugin('output processor', 'SandboxProcessor', cls = TaskOutputProcessor, pargs = (task,)) self._do_shuffle = config.getBool('shuffle', False, onChange = None) self._continuous = config.getBool('continuous', False, onChange = None) self._reportClass = Report.getClass(config.get('abort report', 'LocationReport', onChange = None)) self._showBlocker = True
def __init__(self, config, name): name = (name or self.__class__.__name__).upper().replace('.', '_') NamedPlugin.__init__(self, config, name) self._wait_idle = config.getInt('wait idle', 60, onChange = None) self._wait_work = config.getInt('wait work', 10, onChange = None) self._job_parser = config.getPlugin('job parser', 'JobInfoProcessor', cls = 'JobInfoProcessor', onChange = None)
def __init__(self, config, name, workflow, redraw_event, stream): NamedPlugin.__init__(self, config, name) (self._dirty, self._redraw_event, self._on_height_change) = (False, redraw_event, None) (self._layout_pos, self._layout_height, self._layout_width) = (0, 0, 0) self._dirty_interval = config.get_float('gui refresh interval', 0.2, on_change=None) self._height_interval = config.get_float('gui height interval', 10., on_change=None) (self._dirty_next, self._height_next) = (0, 0) self._stream = stream
def __init__(self, config, name): name = (name or self.__class__.__name__).upper().replace('.', '_') NamedPlugin.__init__(self, config, name) self._wait_idle = config.getInt('wait idle', 60, onChange=None) self._wait_work = config.getInt('wait work', 10, onChange=None) self._job_parser = config.getPlugin('job parser', 'JobInfoProcessor', cls='JobInfoProcessor', onChange=None)
def __init__(self, config, name, broker_prefix, item_name, discover_fun): NamedPlugin.__init__(self, config, name) (self._item_list_start, self._item_list_discovered, self._item_name) = (None, False, item_name) self._num_entries = config.get_int('%s entries' % broker_prefix, 0, on_change=None) self._randomize = config.get_bool('%s randomize' % broker_prefix, False, on_change=None)
def __init__(self, config, name, userOpt, itemName, discoverFun): NamedPlugin.__init__(self, config, name) (self._itemsStart, self._itemsDiscovered, self._itemName) = (None, False, itemName) self._nEntries = config.getInt('%s entries' % userOpt, 0, onChange=None) self._nRandom = config.getBool('%s randomize' % userOpt, False, onChange=None)
def __init__(self, config, name, abort = None): NamedPlugin.__init__(self, config, name) # Initial activity stream sys.stdout = SimpleActivityStream(sys.stdout, register_callback = True) sys.stderr = SimpleActivityStream(sys.stderr) # Workdir settings self._workDir = config.getWorkPath() self._checkSpace = config.getInt('workdir space', 10, onChange = None) # Initialise task module self.task = config.getPlugin(['module', 'task'], cls = TaskModule, tags = [self]) if abort == 'task': return self._log.log(logging.INFO, 'Current task ID: %s', self.task.taskID) self._log.log(logging.INFO, 'Task started on: %s', self.task.taskDate) # Initialise workload management interface self.wms = config.getCompositePlugin('backend', 'grid', 'MultiWMS', cls = WMS, tags = [self, self.task]) # Subsequent config calls also include section "jobs": jobs_config = config.changeView(viewClass = 'TaggedConfigView', addSections = ['jobs'], addTags = [self]) # Initialise monitoring module self.monitor = jobs_config.getCompositePlugin('monitor', 'scripts', 'MultiMonitor', cls = Monitoring, tags = [self, self.task], pargs = (self.task,)) # Initialise job database self.jobManager = jobs_config.getPlugin('job manager', 'SimpleJobManager', cls = JobManager, tags = [self, self.task, self.wms], pargs = (self.task, self.monitor)) if abort == 'jobmanager': return # Prepare work package self.wms.deployTask(self.task, self.monitor, transferSE = config.getState('init', detail = 'storage'), transferSB = config.getState('init', detail = 'sandbox')) # Configure workflow settings self._actionList = jobs_config.getList('action', ['check', 'retrieve', 'submit'], onChange = None) self.duration = 0 if jobs_config.getBool('continuous', False, onChange = None): # legacy option self.duration = -1 self.duration = jobs_config.getTime('duration', self.duration, onChange = None) self._submitFlag = jobs_config.getBool('submission', True, onChange = None) self._submitTime = jobs_config.getTime('submission time requirement', self.task.wallTime, onChange = None) # Initialise GUI (sys.stdout, sys.stderr) = (sys.stdout.finish(), sys.stderr.finish()) self._gui = config.getPlugin('gui', 'SimpleConsole', cls = GUI, onChange = None, pargs = (self,))
def __init__(self, config, name): name = (name or self.__class__.__name__).upper().replace('.', '_') NamedPlugin.__init__(self, config, name) self._wait_idle = config.get_int('wait idle', 60, on_change=None) self._wait_work = config.get_int('wait work', 10, on_change=None) self._job_parser = config.get_plugin('job parser', 'JobInfoProcessor', cls='JobInfoProcessor', on_change=None) self._remote_event_handler = config.get_composited_plugin( ['remote monitor', 'remote event handler'], '', 'MultiRemoteEventHandler', cls=RemoteEventHandler, bind_kwargs={'tags': [self]}, require_plugin=False, on_change=TriggerInit('sandbox')) or RemoteEventHandler(config, 'dummy')
def __init__(self, config, name): NamedPlugin.__init__(self, config, name) initSandbox = changeInitNeeded('sandbox') self._varCheck = validNoVar(config) # Task requirements jobs_config = config.changeView(viewClass = 'TaggedConfigView', addSections = ['jobs'], addTags = [self]) # Move this into parameter manager? self.wallTime = jobs_config.getTime('wall time', onChange = None) self.cpuTime = jobs_config.getTime('cpu time', self.wallTime, onChange = None) self.cpus = jobs_config.getInt('cpus', 1, onChange = None) self.memory = jobs_config.getInt('memory', -1, onChange = None) self.nodeTimeout = jobs_config.getTime('node timeout', -1, onChange = initSandbox) # Compute / get task ID self.taskID = config.get('task id', 'GC' + md5_hex(str(time()))[:12], persistent = True) self.taskDate = config.get('task date', strftime('%Y-%m-%d'), persistent = True, onChange = initSandbox) self.taskConfigName = config.getConfigName() self._job_name_generator = config.getPlugin('job name generator', 'DefaultJobName', cls = JobNamePlugin, pargs = (self,)) # Storage setup storage_config = config.changeView(viewClass = 'TaggedConfigView', setClasses = None, setNames = None, addSections = ['storage'], addTags = [self]) self.taskVariables = { # Space limits 'SCRATCH_UL': storage_config.getInt('scratch space used', 5000, onChange = initSandbox), 'SCRATCH_LL': storage_config.getInt('scratch space left', 1, onChange = initSandbox), 'LANDINGZONE_UL': storage_config.getInt('landing zone space used', 100, onChange = initSandbox), 'LANDINGZONE_LL': storage_config.getInt('landing zone space left', 1, onChange = initSandbox), } storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@') self.seMinSize = storage_config.getInt('se min size', -1, onChange = initSandbox) self.sbInputFiles = config.getPaths('input files', [], onChange = initSandbox) self.sbOutputFiles = config.getList('output files', [], onChange = initSandbox) self.gzipOut = config.getBool('gzip output', True, onChange = initSandbox) self._subst_files = config.getList('subst files', [], onChange = initSandbox) self.dependencies = lmap(str.lower, config.getList('depends', [], onChange = initSandbox)) # Get error messages from gc-run.lib comments self.errorDict = {} self.updateErrorDict(utils.pathShare('gc-run.lib')) # Init parameter source manager psrc_repository = {} self._setupJobParameters(config, psrc_repository) self._pfactory = config.getPlugin('internal parameter factory', 'BasicParameterFactory', cls = ParameterFactory, pargs = (psrc_repository,), tags = [self], inherit = True) self.source = config.getPlugin('parameter adapter', 'TrackedParameterAdapter', cls = ParameterAdapter, pargs = (self._pfactory.getSource(),))
def __init__(self, config, name): NamedPlugin.__init__(self, config, name) initSandbox = changeInitNeeded('sandbox') self._varCheck = validNoVar(config) # Task requirements jobs_config = config.changeView(viewClass = 'TaggedConfigView', addSections = ['jobs'], addTags = [self]) # Move this into parameter manager? self.wallTime = jobs_config.getTime('wall time', onChange = None) self.cpuTime = jobs_config.getTime('cpu time', self.wallTime, onChange = None) self.cpus = jobs_config.getInt('cpus', 1, onChange = None) self.memory = jobs_config.getInt('memory', -1, onChange = None) self.nodeTimeout = jobs_config.getTime('node timeout', -1, onChange = initSandbox) # Compute / get task ID self.taskID = config.get('task id', 'GC' + md5_hex(str(time()))[:12], persistent = True) self.taskDate = config.get('task date', strftime('%Y-%m-%d'), persistent = True, onChange = initSandbox) self.taskConfigName = config.getConfigName() self._job_name_generator = config.getPlugin('job name generator', 'DefaultJobName', cls = JobNamePlugin, pargs = (self,)) # Storage setup storage_config = config.changeView(viewClass = 'TaggedConfigView', setClasses = None, setNames = None, addSections = ['storage'], addTags = [self]) self.taskVariables = { # Space limits 'SCRATCH_UL': storage_config.getInt('scratch space used', 5000, onChange = initSandbox), 'SCRATCH_LL': storage_config.getInt('scratch space left', 1, onChange = initSandbox), 'LANDINGZONE_UL': storage_config.getInt('landing zone space used', 100, onChange = initSandbox), 'LANDINGZONE_LL': storage_config.getInt('landing zone space left', 1, onChange = initSandbox), } storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@') self.seMinSize = storage_config.getInt('se min size', -1, onChange = initSandbox) self.sbInputFiles = config.getPaths('input files', [], onChange = initSandbox) self.sbOutputFiles = config.getList('output files', [], onChange = initSandbox) self.gzipOut = config.getBool('gzip output', True, onChange = initSandbox) self.substFiles = config.getList('subst files', [], onChange = initSandbox) self.dependencies = lmap(str.lower, config.getList('depends', [], onChange = initSandbox)) # Get error messages from gc-run.lib comments self.errorDict = {} self.updateErrorDict(utils.pathShare('gc-run.lib')) # Init parameter source manager self._setupJobParameters(config) self._pfactory = config.getPlugin('internal parameter factory', 'BasicParameterFactory', cls = ParameterFactory, tags = [self], inherit = True) self.source = config.getPlugin('parameter adapter', 'TrackedParameterAdapter', cls = ParameterAdapter, pargs = (self._pfactory.getSource(),))
def __init__(self, config, name, abort = None): NamedPlugin.__init__(self, config, name) # Workdir settings self._workDir = config.getWorkPath() self._checkSpace = config.getInt('workdir space', 10, onChange = None) # Initialise task module self.task = config.getPlugin(['module', 'task'], cls = TaskModule, tags = [self]) if abort == 'task': return utils.vprint('Current task ID: %s' % self.task.taskID, -1) utils.vprint('Task started on %s' % self.task.taskDate, -1) # Initialise workload management interface self.wms = config.getCompositePlugin('backend', 'grid', 'MultiWMS', cls = WMS, tags = [self, self.task]) # Subsequent config calls also include section "jobs": jobs_config = config.changeView(viewClass = 'TaggedConfigView', addSections = ['jobs'], addTags = [self]) # Initialise monitoring module self.monitor = jobs_config.getCompositePlugin('monitor', 'scripts', 'MultiMonitor', cls = Monitoring, tags = [self, self.task], pargs = (self.task,)) # Initialise job database self.jobManager = jobs_config.getPlugin('job manager', 'SimpleJobManager', cls = JobManager, tags = [self, self.task, self.wms], pargs = (self.task, self.monitor)) # Prepare work package self.wms.deployTask(self.task, self.monitor) # Configure workflow settings self._actionList = jobs_config.getList('action', ['check', 'retrieve', 'submit'], onChange = None) self.duration = 0 if jobs_config.getBool('continuous', False, onChange = None): # legacy option self.duration = -1 self.duration = jobs_config.getTime('duration', self.duration, onChange = None) self._submitFlag = jobs_config.getBool('submission', True, onChange = None) self._submitTime = jobs_config.getTime('submission time requirement', self.task.wallTime, onChange = None) # Initialise GUI self._gui = jobs_config.getPlugin('gui', 'SimpleConsole', cls = GUI, onChange = None, pargs = (self,))
def __init__(self, config, name, task=None, backend=None, job_manager=None): NamedPlugin.__init__(self, config, name) # Configure workflow settings jobs_config = config.change_view(view_class='TaggedConfigView', add_sections=['jobs']) self._action_list = jobs_config.get_list('action', ['check', 'retrieve', 'submit'], on_change=None) self._duration = 0 if jobs_config.get_bool('continuous', False, on_change=None): # legacy option self._duration = -1 self._duration = jobs_config.get_time('duration', self._duration, on_change=None) self._submit_flag = jobs_config.get_bool('submission', True, on_change=None) # Work directory settings self._check_space_dn = config.get_work_path() self._check_space = config.get_int('workdir space', 10, on_change=None) self._check_space_timeout = config.get_time('workdir space timeout', 5, on_change=None) self._check_space_log = logging.getLogger('workflow.space') self._check_space_log.addFilter(LogEveryNsec(interval=5 * 60)) # Configure local/job_manager and remote/backend monitoring module jobs_config.get_plugin(['event handler manager'], 'CompatEventHandlerManager', cls=EventHandlerManager, on_change=None) # Initialise task module self.task = config.get_plugin(['module', 'task'], cls=TaskModule, bind_kwargs={'tags': [self]}, override=task) if abort(): return # Initialise workload management interface self.backend = config.get_composited_plugin('backend', default_compositor='MultiWMS', cls=WMS, bind_kwargs={'tags': [self, self.task]}, override=backend) if abort(): return # Initialise job database self.job_manager = jobs_config.get_plugin('job manager', 'SimpleJobManager', cls=JobManager, bind_kwargs={'tags': [self, self.task, self.backend]}, pargs=(self.task,), override=job_manager) # Store submission settings / states self._transfer_se = config.get_state('init', detail='storage') self._transfer_sb = config.get_state('init', detail='sandbox') self._submit_time = jobs_config.get_time('submission time requirement', self.task.wall_time, on_change=None)
def __init__(self, config, name, task, eventhandler): NamedPlugin.__init__(self, config, name) (self._task, self._eventhandler) = (task, eventhandler) self._log_user = logging.getLogger('user') self._log_user_time = logging.getLogger('user.time') self._njobs_limit = config.getInt('jobs', -1, onChange=None) self._njobs_inflight = config.getInt('in flight', -1, onChange=None) self._njobs_inqueue = config.getInt('in queue', -1, onChange=None) self._chunks_enabled = config.getBool('chunks enabled', True, onChange=None) self._chunks_submit = config.getInt('chunks submit', 100, onChange=None) self._chunks_check = config.getInt('chunks check', 100, onChange=None) self._chunks_retrieve = config.getInt('chunks retrieve', 100, onChange=None) self._job_timeout = config.getTime('queue timeout', -1, onChange=None) self._job_retries = config.getInt('max retry', -1, onChange=None) selected = JobSelector.create(config.get('selected', '', onChange=None), task=self._task) self.jobDB = config.getPlugin('job database', 'JobDB', cls=JobDB, pargs=(self.getMaxJobs(self._task), selected)) self._disabled_jobs_logfile = config.getWorkPath('disabled') self._outputProcessor = config.getPlugin('output processor', 'SandboxProcessor', cls=TaskOutputProcessor, pargs=(task, )) self._do_shuffle = config.getBool('shuffle', False, onChange=None) self._reportClass = Report.getClass( config.get('abort report', 'LocationReport', onChange=None)) self._showBlocker = True
def __init__(self, config, name, task): NamedPlugin.__init__(self, config, name) self._local_event_handler = config.get_composited_plugin( ['local monitor', 'local event handler'], 'logmonitor', 'MultiLocalEventHandler', cls=LocalEventHandler, bind_kwargs={'tags': [self, task]}, require_plugin=False, on_change=None) self._local_event_handler = self._local_event_handler or LocalEventHandler(None, '') self._log = logging.getLogger('jobs.manager') self._njobs_limit = config.get_int('jobs', -1, on_change=None) self._njobs_inflight = config.get_int('in flight', -1, on_change=None) self._njobs_inqueue = config.get_int('in queue', -1, on_change=None) self._chunks_enabled = config.get_bool('chunks enabled', True, on_change=None) self._chunks_submit = config.get_int('chunks submit', 100, on_change=None) self._chunks_check = config.get_int('chunks check', 100, on_change=None) self._chunks_retrieve = config.get_int('chunks retrieve', 100, on_change=None) self._timeout_unknown = config.get_time('unknown timeout', -1, on_change=None) self._timeout_queue = config.get_time('queue timeout', -1, on_change=None) self._job_retries = config.get_int('max retry', -1, on_change=None) selected = JobSelector.create(config.get('selected', '', on_change=None), task=task) self.job_db = config.get_plugin('job database', 'TextFileJobDB', cls=JobDB, pargs=(self._get_max_jobs(task), selected), on_change=None) self._disabled_jobs_logfile = config.get_work_path('disabled') self._output_processor = config.get_plugin('output processor', 'SandboxProcessor', cls=TaskOutputProcessor, on_change=None) self._uii = UserInputInterface() self._interactive_cancel = config.is_interactive(['delete jobs', 'cancel jobs'], True) self._interactive_reset = config.is_interactive('reset jobs', True) self._do_shuffle = config.get_bool('shuffle', False, on_change=None) self._abort_report = config.get_plugin('abort report', 'LocationReport', cls=Report, pargs=(self.job_db, task), on_change=None) self._show_blocker = True self._callback_list = []
def __init__(self, config, name, abort=None): NamedPlugin.__init__(self, config, name) # Initial activity stream sys.stdout = SimpleActivityStream(sys.stdout, register_callback=True) sys.stderr = SimpleActivityStream(sys.stderr) # Workdir settings self._workDir = config.getWorkPath() self._checkSpace = config.getInt('workdir space', 10, onChange=None) # Initialise task module self.task = config.getPlugin(['module', 'task'], cls=TaskModule, tags=[self]) if abort == 'task': return self._log.log(logging.INFO, 'Current task ID: %s', self.task.taskID) self._log.log(logging.INFO, 'Task started on: %s', self.task.taskDate) # Initialise workload management interface self.wms = config.getCompositePlugin('backend', 'grid', 'MultiWMS', cls=WMS, tags=[self, self.task]) # Subsequent config calls also include section "jobs": jobs_config = config.changeView(viewClass='TaggedConfigView', addSections=['jobs'], addTags=[self]) # Initialise monitoring module self.monitor = jobs_config.getCompositePlugin('monitor', 'scripts', 'MultiMonitor', cls=Monitoring, tags=[self, self.task], pargs=(self.task, )) # Initialise job database self.jobManager = jobs_config.getPlugin( 'job manager', 'SimpleJobManager', cls=JobManager, tags=[self, self.task, self.wms], pargs=(self.task, self.monitor)) if abort == 'jobmanager': return # Prepare work package self.wms.deployTask(self.task, self.monitor, transferSE=config.getState('init', detail='storage'), transferSB=config.getState('init', detail='sandbox')) # Configure workflow settings self._actionList = jobs_config.getList('action', ['check', 'retrieve', 'submit'], onChange=None) self.duration = 0 if jobs_config.getBool('continuous', False, onChange=None): # legacy option self.duration = -1 self.duration = jobs_config.getTime('duration', self.duration, onChange=None) self._submitFlag = jobs_config.getBool('submission', True, onChange=None) self._submitTime = jobs_config.getTime('submission time requirement', self.task.wallTime, onChange=None) # Initialise GUI (sys.stdout, sys.stderr) = (sys.stdout.finish(), sys.stderr.finish()) self._gui = config.getPlugin('gui', 'SimpleConsole', cls=GUI, onChange=None, pargs=(self, ))
def __init__(self, config, name, task): NamedPlugin.__init__(self, config, name) self._local_event_handler = config.get_composited_plugin( ['local monitor', 'local event handler'], 'logmonitor', 'MultiLocalEventHandler', cls=LocalEventHandler, bind_kwargs={'tags': [self, task]}, require_plugin=False, on_change=None) self._local_event_handler = self._local_event_handler or LocalEventHandler( None, '') self._log = logging.getLogger('jobs.manager') self._njobs_limit = config.get_int('jobs', -1, on_change=None) self._njobs_inflight = config.get_int('in flight', -1, on_change=None) self._njobs_inqueue = config.get_int('in queue', -1, on_change=None) self._chunks_enabled = config.get_bool('chunks enabled', True, on_change=None) self._chunks_submit = config.get_int('chunks submit', 100, on_change=None) self._chunks_check = config.get_int('chunks check', 100, on_change=None) self._chunks_retrieve = config.get_int('chunks retrieve', 100, on_change=None) self._timeout_unknown = config.get_time('unknown timeout', -1, on_change=None) self._timeout_queue = config.get_time('queue timeout', -1, on_change=None) self._job_retries = config.get_int('max retry', -1, on_change=None) selected = JobSelector.create(config.get('selected', '', on_change=None), task=task) self.job_db = config.get_plugin('job database', 'TextFileJobDB', cls=JobDB, pargs=(self._get_max_jobs(task), selected), on_change=None) self._disabled_jobs_logfile = config.get_work_path('disabled') self._output_processor = config.get_plugin('output processor', 'SandboxProcessor', cls=TaskOutputProcessor, on_change=None) self._uii = UserInputInterface() self._interactive_cancel = config.is_interactive( ['delete jobs', 'cancel jobs'], True) self._interactive_reset = config.is_interactive('reset jobs', True) self._do_shuffle = config.get_bool('shuffle', False, on_change=None) self._abort_report = config.get_plugin('abort report', 'LocationReport', cls=Report, pargs=(self.job_db, task), on_change=None) self._show_blocker = True self._callback_list = []
def __init__(self, config, name, task): NamedPlugin.__init__(self, config, name) self._task = task
def __init__(self, config, name, optDefault, optPrefix, varPrefix): NamedPlugin.__init__(self, config, name) (self.smOptPrefix, self.varPrefix) = (optPrefix, varPrefix)
def __init__(self, config, name, job_db, task=None): NamedPlugin.__init__(self, config, name) self._job_class_list = [] for jc_attr in dir(JobClass): if hasattr(getattr(JobClass, jc_attr), 'state_list'): self._job_class_list.append(getattr(JobClass, jc_attr))
def __init__(self, config, name): NamedPlugin.__init__(self, config, name) self.adapter = config.get('parameter adapter', 'TrackedParameterAdapter') self._paramConfig = ParameterConfig(config.changeView(setSections = ['parameters']), self.adapter != 'TrackedParameterAdapter')
def __init__(self, config, name, storage_type, storage_channel, storage_var_prefix): NamedPlugin.__init__(self, config, name) (self._storage_channel, self._storage_var_prefix) = (storage_channel, storage_var_prefix)
def __init__(self, config, name, userOpt, itemName, discoverFun): NamedPlugin.__init__(self, config, name) (self._itemsStart, self._itemsDiscovered, self._itemName) = (None, False, itemName) self._nEntries = config.getInt('%s entries' % userOpt, 0, onChange = None) self._nRandom = config.getBool('%s randomize' % userOpt, False, onChange = None)
def __init__(self, config, name): # Read configuration options and init vars NamedPlugin.__init__(self, config, name) init_sandbox = TriggerInit('sandbox') self._var_checker = NoVarCheck(config) # Task requirements # Move this into parameter manager? jobs_config = config.change_view(view_class='TaggedConfigView', add_sections=['jobs'], add_tags=[self]) self.wall_time = jobs_config.get_time('wall time', on_change=None) self._cpu_time = jobs_config.get_time('cpu time', self.wall_time, on_change=None) self._cores = jobs_config.get_int(['cores', 'cpus'], 1, on_change=None) self._memory = jobs_config.get_int('memory', -1, on_change=None) self._job_timeout = jobs_config.get_time('node timeout', -1, on_change=init_sandbox) # Compute / get task ID self._task_id = config.get('task id', 'GC' + md5_hex(str(time.time()))[:12], persistent=True) self._task_date = config.get('task date', time.strftime('%Y-%m-%d'), persistent=True, on_change=init_sandbox) self._task_time = config.get('task time', time.strftime('%H%M%S'), persistent=True, on_change=init_sandbox) task_name_generator = config.get_plugin('task name generator', 'DefaultTaskName', cls=TaskNamePlugin) self._task_name = task_name_generator.get_name(self) self._task_config_name = config.get_config_name() self._job_name_generator = config.get_plugin('job name generator', 'DefaultJobName', cls=JobNamePlugin) # Storage setup storage_config = config.change_view(view_class='TaggedConfigView', set_classes=None, set_names=None, add_sections=['storage'], add_tags=[self]) scratch_space_used = storage_config.get_int('scratch space used', 5000, on_change=init_sandbox) lz_space_used = storage_config.get_int('landing zone space used', 100, on_change=init_sandbox) self._task_var_dict = { # Space limits 'SCRATCH_UL': scratch_space_used, 'SCRATCH_LL': storage_config.get_int('scratch space left', 1, on_change=init_sandbox), 'LANDINGZONE_UL': lz_space_used, 'LANDINGZONE_LL': storage_config.get_int('landing zone space left', 1, on_change=init_sandbox), } storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@') self._se_min_size = storage_config.get_int('se min size', -1, on_change=init_sandbox) self._disk_min = max(scratch_space_used, lz_space_used) self._sb_in_fn_list = config.get_path_list('input files', [], on_change=init_sandbox) self._sb_out_fn_list = config.get_list('output files', [], on_change=init_sandbox) self._do_gzip_std_output = config.get_bool('gzip output', True, on_change=init_sandbox) self._subst_files = config.get_list('subst files', [], on_change=init_sandbox) self._dependencies = lmap(str.lower, config.get_list('depends', [], on_change=init_sandbox)) # Get error messages from gc-run.lib comments self.map_error_code2msg = {} self._update_map_error_code2msg(get_path_share('gc-run.lib')) # Init parameter source manager psrc_repository = {} self._setup_repository(config, psrc_repository) pfactory = config.get_plugin('internal parameter factory', 'BasicParameterFactory', cls=ParameterFactory, bind_kwargs={'tags': [self], 'inherit': True}) self._source = config.get_plugin('parameter adapter', 'TrackedParameterAdapter', cls=ParameterAdapter, pargs=(pfactory.get_psrc(psrc_repository),)) self._log.log(logging.DEBUG3, 'Using parameter adapter %s', repr(self._source)) self._log.info('Current task ID: %s', self._task_id) self._log.info('Task started on: %s', self._task_date)
def __init__(self, config, name): # Read configuration options and init vars NamedPlugin.__init__(self, config, name) init_sandbox = TriggerInit('sandbox') self._var_checker = NoVarCheck(config) # Task requirements # Move this into parameter manager? jobs_config = config.change_view(view_class='TaggedConfigView', add_sections=['jobs'], add_tags=[self]) self.wall_time = jobs_config.get_time('wall time', on_change=None) self._cpu_time = jobs_config.get_time('cpu time', self.wall_time, on_change=None) self._cores = jobs_config.get_int(['cores', 'cpus'], 1, on_change=None) self._memory = jobs_config.get_int('memory', -1, on_change=None) self._job_timeout = jobs_config.get_time('node timeout', -1, on_change=init_sandbox) # Compute / get task ID self._task_id = config.get('task id', 'GC' + md5_hex(str(time.time()))[:12], persistent=True) self._task_date = config.get('task date', time.strftime('%Y-%m-%d'), persistent=True, on_change=init_sandbox) self._task_time = config.get('task time', time.strftime('%H%M%S'), persistent=True, on_change=init_sandbox) task_name_generator = config.get_plugin('task name generator', 'DefaultTaskName', cls=TaskNamePlugin) self._task_name = task_name_generator.get_name(self) self._task_config_name = config.get_config_name() self._job_name_generator = config.get_plugin('job name generator', 'DefaultJobName', cls=JobNamePlugin) # Storage setup storage_config = config.change_view(view_class='TaggedConfigView', set_classes=None, set_names=None, add_sections=['storage'], add_tags=[self]) scratch_space_used = storage_config.get_int('scratch space used', 5000, on_change=init_sandbox) lz_space_used = storage_config.get_int('landing zone space used', 100, on_change=init_sandbox) self._task_var_dict = { # Space limits 'SCRATCH_UL': scratch_space_used, 'SCRATCH_LL': storage_config.get_int('scratch space left', 1, on_change=init_sandbox), 'LANDINGZONE_UL': lz_space_used, 'LANDINGZONE_LL': storage_config.get_int('landing zone space left', 1, on_change=init_sandbox), } storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@') self._se_min_size = storage_config.get_int('se min size', -1, on_change=init_sandbox) self._disk_min = max(scratch_space_used, lz_space_used) self._sb_in_fn_list = config.get_path_list('input files', [], on_change=init_sandbox) self._sb_out_fn_list = config.get_list('output files', [], on_change=init_sandbox) self._do_gzip_std_output = config.get_bool('gzip output', True, on_change=init_sandbox) self._subst_files = config.get_list('subst files', [], on_change=init_sandbox) self._dependencies = lmap( str.lower, config.get_list('depends', [], on_change=init_sandbox)) # Get error messages from gc-run.lib comments self.map_error_code2msg = {} self._update_map_error_code2msg(get_path_share('gc-run.lib')) # Init parameter source manager psrc_repository = {} self._setup_repository(config, psrc_repository) pfactory = config.get_plugin('internal parameter factory', 'BasicParameterFactory', cls=ParameterFactory, bind_kwargs={ 'tags': [self], 'inherit': True }) self._source = config.get_plugin( 'parameter adapter', 'TrackedParameterAdapter', cls=ParameterAdapter, pargs=(pfactory.get_psrc(psrc_repository), )) self._log.log(logging.DEBUG3, 'Using parameter adapter %s', repr(self._source)) self._log.info('Current task ID: %s', self._task_id) self._log.info('Task started on: %s', self._task_date)
def __init__(self, config, name, abort=None): NamedPlugin.__init__(self, config, name) # Workdir settings self._workDir = config.getWorkPath() self._checkSpace = config.getInt('workdir space', 10, onChange=None) # Initialise task module self.task = config.getPlugin(['module', 'task'], cls=TaskModule, tags=[self]) if abort == 'task': return utils.vprint('Current task ID: %s' % self.task.taskID, -1) utils.vprint('Task started on %s' % self.task.taskDate, -1) # Initialise workload management interface self.wms = config.getCompositePlugin('backend', 'grid', 'MultiWMS', cls=WMS, tags=[self, self.task]) # Subsequent config calls also include section "jobs": jobs_config = config.changeView(viewClass='TaggedConfigView', addSections=['jobs'], addTags=[self]) # Initialise monitoring module self.monitor = jobs_config.getCompositePlugin('monitor', 'scripts', 'MultiMonitor', cls=Monitoring, tags=[self, self.task], pargs=(self.task, )) # Initialise job database self.jobManager = jobs_config.getPlugin( 'job manager', 'SimpleJobManager', cls=JobManager, tags=[self, self.task, self.wms], pargs=(self.task, self.monitor)) # Prepare work package self.wms.deployTask(self.task, self.monitor) # Configure workflow settings self._actionList = jobs_config.getList('action', ['check', 'retrieve', 'submit'], onChange=None) self.duration = 0 if jobs_config.getBool('continuous', False, onChange=None): # legacy option self.duration = -1 self.duration = jobs_config.getTime('duration', self.duration, onChange=None) self._submitFlag = jobs_config.getBool('submission', True, onChange=None) self._submitTime = jobs_config.getTime('submission time requirement', self.task.wallTime, onChange=None) # Initialise GUI self._gui = jobs_config.getPlugin('gui', 'SimpleConsole', cls=GUI, onChange=None, pargs=(self, ))