def __init__(self, config, wmsName):
wmsName = (wmsName
or self.__class__.__name__).upper().replace('.', '_')
NamedPlugin.__init__(self, config, wmsName)
(self.config, self.wmsName) = (config, wmsName)
self._wait_idle = config.getInt('wait idle', 60, onChange=None)
self._wait_work = config.getInt('wait work', 10, onChange=None)
def __init__(self, config, name, task, eventhandler):
NamedPlugin.__init__(self, config, name)
(self._task, self._eventhandler) = (task, eventhandler)
self._log_user = logging.getLogger('user')
self._log_user_time = logging.getLogger('user.time')
self._njobs_limit = config.getInt('jobs', -1, onChange = None)
self._njobs_inflight = config.getInt('in flight', -1, onChange = None)
self._njobs_inqueue = config.getInt('in queue', -1, onChange = None)
self._chunks_submit = config.getInt('chunks submit', 100, onChange = None)
self._chunks_check = config.getInt('chunks check', 100, onChange = None)
self._chunks_retrieve = config.getInt('chunks retrieve', 100, onChange = None)
self._job_timeout = config.getTime('queue timeout', -1, onChange = None)
self._job_retries = config.getInt('max retry', -1, onChange = None)
selected = JobSelector.create(config.get('selected', '', onChange = None), task = self._task)
self.jobDB = config.getPlugin('job database', 'JobDB',
cls = JobDB, pargs = (self.getMaxJobs(self._task), selected))
self._disabled_jobs_logfile = config.getWorkPath('disabled')
self._outputProcessor = config.getPlugin('output processor', 'SandboxProcessor',
cls = TaskOutputProcessor, pargs = (task,))
self._do_shuffle = config.getBool('shuffle', False, onChange = None)
self._continuous = config.getBool('continuous', False, onChange = None)
self._reportClass = Report.getClass(config.get('abort report', 'LocationReport', onChange = None))
self._showBlocker = True
def __init__(self, config, name):
name = (name or self.__class__.__name__).upper().replace('.', '_')
NamedPlugin.__init__(self, config, name)
self._wait_idle = config.getInt('wait idle', 60, onChange = None)
self._wait_work = config.getInt('wait work', 10, onChange = None)
self._job_parser = config.getPlugin('job parser', 'JobInfoProcessor',
cls = 'JobInfoProcessor', onChange = None)
def __init__(self, config, name, workflow, redraw_event, stream):
NamedPlugin.__init__(self, config, name)
(self._dirty, self._redraw_event, self._on_height_change) = (False, redraw_event, None)
(self._layout_pos, self._layout_height, self._layout_width) = (0, 0, 0)
self._dirty_interval = config.get_float('gui refresh interval', 0.2, on_change=None)
self._height_interval = config.get_float('gui height interval', 10., on_change=None)
(self._dirty_next, self._height_next) = (0, 0)
self._stream = stream
def __init__(self, config, name):
name = (name or self.__class__.__name__).upper().replace('.', '_')
NamedPlugin.__init__(self, config, name)
self._wait_idle = config.getInt('wait idle', 60, onChange=None)
self._wait_work = config.getInt('wait work', 10, onChange=None)
self._job_parser = config.getPlugin('job parser',
'JobInfoProcessor',
cls='JobInfoProcessor',
onChange=None)
def __init__(self, config, name, broker_prefix, item_name, discover_fun):
NamedPlugin.__init__(self, config, name)
(self._item_list_start, self._item_list_discovered,
self._item_name) = (None, False, item_name)
self._num_entries = config.get_int('%s entries' % broker_prefix,
0,
on_change=None)
self._randomize = config.get_bool('%s randomize' % broker_prefix,
False,
on_change=None)
def __init__(self, config, name, userOpt, itemName, discoverFun):
NamedPlugin.__init__(self, config, name)
(self._itemsStart, self._itemsDiscovered,
self._itemName) = (None, False, itemName)
self._nEntries = config.getInt('%s entries' % userOpt,
0,
onChange=None)
self._nRandom = config.getBool('%s randomize' % userOpt,
False,
onChange=None)
def __init__(self, config, name, abort = None):
NamedPlugin.__init__(self, config, name)
# Initial activity stream
sys.stdout = SimpleActivityStream(sys.stdout, register_callback = True)
sys.stderr = SimpleActivityStream(sys.stderr)
# Workdir settings
self._workDir = config.getWorkPath()
self._checkSpace = config.getInt('workdir space', 10, onChange = None)
# Initialise task module
self.task = config.getPlugin(['module', 'task'], cls = TaskModule, tags = [self])
if abort == 'task':
return
self._log.log(logging.INFO, 'Current task ID: %s', self.task.taskID)
self._log.log(logging.INFO, 'Task started on: %s', self.task.taskDate)
# Initialise workload management interface
self.wms = config.getCompositePlugin('backend', 'grid', 'MultiWMS',
cls = WMS, tags = [self, self.task])
# Subsequent config calls also include section "jobs":
jobs_config = config.changeView(viewClass = 'TaggedConfigView',
addSections = ['jobs'], addTags = [self])
# Initialise monitoring module
self.monitor = jobs_config.getCompositePlugin('monitor', 'scripts', 'MultiMonitor',
cls = Monitoring, tags = [self, self.task], pargs = (self.task,))
# Initialise job database
self.jobManager = jobs_config.getPlugin('job manager', 'SimpleJobManager',
cls = JobManager, tags = [self, self.task, self.wms], pargs = (self.task, self.monitor))
if abort == 'jobmanager':
return
# Prepare work package
self.wms.deployTask(self.task, self.monitor,
transferSE = config.getState('init', detail = 'storage'),
transferSB = config.getState('init', detail = 'sandbox'))
# Configure workflow settings
self._actionList = jobs_config.getList('action', ['check', 'retrieve', 'submit'], onChange = None)
self.duration = 0
if jobs_config.getBool('continuous', False, onChange = None): # legacy option
self.duration = -1
self.duration = jobs_config.getTime('duration', self.duration, onChange = None)
self._submitFlag = jobs_config.getBool('submission', True, onChange = None)
self._submitTime = jobs_config.getTime('submission time requirement', self.task.wallTime, onChange = None)
# Initialise GUI
(sys.stdout, sys.stderr) = (sys.stdout.finish(), sys.stderr.finish())
self._gui = config.getPlugin('gui', 'SimpleConsole', cls = GUI, onChange = None, pargs = (self,))
def __init__(self, config, name):
name = (name or self.__class__.__name__).upper().replace('.', '_')
NamedPlugin.__init__(self, config, name)
self._wait_idle = config.get_int('wait idle', 60, on_change=None)
self._wait_work = config.get_int('wait work', 10, on_change=None)
self._job_parser = config.get_plugin('job parser', 'JobInfoProcessor',
cls='JobInfoProcessor', on_change=None)
self._remote_event_handler = config.get_composited_plugin(
['remote monitor', 'remote event handler'], '', 'MultiRemoteEventHandler',
cls=RemoteEventHandler, bind_kwargs={'tags': [self]},
require_plugin=False, on_change=TriggerInit('sandbox')) or RemoteEventHandler(config, 'dummy')
def __init__(self, config, name):
name = (name or self.__class__.__name__).upper().replace('.', '_')
NamedPlugin.__init__(self, config, name)
self._wait_idle = config.get_int('wait idle', 60, on_change=None)
self._wait_work = config.get_int('wait work', 10, on_change=None)
self._job_parser = config.get_plugin('job parser', 'JobInfoProcessor',
cls='JobInfoProcessor', on_change=None)
self._remote_event_handler = config.get_composited_plugin(
['remote monitor', 'remote event handler'], '', 'MultiRemoteEventHandler',
cls=RemoteEventHandler, bind_kwargs={'tags': [self]},
require_plugin=False, on_change=TriggerInit('sandbox')) or RemoteEventHandler(config, 'dummy')
def __init__(self, config, name):
NamedPlugin.__init__(self, config, name)
initSandbox = changeInitNeeded('sandbox')
self._varCheck = validNoVar(config)
# Task requirements
jobs_config = config.changeView(viewClass = 'TaggedConfigView', addSections = ['jobs'], addTags = [self]) # Move this into parameter manager?
self.wallTime = jobs_config.getTime('wall time', onChange = None)
self.cpuTime = jobs_config.getTime('cpu time', self.wallTime, onChange = None)
self.cpus = jobs_config.getInt('cpus', 1, onChange = None)
self.memory = jobs_config.getInt('memory', -1, onChange = None)
self.nodeTimeout = jobs_config.getTime('node timeout', -1, onChange = initSandbox)
# Compute / get task ID
self.taskID = config.get('task id', 'GC' + md5_hex(str(time()))[:12], persistent = True)
self.taskDate = config.get('task date', strftime('%Y-%m-%d'), persistent = True, onChange = initSandbox)
self.taskConfigName = config.getConfigName()
self._job_name_generator = config.getPlugin('job name generator', 'DefaultJobName',
cls = JobNamePlugin, pargs = (self,))
# Storage setup
storage_config = config.changeView(viewClass = 'TaggedConfigView',
setClasses = None, setNames = None, addSections = ['storage'], addTags = [self])
self.taskVariables = {
# Space limits
'SCRATCH_UL': storage_config.getInt('scratch space used', 5000, onChange = initSandbox),
'SCRATCH_LL': storage_config.getInt('scratch space left', 1, onChange = initSandbox),
'LANDINGZONE_UL': storage_config.getInt('landing zone space used', 100, onChange = initSandbox),
'LANDINGZONE_LL': storage_config.getInt('landing zone space left', 1, onChange = initSandbox),
}
storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@')
self.seMinSize = storage_config.getInt('se min size', -1, onChange = initSandbox)
self.sbInputFiles = config.getPaths('input files', [], onChange = initSandbox)
self.sbOutputFiles = config.getList('output files', [], onChange = initSandbox)
self.gzipOut = config.getBool('gzip output', True, onChange = initSandbox)
self._subst_files = config.getList('subst files', [], onChange = initSandbox)
self.dependencies = lmap(str.lower, config.getList('depends', [], onChange = initSandbox))
# Get error messages from gc-run.lib comments
self.errorDict = {}
self.updateErrorDict(utils.pathShare('gc-run.lib'))
# Init parameter source manager
psrc_repository = {}
self._setupJobParameters(config, psrc_repository)
self._pfactory = config.getPlugin('internal parameter factory', 'BasicParameterFactory',
cls = ParameterFactory, pargs = (psrc_repository,), tags = [self], inherit = True)
self.source = config.getPlugin('parameter adapter', 'TrackedParameterAdapter',
cls = ParameterAdapter, pargs = (self._pfactory.getSource(),))
def __init__(self, config, name):
NamedPlugin.__init__(self, config, name)
initSandbox = changeInitNeeded('sandbox')
self._varCheck = validNoVar(config)
# Task requirements
jobs_config = config.changeView(viewClass = 'TaggedConfigView', addSections = ['jobs'], addTags = [self]) # Move this into parameter manager?
self.wallTime = jobs_config.getTime('wall time', onChange = None)
self.cpuTime = jobs_config.getTime('cpu time', self.wallTime, onChange = None)
self.cpus = jobs_config.getInt('cpus', 1, onChange = None)
self.memory = jobs_config.getInt('memory', -1, onChange = None)
self.nodeTimeout = jobs_config.getTime('node timeout', -1, onChange = initSandbox)
# Compute / get task ID
self.taskID = config.get('task id', 'GC' + md5_hex(str(time()))[:12], persistent = True)
self.taskDate = config.get('task date', strftime('%Y-%m-%d'), persistent = True, onChange = initSandbox)
self.taskConfigName = config.getConfigName()
self._job_name_generator = config.getPlugin('job name generator', 'DefaultJobName',
cls = JobNamePlugin, pargs = (self,))
# Storage setup
storage_config = config.changeView(viewClass = 'TaggedConfigView',
setClasses = None, setNames = None, addSections = ['storage'], addTags = [self])
self.taskVariables = {
# Space limits
'SCRATCH_UL': storage_config.getInt('scratch space used', 5000, onChange = initSandbox),
'SCRATCH_LL': storage_config.getInt('scratch space left', 1, onChange = initSandbox),
'LANDINGZONE_UL': storage_config.getInt('landing zone space used', 100, onChange = initSandbox),
'LANDINGZONE_LL': storage_config.getInt('landing zone space left', 1, onChange = initSandbox),
}
storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@')
self.seMinSize = storage_config.getInt('se min size', -1, onChange = initSandbox)
self.sbInputFiles = config.getPaths('input files', [], onChange = initSandbox)
self.sbOutputFiles = config.getList('output files', [], onChange = initSandbox)
self.gzipOut = config.getBool('gzip output', True, onChange = initSandbox)
self.substFiles = config.getList('subst files', [], onChange = initSandbox)
self.dependencies = lmap(str.lower, config.getList('depends', [], onChange = initSandbox))
# Get error messages from gc-run.lib comments
self.errorDict = {}
self.updateErrorDict(utils.pathShare('gc-run.lib'))
# Init parameter source manager
self._setupJobParameters(config)
self._pfactory = config.getPlugin('internal parameter factory', 'BasicParameterFactory',
cls = ParameterFactory, tags = [self], inherit = True)
self.source = config.getPlugin('parameter adapter', 'TrackedParameterAdapter',
cls = ParameterAdapter, pargs = (self._pfactory.getSource(),))
def __init__(self, config, name, abort = None):
NamedPlugin.__init__(self, config, name)
# Workdir settings
self._workDir = config.getWorkPath()
self._checkSpace = config.getInt('workdir space', 10, onChange = None)
# Initialise task module
self.task = config.getPlugin(['module', 'task'], cls = TaskModule, tags = [self])
if abort == 'task':
return
utils.vprint('Current task ID: %s' % self.task.taskID, -1)
utils.vprint('Task started on %s' % self.task.taskDate, -1)
# Initialise workload management interface
self.wms = config.getCompositePlugin('backend', 'grid', 'MultiWMS',
cls = WMS, tags = [self, self.task])
# Subsequent config calls also include section "jobs":
jobs_config = config.changeView(viewClass = 'TaggedConfigView',
addSections = ['jobs'], addTags = [self])
# Initialise monitoring module
self.monitor = jobs_config.getCompositePlugin('monitor', 'scripts', 'MultiMonitor',
cls = Monitoring, tags = [self, self.task], pargs = (self.task,))
# Initialise job database
self.jobManager = jobs_config.getPlugin('job manager', 'SimpleJobManager',
cls = JobManager, tags = [self, self.task, self.wms], pargs = (self.task, self.monitor))
# Prepare work package
self.wms.deployTask(self.task, self.monitor)
# Configure workflow settings
self._actionList = jobs_config.getList('action', ['check', 'retrieve', 'submit'], onChange = None)
self.duration = 0
if jobs_config.getBool('continuous', False, onChange = None): # legacy option
self.duration = -1
self.duration = jobs_config.getTime('duration', self.duration, onChange = None)
self._submitFlag = jobs_config.getBool('submission', True, onChange = None)
self._submitTime = jobs_config.getTime('submission time requirement', self.task.wallTime, onChange = None)
# Initialise GUI
self._gui = jobs_config.getPlugin('gui', 'SimpleConsole', cls = GUI, onChange = None, pargs = (self,))
def __init__(self, config, name, task=None, backend=None, job_manager=None):
NamedPlugin.__init__(self, config, name)
# Configure workflow settings
jobs_config = config.change_view(view_class='TaggedConfigView', add_sections=['jobs'])
self._action_list = jobs_config.get_list('action',
['check', 'retrieve', 'submit'], on_change=None)
self._duration = 0
if jobs_config.get_bool('continuous', False, on_change=None): # legacy option
self._duration = -1
self._duration = jobs_config.get_time('duration', self._duration, on_change=None)
self._submit_flag = jobs_config.get_bool('submission', True, on_change=None)
# Work directory settings
self._check_space_dn = config.get_work_path()
self._check_space = config.get_int('workdir space', 10, on_change=None)
self._check_space_timeout = config.get_time('workdir space timeout', 5, on_change=None)
self._check_space_log = logging.getLogger('workflow.space')
self._check_space_log.addFilter(LogEveryNsec(interval=5 * 60))
# Configure local/job_manager and remote/backend monitoring module
jobs_config.get_plugin(['event handler manager'], 'CompatEventHandlerManager',
cls=EventHandlerManager, on_change=None)
# Initialise task module
self.task = config.get_plugin(['module', 'task'], cls=TaskModule,
bind_kwargs={'tags': [self]}, override=task)
if abort():
return
# Initialise workload management interface
self.backend = config.get_composited_plugin('backend', default_compositor='MultiWMS', cls=WMS,
bind_kwargs={'tags': [self, self.task]}, override=backend)
if abort():
return
# Initialise job database
self.job_manager = jobs_config.get_plugin('job manager', 'SimpleJobManager', cls=JobManager,
bind_kwargs={'tags': [self, self.task, self.backend]}, pargs=(self.task,), override=job_manager)
# Store submission settings / states
self._transfer_se = config.get_state('init', detail='storage')
self._transfer_sb = config.get_state('init', detail='sandbox')
self._submit_time = jobs_config.get_time('submission time requirement',
self.task.wall_time, on_change=None)
def __init__(self, config, name, task, eventhandler):
NamedPlugin.__init__(self, config, name)
(self._task, self._eventhandler) = (task, eventhandler)
self._log_user = logging.getLogger('user')
self._log_user_time = logging.getLogger('user.time')
self._njobs_limit = config.getInt('jobs', -1, onChange=None)
self._njobs_inflight = config.getInt('in flight', -1, onChange=None)
self._njobs_inqueue = config.getInt('in queue', -1, onChange=None)
self._chunks_enabled = config.getBool('chunks enabled',
True,
onChange=None)
self._chunks_submit = config.getInt('chunks submit',
100,
onChange=None)
self._chunks_check = config.getInt('chunks check', 100, onChange=None)
self._chunks_retrieve = config.getInt('chunks retrieve',
100,
onChange=None)
self._job_timeout = config.getTime('queue timeout', -1, onChange=None)
self._job_retries = config.getInt('max retry', -1, onChange=None)
selected = JobSelector.create(config.get('selected', '',
onChange=None),
task=self._task)
self.jobDB = config.getPlugin('job database',
'JobDB',
cls=JobDB,
pargs=(self.getMaxJobs(self._task),
selected))
self._disabled_jobs_logfile = config.getWorkPath('disabled')
self._outputProcessor = config.getPlugin('output processor',
'SandboxProcessor',
cls=TaskOutputProcessor,
pargs=(task, ))
self._do_shuffle = config.getBool('shuffle', False, onChange=None)
self._reportClass = Report.getClass(
config.get('abort report', 'LocationReport', onChange=None))
self._showBlocker = True
def __init__(self, config, name, task):
NamedPlugin.__init__(self, config, name)
self._local_event_handler = config.get_composited_plugin(
['local monitor', 'local event handler'], 'logmonitor', 'MultiLocalEventHandler',
cls=LocalEventHandler, bind_kwargs={'tags': [self, task]},
require_plugin=False, on_change=None)
self._local_event_handler = self._local_event_handler or LocalEventHandler(None, '')
self._log = logging.getLogger('jobs.manager')
self._njobs_limit = config.get_int('jobs', -1, on_change=None)
self._njobs_inflight = config.get_int('in flight', -1, on_change=None)
self._njobs_inqueue = config.get_int('in queue', -1, on_change=None)
self._chunks_enabled = config.get_bool('chunks enabled', True, on_change=None)
self._chunks_submit = config.get_int('chunks submit', 100, on_change=None)
self._chunks_check = config.get_int('chunks check', 100, on_change=None)
self._chunks_retrieve = config.get_int('chunks retrieve', 100, on_change=None)
self._timeout_unknown = config.get_time('unknown timeout', -1, on_change=None)
self._timeout_queue = config.get_time('queue timeout', -1, on_change=None)
self._job_retries = config.get_int('max retry', -1, on_change=None)
selected = JobSelector.create(config.get('selected', '', on_change=None), task=task)
self.job_db = config.get_plugin('job database', 'TextFileJobDB',
cls=JobDB, pargs=(self._get_max_jobs(task), selected), on_change=None)
self._disabled_jobs_logfile = config.get_work_path('disabled')
self._output_processor = config.get_plugin('output processor', 'SandboxProcessor',
cls=TaskOutputProcessor, on_change=None)
self._uii = UserInputInterface()
self._interactive_cancel = config.is_interactive(['delete jobs', 'cancel jobs'], True)
self._interactive_reset = config.is_interactive('reset jobs', True)
self._do_shuffle = config.get_bool('shuffle', False, on_change=None)
self._abort_report = config.get_plugin('abort report', 'LocationReport',
cls=Report, pargs=(self.job_db, task), on_change=None)
self._show_blocker = True
self._callback_list = []
def __init__(self, config, name, abort=None):
NamedPlugin.__init__(self, config, name)
# Initial activity stream
sys.stdout = SimpleActivityStream(sys.stdout, register_callback=True)
sys.stderr = SimpleActivityStream(sys.stderr)
# Workdir settings
self._workDir = config.getWorkPath()
self._checkSpace = config.getInt('workdir space', 10, onChange=None)
# Initialise task module
self.task = config.getPlugin(['module', 'task'],
cls=TaskModule,
tags=[self])
if abort == 'task':
return
self._log.log(logging.INFO, 'Current task ID: %s', self.task.taskID)
self._log.log(logging.INFO, 'Task started on: %s', self.task.taskDate)
# Initialise workload management interface
self.wms = config.getCompositePlugin('backend',
'grid',
'MultiWMS',
cls=WMS,
tags=[self, self.task])
# Subsequent config calls also include section "jobs":
jobs_config = config.changeView(viewClass='TaggedConfigView',
addSections=['jobs'],
addTags=[self])
# Initialise monitoring module
self.monitor = jobs_config.getCompositePlugin('monitor',
'scripts',
'MultiMonitor',
cls=Monitoring,
tags=[self, self.task],
pargs=(self.task, ))
# Initialise job database
self.jobManager = jobs_config.getPlugin(
'job manager',
'SimpleJobManager',
cls=JobManager,
tags=[self, self.task, self.wms],
pargs=(self.task, self.monitor))
if abort == 'jobmanager':
return
# Prepare work package
self.wms.deployTask(self.task,
self.monitor,
transferSE=config.getState('init',
detail='storage'),
transferSB=config.getState('init',
detail='sandbox'))
# Configure workflow settings
self._actionList = jobs_config.getList('action',
['check', 'retrieve', 'submit'],
onChange=None)
self.duration = 0
if jobs_config.getBool('continuous', False,
onChange=None): # legacy option
self.duration = -1
self.duration = jobs_config.getTime('duration',
self.duration,
onChange=None)
self._submitFlag = jobs_config.getBool('submission',
True,
onChange=None)
self._submitTime = jobs_config.getTime('submission time requirement',
self.task.wallTime,
onChange=None)
# Initialise GUI
(sys.stdout, sys.stderr) = (sys.stdout.finish(), sys.stderr.finish())
self._gui = config.getPlugin('gui',
'SimpleConsole',
cls=GUI,
onChange=None,
pargs=(self, ))
def __init__(self, config, name, task):
NamedPlugin.__init__(self, config, name)
self._local_event_handler = config.get_composited_plugin(
['local monitor', 'local event handler'],
'logmonitor',
'MultiLocalEventHandler',
cls=LocalEventHandler,
bind_kwargs={'tags': [self, task]},
require_plugin=False,
on_change=None)
self._local_event_handler = self._local_event_handler or LocalEventHandler(
None, '')
self._log = logging.getLogger('jobs.manager')
self._njobs_limit = config.get_int('jobs', -1, on_change=None)
self._njobs_inflight = config.get_int('in flight', -1, on_change=None)
self._njobs_inqueue = config.get_int('in queue', -1, on_change=None)
self._chunks_enabled = config.get_bool('chunks enabled',
True,
on_change=None)
self._chunks_submit = config.get_int('chunks submit',
100,
on_change=None)
self._chunks_check = config.get_int('chunks check',
100,
on_change=None)
self._chunks_retrieve = config.get_int('chunks retrieve',
100,
on_change=None)
self._timeout_unknown = config.get_time('unknown timeout',
-1,
on_change=None)
self._timeout_queue = config.get_time('queue timeout',
-1,
on_change=None)
self._job_retries = config.get_int('max retry', -1, on_change=None)
selected = JobSelector.create(config.get('selected',
'',
on_change=None),
task=task)
self.job_db = config.get_plugin('job database',
'TextFileJobDB',
cls=JobDB,
pargs=(self._get_max_jobs(task),
selected),
on_change=None)
self._disabled_jobs_logfile = config.get_work_path('disabled')
self._output_processor = config.get_plugin('output processor',
'SandboxProcessor',
cls=TaskOutputProcessor,
on_change=None)
self._uii = UserInputInterface()
self._interactive_cancel = config.is_interactive(
['delete jobs', 'cancel jobs'], True)
self._interactive_reset = config.is_interactive('reset jobs', True)
self._do_shuffle = config.get_bool('shuffle', False, on_change=None)
self._abort_report = config.get_plugin('abort report',
'LocationReport',
cls=Report,
pargs=(self.job_db, task),
on_change=None)
self._show_blocker = True
self._callback_list = []
def __init__(self, config, name, task): NamedPlugin.__init__(self, config, name) self._task = task
def __init__(self, config, name, optDefault, optPrefix, varPrefix): NamedPlugin.__init__(self, config, name) (self.smOptPrefix, self.varPrefix) = (optPrefix, varPrefix)
def __init__(self, config, name, broker_prefix, item_name, discover_fun):
NamedPlugin.__init__(self, config, name)
(self._item_list_start, self._item_list_discovered, self._item_name) = (None, False, item_name)
self._num_entries = config.get_int('%s entries' % broker_prefix, 0, on_change=None)
self._randomize = config.get_bool('%s randomize' % broker_prefix, False, on_change=None)
def __init__(self, config, name, job_db, task=None):
NamedPlugin.__init__(self, config, name)
self._job_class_list = []
for jc_attr in dir(JobClass):
if hasattr(getattr(JobClass, jc_attr), 'state_list'):
self._job_class_list.append(getattr(JobClass, jc_attr))
def __init__(self, config, name, job_db, task=None): NamedPlugin.__init__(self, config, name) self._job_class_list = [] for jc_attr in dir(JobClass): if hasattr(getattr(JobClass, jc_attr), 'state_list'): self._job_class_list.append(getattr(JobClass, jc_attr))
def __init__(self, config, name, optDefault, optPrefix, varPrefix):
NamedPlugin.__init__(self, config, name)
(self.smOptPrefix, self.varPrefix) = (optPrefix, varPrefix)
def __init__(self, config, name):
NamedPlugin.__init__(self, config, name)
self.adapter = config.get('parameter adapter', 'TrackedParameterAdapter')
self._paramConfig = ParameterConfig(config.changeView(setSections = ['parameters']), self.adapter != 'TrackedParameterAdapter')
def __init__(self, config, name, storage_type, storage_channel,
storage_var_prefix):
NamedPlugin.__init__(self, config, name)
(self._storage_channel,
self._storage_var_prefix) = (storage_channel, storage_var_prefix)
def __init__(self, config, name, userOpt, itemName, discoverFun):
NamedPlugin.__init__(self, config, name)
(self._itemsStart, self._itemsDiscovered, self._itemName) = (None, False, itemName)
self._nEntries = config.getInt('%s entries' % userOpt, 0, onChange = None)
self._nRandom = config.getBool('%s randomize' % userOpt, False, onChange = None)
def __init__(self, config, name): # Read configuration options and init vars
NamedPlugin.__init__(self, config, name)
init_sandbox = TriggerInit('sandbox')
self._var_checker = NoVarCheck(config)
# Task requirements
# Move this into parameter manager?
jobs_config = config.change_view(view_class='TaggedConfigView',
add_sections=['jobs'], add_tags=[self])
self.wall_time = jobs_config.get_time('wall time', on_change=None)
self._cpu_time = jobs_config.get_time('cpu time', self.wall_time, on_change=None)
self._cores = jobs_config.get_int(['cores', 'cpus'], 1, on_change=None)
self._memory = jobs_config.get_int('memory', -1, on_change=None)
self._job_timeout = jobs_config.get_time('node timeout', -1, on_change=init_sandbox)
# Compute / get task ID
self._task_id = config.get('task id', 'GC' + md5_hex(str(time.time()))[:12], persistent=True)
self._task_date = config.get('task date', time.strftime('%Y-%m-%d'),
persistent=True, on_change=init_sandbox)
self._task_time = config.get('task time', time.strftime('%H%M%S'),
persistent=True, on_change=init_sandbox)
task_name_generator = config.get_plugin('task name generator',
'DefaultTaskName', cls=TaskNamePlugin)
self._task_name = task_name_generator.get_name(self)
self._task_config_name = config.get_config_name()
self._job_name_generator = config.get_plugin('job name generator',
'DefaultJobName', cls=JobNamePlugin)
# Storage setup
storage_config = config.change_view(view_class='TaggedConfigView',
set_classes=None, set_names=None, add_sections=['storage'], add_tags=[self])
scratch_space_used = storage_config.get_int('scratch space used', 5000, on_change=init_sandbox)
lz_space_used = storage_config.get_int('landing zone space used', 100, on_change=init_sandbox)
self._task_var_dict = {
# Space limits
'SCRATCH_UL': scratch_space_used,
'SCRATCH_LL': storage_config.get_int('scratch space left', 1, on_change=init_sandbox),
'LANDINGZONE_UL': lz_space_used,
'LANDINGZONE_LL': storage_config.get_int('landing zone space left', 1, on_change=init_sandbox),
}
storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@')
self._se_min_size = storage_config.get_int('se min size', -1, on_change=init_sandbox)
self._disk_min = max(scratch_space_used, lz_space_used)
self._sb_in_fn_list = config.get_path_list('input files', [], on_change=init_sandbox)
self._sb_out_fn_list = config.get_list('output files', [], on_change=init_sandbox)
self._do_gzip_std_output = config.get_bool('gzip output', True, on_change=init_sandbox)
self._subst_files = config.get_list('subst files', [], on_change=init_sandbox)
self._dependencies = lmap(str.lower, config.get_list('depends', [], on_change=init_sandbox))
# Get error messages from gc-run.lib comments
self.map_error_code2msg = {}
self._update_map_error_code2msg(get_path_share('gc-run.lib'))
# Init parameter source manager
psrc_repository = {}
self._setup_repository(config, psrc_repository)
pfactory = config.get_plugin('internal parameter factory', 'BasicParameterFactory',
cls=ParameterFactory, bind_kwargs={'tags': [self], 'inherit': True})
self._source = config.get_plugin('parameter adapter', 'TrackedParameterAdapter',
cls=ParameterAdapter, pargs=(pfactory.get_psrc(psrc_repository),))
self._log.log(logging.DEBUG3, 'Using parameter adapter %s', repr(self._source))
self._log.info('Current task ID: %s', self._task_id)
self._log.info('Task started on: %s', self._task_date)
def __init__(self, config, name, task):
NamedPlugin.__init__(self, config, name)
self._task = task
def __init__(self, config,
name): # Read configuration options and init vars
NamedPlugin.__init__(self, config, name)
init_sandbox = TriggerInit('sandbox')
self._var_checker = NoVarCheck(config)
# Task requirements
# Move this into parameter manager?
jobs_config = config.change_view(view_class='TaggedConfigView',
add_sections=['jobs'],
add_tags=[self])
self.wall_time = jobs_config.get_time('wall time', on_change=None)
self._cpu_time = jobs_config.get_time('cpu time',
self.wall_time,
on_change=None)
self._cores = jobs_config.get_int(['cores', 'cpus'], 1, on_change=None)
self._memory = jobs_config.get_int('memory', -1, on_change=None)
self._job_timeout = jobs_config.get_time('node timeout',
-1,
on_change=init_sandbox)
# Compute / get task ID
self._task_id = config.get('task id',
'GC' + md5_hex(str(time.time()))[:12],
persistent=True)
self._task_date = config.get('task date',
time.strftime('%Y-%m-%d'),
persistent=True,
on_change=init_sandbox)
self._task_time = config.get('task time',
time.strftime('%H%M%S'),
persistent=True,
on_change=init_sandbox)
task_name_generator = config.get_plugin('task name generator',
'DefaultTaskName',
cls=TaskNamePlugin)
self._task_name = task_name_generator.get_name(self)
self._task_config_name = config.get_config_name()
self._job_name_generator = config.get_plugin('job name generator',
'DefaultJobName',
cls=JobNamePlugin)
# Storage setup
storage_config = config.change_view(view_class='TaggedConfigView',
set_classes=None,
set_names=None,
add_sections=['storage'],
add_tags=[self])
scratch_space_used = storage_config.get_int('scratch space used',
5000,
on_change=init_sandbox)
lz_space_used = storage_config.get_int('landing zone space used',
100,
on_change=init_sandbox)
self._task_var_dict = {
# Space limits
'SCRATCH_UL':
scratch_space_used,
'SCRATCH_LL':
storage_config.get_int('scratch space left',
1,
on_change=init_sandbox),
'LANDINGZONE_UL':
lz_space_used,
'LANDINGZONE_LL':
storage_config.get_int('landing zone space left',
1,
on_change=init_sandbox),
}
storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@')
self._se_min_size = storage_config.get_int('se min size',
-1,
on_change=init_sandbox)
self._disk_min = max(scratch_space_used, lz_space_used)
self._sb_in_fn_list = config.get_path_list('input files', [],
on_change=init_sandbox)
self._sb_out_fn_list = config.get_list('output files', [],
on_change=init_sandbox)
self._do_gzip_std_output = config.get_bool('gzip output',
True,
on_change=init_sandbox)
self._subst_files = config.get_list('subst files', [],
on_change=init_sandbox)
self._dependencies = lmap(
str.lower, config.get_list('depends', [], on_change=init_sandbox))
# Get error messages from gc-run.lib comments
self.map_error_code2msg = {}
self._update_map_error_code2msg(get_path_share('gc-run.lib'))
# Init parameter source manager
psrc_repository = {}
self._setup_repository(config, psrc_repository)
pfactory = config.get_plugin('internal parameter factory',
'BasicParameterFactory',
cls=ParameterFactory,
bind_kwargs={
'tags': [self],
'inherit': True
})
self._source = config.get_plugin(
'parameter adapter',
'TrackedParameterAdapter',
cls=ParameterAdapter,
pargs=(pfactory.get_psrc(psrc_repository), ))
self._log.log(logging.DEBUG3, 'Using parameter adapter %s',
repr(self._source))
self._log.info('Current task ID: %s', self._task_id)
self._log.info('Task started on: %s', self._task_date)
def __init__(self, config, name, abort=None):
NamedPlugin.__init__(self, config, name)
# Workdir settings
self._workDir = config.getWorkPath()
self._checkSpace = config.getInt('workdir space', 10, onChange=None)
# Initialise task module
self.task = config.getPlugin(['module', 'task'],
cls=TaskModule,
tags=[self])
if abort == 'task':
return
utils.vprint('Current task ID: %s' % self.task.taskID, -1)
utils.vprint('Task started on %s' % self.task.taskDate, -1)
# Initialise workload management interface
self.wms = config.getCompositePlugin('backend',
'grid',
'MultiWMS',
cls=WMS,
tags=[self, self.task])
# Subsequent config calls also include section "jobs":
jobs_config = config.changeView(viewClass='TaggedConfigView',
addSections=['jobs'],
addTags=[self])
# Initialise monitoring module
self.monitor = jobs_config.getCompositePlugin('monitor',
'scripts',
'MultiMonitor',
cls=Monitoring,
tags=[self, self.task],
pargs=(self.task, ))
# Initialise job database
self.jobManager = jobs_config.getPlugin(
'job manager',
'SimpleJobManager',
cls=JobManager,
tags=[self, self.task, self.wms],
pargs=(self.task, self.monitor))
# Prepare work package
self.wms.deployTask(self.task, self.monitor)
# Configure workflow settings
self._actionList = jobs_config.getList('action',
['check', 'retrieve', 'submit'],
onChange=None)
self.duration = 0
if jobs_config.getBool('continuous', False,
onChange=None): # legacy option
self.duration = -1
self.duration = jobs_config.getTime('duration',
self.duration,
onChange=None)
self._submitFlag = jobs_config.getBool('submission',
True,
onChange=None)
self._submitTime = jobs_config.getTime('submission time requirement',
self.task.wallTime,
onChange=None)
# Initialise GUI
self._gui = jobs_config.getPlugin('gui',
'SimpleConsole',
cls=GUI,
onChange=None,
pargs=(self, ))
