def __init__(self, config, name):
NamedPlugin.__init__(self, config, name)
self._workDir = config.getWorkPath()
# Initialise task module
self.task = config.getPlugin(['task', 'module'], cls = TaskModule, tags = [self]).getInstance()
utils.vprint('Current task ID: %s' % self.task.taskID, -1)
utils.vprint('Task started on %s' % self.task.taskDate, -1)
# Initialise monitoring module
self.monitor = config.getCompositePlugin('monitor', 'scripts', 'MultiMonitor',
cls = Monitoring, tags = [self, self.task]).getInstance(self.task)
# Initialise workload management interface
self.wms = config.getCompositePlugin('backend', 'grid', 'MultiWMS',
cls = WMS, tags = [self, self.task]).getInstance()
# Initialise job database
jobManagerCls = config.getPlugin('job manager', 'SimpleJobManager',
cls = JobManager, tags = [self, self.task, self.wms])
self.jobManager = jobManagerCls.getInstance(self.task, self.monitor)
# Prepare work package
self.wms.deployTask(self.task, self.monitor)
configJobs = config.changeView(viewClass = TaggedConfigView, addSections = ['jobs'], addTags = [self])
self._actionList = configJobs.getList('action', ['check', 'retrieve', 'submit'], onChange = None)
self.runContinuous = configJobs.getBool('continuous', False, onChange = None)
self._checkSpace = config.getInt('workdir space', 10, onChange = None)
self._submitFlag = config.getBool('submission', True, onChange = None)
guiClass = config.getPlugin('gui', 'SimpleConsole', cls = GUI, onChange = None)
self._gui = guiClass.getInstance(config, self)
def __init__(self, config, name):
NamedPlugin.__init__(self, config, name)
initSandbox = changeInitNeeded('sandbox')
# Task requirements
configJobs = config.changeView(viewClass = TaggedConfigView, addSections = ['jobs'], addTags = [self]) # Move this into parameter manager?
self.wallTime = configJobs.getTime('wall time', onChange = None)
self.cpuTime = configJobs.getTime('cpu time', self.wallTime, onChange = None)
self.cpus = configJobs.getInt('cpus', 1, onChange = None)
self.memory = configJobs.getInt('memory', -1, onChange = None)
self.nodeTimeout = configJobs.getTime('node timeout', -1, onChange = initSandbox)
# Compute / get task ID
self.taskID = config.get('task id', 'GC' + md5(str(time())).hexdigest()[:12], persistent = True)
self.taskDate = config.get('task date', strftime('%Y-%m-%d'), persistent = True, onChange = initSandbox)
self.taskConfigName = config.getConfigName()
# Storage setup
configStorage = config.changeView(viewClass = TaggedConfigView,
setClasses = None, setNames = None, addSections = ['storage'], addTags = [self])
self.taskVariables = {
# Space limits
'SCRATCH_UL': configStorage.getInt('scratch space used', 5000, onChange = initSandbox),
'SCRATCH_LL': configStorage.getInt('scratch space left', 1, onChange = initSandbox),
'LANDINGZONE_UL': configStorage.getInt('landing zone space used', 100, onChange = initSandbox),
'LANDINGZONE_LL': configStorage.getInt('landing zone space left', 1, onChange = initSandbox),
}
configStorage.set('se output pattern', 'job_@GC_JOB_ID@_@X@')
self.seMinSize = configStorage.getInt('se min size', -1, onChange = initSandbox)
self.sbInputFiles = config.getPaths('input files', [], onChange = initSandbox)
self.sbOutputFiles = config.getList('output files', [], onChange = initSandbox)
self.gzipOut = config.getBool('gzip output', True, onChange = initSandbox)
self.substFiles = config.getList('subst files', [], onChange = initSandbox)
self.dependencies = map(str.lower, config.getList('depends', [], onChange = initSandbox))
# Get error messages from gc-run.lib comments
self.errorDict = dict(self.updateErrorDict(utils.pathShare('gc-run.lib')))
# Init parameter source manager
pm = config.getPlugin('parameter factory', 'SimpleParameterFactory',
cls = ParameterFactory, inherit = True).getInstance()
configParam = config.changeView(viewClass = TaggedConfigView, addSections = ['parameters'], addTags = [self])
self.setupJobParameters(configParam, pm)
self.source = pm.getSource(configParam)
def __init__(self, config, name, task, eventhandler):
NamedPlugin.__init__(self, config, name)
(self._task, self._eventhandler) = (task, eventhandler)
self.jobLimit = config.getInt('jobs', -1, onChange = None)
selected = JobSelector.create(config.get('selected', '', onChange = None), task = self._task)
jobDBClass = config.getPlugin('jobdb', 'JobDB', cls = JobDB)
self.jobDB = jobDBClass.getInstance(config, self.getMaxJobs(self._task), selected)
self.disableLog = config.getWorkPath('disabled')
outputProcessorClass = config.getPlugin('output processor', 'SandboxProcessor', cls = TaskOutputProcessor)
self._outputProcessor = outputProcessorClass.getInstance(task)
self.timeout = config.getTime('queue timeout', -1, onChange = None)
self.inFlight = config.getInt('in flight', -1, onChange = None)
self.inQueue = config.getInt('in queue', -1, onChange = None)
self.doShuffle = config.getBool('shuffle', False, onChange = None)
self.maxRetry = config.getInt('max retry', -1, onChange = None)
self.continuous = config.getBool('continuous', False, onChange = None)
self._reportClass = config.getPlugin('abort report', 'LocationReport', cls = Report, onChange = None)
def __init__(self, config, name, userOpt, itemName, discoverFun):
NamedPlugin.__init__(self, config, name)
(self._itemsStart, self._itemsDiscovered, self._itemName) = (None, False, itemName)
self._nEntries = config.getInt('%s entries' % userOpt, 0, onChange = None)
self._nRandom = config.getBool('%s randomize' % userOpt, False, onChange = None)
def __init__(self, config, name, task, submodules = []):
NamedPlugin.__init__(self, config, name)
self._log = logging.getLogger('monitoring')
(self.config, self.task, self.submodules) = (config, task, submodules)
def __init__(self, config, name):
NamedPlugin.__init__(self, config, name)
self.adapter = config.get('parameter adapter', 'TrackedParameterAdapter')
self.paramConfig = ParameterConfig(config.changeView(setSections = ['parameters']), self.adapter != 'TrackedParameterAdapter')
def __init__(self, config, wmsName):
wmsName = utils.QM(wmsName, wmsName, self.__class__.__name__).upper().replace('.', '_')
NamedPlugin.__init__(self, config, wmsName)
(self.config, self.wmsName) = (config, wmsName)
self._wait_idle = config.getInt('wait idle', 60, onChange = None)
self._wait_work = config.getInt('wait work', 10, onChange = None)