def __init__(self, config, name, optDefault, optPrefix, varPrefix):
StorageManager.__init__(self, config, name, optDefault, optPrefix, varPrefix)
normSEPath = lambda x: utils.QM(x[0] == '/', 'dir:///%s' % x.lstrip('/'), x)
self.defPaths = config.getList('%s path' % optDefault, [], onValid = validNoVar(config), parseItem = normSEPath)
self.smPaths = config.getList('%s path' % optPrefix, self.defPaths, onValid = validNoVar(config), parseItem = normSEPath)
self.smFiles = config.getList('%s files' % optPrefix, [])
self.smPattern = config.get('%s pattern' % optPrefix, '@X@')
self.smTimeout = config.getTime('%s timeout' % optPrefix, 2*60*60)
self.smForce = config.getBool('%s force' % optPrefix, True)
def __init__(self, config, name, optDefault, optPrefix, varPrefix):
StorageManager.__init__(self, config, name, optDefault, optPrefix,
varPrefix)
normSEPath = lambda x: utils.QM(x[0] == '/', 'dir:///%s' % x.lstrip(
'/'), x)
self.defPaths = config.getList('%s path' % optDefault, [],
onValid=validNoVar(config),
parseItem=normSEPath)
self.smPaths = config.getList('%s path' % optPrefix,
self.defPaths,
onValid=validNoVar(config),
parseItem=normSEPath)
self.smFiles = config.getList('%s files' % optPrefix, [])
self.smPattern = config.get('%s pattern' % optPrefix, '@X@')
self.smTimeout = config.getTime('%s timeout' % optPrefix, 2 * 60 * 60)
self.smForce = config.getBool('%s force' % optPrefix, True)
def __init__(self, config, name):
NamedPlugin.__init__(self, config, name)
initSandbox = changeInitNeeded('sandbox')
self._varCheck = validNoVar(config)
# Task requirements
jobs_config = config.changeView(viewClass = 'TaggedConfigView', addSections = ['jobs'], addTags = [self]) # Move this into parameter manager?
self.wallTime = jobs_config.getTime('wall time', onChange = None)
self.cpuTime = jobs_config.getTime('cpu time', self.wallTime, onChange = None)
self.cpus = jobs_config.getInt('cpus', 1, onChange = None)
self.memory = jobs_config.getInt('memory', -1, onChange = None)
self.nodeTimeout = jobs_config.getTime('node timeout', -1, onChange = initSandbox)
# Compute / get task ID
self.taskID = config.get('task id', 'GC' + md5_hex(str(time()))[:12], persistent = True)
self.taskDate = config.get('task date', strftime('%Y-%m-%d'), persistent = True, onChange = initSandbox)
self.taskConfigName = config.getConfigName()
self._job_name_generator = config.getPlugin('job name generator', 'DefaultJobName',
cls = JobNamePlugin, pargs = (self,))
# Storage setup
storage_config = config.changeView(viewClass = 'TaggedConfigView',
setClasses = None, setNames = None, addSections = ['storage'], addTags = [self])
self.taskVariables = {
# Space limits
'SCRATCH_UL': storage_config.getInt('scratch space used', 5000, onChange = initSandbox),
'SCRATCH_LL': storage_config.getInt('scratch space left', 1, onChange = initSandbox),
'LANDINGZONE_UL': storage_config.getInt('landing zone space used', 100, onChange = initSandbox),
'LANDINGZONE_LL': storage_config.getInt('landing zone space left', 1, onChange = initSandbox),
}
storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@')
self.seMinSize = storage_config.getInt('se min size', -1, onChange = initSandbox)
self.sbInputFiles = config.getPaths('input files', [], onChange = initSandbox)
self.sbOutputFiles = config.getList('output files', [], onChange = initSandbox)
self.gzipOut = config.getBool('gzip output', True, onChange = initSandbox)
self._subst_files = config.getList('subst files', [], onChange = initSandbox)
self.dependencies = lmap(str.lower, config.getList('depends', [], onChange = initSandbox))
# Get error messages from gc-run.lib comments
self.errorDict = {}
self.updateErrorDict(utils.pathShare('gc-run.lib'))
# Init parameter source manager
psrc_repository = {}
self._setupJobParameters(config, psrc_repository)
self._pfactory = config.getPlugin('internal parameter factory', 'BasicParameterFactory',
cls = ParameterFactory, pargs = (psrc_repository,), tags = [self], inherit = True)
self.source = config.getPlugin('parameter adapter', 'TrackedParameterAdapter',
cls = ParameterAdapter, pargs = (self._pfactory.getSource(),))
def __init__(self, config, name):
NamedPlugin.__init__(self, config, name)
initSandbox = changeInitNeeded('sandbox')
self._varCheck = validNoVar(config)
# Task requirements
jobs_config = config.changeView(viewClass = 'TaggedConfigView', addSections = ['jobs'], addTags = [self]) # Move this into parameter manager?
self.wallTime = jobs_config.getTime('wall time', onChange = None)
self.cpuTime = jobs_config.getTime('cpu time', self.wallTime, onChange = None)
self.cpus = jobs_config.getInt('cpus', 1, onChange = None)
self.memory = jobs_config.getInt('memory', -1, onChange = None)
self.nodeTimeout = jobs_config.getTime('node timeout', -1, onChange = initSandbox)
# Compute / get task ID
self.taskID = config.get('task id', 'GC' + md5_hex(str(time()))[:12], persistent = True)
self.taskDate = config.get('task date', strftime('%Y-%m-%d'), persistent = True, onChange = initSandbox)
self.taskConfigName = config.getConfigName()
self._job_name_generator = config.getPlugin('job name generator', 'DefaultJobName',
cls = JobNamePlugin, pargs = (self,))
# Storage setup
storage_config = config.changeView(viewClass = 'TaggedConfigView',
setClasses = None, setNames = None, addSections = ['storage'], addTags = [self])
self.taskVariables = {
# Space limits
'SCRATCH_UL': storage_config.getInt('scratch space used', 5000, onChange = initSandbox),
'SCRATCH_LL': storage_config.getInt('scratch space left', 1, onChange = initSandbox),
'LANDINGZONE_UL': storage_config.getInt('landing zone space used', 100, onChange = initSandbox),
'LANDINGZONE_LL': storage_config.getInt('landing zone space left', 1, onChange = initSandbox),
}
storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@')
self.seMinSize = storage_config.getInt('se min size', -1, onChange = initSandbox)
self.sbInputFiles = config.getPaths('input files', [], onChange = initSandbox)
self.sbOutputFiles = config.getList('output files', [], onChange = initSandbox)
self.gzipOut = config.getBool('gzip output', True, onChange = initSandbox)
self.substFiles = config.getList('subst files', [], onChange = initSandbox)
self.dependencies = lmap(str.lower, config.getList('depends', [], onChange = initSandbox))
# Get error messages from gc-run.lib comments
self.errorDict = {}
self.updateErrorDict(utils.pathShare('gc-run.lib'))
# Init parameter source manager
self._setupJobParameters(config)
self._pfactory = config.getPlugin('internal parameter factory', 'BasicParameterFactory',
cls = ParameterFactory, tags = [self], inherit = True)
self.source = config.getPlugin('parameter adapter', 'TrackedParameterAdapter',
cls = ParameterAdapter, pargs = (self._pfactory.getSource(),))