def __init__(self, config, name, optDefault, optPrefix, varPrefix): StorageManager.__init__(self, config, name, optDefault, optPrefix, varPrefix) normSEPath = lambda x: utils.QM(x[0] == '/', 'dir:///%s' % x.lstrip('/'), x) self.defPaths = config.getList('%s path' % optDefault, [], onValid = validNoVar(config), parseItem = normSEPath) self.smPaths = config.getList('%s path' % optPrefix, self.defPaths, onValid = validNoVar(config), parseItem = normSEPath) self.smFiles = config.getList('%s files' % optPrefix, []) self.smPattern = config.get('%s pattern' % optPrefix, '@X@') self.smTimeout = config.getTime('%s timeout' % optPrefix, 2*60*60) self.smForce = config.getBool('%s force' % optPrefix, True)
def __init__(self, config, name, optDefault, optPrefix, varPrefix): StorageManager.__init__(self, config, name, optDefault, optPrefix, varPrefix) normSEPath = lambda x: utils.QM(x[0] == '/', 'dir:///%s' % x.lstrip( '/'), x) self.defPaths = config.getList('%s path' % optDefault, [], onValid=validNoVar(config), parseItem=normSEPath) self.smPaths = config.getList('%s path' % optPrefix, self.defPaths, onValid=validNoVar(config), parseItem=normSEPath) self.smFiles = config.getList('%s files' % optPrefix, []) self.smPattern = config.get('%s pattern' % optPrefix, '@X@') self.smTimeout = config.getTime('%s timeout' % optPrefix, 2 * 60 * 60) self.smForce = config.getBool('%s force' % optPrefix, True)
def __init__(self, config, name): NamedPlugin.__init__(self, config, name) initSandbox = changeInitNeeded('sandbox') self._varCheck = validNoVar(config) # Task requirements jobs_config = config.changeView(viewClass = 'TaggedConfigView', addSections = ['jobs'], addTags = [self]) # Move this into parameter manager? self.wallTime = jobs_config.getTime('wall time', onChange = None) self.cpuTime = jobs_config.getTime('cpu time', self.wallTime, onChange = None) self.cpus = jobs_config.getInt('cpus', 1, onChange = None) self.memory = jobs_config.getInt('memory', -1, onChange = None) self.nodeTimeout = jobs_config.getTime('node timeout', -1, onChange = initSandbox) # Compute / get task ID self.taskID = config.get('task id', 'GC' + md5_hex(str(time()))[:12], persistent = True) self.taskDate = config.get('task date', strftime('%Y-%m-%d'), persistent = True, onChange = initSandbox) self.taskConfigName = config.getConfigName() self._job_name_generator = config.getPlugin('job name generator', 'DefaultJobName', cls = JobNamePlugin, pargs = (self,)) # Storage setup storage_config = config.changeView(viewClass = 'TaggedConfigView', setClasses = None, setNames = None, addSections = ['storage'], addTags = [self]) self.taskVariables = { # Space limits 'SCRATCH_UL': storage_config.getInt('scratch space used', 5000, onChange = initSandbox), 'SCRATCH_LL': storage_config.getInt('scratch space left', 1, onChange = initSandbox), 'LANDINGZONE_UL': storage_config.getInt('landing zone space used', 100, onChange = initSandbox), 'LANDINGZONE_LL': storage_config.getInt('landing zone space left', 1, onChange = initSandbox), } storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@') self.seMinSize = storage_config.getInt('se min size', -1, onChange = initSandbox) self.sbInputFiles = config.getPaths('input files', [], onChange = initSandbox) self.sbOutputFiles = config.getList('output files', [], onChange = initSandbox) self.gzipOut = config.getBool('gzip output', True, onChange = initSandbox) self._subst_files = config.getList('subst files', [], onChange = initSandbox) self.dependencies = lmap(str.lower, config.getList('depends', [], onChange = initSandbox)) # Get error messages from gc-run.lib comments self.errorDict = {} self.updateErrorDict(utils.pathShare('gc-run.lib')) # Init parameter source manager psrc_repository = {} self._setupJobParameters(config, psrc_repository) self._pfactory = config.getPlugin('internal parameter factory', 'BasicParameterFactory', cls = ParameterFactory, pargs = (psrc_repository,), tags = [self], inherit = True) self.source = config.getPlugin('parameter adapter', 'TrackedParameterAdapter', cls = ParameterAdapter, pargs = (self._pfactory.getSource(),))
def __init__(self, config, name): NamedPlugin.__init__(self, config, name) initSandbox = changeInitNeeded('sandbox') self._varCheck = validNoVar(config) # Task requirements jobs_config = config.changeView(viewClass = 'TaggedConfigView', addSections = ['jobs'], addTags = [self]) # Move this into parameter manager? self.wallTime = jobs_config.getTime('wall time', onChange = None) self.cpuTime = jobs_config.getTime('cpu time', self.wallTime, onChange = None) self.cpus = jobs_config.getInt('cpus', 1, onChange = None) self.memory = jobs_config.getInt('memory', -1, onChange = None) self.nodeTimeout = jobs_config.getTime('node timeout', -1, onChange = initSandbox) # Compute / get task ID self.taskID = config.get('task id', 'GC' + md5_hex(str(time()))[:12], persistent = True) self.taskDate = config.get('task date', strftime('%Y-%m-%d'), persistent = True, onChange = initSandbox) self.taskConfigName = config.getConfigName() self._job_name_generator = config.getPlugin('job name generator', 'DefaultJobName', cls = JobNamePlugin, pargs = (self,)) # Storage setup storage_config = config.changeView(viewClass = 'TaggedConfigView', setClasses = None, setNames = None, addSections = ['storage'], addTags = [self]) self.taskVariables = { # Space limits 'SCRATCH_UL': storage_config.getInt('scratch space used', 5000, onChange = initSandbox), 'SCRATCH_LL': storage_config.getInt('scratch space left', 1, onChange = initSandbox), 'LANDINGZONE_UL': storage_config.getInt('landing zone space used', 100, onChange = initSandbox), 'LANDINGZONE_LL': storage_config.getInt('landing zone space left', 1, onChange = initSandbox), } storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@') self.seMinSize = storage_config.getInt('se min size', -1, onChange = initSandbox) self.sbInputFiles = config.getPaths('input files', [], onChange = initSandbox) self.sbOutputFiles = config.getList('output files', [], onChange = initSandbox) self.gzipOut = config.getBool('gzip output', True, onChange = initSandbox) self.substFiles = config.getList('subst files', [], onChange = initSandbox) self.dependencies = lmap(str.lower, config.getList('depends', [], onChange = initSandbox)) # Get error messages from gc-run.lib comments self.errorDict = {} self.updateErrorDict(utils.pathShare('gc-run.lib')) # Init parameter source manager self._setupJobParameters(config) self._pfactory = config.getPlugin('internal parameter factory', 'BasicParameterFactory', cls = ParameterFactory, tags = [self], inherit = True) self.source = config.getPlugin('parameter adapter', 'TrackedParameterAdapter', cls = ParameterAdapter, pargs = (self._pfactory.getSource(),))