def tree2expr(self, node): if isinstance(node, tuple): (operator, args) = node if operator == 'lookup': assert(len(args) == 2) return self._createVarSource(tree2names(args[0]), tree2names(args[1])) elif operator == 'ref': assert(len(args) == 1) refTypeDefault = 'dataset' if args[0] not in DataParameterSource.datasetsAvailable: refTypeDefault = 'csv' refType = self._paramConfig.get(args[0], 'type', refTypeDefault) if refType == 'dataset': return [DataParameterSource.create(self._paramConfig, args[0])] elif refType == 'csv': return [CSVParameterSource.create(self._paramConfig, args[0])] raise APIError('Unknown reference type: "%s"' % refType) else: args_complete = lchain(imap(self.tree2expr, args)) if operator == '*': return self.combineSources(CrossParameterSource, args_complete) elif operator == '+': return self.combineSources(ChainParameterSource, args_complete) elif operator == ',': return self.combineSources(ZipLongParameterSource, args_complete) raise APIError('Unknown token: "%s"' % operator) elif isinstance(node, int): return [node] else: return self._createVarSource([node], None)
def _getUserSource(self, pExpr): tokens = tokenize(pExpr, lchain([self._precedence.keys(), list('()[]<>{}')])) tokens = list(tok2inlinetok(tokens, list(self._precedence.keys()))) self._log.debug('Parsing parameter string: "%s"', str.join(' ', imap(str, tokens))) tree = tok2tree(tokens, self._precedence) source = self._tree2expr(tree) for (PSourceClass, args) in self._nestedSources: source = PSourceClass(source, *args) return source
def _getJobsOutput(self, ids): if not len(ids): raise StopIteration activity = utils.ActivityLog('retrieving job outputs') for wmsId, jobNum in ids: path = self._getSandbox(wmsId) if path is None: yield (jobNum, None) continue # Cleanup sandbox outFiles = lchain(imap(lambda pat: glob.glob(os.path.join(path, pat)), self.outputFiles)) utils.removeFiles(ifilter(lambda x: x not in outFiles, imap(lambda fn: os.path.join(path, fn), os.listdir(path)))) yield (jobNum, path) del activity
def _getSandboxFiles(self, task, monitor, smList): # Prepare all input files depList = set( ichain(imap(lambda x: x.getDependencies(), [task] + smList))) depPaths = lmap(lambda pkg: utils.pathShare('', pkg=pkg), os.listdir(utils.pathPKG())) depFiles = lmap( lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList) taskEnv = utils.mergeDicts( imap(lambda x: x.getTaskConfig(), [monitor, task] + smList)) taskEnv.update({ 'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(), 'GC_WMS_NAME': self.wmsName }) taskConfig = sorted( utils.DictFormat(escapeString=True).format( taskEnv, format='export %s%s%s\n')) varMappingDict = dict( izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys())) varMappingDict.update(task.getVarMapping()) varMapping = sorted( utils.DictFormat(delimeter=' ').format(varMappingDict, format='%s%s%s\n')) # Resolve wildcards in task input files def getTaskFiles(): for f in task.getSBInFiles(): matched = glob.glob(f.pathAbs) if matched != []: for match in matched: yield match else: yield f.pathAbs return lchain([ monitor.getFiles(), depFiles, getTaskFiles(), [ VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping) ] ])
def _getUserSource(self, pExpr, parent): tokens = tokenize(pExpr, lchain([self.precedence.keys(), list('()[]<>')])) tokens = list(tok2inlinetok(tokens, list(self.precedence.keys()))) utils.vprint('Parsing parameter string: "%s"' % str.join(' ', imap(str, tokens)), 0) tree = tok2tree(tokens, self.precedence) source_list = self.tree2expr(tree) if DataParameterSource.datasetsAvailable and not DataParameterSource.datasetsUsed: source_list.insert(0, DataParameterSource.create()) if parent: source_list.append(parent) if len(lfilter(lambda p: p.getMaxParameters() is not None, source_list)) > 1: source = self.combineSources(CrossParameterSource, source_list) else: source = self.combineSources(ZipLongParameterSource, source_list) # zip more efficient assert(len(source) == 1) source = source[0] for (PSourceClass, args) in self.elevatedSwitch: source = PSourceClass(source, *args) utils.vprint('Parsing output: %r' % source, 0) return source
def _getSandboxFiles(self, task, monitor, smList): # Prepare all input files depList = set(ichain(imap(lambda x: x.getDependencies(), [task] + smList))) depPaths = lmap(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathPKG())) depFiles = lmap(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList) taskEnv = utils.mergeDicts(imap(lambda x: x.getTaskConfig(), [monitor, task] + smList)) taskEnv.update({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(), 'GC_WMS_NAME': self.wmsName}) taskConfig = sorted(utils.DictFormat(escapeString = True).format(taskEnv, format = 'export %s%s%s\n')) varMappingDict = dict(izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys())) varMappingDict.update(task.getVarMapping()) varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n')) # Resolve wildcards in task input files def getTaskFiles(): for f in task.getSBInFiles(): matched = glob.glob(f.pathAbs) if matched != []: for match in matched: yield match else: yield f.pathAbs return lchain([monitor.getFiles(), depFiles, getTaskFiles(), [VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping)]])
def getVarNames(self): # Take task variables and the variables from the parameter source return lchain([self.getTaskConfig().keys(), self.source.getJobKeys()])
def getFiles(self): return lchain(lmap(lambda h: h.getFiles(), self._handlers) + [self.getScript()])
def getScript(self): return lchain(imap(lambda h: h.getScript(), self._handlers))
def getVarNames(self): # Take task variables and the variables from the parameter source return lchain([self.getTaskConfig().keys(), self.source.getJobKeys()])
def getNeededKeys(self, splitter): return lchain( imap(lambda p: p.getNeededKeys(splitter), self._processorList))
def getKeys(self): return lchain(imap(lambda p: p.getKeys(), self._processorList))
def getActiveKeys(kUser, kGuard, gIdx): return kUser + (kGuard or lchain(imap(lambda x: x.getGuards()[gIdx], self.scanner)))
def getFiles(self): return lchain( lmap(lambda h: h.getFiles(), self._handlers) + [self.getScript()])
def getScript(self): return lchain(imap(lambda h: h.getScript(), self._handlers))
def getNeededKeys(self, splitter): return lchain(imap(lambda p: p.getNeededKeys(splitter), self._processorList))
def getKeys(self): return lchain(imap(lambda p: p.getKeys(), self._processorList))