def tree2expr(self, node):
if isinstance(node, tuple):
(operator, args) = node
if operator == 'lookup':
assert(len(args) == 2)
return self._createVarSource(tree2names(args[0]), tree2names(args[1]))
elif operator == 'ref':
assert(len(args) == 1)
refTypeDefault = 'dataset'
if args[0] not in DataParameterSource.datasetsAvailable:
refTypeDefault = 'csv'
refType = self._paramConfig.get(args[0], 'type', refTypeDefault)
if refType == 'dataset':
return [DataParameterSource.create(self._paramConfig, args[0])]
elif refType == 'csv':
return [CSVParameterSource.create(self._paramConfig, args[0])]
raise APIError('Unknown reference type: "%s"' % refType)
else:
args_complete = lchain(imap(self.tree2expr, args))
if operator == '*':
return self.combineSources(CrossParameterSource, args_complete)
elif operator == '+':
return self.combineSources(ChainParameterSource, args_complete)
elif operator == ',':
return self.combineSources(ZipLongParameterSource, args_complete)
raise APIError('Unknown token: "%s"' % operator)
elif isinstance(node, int):
return [node]
else:
return self._createVarSource([node], None)
def _getUserSource(self, pExpr):
tokens = tokenize(pExpr, lchain([self._precedence.keys(), list('()[]<>{}')]))
tokens = list(tok2inlinetok(tokens, list(self._precedence.keys())))
self._log.debug('Parsing parameter string: "%s"', str.join(' ', imap(str, tokens)))
tree = tok2tree(tokens, self._precedence)
source = self._tree2expr(tree)
for (PSourceClass, args) in self._nestedSources:
source = PSourceClass(source, *args)
return source
def _getJobsOutput(self, ids):
if not len(ids):
raise StopIteration
activity = utils.ActivityLog('retrieving job outputs')
for wmsId, jobNum in ids:
path = self._getSandbox(wmsId)
if path is None:
yield (jobNum, None)
continue
# Cleanup sandbox
outFiles = lchain(imap(lambda pat: glob.glob(os.path.join(path, pat)), self.outputFiles))
utils.removeFiles(ifilter(lambda x: x not in outFiles, imap(lambda fn: os.path.join(path, fn), os.listdir(path))))
yield (jobNum, path)
del activity
def _getSandboxFiles(self, task, monitor, smList):
# Prepare all input files
depList = set(
ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
depPaths = lmap(lambda pkg: utils.pathShare('', pkg=pkg),
os.listdir(utils.pathPKG()))
depFiles = lmap(
lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths),
depList)
taskEnv = utils.mergeDicts(
imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
taskEnv.update({
'GC_DEPFILES': str.join(' ', depList),
'GC_USERNAME': self._token.getUsername(),
'GC_WMS_NAME': self.wmsName
})
taskConfig = sorted(
utils.DictFormat(escapeString=True).format(
taskEnv, format='export %s%s%s\n'))
varMappingDict = dict(
izip(monitor.getTaskConfig().keys(),
monitor.getTaskConfig().keys()))
varMappingDict.update(task.getVarMapping())
varMapping = sorted(
utils.DictFormat(delimeter=' ').format(varMappingDict,
format='%s%s%s\n'))
# Resolve wildcards in task input files
def getTaskFiles():
for f in task.getSBInFiles():
matched = glob.glob(f.pathAbs)
if matched != []:
for match in matched:
yield match
else:
yield f.pathAbs
return lchain([
monitor.getFiles(), depFiles,
getTaskFiles(),
[
VirtualFile('_config.sh', taskConfig),
VirtualFile('_varmap.dat', varMapping)
]
])
def _getUserSource(self, pExpr, parent):
tokens = tokenize(pExpr, lchain([self.precedence.keys(), list('()[]<>')]))
tokens = list(tok2inlinetok(tokens, list(self.precedence.keys())))
utils.vprint('Parsing parameter string: "%s"' % str.join(' ', imap(str, tokens)), 0)
tree = tok2tree(tokens, self.precedence)
source_list = self.tree2expr(tree)
if DataParameterSource.datasetsAvailable and not DataParameterSource.datasetsUsed:
source_list.insert(0, DataParameterSource.create())
if parent:
source_list.append(parent)
if len(lfilter(lambda p: p.getMaxParameters() is not None, source_list)) > 1:
source = self.combineSources(CrossParameterSource, source_list)
else:
source = self.combineSources(ZipLongParameterSource, source_list) # zip more efficient
assert(len(source) == 1)
source = source[0]
for (PSourceClass, args) in self.elevatedSwitch:
source = PSourceClass(source, *args)
utils.vprint('Parsing output: %r' % source, 0)
return source
def _getSandboxFiles(self, task, monitor, smList):
# Prepare all input files
depList = set(ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
depPaths = lmap(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathPKG()))
depFiles = lmap(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList)
taskEnv = utils.mergeDicts(imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
taskEnv.update({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(),
'GC_WMS_NAME': self.wmsName})
taskConfig = sorted(utils.DictFormat(escapeString = True).format(taskEnv, format = 'export %s%s%s\n'))
varMappingDict = dict(izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys()))
varMappingDict.update(task.getVarMapping())
varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n'))
# Resolve wildcards in task input files
def getTaskFiles():
for f in task.getSBInFiles():
matched = glob.glob(f.pathAbs)
if matched != []:
for match in matched:
yield match
else:
yield f.pathAbs
return lchain([monitor.getFiles(), depFiles, getTaskFiles(),
[VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping)]])
def getVarNames(self):
# Take task variables and the variables from the parameter source
return lchain([self.getTaskConfig().keys(), self.source.getJobKeys()])
def getFiles(self): return lchain(lmap(lambda h: h.getFiles(), self._handlers) + [self.getScript()])
def getScript(self): return lchain(imap(lambda h: h.getScript(), self._handlers))
def getVarNames(self): # Take task variables and the variables from the parameter source return lchain([self.getTaskConfig().keys(), self.source.getJobKeys()])
def getNeededKeys(self, splitter):
return lchain(
imap(lambda p: p.getNeededKeys(splitter), self._processorList))
def getKeys(self):
return lchain(imap(lambda p: p.getKeys(), self._processorList))
def getActiveKeys(kUser, kGuard, gIdx): return kUser + (kGuard or lchain(imap(lambda x: x.getGuards()[gIdx], self.scanner)))
def getFiles(self):
return lchain(
lmap(lambda h: h.getFiles(), self._handlers) + [self.getScript()])
def getScript(self):
return lchain(imap(lambda h: h.getScript(), self._handlers))
def getNeededKeys(self, splitter): return lchain(imap(lambda p: p.getNeededKeys(splitter), self._processorList))
def getKeys(self): return lchain(imap(lambda p: p.getKeys(), self._processorList))
