def _get_sandbox_file_list(self, task, sm_list):
# Prepare all input files
dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list)))
dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep,
lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list)
task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(),
*imap(lambda x: x.get_task_dict(), [task] + sm_list))
task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list),
'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name})
task_config_str_list = DictFormat(escape_strings=True).format(
task_config_dict, format='export %s%s%s\n')
vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(),
self._remote_event_handler.get_mon_env_dict().keys()))
vn_alias_dict.update(task.get_var_alias_map())
vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n')
# Resolve wildcards in task input files
def _get_task_fn_list():
for fpi in task.get_sb_in_fpi_list():
matched = glob.glob(fpi.path_abs)
if matched != []:
for match in matched:
yield match
else:
yield fpi.path_abs
return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [
VirtualFile('_config.sh', sorted(task_config_str_list)),
VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def _get_sandbox_file_list(self, task, sm_list):
# Prepare all input files
dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list)))
dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep,
lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list)
task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(),
*imap(lambda x: x.get_task_dict(), [task] + sm_list))
task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list),
'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name})
task_config_str_list = DictFormat(escape_strings=True).format(
task_config_dict, format='export %s%s%s\n')
vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(),
self._remote_event_handler.get_mon_env_dict().keys()))
vn_alias_dict.update(task.get_var_alias_map())
vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n')
# Resolve wildcards in task input files
def _get_task_fn_list():
for fpi in task.get_sb_in_fpi_list():
matched = glob.glob(fpi.path_abs)
if matched != []:
for match in matched:
yield match
else:
yield fpi.path_abs
return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [
VirtualFile('_config.sh', sorted(task_config_str_list)),
VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def join_config_locations(opt_first, *opt_list): if isinstance(opt_first, (list, tuple)): # first option is a list - expand the first parameter if not opt_list: # only first option -> clean and return return lmap(str.strip, opt_first) return lchain(imap(lambda opt: join_config_locations(opt.strip(), *opt_list), opt_first)) if not opt_list: # only first option -> clean and return return [opt_first.strip()] return lmap(lambda opt: (opt_first + ' ' + opt).strip(), join_config_locations(*opt_list))
def _getUserSource(self, pExpr):
tokens = tokenize(pExpr, lchain([self._precedence.keys(), list('()[]<>{}')]))
tokens = list(tok2inlinetok(tokens, list(self._precedence.keys())))
self._log.debug('Parsing parameter string: "%s"', str.join(' ', imap(str, tokens)))
tree = tok2tree(tokens, self._precedence)
source = self._tree2expr(tree)
for (PSourceClass, args) in self._nestedSources:
source = PSourceClass.createInstance(PSourceClass.__name__, source, *args)
return source
def join_config_locations(opt_first, *opt_list):
if isinstance(
opt_first,
(list, tuple)): # first option is a list - expand the first parameter
if not opt_list: # only first option -> clean and return
return lmap(str.strip, opt_first)
return lchain(
imap(lambda opt: join_config_locations(opt.strip(), *opt_list),
opt_first))
if not opt_list: # only first option -> clean and return
return [opt_first.strip()]
return lmap(lambda opt: (opt_first + ' ' + opt).strip(),
join_config_locations(*opt_list))
def _getJobsOutput(self, ids):
if not len(ids):
raise StopIteration
activity = Activity('retrieving %d job outputs' % len(ids))
for gcID, jobNum in ids:
path = self._sandbox_helper.get_sandbox(gcID)
if path is None:
yield (jobNum, None)
continue
# Cleanup sandbox
outFiles = lchain(imap(lambda pat: glob.glob(os.path.join(path, pat)), self.outputFiles))
utils.removeFiles(ifilter(lambda x: x not in outFiles, imap(lambda fn: os.path.join(path, fn), os.listdir(path))))
yield (jobNum, path)
activity.finish()
def _getSandboxFiles(self, task, monitor, smList):
# Prepare all input files
depList = set(
ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
depPaths = lmap(lambda pkg: utils.pathShare('', pkg=pkg),
os.listdir(utils.pathPKG()))
depFiles = lmap(
lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths),
depList)
taskEnv = utils.mergeDicts(
imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
taskEnv.update({
'GC_DEPFILES': str.join(' ', depList),
'GC_USERNAME': self._token.getUsername(),
'GC_WMS_NAME': self._name
})
taskConfig = sorted(
utils.DictFormat(escapeString=True).format(
taskEnv, format='export %s%s%s\n'))
varMappingDict = dict(
izip(monitor.getTaskConfig().keys(),
monitor.getTaskConfig().keys()))
varMappingDict.update(task.getVarMapping())
varMapping = sorted(
utils.DictFormat(delimeter=' ').format(varMappingDict,
format='%s%s%s\n'))
# Resolve wildcards in task input files
def getTaskFiles():
for f in task.getSBInFiles():
matched = glob.glob(f.pathAbs)
if matched != []:
for match in matched:
yield match
else:
yield f.pathAbs
return lchain([
monitor.getFiles(), depFiles,
getTaskFiles(),
[
VirtualFile('_config.sh', taskConfig),
VirtualFile('_varmap.dat', varMapping)
]
])
def _join_config_locations(*opt_list):
opt_first = opt_list[0]
opt_list = opt_list[1:]
if isinstance(
opt_first,
(list, tuple)): # first option is a list - expand the first parameter
if not opt_list: # only first option -> clean and return
return lmap(str.strip, opt_first)
return lchain(
imap(lambda opt: _join_config_locations(opt.strip(), *opt_list),
opt_first))
if not opt_list: # only first option -> clean and return
return [opt_first.strip()]
def _do_join(opt):
return (opt_first + ' ' + opt).strip()
return lmap(_do_join, _join_config_locations(*opt_list))
def _get_jobs_output(self, gc_id_jobnum_list):
if not len(gc_id_jobnum_list):
raise StopIteration
activity = Activity('retrieving %d job outputs' % len(gc_id_jobnum_list))
for gc_id, jobnum in gc_id_jobnum_list:
path = self._sandbox_helper.get_sandbox(gc_id)
if path is None:
yield (jobnum, None)
continue
# Cleanup sandbox
output_fn_list = lchain(imap(lambda pat: glob.glob(os.path.join(path, pat)),
self._output_fn_list))
remove_files(ifilter(lambda x: x not in output_fn_list,
imap(lambda fn: os.path.join(path, fn), os.listdir(path))))
yield (jobnum, path)
activity.finish()
def _getSandboxFiles(self, task, monitor, smList):
# Prepare all input files
depList = set(ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
depPaths = lmap(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathPKG()))
depFiles = lmap(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList)
taskEnv = utils.mergeDicts(imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
taskEnv.update({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(),
'GC_WMS_NAME': self._name})
taskConfig = sorted(utils.DictFormat(escapeString = True).format(taskEnv, format = 'export %s%s%s\n'))
varMappingDict = dict(izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys()))
varMappingDict.update(task.getVarMapping())
varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n'))
# Resolve wildcards in task input files
def getTaskFiles():
for f in task.getSBInFiles():
matched = glob.glob(f.pathAbs)
if matched != []:
for match in matched:
yield match
else:
yield f.pathAbs
return lchain([monitor.getFiles(), depFiles, getTaskFiles(),
[VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping)]])
def _get_jobs_output(self, gc_id_jobnum_list):
if not len(gc_id_jobnum_list):
raise StopIteration
activity = Activity('retrieving %d job outputs' %
len(gc_id_jobnum_list))
for gc_id, jobnum in gc_id_jobnum_list:
path = self._sandbox_helper.get_sandbox(gc_id)
if path is None:
yield (jobnum, None)
continue
# Cleanup sandbox
output_fn_list = lchain(
imap(lambda pat: glob.glob(os.path.join(path, pat)),
self._output_fn_list))
remove_files(
ifilter(
lambda x: x not in output_fn_list,
imap(lambda fn: os.path.join(path, fn), os.listdir(path))))
yield (jobnum, path)
activity.finish()
def get_used_psrc_list(self): return [self] + lchain(imap(lambda psrc: psrc.get_used_psrc_list(), self._psrc_list))
def _get_active_hash_input(prefix, guard_entry_idx): hash_input_list_user = scanner_config.get_list(join_config_locations(prefix, 'hash keys'), []) hash_input_list_guard = scanner_config.get_list(join_config_locations(prefix, 'guard override'), lchain(imap(lambda scanner: scanner.get_guard_keysets()[guard_entry_idx], self._scanner_list))) return hash_input_list_user + hash_input_list_guard
def get_partition_metadata(self): return lchain(imap(lambda p: p.get_partition_metadata() or [], self._processor_list))
def getKeys(self): return lchain(imap(lambda p: p.getKeys() or [], self._processorList))
def get_script(self):
return lchain(imap(lambda h: h.get_script(), self._handlers))
def get_script(self): return lchain(imap(lambda h: h.get_script(), self._handlers))
def getNeededKeys(self, splitter): return lchain(imap(lambda p: p.getNeededKeys(splitter) or [], self._processorList))
def getActiveKeys(kUser, kGuard, gIdx):
return kUser + (kGuard or lchain(
imap(lambda x: x.getGuards()[gIdx], self._scanner)))
def getFiles(self): return lchain(lmap(lambda h: h.getFiles(), self._handlers) + [self.getScript()])
def get_partition_metadata(self):
return lchain(
imap(lambda p: p.get_partition_metadata() or [],
self._processor_list))
def get_needed_vn_list(self, splitter):
return lchain(
imap(lambda p: p.get_needed_vn_list(splitter) or [],
self._processor_list))
def getUsedSources(self): return [self] + lchain(imap(lambda ps: ps.getUsedSources(), self._psourceList))
def get_parameter_deps(self): return lchain(imap(lambda psrc: psrc.get_parameter_deps(), self._psrc_list))
def _getQueryArgs(self): """ClassAd names to query Condor with""" qqM = self._getQueueQueryMap() return lchain(qqM.values())
def get_used_psrc_list(self):
return [self] + lchain(
imap(lambda psrc: psrc.get_used_psrc_list(), self._psrc_list))
def get_file_list(self): return lchain(lmap(lambda h: h.get_file_list(), self._handlers) + [self.get_script()])
def getVarNames(self): # Take task variables and the variables from the parameter source return lchain([self.getTaskConfig().keys(), self.source.getJobKeys()])
def get_file_list(self):
return lchain(
lmap(lambda h: h.get_file_list(), self._handlers) +
[self.get_script()])
def get_resync_request(self):
return lchain(
imap(lambda psrc: psrc.get_resync_request(), self._psrc_list))
def getActiveKeys(kUser, kGuard, gIdx): return kUser + (kGuard or lchain(imap(lambda x: x.getGuards()[gIdx], self._scanner)))
def getKeys(self):
return lchain(imap(lambda p: p.getKeys() or [], self._processorList))
def get_needed_vn_list(self, splitter): return lchain(imap(lambda p: p.get_needed_vn_list(splitter) or [], self._processor_list))
def getNeededKeys(self, splitter):
return lchain(
imap(lambda p: p.getNeededKeys(splitter) or [],
self._processorList))
def _getQueryArgs(self):
"""ClassAd names to query Condor with"""
qqM = self._getQueueQueryMap()
return lchain(qqM.values())
def get_parameter_deps(self):
return lchain(
imap(lambda psrc: psrc.get_parameter_deps(), self._psrc_list))
def getFiles(self):
return lchain(
lmap(lambda h: h.getFiles(), self._handlers) + [self.getScript()])
def get_resync_request(self): return lchain(imap(lambda psrc: psrc.get_resync_request(), self._psrc_list))
