def _get_sandbox_file_list(self, task, sm_list): # Prepare all input files dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list))) dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep, lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list) task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(), *imap(lambda x: x.get_task_dict(), [task] + sm_list)) task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list), 'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name}) task_config_str_list = DictFormat(escape_strings=True).format( task_config_dict, format='export %s%s%s\n') vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(), self._remote_event_handler.get_mon_env_dict().keys())) vn_alias_dict.update(task.get_var_alias_map()) vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n') # Resolve wildcards in task input files def _get_task_fn_list(): for fpi in task.get_sb_in_fpi_list(): matched = glob.glob(fpi.path_abs) if matched != []: for match in matched: yield match else: yield fpi.path_abs return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [ VirtualFile('_config.sh', sorted(task_config_str_list)), VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def _get_sandbox_file_list(self, task, sm_list): # Prepare all input files dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list))) dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep, lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list) task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(), *imap(lambda x: x.get_task_dict(), [task] + sm_list)) task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list), 'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name}) task_config_str_list = DictFormat(escape_strings=True).format( task_config_dict, format='export %s%s%s\n') vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(), self._remote_event_handler.get_mon_env_dict().keys())) vn_alias_dict.update(task.get_var_alias_map()) vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n') # Resolve wildcards in task input files def _get_task_fn_list(): for fpi in task.get_sb_in_fpi_list(): matched = glob.glob(fpi.path_abs) if matched != []: for match in matched: yield match else: yield fpi.path_abs return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [ VirtualFile('_config.sh', sorted(task_config_str_list)), VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def join_config_locations(opt_first, *opt_list): if isinstance(opt_first, (list, tuple)): # first option is a list - expand the first parameter if not opt_list: # only first option -> clean and return return lmap(str.strip, opt_first) return lchain(imap(lambda opt: join_config_locations(opt.strip(), *opt_list), opt_first)) if not opt_list: # only first option -> clean and return return [opt_first.strip()] return lmap(lambda opt: (opt_first + ' ' + opt).strip(), join_config_locations(*opt_list))
def _getUserSource(self, pExpr): tokens = tokenize(pExpr, lchain([self._precedence.keys(), list('()[]<>{}')])) tokens = list(tok2inlinetok(tokens, list(self._precedence.keys()))) self._log.debug('Parsing parameter string: "%s"', str.join(' ', imap(str, tokens))) tree = tok2tree(tokens, self._precedence) source = self._tree2expr(tree) for (PSourceClass, args) in self._nestedSources: source = PSourceClass.createInstance(PSourceClass.__name__, source, *args) return source
def join_config_locations(opt_first, *opt_list): if isinstance( opt_first, (list, tuple)): # first option is a list - expand the first parameter if not opt_list: # only first option -> clean and return return lmap(str.strip, opt_first) return lchain( imap(lambda opt: join_config_locations(opt.strip(), *opt_list), opt_first)) if not opt_list: # only first option -> clean and return return [opt_first.strip()] return lmap(lambda opt: (opt_first + ' ' + opt).strip(), join_config_locations(*opt_list))
def _getJobsOutput(self, ids): if not len(ids): raise StopIteration activity = Activity('retrieving %d job outputs' % len(ids)) for gcID, jobNum in ids: path = self._sandbox_helper.get_sandbox(gcID) if path is None: yield (jobNum, None) continue # Cleanup sandbox outFiles = lchain(imap(lambda pat: glob.glob(os.path.join(path, pat)), self.outputFiles)) utils.removeFiles(ifilter(lambda x: x not in outFiles, imap(lambda fn: os.path.join(path, fn), os.listdir(path)))) yield (jobNum, path) activity.finish()
def _getSandboxFiles(self, task, monitor, smList): # Prepare all input files depList = set( ichain(imap(lambda x: x.getDependencies(), [task] + smList))) depPaths = lmap(lambda pkg: utils.pathShare('', pkg=pkg), os.listdir(utils.pathPKG())) depFiles = lmap( lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList) taskEnv = utils.mergeDicts( imap(lambda x: x.getTaskConfig(), [monitor, task] + smList)) taskEnv.update({ 'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(), 'GC_WMS_NAME': self._name }) taskConfig = sorted( utils.DictFormat(escapeString=True).format( taskEnv, format='export %s%s%s\n')) varMappingDict = dict( izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys())) varMappingDict.update(task.getVarMapping()) varMapping = sorted( utils.DictFormat(delimeter=' ').format(varMappingDict, format='%s%s%s\n')) # Resolve wildcards in task input files def getTaskFiles(): for f in task.getSBInFiles(): matched = glob.glob(f.pathAbs) if matched != []: for match in matched: yield match else: yield f.pathAbs return lchain([ monitor.getFiles(), depFiles, getTaskFiles(), [ VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping) ] ])
def _join_config_locations(*opt_list): opt_first = opt_list[0] opt_list = opt_list[1:] if isinstance( opt_first, (list, tuple)): # first option is a list - expand the first parameter if not opt_list: # only first option -> clean and return return lmap(str.strip, opt_first) return lchain( imap(lambda opt: _join_config_locations(opt.strip(), *opt_list), opt_first)) if not opt_list: # only first option -> clean and return return [opt_first.strip()] def _do_join(opt): return (opt_first + ' ' + opt).strip() return lmap(_do_join, _join_config_locations(*opt_list))
def _get_jobs_output(self, gc_id_jobnum_list): if not len(gc_id_jobnum_list): raise StopIteration activity = Activity('retrieving %d job outputs' % len(gc_id_jobnum_list)) for gc_id, jobnum in gc_id_jobnum_list: path = self._sandbox_helper.get_sandbox(gc_id) if path is None: yield (jobnum, None) continue # Cleanup sandbox output_fn_list = lchain(imap(lambda pat: glob.glob(os.path.join(path, pat)), self._output_fn_list)) remove_files(ifilter(lambda x: x not in output_fn_list, imap(lambda fn: os.path.join(path, fn), os.listdir(path)))) yield (jobnum, path) activity.finish()
def _getSandboxFiles(self, task, monitor, smList): # Prepare all input files depList = set(ichain(imap(lambda x: x.getDependencies(), [task] + smList))) depPaths = lmap(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathPKG())) depFiles = lmap(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList) taskEnv = utils.mergeDicts(imap(lambda x: x.getTaskConfig(), [monitor, task] + smList)) taskEnv.update({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(), 'GC_WMS_NAME': self._name}) taskConfig = sorted(utils.DictFormat(escapeString = True).format(taskEnv, format = 'export %s%s%s\n')) varMappingDict = dict(izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys())) varMappingDict.update(task.getVarMapping()) varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n')) # Resolve wildcards in task input files def getTaskFiles(): for f in task.getSBInFiles(): matched = glob.glob(f.pathAbs) if matched != []: for match in matched: yield match else: yield f.pathAbs return lchain([monitor.getFiles(), depFiles, getTaskFiles(), [VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping)]])
def _get_jobs_output(self, gc_id_jobnum_list): if not len(gc_id_jobnum_list): raise StopIteration activity = Activity('retrieving %d job outputs' % len(gc_id_jobnum_list)) for gc_id, jobnum in gc_id_jobnum_list: path = self._sandbox_helper.get_sandbox(gc_id) if path is None: yield (jobnum, None) continue # Cleanup sandbox output_fn_list = lchain( imap(lambda pat: glob.glob(os.path.join(path, pat)), self._output_fn_list)) remove_files( ifilter( lambda x: x not in output_fn_list, imap(lambda fn: os.path.join(path, fn), os.listdir(path)))) yield (jobnum, path) activity.finish()
def get_used_psrc_list(self): return [self] + lchain(imap(lambda psrc: psrc.get_used_psrc_list(), self._psrc_list))
def _get_active_hash_input(prefix, guard_entry_idx): hash_input_list_user = scanner_config.get_list(join_config_locations(prefix, 'hash keys'), []) hash_input_list_guard = scanner_config.get_list(join_config_locations(prefix, 'guard override'), lchain(imap(lambda scanner: scanner.get_guard_keysets()[guard_entry_idx], self._scanner_list))) return hash_input_list_user + hash_input_list_guard
def get_partition_metadata(self): return lchain(imap(lambda p: p.get_partition_metadata() or [], self._processor_list))
def getKeys(self): return lchain(imap(lambda p: p.getKeys() or [], self._processorList))
def get_script(self): return lchain(imap(lambda h: h.get_script(), self._handlers))
def get_script(self): return lchain(imap(lambda h: h.get_script(), self._handlers))
def getNeededKeys(self, splitter): return lchain(imap(lambda p: p.getNeededKeys(splitter) or [], self._processorList))
def getActiveKeys(kUser, kGuard, gIdx): return kUser + (kGuard or lchain( imap(lambda x: x.getGuards()[gIdx], self._scanner)))
def getFiles(self): return lchain(lmap(lambda h: h.getFiles(), self._handlers) + [self.getScript()])
def get_partition_metadata(self): return lchain( imap(lambda p: p.get_partition_metadata() or [], self._processor_list))
def get_needed_vn_list(self, splitter): return lchain( imap(lambda p: p.get_needed_vn_list(splitter) or [], self._processor_list))
def getUsedSources(self): return [self] + lchain(imap(lambda ps: ps.getUsedSources(), self._psourceList))
def get_parameter_deps(self): return lchain(imap(lambda psrc: psrc.get_parameter_deps(), self._psrc_list))
def _getQueryArgs(self): """ClassAd names to query Condor with""" qqM = self._getQueueQueryMap() return lchain(qqM.values())
def get_used_psrc_list(self): return [self] + lchain( imap(lambda psrc: psrc.get_used_psrc_list(), self._psrc_list))
def get_file_list(self): return lchain(lmap(lambda h: h.get_file_list(), self._handlers) + [self.get_script()])
def getVarNames(self): # Take task variables and the variables from the parameter source return lchain([self.getTaskConfig().keys(), self.source.getJobKeys()])
def get_file_list(self): return lchain( lmap(lambda h: h.get_file_list(), self._handlers) + [self.get_script()])
def get_resync_request(self): return lchain( imap(lambda psrc: psrc.get_resync_request(), self._psrc_list))
def getActiveKeys(kUser, kGuard, gIdx): return kUser + (kGuard or lchain(imap(lambda x: x.getGuards()[gIdx], self._scanner)))
def getKeys(self): return lchain(imap(lambda p: p.getKeys() or [], self._processorList))
def get_needed_vn_list(self, splitter): return lchain(imap(lambda p: p.get_needed_vn_list(splitter) or [], self._processor_list))
def getNeededKeys(self, splitter): return lchain( imap(lambda p: p.getNeededKeys(splitter) or [], self._processorList))
def _getQueryArgs(self): """ClassAd names to query Condor with""" qqM = self._getQueueQueryMap() return lchain(qqM.values())
def get_parameter_deps(self): return lchain( imap(lambda psrc: psrc.get_parameter_deps(), self._psrc_list))
def getFiles(self): return lchain( lmap(lambda h: h.getFiles(), self._handlers) + [self.getScript()])
def get_resync_request(self): return lchain(imap(lambda psrc: psrc.get_resync_request(), self._psrc_list))