def run(self, log, info): used_engines = [] log.debug("All available engines: %s", info['ENGINES']) for engine in info['ENGINES'].split(" "): key = 'RUN' + engine.upper() if key in info and info[key] == 'True': used_engines.append(engine) log.debug("Effectively used engines: %s" % used_engines) if not isinstance(info[Keys.DATASET_CODE],list): info[Keys.DATASET_CODE] = [info[Keys.DATASET_CODE]] runs = len(info[Keys.DATASET_CODE]) log.debug("Number of samples: %d" % runs) for i in range(runs): collectedconfig = {} for engine in used_engines: path = "%s.ini_%d" % (engine, i) if not os.path.exists(path): raise RuntimeError("Required infofile not found " + path) else: log.debug("Found infofile "+path) engineconfig = infohandler.get_handler(path).read(path) collectedconfig = dicts.merge(collectedconfig, engineconfig, priority='append') for key in collectedconfig.keys(): if isinstance(collectedconfig[key], list): collectedconfig[key] = dicts.unify(collectedconfig[key]) collector_path = "%s_%d" % (info[Keys.MERGED], i) infohandler.get_handler(info[Keys.MERGED]).write(collectedconfig, collector_path) log.debug('Wrote outfile ' + collector_path) return info
def run(self, info): paths = sorted(glob.glob(info[Keys.MERGE] + "_*")) #read in config_container = {} nofiles = len(paths) if nofiles == 0: raise RuntimeError("No files to merge found!") for path in paths: logging.debug("Reading %s", path) config = infohandler.get_handler(path).read(path) lastjob = config[Keys.SUBJOBLIST][-1] checksum = int(lastjob.split(Keys.SUBJOBSEP)[2]) if nofiles != checksum: raise RuntimeError( "Number of inputfiles %d and checksum %d do not match" % (nofiles, checksum)) #append the current config to the ones with same parent subjobs parentjoblist = config[Keys.SUBJOBLIST][:-1] parentjobstr = self.parentjobs_to_str(parentjoblist) #remove one level from subjoblist config[Keys.SUBJOBLIST] = parentjoblist if not config[Keys.SUBJOBLIST]: del config[Keys.SUBJOBLIST] if parentjobstr in config_container: config_container[parentjobstr] = dicts.merge(config_container[parentjobstr], \ config, priority='append') else: config_container[parentjobstr] = config #unify (only possible after all collected) for config in config_container.values(): for key in config.keys(): if key == Keys.SUBJOBLIST: config[key] = dicts.unify(config[key], unlist_single=False) continue if isinstance(config[key], list): config[key] = dicts.unify(config[key]) #write back for i, config in enumerate(config_container.values()): path = info[Keys.MERGED] + '_' + str(i) logging.debug("Writing out %s", path) infohandler.get_handler(path).write(config, path) return info
def run(self, log, info): paths = sorted(glob.glob(info[Keys.MERGE] + "_*")) #read in config_container = {} nofiles = len(paths) if nofiles == 0: raise RuntimeError("No files to merge found!") for path in paths: log.debug("Reading " + path) config = infohandler.get_handler(path).read(path) lastjob = config[Keys.SUBJOBLIST][-1] checksum = int(lastjob.split(Keys.SUBJOBSEP)[2]) if nofiles != checksum: raise RuntimeError("Number of inputfiles %d and checksum %d do not match" % (nofiles, checksum)) #append the current config to the ones with same parent subjobs parentjoblist = config[Keys.SUBJOBLIST][:-1] parentjobstr = self.parentjobs_to_str(parentjoblist) #remove one level from subjoblist config[Keys.SUBJOBLIST] = parentjoblist if not config[Keys.SUBJOBLIST]: del config[Keys.SUBJOBLIST] if parentjobstr in config_container: config_container[parentjobstr] = dicts.merge(config_container[parentjobstr], config, priority='append') else: config_container[parentjobstr] = config #unify (only possible after all collected) for config in config_container.values(): for key in config.keys(): if key == Keys.SUBJOBLIST: config[key] = dicts.unify(config[key], unlist_single=False) continue if isinstance(config[key], list): config[key] = dicts.unify(config[key]) #write back for i, config in enumerate(config_container.values()): path = info[Keys.MERGED] + '_' + str(i) log.debug("Writing out " + path) infohandler.get_handler(path).write(config, path) return info
def run(self, info): infoh = get_handler(info[Keys.BRANCH]) tobranch = info[Keys.BRANCH].split(" ") del info[Keys.BRANCH] for branch in tobranch: logging.info("Branching %s", branch) info = info.copy() infoh.write(info, branch) return info
def run(self, log, info): ih = get_handler(info[Keys.BRANCH]) tobranch = info[Keys.BRANCH].split(" ") del info[Keys.BRANCH] for branch in tobranch: log.info("Branching " + branch) info = info.copy() ih.write(info, branch) return info
def setup(self, app_args): # basic arguments for every node basic_args = [Argument(Keys.INPUT, KeyHelp.INPUT, default=''), Argument(Keys.OUTPUT, KeyHelp.OUTPUT, default=''), Argument(Keys.MODULE, KeyHelp.MODULE, default=''), Argument(Keys.LOG_LEVEL, KeyHelp.LOG_LEVEL, default="DEBUG")] # Fixme: Prettify WORKDIR creation system # WORKDIR: if WORKDIR is defined add related args for i, arg in enumerate(app_args): if arg.name == Keys.WORKDIR: app_args.insert(i + 1, Argument(Keys.BASEDIR, KeyHelp.BASEDIR, default='.')) app_args.insert(i + 2, Argument(Keys.JOB_ID, KeyHelp.JOB_ID, default='')) app_args.insert(i + 3, Argument(Keys.SUBJOBLIST, KeyHelp.SUBJOBLIST, default='')) app_args.insert(i + 4, Argument(Keys.NAME, KeyHelp.NAME, default=self.__class__.__name__)) break defaults, cliargs = parse_sysargs(basic_args + app_args) # construct info from defaults < info < commandlineargs ih = get_handler(cliargs.get(Keys.INPUT, None)) fileinfo = ih.read(cliargs.get(Keys.INPUT, None)) info = dicts.merge(cliargs, dicts.merge(fileinfo, defaults)) # setup logging log = Logger.create(info[Keys.LOG_LEVEL]) # request by malars: show dataset prominent in logger if Keys.DATASET_CODE in info: if not isinstance(info[Keys.DATASET_CODE], list): if Keys.MZXML in info and not isinstance(info[Keys.MZXML], list): log.info("Dataset is %s (%s)" % (info[Keys.DATASET_CODE], os.path.basename(info[Keys.MZXML]))) else: log.info("Dataset is %s" % info[Keys.DATASET_CODE]) else: log.debug("Datasets are %s" % info[Keys.DATASET_CODE]) # WORKDIR: create WORKDIR (only after mk log) info = dirs.create_workdir(log, info) # filter to requested args if Keys.ALL_ARGS in info: # if ALL_ARGS is set give whole info to app... req_info = info else: req_info = {} # ...otherwise copy only explicitly requested args to app for key in [arg.name for arg in basic_args + app_args]: if key in info: req_info[key] = info[key] log.debug("info for app: %s" % req_info) return log, req_info, info
def test1_arg_priority(self): """Test the argument priorities. Disabled.""" sys.argv = ['--INPUT', 'fcuk.ini', '--OUTPUT', 'kufc.ini'] # basic arguments for every node basic_args = [ Argument(Keys.INPUT, KeyHelp.INPUT, default=''), Argument(Keys.OUTPUT, KeyHelp.OUTPUT, default=''), Argument(Keys.MODULE, KeyHelp.MODULE, default=''), Argument(Keys.LOG_LEVEL, KeyHelp.LOG_LEVEL, default="DEBUG") ] # WORKDIR: if WORKDIR is defined add related args app_args = [] for i, arg in enumerate(app_args): if arg.name == Keys.WORKDIR: app_args.insert( i + 1, Argument(Keys.BASEDIR, KeyHelp.BASEDIR, default='.')) app_args.insert( i + 2, Argument(Keys.JOB_ID, KeyHelp.JOB_ID, default='')) app_args.insert( i + 3, Argument(Keys.SUBJOBLIST, KeyHelp.SUBJOBLIST, default='')) app_args.insert( i + 4, Argument(Keys.NAME, KeyHelp.NAME, default=self.__class__.__name__)) break blub = basic_args + app_args defaults, cliargs = parse_sysargs(blub) # construct info from defaults < info < commandlineargs inifile = cliargs.get(Keys.INPUT, None) info_fh = get_handler(inifile) fileinfo = info_fh.read(inifile) info = dicts.merge(cliargs, dicts.merge(fileinfo, defaults)) # request by malars: show dataset prominent in logger if Keys.DATASET_CODE in info: if not isinstance(info[Keys.DATASET_CODE], list): if Keys.MZXML in info and not isinstance( info[Keys.MZXML], list): logging.info("Dataset is %s (%s)", info[Keys.DATASET_CODE], os.path.basename(info[Keys.MZXML])) else: logging.info("Dataset is %s", info[Keys.DATASET_CODE]) else: logging.debug("Datasets are %s", info[Keys.DATASET_CODE])
def run(self, log, info): basename = info[Keys.SPLIT] key = info[Keys.SPLIT_KEY] value = info.get(key,"") if not isinstance(value, list): value = [value] info = info.copy() del info[Keys.SPLIT] del info[Keys.SPLIT_KEY] if info.get(Keys.SUBJOBLIST, "") == "": info[Keys.SUBJOBLIST] = [] for i, val in enumerate(value): infocopy = copy.deepcopy(info) infocopy[key] = val infocopy[Keys.SUBJOBLIST].append("%s%s%d%s%d" % (key, Keys.SUBJOBSEP, i, Keys.SUBJOBSEP, len(value))) path = basename + "_" + str(i) log.debug("Writing split file " + path) get_handler(basename).write(infocopy, path) return info
def run(self, info): basename = info[Keys.SPLIT] key = info[Keys.SPLIT_KEY] value = info.get(key, "") if not isinstance(value, list): value = [value] info = info.copy() del info[Keys.SPLIT] del info[Keys.SPLIT_KEY] if info.get(Keys.SUBJOBLIST, "") == "": info[Keys.SUBJOBLIST] = [] for i, val in enumerate(value): infocopy = copy.deepcopy(info) infocopy[key] = val infocopy[Keys.SUBJOBLIST].append("%s%s%d%s%d" % (key, Keys.SUBJOBSEP, i, \ Keys.SUBJOBSEP, len(value))) path = basename + "_" + str(i) logging.debug("Writing split file %s", path) get_handler(basename).write(infocopy, path) return info
def run(self, log, info): ih = get_handler(info[Keys.COLLATE]) paths = info[Keys.COLLATE].split(" ") del info[Keys.COLLATE] collector_config = info.copy() #read in for path in paths: log.debug('collating file [%s]' % path) config = ih.read(path) collector_config = dicts.merge(collector_config, config, priority='append') #unify for key in collector_config.keys(): collector_config[key] = dicts.unify(collector_config[key]) #write back return collector_config
def test1_arg_priority(self): return sys.argv = [ '--INPUT', 'fcuk.ini', '--OUTPUT', 'kufc.ini'] # basic arguments for every node basic_args = [Argument(Keys.INPUT, KeyHelp.INPUT, default=''), Argument(Keys.OUTPUT, KeyHelp.OUTPUT, default=''), Argument(Keys.MODULE, KeyHelp.MODULE, default=''), Argument(Keys.LOG_LEVEL, KeyHelp.LOG_LEVEL, default="DEBUG")] # Fixme: Prettify WORKDIR creation system # WORKDIR: if WORKDIR is defined add related args app_args = [] for i, arg in enumerate(app_args): if arg.name == Keys.WORKDIR: app_args.insert(i + 1, Argument(Keys.BASEDIR, KeyHelp.BASEDIR, default='.')) app_args.insert(i + 2, Argument(Keys.JOB_ID, KeyHelp.JOB_ID, default='')) app_args.insert(i + 3, Argument(Keys.SUBJOBLIST, KeyHelp.SUBJOBLIST, default='')) app_args.insert(i + 4, Argument(Keys.NAME, KeyHelp.NAME, default=self.__class__.__name__)) break blub = basic_args + app_args defaults, cliargs = parse_sysargs(blub) # construct info from defaults < info < commandlineargs inifile = cliargs.get(Keys.INPUT, None) ih = get_handler(inifile) fileinfo = ih.read(inifile) info = dicts.merge(cliargs, dicts.merge(fileinfo, defaults)) # setup logging log = Logger.create(info[Keys.LOG_LEVEL]) # request by malars: show dataset prominent in logger if Keys.DATASET_CODE in info: if not isinstance(info[Keys.DATASET_CODE], list): if Keys.MZXML in info and not isinstance(info[Keys.MZXML], list): log.info("Dataset is %s (%s)" % (info[Keys.DATASET_CODE], os.path.basename(info[Keys.MZXML]))) else: log.info("Dataset is %s" % info[Keys.DATASET_CODE]) else: log.debug("Datasets are %s" % info[Keys.DATASET_CODE])
def run(self, info): infoh = get_handler(info[Keys.COLLATE]) paths = info[Keys.COLLATE].split(" ") del info[Keys.COLLATE] collector_config = info.copy() #read in for path in paths: logging.debug('collating file [%s]', path) config = infoh.read(path) collector_config = dicts.merge(collector_config, config, priority='append') #unify for key in collector_config.keys(): collector_config[key] = dicts.unify(collector_config[key]) #write back return collector_config
def teardown(self, info): infh = get_handler(info.get(Keys.OUTPUT)) infh.write(info, info.get(Keys.OUTPUT))
def setup(self, app_args): # basic arguments for every node basic_args = [ Argument(Keys.INPUT, KeyHelp.INPUT, default=''), Argument(Keys.OUTPUT, KeyHelp.OUTPUT, default=''), Argument(Keys.MODULE, KeyHelp.MODULE, default=''), Argument(Keys.LOG_LEVEL, KeyHelp.LOG_LEVEL, default="INFO") ] # WORKDIR: if WORKDIR is defined add related args for i, arg in enumerate(app_args): if arg.name == Keys.WORKDIR: app_args.insert( i + 1, Argument(Keys.BASEDIR, KeyHelp.BASEDIR, default='.')) app_args.insert( i + 2, Argument(Keys.JOB_ID, KeyHelp.JOB_ID, default='')) app_args.insert( i + 3, Argument(Keys.SUBJOBLIST, KeyHelp.SUBJOBLIST, default='')) app_args.insert( i + 4, Argument(Keys.NAME, KeyHelp.NAME, default=self.__class__.__name__)) break defaults, cliargs = parse_sysargs(basic_args + app_args) # construct info from defaults < info < commandlineargs infh = get_handler(cliargs.get(Keys.INPUT, None)) fileinfo = infh.read(cliargs.get(Keys.INPUT, None)) info = dicts.merge(cliargs, dicts.merge(fileinfo, defaults)) # request by malars: show dataset prominent in logger if Keys.DATASET_CODE in info: if not isinstance(info[Keys.DATASET_CODE], list): if Keys.MZXML in info and not isinstance( info[Keys.MZXML], list): logging.info("Dataset is %s (%s)", info[Keys.DATASET_CODE], os.path.basename(info[Keys.MZXML])) else: logging.info("Dataset is %s", info[Keys.DATASET_CODE]) else: logging.debug("Datasets are %s", info[Keys.DATASET_CODE]) # WORKDIR: create WORKDIR (only after mk) info = dirs.create_workdir(info) # filter to requested args if Keys.ALL_ARGS in info: # if ALL_ARGS is set give whole info to app... req_info = info else: req_info = {} # ...otherwise copy only explicitly requested args to app for key in [arg.name for arg in basic_args + app_args]: if key in info: req_info[key] = info[key] logging.debug("info for app: %s", req_info) return req_info, info
def teardown(self, log, info): ih = get_handler(info.get(Keys.OUTPUT)) ih.write(info, info.get(Keys.OUTPUT))