def run(self, log, info):
used_engines = []
log.debug("All available engines: %s", info['ENGINES'])
for engine in info['ENGINES'].split(" "):
key = 'RUN' + engine.upper()
if key in info and info[key] == 'True':
used_engines.append(engine)
log.debug("Effectively used engines: %s" % used_engines)
if not isinstance(info[Keys.DATASET_CODE],list):
info[Keys.DATASET_CODE] = [info[Keys.DATASET_CODE]]
runs = len(info[Keys.DATASET_CODE])
log.debug("Number of samples: %d" % runs)
for i in range(runs):
collectedconfig = {}
for engine in used_engines:
path = "%s.ini_%d" % (engine, i)
if not os.path.exists(path):
raise RuntimeError("Required infofile not found " + path)
else:
log.debug("Found infofile "+path)
engineconfig = infohandler.get_handler(path).read(path)
collectedconfig = dicts.merge(collectedconfig, engineconfig, priority='append')
for key in collectedconfig.keys():
if isinstance(collectedconfig[key], list):
collectedconfig[key] = dicts.unify(collectedconfig[key])
collector_path = "%s_%d" % (info[Keys.MERGED], i)
infohandler.get_handler(info[Keys.MERGED]).write(collectedconfig, collector_path)
log.debug('Wrote outfile ' + collector_path)
return info
def run(self, info):
paths = sorted(glob.glob(info[Keys.MERGE] + "_*"))
#read in
config_container = {}
nofiles = len(paths)
if nofiles == 0:
raise RuntimeError("No files to merge found!")
for path in paths:
logging.debug("Reading %s", path)
config = infohandler.get_handler(path).read(path)
lastjob = config[Keys.SUBJOBLIST][-1]
checksum = int(lastjob.split(Keys.SUBJOBSEP)[2])
if nofiles != checksum:
raise RuntimeError(
"Number of inputfiles %d and checksum %d do not match" %
(nofiles, checksum))
#append the current config to the ones with same parent subjobs
parentjoblist = config[Keys.SUBJOBLIST][:-1]
parentjobstr = self.parentjobs_to_str(parentjoblist)
#remove one level from subjoblist
config[Keys.SUBJOBLIST] = parentjoblist
if not config[Keys.SUBJOBLIST]:
del config[Keys.SUBJOBLIST]
if parentjobstr in config_container:
config_container[parentjobstr] = dicts.merge(config_container[parentjobstr], \
config, priority='append')
else:
config_container[parentjobstr] = config
#unify (only possible after all collected)
for config in config_container.values():
for key in config.keys():
if key == Keys.SUBJOBLIST:
config[key] = dicts.unify(config[key], unlist_single=False)
continue
if isinstance(config[key], list):
config[key] = dicts.unify(config[key])
#write back
for i, config in enumerate(config_container.values()):
path = info[Keys.MERGED] + '_' + str(i)
logging.debug("Writing out %s", path)
infohandler.get_handler(path).write(config, path)
return info
def run(self, log, info):
paths = sorted(glob.glob(info[Keys.MERGE] + "_*"))
#read in
config_container = {}
nofiles = len(paths)
if nofiles == 0:
raise RuntimeError("No files to merge found!")
for path in paths:
log.debug("Reading " + path)
config = infohandler.get_handler(path).read(path)
lastjob = config[Keys.SUBJOBLIST][-1]
checksum = int(lastjob.split(Keys.SUBJOBSEP)[2])
if nofiles != checksum:
raise RuntimeError("Number of inputfiles %d and checksum %d do not match" % (nofiles, checksum))
#append the current config to the ones with same parent subjobs
parentjoblist = config[Keys.SUBJOBLIST][:-1]
parentjobstr = self.parentjobs_to_str(parentjoblist)
#remove one level from subjoblist
config[Keys.SUBJOBLIST] = parentjoblist
if not config[Keys.SUBJOBLIST]:
del config[Keys.SUBJOBLIST]
if parentjobstr in config_container:
config_container[parentjobstr] = dicts.merge(config_container[parentjobstr], config, priority='append')
else:
config_container[parentjobstr] = config
#unify (only possible after all collected)
for config in config_container.values():
for key in config.keys():
if key == Keys.SUBJOBLIST:
config[key] = dicts.unify(config[key], unlist_single=False)
continue
if isinstance(config[key], list):
config[key] = dicts.unify(config[key])
#write back
for i, config in enumerate(config_container.values()):
path = info[Keys.MERGED] + '_' + str(i)
log.debug("Writing out " + path)
infohandler.get_handler(path).write(config, path)
return info
def run(self, info):
infoh = get_handler(info[Keys.BRANCH])
tobranch = info[Keys.BRANCH].split(" ")
del info[Keys.BRANCH]
for branch in tobranch:
logging.info("Branching %s", branch)
info = info.copy()
infoh.write(info, branch)
return info
def run(self, log, info):
ih = get_handler(info[Keys.BRANCH])
tobranch = info[Keys.BRANCH].split(" ")
del info[Keys.BRANCH]
for branch in tobranch:
log.info("Branching " + branch)
info = info.copy()
ih.write(info, branch)
return info
def setup(self, app_args):
# basic arguments for every node
basic_args = [Argument(Keys.INPUT, KeyHelp.INPUT, default=''),
Argument(Keys.OUTPUT, KeyHelp.OUTPUT, default=''),
Argument(Keys.MODULE, KeyHelp.MODULE, default=''),
Argument(Keys.LOG_LEVEL, KeyHelp.LOG_LEVEL, default="DEBUG")]
# Fixme: Prettify WORKDIR creation system
# WORKDIR: if WORKDIR is defined add related args
for i, arg in enumerate(app_args):
if arg.name == Keys.WORKDIR:
app_args.insert(i + 1, Argument(Keys.BASEDIR, KeyHelp.BASEDIR, default='.'))
app_args.insert(i + 2, Argument(Keys.JOB_ID, KeyHelp.JOB_ID, default=''))
app_args.insert(i + 3, Argument(Keys.SUBJOBLIST, KeyHelp.SUBJOBLIST, default=''))
app_args.insert(i + 4, Argument(Keys.NAME, KeyHelp.NAME, default=self.__class__.__name__))
break
defaults, cliargs = parse_sysargs(basic_args + app_args)
# construct info from defaults < info < commandlineargs
ih = get_handler(cliargs.get(Keys.INPUT, None))
fileinfo = ih.read(cliargs.get(Keys.INPUT, None))
info = dicts.merge(cliargs, dicts.merge(fileinfo, defaults))
# setup logging
log = Logger.create(info[Keys.LOG_LEVEL])
# request by malars: show dataset prominent in logger
if Keys.DATASET_CODE in info:
if not isinstance(info[Keys.DATASET_CODE], list):
if Keys.MZXML in info and not isinstance(info[Keys.MZXML], list):
log.info("Dataset is %s (%s)" % (info[Keys.DATASET_CODE], os.path.basename(info[Keys.MZXML])))
else:
log.info("Dataset is %s" % info[Keys.DATASET_CODE])
else:
log.debug("Datasets are %s" % info[Keys.DATASET_CODE])
# WORKDIR: create WORKDIR (only after mk log)
info = dirs.create_workdir(log, info)
# filter to requested args
if Keys.ALL_ARGS in info:
# if ALL_ARGS is set give whole info to app...
req_info = info
else:
req_info = {}
# ...otherwise copy only explicitly requested args to app
for key in [arg.name for arg in basic_args + app_args]:
if key in info:
req_info[key] = info[key]
log.debug("info for app: %s" % req_info)
return log, req_info, info
def test1_arg_priority(self):
"""Test the argument priorities. Disabled."""
sys.argv = ['--INPUT', 'fcuk.ini', '--OUTPUT', 'kufc.ini']
# basic arguments for every node
basic_args = [
Argument(Keys.INPUT, KeyHelp.INPUT, default=''),
Argument(Keys.OUTPUT, KeyHelp.OUTPUT, default=''),
Argument(Keys.MODULE, KeyHelp.MODULE, default=''),
Argument(Keys.LOG_LEVEL, KeyHelp.LOG_LEVEL, default="DEBUG")
]
# WORKDIR: if WORKDIR is defined add related args
app_args = []
for i, arg in enumerate(app_args):
if arg.name == Keys.WORKDIR:
app_args.insert(
i + 1, Argument(Keys.BASEDIR, KeyHelp.BASEDIR,
default='.'))
app_args.insert(
i + 2, Argument(Keys.JOB_ID, KeyHelp.JOB_ID, default=''))
app_args.insert(
i + 3,
Argument(Keys.SUBJOBLIST, KeyHelp.SUBJOBLIST, default=''))
app_args.insert(
i + 4,
Argument(Keys.NAME,
KeyHelp.NAME,
default=self.__class__.__name__))
break
blub = basic_args + app_args
defaults, cliargs = parse_sysargs(blub)
# construct info from defaults < info < commandlineargs
inifile = cliargs.get(Keys.INPUT, None)
info_fh = get_handler(inifile)
fileinfo = info_fh.read(inifile)
info = dicts.merge(cliargs, dicts.merge(fileinfo, defaults))
# request by malars: show dataset prominent in logger
if Keys.DATASET_CODE in info:
if not isinstance(info[Keys.DATASET_CODE], list):
if Keys.MZXML in info and not isinstance(
info[Keys.MZXML], list):
logging.info("Dataset is %s (%s)", info[Keys.DATASET_CODE],
os.path.basename(info[Keys.MZXML]))
else:
logging.info("Dataset is %s", info[Keys.DATASET_CODE])
else:
logging.debug("Datasets are %s", info[Keys.DATASET_CODE])
def run(self, log, info):
basename = info[Keys.SPLIT]
key = info[Keys.SPLIT_KEY]
value = info.get(key,"")
if not isinstance(value, list):
value = [value]
info = info.copy()
del info[Keys.SPLIT]
del info[Keys.SPLIT_KEY]
if info.get(Keys.SUBJOBLIST, "") == "":
info[Keys.SUBJOBLIST] = []
for i, val in enumerate(value):
infocopy = copy.deepcopy(info)
infocopy[key] = val
infocopy[Keys.SUBJOBLIST].append("%s%s%d%s%d" % (key, Keys.SUBJOBSEP, i, Keys.SUBJOBSEP, len(value)))
path = basename + "_" + str(i)
log.debug("Writing split file " + path)
get_handler(basename).write(infocopy, path)
return info
def run(self, info):
basename = info[Keys.SPLIT]
key = info[Keys.SPLIT_KEY]
value = info.get(key, "")
if not isinstance(value, list):
value = [value]
info = info.copy()
del info[Keys.SPLIT]
del info[Keys.SPLIT_KEY]
if info.get(Keys.SUBJOBLIST, "") == "":
info[Keys.SUBJOBLIST] = []
for i, val in enumerate(value):
infocopy = copy.deepcopy(info)
infocopy[key] = val
infocopy[Keys.SUBJOBLIST].append("%s%s%d%s%d" % (key, Keys.SUBJOBSEP, i, \
Keys.SUBJOBSEP, len(value)))
path = basename + "_" + str(i)
logging.debug("Writing split file %s", path)
get_handler(basename).write(infocopy, path)
return info
def run(self, log, info):
ih = get_handler(info[Keys.COLLATE])
paths = info[Keys.COLLATE].split(" ")
del info[Keys.COLLATE]
collector_config = info.copy()
#read in
for path in paths:
log.debug('collating file [%s]' % path)
config = ih.read(path)
collector_config = dicts.merge(collector_config, config, priority='append')
#unify
for key in collector_config.keys():
collector_config[key] = dicts.unify(collector_config[key])
#write back
return collector_config
def test1_arg_priority(self):
return
sys.argv = [ '--INPUT', 'fcuk.ini', '--OUTPUT', 'kufc.ini']
# basic arguments for every node
basic_args = [Argument(Keys.INPUT, KeyHelp.INPUT, default=''),
Argument(Keys.OUTPUT, KeyHelp.OUTPUT, default=''),
Argument(Keys.MODULE, KeyHelp.MODULE, default=''),
Argument(Keys.LOG_LEVEL, KeyHelp.LOG_LEVEL, default="DEBUG")]
# Fixme: Prettify WORKDIR creation system
# WORKDIR: if WORKDIR is defined add related args
app_args = []
for i, arg in enumerate(app_args):
if arg.name == Keys.WORKDIR:
app_args.insert(i + 1, Argument(Keys.BASEDIR, KeyHelp.BASEDIR, default='.'))
app_args.insert(i + 2, Argument(Keys.JOB_ID, KeyHelp.JOB_ID, default=''))
app_args.insert(i + 3, Argument(Keys.SUBJOBLIST, KeyHelp.SUBJOBLIST, default=''))
app_args.insert(i + 4, Argument(Keys.NAME, KeyHelp.NAME, default=self.__class__.__name__))
break
blub = basic_args + app_args
defaults, cliargs = parse_sysargs(blub)
# construct info from defaults < info < commandlineargs
inifile = cliargs.get(Keys.INPUT, None)
ih = get_handler(inifile)
fileinfo = ih.read(inifile)
info = dicts.merge(cliargs, dicts.merge(fileinfo, defaults))
# setup logging
log = Logger.create(info[Keys.LOG_LEVEL])
# request by malars: show dataset prominent in logger
if Keys.DATASET_CODE in info:
if not isinstance(info[Keys.DATASET_CODE], list):
if Keys.MZXML in info and not isinstance(info[Keys.MZXML], list):
log.info("Dataset is %s (%s)" % (info[Keys.DATASET_CODE], os.path.basename(info[Keys.MZXML])))
else:
log.info("Dataset is %s" % info[Keys.DATASET_CODE])
else:
log.debug("Datasets are %s" % info[Keys.DATASET_CODE])
def run(self, info):
infoh = get_handler(info[Keys.COLLATE])
paths = info[Keys.COLLATE].split(" ")
del info[Keys.COLLATE]
collector_config = info.copy()
#read in
for path in paths:
logging.debug('collating file [%s]', path)
config = infoh.read(path)
collector_config = dicts.merge(collector_config,
config,
priority='append')
#unify
for key in collector_config.keys():
collector_config[key] = dicts.unify(collector_config[key])
#write back
return collector_config
def teardown(self, info):
infh = get_handler(info.get(Keys.OUTPUT))
infh.write(info, info.get(Keys.OUTPUT))
def setup(self, app_args):
# basic arguments for every node
basic_args = [
Argument(Keys.INPUT, KeyHelp.INPUT, default=''),
Argument(Keys.OUTPUT, KeyHelp.OUTPUT, default=''),
Argument(Keys.MODULE, KeyHelp.MODULE, default=''),
Argument(Keys.LOG_LEVEL, KeyHelp.LOG_LEVEL, default="INFO")
]
# WORKDIR: if WORKDIR is defined add related args
for i, arg in enumerate(app_args):
if arg.name == Keys.WORKDIR:
app_args.insert(
i + 1, Argument(Keys.BASEDIR, KeyHelp.BASEDIR,
default='.'))
app_args.insert(
i + 2, Argument(Keys.JOB_ID, KeyHelp.JOB_ID, default=''))
app_args.insert(
i + 3,
Argument(Keys.SUBJOBLIST, KeyHelp.SUBJOBLIST, default=''))
app_args.insert(
i + 4,
Argument(Keys.NAME,
KeyHelp.NAME,
default=self.__class__.__name__))
break
defaults, cliargs = parse_sysargs(basic_args + app_args)
# construct info from defaults < info < commandlineargs
infh = get_handler(cliargs.get(Keys.INPUT, None))
fileinfo = infh.read(cliargs.get(Keys.INPUT, None))
info = dicts.merge(cliargs, dicts.merge(fileinfo, defaults))
# request by malars: show dataset prominent in logger
if Keys.DATASET_CODE in info:
if not isinstance(info[Keys.DATASET_CODE], list):
if Keys.MZXML in info and not isinstance(
info[Keys.MZXML], list):
logging.info("Dataset is %s (%s)", info[Keys.DATASET_CODE],
os.path.basename(info[Keys.MZXML]))
else:
logging.info("Dataset is %s", info[Keys.DATASET_CODE])
else:
logging.debug("Datasets are %s", info[Keys.DATASET_CODE])
# WORKDIR: create WORKDIR (only after mk)
info = dirs.create_workdir(info)
# filter to requested args
if Keys.ALL_ARGS in info:
# if ALL_ARGS is set give whole info to app...
req_info = info
else:
req_info = {}
# ...otherwise copy only explicitly requested args to app
for key in [arg.name for arg in basic_args + app_args]:
if key in info:
req_info[key] = info[key]
logging.debug("info for app: %s", req_info)
return req_info, info
def teardown(self, log, info):
ih = get_handler(info.get(Keys.OUTPUT))
ih.write(info, info.get(Keys.OUTPUT))
