def main(self):
logging.info("Build prerequisite pnlpipe_software")
combos = read_combos(self.parent.pipeline_name)
software = set(
_concat([get_software(combo).items() for combo in combos]))
for name, version in software:
module = pnlpipe_software.import_module(name)
logging.info("Make {}".format(module.get_path(version)))
module.make(version)
def main(self, softwareModule):
if not softwareModule:
logger.info(
"Missing pnlpipe_software module argument, e.g. BRAINSTools")
return 1
softwareModule = pnlpipe_software.import_module(softwareModule)
if self.ver:
logger.info("Make '{}', version: {}".format(
softwareModule.__name__, self.ver))
softwareModule.make(self.ver)
else:
logger.info("Make '{}'".format(softwareModule.get_path()))
softwareModule.make()
def make_env_files(pipeline_name, use_full_paths=False):
# first delete existing files in case they are stale
outdir = local.path(params_file(pipeline_name)).dirname
for f in outdir // ('_' + pipeline_name + '*.sh'):
f.delete()
for paramid, combo, caseids in read_grouped_combos(pipeline_name):
pipeline = make_pipeline(pipeline_name, combo, caseids[0])
envfile = outdir / ('{}_env{}.sh'.format(pipeline_name, paramid))
logging.info("Make '{}'".format(envfile))
with open(envfile, 'w') as f:
f.write('# Parameter combination {}\n'.format(paramid))
printVertical(combo, prepend='# ', fd=f)
f.write('\n')
# Generated output paths
for tag, node in pipeline.items():
# if self.use_full_paths:
nodepath = escape_path(node.output())
# else:
# from pnlpipe_cli.pipecmd.symlink import toSymlink
# path = toSymlink(caseid, pipeline_name, tag,
# node.output(), paramid)
f.write('export {}={}\n\n'.format(tag, nodepath))
f.write('export {}={}\n\n'.format(OBSID_KEY, caseids[0]))
# Software environment
env_dicts = []
for softname, version in software_params(combo).items():
software_module = pnlpipe_software.import_module(softname)
if hasattr(software_module, 'env_dict'):
env_dicts.append(software_module.env_dict(version))
softVars = pnlpipe_software.composeEnvDicts(env_dicts)
for var, val in softVars.items():
f.write('export {}={}\n\n'.format(var, val))
# TODO remove ad hoc addition of pnlscripts?
f.write("export PATH={}:$PATH\n".format(local.path('pnlscripts')))
def main(self):
if not self.paramid:
self.paramid = 1
grouped_combos = read_grouped_combos(self.parent.pipeline_name)
self.validate(len(grouped_combos))
for paramid, combo, caseids in grouped_combos:
if paramid != self.paramid:
continue
pipeline = make_pipeline(self.parent.pipeline_name, combo,
caseids[0])
_print(
"# Shell environment setup for pipeline {} (out of {})".format(
paramid, len(grouped_combos)))
_print("Parameters:")
printVertical(combo, prepend='# ')
for tag, node in pipeline.items():
nodepath = _escape_path(node.output())
print('export {}={}\n'.format(tag, nodepath))
print('export {}={}\n'.format('caseid', caseids[0]))
# Software environment
env_dicts = []
for softname, version in get_software(combo).items():
software_module = pnlpipe_software.import_module(softname)
if hasattr(software_module, 'env_dict'):
env_dicts.append(software_module.env_dict(version))
softVars = pnlpipe_software.composeEnvDicts(env_dicts)
for var, val in softVars.items():
print('export {}={}\n'.format(var, val))
print("export PATH={}:$PATH\n".format(local.path('pnlscripts')))
def main(self, *arg_caseids):
pipeline_name = self.parent.pipeline_name
log.info('Run {} pipeline'.format(pipeline_name))
target = self.want
if not target:
target = pnlpipe_pipelines.default_target(pipeline_name)
log.info('Check that prerequisite software exists')
missing_modules = []
missing_paths = []
grouped_combos = read_grouped_combos(pipeline_name, assert_valid_combos=True)
self.validate(len(grouped_combos))
# Check that prerequisite software is installed
software = set(_concat([get_software(combo).items() for (_,combo,_) in grouped_combos]))
for name, version in software:
modulefile = pnlpipe_software.module_file(name)
if not modulefile.exists():
missing_modules.append(modulefile)
continue
module = pnlpipe_software.import_module(name)
softpath = local.path(module.get_path(version))
if not softpath.exists():
missing_paths.append(module.get_path(version))
continue
log.info("{} exists".format(softpath))
for f in missing_modules:
log.critical("missing {}".format(f))
for p in missing_paths:
log.critical("missing: {}".format(p))
if missing_modules:
sys.exit(1)
if missing_paths:
errmsg = """
Some pnlpipe_software components are missing and so some parts of the pipeline won't run.
Run './pnlpipe {} setup' to build all prequisite pnlpipe_software.
""".format(pipeline_name)
errmsgFS = """
Some pnlpipe_software components are missing and so some parts of the pipeline won't run.
Run './pnlpipe {} setup' to build all prequisite pnlpipe_software and make sure the environment variable FREESURFER_HOME is set.
""".format(pipeline_name)
# TODO remove ad hoc message for FreeSurfer?
for soft in missing_paths:
if 'FREESURFER_HOME' in soft:
print(errmsgFS)
sys.exit(1)
# raise Exception(errmsgFS)
# raise Exception(errmsg)
print(errmsg)
sys.exit(1)
for paramid, combo, caseids in grouped_combos:
if self.paramid and paramid != self.paramid:
continue
print('')
print("## Pipeline {} ({} cases)".format(paramid, len(caseids)))
caseids = list(arg_caseids) if arg_caseids else caseids
print('')
print('Parameters:')
printVertical(dict(combo, caseids=caseids), keys=combo.keys() + ['caseids'])
for caseid in caseids:
# print(['caseid']+combo.keys())
# print(dict(combo,caseid=caseid))
print('')
# fullCombo = {}
# fullCombo.update(combo)
# fullCombo[OBSID_KEY] = caseid
# print('Parameters:')
# printVertical(fullCombo, keys=[OBSID_KEY] + combo.keys())
print('Update caseid: {}'.format(caseid))
pipeline = make_pipeline(pipeline_name, combo, caseid=caseid)
log.info("Make target tagged '{}'".format(target))
if self.question:
nodeAndReason = upToDate(pipeline[target])
if nodeAndReason:
print("'{}' is stale".format(pipeline[target].show()))
print("Reason: {}: {}".format(nodeAndReason[0].show(), nodeAndReason[1]))
print()
else:
print("{}: is up to date.".format(pipeline[target].show()))
else:
update(pipeline[target])