def readComboPaths(pipeline_name):
result = []
make_pipelineFn = pnlpipe_pipelines.get_make_pipeline(pipeline_name)
# for each parameter values combo (a parameter point without caseid)
for i, (combo, caseids) in enumerate(read_grouped_combos(pipeline_name)):
iStr = str(i)
comboPaths = {
'pipelineName': local.path(paramsFile).stem,
'paramCombo': paramCombo,
'paths': defaultdict(list),
'paramId': i,
'num': len(caseids),
'caseids': caseids
}
for caseid in caseids:
args = dict(paramCombo, caseid=caseid)
args = {k: v for k, v in args.items() if not k.startswith('_')}
subjectPipeline = make_pipelineFn(**args)
for pipelineKey, node in subjectPipeline.items():
if pipelineKey.startswith('_'):
continue
_assert_is_node(node, pipelineKey)
p = SubjectPath(caseid=caseid,
pipelineKey=pipelineKey,
path=node.output())
comboPaths['paths'][pipelineKey].append(p)
result.append(comboPaths)
return result
def make_pipeline(pipeline_name, combo, caseid):
if not isinstance(combo, dict):
raise TypeError(
"make_pipeline: expects parameter combination dictionary")
args = dict(combo, caseid=caseid)
make_pipelineFn = pnlpipe_pipelines.get_make_pipeline(pipeline_name)
args = {k: v for k, v in args.items() if not k.startswith('_')}
pipeline = make_pipelineFn(**args)
if not pipeline:
raise Exception(
"make_pipeline(..) returned None, did you forget to return dictionary in pnlpipe_pipelines/pipeline_{}?"
.format(pipeline_name))
return pipeline
def main(self):
make_pipelineFn = pnlpipe_pipelines.get_make_pipeline(
self.parent.pipeline_name)
print(make_pipelineFn.__doc__ or 'No description.')
grouped_combos = read_grouped_combos(self.parent.pipeline_name)
print("There are {} parameter combination(s) defined in '{}'.".format(
len(grouped_combos),
self.parent.params_file.relative_to(local.cwd)))
for paramid, combo, caseids in grouped_combos:
counts = {}
caseids = caseids or [None]
_print()
print("## Parameter Combination {} ({} case(s))".format(
paramid, len(caseids)),
file=sys.stderr)
_print()
_print('Parameters:')
printVertical(dict(combo, caseids=caseids),
keys=list(combo.keys()) + ['caseids'])
pipeline = make_pipeline(self.parent.pipeline_name, combo,
caseids[0])
pathsmap = {'caseid_placeholder': caseids[0]}
for tag, node in pipeline.items():
pathsmap[tag] = local.path(node.output()).relative_to(
local.cwd)
_print()
_print('Paths:')
printVertical(pathsmap, keys=['caseid_placeholder'] + \
[k for k in list(pathsmap.keys()) if k not in ['caseid_placeholder']])
for caseid in caseids:
pipeline = make_pipeline(self.parent.pipeline_name, combo,
caseid)
for tag, node in pipeline.items():
if local.path(node.output()).exists():
counts[tag] = counts.get(tag, 0) + 1
else:
counts[tag] = counts.get(tag, 0) + 0
counts['#cases'] = len(caseids)
print('', file=sys.stderr)
header = list(counts.keys())
header.remove('#cases')
printTable(counts, header + ['#cases'])