def test_import_overwrite(config_3):
stages, cfg = fast_flow.read_sequence_yaml(str(config_3), backend="tests.fake_scribbler_to_test", return_cfg=True)
assert len(stages) == 3
assert "my_first_stage" in cfg
assert "my_second_stage.0" in cfg
assert "my_second_stage.1" in cfg
assert "my_third_stage" not in cfg
def main(args=None):
args = process_args(args)
sequence = fast_flow.read_sequence_yaml(args.sequence_cfg,
output_dir=args.outdir)
datasets = fast_curator.read.from_yaml(args.dataset_cfg)
mkdir_p(args.outdir)
process = atup.AtUproot(
args.outdir,
quiet=args.quiet,
parallel_mode=args.mode,
process=args.ncores,
max_blocks_per_dataset=args.nblocks_per_dataset,
max_blocks_per_process=args.nblocks_per_sample,
nevents_per_block=args.blocksize,
profile=args.profile,
profile_out_path="profile.txt",
)
sequence = [
(s, s.collector() if hasattr(s, "collector") else DummyCollector())
for s in sequence
]
ret_val = process.run(datasets, sequence)
print(ret_val)
return 0
def test_read_sequence_yaml(config_1):
stages = fast_flow.read_sequence_yaml(str(config_1))
assert len(stages) == 2
assert isinstance(stages[0], fakes.FakeScribbler)
assert isinstance(stages[1], fakes.FakeScribblerArgs)
assert stages[1].an_int == 3
assert stages[1].a_str == "hello world"
assert len(stages[1].other_args) == 2
def test_read_return_cfg(config_2):
stages, cfg = fast_flow.read_sequence_yaml(str(config_2), backend=fakes, return_cfg=True)
assert len(stages) == 5
assert len(cfg) == 7
assert "stages" in cfg
assert "my_first_stage.0" in cfg
assert "my_first_stage.1" in cfg
assert "my_second_stage.0" in cfg
assert "my_second_stage.1" in cfg
assert "my_third_stage" in cfg
def main(args=None):
args = create_parser().parse_args(args)
sequence = fast_flow.read_sequence_yaml(args.sequence_cfg, output_dir=args.outdir)
datasets = fast_curator.read.from_yaml(args.dataset_cfg)
mkdir_p(args.outdir)
_, ret_val = run_carpenter(sequence, datasets, args)
print(ret_val)
return 0
def main(args=None):
args = create_parser().parse_args(args)
sequence = fast_flow.read_sequence_yaml(args.sequence_cfg,
output_dir=args.outdir,
backend="fast_carpenter")
datasets = fast_curator.read.from_yaml(args.dataset_cfg)
backend = get_backend(args.mode)
mkdir_p(args.outdir)
results, _ = backend.execute(sequence, datasets, args)
print("Summary of results")
print(results)
print("Output written to directory '%s'" % args.outdir)
return 0
def main(args=None):
args = create_parser().parse_args(args)
sequence, seq_cfg = fast_flow.read_sequence_yaml(args.sequence_cfg,
output_dir=args.outdir,
backend="fast_carpenter",
return_cfg=True)
datasets = fast_curator.read.from_yaml(args.dataset_cfg)
backend = get_backend(args.mode)
mkdir_p(args.outdir)
if args.bookkeeping:
book_keeping_file = os.path.join(args.outdir, "book-keeping.tar.gz")
write_booking(book_keeping_file, seq_cfg, datasets, cmd_line_args=args)
results, _ = backend.execute(sequence, datasets, args)
print("Summary of results")
print(results)
print("Output written to directory '%s'" % args.outdir)
return 0
parser.add_argument("-o",
"--outdir",
type=str,
default=None,
help="Output directory (if wanted)")
args = parser.parse_args()
return args
def process_sequence(sequence, nevents):
"""
This function is left up to the user to define. This is also where the
interaction between the sequences described in the config file is really
defined (eg what each step must return, what it is given, how it should be
called, etc).
"""
for i in range(nevents):
data = {}
data["iteration"] = i
for step in sequence:
step(data)
print(data)
if __name__ == "__main__":
args = parse_args()
sequence = read_sequence_yaml(args.sequence_cfg, output_dir=args.outdir)
process_sequence(sequence, nevents=args.nevents)
def read_processing_cfg(fname, out_dir):
sequence = read_sequence_yaml(fname, backend=stages, output_dir=out_dir)
return sequence