def test_release_at_earliest_opportunity(self):
manager = ParallelRunnerManager()
manager.start()
log = manager.list()
pipeline = Pipeline(
[
node(source, None, "first"),
node(identity, "first", "second"),
node(sink, "second", None),
]
)
catalog = DataCatalog(
{
"first": manager.LoggingDataSet(log, "first"),
"second": manager.LoggingDataSet(log, "second"),
}
)
ParallelRunner().run(pipeline, catalog)
# we want to see "release first" before "load second"
assert list(log) == [
("load", "first"),
("release", "first"),
("load", "second"),
("release", "second"),
]
def test_dont_release_inputs_and_outputs(self):
manager = ParallelRunnerManager()
manager.start()
log = Manager().dict(log=[])
pipeline = Pipeline(
[node(identity, "in", "middle"),
node(identity, "middle", "out")])
catalog = DataCatalog({
"in": manager.LoggingDataSet(log, "in", "stuff"),
"middle": manager.LoggingDataSet(log, "middle"),
"out": manager.LoggingDataSet(log, "out"),
})
ParallelRunner().run(pipeline, catalog)
# we don't want to see release in or out in here
assert log["log"] == [("load", "in"), ("load", "middle"),
("release", "middle")]
def test_count_multiple_loads(self):
manager = ParallelRunnerManager()
manager.start()
log = Manager().dict(log=[])
pipeline = Pipeline([
node(source, None, "dataset"),
node(sink, "dataset", None, name="bob"),
node(sink, "dataset", None, name="fred"),
])
catalog = DataCatalog(
{"dataset": manager.LoggingDataSet(log, "dataset")})
ParallelRunner().run(pipeline, catalog)
# we want to the release after both the loads
assert log["log"] == [
("load", "dataset"),
("load", "dataset"),
("release", "dataset"),
]