def test_checkpoints__rewrites():
"""
Test that only parts that were changed are executed if a pipeline is
changed.
"""
_checkpoints = {}
calls = {}
transform = chained(
_count(calls, 'step 1', lambda x: x * 2),
checkpoint(target=_checkpoints),
_count(calls, 'step 2', lambda x: x - 3),
checkpoint(target=_checkpoints),
_count(calls, 'step 3', lambda x: x),
)
assert apply(transform, 5, rewrites=[add_checkpoints]) == 7
assert calls == {'step 1': 1, 'step 2': 1, 'step 3': 1}
assert apply(transform, 5, rewrites=[add_checkpoints]) == 7
assert calls == {'step 1': 1, 'step 2': 1, 'step 3': 2}
transform = chained(
_count(calls, 'step 1', lambda x: x * 2),
checkpoint(target=_checkpoints),
_count(calls, 'alt-step 2', lambda x: x),
checkpoint(target=_checkpoints),
_count(calls, 'step 3', lambda x: x),
)
assert apply(transform, 5, rewrites=[add_checkpoints]) == 10
assert calls == {'step 1': 1, 'step 2': 1, 'alt-step 2': 1, 'step 3': 3}
assert apply(transform, 5, rewrites=[add_checkpoints]) == 10
assert calls == {'step 1': 1, 'step 2': 1, 'alt-step 2': 1, 'step 3': 4}
def test_checkpoints__rewrites():
"""
Test that only parts that were changed are executed if a pipeline is
changed.
"""
_checkpoints = {}
calls = {}
transform = chained(
_count(calls, 'step 1', lambda x: x * 2),
checkpoint(target=_checkpoints),
_count(calls, 'step 2', lambda x: x - 3),
checkpoint(target=_checkpoints),
_count(calls, 'step 3', lambda x: x),
)
assert apply(transform, 5, rewrites=[add_checkpoints]) == 7
assert calls == {'step 1': 1, 'step 2': 1, 'step 3': 1}
assert apply(transform, 5, rewrites=[add_checkpoints]) == 7
assert calls == {'step 1': 1, 'step 2': 1, 'step 3': 2}
transform = chained(
_count(calls, 'step 1', lambda x: x * 2),
checkpoint(target=_checkpoints),
_count(calls, 'alt-step 2', lambda x: x),
checkpoint(target=_checkpoints),
_count(calls, 'step 3', lambda x: x),
)
assert apply(transform, 5, rewrites=[add_checkpoints]) == 10
assert calls == {'step 1': 1, 'step 2': 1, 'alt-step 2': 1, 'step 3': 3}
assert apply(transform, 5, rewrites=[add_checkpoints]) == 10
assert calls == {'step 1': 1, 'step 2': 1, 'alt-step 2': 1, 'step 3': 4}
def test_checkpoints__repr():
"""
Test that only parts that were changed are executed if a pipeline is
changed.
"""
_checkpoints = {}
transform = add_checkpoints(
chained(lambda x: x * 2, checkpoint(target=_checkpoints), lambda x: x))
repr(transform)
def test_checkpoints__repr():
"""
Test that only parts that were changed are executed if a pipeline is
changed.
"""
_checkpoints = {}
transform = add_checkpoints(chained(
lambda x: x * 2,
checkpoint(target=_checkpoints),
lambda x: x
))
repr(transform)
def test_checkpoints__single_no_rewrite():
_checkpoints = {}
calls = {}
transform = chained(_count(calls, 'step 1', lambda x: x * 2),
checkpoint(target=_checkpoints),
_count(calls, 'step 2', lambda x: x - 3))
assert apply(transform, 5) == 7
assert calls == {'step 1': 1, 'step 2': 1}
assert apply(transform, 5) == 7
assert calls == {'step 1': 2, 'step 2': 2}
def test_checkpoint__not_rewritten():
"""checkpoints are currently ignored.
"""
_checkpoints = {}
transform = chained(
map(lambda x: x * 2),
checkpoint(target=_checkpoints),
map(lambda x: x - 3),
)
seq = [1, 2, 3, 4, 5]
actual = apply_to_local(transform, seq, npartitions=3)
assert actual == [-1, 1, 3, 5, 7]
def test_checkpoints__single_no_rewrite():
_checkpoints = {}
calls = {}
transform = chained(
_count(calls, 'step 1', lambda x: x * 2),
checkpoint(target=_checkpoints),
_count(calls, 'step 2', lambda x: x - 3)
)
assert apply(transform, 5) == 7
assert calls == {'step 1': 1, 'step 2': 1}
assert apply(transform, 5) == 7
assert calls == {'step 1': 2, 'step 2': 2}
def test_checkpoints__single():
_checkpoints = {}
calls = {}
transform = chained(_count(calls, 'step 1', lambda x: x * 2),
checkpoint(target=_checkpoints),
_count(calls, 'step 2', lambda x: x - 3))
assert apply(transform, 5, rewrites=[add_checkpoints]) == 7
assert calls == {'step 1': 1, 'step 2': 1}
assert apply(transform, 5, rewrites=[add_checkpoints]) == 7
assert calls == {'step 1': 1, 'step 2': 2}
assert apply(transform, 5, rewrites=[add_checkpoints]) == 7
assert calls == {'step 1': 1, 'step 2': 3}
_checkpoints.clear()
assert apply(transform, 5, rewrites=[add_checkpoints]) == 7
assert calls == {'step 1': 2, 'step 2': 4}
def test_checkpoints__single():
_checkpoints = {}
calls = {}
transform = chained(
_count(calls, 'step 1', lambda x: x * 2),
checkpoint(target=_checkpoints),
_count(calls, 'step 2', lambda x: x - 3)
)
assert apply(transform, 5, rewrites=[add_checkpoints]) == 7
assert calls == {'step 1': 1, 'step 2': 1}
assert apply(transform, 5, rewrites=[add_checkpoints]) == 7
assert calls == {'step 1': 1, 'step 2': 2}
assert apply(transform, 5, rewrites=[add_checkpoints]) == 7
assert calls == {'step 1': 1, 'step 2': 3}
_checkpoints.clear()
assert apply(transform, 5, rewrites=[add_checkpoints]) == 7
assert calls == {'step 1': 2, 'step 2': 4}