def test_checkpoints__rewrites(): """ Test that only parts that were changed are executed if a pipeline is changed. """ _checkpoints = {} calls = {} transform = chained( _count(calls, 'step 1', lambda x: x * 2), checkpoint(target=_checkpoints), _count(calls, 'step 2', lambda x: x - 3), checkpoint(target=_checkpoints), _count(calls, 'step 3', lambda x: x), ) assert apply(transform, 5, rewrites=[add_checkpoints]) == 7 assert calls == {'step 1': 1, 'step 2': 1, 'step 3': 1} assert apply(transform, 5, rewrites=[add_checkpoints]) == 7 assert calls == {'step 1': 1, 'step 2': 1, 'step 3': 2} transform = chained( _count(calls, 'step 1', lambda x: x * 2), checkpoint(target=_checkpoints), _count(calls, 'alt-step 2', lambda x: x), checkpoint(target=_checkpoints), _count(calls, 'step 3', lambda x: x), ) assert apply(transform, 5, rewrites=[add_checkpoints]) == 10 assert calls == {'step 1': 1, 'step 2': 1, 'alt-step 2': 1, 'step 3': 3} assert apply(transform, 5, rewrites=[add_checkpoints]) == 10 assert calls == {'step 1': 1, 'step 2': 1, 'alt-step 2': 1, 'step 3': 4}
def test_checkpoints__repr(): """ Test that only parts that were changed are executed if a pipeline is changed. """ _checkpoints = {} transform = add_checkpoints( chained(lambda x: x * 2, checkpoint(target=_checkpoints), lambda x: x)) repr(transform)
def test_checkpoints__repr(): """ Test that only parts that were changed are executed if a pipeline is changed. """ _checkpoints = {} transform = add_checkpoints(chained( lambda x: x * 2, checkpoint(target=_checkpoints), lambda x: x )) repr(transform)
def test_checkpoints__single_no_rewrite(): _checkpoints = {} calls = {} transform = chained(_count(calls, 'step 1', lambda x: x * 2), checkpoint(target=_checkpoints), _count(calls, 'step 2', lambda x: x - 3)) assert apply(transform, 5) == 7 assert calls == {'step 1': 1, 'step 2': 1} assert apply(transform, 5) == 7 assert calls == {'step 1': 2, 'step 2': 2}
def test_checkpoint__not_rewritten(): """checkpoints are currently ignored. """ _checkpoints = {} transform = chained( map(lambda x: x * 2), checkpoint(target=_checkpoints), map(lambda x: x - 3), ) seq = [1, 2, 3, 4, 5] actual = apply_to_local(transform, seq, npartitions=3) assert actual == [-1, 1, 3, 5, 7]
def test_checkpoints__single_no_rewrite(): _checkpoints = {} calls = {} transform = chained( _count(calls, 'step 1', lambda x: x * 2), checkpoint(target=_checkpoints), _count(calls, 'step 2', lambda x: x - 3) ) assert apply(transform, 5) == 7 assert calls == {'step 1': 1, 'step 2': 1} assert apply(transform, 5) == 7 assert calls == {'step 1': 2, 'step 2': 2}
def test_checkpoints__single(): _checkpoints = {} calls = {} transform = chained(_count(calls, 'step 1', lambda x: x * 2), checkpoint(target=_checkpoints), _count(calls, 'step 2', lambda x: x - 3)) assert apply(transform, 5, rewrites=[add_checkpoints]) == 7 assert calls == {'step 1': 1, 'step 2': 1} assert apply(transform, 5, rewrites=[add_checkpoints]) == 7 assert calls == {'step 1': 1, 'step 2': 2} assert apply(transform, 5, rewrites=[add_checkpoints]) == 7 assert calls == {'step 1': 1, 'step 2': 3} _checkpoints.clear() assert apply(transform, 5, rewrites=[add_checkpoints]) == 7 assert calls == {'step 1': 2, 'step 2': 4}
def test_checkpoints__single(): _checkpoints = {} calls = {} transform = chained( _count(calls, 'step 1', lambda x: x * 2), checkpoint(target=_checkpoints), _count(calls, 'step 2', lambda x: x - 3) ) assert apply(transform, 5, rewrites=[add_checkpoints]) == 7 assert calls == {'step 1': 1, 'step 2': 1} assert apply(transform, 5, rewrites=[add_checkpoints]) == 7 assert calls == {'step 1': 1, 'step 2': 2} assert apply(transform, 5, rewrites=[add_checkpoints]) == 7 assert calls == {'step 1': 1, 'step 2': 3} _checkpoints.clear() assert apply(transform, 5, rewrites=[add_checkpoints]) == 7 assert calls == {'step 1': 2, 'step 2': 4}