def test_add_step(self):
def test_function(pipeline, step_id, var1, var2):
result = {
'status': success,
'next_id': pipeline.next_id,
'step_id': step_id,
'sum': var1+var2
}
return result
pipe = Pipeline()
pipe.add_step(test_function, ['tag1','tag2'], var1=5, var2=10)
step = pipe.steps[0]
assert len(pipe.steps)==1
assert pipe.next_id==1
assert step.func==test_function
assert set(step.tags)==set(['tag1','tag2'])
assert step.func_kwargs=={'var1':5,'var2':10}
assert step.results==None
def test_run_advanced(self, tmpdir):
temp_path = os.path.join(str(tmpdir), 'temp')
log_path = os.path.join(str(tmpdir), 'log')
paths = {
'temp': temp_path,
'log': log_path
}
pipe = Pipeline(paths=paths, create_paths=True)
pipe.add_step(test_func1, ['func1'], var1=3, var2=4)
pipe.add_step(test_func2, ['func2'], var1=25, var2=10)
pipe.add_step(test_func2, ['func2'], var1=1, var2=0)
pipe.add_step(test_func3, var1=1, var2=0)
with pytest.raises(PipelineError):
pipe.run()
pipe = dill.load(open(os.path.join(paths['log'], 'pipeline.p'), 'rb'))
assert pipe.run_step_idx==2
assert pipe.steps[0].results=={'next_id': 4, 'status': 'success', 'step_id': 0, 'sum': 7}
with pytest.raises(ZeroDivisionError):
for step in pipe.steps:
step.results = None
pipe.run(resume=True, ignore_errors=True)
new_pipe = dill.load(open(os.path.join(paths['log'], 'pipeline.p'), 'rb'))
result = new_pipe.run(start_idx=1, ignore_errors=True, ignore_exceptions=True)
assert result['status']=='success'
assert new_pipe.steps[0].results==None
assert new_pipe.steps[1].results=={'diff': 2.5, 'status': 'success'}
assert new_pipe.steps[2].results=={'error': 'Division by 0', 'status': 'error'}
assert new_pipe.steps[3].results['status']=='error'
def test_run_basic(self, tmpdir):
temp_path = os.path.join(str(tmpdir), 'temp')
log_path = os.path.join(str(tmpdir), 'log')
paths = {
'temp': temp_path,
'log': log_path
}
pipe = Pipeline(paths=paths, create_paths=True)
pipe.add_step(test_func1, ['func1'], var1=3, var2=4)
pipe.add_step(test_func2, ['func2'], var1=25, var2=10)
pipe.add_step(test_func2, ['func2'], var1=1, var2=0)
pipe.add_step(test_func3, var1=1, var2=0)
result = pipe.run(ignore_errors=True, ignore_exceptions=True)
assert result['status']=='success'
assert pipe.steps[0].results=={'next_id': 4, 'status': 'success', 'step_id': 0, 'sum': 7}
assert pipe.steps[1].results=={'diff': 2.5, 'status': 'success'}
assert pipe.steps[2].results=={'error': 'Division by 0', 'status': 'error'}
assert pipe.steps[3].results['status']=='error'
for step in pipe.steps:
step.results = None
result = pipe.run(['func2'], ignore_errors=True, ignore_exceptions=True)
assert pipe.steps[0].results==None
assert pipe.steps[1].results=={'diff': 2.5, 'status': 'success'}
assert pipe.steps[2].results=={'error': 'Division by 0', 'status': 'error'}
assert pipe.steps[3].results==None
for step in pipe.steps:
step.results = None
result = pipe.run(ignore_tags=['func2'], ignore_errors=True, ignore_exceptions=True)
assert pipe.steps[0].results=={'next_id': 4, 'status': 'success', 'step_id': 0, 'sum': 7}
assert pipe.steps[1].results==None
assert pipe.steps[2].results==None
assert pipe.steps[3].results['status']=='error'