def test_upload_after_task_build(tmp_directory):
dag = DAG()
product = File('file.txt')
product.upload = Mock(wraps=product.upload)
task = PythonCallable(_touch, product, dag=dag)
task.build()
product.upload.assert_called_once()
def test_building_a_single_task_when_rendered_upstream():
dag = DAG()
t1 = PythonCallable(touch, File('1.txt'), dag, name=1)
t2 = PythonCallable(touch_w_upstream, File('2.txt'), dag, name=2)
t1 >> t2
dag.render()
t2.build()
def test_building_a_single_task_when_has_unrendered_upstream():
dag = DAG()
t1 = PythonCallable(touch, File('1.txt'), dag, name=1)
t2 = PythonCallable(touch_w_upstream, File('2.txt'), dag, name=2)
t1 >> t2
with pytest.raises(TaskBuildError) as excinfo:
t2.build()
msg = ('Cannot directly build task "2" as it has upstream dependencies'
', call dag.render() first')
assert msg == str(excinfo.value)
def test_task_build_clears_cached_status(tmp_directory):
dag = DAG()
t = PythonCallable(touch, File('my_file'), dag)
t.render()
assert t.product._outdated_data_dependencies_status is None
assert t.product._outdated_code_dependency_status is None
t.status()
assert t.product._outdated_data_dependencies_status is not None
assert t.product._outdated_code_dependency_status is not None
t.build()
assert t.product._outdated_data_dependencies_status is None
assert t.product._outdated_code_dependency_status is None
def test_task_report_after_building(tmp_directory):
dag = DAG()
t = PythonCallable(touch_root, File('some_file.txt'), dag, name='task')
t.render()
report = t.build()
assert report['Ran?']
assert report['Elapsed (s)']
assert report['name'] == 'task'
def test_outdated_data_simple_dependency(tmp_directory):
""" A -> B
"""
dag = DAG()
fa = Path('a.txt')
fb = Path('b.txt')
ta = PythonCallable(touch_root, File(fa), dag, 'ta')
tb = PythonCallable(touch, File(fb), dag, 'tb')
ta >> tb
ta.render()
tb.render()
assert not ta.product.exists()
assert not tb.product.exists()
assert ta.product._is_outdated()
assert tb.product._is_outdated()
dag.build()
dag._clear_metadata()
# they both exist now
assert ta.product.exists()
assert tb.product.exists()
# and arent outdated...
assert not ta.product._is_outdated()
assert not tb.product._is_outdated()
# let's make b outdated
ta.build(force=True)
dag._clear_metadata()
assert not ta.product._is_outdated()
assert tb.product._is_outdated()
def test_dag_render_step_by_step_w_skipped(tmp_directory):
dag = DAG()
t1 = PythonCallable(touch_root, File('t1.txt'), dag, name='t1')
t21 = PythonCallable(touch, File('t21.txt'), dag, name='t21')
t22 = PythonCallable(touch, File('t22.txt'), dag, name='t22')
t3 = PythonCallable(touch, File('t3.txt'), dag, name='t3')
t1 >> t21
t1 >> t22
(t21 + t22) >> t3
assert (set(t.exec_status
for t in dag.values()) == {TaskStatus.WaitingRender})
dag.render()
t1.build()
dag.render()
assert t1.exec_status == TaskStatus.Skipped
assert t21.exec_status == TaskStatus.WaitingExecution
assert t22.exec_status == TaskStatus.WaitingExecution
assert t3.exec_status == TaskStatus.WaitingUpstream
t21.build()
dag.render()
assert t1.exec_status == TaskStatus.Skipped
assert t21.exec_status == TaskStatus.Skipped
assert t22.exec_status == TaskStatus.WaitingExecution
assert t3.exec_status == TaskStatus.WaitingUpstream
t22.build()
dag.render()
assert t1.exec_status == TaskStatus.Skipped
assert t21.exec_status == TaskStatus.Skipped
assert t22.exec_status == TaskStatus.Skipped
assert t3.exec_status == TaskStatus.WaitingExecution
t3.build()
dag.render()
assert t1.exec_status == TaskStatus.Skipped
assert t21.exec_status == TaskStatus.Skipped
assert t22.exec_status == TaskStatus.Skipped
assert t3.exec_status == TaskStatus.Skipped
def test_many_upstream(tmp_directory):
""" {A, B} -> C
"""
dag = DAG()
fa = Path('a.txt')
fb = Path('b.txt')
fc = Path('c.txt')
ta = PythonCallable(touch_root, File(fa), dag, 'ta')
tb = PythonCallable(touch_root, File(fb), dag, 'tb')
tc = PythonCallable(touch, File(fc), dag, 'tc')
(ta + tb) >> tc
dag.build()
assert ta.product.exists()
assert tb.product.exists()
assert tc.product.exists()
assert not ta.product._is_outdated()
assert not tb.product._is_outdated()
assert not tc.product._is_outdated()
ta.build(force=True)
dag._clear_metadata()
assert not ta.product._is_outdated()
assert not tb.product._is_outdated()
assert tc.product._is_outdated()
dag.build()
tb.build(force=True)
dag._clear_metadata()
assert not ta.product._is_outdated()
assert not tb.product._is_outdated()
assert tc.product._is_outdated()
def test_build_a_single_task(tmp_directory):
dag = DAG()
t = PythonCallable(touch, File('1.txt'), dag)
assert t.build()
