def test_upload_after_task_build(tmp_directory): dag = DAG() product = File('file.txt') product.upload = Mock(wraps=product.upload) task = PythonCallable(_touch, product, dag=dag) task.build() product.upload.assert_called_once()
def test_building_a_single_task_when_rendered_upstream(): dag = DAG() t1 = PythonCallable(touch, File('1.txt'), dag, name=1) t2 = PythonCallable(touch_w_upstream, File('2.txt'), dag, name=2) t1 >> t2 dag.render() t2.build()
def test_building_a_single_task_when_has_unrendered_upstream(): dag = DAG() t1 = PythonCallable(touch, File('1.txt'), dag, name=1) t2 = PythonCallable(touch_w_upstream, File('2.txt'), dag, name=2) t1 >> t2 with pytest.raises(TaskBuildError) as excinfo: t2.build() msg = ('Cannot directly build task "2" as it has upstream dependencies' ', call dag.render() first') assert msg == str(excinfo.value)
def test_task_build_clears_cached_status(tmp_directory): dag = DAG() t = PythonCallable(touch, File('my_file'), dag) t.render() assert t.product._outdated_data_dependencies_status is None assert t.product._outdated_code_dependency_status is None t.status() assert t.product._outdated_data_dependencies_status is not None assert t.product._outdated_code_dependency_status is not None t.build() assert t.product._outdated_data_dependencies_status is None assert t.product._outdated_code_dependency_status is None
def test_task_report_after_building(tmp_directory): dag = DAG() t = PythonCallable(touch_root, File('some_file.txt'), dag, name='task') t.render() report = t.build() assert report['Ran?'] assert report['Elapsed (s)'] assert report['name'] == 'task'
def test_outdated_data_simple_dependency(tmp_directory): """ A -> B """ dag = DAG() fa = Path('a.txt') fb = Path('b.txt') ta = PythonCallable(touch_root, File(fa), dag, 'ta') tb = PythonCallable(touch, File(fb), dag, 'tb') ta >> tb ta.render() tb.render() assert not ta.product.exists() assert not tb.product.exists() assert ta.product._is_outdated() assert tb.product._is_outdated() dag.build() dag._clear_metadata() # they both exist now assert ta.product.exists() assert tb.product.exists() # and arent outdated... assert not ta.product._is_outdated() assert not tb.product._is_outdated() # let's make b outdated ta.build(force=True) dag._clear_metadata() assert not ta.product._is_outdated() assert tb.product._is_outdated()
def test_dag_render_step_by_step_w_skipped(tmp_directory): dag = DAG() t1 = PythonCallable(touch_root, File('t1.txt'), dag, name='t1') t21 = PythonCallable(touch, File('t21.txt'), dag, name='t21') t22 = PythonCallable(touch, File('t22.txt'), dag, name='t22') t3 = PythonCallable(touch, File('t3.txt'), dag, name='t3') t1 >> t21 t1 >> t22 (t21 + t22) >> t3 assert (set(t.exec_status for t in dag.values()) == {TaskStatus.WaitingRender}) dag.render() t1.build() dag.render() assert t1.exec_status == TaskStatus.Skipped assert t21.exec_status == TaskStatus.WaitingExecution assert t22.exec_status == TaskStatus.WaitingExecution assert t3.exec_status == TaskStatus.WaitingUpstream t21.build() dag.render() assert t1.exec_status == TaskStatus.Skipped assert t21.exec_status == TaskStatus.Skipped assert t22.exec_status == TaskStatus.WaitingExecution assert t3.exec_status == TaskStatus.WaitingUpstream t22.build() dag.render() assert t1.exec_status == TaskStatus.Skipped assert t21.exec_status == TaskStatus.Skipped assert t22.exec_status == TaskStatus.Skipped assert t3.exec_status == TaskStatus.WaitingExecution t3.build() dag.render() assert t1.exec_status == TaskStatus.Skipped assert t21.exec_status == TaskStatus.Skipped assert t22.exec_status == TaskStatus.Skipped assert t3.exec_status == TaskStatus.Skipped
def test_many_upstream(tmp_directory): """ {A, B} -> C """ dag = DAG() fa = Path('a.txt') fb = Path('b.txt') fc = Path('c.txt') ta = PythonCallable(touch_root, File(fa), dag, 'ta') tb = PythonCallable(touch_root, File(fb), dag, 'tb') tc = PythonCallable(touch, File(fc), dag, 'tc') (ta + tb) >> tc dag.build() assert ta.product.exists() assert tb.product.exists() assert tc.product.exists() assert not ta.product._is_outdated() assert not tb.product._is_outdated() assert not tc.product._is_outdated() ta.build(force=True) dag._clear_metadata() assert not ta.product._is_outdated() assert not tb.product._is_outdated() assert tc.product._is_outdated() dag.build() tb.build(force=True) dag._clear_metadata() assert not ta.product._is_outdated() assert not tb.product._is_outdated() assert tc.product._is_outdated()
def test_build_a_single_task(tmp_directory): dag = DAG() t = PythonCallable(touch, File('1.txt'), dag) assert t.build()