def test_build_partially_with_wildcard(tmp_directory): dag = DAG(executor=Serial(build_in_subprocess=False)) PythonCallable(touch_root, File('a-1.txt'), dag, name='a-1') PythonCallable(touch_root, File('a-2.txt'), dag, name='a-2') PythonCallable(touch_root, File('b.txt'), dag, name='b') dag.build_partially('a-*') assert Path('a-1.txt').exists() assert Path('a-2.txt').exists() assert not Path('b.txt').exists()
def test_build_partially_with_wildcard_skip_upstream(tmp_directory): dag = DAG(executor=Serial(build_in_subprocess=False)) root = PythonCallable(touch_root, File('root.txt'), dag, name='root') a1 = PythonCallable(touch, File('a-1.txt'), dag, name='a-1') root >> a1 PythonCallable(touch_root, File('a-2.txt'), dag, name='a-2') PythonCallable(touch_root, File('b.txt'), dag, name='b') dag.build_partially('a-*', skip_upstream=True) assert not Path('root.txt').exists() assert Path('a-1.txt').exists() assert Path('a-2.txt').exists() assert not Path('b.txt').exists()
def test_partial_build(tmp_directory): dag = DAG('dag') ta = PythonCallable(touch_root, File(Path('a.txt')), dag, 'ta') tb = PythonCallable(touch, File(Path('b.txt')), dag, 'tb') tc = PythonCallable(touch, File(Path('c.txt')), dag, 'tc') td = PythonCallable(touch, File(Path('d.txt')), dag, 'td') te = PythonCallable(touch, File(Path('e.txt')), dag, 'te') ta >> tb >> tc tb >> td >> te table = dag.build_partially('tc') assert set(table['name']) == {'ta', 'tb', 'tc'} assert all(table['Ran?'])
def test_build_partially(tmp_directory, executor): dag = DAG(executor=executor) PythonCallable(touch_root, File('a.txt'), dag, name='a') PythonCallable(touch_root, File('b.txt'), dag, name='b') report = dag.build_partially('b') # check it only ran task b assert report['Ran?'] == [True] assert report['name'] == ['b'] # task status in original dag are the same assert (set(t.exec_status for t in dag.values()) == {TaskStatus.WaitingRender}) # this triggers metadata loading for the first time dag.render() # new task status reflect partial execution assert ({n: t.exec_status for n, t in dag.items()} == { 'a': TaskStatus.WaitingExecution, 'b': TaskStatus.Skipped, })