def test_raises_render_error_if_missing_param_in_product():
dag = DAG('my dag')
ta = ShellScript('echo "a" > {{product}}',
File('a_{{name}}.txt'),
dag,
name='my task')
with pytest.raises(RenderError):
ta.render()
def test_raises_render_error_if_missing_param_in_code():
dag = DAG('my dag')
ta = ShellScript('{{command}} "a" > {{product}}',
File('a.txt'),
dag,
name='my task')
with pytest.raises(RenderError):
ta.render()
def test_non_existent_file():
dag = DAG()
f = File('file.txt')
ta = ShellScript('echo hi > {{product}}', f, dag, 'ta')
ta.render()
assert not f.exists()
assert f._outdated()
assert f._outdated_code_dependency()
assert not f._outdated_data_dependencies()
def test_raises_render_error_if_extra_param_in_code():
dag = DAG('my dag')
ta = ShellScript('echo "a" > {{product}}',
File('a.txt'),
dag,
name='my task',
params=dict(extra_param=1))
with pytest.raises(RenderError):
ta.render()
def test_can_create_task_with_many_products():
dag = DAG()
fa1 = File('a1.txt')
fa2 = File('a2.txt')
ta = ShellScript('echo {{product}}', [fa1, fa2], dag, 'ta')
ta.render()
assert not ta.product.exists()
assert ta.product._outdated()
assert ta.product._outdated_code_dependency()
assert not ta.product._outdated_data_dependencies()
def test_raises_render_error_if_non_existing_dependency_used():
dag = DAG('my dag')
ta = ShellScript('echo "a" > {{product}}', File('a.txt'), dag, name='bash')
tb = ShellScript('cat {{upstream.not_valid}} > {{product}}',
File('b.txt'),
dag,
name='bash2')
ta >> tb
with pytest.raises(RenderError):
tb.render()
def test_outdated_data_simple_dependency(tmp_directory):
""" A -> B
"""
dag = DAG()
fa = Path('a.txt')
fb = Path('b.txt')
ta = ShellScript('touch {{product}}', File(fa), dag, 'ta')
tb = ShellScript('cat {{upstream["ta"]}} > {{product}}', File(fb), dag,
'tb')
ta >> tb
ta.render()
tb.render()
assert not ta.product.exists()
assert not tb.product.exists()
assert ta.product._outdated()
assert tb.product._outdated()
dag.build()
dag._clear_cached_outdated_status()
# they both exist now
assert ta.product.exists()
assert tb.product.exists()
# and arent outdated...
assert not ta.product._outdated()
assert not tb.product._outdated()
# let's make b outdated
ta.build(force=True)
dag._clear_cached_outdated_status()
assert not ta.product._outdated()
assert tb.product._outdated()
def test_shows_warning_if_unused_dependencies():
dag = DAG('dag')
ta = ShellScript('echo "a" > {{product}}', File('a.txt'), dag, 'ta')
tb = ShellScript('cat {{upstream["ta"]}} > {{product}}', File('b.txt'),
dag, 'tb')
tc = ShellScript('cat {{upstream["tb"]}} > {{product}}', File('c.txt'),
dag, 'tc')
ta >> tb >> tc
ta >> tc
ta.render()
tb.render()
with pytest.warns(UserWarning):
tc.render()