def test_many_upstream(tmp_directory):
""" {A, B} -> C
"""
dag = DAG()
fa = Path('a.txt')
fb = Path('b.txt')
fc = Path('c.txt')
ta = ShellScript('touch {{product}}', File(fa),
dag, 'ta')
tb = ShellScript('touch {{product}} > {{product}}', File(fb),
dag, 'tb')
tc = ShellScript('cat {{upstream["ta"]}} {{upstream["tb"]}} > {{product}}',
File(fc), dag, 'tc')
(ta + tb) >> tc
dag.build()
assert ta.product.exists()
assert tb.product.exists()
assert tc.product.exists()
assert not ta.product._outdated()
assert not tb.product._outdated()
assert not tc.product._outdated()
ta.build(force=True)
dag._clear_cached_outdated_status()
assert not ta.product._outdated()
assert not tb.product._outdated()
assert tc.product._outdated()
dag.build()
tb.build(force=True)
dag._clear_cached_outdated_status()
assert not ta.product._outdated()
assert not tb.product._outdated()
assert tc.product._outdated()
def test_outdated_data_simple_dependency(tmp_directory):
""" A -> B
"""
dag = DAG()
fa = Path('a.txt')
fb = Path('b.txt')
ta = ShellScript('touch {{product}}', File(fa), dag, 'ta')
tb = ShellScript('cat {{upstream["ta"]}} > {{product}}', File(fb), dag,
'tb')
ta >> tb
ta.render()
tb.render()
assert not ta.product.exists()
assert not tb.product.exists()
assert ta.product._outdated()
assert tb.product._outdated()
dag.build()
dag._clear_cached_outdated_status()
# they both exist now
assert ta.product.exists()
assert tb.product.exists()
# and arent outdated...
assert not ta.product._outdated()
assert not tb.product._outdated()
# let's make b outdated
ta.build(force=True)
dag._clear_cached_outdated_status()
assert not ta.product._outdated()
assert tb.product._outdated()
def test_task_change_in_status():
dag = DAG('dag')
ta = ShellScript('echo "a" > {{product}}', File('a.txt'), dag, 'ta')
tb = ShellScript('cat {{upstream["ta"]}} > {{product}}', File('b.txt'),
dag, 'tb')
tc = ShellScript('cat {{upstream["tb"]}} > {{product}}', File('c.txt'),
dag, 'tc')
assert all([t._status == TaskStatus.WaitingRender for t in [ta, tb, tc]])
ta >> tb >> tc
dag.render()
assert (ta._status == TaskStatus.WaitingExecution
and tb._status == TaskStatus.WaitingUpstream
and tc._status == TaskStatus.WaitingUpstream)
ta.build()
assert (ta._status == TaskStatus.Executed
and tb._status == TaskStatus.WaitingExecution
and tc._status == TaskStatus.WaitingUpstream)
tb.build()
assert (ta._status == TaskStatus.Executed
and tb._status == TaskStatus.Executed
and tc._status == TaskStatus.WaitingExecution)
tc.build()
assert all([t._status == TaskStatus.Executed for t in [ta, tb, tc]])