def test_job_never_ready(self, walk_class, setup_sbx):
"""Trying to run a job repeatedly returning notready."""
actions = DAG()
actions.add_vertex("1.notready:always")
c = walk_class(actions)
job = c.saved_jobs["1.notready:always"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.notready
assert c.job_status == {"1.notready:always": ReturnValue.notready}
assert c.requeued == {"1.notready:always": 3}
# In the situation where we are using fingerprints,
# verify the behavior when re-doing a walk with
# the same DAG.
if walk_class == FingerprintWalk:
r2 = walk_class(actions)
job = r2.saved_jobs["1.notready:always"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.notready
assert r2.job_status == {"1.notready:always": ReturnValue.notready}
assert r2.requeued == {"1.notready:always": 3}
def test_requeue(self):
"""Requeue test.
Same as previous example except that all tests are requeued
once.
"""
results = {}
def collect(job):
if job.uid not in results:
results[job.uid] = True
return True
else:
return False
# This time test with two interdependent jobs
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(Scheduler.simple_provider(NopJob),
tokens=2, collect=collect)
s.run(dag)
assert s.max_active_jobs == 2
assert results['1']
assert results['2']
def test_cycle_detection():
d = DAG()
d.add_vertex('a')
d.add_vertex('b')
d.update_vertex('a', predecessors=['b'])
with pytest.raises(DAGError):
d.update_vertex('b', data='newb', predecessors=['a'])
# Ensure that DAG is still valid and that previous
# update_vertex has no effect
result = []
for vertex_id, data in d:
result.append(vertex_id)
assert data is None
assert result == ['b', 'a']
# Force the creation of a cycle
d.update_vertex('b', data='newb', predecessors=['a'], enable_checks=False)
# Verify that the cycle is detected
with pytest.raises(DAGError):
d.check()
# Verify that some functions do not hang when a cycle is present
assert len(d.get_closure('b')) == 2
assert str(d)
assert d.as_dot()
with pytest.raises(DAGError):
d.reverse_graph()
def test_job_depending_on_job_with_no_predicted_fingerprint_failed(setup_sbx):
"""Test case where job depends on failed job with late fingerprint."""
actions = DAG()
actions.add_vertex("fingerprint_after_job.bad")
actions.add_vertex("2", predecessors=["fingerprint_after_job.bad"])
r1 = FingerprintWalk(actions)
assert (
r1.compute_fingerprint("fingerprint_after_job.bad", None, is_prediction=True)
is None
)
# Check the status of the first job ('fingerprint_after_job.bad').
# It should be a real job that returned a failure.
job = r1.saved_jobs["fingerprint_after_job.bad"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue(1)
# Check the status of the second job ('2'); because that job depends
# on a job that failed, it should show that the job was skipped.
job = r1.saved_jobs["2"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.force_fail
# Check that no job was requeued.
assert r1.requeued == {}
def test_keyboard_interrupt(self):
"""Ensure that jobs can be interrupted."""
results = {}
pytest.importorskip('psutil')
def get_job(uid, data, predecessors, notify_end):
return NopJob(uid, data, notify_end)
def collect(job):
results[job.uid] = job
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(get_job, tokens=2, collect=collect, job_timeout=2)
# fake log_state that will raise a KeyboardInterrupt
def fake_log_state():
raise KeyboardInterrupt
s.log_state = fake_log_state
with pytest.raises(KeyboardInterrupt):
s.run(dag)
for k, v in results.items():
assert v.interrupted
def test_job_not_ready_then_ok(self, walk_class, setup_sbx):
"""Rerunning a job that first returned notready."""
actions = DAG()
actions.add_vertex("1.notready:once")
c = walk_class(actions)
job = c.saved_jobs["1.notready:once"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert c.job_status == {"1.notready:once": ReturnValue.success}
assert c.requeued == {"1.notready:once": 1}
# In the situation where we are using fingerprints,
# verify the behavior when re-doing a walk with
# the same DAG.
if walk_class == FingerprintWalk:
r2 = walk_class(actions)
job = r2.saved_jobs["1.notready:once"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
assert r2.job_status == {"1.notready:once": ReturnValue.skip}
assert r2.requeued == {}
def test_bad_job_no_predecessors(self, walk_class, setup_sbx):
"""Simple case of a leaf job failing."""
actions = DAG()
actions.add_vertex("1.bad")
c = walk_class(actions)
job = c.saved_jobs["1.bad"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue(1)
assert c.job_status == {"1.bad": ReturnValue(1)}
assert c.requeued == {}
# In the situation where we are using fingerprints,
# verify the behavior when re-doing a walk with
# the same DAG.
if walk_class == FingerprintWalk:
r2 = walk_class(actions)
job = r2.saved_jobs["1.bad"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue(1)
assert r2.job_status == {"1.bad": ReturnValue(1)}
assert r2.requeued == {}
def test_minimal_run2(self):
"""Test with two interdependent jobs."""
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2', predecessors=['1'])
s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2)
s.run(dag)
assert s.max_active_jobs == 1
def test_minimal_run(self):
"""Test with only two independent jobs."""
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2)
s.run(dag)
assert s.max_active_jobs == 2
def test_iter_with_busy_state():
d = DAG()
d.add_vertex('a')
d.add_vertex('b', predecessors=['a'])
it = DAGIterator(d, enable_busy_state=True)
for nid, data in it:
if nid is None:
it.leave('a')
def test_iter_with_busy_state():
d = DAG()
d.add_vertex("a")
d.add_vertex("b", predecessors=["a"])
it = DAGIterator(d, enable_busy_state=True)
for nid, _ in it:
if nid is None:
it.leave("a")
def test_inexisting():
d = DAG()
d.add_vertex("a")
assert "a" in d
d.update_vertex("a", data="NOT B", predecessors=["b"], enable_checks=False)
assert "b" not in d
assert d["a"] == "NOT B"
with pytest.raises(DAGError):
d.check()
def test_iter_with_busy_state():
d = DAG()
d.add_vertex('a')
d.add_vertex('b', predecessors=['a'])
it = DAGIterator(d, enable_busy_state=True)
for nid, data in it:
if nid is None:
it.leave('a')
def test_inexisting():
d = DAG()
d.add_vertex('a')
assert 'a' in d
d.update_vertex('a', data='NOT B', predecessors=['b'], enable_checks=False)
assert 'b' not in d
assert d['a'] == 'NOT B'
with pytest.raises(DAGError):
d.check()
def test_simple_dag():
d = DAG()
d.add_vertex('a')
d.add_vertex('b')
d.add_vertex('c')
result = []
for vertex_id, data in d:
result.append(vertex_id)
result.sort()
assert result == ['a', 'b', 'c']
def test_simple_dag():
d = DAG()
d.add_vertex('a')
d.add_vertex('b')
d.add_vertex('c')
result = []
for vertex_id, data in d:
result.append(vertex_id)
result.sort()
assert result == ['a', 'b', 'c']
def test_simple_dag():
d = DAG()
d.add_vertex("a")
d.add_vertex("b")
d.add_vertex("c")
result = []
for vertex_id, _ in d:
result.append(vertex_id)
result.sort()
assert result == ["a", "b", "c"]
assert d.check() is None
def test_cycle():
d = DAG()
d.add_vertex('a')
d.add_vertex('b')
d.update_vertex('a', predecessors=['b'])
d.add_vertex('c', predecessors=['b'])
d.update_vertex('b', predecessors=['c'], enable_checks=False)
with pytest.raises(DAGError):
d.check()
with pytest.raises(DAGError):
d.get_context('b')
def test_cycle():
d = DAG()
d.add_vertex("a")
d.add_vertex("b")
d.update_vertex("a", predecessors=["b"])
d.add_vertex("c", predecessors=["b"])
d.update_vertex("b", predecessors=["c"], enable_checks=False)
with pytest.raises(DAGError):
d.check()
with pytest.raises(DAGError):
d.get_context("b")
def test_ordering(self):
"""Test that jobs are ordered correctly."""
results = []
def collect(job):
results.append(job.uid)
dag = DAG()
dag.add_vertex("3")
dag.add_vertex("0")
dag.add_vertex("1")
s = Scheduler(Scheduler.simple_provider(NopJob), tokens=1, collect=collect)
s.run(dag)
assert tuple(results) == ("0", "1", "3")
def test_cycle_detection():
d = DAG()
d.add_vertex('a')
d.add_vertex('b')
d.update_vertex('a', predecessors=['b'])
with pytest.raises(DAGError):
d.update_vertex('b', data='newb', predecessors=['a'])
# Ensure that DAG is still valid and that previous
# update_vertex has no effect
result = []
for vertex_id, data in d:
result.append(vertex_id)
assert data is None
assert result == ['b', 'a']
def test_cycle_detection():
d = DAG()
d.add_vertex('a')
d.add_vertex('b')
d.update_vertex('a', predecessors=['b'])
with pytest.raises(DAGError):
d.update_vertex('b', data='newb', predecessors=['a'])
# Ensure that DAG is still valid and that previous
# update_vertex has no effect
result = []
for vertex_id, data in d:
result.append(vertex_id)
assert data is None
assert result == ['b', 'a']
def test_add_vertex():
d = DAG()
# add_vertex should fail in case a dep does not exist
with pytest.raises(DAGError):
d.add_vertex("a", predecessors=["b"])
# check order of a iteration with simple dependency
d.add_vertex("b")
d.add_vertex("a", predecessors=["b"])
result = []
for vertex_id, _ in d:
result.append(vertex_id)
assert result == ["b", "a"]
# check that add_vertex fails on attempt to add already existing nodde
with pytest.raises(DAGError):
d.add_vertex("a")
# check update with new dependency
d.add_vertex("c")
d.update_vertex("b", predecessors=["c"])
assert d.get_predecessors("b") == frozenset(["c"])
assert d.vertex_predecessors == {
"a": frozenset(["b"]),
"b": frozenset(["c"]),
"c": frozenset([]),
}
result = []
for vertex_id, _ in d:
result.append(vertex_id)
assert result == ["c", "b", "a"]
d.update_vertex("a", data="datafora_")
d.update_vertex("c", data="dataforc_")
result = []
compound_data = ""
for vertex_id, data in d:
if data is not None:
compound_data += data
result.append(vertex_id)
assert result == ["c", "b", "a"]
assert compound_data == "dataforc_datafora_"
def test_add_vertex():
d = DAG()
# add_vertex should fail in case a dep does not exist
with pytest.raises(DAGError):
d.add_vertex('a', predecessors=['b'])
# check order of a iteration with simple dependency
d.add_vertex('b')
d.add_vertex('a', predecessors=['b'])
result = []
for vertex_id, data in d:
result.append(vertex_id)
assert result == ['b', 'a']
# check that add_vertex fails on attempt to add already existing nodde
with pytest.raises(DAGError):
d.add_vertex('a')
# check update with new dependency
d.add_vertex('c')
d.update_vertex('b', predecessors=['c'])
assert d.get_predecessors('b') == frozenset(['c'])
assert d.vertex_predecessors == {
'a': frozenset(['b']),
'b': frozenset(['c']),
'c': frozenset([])
}
result = []
for vertex_id, data in d:
result.append(vertex_id)
assert result == ['c', 'b', 'a']
d.update_vertex('a', data='datafora_')
d.update_vertex('c', data='dataforc_')
result = []
compound_data = ''
for vertex_id, data in d:
if data is not None:
compound_data += data
result.append(vertex_id)
assert result == ['c', 'b', 'a']
assert compound_data == 'dataforc_datafora_'
def test_do_nothing_job(self, walk_class, setup_sbx):
"""Test DAG leading us to create a DoNothingJob object."""
actions = DAG()
actions.add_vertex("1.do-nothing")
actions.add_vertex("2", predecessors=["1.do-nothing"])
c = walk_class(actions)
job = c.saved_jobs["1.do-nothing"]
assert isinstance(job, DoNothingJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
job = c.saved_jobs["2"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert c.job_status == {
"1.do-nothing": ReturnValue.success,
"2": ReturnValue.success,
}
assert c.requeued == {}
# In the situation where we are using fingerprints,
# verify the behavior when re-doing a walk with
# the same DAG.
if walk_class == FingerprintWalk:
r2 = walk_class(actions)
job = r2.saved_jobs["1.do-nothing"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
job = r2.saved_jobs["2"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
assert r2.job_status == {
"1.do-nothing": ReturnValue.skip,
"2": ReturnValue.skip,
}
assert r2.requeued == {}
def test_failed_predecessor(self, walk_class, setup_sbx):
"""Simulate the scenarior when a predecessor failed."""
actions = DAG()
actions.add_vertex("1.bad")
actions.add_vertex("2", predecessors=["1.bad"])
c = walk_class(actions)
job = c.saved_jobs["1.bad"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue(1)
job = c.saved_jobs["2"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.force_fail
assert c.job_status == {
"1.bad": ReturnValue(1),
"2": ReturnValue.force_fail
}
assert c.requeued == {}
# In the situation where we are using fingerprints,
# verify the behavior when re-doing a walk with
# the same DAG.
if walk_class == FingerprintWalk:
r2 = walk_class(actions)
job = r2.saved_jobs["1.bad"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue(1)
job = r2.saved_jobs["2"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.force_fail
assert r2.job_status == {
"1.bad": ReturnValue(1),
"2": ReturnValue.force_fail,
}
assert r2.requeued == {}
def test_timeout(self):
"""Ensure that jobs are interrupted correctly on timeout."""
results = {}
pytest.importorskip('psutil')
def get_job(uid, data, predecessors, notify_end):
return SleepJob(uid, data, notify_end)
def collect(job):
results[job.uid] = job
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(get_job, tokens=2, collect=collect, job_timeout=2)
s.run(dag)
for k, v in results.items():
assert v.interrupted
def test_shortest_path():
d = DAG()
d.add_vertex("a")
d.add_vertex("b")
d.update_vertex("a", predecessors=["b"])
d.update_vertex("b", predecessors=["a"], enable_checks=False)
assert d.shortest_path("a", "a") == ["a", "b", "a"]
d = DAG()
d.add_vertex("a")
d.update_vertex("a", predecessors=["a"], enable_checks=False)
assert d.shortest_path("a", "a") == ["a", "a"]
d = DAG()
d.add_vertex("a")
d.add_vertex("b", predecessors=["a"])
d.add_vertex("c", predecessors=["b"])
d.add_vertex("d", predecessors=["c", "a"])
assert d.shortest_path("a", "d") == ["a", "d"]
assert d.shortest_path("d", "a") is None
def test_shortest_path():
d = DAG()
d.add_vertex('a')
d.add_vertex('b')
d.update_vertex('a', predecessors=['b'])
d.update_vertex('b', predecessors=['a'], enable_checks=False)
assert d.shortest_path('a', 'a') == ['a', 'b', 'a']
d = DAG()
d.add_vertex('a')
d.update_vertex('a', predecessors=['a'], enable_checks=False)
assert d.shortest_path('a', 'a') == ['a', 'a']
d = DAG()
d.add_vertex('a')
d.add_vertex('b', predecessors=['a'])
d.add_vertex('c', predecessors=['b'])
d.add_vertex('d', predecessors=['c', 'a'])
assert d.shortest_path('a', 'd') == ['a', 'd']
assert d.shortest_path('d', 'a') is None
def test_add_vertex():
d = DAG()
# add_vertex should fail in case a dep does not exist
with pytest.raises(DAGError):
d.add_vertex('a', predecessors=['b'])
# check order of a iteration with simple dependency
d.add_vertex('b')
d.add_vertex('a', predecessors=['b'])
result = []
for vertex_id, data in d:
result.append(vertex_id)
assert result == ['b', 'a']
# check that add_vertex fails on attempt to add already existing nodde
with pytest.raises(DAGError):
d.add_vertex('a')
# check update with new dependency
d.add_vertex('c')
d.update_vertex('b', predecessors=['c'])
result = []
for vertex_id, data in d:
result.append(vertex_id)
assert result == ['c', 'b', 'a']
d.update_vertex('a', data='datafora_')
d.update_vertex('c', data='dataforc_')
result = []
compound_data = ''
for vertex_id, data in d:
if data is not None:
compound_data += data
result.append(vertex_id)
assert result == ['c', 'b', 'a']
assert compound_data == 'dataforc_datafora_'
def test_dag_merge():
d = DAG()
d.add_vertex('b')
d.add_vertex('a', predecessors=['b'])
d2 = DAG()
d2.add_vertex('c')
d2.add_vertex('b', predecessors=['c'])
d2.add_vertex('a', predecessors=['c'])
d3 = d | d2
result = []
for vertex_id, data in d3:
result.append(vertex_id)
assert data is None
assert result == ['c', 'b', 'a']
def test_dag_merge():
d = DAG()
d.add_vertex("b")
d.add_vertex("a", predecessors=["b"])
d2 = DAG()
d2.add_vertex("c")
d2.add_vertex("b", predecessors=["c"])
d2.add_vertex("a", predecessors=["c"])
d3 = d | d2
result = []
for vertex_id, data in d3:
result.append(vertex_id)
assert data is None
assert result == ["c", "b", "a"]
def test_skip(self):
"""Simple example in which all the tests are skipped."""
results = {}
def get_job(uid, data, predecessors, notify_end):
result = NopJob(uid, data, notify_end)
result.should_skip = True
return result
def collect(job):
results[job.uid] = job.timing_info
# This time test with two interdependent jobs
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(get_job, tokens=2, collect=collect)
s.run(dag)
# Check start_time end_time to be sure tests have not been run
for k, v in results.items():
assert v.start_time is None
assert v.stop_time is None
def test_dag_merge():
d = DAG()
d.add_vertex('b')
d.add_vertex('a', predecessors=['b'])
d2 = DAG()
d2.add_vertex('c')
d2.add_vertex('b', predecessors=['c'])
d2.add_vertex('a', predecessors=['c'])
d3 = d | d2
result = []
for vertex_id, data in d3:
result.append(vertex_id)
assert data is None
assert result == ['c', 'b', 'a']
def test_predecessor_with_no_fingerprint(setup_sbx):
actions = DAG()
actions.add_vertex("1")
actions.add_vertex("2.no_fingerprint", predecessors=["1"])
actions.add_vertex("3", predecessors=["2.no_fingerprint"])
actions.add_vertex("4", predecessors=["3"])
# Execute our planned actions for the first time...
r1 = FingerprintWalk(actions)
for uid in ("1", "2.no_fingerprint", "3", "4"):
job = r1.saved_jobs[uid]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert r1.job_status == {
"1": ReturnValue.success,
"2.no_fingerprint": ReturnValue.success,
"3": ReturnValue.success,
"4": ReturnValue.success,
}
assert r1.requeued == {}
# Re-execute the plan a second time. Because '2.no_fingerprint'
# has no fingerprint, both '2.no_fingerprint', but also the node
# that depends directly on it should be re-executed. Node '4,
# on the other hand, should only be re-executed if Node "3"'s
# fingerprint changed. The way things are set up in this testsuite
# is such that the fingerprint remained the same, so '4' is not
# expected to be re-run.
r2 = FingerprintWalk(actions)
for uid in ("1", "4"):
job = r2.saved_jobs[uid]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
for uid in ("2.no_fingerprint", "3"):
job = r2.saved_jobs[uid]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert r2.job_status == {
"1": ReturnValue.skip,
"2.no_fingerprint": ReturnValue.success,
"3": ReturnValue.success,
"4": ReturnValue.skip,
}
assert r2.requeued == {}
def schedule(self, resolver):
"""Compute a DAG of scheduled actions.
:param resolver: a function that helps the scheduler resolve cases
for which a decision should be taken
:type resolver: (Action, Decision) -> bool
"""
rev = self.tree.reverse_graph()
uploads = []
dag = DAG()
for uid, action in rev:
if uid == 'root':
# Root node is alway in the final DAG
dag.add_vertex(uid, action)
elif isinstance(action, Decision):
# Decision node does not appears in the final DAG but we need
# to apply the triggers based on the current list of scheduled
# actions.
action.apply_triggers(dag)
elif isinstance(action, UploadComponent):
uploads.append((action,
self.tree.vertex_predecessors[uid]))
else:
# Compute the list of successors for the current node (i.e:
# predecessors in the reversed graph). Ignore UploadComponent
# nodes as they will be processed only once the scheduling
# is done.
preds = list([k for k in rev.vertex_predecessors[uid]
if not isinstance(rev[k], UploadComponent)])
if len(preds) == 1 and isinstance(rev[preds[0]], Decision):
decision = rev[preds[0]]
# The current node addition is driven by a decision
# First check that the parent of the decision is
# scheduled. If not discard the item.
if decision.initiator not in dag:
continue
# Now check decision made. If the decision cannot be made
# delegate to the resolve function.
choice = decision.get_decision()
if choice == uid:
dag.add_vertex(uid, action)
dag.update_vertex(decision.initiator,
predecessors=[uid],
enable_checks=False)
elif choice is None:
# delegate to resolver
try:
if resolver(action, decision):
dag.add_vertex(uid, action)
dag.update_vertex(decision.initiator,
predecessors=[uid],
enable_checks=False)
except SchedulingError as e:
# In order to help the analysis of a scheduling
# error compute the explicit initiators of that
# action
dag.add_vertex(uid, action)
dag.update_vertex(decision.initiator,
predecessors=[action.uid],
enable_checks=False)
rev_graph = dag.reverse_graph()
# Initiators are explicit actions (connected to
# 'root') that are in the closure of the failing
# node.
initiators = [
iuid for iuid in rev_graph.get_closure(uid)
if 'root'
in rev_graph.vertex_predecessors[iuid]]
raise SchedulingError(e.message, uid=uid,
initiators=initiators)
else:
# An action is scheduled only if one of its successors is
# scheduled.
successors = [k for k in preds if k in dag]
if successors:
dag.add_vertex(uid, action)
for a in successors:
dag.update_vertex(a,
predecessors=[uid],
enable_checks=False)
# Handle UploadComponent nodes. Add the node only if all predecessors
# are scheduled.
for action, predecessors in uploads:
if len([p for p in predecessors if p not in dag]) == 0:
dag.update_vertex(action.uid,
action,
predecessors=predecessors,
enable_checks=False)
# connect upload to the root node
dag.update_vertex('root', predecessors=[action.uid])
return dag
def test_dag_len():
d = DAG()
d.add_vertex('a')
d.add_vertex('b')
d.update_vertex('a', predecessors=['b'])
assert len(d) == 2
class MainLoop(object):
"""Run a list of jobs."""
def __init__(self,
item_list,
run_job,
collect_result,
parallelism=None,
abort_file=None,
dyn_poll_interval=True):
"""Launch loop.
:param item_list: a list of jobs or a dag
:param run_job: a function that takes a job for argument and
return the spawned process (:class:`e3.os_process.Run` object).
Its prototype should be ``func (name, job_info)`` with name the job
identifier and job_info the related information, passed in a tuple
(slot_number, job_retry). Note that if you want to take advantage
of the parallelism the spawned process should be launched in
background (ie with bg=True when using :class:`e3.os_process.Run`).
If run_job returns SKIP_EXECUTION instead of a Run object
the mainloop will directly call collect_result without waiting.
:param collect_result: a function called when a job is finished. The
prototype should be func (name, process, job_info). If
collect_result raise NeedRequeue then the test will be requeued.
job_info is a tuple: (slot_number, job_nb_retry)
:param parallelism: number of workers
:type parallelism: int | None
:param abort_file: If specified, the loop will abort if the file is
present
:type abort_file: str | None
:param dyn_poll_interval: If True the interval between each polling
iteration is automatically updated. Otherwise it's set to 0.1
seconds.
:type dyn_poll_interval: bool
"""
e = e3.env.Env()
self.parallelism = e.get_attr("main_options.mainloop_jobs",
default_value=1,
forced_value=parallelism)
self.abort_file = e.get_attr("main_options.mainloop_abort_file",
default_value=None,
forced_value=abort_file)
if self.parallelism == 0:
if e.build.cpu.cores != UNKNOWN:
self.parallelism = e.build.cpu.cores
else:
self.parallelism = 1
e3.log.debug("start main loop with %d workers (abort on %s)",
self.parallelism, self.abort_file)
self.workers = [None] * self.parallelism
self.locked_items = [None] * self.parallelism
if not isinstance(item_list, DAG):
self.item_list = DAG()
for index, item in enumerate(item_list):
self.item_list.add_vertex(str(index), item)
else:
self.item_list = item_list
self.iterator = DAGIterator(self.item_list, enable_busy_state=True)
self.collect_result = collect_result
active_workers = 0
max_active_workers = self.parallelism
poll_sleep = 0.1
no_free_item = False
try:
while True:
# Check for abortion
if self.abort_file is not None and \
os.path.isfile(self.abort_file):
logger.info('Aborting: file %s has been found',
self.abort_file)
self.abort()
return # Exit the loop
# Find free workers
for slot, worker in enumerate(self.workers):
if worker is None:
# a worker slot is free so use it for next job
next_id, next_job = next(self.iterator)
if next_job is None:
no_free_item = True
break
else:
self.locked_items[slot] = next_id
self.workers[slot] = Worker(next_job,
run_job,
collect_result,
slot)
active_workers += 1
poll_counter = 0
e3.log.debug('Wait for free worker')
while active_workers >= max_active_workers or no_free_item:
# All worker are occupied so wait for one to finish
poll_counter += 1
for slot, worker in enumerate(self.workers):
if worker is None:
continue
# Test if the worker is still active and have more
# job pending
if not (worker.poll() or worker.execute_next()):
# If not the case free the worker slot
active_workers -= 1
self.workers[slot] = None
self.iterator.leave(self.locked_items[slot])
no_free_item = False
self.locked_items[slot] = None
sleep(poll_sleep)
if dyn_poll_interval:
poll_sleep = compute_next_dyn_poll(poll_counter,
poll_sleep)
except (StopIteration, KeyboardInterrupt) as e:
if e.__class__ == KeyboardInterrupt:
# Got ^C, abort the mainloop
logger.error("User interrupt")
# All the jobs are finished
while active_workers > 0:
for slot, worker in enumerate(self.workers):
if worker is None:
continue
# Test if the worker is still active and ignore any
# job pending
try:
still_running = worker.poll()
except TooManyErrors:
still_running = False
# We're not spawing more jobs so we can safely
# ignore all TooManyErrors exceptions.
if not still_running:
active_workers -= 1
self.workers[slot] = None
sleep(0.1)
if e.__class__ == KeyboardInterrupt:
self.abort()
raise
except TooManyErrors:
# too many failures, abort the execution
logger.error("Too many errors, aborting")
self.abort()
def abort(self):
"""Abort the loop."""
# First force release of all elements to ensure that iteration
# on the remaining DAG elements won't be blocked
for job_id in self.locked_items:
if job_id is not None:
self.item_list.release(job_id)
# Wait for worker still active if necessary
if self.abort_file is not None and os.path.isfile(self.abort_file):
for worker in self.workers:
if worker is not None:
worker.wait()
# Mark remaining jobs as skipped
for job_id, job_list in self.iterator:
self.item_list.release(job_id)
if not isinstance(job_list, list):
job_list = [job_list]
for job in job_list:
self.collect_result(job, SKIP_EXECUTION, None)
def test_dag_str():
d = DAG()
d.add_vertex('a')
d.add_vertex('b')
d.update_vertex('a', predecessors=['b'])
assert str(d)
