def test_job_never_ready(self, walk_class, setup_sbx): """Trying to run a job repeatedly returning notready.""" actions = DAG() actions.add_vertex("1.notready:always") c = walk_class(actions) job = c.saved_jobs["1.notready:always"] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue.notready assert c.job_status == {"1.notready:always": ReturnValue.notready} assert c.requeued == {"1.notready:always": 3} # In the situation where we are using fingerprints, # verify the behavior when re-doing a walk with # the same DAG. if walk_class == FingerprintWalk: r2 = walk_class(actions) job = r2.saved_jobs["1.notready:always"] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue.notready assert r2.job_status == {"1.notready:always": ReturnValue.notready} assert r2.requeued == {"1.notready:always": 3}
def test_requeue(self): """Requeue test. Same as previous example except that all tests are requeued once. """ results = {} def collect(job): if job.uid not in results: results[job.uid] = True return True else: return False # This time test with two interdependent jobs dag = DAG() dag.add_vertex('1') dag.add_vertex('2') s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2, collect=collect) s.run(dag) assert s.max_active_jobs == 2 assert results['1'] assert results['2']
def test_cycle_detection(): d = DAG() d.add_vertex('a') d.add_vertex('b') d.update_vertex('a', predecessors=['b']) with pytest.raises(DAGError): d.update_vertex('b', data='newb', predecessors=['a']) # Ensure that DAG is still valid and that previous # update_vertex has no effect result = [] for vertex_id, data in d: result.append(vertex_id) assert data is None assert result == ['b', 'a'] # Force the creation of a cycle d.update_vertex('b', data='newb', predecessors=['a'], enable_checks=False) # Verify that the cycle is detected with pytest.raises(DAGError): d.check() # Verify that some functions do not hang when a cycle is present assert len(d.get_closure('b')) == 2 assert str(d) assert d.as_dot() with pytest.raises(DAGError): d.reverse_graph()
def test_job_depending_on_job_with_no_predicted_fingerprint_failed(setup_sbx): """Test case where job depends on failed job with late fingerprint.""" actions = DAG() actions.add_vertex("fingerprint_after_job.bad") actions.add_vertex("2", predecessors=["fingerprint_after_job.bad"]) r1 = FingerprintWalk(actions) assert ( r1.compute_fingerprint("fingerprint_after_job.bad", None, is_prediction=True) is None ) # Check the status of the first job ('fingerprint_after_job.bad'). # It should be a real job that returned a failure. job = r1.saved_jobs["fingerprint_after_job.bad"] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue(1) # Check the status of the second job ('2'); because that job depends # on a job that failed, it should show that the job was skipped. job = r1.saved_jobs["2"] assert isinstance(job, EmptyJob) assert job.should_skip is True assert job.status == ReturnValue.force_fail # Check that no job was requeued. assert r1.requeued == {}
def test_keyboard_interrupt(self): """Ensure that jobs can be interrupted.""" results = {} pytest.importorskip('psutil') def get_job(uid, data, predecessors, notify_end): return NopJob(uid, data, notify_end) def collect(job): results[job.uid] = job dag = DAG() dag.add_vertex('1') dag.add_vertex('2') s = Scheduler(get_job, tokens=2, collect=collect, job_timeout=2) # fake log_state that will raise a KeyboardInterrupt def fake_log_state(): raise KeyboardInterrupt s.log_state = fake_log_state with pytest.raises(KeyboardInterrupt): s.run(dag) for k, v in results.items(): assert v.interrupted
def test_job_not_ready_then_ok(self, walk_class, setup_sbx): """Rerunning a job that first returned notready.""" actions = DAG() actions.add_vertex("1.notready:once") c = walk_class(actions) job = c.saved_jobs["1.notready:once"] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue.success assert c.job_status == {"1.notready:once": ReturnValue.success} assert c.requeued == {"1.notready:once": 1} # In the situation where we are using fingerprints, # verify the behavior when re-doing a walk with # the same DAG. if walk_class == FingerprintWalk: r2 = walk_class(actions) job = r2.saved_jobs["1.notready:once"] assert isinstance(job, EmptyJob) assert job.should_skip is True assert job.status == ReturnValue.skip assert r2.job_status == {"1.notready:once": ReturnValue.skip} assert r2.requeued == {}
def test_bad_job_no_predecessors(self, walk_class, setup_sbx): """Simple case of a leaf job failing.""" actions = DAG() actions.add_vertex("1.bad") c = walk_class(actions) job = c.saved_jobs["1.bad"] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue(1) assert c.job_status == {"1.bad": ReturnValue(1)} assert c.requeued == {} # In the situation where we are using fingerprints, # verify the behavior when re-doing a walk with # the same DAG. if walk_class == FingerprintWalk: r2 = walk_class(actions) job = r2.saved_jobs["1.bad"] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue(1) assert r2.job_status == {"1.bad": ReturnValue(1)} assert r2.requeued == {}
def test_minimal_run2(self): """Test with two interdependent jobs.""" dag = DAG() dag.add_vertex('1') dag.add_vertex('2', predecessors=['1']) s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2) s.run(dag) assert s.max_active_jobs == 1
def test_minimal_run(self): """Test with only two independent jobs.""" dag = DAG() dag.add_vertex('1') dag.add_vertex('2') s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2) s.run(dag) assert s.max_active_jobs == 2
def test_iter_with_busy_state(): d = DAG() d.add_vertex('a') d.add_vertex('b', predecessors=['a']) it = DAGIterator(d, enable_busy_state=True) for nid, data in it: if nid is None: it.leave('a')
def test_iter_with_busy_state(): d = DAG() d.add_vertex("a") d.add_vertex("b", predecessors=["a"]) it = DAGIterator(d, enable_busy_state=True) for nid, _ in it: if nid is None: it.leave("a")
def test_inexisting(): d = DAG() d.add_vertex("a") assert "a" in d d.update_vertex("a", data="NOT B", predecessors=["b"], enable_checks=False) assert "b" not in d assert d["a"] == "NOT B" with pytest.raises(DAGError): d.check()
def test_inexisting(): d = DAG() d.add_vertex('a') assert 'a' in d d.update_vertex('a', data='NOT B', predecessors=['b'], enable_checks=False) assert 'b' not in d assert d['a'] == 'NOT B' with pytest.raises(DAGError): d.check()
def test_simple_dag(): d = DAG() d.add_vertex('a') d.add_vertex('b') d.add_vertex('c') result = [] for vertex_id, data in d: result.append(vertex_id) result.sort() assert result == ['a', 'b', 'c']
def test_simple_dag(): d = DAG() d.add_vertex("a") d.add_vertex("b") d.add_vertex("c") result = [] for vertex_id, _ in d: result.append(vertex_id) result.sort() assert result == ["a", "b", "c"] assert d.check() is None
def test_cycle(): d = DAG() d.add_vertex('a') d.add_vertex('b') d.update_vertex('a', predecessors=['b']) d.add_vertex('c', predecessors=['b']) d.update_vertex('b', predecessors=['c'], enable_checks=False) with pytest.raises(DAGError): d.check() with pytest.raises(DAGError): d.get_context('b')
def test_cycle(): d = DAG() d.add_vertex("a") d.add_vertex("b") d.update_vertex("a", predecessors=["b"]) d.add_vertex("c", predecessors=["b"]) d.update_vertex("b", predecessors=["c"], enable_checks=False) with pytest.raises(DAGError): d.check() with pytest.raises(DAGError): d.get_context("b")
def test_ordering(self): """Test that jobs are ordered correctly.""" results = [] def collect(job): results.append(job.uid) dag = DAG() dag.add_vertex("3") dag.add_vertex("0") dag.add_vertex("1") s = Scheduler(Scheduler.simple_provider(NopJob), tokens=1, collect=collect) s.run(dag) assert tuple(results) == ("0", "1", "3")
def test_cycle_detection(): d = DAG() d.add_vertex('a') d.add_vertex('b') d.update_vertex('a', predecessors=['b']) with pytest.raises(DAGError): d.update_vertex('b', data='newb', predecessors=['a']) # Ensure that DAG is still valid and that previous # update_vertex has no effect result = [] for vertex_id, data in d: result.append(vertex_id) assert data is None assert result == ['b', 'a']
def test_add_vertex(): d = DAG() # add_vertex should fail in case a dep does not exist with pytest.raises(DAGError): d.add_vertex("a", predecessors=["b"]) # check order of a iteration with simple dependency d.add_vertex("b") d.add_vertex("a", predecessors=["b"]) result = [] for vertex_id, _ in d: result.append(vertex_id) assert result == ["b", "a"] # check that add_vertex fails on attempt to add already existing nodde with pytest.raises(DAGError): d.add_vertex("a") # check update with new dependency d.add_vertex("c") d.update_vertex("b", predecessors=["c"]) assert d.get_predecessors("b") == frozenset(["c"]) assert d.vertex_predecessors == { "a": frozenset(["b"]), "b": frozenset(["c"]), "c": frozenset([]), } result = [] for vertex_id, _ in d: result.append(vertex_id) assert result == ["c", "b", "a"] d.update_vertex("a", data="datafora_") d.update_vertex("c", data="dataforc_") result = [] compound_data = "" for vertex_id, data in d: if data is not None: compound_data += data result.append(vertex_id) assert result == ["c", "b", "a"] assert compound_data == "dataforc_datafora_"
def test_add_vertex(): d = DAG() # add_vertex should fail in case a dep does not exist with pytest.raises(DAGError): d.add_vertex('a', predecessors=['b']) # check order of a iteration with simple dependency d.add_vertex('b') d.add_vertex('a', predecessors=['b']) result = [] for vertex_id, data in d: result.append(vertex_id) assert result == ['b', 'a'] # check that add_vertex fails on attempt to add already existing nodde with pytest.raises(DAGError): d.add_vertex('a') # check update with new dependency d.add_vertex('c') d.update_vertex('b', predecessors=['c']) assert d.get_predecessors('b') == frozenset(['c']) assert d.vertex_predecessors == { 'a': frozenset(['b']), 'b': frozenset(['c']), 'c': frozenset([]) } result = [] for vertex_id, data in d: result.append(vertex_id) assert result == ['c', 'b', 'a'] d.update_vertex('a', data='datafora_') d.update_vertex('c', data='dataforc_') result = [] compound_data = '' for vertex_id, data in d: if data is not None: compound_data += data result.append(vertex_id) assert result == ['c', 'b', 'a'] assert compound_data == 'dataforc_datafora_'
def test_do_nothing_job(self, walk_class, setup_sbx): """Test DAG leading us to create a DoNothingJob object.""" actions = DAG() actions.add_vertex("1.do-nothing") actions.add_vertex("2", predecessors=["1.do-nothing"]) c = walk_class(actions) job = c.saved_jobs["1.do-nothing"] assert isinstance(job, DoNothingJob) assert job.should_skip is False assert job.status == ReturnValue.success job = c.saved_jobs["2"] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue.success assert c.job_status == { "1.do-nothing": ReturnValue.success, "2": ReturnValue.success, } assert c.requeued == {} # In the situation where we are using fingerprints, # verify the behavior when re-doing a walk with # the same DAG. if walk_class == FingerprintWalk: r2 = walk_class(actions) job = r2.saved_jobs["1.do-nothing"] assert isinstance(job, EmptyJob) assert job.should_skip is True assert job.status == ReturnValue.skip job = r2.saved_jobs["2"] assert isinstance(job, EmptyJob) assert job.should_skip is True assert job.status == ReturnValue.skip assert r2.job_status == { "1.do-nothing": ReturnValue.skip, "2": ReturnValue.skip, } assert r2.requeued == {}
def test_failed_predecessor(self, walk_class, setup_sbx): """Simulate the scenarior when a predecessor failed.""" actions = DAG() actions.add_vertex("1.bad") actions.add_vertex("2", predecessors=["1.bad"]) c = walk_class(actions) job = c.saved_jobs["1.bad"] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue(1) job = c.saved_jobs["2"] assert isinstance(job, EmptyJob) assert job.should_skip is True assert job.status == ReturnValue.force_fail assert c.job_status == { "1.bad": ReturnValue(1), "2": ReturnValue.force_fail } assert c.requeued == {} # In the situation where we are using fingerprints, # verify the behavior when re-doing a walk with # the same DAG. if walk_class == FingerprintWalk: r2 = walk_class(actions) job = r2.saved_jobs["1.bad"] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue(1) job = r2.saved_jobs["2"] assert isinstance(job, EmptyJob) assert job.should_skip is True assert job.status == ReturnValue.force_fail assert r2.job_status == { "1.bad": ReturnValue(1), "2": ReturnValue.force_fail, } assert r2.requeued == {}
def test_timeout(self): """Ensure that jobs are interrupted correctly on timeout.""" results = {} pytest.importorskip('psutil') def get_job(uid, data, predecessors, notify_end): return SleepJob(uid, data, notify_end) def collect(job): results[job.uid] = job dag = DAG() dag.add_vertex('1') dag.add_vertex('2') s = Scheduler(get_job, tokens=2, collect=collect, job_timeout=2) s.run(dag) for k, v in results.items(): assert v.interrupted
def test_shortest_path(): d = DAG() d.add_vertex("a") d.add_vertex("b") d.update_vertex("a", predecessors=["b"]) d.update_vertex("b", predecessors=["a"], enable_checks=False) assert d.shortest_path("a", "a") == ["a", "b", "a"] d = DAG() d.add_vertex("a") d.update_vertex("a", predecessors=["a"], enable_checks=False) assert d.shortest_path("a", "a") == ["a", "a"] d = DAG() d.add_vertex("a") d.add_vertex("b", predecessors=["a"]) d.add_vertex("c", predecessors=["b"]) d.add_vertex("d", predecessors=["c", "a"]) assert d.shortest_path("a", "d") == ["a", "d"] assert d.shortest_path("d", "a") is None
def test_shortest_path(): d = DAG() d.add_vertex('a') d.add_vertex('b') d.update_vertex('a', predecessors=['b']) d.update_vertex('b', predecessors=['a'], enable_checks=False) assert d.shortest_path('a', 'a') == ['a', 'b', 'a'] d = DAG() d.add_vertex('a') d.update_vertex('a', predecessors=['a'], enable_checks=False) assert d.shortest_path('a', 'a') == ['a', 'a'] d = DAG() d.add_vertex('a') d.add_vertex('b', predecessors=['a']) d.add_vertex('c', predecessors=['b']) d.add_vertex('d', predecessors=['c', 'a']) assert d.shortest_path('a', 'd') == ['a', 'd'] assert d.shortest_path('d', 'a') is None
def test_add_vertex(): d = DAG() # add_vertex should fail in case a dep does not exist with pytest.raises(DAGError): d.add_vertex('a', predecessors=['b']) # check order of a iteration with simple dependency d.add_vertex('b') d.add_vertex('a', predecessors=['b']) result = [] for vertex_id, data in d: result.append(vertex_id) assert result == ['b', 'a'] # check that add_vertex fails on attempt to add already existing nodde with pytest.raises(DAGError): d.add_vertex('a') # check update with new dependency d.add_vertex('c') d.update_vertex('b', predecessors=['c']) result = [] for vertex_id, data in d: result.append(vertex_id) assert result == ['c', 'b', 'a'] d.update_vertex('a', data='datafora_') d.update_vertex('c', data='dataforc_') result = [] compound_data = '' for vertex_id, data in d: if data is not None: compound_data += data result.append(vertex_id) assert result == ['c', 'b', 'a'] assert compound_data == 'dataforc_datafora_'
def test_dag_merge(): d = DAG() d.add_vertex('b') d.add_vertex('a', predecessors=['b']) d2 = DAG() d2.add_vertex('c') d2.add_vertex('b', predecessors=['c']) d2.add_vertex('a', predecessors=['c']) d3 = d | d2 result = [] for vertex_id, data in d3: result.append(vertex_id) assert data is None assert result == ['c', 'b', 'a']
def test_dag_merge(): d = DAG() d.add_vertex("b") d.add_vertex("a", predecessors=["b"]) d2 = DAG() d2.add_vertex("c") d2.add_vertex("b", predecessors=["c"]) d2.add_vertex("a", predecessors=["c"]) d3 = d | d2 result = [] for vertex_id, data in d3: result.append(vertex_id) assert data is None assert result == ["c", "b", "a"]
def test_skip(self): """Simple example in which all the tests are skipped.""" results = {} def get_job(uid, data, predecessors, notify_end): result = NopJob(uid, data, notify_end) result.should_skip = True return result def collect(job): results[job.uid] = job.timing_info # This time test with two interdependent jobs dag = DAG() dag.add_vertex('1') dag.add_vertex('2') s = Scheduler(get_job, tokens=2, collect=collect) s.run(dag) # Check start_time end_time to be sure tests have not been run for k, v in results.items(): assert v.start_time is None assert v.stop_time is None
def test_predecessor_with_no_fingerprint(setup_sbx): actions = DAG() actions.add_vertex("1") actions.add_vertex("2.no_fingerprint", predecessors=["1"]) actions.add_vertex("3", predecessors=["2.no_fingerprint"]) actions.add_vertex("4", predecessors=["3"]) # Execute our planned actions for the first time... r1 = FingerprintWalk(actions) for uid in ("1", "2.no_fingerprint", "3", "4"): job = r1.saved_jobs[uid] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue.success assert r1.job_status == { "1": ReturnValue.success, "2.no_fingerprint": ReturnValue.success, "3": ReturnValue.success, "4": ReturnValue.success, } assert r1.requeued == {} # Re-execute the plan a second time. Because '2.no_fingerprint' # has no fingerprint, both '2.no_fingerprint', but also the node # that depends directly on it should be re-executed. Node '4, # on the other hand, should only be re-executed if Node "3"'s # fingerprint changed. The way things are set up in this testsuite # is such that the fingerprint remained the same, so '4' is not # expected to be re-run. r2 = FingerprintWalk(actions) for uid in ("1", "4"): job = r2.saved_jobs[uid] assert isinstance(job, EmptyJob) assert job.should_skip is True assert job.status == ReturnValue.skip for uid in ("2.no_fingerprint", "3"): job = r2.saved_jobs[uid] assert isinstance(job, ControlledJob) assert job.should_skip is False assert job.status == ReturnValue.success assert r2.job_status == { "1": ReturnValue.skip, "2.no_fingerprint": ReturnValue.success, "3": ReturnValue.success, "4": ReturnValue.skip, } assert r2.requeued == {}
def schedule(self, resolver): """Compute a DAG of scheduled actions. :param resolver: a function that helps the scheduler resolve cases for which a decision should be taken :type resolver: (Action, Decision) -> bool """ rev = self.tree.reverse_graph() uploads = [] dag = DAG() for uid, action in rev: if uid == 'root': # Root node is alway in the final DAG dag.add_vertex(uid, action) elif isinstance(action, Decision): # Decision node does not appears in the final DAG but we need # to apply the triggers based on the current list of scheduled # actions. action.apply_triggers(dag) elif isinstance(action, UploadComponent): uploads.append((action, self.tree.vertex_predecessors[uid])) else: # Compute the list of successors for the current node (i.e: # predecessors in the reversed graph). Ignore UploadComponent # nodes as they will be processed only once the scheduling # is done. preds = list([k for k in rev.vertex_predecessors[uid] if not isinstance(rev[k], UploadComponent)]) if len(preds) == 1 and isinstance(rev[preds[0]], Decision): decision = rev[preds[0]] # The current node addition is driven by a decision # First check that the parent of the decision is # scheduled. If not discard the item. if decision.initiator not in dag: continue # Now check decision made. If the decision cannot be made # delegate to the resolve function. choice = decision.get_decision() if choice == uid: dag.add_vertex(uid, action) dag.update_vertex(decision.initiator, predecessors=[uid], enable_checks=False) elif choice is None: # delegate to resolver try: if resolver(action, decision): dag.add_vertex(uid, action) dag.update_vertex(decision.initiator, predecessors=[uid], enable_checks=False) except SchedulingError as e: # In order to help the analysis of a scheduling # error compute the explicit initiators of that # action dag.add_vertex(uid, action) dag.update_vertex(decision.initiator, predecessors=[action.uid], enable_checks=False) rev_graph = dag.reverse_graph() # Initiators are explicit actions (connected to # 'root') that are in the closure of the failing # node. initiators = [ iuid for iuid in rev_graph.get_closure(uid) if 'root' in rev_graph.vertex_predecessors[iuid]] raise SchedulingError(e.message, uid=uid, initiators=initiators) else: # An action is scheduled only if one of its successors is # scheduled. successors = [k for k in preds if k in dag] if successors: dag.add_vertex(uid, action) for a in successors: dag.update_vertex(a, predecessors=[uid], enable_checks=False) # Handle UploadComponent nodes. Add the node only if all predecessors # are scheduled. for action, predecessors in uploads: if len([p for p in predecessors if p not in dag]) == 0: dag.update_vertex(action.uid, action, predecessors=predecessors, enable_checks=False) # connect upload to the root node dag.update_vertex('root', predecessors=[action.uid]) return dag
def test_dag_len(): d = DAG() d.add_vertex('a') d.add_vertex('b') d.update_vertex('a', predecessors=['b']) assert len(d) == 2
class MainLoop(object): """Run a list of jobs.""" def __init__(self, item_list, run_job, collect_result, parallelism=None, abort_file=None, dyn_poll_interval=True): """Launch loop. :param item_list: a list of jobs or a dag :param run_job: a function that takes a job for argument and return the spawned process (:class:`e3.os_process.Run` object). Its prototype should be ``func (name, job_info)`` with name the job identifier and job_info the related information, passed in a tuple (slot_number, job_retry). Note that if you want to take advantage of the parallelism the spawned process should be launched in background (ie with bg=True when using :class:`e3.os_process.Run`). If run_job returns SKIP_EXECUTION instead of a Run object the mainloop will directly call collect_result without waiting. :param collect_result: a function called when a job is finished. The prototype should be func (name, process, job_info). If collect_result raise NeedRequeue then the test will be requeued. job_info is a tuple: (slot_number, job_nb_retry) :param parallelism: number of workers :type parallelism: int | None :param abort_file: If specified, the loop will abort if the file is present :type abort_file: str | None :param dyn_poll_interval: If True the interval between each polling iteration is automatically updated. Otherwise it's set to 0.1 seconds. :type dyn_poll_interval: bool """ e = e3.env.Env() self.parallelism = e.get_attr("main_options.mainloop_jobs", default_value=1, forced_value=parallelism) self.abort_file = e.get_attr("main_options.mainloop_abort_file", default_value=None, forced_value=abort_file) if self.parallelism == 0: if e.build.cpu.cores != UNKNOWN: self.parallelism = e.build.cpu.cores else: self.parallelism = 1 e3.log.debug("start main loop with %d workers (abort on %s)", self.parallelism, self.abort_file) self.workers = [None] * self.parallelism self.locked_items = [None] * self.parallelism if not isinstance(item_list, DAG): self.item_list = DAG() for index, item in enumerate(item_list): self.item_list.add_vertex(str(index), item) else: self.item_list = item_list self.iterator = DAGIterator(self.item_list, enable_busy_state=True) self.collect_result = collect_result active_workers = 0 max_active_workers = self.parallelism poll_sleep = 0.1 no_free_item = False try: while True: # Check for abortion if self.abort_file is not None and \ os.path.isfile(self.abort_file): logger.info('Aborting: file %s has been found', self.abort_file) self.abort() return # Exit the loop # Find free workers for slot, worker in enumerate(self.workers): if worker is None: # a worker slot is free so use it for next job next_id, next_job = next(self.iterator) if next_job is None: no_free_item = True break else: self.locked_items[slot] = next_id self.workers[slot] = Worker(next_job, run_job, collect_result, slot) active_workers += 1 poll_counter = 0 e3.log.debug('Wait for free worker') while active_workers >= max_active_workers or no_free_item: # All worker are occupied so wait for one to finish poll_counter += 1 for slot, worker in enumerate(self.workers): if worker is None: continue # Test if the worker is still active and have more # job pending if not (worker.poll() or worker.execute_next()): # If not the case free the worker slot active_workers -= 1 self.workers[slot] = None self.iterator.leave(self.locked_items[slot]) no_free_item = False self.locked_items[slot] = None sleep(poll_sleep) if dyn_poll_interval: poll_sleep = compute_next_dyn_poll(poll_counter, poll_sleep) except (StopIteration, KeyboardInterrupt) as e: if e.__class__ == KeyboardInterrupt: # Got ^C, abort the mainloop logger.error("User interrupt") # All the jobs are finished while active_workers > 0: for slot, worker in enumerate(self.workers): if worker is None: continue # Test if the worker is still active and ignore any # job pending try: still_running = worker.poll() except TooManyErrors: still_running = False # We're not spawing more jobs so we can safely # ignore all TooManyErrors exceptions. if not still_running: active_workers -= 1 self.workers[slot] = None sleep(0.1) if e.__class__ == KeyboardInterrupt: self.abort() raise except TooManyErrors: # too many failures, abort the execution logger.error("Too many errors, aborting") self.abort() def abort(self): """Abort the loop.""" # First force release of all elements to ensure that iteration # on the remaining DAG elements won't be blocked for job_id in self.locked_items: if job_id is not None: self.item_list.release(job_id) # Wait for worker still active if necessary if self.abort_file is not None and os.path.isfile(self.abort_file): for worker in self.workers: if worker is not None: worker.wait() # Mark remaining jobs as skipped for job_id, job_list in self.iterator: self.item_list.release(job_id) if not isinstance(job_list, list): job_list = [job_list] for job in job_list: self.collect_result(job, SKIP_EXECUTION, None)
def test_dag_str(): d = DAG() d.add_vertex('a') d.add_vertex('b') d.update_vertex('a', predecessors=['b']) assert str(d)