def test_scheduling_time(): test_graph = TaskGraph() a = test_graph.new_task("A", duration=3, output_size=1) b = test_graph.new_task("B", duration=1, output_size=1) c = test_graph.new_task("C", duration=1, output_size=1) d = test_graph.new_task("D", duration=1, output_size=1) b.add_input(a) c.add_input(b) d.add_input(c) times = [] class Scheduler(SchedulerBase): def schedule(self, update): if not self.task_graph.tasks: return simulator = self._simulator times.append(simulator.env.now) for t in update.new_ready_tasks: self.assign(self.workers[0], t) scheduler = Scheduler("x", "0") simulator = do_sched_test( test_graph, 1, scheduler, SimpleNetModel(bandwidth=2), scheduling_time=2, return_simulator=True) runtime_state = simulator.runtime_state assert times == [0, 5, 8, 11, 14] assert runtime_state.task_info(a).end_time == 5 assert runtime_state.task_info(b).end_time == 8 assert runtime_state.task_info(c).end_time == 11 assert runtime_state.task_info(d).end_time == 14
def test_trace_task_fetch(): tg = TaskGraph() a = tg.new_task(output_size=5, duration=2) b = tg.new_task(output_size=3, duration=3) b.add_input(a) c = tg.new_task(duration=4) c.add_input(b) simulator = do_sched_test(tg, [1, 1], fixed_scheduler([ (0, a, 0), (1, b, 0), (0, c, 0) ]), netmodel=SimpleNetModel(1), trace=True, return_simulator=True) workers = simulator.workers fetch_start_events = [e for e in simulator.trace_events if isinstance(e, FetchStartTraceEvent)] assert fetch_start_events == [ FetchStartTraceEvent(2, workers[1], workers[0], a.output), FetchStartTraceEvent(10, workers[0], workers[1], b.output), ] fetch_end_events = [e for e in simulator.trace_events if isinstance(e, FetchEndTraceEvent)] assert fetch_end_events == [ FetchEndTraceEvent(7, workers[1], workers[0], a.output), FetchEndTraceEvent(13, workers[0], workers[1], b.output), ]
def test_simulator_reschedule_scheduled_download(): test_graph = TaskGraph() s = [test_graph.new_task("S{}".format(i), duration=0, cpus=1, output_size=10) for i in range(10)] a1 = test_graph.new_task("A1", duration=10, cpus=1) b = test_graph.new_task("B", duration=1, cpus=1) c = test_graph.new_task("C", duration=2, cpus=1) a1.add_inputs(s) assignments = [ [(0, x, 100) for x in s] + [ (1, a1, 0), (0, b, 10), (1, c, 10), ], [], [(2, a1, 0), ] ] scheduler = fixed_scheduler(assignments, steps=True, reassigning=True) scheduler._disable_cleanup = True simulator = do_sched_test(test_graph, [1, 1, 1], scheduler, trace=True, return_simulator=True, netmodel=SimpleNetModel(1)) assert simulator.env.now > 40 available = set() for x in s: available.add( frozenset(w.worker_id for w in scheduler.task_graph.objects[x.output.id].availability)) assert frozenset({0, 1, 2}) in available assert frozenset({0, 2}) in available assert simulator.runtime_state.task_info(a1).assigned_workers == [simulator.workers[2]]
def test_worker_max_downloads_per_worker(): g = TaskGraph() a = g.new_task("a", duration=0, outputs=[1, 1, 1, 1]) b = g.new_task("b", duration=0) b.add_inputs(a.outputs) s = fixed_scheduler([ (0, a, 0), (1, b, 0), ]) assert do_sched_test(g, [1, 1], s, SimpleNetModel()) == 2 assert do_sched_test( g, [Worker(), Worker(max_downloads_per_worker=1)], s, SimpleNetModel()) == 4 assert do_sched_test( g, [Worker(), Worker(max_downloads_per_worker=2)], s, SimpleNetModel()) == 2 assert do_sched_test( g, [Worker(), Worker(max_downloads_per_worker=3)], s, SimpleNetModel()) == 2 assert do_sched_test( g, [Worker(), Worker(max_downloads_per_worker=4)], s, SimpleNetModel()) == 1 assert do_sched_test( g, [Worker(), Worker(max_downloads_per_worker=5)], s, SimpleNetModel()) == 1
def test_worker_priority_block(): g = TaskGraph() a = g.new_task("a", duration=1) b = g.new_task("b", duration=1, cpus=3) c = g.new_task("c", duration=1) s = fixed_scheduler([(0, a, 3), (0, b, 2), (0, c, 1)]) w = [Worker(cpus=3)] simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True) runtime_state = simulator.runtime_state assert runtime_state.task_info(a).end_time == pytest.approx(1) assert runtime_state.task_info(b).end_time == pytest.approx(2) assert runtime_state.task_info(c).end_time == pytest.approx(1) s = fixed_scheduler([(0, a, 3), (0, b, 2, 2), (0, c, 1)]) w = [Worker(cpus=3)] simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True) runtime_state = simulator.runtime_state assert runtime_state.task_info(a).end_time == pytest.approx(1) assert runtime_state.task_info(b).end_time == pytest.approx(2) assert runtime_state.task_info(c).end_time == pytest.approx(3)
def test_worker_running_tasks(): test_graph = TaskGraph() test_graph.new_task("X", duration=10) a = test_graph.new_task("A", duration=1, output_size=1) b = test_graph.new_task("B", duration=8, output_size=1) b.add_input(a) remaining_times = [] class Scheduler(SchedulerBase): scheduled = False def schedule(self, update): if not self.task_graph.tasks: return simulator = self._simulator remaining_times.append([[ t.remaining_time(simulator.env.now) for t in w.running_tasks.values() ] for w in simulator.workers]) if not self.scheduled: tasks = self.task_graph.tasks self.scheduled = True self.assign(self.workers[0], tasks[0]) self.assign(self.workers[1], tasks[1]) self.assign(self.workers[1], tasks[2]) else: return () scheduler = Scheduler("x", "0") do_sched_test(test_graph, 2, scheduler) assert remaining_times == [[[], []], [[9], []], [[1], []], [[], []]]
def test_trace_task_execution(): tg = TaskGraph() a = tg.new_task(output_size=0, duration=2) b = tg.new_task(output_size=0, duration=3) b.add_input(a) c = tg.new_task(duration=4) c.add_input(b) simulator = do_sched_test(tg, [1, 1], fixed_scheduler([ (0, a, 0), (1, b, 0), (0, c, 0) ]), trace=True, return_simulator=True) start_events = [e for e in simulator.trace_events if isinstance(e, TaskStartTraceEvent)] assert start_events == [ TaskStartTraceEvent(0, simulator.workers[0], a), TaskStartTraceEvent(2, simulator.workers[1], b), TaskStartTraceEvent(5, simulator.workers[0], c) ] end_events = [e for e in simulator.trace_events if isinstance(e, TaskEndTraceEvent)] assert end_events == [ TaskEndTraceEvent(2, simulator.workers[0], a), TaskEndTraceEvent(5, simulator.workers[1], b), TaskEndTraceEvent(9, simulator.workers[0], c) ]
def test_simulator_no_events(): task_graph = TaskGraph() task_graph.new_task("A", duration=1) scheduler = DoNothingScheduler() with pytest.raises(RuntimeError): do_sched_test(task_graph, 1, scheduler)
def test_simulator_cpus1(): test_graph = TaskGraph() test_graph.new_task("A", duration=1, cpus=1) test_graph.new_task("B", duration=1, cpus=2) scheduler = AllOnOneScheduler() assert do_sched_test(test_graph, [2], scheduler) == 2 scheduler = AllOnOneScheduler() assert do_sched_test(test_graph, [3], scheduler) == 1
def test_more_outputs_from_same_source(): test_graph = TaskGraph() a = test_graph.new_task("A", duration=1, outputs=[1, 1, 1]) b = test_graph.new_task("B", duration=1) b.add_input(a.outputs[0]) b.add_input(a.outputs[2]) s = fixed_scheduler([ (0, a, 0), (0, b, 0), ]) assert do_sched_test(test_graph, [1], s) == 2
def test_simulator_local_reassign(): test_graph = TaskGraph() a0 = test_graph.new_task("A0", duration=1, output_size=1) a1 = test_graph.new_task("A1", duration=1, output_size=1) a2 = test_graph.new_task("A2", duration=1, cpus=1, output_size=10) a2.add_inputs([a1, a0]) class Scheduler(SchedulerBase): def start(self): self.done = False return super().start() def schedule(self, update): if not self.task_graph.tasks or self.done: return t = self.task_graph.tasks[a2.id] for o in t.inputs: assert not o.scheduled for o in t.outputs: assert not o.scheduled w1 = self.workers[1] w2 = self.workers[2] self.assign(w1, t) for o in t.inputs: assert o.scheduled == {w1} for o in t.outputs: assert o.scheduled == {w1} self.assign(w2, t) for o in t.inputs: assert o.scheduled == {w2} for o in t.outputs: assert o.scheduled == {w2} for t in self.task_graph.tasks.values(): self.assign(w1, t) self.done = True scheduler = Scheduler("test", "0", True) do_sched_test(test_graph, [1, 1, 1], scheduler, trace=True, netmodel=SimpleNetModel(1))
def plan_reverse_cherry1(): """ a1/10/1 a2/10/1 \ / \ / a3 """ # noqa task_graph = TaskGraph() a1 = task_graph.new_task("a1", 10, 1) a2 = task_graph.new_task("a2", 10, 1) a3 = task_graph.new_task("a3", 1) a3.add_input(a1) a3.add_input(a2) return task_graph
def test_simulator_task_start_notify(): test_graph = TaskGraph() a0 = test_graph.new_task("A0", duration=1, output_size=1) a1 = test_graph.new_task("A1", duration=10, cpus=1, output_size=1) a2 = test_graph.new_task("A2", duration=10, cpus=1, output_size=1) a3 = test_graph.new_task("A3", duration=3, cpus=1) a1.add_input(a0) a2.add_input(a1) triggered = [False, False] class Scheduler(SchedulerBase): def start(self): self.step = 0 return super().start() def schedule(self, update): def tg(task): return self.task_graph.tasks[task.id] if not self.task_graph.tasks: return self.step += 1 if self.step == 1: self.assign(self.workers[0], tg(a0)) self.assign(self.workers[0], tg(a1)) self.assign(self.workers[0], tg(a2)) self.assign(self.workers[1], tg(a3)) elif tg(a3) in update.new_started_tasks: assert not triggered[0] and not triggered[1] triggered[0] = True assert tg(a3).running elif tg(a3).state == TaskState.Finished and tg(a3) in update.new_finished_tasks: assert triggered[0] assert not triggered[1] triggered[1] = True assert not tg(a0).running assert tg(a1).running assert not tg(a2).running assert not tg(a3).running scheduler = Scheduler("test", "0", task_start_notification=True) do_sched_test(test_graph, [1, 1, 1], scheduler, trace=True, netmodel=SimpleNetModel(1)) assert triggered[1] and triggered[0]
def test_random_levels(): graph = TaskGraph() random_levels([3, 10, 5, 1], [0, 3, 2, 3], lambda: graph.new_task(output_size=1)) check_graph(graph) assert graph.task_count == 19 assert len(list(graph.arcs)) == 43
def test_worker_estimate_earliest_time_offset_now(): now = 0 tg = TaskGraph() t0 = tg.new_task(expected_duration=3, cpus=1) t1 = tg.new_task(expected_duration=5, cpus=1) t2 = tg.new_task(expected_duration=3, cpus=2) tg = create_scheduler_graph(tg) tg.tasks[t0.id].start_time = now tg.tasks[t1.id].start_time = now worker = SchedulerWorker(0, cpus=2) worker.scheduled_tasks = [] worker.running_tasks.update((tg.tasks[t0.id], tg.tasks[t1.id])) assert worker_estimate_earliest_time(worker, tg.tasks[t2.id], now + 2) == 3
def test_compute_b_level_multiple_outputs(): tg = TaskGraph() a = tg.new_task(outputs=[2, 4], expected_duration=0) b = tg.new_task(outputs=[5], expected_duration=0) c = tg.new_task(outputs=[2], expected_duration=0) d = tg.new_task(expected_duration=0) b.add_input(a.outputs[0]) c.add_input(a.outputs[1]) d.add_inputs((b, c)) blevel = compute_b_level_duration_size(tg, get_size_estimate) assert blevel[a] == 7 assert blevel[b] == 5 assert blevel[c] == 2 assert blevel[d] == 0
def test_simulator_reassign_failed(): test_graph = TaskGraph() a0 = test_graph.new_task("A0", duration=1, output_size=1) a1 = test_graph.new_task("A1", duration=5, cpus=1) a2 = test_graph.new_task("A2", duration=3, cpus=1, output_size=1) a3 = test_graph.new_task("A3", duration=1, cpus=1) a1.add_input(a0) a3.add_input(a2) test_update = [] class Scheduler(SchedulerBase): def start(self): self.step = 0 return super().start() def schedule(self, update): def tg(task): return self.task_graph.tasks[task.id] if not self.task_graph.tasks: return self.step += 1 if self.step == 1: self.assign(self.workers[3], tg(a0)) self.assign(self.workers[0], tg(a1)) self.assign(self.workers[1], tg(a2), 10) self.assign(self.workers[1], tg(a3), 1) elif self.step == 4: self.assign(self.workers[2], tg(a1)) elif self.step == 5: test_update.append(update) scheduler = Scheduler("test", "0", True) scheduler._disable_cleanup = True do_sched_test(test_graph, [1, 1, 1, 1], scheduler, trace=True, netmodel=SimpleNetModel(1)) assert test_update[0].reassign_failed[0].id == a1.id assert test_update[0].reassign_failed[0].scheduled_worker.worker_id == 0 assert {w.worker_id for w in scheduler.task_graph.objects[a0.output.id].scheduled} == {0, 3}
def test_worker_max_downloads_global(): g = TaskGraph() a1, a2, a3, a4 = [ g.new_task("a{}".format(i), duration=0, output_size=1) for i in range(4) ] b = g.new_task("b", duration=0) b.add_inputs([a1, a2, a3, a4]) s = fixed_scheduler([ (0, a1, 0), (1, a2, 0), (2, a3, 0), # worker is 2! (2, a4, 0), # worker is also 2! (4, b, 0), ]) def make_workers(max_downloads, max_downloads_per_worker=2): return [ Worker(), Worker(), Worker(), Worker(), Worker(max_downloads=max_downloads, max_downloads_per_worker=max_downloads_per_worker) ] assert do_sched_test(g, make_workers(1), s, SimpleNetModel()) == pytest.approx(4) assert do_sched_test(g, make_workers(2), s, SimpleNetModel()) == pytest.approx(2) assert do_sched_test(g, make_workers(3), s, SimpleNetModel()) == pytest.approx(2) assert do_sched_test(g, make_workers(3), s, SimpleNetModel()) == pytest.approx(2) assert do_sched_test(g, make_workers(4), s, SimpleNetModel()) == pytest.approx(1) assert do_sched_test(g, make_workers(4, 1), s, SimpleNetModel()) == pytest.approx(2) assert do_sched_test(g, make_workers(3, 1), s, SimpleNetModel()) == pytest.approx(2)
def test_simulator_reschedule_too_late(): test_graph = TaskGraph() source = test_graph.new_task("S", duration=0, cpus=1, output_size=10) a1 = test_graph.new_task("A1", duration=10, cpus=1) b = test_graph.new_task("B", duration=1, cpus=1) assignments = [[ (0, source, 100), (1, a1, 0), (0, b, 10), ], [ (0, a1, 0), ]] simulator = do_sched_test(test_graph, [1, 1, 1], fixed_scheduler(assignments, steps=True, reassigning=True), trace=True, return_simulator=True, netmodel=SimpleNetModel(1)) assert simulator.env.now == 10 assert simulator.runtime_state.task_info(a1).assigned_workers == [simulator.workers[1]]
def test_worker_download_priorities2(): g = TaskGraph() a = g.new_task("a", duration=0, outputs=[2, 2]) b = g.new_task("b", duration=4, output_size=2) d = g.new_task("d", duration=1) a2 = g.new_task("a2", duration=1) a2.add_input(a.outputs[0]) b2 = g.new_task("b", duration=1, output_size=1) b2.add_input(a.outputs[1]) b2.add_input(b) s = fixed_scheduler([(0, a), (0, b), (1, d, 3), (1, a2, 1), (1, b2, 2)]) w = [Worker(cpus=3), Worker(cpus=1, max_downloads=1)] simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True) assert simulator.runtime_state.task_info(a2).end_time == pytest.approx(3) assert simulator.runtime_state.task_info(b2).end_time == pytest.approx(7)
def test_worker_freecpus(): test_graph = TaskGraph() test_graph.new_task("A", duration=10, cpus=2, output_size=1) test_graph.new_task("B", duration=8, cpus=3, output_size=1) c = test_graph.new_task("C", duration=1, cpus=1, output_size=1) d = test_graph.new_task("D", duration=3, cpus=3, output_size=1) d.add_input(c) free_cpus = [] class Scheduler(SchedulerBase): def schedule(self, update): if not self.task_graph.tasks: return worker = self._simulator.workers[0] free_cpus.append(worker.free_cpus) for t in update.new_ready_tasks: self.assign(self.workers[worker.id], t) scheduler = Scheduler("x", "0") do_sched_test(test_graph, [10], scheduler) assert free_cpus == [10, 5, 5, 8, 10]
def test_worker_download_priorities1(): SIZE = 20 g = TaskGraph() a = g.new_task("a", duration=0, outputs=[1] * SIZE) b = [g.new_task("b{}".format(i), duration=0) for i in range(SIZE)] for i, t in enumerate(b): t.add_input(a.outputs[i]) r = random.Random(42) priorities = list(range(SIZE)) r.shuffle(priorities) s = fixed_scheduler([(0, a, 0)] + [(1, t, p) for t, p in zip(b, priorities)]) w = [Worker(), Worker(max_downloads=2, max_downloads_per_worker=2)] simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True) runtime_state = simulator.runtime_state for t, p in zip(b, priorities): assert runtime_state.task_info(t).end_time == pytest.approx( (SIZE - p - 1) // 2 + 1)
def test_simulator_reschedule_no_download(): test_graph = TaskGraph() a1 = test_graph.new_task("A1", duration=10, cpus=1) a2 = test_graph.new_task("A2", duration=10, cpus=1) b = test_graph.new_task("B", duration=1, cpus=1) c = test_graph.new_task("C", duration=1, cpus=1) d = test_graph.new_task("D", duration=1, cpus=1) e = test_graph.new_task("E", duration=1, cpus=1) assignments = [[ (0, a1, 0), (1, a2, 0), (0, b, 10), (0, c, 9), (0, d, 8), (0, e, 7), ], [ (1, a1, 0), ], [ (None, a1, 0) ], [ (0, a1, 0) ], [ (2, a1, 0) ]] simulator = do_sched_test(test_graph, [1, 1, 1], fixed_scheduler(assignments, steps=True, reassigning=True), trace=True, return_simulator=True) assert simulator.env.now == 14 assert simulator.runtime_state.task_info(a1).assigned_workers == [simulator.workers[0]] # Test the same without allowed reassigning with pytest.raises(Exception): do_sched_test(test_graph, [1, 1, 1], fixed_scheduler(assignments, steps=True, reassigning=False))
def test_task_zerocost(): test_graph = TaskGraph() a = test_graph.new_task("A", duration=1, output_size=100) b = test_graph.new_task("B", duration=1, output_size=50) c = test_graph.new_task("C", duration=8) c.add_inputs((a, b)) d = test_graph.new_task("D", duration=1, outputs=[0]) e = test_graph.new_task("E", duration=1, outputs=[0]) e.add_input(d) class Scheduler(StaticScheduler): def static_schedule(self): if not self.workers or not self.task_graph.tasks: return tasks = self.task_graph.tasks self.assign(self.workers[0], tasks[0]) self.assign(self.workers[1], tasks[1]) self.assign(self.workers[2], tasks[3]) self.assign(self.workers[0], tasks[4]) self.assign(self.workers[2], tasks[2]) scheduler = Scheduler("x", "0") do_sched_test(test_graph, 3, scheduler, SimpleNetModel(bandwidth=2))
def test_worker_execute_priorities(): SIZE = 20 g = TaskGraph() b = [g.new_task("b{}".format(i), duration=1) for i in range(SIZE)] r = random.Random(42) priorities = list(range(SIZE)) r.shuffle(priorities) s = fixed_scheduler([(0, t, p) for t, p in zip(b, priorities)]) simulator = do_sched_test(g, [Worker(cpus=2)], s, return_simulator=True) runtime_state = simulator.runtime_state for t, p in zip(b, priorities): assert runtime_state.task_info(t).end_time == pytest.approx( (SIZE - p - 1) // 2 + 1)
def deserialize_graph(tasks): graph = TaskGraph() id_to_task = {} for t in tasks: task = graph.new_task(duration=t["d"], expected_duration=t["e_d"], cpus=t["cpus"], outputs=[o["s"] for o in t["outputs"]]) for (index, output) in enumerate(task.outputs): output.expected_size = t["outputs"][index]["e_s"] id_to_task[len(id_to_task)] = task for (i, t) in enumerate(tasks): for (parent, output_index) in t["inputs"]: parent = id_to_task[parent] id_to_task[i].add_input(parent.outputs[output_index]) return graph
def plan1(): """ a1/1 a2/3/3 | | a3/1 | a4/1/6 |\ / /| o o |/ | | a5/1 a6/6 a7/2 | \ | / | \ | / \--- a8/1 """ # noqa task_graph = TaskGraph() tasks = [] oid = 0 for i, (duration, outputs) in enumerate([ (2, [1]), # a1 (3, [3]), # a2 (2, [1, 1]), # a3 (1, [6]), # a4 (1, [1]), # a5 (6, [1]), # a6 (1, [2]), # a7 (1, []) # a8 ]): objects = [] for size in outputs: objects.append(DataObject(oid, size, size)) oid += 1 tasks.append( task_graph.new_task("a{}".format(i + 1), duration=duration, expected_duration=duration, outputs=objects)) a1, a2, a3, a4, a5, a6, a7, a8 = tasks a3.add_input(a1) a5.add_inputs([a3.outputs[0], a2, a4]) a6.add_input(a4) a8.add_inputs([a5, a6, a7, a3.outputs[1]]) task_graph.validate() return task_graph
def test_is_descendant(): graph = TaskGraph() n1, n2, n3, n4, n5 = [ graph.new_task(output_size=1 if i != 5 else None) for i in range(5) ] n1.add_input(n2) n1.add_input(n4) n2.add_input(n3) n2.add_input(n4) assert not n1.is_predecessor_of(n4) assert n4.is_predecessor_of(n1) assert not n2.is_predecessor_of(n4) assert n4.is_predecessor_of(n2) assert n2.is_predecessor_of(n1) assert not n1.is_predecessor_of(n5) assert not n5.is_predecessor_of(n1) assert not n1.is_predecessor_of(n1)
def test_simulator_cpus3(): test_graph = TaskGraph() test_graph.new_task("A", duration=3, cpus=1) test_graph.new_task("B", duration=1, cpus=2) test_graph.new_task("C", duration=1, cpus=1) test_graph.new_task("D", duration=1, cpus=3) test_graph.new_task("E", duration=1, cpus=1) test_graph.new_task("F", duration=1, cpus=1) scheduler = AllOnOneScheduler() assert do_sched_test(test_graph, [3], scheduler) == 4 scheduler = AllOnOneScheduler() assert do_sched_test(test_graph, [4], scheduler) == 3 scheduler = AllOnOneScheduler() assert do_sched_test(test_graph, [5], scheduler) == 3
def test_random_dependencies(): graph = TaskGraph() random_dependencies(10, 0.2, lambda: graph.new_task(output_size=1)) assert graph.task_count == 10 check_graph(graph)