def test_worker_priority_block():
g = TaskGraph()
a = g.new_task("a", duration=1)
b = g.new_task("b", duration=1, cpus=3)
c = g.new_task("c", duration=1)
s = fixed_scheduler([(0, a, 3), (0, b, 2), (0, c, 1)])
w = [Worker(cpus=3)]
simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True)
runtime_state = simulator.runtime_state
assert runtime_state.task_info(a).end_time == pytest.approx(1)
assert runtime_state.task_info(b).end_time == pytest.approx(2)
assert runtime_state.task_info(c).end_time == pytest.approx(1)
s = fixed_scheduler([(0, a, 3), (0, b, 2, 2), (0, c, 1)])
w = [Worker(cpus=3)]
simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True)
runtime_state = simulator.runtime_state
assert runtime_state.task_info(a).end_time == pytest.approx(1)
assert runtime_state.task_info(b).end_time == pytest.approx(2)
assert runtime_state.task_info(c).end_time == pytest.approx(3)
def test_scheduling_time():
test_graph = TaskGraph()
a = test_graph.new_task("A", duration=3, output_size=1)
b = test_graph.new_task("B", duration=1, output_size=1)
c = test_graph.new_task("C", duration=1, output_size=1)
d = test_graph.new_task("D", duration=1, output_size=1)
b.add_input(a)
c.add_input(b)
d.add_input(c)
times = []
class Scheduler(SchedulerBase):
def schedule(self, update):
if not self.task_graph.tasks:
return
simulator = self._simulator
times.append(simulator.env.now)
for t in update.new_ready_tasks:
self.assign(self.workers[0], t)
scheduler = Scheduler("x", "0")
simulator = do_sched_test(
test_graph, 1, scheduler,
SimpleNetModel(bandwidth=2),
scheduling_time=2, return_simulator=True)
runtime_state = simulator.runtime_state
assert times == [0, 5, 8, 11, 14]
assert runtime_state.task_info(a).end_time == 5
assert runtime_state.task_info(b).end_time == 8
assert runtime_state.task_info(c).end_time == 11
assert runtime_state.task_info(d).end_time == 14
def test_simulator_reschedule_scheduled_download():
test_graph = TaskGraph()
s = [test_graph.new_task("S{}".format(i), duration=0, cpus=1, output_size=10)
for i in range(10)]
a1 = test_graph.new_task("A1", duration=10, cpus=1)
b = test_graph.new_task("B", duration=1, cpus=1)
c = test_graph.new_task("C", duration=2, cpus=1)
a1.add_inputs(s)
assignments = [
[(0, x, 100) for x in s] + [
(1, a1, 0),
(0, b, 10),
(1, c, 10),
], [], [(2, a1, 0), ]
]
scheduler = fixed_scheduler(assignments, steps=True, reassigning=True)
scheduler._disable_cleanup = True
simulator = do_sched_test(test_graph, [1, 1, 1],
scheduler,
trace=True, return_simulator=True, netmodel=SimpleNetModel(1))
assert simulator.env.now > 40
available = set()
for x in s:
available.add(
frozenset(w.worker_id for w in scheduler.task_graph.objects[x.output.id].availability))
assert frozenset({0, 1, 2}) in available
assert frozenset({0, 2}) in available
assert simulator.runtime_state.task_info(a1).assigned_workers == [simulator.workers[2]]
def test_trace_task_fetch():
tg = TaskGraph()
a = tg.new_task(output_size=5, duration=2)
b = tg.new_task(output_size=3, duration=3)
b.add_input(a)
c = tg.new_task(duration=4)
c.add_input(b)
simulator = do_sched_test(tg, [1, 1], fixed_scheduler([
(0, a, 0),
(1, b, 0),
(0, c, 0)
]), netmodel=SimpleNetModel(1), trace=True, return_simulator=True)
workers = simulator.workers
fetch_start_events = [e for e in simulator.trace_events if isinstance(e, FetchStartTraceEvent)]
assert fetch_start_events == [
FetchStartTraceEvent(2, workers[1], workers[0], a.output),
FetchStartTraceEvent(10, workers[0], workers[1], b.output),
]
fetch_end_events = [e for e in simulator.trace_events if isinstance(e, FetchEndTraceEvent)]
assert fetch_end_events == [
FetchEndTraceEvent(7, workers[1], workers[0], a.output),
FetchEndTraceEvent(13, workers[0], workers[1], b.output),
]
def test_estimate_schedule(plan1):
netmodel = SimpleNetModel(1)
workers = [SchedulerWorker(i, cpus=4) for i in range(4)]
tg = create_scheduler_graph(plan1)
tasks = tg.tasks.values()
schedule = [TaskAssignment(w, t) for (w, t) in zip(itertools.cycle(workers), tasks)]
assert estimate_schedule(schedule, netmodel) == 16
def test_worker_max_downloads_per_worker():
g = TaskGraph()
a = g.new_task("a", duration=0, outputs=[1, 1, 1, 1])
b = g.new_task("b", duration=0)
b.add_inputs(a.outputs)
s = fixed_scheduler([
(0, a, 0),
(1, b, 0),
])
assert do_sched_test(g, [1, 1], s, SimpleNetModel()) == 2
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=1)], s,
SimpleNetModel()) == 4
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=2)], s,
SimpleNetModel()) == 2
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=3)], s,
SimpleNetModel()) == 2
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=4)], s,
SimpleNetModel()) == 1
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=5)], s,
SimpleNetModel()) == 1
def test_scheduler_blevel_gt(plan1):
# 2w, simple
for _ in range(50):
scheduler = BlevelGtScheduler()
scheduler._disable_cleanup = True
assert do_sched_test(plan1, 2, scheduler, SimpleNetModel()) in [13, 16]
sizes = set()
for obj in scheduler.task_graph.objects.values():
assert len(obj.placing) == 1
sizes.add(len(obj.availability))
assert sizes == {1, 2}
def test_simulator_local_reassign():
test_graph = TaskGraph()
a0 = test_graph.new_task("A0", duration=1, output_size=1)
a1 = test_graph.new_task("A1", duration=1, output_size=1)
a2 = test_graph.new_task("A2", duration=1, cpus=1, output_size=10)
a2.add_inputs([a1, a0])
class Scheduler(SchedulerBase):
def start(self):
self.done = False
return super().start()
def schedule(self, update):
if not self.task_graph.tasks or self.done:
return
t = self.task_graph.tasks[a2.id]
for o in t.inputs:
assert not o.scheduled
for o in t.outputs:
assert not o.scheduled
w1 = self.workers[1]
w2 = self.workers[2]
self.assign(w1, t)
for o in t.inputs:
assert o.scheduled == {w1}
for o in t.outputs:
assert o.scheduled == {w1}
self.assign(w2, t)
for o in t.inputs:
assert o.scheduled == {w2}
for o in t.outputs:
assert o.scheduled == {w2}
for t in self.task_graph.tasks.values():
self.assign(w1, t)
self.done = True
scheduler = Scheduler("test", "0", True)
do_sched_test(test_graph, [1, 1, 1],
scheduler,
trace=True,
netmodel=SimpleNetModel(1))
def test_scheduler_random(plan1):
# 1w, instant
assert 17 == do_sched_test(plan1, 1, RandomScheduler())
# 2w, instant
for _ in range(50):
assert 9 <= do_sched_test(plan1, 2, RandomScheduler()) <= 12
# 3w, instant
for _ in range(50):
assert 8 <= do_sched_test(plan1, 3, RandomScheduler()) <= 9
# 2w, simple
for _ in range(50):
assert 13 <= do_sched_test(plan1, 2, RandomScheduler(), SimpleNetModel()) <= 20
def test_simulator_task_start_notify():
test_graph = TaskGraph()
a0 = test_graph.new_task("A0", duration=1, output_size=1)
a1 = test_graph.new_task("A1", duration=10, cpus=1, output_size=1)
a2 = test_graph.new_task("A2", duration=10, cpus=1, output_size=1)
a3 = test_graph.new_task("A3", duration=3, cpus=1)
a1.add_input(a0)
a2.add_input(a1)
triggered = [False, False]
class Scheduler(SchedulerBase):
def start(self):
self.step = 0
return super().start()
def schedule(self, update):
def tg(task):
return self.task_graph.tasks[task.id]
if not self.task_graph.tasks:
return
self.step += 1
if self.step == 1:
self.assign(self.workers[0], tg(a0))
self.assign(self.workers[0], tg(a1))
self.assign(self.workers[0], tg(a2))
self.assign(self.workers[1], tg(a3))
elif tg(a3) in update.new_started_tasks:
assert not triggered[0] and not triggered[1]
triggered[0] = True
assert tg(a3).running
elif tg(a3).state == TaskState.Finished and tg(a3) in update.new_finished_tasks:
assert triggered[0]
assert not triggered[1]
triggered[1] = True
assert not tg(a0).running
assert tg(a1).running
assert not tg(a2).running
assert not tg(a3).running
scheduler = Scheduler("test", "0", task_start_notification=True)
do_sched_test(test_graph, [1, 1, 1],
scheduler,
trace=True, netmodel=SimpleNetModel(1))
assert triggered[1] and triggered[0]
def test_simulator_reassign_failed():
test_graph = TaskGraph()
a0 = test_graph.new_task("A0", duration=1, output_size=1)
a1 = test_graph.new_task("A1", duration=5, cpus=1)
a2 = test_graph.new_task("A2", duration=3, cpus=1, output_size=1)
a3 = test_graph.new_task("A3", duration=1, cpus=1)
a1.add_input(a0)
a3.add_input(a2)
test_update = []
class Scheduler(SchedulerBase):
def start(self):
self.step = 0
return super().start()
def schedule(self, update):
def tg(task):
return self.task_graph.tasks[task.id]
if not self.task_graph.tasks:
return
self.step += 1
if self.step == 1:
self.assign(self.workers[3], tg(a0))
self.assign(self.workers[0], tg(a1))
self.assign(self.workers[1], tg(a2), 10)
self.assign(self.workers[1], tg(a3), 1)
elif self.step == 4:
self.assign(self.workers[2], tg(a1))
elif self.step == 5:
test_update.append(update)
scheduler = Scheduler("test", "0", True)
scheduler._disable_cleanup = True
do_sched_test(test_graph, [1, 1, 1, 1],
scheduler,
trace=True, netmodel=SimpleNetModel(1))
assert test_update[0].reassign_failed[0].id == a1.id
assert test_update[0].reassign_failed[0].scheduled_worker.worker_id == 0
assert {w.worker_id for w in scheduler.task_graph.objects[a0.output.id].scheduled} == {0, 3}
def test_simulator_reschedule_too_late():
test_graph = TaskGraph()
source = test_graph.new_task("S", duration=0, cpus=1, output_size=10)
a1 = test_graph.new_task("A1", duration=10, cpus=1)
b = test_graph.new_task("B", duration=1, cpus=1)
assignments = [[
(0, source, 100),
(1, a1, 0),
(0, b, 10),
], [
(0, a1, 0),
]]
simulator = do_sched_test(test_graph, [1, 1, 1],
fixed_scheduler(assignments, steps=True, reassigning=True),
trace=True, return_simulator=True, netmodel=SimpleNetModel(1))
assert simulator.env.now == 10
assert simulator.runtime_state.task_info(a1).assigned_workers == [simulator.workers[1]]
def test_worker_download_priorities2():
g = TaskGraph()
a = g.new_task("a", duration=0, outputs=[2, 2])
b = g.new_task("b", duration=4, output_size=2)
d = g.new_task("d", duration=1)
a2 = g.new_task("a2", duration=1)
a2.add_input(a.outputs[0])
b2 = g.new_task("b", duration=1, output_size=1)
b2.add_input(a.outputs[1])
b2.add_input(b)
s = fixed_scheduler([(0, a), (0, b), (1, d, 3), (1, a2, 1), (1, b2, 2)])
w = [Worker(cpus=3), Worker(cpus=1, max_downloads=1)]
simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True)
assert simulator.runtime_state.task_info(a2).end_time == pytest.approx(3)
assert simulator.runtime_state.task_info(b2).end_time == pytest.approx(7)
def test_worker_max_downloads_global():
g = TaskGraph()
a1, a2, a3, a4 = [
g.new_task("a{}".format(i), duration=0, output_size=1)
for i in range(4)
]
b = g.new_task("b", duration=0)
b.add_inputs([a1, a2, a3, a4])
s = fixed_scheduler([
(0, a1, 0),
(1, a2, 0),
(2, a3, 0), # worker is 2!
(2, a4, 0), # worker is also 2!
(4, b, 0),
])
def make_workers(max_downloads, max_downloads_per_worker=2):
return [
Worker(),
Worker(),
Worker(),
Worker(),
Worker(max_downloads=max_downloads,
max_downloads_per_worker=max_downloads_per_worker)
]
assert do_sched_test(g, make_workers(1), s,
SimpleNetModel()) == pytest.approx(4)
assert do_sched_test(g, make_workers(2), s,
SimpleNetModel()) == pytest.approx(2)
assert do_sched_test(g, make_workers(3), s,
SimpleNetModel()) == pytest.approx(2)
assert do_sched_test(g, make_workers(3), s,
SimpleNetModel()) == pytest.approx(2)
assert do_sched_test(g, make_workers(4), s,
SimpleNetModel()) == pytest.approx(1)
assert do_sched_test(g, make_workers(4, 1), s,
SimpleNetModel()) == pytest.approx(2)
assert do_sched_test(g, make_workers(3, 1), s,
SimpleNetModel()) == pytest.approx(2)
def test_worker_download_priorities1():
SIZE = 20
g = TaskGraph()
a = g.new_task("a", duration=0, outputs=[1] * SIZE)
b = [g.new_task("b{}".format(i), duration=0) for i in range(SIZE)]
for i, t in enumerate(b):
t.add_input(a.outputs[i])
r = random.Random(42)
priorities = list(range(SIZE))
r.shuffle(priorities)
s = fixed_scheduler([(0, a, 0)] + [(1, t, p)
for t, p in zip(b, priorities)])
w = [Worker(), Worker(max_downloads=2, max_downloads_per_worker=2)]
simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True)
runtime_state = simulator.runtime_state
for t, p in zip(b, priorities):
assert runtime_state.task_info(t).end_time == pytest.approx(
(SIZE - p - 1) // 2 + 1)
def test_task_zerocost():
test_graph = TaskGraph()
a = test_graph.new_task("A", duration=1, output_size=100)
b = test_graph.new_task("B", duration=1, output_size=50)
c = test_graph.new_task("C", duration=8)
c.add_inputs((a, b))
d = test_graph.new_task("D", duration=1, outputs=[0])
e = test_graph.new_task("E", duration=1, outputs=[0])
e.add_input(d)
class Scheduler(StaticScheduler):
def static_schedule(self):
if not self.workers or not self.task_graph.tasks:
return
tasks = self.task_graph.tasks
self.assign(self.workers[0], tasks[0])
self.assign(self.workers[1], tasks[1])
self.assign(self.workers[2], tasks[3])
self.assign(self.workers[0], tasks[4])
self.assign(self.workers[2], tasks[2])
scheduler = Scheduler("x", "0")
do_sched_test(test_graph, 3, scheduler, SimpleNetModel(bandwidth=2))
def test_scheduler_tlevel(plan1):
assert do_sched_test(plan1, 2, TlevelScheduler(), SimpleNetModel()) == 17
def test_scheduler_mcp(plan1):
assert do_sched_test(plan1, 2, MCPScheduler(), SimpleNetModel()) == 15
def test_scheduler_dls(plan1):
assert do_sched_test(plan1, 2, DLSScheduler(), SimpleNetModel()) == 15
def test_scheduler_camp(plan1):
for _ in range(10):
assert 10 <= do_sched_test(plan1, 2, Camp2Scheduler(),
SimpleNetModel()) <= 18
def test_scheduler_etf(plan1):
assert do_sched_test(plan1, 2, ETFScheduler(), SimpleNetModel()) == 17
def test_scheduler_tlevel_gt(plan1):
for _ in range(50):
scheduler = TlevelGtScheduler()
assert 14 <= do_sched_test(plan1, 2, scheduler, SimpleNetModel()) <= 17
def test_scheduler_random_assign(plan1):
for _ in range(50):
assert 10 <= do_sched_test(plan1, 2, RandomAssignScheduler(),
SimpleNetModel()) <= 25
assert 9 <= do_sched_test(plan1, 3, RandomAssignScheduler(),
SimpleNetModel()) <= 25
def create_netmodel(self):
return SimpleNetModel(self.network_bandwidth)
def test_scheduler_random_gt(plan1):
# 2w, simple
for _ in range(50):
assert 13 <= do_sched_test(plan1, 2, RandomGtScheduler(),
SimpleNetModel()) <= 19
def test_scheduler_ws(plan1):
assert 12 <= do_sched_test(plan1, 2, WorkStealingScheduler(),
SimpleNetModel()) <= 18
def test_scheduler_lc(plan1):
assert 11 <= do_sched_test(plan1, 2, LcScheduler(), SimpleNetModel()) <= 18
def test_scheduler_genetic(plan1):
assert 10 <= do_sched_test(plan1, 2, GeneticScheduler(),
SimpleNetModel()) <= 20