def test_worker_priority_block():
g = TaskGraph()
a = g.new_task("a", duration=1)
b = g.new_task("b", duration=1, cpus=3)
c = g.new_task("c", duration=1)
s = fixed_scheduler([(0, a, 3), (0, b, 2), (0, c, 1)])
w = [Worker(cpus=3)]
simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True)
runtime_state = simulator.runtime_state
assert runtime_state.task_info(a).end_time == pytest.approx(1)
assert runtime_state.task_info(b).end_time == pytest.approx(2)
assert runtime_state.task_info(c).end_time == pytest.approx(1)
s = fixed_scheduler([(0, a, 3), (0, b, 2, 2), (0, c, 1)])
w = [Worker(cpus=3)]
simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True)
runtime_state = simulator.runtime_state
assert runtime_state.task_info(a).end_time == pytest.approx(1)
assert runtime_state.task_info(b).end_time == pytest.approx(2)
assert runtime_state.task_info(c).end_time == pytest.approx(3)
def test_scheduling_time():
test_graph = TaskGraph()
a = test_graph.new_task("A", duration=3, output_size=1)
b = test_graph.new_task("B", duration=1, output_size=1)
c = test_graph.new_task("C", duration=1, output_size=1)
d = test_graph.new_task("D", duration=1, output_size=1)
b.add_input(a)
c.add_input(b)
d.add_input(c)
times = []
class Scheduler(SchedulerBase):
def schedule(self, new_ready, new_finished):
times.append(self.simulator.env.now)
workers = self.simulator.workers
return [
TaskAssignment(workers[0], t)
for t in new_ready
]
scheduler = Scheduler()
simulator = do_sched_test(
test_graph, 1, scheduler,
SimpleNetModel(bandwidth=2),
scheduling_time=2, return_simulator=True)
runtime_state = simulator.runtime_state
assert times == [0, 5, 8, 11, 14]
assert runtime_state.task_info(a).end_time == 5
assert runtime_state.task_info(b).end_time == 8
assert runtime_state.task_info(c).end_time == 11
assert runtime_state.task_info(d).end_time == 14
def test_trace_task_fetch():
tg = TaskGraph()
a = tg.new_task(output_size=5, duration=2)
b = tg.new_task(output_size=3, duration=3)
b.add_input(a)
c = tg.new_task(duration=4)
c.add_input(b)
simulator = do_sched_test(tg, [1, 1],
fixed_scheduler([(0, a, 0), (1, b, 0),
(0, c, 0)]),
netmodel=SimpleNetModel(1),
trace=True,
return_simulator=True)
workers = simulator.workers
fetch_start_events = [
e for e in simulator.trace_events
if isinstance(e, FetchStartTraceEvent)
]
assert fetch_start_events == [
FetchStartTraceEvent(2, workers[1], workers[0], a.output),
FetchStartTraceEvent(10, workers[0], workers[1], b.output),
]
fetch_end_events = [
e for e in simulator.trace_events if isinstance(e, FetchEndTraceEvent)
]
assert fetch_end_events == [
FetchEndTraceEvent(7, workers[1], workers[0], a.output),
FetchEndTraceEvent(13, workers[0], workers[1], b.output),
]
def test_estimate_schedule(plan1):
netmodel = SimpleNetModel(1)
workers = [Worker(cpus=4) for _ in range(4)]
schedule = [TaskAssignment(w, t) for (w, t) in zip(itertools.cycle(workers), plan1.tasks)]
assert estimate_schedule(schedule, plan1, netmodel) == 16
def test_task_zerocost():
test_graph = TaskGraph()
a = test_graph.new_task("A", duration=1, output_size=100)
b = test_graph.new_task("B", duration=1, output_size=50)
c = test_graph.new_task("C", duration=8)
c.add_inputs((a, b))
d = test_graph.new_task("D", duration=1, outputs=[0])
e = test_graph.new_task("E", duration=1, outputs=[0])
e.add_input(d)
class Scheduler(StaticScheduler):
def static_schedule(self):
workers = self.simulator.workers
tasks = self.simulator.task_graph.tasks
return [
TaskAssignment(workers[0], tasks[0]),
TaskAssignment(workers[1], tasks[1]),
TaskAssignment(workers[2], tasks[3]),
TaskAssignment(workers[0], tasks[4]),
TaskAssignment(workers[2], tasks[2])
]
scheduler = Scheduler()
do_sched_test(test_graph, 3, scheduler, SimpleNetModel(bandwidth=2))
def test_estimate_schedule_zero_expected_time(plan1):
netmodel = SimpleNetModel(1)
workers = [Worker(cpus=4) for _ in range(4)]
plan1.tasks[1].expected_duration = 0
plan1.tasks[5].expected_duration = 0
schedule = [TaskAssignment(w, t) for (w, t) in zip(itertools.cycle(workers), plan1.tasks)]
assert estimate_schedule(schedule, plan1, netmodel) == 15
def test_worker_max_downloads_per_worker():
g = TaskGraph()
a = g.new_task("a", duration=0, outputs=[1, 1, 1, 1])
b = g.new_task("b", duration=0)
b.add_inputs(a.outputs)
s = fixed_scheduler([
(0, a, 0),
(1, b, 0),
])
assert do_sched_test(g, [1, 1], s, SimpleNetModel()) == 2
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=1)], s,
SimpleNetModel()) == 4
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=2)], s,
SimpleNetModel()) == 2
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=3)], s,
SimpleNetModel()) == 2
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=4)], s,
SimpleNetModel()) == 1
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=5)], s,
SimpleNetModel()) == 1
def test_scheduler_random(plan1):
# 1w, instant
assert 17 == do_sched_test(plan1, 1, RandomScheduler())
# 2w, instant
for _ in range(50):
assert 9 <= do_sched_test(plan1, 2, RandomScheduler()) <= 12
# 3w, instant
for _ in range(50):
assert 8 <= do_sched_test(plan1, 3, RandomScheduler()) <= 9
# 2w, simple
for _ in range(50):
assert 13 <= do_sched_test(plan1, 2, RandomScheduler(),
SimpleNetModel()) <= 20
def test_worker_download_priorities2():
g = TaskGraph()
a = g.new_task("a", duration=0, outputs=[2, 2])
b = g.new_task("b", duration=4, output_size=2)
d = g.new_task("d", duration=1)
a2 = g.new_task("a2", duration=1)
a2.add_input(a.outputs[0])
b2 = g.new_task("b", duration=1, output_size=1)
b2.add_input(a.outputs[1])
b2.add_input(b)
s = fixed_scheduler([(0, a), (0, b), (1, d, 3), (1, a2, 1), (1, b2, 2)])
w = [Worker(cpus=3), Worker(cpus=1, max_downloads=1)]
simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True)
assert simulator.runtime_state.task_info(a2).end_time == pytest.approx(3)
assert simulator.runtime_state.task_info(b2).end_time == pytest.approx(7)
def test_worker_max_downloads_global():
g = TaskGraph()
a1, a2, a3, a4 = [
g.new_task("a{}".format(i), duration=0, output_size=1)
for i in range(4)
]
b = g.new_task("b", duration=0)
b.add_inputs([a1, a2, a3, a4])
s = fixed_scheduler([
(0, a1, 0),
(1, a2, 0),
(2, a3, 0), # worker is 2!
(2, a4, 0), # worker is also 2!
(4, b, 0),
])
def make_workers(max_downloads, max_downloads_per_worker=2):
return [
Worker(),
Worker(),
Worker(),
Worker(),
Worker(max_downloads=max_downloads,
max_downloads_per_worker=max_downloads_per_worker)
]
assert do_sched_test(g, make_workers(1), s,
SimpleNetModel()) == pytest.approx(4)
assert do_sched_test(g, make_workers(2), s,
SimpleNetModel()) == pytest.approx(2)
assert do_sched_test(g, make_workers(3), s,
SimpleNetModel()) == pytest.approx(2)
assert do_sched_test(g, make_workers(3), s,
SimpleNetModel()) == pytest.approx(2)
assert do_sched_test(g, make_workers(4), s,
SimpleNetModel()) == pytest.approx(1)
assert do_sched_test(g, make_workers(4, 1), s,
SimpleNetModel()) == pytest.approx(2)
assert do_sched_test(g, make_workers(3, 1), s,
SimpleNetModel()) == pytest.approx(2)
def test_worker_download_priorities1():
SIZE = 20
g = TaskGraph()
a = g.new_task("a", duration=0, outputs=[1] * SIZE)
b = [g.new_task("b{}".format(i), duration=0) for i in range(SIZE)]
for i, t in enumerate(b):
t.add_input(a.outputs[i])
r = random.Random(42)
priorities = list(range(SIZE))
r.shuffle(priorities)
s = fixed_scheduler([(0, a, 0)] + [(1, t, p)
for t, p in zip(b, priorities)])
w = [Worker(), Worker(max_downloads=2, max_downloads_per_worker=2)]
simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True)
runtime_state = simulator.runtime_state
for t, p in zip(b, priorities):
assert runtime_state.task_info(t).end_time == pytest.approx(
(SIZE - p - 1) // 2 + 1)
def test_scheduler_random_assign(plan1):
for _ in range(50):
assert 10 <= do_sched_test(plan1, 2, RandomAssignScheduler(),
SimpleNetModel()) <= 25
assert 9 <= do_sched_test(plan1, 3, RandomAssignScheduler(),
SimpleNetModel()) <= 25
def test_scheduler_genetic(plan1):
assert 10 <= do_sched_test(plan1, 2, GeneticScheduler(),
SimpleNetModel()) <= 20
def test_scheduler_mcp(plan1):
assert do_sched_test(plan1, 2, MCPScheduler(), SimpleNetModel()) == 15
def test_scheduler_etf(plan1):
assert do_sched_test(plan1, 2, ETFScheduler(), SimpleNetModel()) == 15
def test_scheduler_last(plan1):
assert do_sched_test(plan1, 2, LASTScheduler(), SimpleNetModel()) == 17
def test_scheduler_dls(plan1):
assert do_sched_test(plan1, 2, DLSScheduler(), SimpleNetModel()) == 14
def test_scheduler_camp(plan1):
for _ in range(10):
assert 10 <= do_sched_test(plan1, 2, Camp2Scheduler(),
SimpleNetModel()) <= 18
def test_scheduler_random_gt(plan1):
# 2w, simple
for _ in range(50):
assert 13 <= do_sched_test(plan1, 2, RandomGtScheduler(),
SimpleNetModel()) <= 19
def test_scheduler_blevel_gt(plan1):
# 2w, simple
for _ in range(50):
assert do_sched_test(plan1, 2, BlevelGtScheduler(),
SimpleNetModel()) in [13, 16]