def do_sched_test(task_graph,
workers,
scheduler,
netmodel=None,
trace=False,
return_simulator=False,
min_scheduling_interval=None,
scheduling_time=None):
if netmodel is None:
netmodel = InstantNetModel()
if isinstance(workers, int):
workers = [Worker() for _ in range(workers)]
elif isinstance(workers[0], int):
workers = [Worker(cpus=cpus) for cpus in workers]
else:
assert isinstance(workers[0], Worker)
simulator = Simulator(task_graph,
workers,
scheduler,
netmodel,
trace=trace,
scheduling_time=scheduling_time,
min_scheduling_interval=min_scheduling_interval)
result = simulator.run()
if return_simulator:
return simulator
else:
return result
def test_worker_priority_block():
g = TaskGraph()
a = g.new_task("a", duration=1)
b = g.new_task("b", duration=1, cpus=3)
c = g.new_task("c", duration=1)
s = fixed_scheduler([(0, a, 3), (0, b, 2), (0, c, 1)])
w = [Worker(cpus=3)]
simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True)
runtime_state = simulator.runtime_state
assert runtime_state.task_info(a).end_time == pytest.approx(1)
assert runtime_state.task_info(b).end_time == pytest.approx(2)
assert runtime_state.task_info(c).end_time == pytest.approx(1)
s = fixed_scheduler([(0, a, 3), (0, b, 2, 2), (0, c, 1)])
w = [Worker(cpus=3)]
simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True)
runtime_state = simulator.runtime_state
assert runtime_state.task_info(a).end_time == pytest.approx(1)
assert runtime_state.task_info(b).end_time == pytest.approx(2)
assert runtime_state.task_info(c).end_time == pytest.approx(3)
def test_worker_max_downloads_per_worker():
g = TaskGraph()
a = g.new_task("a", duration=0, outputs=[1, 1, 1, 1])
b = g.new_task("b", duration=0)
b.add_inputs(a.outputs)
s = fixed_scheduler([
(0, a, 0),
(1, b, 0),
])
assert do_sched_test(g, [1, 1], s, SimpleNetModel()) == 2
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=1)], s,
SimpleNetModel()) == 4
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=2)], s,
SimpleNetModel()) == 2
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=3)], s,
SimpleNetModel()) == 2
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=4)], s,
SimpleNetModel()) == 1
assert do_sched_test(
g, [Worker(), Worker(max_downloads_per_worker=5)], s,
SimpleNetModel()) == 1
def make_workers(max_downloads, max_downloads_per_worker=2):
return [
Worker(),
Worker(),
Worker(),
Worker(),
Worker(max_downloads=max_downloads,
max_downloads_per_worker=max_downloads_per_worker)
]
def test_worker_estimate_earliest_time_offset_now():
tg = TaskGraph()
t0 = tg.new_task(expected_duration=3, cpus=1)
t1 = tg.new_task(expected_duration=5, cpus=1)
t2 = tg.new_task(expected_duration=3, cpus=2)
worker = Worker(cpus=2)
worker.assignments = [
TaskAssignment(worker, t0),
TaskAssignment(worker, t1)
]
worker.running_tasks[t0] = RunningTask(t0, 0)
worker.running_tasks[t1] = RunningTask(t1, 0)
assert worker_estimate_earliest_time(worker, t2, 2) == 3
def test_estimate_schedule(plan1):
netmodel = SimpleNetModel(1)
workers = [Worker(cpus=4) for _ in range(4)]
schedule = [TaskAssignment(w, t) for (w, t) in zip(itertools.cycle(workers), plan1.tasks)]
assert estimate_schedule(schedule, plan1, netmodel) == 16
def create_netmodel(cclass=MinMaxFlowNetModel):
env = simpy.Environment()
workers = [Worker() for _ in range(4)]
for i, w in enumerate(workers):
w.id = i
netmodel = cclass(100)
netmodel.init(env, workers)
return netmodel, env, workers
def test_estimate_schedule_zero_expected_time(plan1):
netmodel = SimpleNetModel(1)
workers = [Worker(cpus=4) for _ in range(4)]
plan1.tasks[1].expected_duration = 0
plan1.tasks[5].expected_duration = 0
schedule = [TaskAssignment(w, t) for (w, t) in zip(itertools.cycle(workers), plan1.tasks)]
assert estimate_schedule(schedule, plan1, netmodel) == 15
def test_worker_estimate_earliest_time():
tg = TaskGraph()
t0 = tg.new_task(expected_duration=3, cpus=2)
t1 = tg.new_task(expected_duration=5, cpus=1)
t2 = tg.new_task(expected_duration=4, cpus=3)
t4 = tg.new_task(expected_duration=4, cpus=2)
t5 = tg.new_task(expected_duration=4, cpus=2)
worker = Worker(cpus=4)
worker.assignments = [
TaskAssignment(worker, t0),
TaskAssignment(worker, t1),
TaskAssignment(worker, t2),
TaskAssignment(worker, t4)
]
worker.running_tasks[t0] = RunningTask(t0, 0)
worker.running_tasks[t1] = RunningTask(t1, 0)
assert worker_estimate_earliest_time(worker, t5, 0) == 7
def test_worker_download_priorities2():
g = TaskGraph()
a = g.new_task("a", duration=0, outputs=[2, 2])
b = g.new_task("b", duration=4, output_size=2)
d = g.new_task("d", duration=1)
a2 = g.new_task("a2", duration=1)
a2.add_input(a.outputs[0])
b2 = g.new_task("b", duration=1, output_size=1)
b2.add_input(a.outputs[1])
b2.add_input(b)
s = fixed_scheduler([(0, a), (0, b), (1, d, 3), (1, a2, 1), (1, b2, 2)])
w = [Worker(cpus=3), Worker(cpus=1, max_downloads=1)]
simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True)
assert simulator.runtime_state.task_info(a2).end_time == pytest.approx(3)
assert simulator.runtime_state.task_info(b2).end_time == pytest.approx(7)
def test_worker_download_priorities1():
SIZE = 20
g = TaskGraph()
a = g.new_task("a", duration=0, outputs=[1] * SIZE)
b = [g.new_task("b{}".format(i), duration=0) for i in range(SIZE)]
for i, t in enumerate(b):
t.add_input(a.outputs[i])
r = random.Random(42)
priorities = list(range(SIZE))
r.shuffle(priorities)
s = fixed_scheduler([(0, a, 0)] + [(1, t, p)
for t, p in zip(b, priorities)])
w = [Worker(), Worker(max_downloads=2, max_downloads_per_worker=2)]
simulator = do_sched_test(g, w, s, SimpleNetModel(), return_simulator=True)
runtime_state = simulator.runtime_state
for t, p in zip(b, priorities):
assert runtime_state.task_info(t).end_time == pytest.approx(
(SIZE - p - 1) // 2 + 1)
def test_worker_execute_priorities():
SIZE = 20
g = TaskGraph()
b = [g.new_task("b{}".format(i), duration=1) for i in range(SIZE)]
r = random.Random(42)
priorities = list(range(SIZE))
r.shuffle(priorities)
s = fixed_scheduler([(0, t, p) for t, p in zip(b, priorities)])
simulator = do_sched_test(g, [Worker(cpus=2)], s, return_simulator=True)
runtime_state = simulator.runtime_state
for t, p in zip(b, priorities):
assert runtime_state.task_info(t).end_time == pytest.approx(
(SIZE - p - 1) // 2 + 1)
def create_worker(wargs):
if instance.netmodel == "simple":
return Worker(**wargs,
max_downloads=inf,
max_downloads_per_worker=inf)
return Worker(**wargs)