def __init__(self, wid, workflow, algorithm, env):
self.id = wid
if algorithm is 'heft':
self.solution = shadow_heft(workflow)
else:
sys.exit("Other algorithms are not supported")
# DO Task execution things here
taskid = 0
ast = 1
aft = 2
self.tasks = []
task_order = []
# The solution object is now how we get information on allocatiosn from SHADOW
for task in self.solution.task_allocations:
allocation = self.solution.task_allocations.get(task)
taskobj = Task(task.tid, env)
taskobj.est = allocation.ast
taskobj.eft = allocation.aft
taskobj.duration = taskobj.eft - taskobj.est
taskobj.machine_id = allocation.machine
taskobj.flops = task.flops_demand
taskobj.pred = list(workflow.graph.predecessors(task))
self.tasks.append(taskobj)
self.tasks.sort(key=lambda x: x.est)
self.exec_order = self.solution.execution_order
self.start_time = None
self.priority = 0
self.status = WorkflowStatus.UNSCHEDULED
self.delayed = None
def testTaskDoWorkWithDelay(self):
dm = copy.copy(self.dm_delay)
t = Task(self.task_id,
est=0,
eft=11,
machine=None,
predecessors=None,
flops=0,
memory=0,
io=0,
delay=dm)
self.env.process(t.do_work(self.env, None))
self.env.run()
self.assertEqual(12, t.aft)
self.assertTrue(t.delay_flag)
def test_double_allocation(self):
"""
Given an existing schedule, add multiple allocations to ensure
duplicates do not exist
"""
task = Task('test_0', 0, 2, self.machine, [])
dup_task = Task('test_2', 8, 12, self.machine, [])
existing_schedule = {task: self.machine, dup_task: self.machine}
new_schedule, new_pairs = self.scheduler._process_current_schedule(
existing_schedule, allocation_pairs={}, workflow_id='test_id')
self.assertFalse(task in self.cluster._tasks['running'])
self.env.run(until=1)
self.assertTrue(task in self.cluster._tasks['running'])
self.assertTrue(dup_task in new_schedule)
self.assertFalse(task in new_schedule)
self.assertTrue(task.id in new_pairs)
def testTaskDoWorkWithOutDelay(self):
dm = copy.copy(self.dm_nodelay)
t = Task(self.task_id,
est=0,
eft=11,
machine=None,
predecessors=None,
flops=0,
memory=0,
io=0,
delay=dm)
# t.ast = 0
t.duration = t.eft - t.est
self.env.process(t.do_work(self.env, None))
self.env.run()
self.assertEqual(11, t.aft)
def testTaskWithOutDelay(self):
dm = copy.copy(self.dm_nodelay)
t = Task(self.task_id,
est=0,
eft=11,
machine=None,
predecessors=None,
flops=0,
memory=0,
io=0,
delay=dm)
# t.est = 0
# t.eft = 11
# t.duration = t.eft - t.est
t.ast = 0
delay = t._calc_task_delay()
self.assertEqual(0, delay - t.duration)
def test_duplication_allocation(self):
ret = self.env.process(
self.cluster.allocate_task_to_cluster(self.task, self.machine))
self.env.run(until=1)
newtask = Task('test_2', 8, 12, self.machine, [])
new_ret = self.env.process(
self.cluster.allocate_task_to_cluster(newtask, self.machine))
self.assertRaises(RuntimeError, self.env.run, until=2)
def test_task_init(self):
t = Task(tid=self.task_id,
est=self.est,
eft=self.eft,
machine=self.machine,
predecessors=None,
flops=0,
memory=0,
io=0,
delay=self.dm)
def generate_plan(self, clock, cluster, buffer, observation, max_ingest):
"""
For this StaticPlanning example, we are using the SHADOW static
scheduling library to produce static plans. There are a couple of
Parameters
----------
observation
buffer
cluster
clock : int
Current simulation time (usually provided through `env.now`)
max_ingest
Returns
-------
"""
if observation.ast is None:
raise RuntimeError(f'Observation AST must be updated before plan')
workflow = self._initialise_shadow_workflows(observation, cluster)
solution = self._run_scheduling(workflow)
est = self._calc_workflow_est(observation, buffer)
eft = solution.makespan
tasks = []
for task in solution.task_allocations:
allocation = solution.task_allocations.get(task)
tid = self._create_observation_task_id(task.tid, observation,
clock)
dm = copy.copy(self.delay_model)
pred = list(workflow.graph.predecessors(task))
predecessors = [
self._create_observation_task_id(x.tid, observation, clock)
for x in pred
]
edge_costs = {}
# Get the data transfer costs
data = dict(workflow.graph.pred[task])
for element in data:
nm = self._create_observation_task_id(element.tid, observation,
clock)
val = data[element]['data_size']
edge_costs[nm] = val
taskobj = Task(tid, allocation.ast, allocation.aft,
allocation.machine.id, predecessors,
task.flops_demand, 0, edge_costs, dm)
tasks.append(taskobj)
tasks.sort(key=lambda x: x.est)
exec_order = solution.execution_order
return WorkflowPlan(observation.name, est, eft, tasks, exec_order,
WorkflowStatus.SCHEDULED, max_ingest)
def _generate_ingest_tasks(self, demand, duration):
"""
Parameters
----------
demand : int
Number of machines that are provisioned for ingest
duration : int
Duration of observation (in simulation timesteps)
Returns
-------
tasks : list()
List of core.Planner.Task objects
"""
tasks = []
for i in range(demand):
t = Task(i, env=self.env)
t.duration = duration
t.task_status = TaskStatus.SCHEDULED
tasks.append(t)
return tasks
def setUp(self) -> None:
"""
TODO
Set up tasks and cluster to attempt allocation of tasks to resources
"""
self.env = simpy.Environment()
config = Config(CONFIG)
self.cluster = Cluster(env=self.env, config=config)
self.telescope = Telescope(self.env,
config,
planner=None,
scheduler=None)
self.observation = self.telescope.observations[0]
self.machine = self.cluster.machines[0]
self.task = Task('test_0', 0, 2, self.machine, [])
def generate_plan(self, clock, cluster, buffer, observation, max_ingest):
"""
Parameters
----------
cluster
clock
buffer
observation
"""
plan = None
if self.algorithm is 'batch':
graph = self._workflow_to_nx(observation.workflow)
est = self._calc_workflow_est(observation, buffer)
tasks = []
exec_order = list(nx.algorithms.topological_sort(graph))
for task in exec_order:
tid = self._create_observation_task_id(task, observation,
clock)
dm = copy.copy(self.delay_model)
pred = list(graph.predecessors(task))
predecessors = [
self._create_observation_task_id(x, observation, clock)
for x in pred
]
edge_costs = {}
# Get the data transfer costs
data = dict(graph.pred[task])
for element in data:
nm = self._create_observation_task_id(
element, observation, clock)
val = data[element]['data_size']
edge_costs[nm] = val
taskobj = Task(tid, 0, 0, None, predecessors,
graph.nodes[task]['comp'], 0, edge_costs, dm)
tasks.append(taskobj)
# exec_order = list(nx.algorithms.topological_sort(graph))
return WorkflowPlan(observation.name, est, -1, tasks, exec_order,
WorkflowStatus.SCHEDULED, max_ingest)
else:
raise RuntimeError(
f'{self.algorithm} is not supported by {str(self)}')
def test_task_setup(self):
task = Task(0, env=self.env)