def __init__(self, wid, workflow, algorithm, env): self.id = wid if algorithm is 'heft': self.solution = shadow_heft(workflow) else: sys.exit("Other algorithms are not supported") # DO Task execution things here taskid = 0 ast = 1 aft = 2 self.tasks = [] task_order = [] # The solution object is now how we get information on allocatiosn from SHADOW for task in self.solution.task_allocations: allocation = self.solution.task_allocations.get(task) taskobj = Task(task.tid, env) taskobj.est = allocation.ast taskobj.eft = allocation.aft taskobj.duration = taskobj.eft - taskobj.est taskobj.machine_id = allocation.machine taskobj.flops = task.flops_demand taskobj.pred = list(workflow.graph.predecessors(task)) self.tasks.append(taskobj) self.tasks.sort(key=lambda x: x.est) self.exec_order = self.solution.execution_order self.start_time = None self.priority = 0 self.status = WorkflowStatus.UNSCHEDULED self.delayed = None
def testTaskDoWorkWithDelay(self): dm = copy.copy(self.dm_delay) t = Task(self.task_id, est=0, eft=11, machine=None, predecessors=None, flops=0, memory=0, io=0, delay=dm) self.env.process(t.do_work(self.env, None)) self.env.run() self.assertEqual(12, t.aft) self.assertTrue(t.delay_flag)
def test_double_allocation(self): """ Given an existing schedule, add multiple allocations to ensure duplicates do not exist """ task = Task('test_0', 0, 2, self.machine, []) dup_task = Task('test_2', 8, 12, self.machine, []) existing_schedule = {task: self.machine, dup_task: self.machine} new_schedule, new_pairs = self.scheduler._process_current_schedule( existing_schedule, allocation_pairs={}, workflow_id='test_id') self.assertFalse(task in self.cluster._tasks['running']) self.env.run(until=1) self.assertTrue(task in self.cluster._tasks['running']) self.assertTrue(dup_task in new_schedule) self.assertFalse(task in new_schedule) self.assertTrue(task.id in new_pairs)
def testTaskDoWorkWithOutDelay(self): dm = copy.copy(self.dm_nodelay) t = Task(self.task_id, est=0, eft=11, machine=None, predecessors=None, flops=0, memory=0, io=0, delay=dm) # t.ast = 0 t.duration = t.eft - t.est self.env.process(t.do_work(self.env, None)) self.env.run() self.assertEqual(11, t.aft)
def testTaskWithOutDelay(self): dm = copy.copy(self.dm_nodelay) t = Task(self.task_id, est=0, eft=11, machine=None, predecessors=None, flops=0, memory=0, io=0, delay=dm) # t.est = 0 # t.eft = 11 # t.duration = t.eft - t.est t.ast = 0 delay = t._calc_task_delay() self.assertEqual(0, delay - t.duration)
def test_duplication_allocation(self): ret = self.env.process( self.cluster.allocate_task_to_cluster(self.task, self.machine)) self.env.run(until=1) newtask = Task('test_2', 8, 12, self.machine, []) new_ret = self.env.process( self.cluster.allocate_task_to_cluster(newtask, self.machine)) self.assertRaises(RuntimeError, self.env.run, until=2)
def test_task_init(self): t = Task(tid=self.task_id, est=self.est, eft=self.eft, machine=self.machine, predecessors=None, flops=0, memory=0, io=0, delay=self.dm)
def generate_plan(self, clock, cluster, buffer, observation, max_ingest): """ For this StaticPlanning example, we are using the SHADOW static scheduling library to produce static plans. There are a couple of Parameters ---------- observation buffer cluster clock : int Current simulation time (usually provided through `env.now`) max_ingest Returns ------- """ if observation.ast is None: raise RuntimeError(f'Observation AST must be updated before plan') workflow = self._initialise_shadow_workflows(observation, cluster) solution = self._run_scheduling(workflow) est = self._calc_workflow_est(observation, buffer) eft = solution.makespan tasks = [] for task in solution.task_allocations: allocation = solution.task_allocations.get(task) tid = self._create_observation_task_id(task.tid, observation, clock) dm = copy.copy(self.delay_model) pred = list(workflow.graph.predecessors(task)) predecessors = [ self._create_observation_task_id(x.tid, observation, clock) for x in pred ] edge_costs = {} # Get the data transfer costs data = dict(workflow.graph.pred[task]) for element in data: nm = self._create_observation_task_id(element.tid, observation, clock) val = data[element]['data_size'] edge_costs[nm] = val taskobj = Task(tid, allocation.ast, allocation.aft, allocation.machine.id, predecessors, task.flops_demand, 0, edge_costs, dm) tasks.append(taskobj) tasks.sort(key=lambda x: x.est) exec_order = solution.execution_order return WorkflowPlan(observation.name, est, eft, tasks, exec_order, WorkflowStatus.SCHEDULED, max_ingest)
def _generate_ingest_tasks(self, demand, duration): """ Parameters ---------- demand : int Number of machines that are provisioned for ingest duration : int Duration of observation (in simulation timesteps) Returns ------- tasks : list() List of core.Planner.Task objects """ tasks = [] for i in range(demand): t = Task(i, env=self.env) t.duration = duration t.task_status = TaskStatus.SCHEDULED tasks.append(t) return tasks
def setUp(self) -> None: """ TODO Set up tasks and cluster to attempt allocation of tasks to resources """ self.env = simpy.Environment() config = Config(CONFIG) self.cluster = Cluster(env=self.env, config=config) self.telescope = Telescope(self.env, config, planner=None, scheduler=None) self.observation = self.telescope.observations[0] self.machine = self.cluster.machines[0] self.task = Task('test_0', 0, 2, self.machine, [])
def generate_plan(self, clock, cluster, buffer, observation, max_ingest): """ Parameters ---------- cluster clock buffer observation """ plan = None if self.algorithm is 'batch': graph = self._workflow_to_nx(observation.workflow) est = self._calc_workflow_est(observation, buffer) tasks = [] exec_order = list(nx.algorithms.topological_sort(graph)) for task in exec_order: tid = self._create_observation_task_id(task, observation, clock) dm = copy.copy(self.delay_model) pred = list(graph.predecessors(task)) predecessors = [ self._create_observation_task_id(x, observation, clock) for x in pred ] edge_costs = {} # Get the data transfer costs data = dict(graph.pred[task]) for element in data: nm = self._create_observation_task_id( element, observation, clock) val = data[element]['data_size'] edge_costs[nm] = val taskobj = Task(tid, 0, 0, None, predecessors, graph.nodes[task]['comp'], 0, edge_costs, dm) tasks.append(taskobj) # exec_order = list(nx.algorithms.topological_sort(graph)) return WorkflowPlan(observation.name, est, -1, tasks, exec_order, WorkflowStatus.SCHEDULED, max_ingest) else: raise RuntimeError( f'{self.algorithm} is not supported by {str(self)}')
def test_task_setup(self): task = Task(0, env=self.env)