class _TestComparisonNonOperator(_TestComparison): valid_types = ( # empty iterable set(), (), # Iterable[int] ProcSet(0), ProcInt(0), {0}, pytest.param((i * i for i in range(4)), id='(i*i for i in range(4))'), # Iterable[ProcInt] (ProcInt(0), ProcInt(1)), # Iterable[ProcSet] (ProcSet(0), ProcSet(1)), # Iterable[Union[int, ProcInt, ProcSet]] (0, ProcInt(1), ProcSet(2)), ) incompatible_types = ( # not iterable None, 0, # iterable of wrong type 'bad-iterable', {None}, (0, ProcInt(1), None), )
def on_host_power_state_changed(self, sender: Host) -> None: if sender.id in self.__last_pstate_id and sender.pstate and self.__last_pstate_id[ sender.id] != sender.pstate.id: assert sender.pstate assert self.simulator self.__last_pstate_id[sender.id] = sender.pstate.id if sender.pstate.type == PowerStateType.SWITCHING_OFF: new_pstate_id = -2 elif sender.pstate.type == PowerStateType.SWITCHING_ON: new_pstate_id = -1 else: new_pstate_id = sender.pstate.id if self.__info['time'][ -1] == self.simulator.current_time and self.__info[ 'new_pstate'][-1] == new_pstate_id: # Update last record procset = ProcSet.from_str(self.__info['machine_id'][-1]) procset.update(ProcSet(sender.id)) self.__info['machine_id'][-1] = str(procset) else: # Record new state self.__info['time'].append(self.simulator.current_time) self.__info['machine_id'].append(str(sender.id)) self.__info['new_pstate'].append(new_pstate_id)
def test_deepcopy_nonempty(self): pset = ProcSet(ProcInt(0, 3)) dcopy_pset = copy.deepcopy(pset) assert dcopy_pset == pset assert dcopy_pset is not pset assert dcopy_pset._itvs is not pset._itvs pset |= ProcSet(ProcInt(128, 255)) assert dcopy_pset != pset
def test_copy_empty(self): pset = ProcSet() copy_pset = copy.copy(pset) assert copy_pset == pset assert copy_pset is not pset assert copy_pset._itvs is not pset._itvs pset |= ProcSet(ProcInt(128, 255)) assert copy_pset != pset
def free_slots(self, begin_time=0, end_time=None): ''' :returns: a DataFrame (compatible with a JobSet) that contains all the not overlapping square free slots of this JobSet maximzing the time. It can be transform to a JobSet to be plot as gantt chart. ''' # slots_time contains tuple of # (slot_begin_time,free_resources_intervals) free_interval_serie = self.free_intervals(begin_time, end_time) slots_time = [(free_interval_serie.time[0], ProcSet(self.res_bounds))] new_slots_time = slots_time columns = ['jobID', 'allocated_resources', 'starting_time', 'finish_time', 'execution_time', 'submission_time'] free_slots_df = pd.DataFrame(columns=columns) prev_free_itvs = ProcSet(self.res_bounds) slots = 0 for i, curr_row in free_interval_serie.iterrows(): if i == 0: continue new_slots_time = [] curr_time = curr_row.time taken_resources = prev_free_itvs - curr_row.free_itvs freed_resources = curr_row.free_itvs - prev_free_itvs if i == len(free_interval_serie) - 1: taken_resources = ProcSet(self.res_bounds) if taken_resources: # slot ends: store it and update free slot for begin_time, itvs in slots_time: to_update = itvs & taken_resources if to_update: # store new slots slots = slots + 1 new_slot = [str(slots), to_update, begin_time, curr_time, curr_time - begin_time, begin_time] free_slots_df.loc[slots] = new_slot # remove free slots free_res = itvs - to_update if free_res: new_slots_time.append((begin_time, free_res)) else: new_slots_time.append((begin_time, itvs)) if freed_resources: # slots begin: udpate free slot if not new_slots_time: new_slots_time = slots_time new_slots_time.append((curr_time, freed_resources)) # update previous prev_free_itvs = curr_row.free_itvs # clean slots_free slots_time = new_slots_time return free_slots_df
def __init__(self, timestamp: float, resources: Sequence[int], state_id: int) -> None: if not resources: raise ValueError('Expected `resources` argument to have at least ' 'one resource, got {}.'.format(resources)) super().__init__(timestamp, BatsimRequestType.SET_RESOURCE_STATE) self.resources = ProcSet(*resources) self.state = state_id
def test_copy_nested(self): pset = ProcSet(ProcInt(0, 3)) nested = {0: pset, 1: [pset]} copy_nested = copy.copy(nested) assert copy_nested[0] == pset assert copy_nested[0] is pset assert copy_nested[0] == copy_nested[1][0] assert copy_nested[0] is copy_nested[1][0] pset |= ProcSet(ProcInt(128, 255)) assert copy_nested[0] == pset assert copy_nested[0] == copy_nested[1][0]
def test_comparison(self, testcase): left_pset = ProcSet(*testcase.left) right_pset = ProcSet(*testcase.right) # apply comparison method by name comparison = getattr(left_pset, self.method)(right_pset) set_comparison = getattr(set(left_pset), self.method)(set(right_pset)) # check correctness of result assert isinstance(comparison, bool) assert comparison == set_comparison
def test_deepcopy_nested(self): pset = ProcSet(ProcInt(0, 3)) nested = {0: pset, 1: [pset]} dcopy_nested = copy.deepcopy(nested) assert dcopy_nested[0] == pset assert dcopy_nested[0] is not pset assert dcopy_nested[0] == dcopy_nested[1][0] assert dcopy_nested[0] is dcopy_nested[1][0] pset |= ProcSet(ProcInt(128, 255)) assert dcopy_nested[0] != pset assert dcopy_nested[0] == dcopy_nested[1][0]
def _schedule_data_staging( self, job: Job, source: StorageResource, destination: StorageResource, walltime=None ): allocation = StorageAllocation( start_time=self.time, walltime=walltime if walltime else -1, resources=[source, destination], job=job ) allocation.allocate_all(self) job._allocation = allocation job._batsim_job.allocation = ProcSet(source.id, destination.id) job._batsim_job.storage_mapping = { 'source': source.id, 'destination': destination.id } self._batsim.execute_job(job._batsim_job) # Copy from the Job.schedule() method; job.py:569 self.info( "Scheduled job ({job})", job=job, type="job_scheduled") job._scheduled = True job._start_time = self.time job._jobs_list.update_element(self)
def test_request(self): r = SetResourceStateBatsimRequest(10, [1, 2, 3], 1) assert r.timestamp == 10 assert r.type == BatsimRequestType.SET_RESOURCE_STATE assert str(r.resources) == str(ProcSet(*[1, 2, 3])) assert r.state == 1
def add_resource(self, resource_id: int) -> None: """ Add a resource id to the request. Args: resource_id: The resource id. """ self.resources.update(ProcSet(resource_id))
def generate_stage_jobs(self, job_info, stage): assert stage in ['in', 'out'] job = job_info.job register_profiles = {} register_jobs = [] for storage_id, nb_bytes in job_info.burst_buffers.items(): job_id = job.id + '_' + stage + '_' + str(storage_id) profile_id = job.profile + '_' + job_id storage = self.storage_id_to_name[storage_id] if stage == 'in': profile = {'type': 'data_staging', 'nb_bytes': nb_bytes, 'from': 'pfs', 'to': storage} else: profile = {'type': 'data_staging', 'nb_bytes': nb_bytes, 'from': storage, 'to': 'pfs'} register_profiles[profile_id] = profile stage_job = Job(id=job_id, subtime=0, walltime=-1, res=1, profile=profile_id, json_dict="") stage_job.allocation = ProcSet(storage_id, self.pfs) stage_job.storage_mapping = {'pfs': self.pfs, storage: storage_id} register_jobs.append(stage_job) self.bs.register_profiles(job.workload, register_profiles) for rj in register_jobs: self.bs.register_job( id=rj.id, res=rj.requested_resources, walltime=rj.requested_time, profile_name=rj.profile ) return register_jobs
def test_add_resource(self): r = SetResourceStateBatsimRequest(10, [1, 2, 3], 1) r.add_resource(9) r.add_resource(10) r.add_resource(15) assert str(r.resources) == str(ProcSet(*[1, 2, 3, 9, 10, 15]))
def test_disjoint(self): pset = ProcSet(ProcInt(0, 3), ProcInt(7, 15)) assert str(pset) == '0-3 7-15' assert format(pset, ':,') == '0:3,7:15' assert format(pset) == str(pset) assert format(pset, '') == str(pset) assert repr(pset) == 'ProcSet((0, 3), (7, 15))' assert pset == eval(repr(pset))
def test_incompatible_operand_type(self, incompatible): pset = ProcSet() # apply comparison method by name comparison = getattr(pset, self.method)(incompatible) # check correctness of result assert comparison is NotImplemented
def test_valid_operand_type(self, valid): pset = ProcSet() # apply comparison method by name comparison = getattr(pset, self.method)(valid) # check correctness of result assert isinstance(comparison, bool)
def test_contiguous(self): pset = ProcSet(ProcInt(0, 7)) assert str(pset) == '0-7' assert format(pset, ':,') == '0:7' assert format(pset) == str(pset) assert format(pset, '') == str(pset) assert repr(pset) == 'ProcSet((0, 7))' assert pset == eval(repr(pset))
def test_request(self): mapping = {"pfs": 1, "nfs": 3} r = ExecuteJobBatsimRequest(10, "w!1", [1, 2, 3], mapping) assert r.timestamp == 10 assert r.type == BatsimRequestType.EXECUTE_JOB assert r.job_id == "w!1" assert str(r.alloc) == str(ProcSet(*[1, 2, 3])) assert r.storage_mapping == mapping
def test_small(self): pset = ProcSet(ProcInt(0, 1)) assert str(pset) == '0-1' assert format(pset, ':,') == '0:1' assert format(pset) == str(pset) assert format(pset, '') == str(pset) assert repr(pset) == 'ProcSet((0, 1))' assert pset == eval(repr(pset))
def test_empty(self): pset = ProcSet() assert str(pset) == '' assert format(pset, ':,') == '' assert format(pset) == str(pset) assert format(pset, '') == str(pset) assert repr(pset) == 'ProcSet()' assert pset == eval(repr(pset))
def test_single_point(self): pset = ProcSet(ProcInt(0, 0)) assert str(pset) == '0' assert format(pset, ':,') == '0' assert format(pset) == str(pset) assert format(pset, '') == str(pset) assert repr(pset) == 'ProcSet(0)' assert pset == eval(repr(pset))
def __init__(self, timestamp: float, data: dict) -> None: super().__init__(timestamp, BatsimEventType.NOTIFY) self.__notify_type = BatsimNotifyType[data["type"].upper()] self.__resources: Optional[Sequence[int]] = None external_events = (BatsimNotifyType.EVENT_MACHINE_AVAILABLE, BatsimNotifyType.EVENT_MACHINE_UNAVAILABLE) if self.__notify_type in external_events: self.__resources = list(ProcSet.from_str(data["resources"]))
class SetResourceStateBatsimRequest(BatsimRequest): """ Batsim Set Resource State request class. This request tells batsim to change the power state of a resource. Args: timestamp: The time which the request should occur. resources: A sequence of resources id. state_id: The power state id. Raises: ValueError: In case of invalid sequence size. AssertionError: In case of invalid arguments type. Attributes: resources: A set of resources id. state: The resource power state id to switch. """ def __init__(self, timestamp: float, resources: Sequence[int], state_id: int) -> None: if not resources: raise ValueError('Expected `resources` argument to have at least ' 'one resource, got {}.'.format(resources)) super().__init__(timestamp, BatsimRequestType.SET_RESOURCE_STATE) self.resources = ProcSet(*resources) self.state = state_id def add_resource(self, resource_id: int) -> None: """ Add a resource id to the request. Args: resource_id: The resource id. """ self.resources.update(ProcSet(resource_id)) def _get_data_dict(self) -> dict: """ Batsim data dict. Returns: A dict with the properties following Batsim format. """ return {"resources": str(self.resources), "state": str(self.state)}
def get_notify_machine_available(timestamp: float = 0, resources: Sequence[int] = (1, )) -> dict: d = { "timestamp": timestamp, "type": "NOTIFY", "data": { "type": "EVENT_MACHINE_AVAILABLE", "resources": str(ProcSet(*resources)), "timestamp": timestamp }, } return d
def get_resource_state_changed(timestamp: float = 0, resources: Sequence[int] = (1, 2, 3), state: int = 0) -> dict: d = { "timestamp": timestamp, "type": "RESOURCE_STATE_CHANGED", "data": { "resources": str(ProcSet(*resources)), "state": str(state) } } return d
def get_set_resource_state(timestamp: float = 0, resources: Sequence[int] = (1, 2, 3), state: str = "1") -> dict: d = { "timestamp": timestamp, "type": "SET_RESOURCE_STATE", "data": { "resources": str(ProcSet(*resources)), "state": state } } return d
def onNoMoreJobsInWorkloads(self): pstates_to_change = [] self.end_Workload = True for r in self.idle_machines: self.idle_machines.remove(r) self.machine_wait[r] = -1 self.switching_OFF_machines.add(r) self.machines_states[r] = State.SwitchingOFF.value pstates_to_change.append((PState.Sleep.value, (r, r))) for (val, (r1, r2)) in pstates_to_change: self.bs.set_resource_state(ProcSet(r1), val)
def change_resource_states(self) -> None: """Sends resource state changes to Batsim. This alters the Cores P-states in the simulation, thus affecting computational capability and power consumption. """ for pstate in (0, 1, 2, 3): resources = [ i for i, s in self.resource_manager.state_changes.items() if s == pstate ] if resources: self.bs.set_resource_state(ProcSet(*resources), pstate) # Reset for next decision step self.resource_manager.state_changes = {}
def __init__(self, timestamp: float, job_id: str, alloc: Sequence[int], storage_mapping: Dict[str, int] = None) -> None: super().__init__(timestamp, BatsimRequestType.EXECUTE_JOB) if storage_mapping: assert isinstance(storage_mapping, dict) assert all( isinstance(k, str) and isinstance(v, int) for k, v in storage_mapping.items()) self.job_id = job_id self.storage_mapping = storage_mapping self.alloc = ProcSet(*alloc)