class _TestComparisonNonOperator(_TestComparison):
valid_types = (
# empty iterable
set(),
(),
# Iterable[int]
ProcSet(0),
ProcInt(0),
{0},
pytest.param((i * i for i in range(4)), id='(i*i for i in range(4))'),
# Iterable[ProcInt]
(ProcInt(0), ProcInt(1)),
# Iterable[ProcSet]
(ProcSet(0), ProcSet(1)),
# Iterable[Union[int, ProcInt, ProcSet]]
(0, ProcInt(1), ProcSet(2)),
)
incompatible_types = (
# not iterable
None,
0,
# iterable of wrong type
'bad-iterable',
{None},
(0, ProcInt(1), None),
)
def on_host_power_state_changed(self, sender: Host) -> None:
if sender.id in self.__last_pstate_id and sender.pstate and self.__last_pstate_id[
sender.id] != sender.pstate.id:
assert sender.pstate
assert self.simulator
self.__last_pstate_id[sender.id] = sender.pstate.id
if sender.pstate.type == PowerStateType.SWITCHING_OFF:
new_pstate_id = -2
elif sender.pstate.type == PowerStateType.SWITCHING_ON:
new_pstate_id = -1
else:
new_pstate_id = sender.pstate.id
if self.__info['time'][
-1] == self.simulator.current_time and self.__info[
'new_pstate'][-1] == new_pstate_id:
# Update last record
procset = ProcSet.from_str(self.__info['machine_id'][-1])
procset.update(ProcSet(sender.id))
self.__info['machine_id'][-1] = str(procset)
else:
# Record new state
self.__info['time'].append(self.simulator.current_time)
self.__info['machine_id'].append(str(sender.id))
self.__info['new_pstate'].append(new_pstate_id)
def test_deepcopy_nonempty(self):
pset = ProcSet(ProcInt(0, 3))
dcopy_pset = copy.deepcopy(pset)
assert dcopy_pset == pset
assert dcopy_pset is not pset
assert dcopy_pset._itvs is not pset._itvs
pset |= ProcSet(ProcInt(128, 255))
assert dcopy_pset != pset
def test_copy_empty(self):
pset = ProcSet()
copy_pset = copy.copy(pset)
assert copy_pset == pset
assert copy_pset is not pset
assert copy_pset._itvs is not pset._itvs
pset |= ProcSet(ProcInt(128, 255))
assert copy_pset != pset
def free_slots(self, begin_time=0, end_time=None):
'''
:returns: a DataFrame (compatible with a JobSet) that contains all
the not overlapping square free slots of this JobSet maximzing the
time. It can be transform to a JobSet to be plot as gantt chart.
'''
# slots_time contains tuple of
# (slot_begin_time,free_resources_intervals)
free_interval_serie = self.free_intervals(begin_time, end_time)
slots_time = [(free_interval_serie.time[0], ProcSet(self.res_bounds))]
new_slots_time = slots_time
columns = ['jobID', 'allocated_resources',
'starting_time', 'finish_time', 'execution_time',
'submission_time']
free_slots_df = pd.DataFrame(columns=columns)
prev_free_itvs = ProcSet(self.res_bounds)
slots = 0
for i, curr_row in free_interval_serie.iterrows():
if i == 0:
continue
new_slots_time = []
curr_time = curr_row.time
taken_resources = prev_free_itvs - curr_row.free_itvs
freed_resources = curr_row.free_itvs - prev_free_itvs
if i == len(free_interval_serie) - 1:
taken_resources = ProcSet(self.res_bounds)
if taken_resources:
# slot ends: store it and update free slot
for begin_time, itvs in slots_time:
to_update = itvs & taken_resources
if to_update:
# store new slots
slots = slots + 1
new_slot = [str(slots),
to_update,
begin_time,
curr_time,
curr_time - begin_time,
begin_time]
free_slots_df.loc[slots] = new_slot
# remove free slots
free_res = itvs - to_update
if free_res:
new_slots_time.append((begin_time, free_res))
else:
new_slots_time.append((begin_time, itvs))
if freed_resources:
# slots begin: udpate free slot
if not new_slots_time:
new_slots_time = slots_time
new_slots_time.append((curr_time, freed_resources))
# update previous
prev_free_itvs = curr_row.free_itvs
# clean slots_free
slots_time = new_slots_time
return free_slots_df
def __init__(self, timestamp: float, resources: Sequence[int],
state_id: int) -> None:
if not resources:
raise ValueError('Expected `resources` argument to have at least '
'one resource, got {}.'.format(resources))
super().__init__(timestamp, BatsimRequestType.SET_RESOURCE_STATE)
self.resources = ProcSet(*resources)
self.state = state_id
def test_copy_nested(self):
pset = ProcSet(ProcInt(0, 3))
nested = {0: pset, 1: [pset]}
copy_nested = copy.copy(nested)
assert copy_nested[0] == pset
assert copy_nested[0] is pset
assert copy_nested[0] == copy_nested[1][0]
assert copy_nested[0] is copy_nested[1][0]
pset |= ProcSet(ProcInt(128, 255))
assert copy_nested[0] == pset
assert copy_nested[0] == copy_nested[1][0]
def test_comparison(self, testcase):
left_pset = ProcSet(*testcase.left)
right_pset = ProcSet(*testcase.right)
# apply comparison method by name
comparison = getattr(left_pset, self.method)(right_pset)
set_comparison = getattr(set(left_pset), self.method)(set(right_pset))
# check correctness of result
assert isinstance(comparison, bool)
assert comparison == set_comparison
def test_deepcopy_nested(self):
pset = ProcSet(ProcInt(0, 3))
nested = {0: pset, 1: [pset]}
dcopy_nested = copy.deepcopy(nested)
assert dcopy_nested[0] == pset
assert dcopy_nested[0] is not pset
assert dcopy_nested[0] == dcopy_nested[1][0]
assert dcopy_nested[0] is dcopy_nested[1][0]
pset |= ProcSet(ProcInt(128, 255))
assert dcopy_nested[0] != pset
assert dcopy_nested[0] == dcopy_nested[1][0]
def _schedule_data_staging(
self,
job: Job,
source: StorageResource,
destination: StorageResource,
walltime=None
):
allocation = StorageAllocation(
start_time=self.time,
walltime=walltime if walltime else -1,
resources=[source, destination],
job=job
)
allocation.allocate_all(self)
job._allocation = allocation
job._batsim_job.allocation = ProcSet(source.id, destination.id)
job._batsim_job.storage_mapping = {
'source': source.id,
'destination': destination.id
}
self._batsim.execute_job(job._batsim_job)
# Copy from the Job.schedule() method; job.py:569
self.info(
"Scheduled job ({job})",
job=job,
type="job_scheduled")
job._scheduled = True
job._start_time = self.time
job._jobs_list.update_element(self)
def test_request(self):
r = SetResourceStateBatsimRequest(10, [1, 2, 3], 1)
assert r.timestamp == 10
assert r.type == BatsimRequestType.SET_RESOURCE_STATE
assert str(r.resources) == str(ProcSet(*[1, 2, 3]))
assert r.state == 1
def add_resource(self, resource_id: int) -> None:
""" Add a resource id to the request.
Args:
resource_id: The resource id.
"""
self.resources.update(ProcSet(resource_id))
def generate_stage_jobs(self, job_info, stage):
assert stage in ['in', 'out']
job = job_info.job
register_profiles = {}
register_jobs = []
for storage_id, nb_bytes in job_info.burst_buffers.items():
job_id = job.id + '_' + stage + '_' + str(storage_id)
profile_id = job.profile + '_' + job_id
storage = self.storage_id_to_name[storage_id]
if stage == 'in':
profile = {'type': 'data_staging', 'nb_bytes': nb_bytes, 'from': 'pfs', 'to': storage}
else:
profile = {'type': 'data_staging', 'nb_bytes': nb_bytes, 'from': storage, 'to': 'pfs'}
register_profiles[profile_id] = profile
stage_job = Job(id=job_id, subtime=0, walltime=-1, res=1, profile=profile_id, json_dict="")
stage_job.allocation = ProcSet(storage_id, self.pfs)
stage_job.storage_mapping = {'pfs': self.pfs, storage: storage_id}
register_jobs.append(stage_job)
self.bs.register_profiles(job.workload, register_profiles)
for rj in register_jobs:
self.bs.register_job(
id=rj.id,
res=rj.requested_resources,
walltime=rj.requested_time,
profile_name=rj.profile
)
return register_jobs
def test_add_resource(self):
r = SetResourceStateBatsimRequest(10, [1, 2, 3], 1)
r.add_resource(9)
r.add_resource(10)
r.add_resource(15)
assert str(r.resources) == str(ProcSet(*[1, 2, 3, 9, 10, 15]))
def test_disjoint(self):
pset = ProcSet(ProcInt(0, 3), ProcInt(7, 15))
assert str(pset) == '0-3 7-15'
assert format(pset, ':,') == '0:3,7:15'
assert format(pset) == str(pset)
assert format(pset, '') == str(pset)
assert repr(pset) == 'ProcSet((0, 3), (7, 15))'
assert pset == eval(repr(pset))
def test_incompatible_operand_type(self, incompatible):
pset = ProcSet()
# apply comparison method by name
comparison = getattr(pset, self.method)(incompatible)
# check correctness of result
assert comparison is NotImplemented
def test_valid_operand_type(self, valid):
pset = ProcSet()
# apply comparison method by name
comparison = getattr(pset, self.method)(valid)
# check correctness of result
assert isinstance(comparison, bool)
def test_contiguous(self):
pset = ProcSet(ProcInt(0, 7))
assert str(pset) == '0-7'
assert format(pset, ':,') == '0:7'
assert format(pset) == str(pset)
assert format(pset, '') == str(pset)
assert repr(pset) == 'ProcSet((0, 7))'
assert pset == eval(repr(pset))
def test_request(self):
mapping = {"pfs": 1, "nfs": 3}
r = ExecuteJobBatsimRequest(10, "w!1", [1, 2, 3], mapping)
assert r.timestamp == 10
assert r.type == BatsimRequestType.EXECUTE_JOB
assert r.job_id == "w!1"
assert str(r.alloc) == str(ProcSet(*[1, 2, 3]))
assert r.storage_mapping == mapping
def test_small(self):
pset = ProcSet(ProcInt(0, 1))
assert str(pset) == '0-1'
assert format(pset, ':,') == '0:1'
assert format(pset) == str(pset)
assert format(pset, '') == str(pset)
assert repr(pset) == 'ProcSet((0, 1))'
assert pset == eval(repr(pset))
def test_empty(self):
pset = ProcSet()
assert str(pset) == ''
assert format(pset, ':,') == ''
assert format(pset) == str(pset)
assert format(pset, '') == str(pset)
assert repr(pset) == 'ProcSet()'
assert pset == eval(repr(pset))
def test_single_point(self):
pset = ProcSet(ProcInt(0, 0))
assert str(pset) == '0'
assert format(pset, ':,') == '0'
assert format(pset) == str(pset)
assert format(pset, '') == str(pset)
assert repr(pset) == 'ProcSet(0)'
assert pset == eval(repr(pset))
def __init__(self, timestamp: float, data: dict) -> None:
super().__init__(timestamp, BatsimEventType.NOTIFY)
self.__notify_type = BatsimNotifyType[data["type"].upper()]
self.__resources: Optional[Sequence[int]] = None
external_events = (BatsimNotifyType.EVENT_MACHINE_AVAILABLE,
BatsimNotifyType.EVENT_MACHINE_UNAVAILABLE)
if self.__notify_type in external_events:
self.__resources = list(ProcSet.from_str(data["resources"]))
class SetResourceStateBatsimRequest(BatsimRequest):
""" Batsim Set Resource State request class.
This request tells batsim to change the power state of a resource.
Args:
timestamp: The time which the request should occur.
resources: A sequence of resources id.
state_id: The power state id.
Raises:
ValueError: In case of invalid sequence size.
AssertionError: In case of invalid arguments type.
Attributes:
resources: A set of resources id.
state: The resource power state id to switch.
"""
def __init__(self, timestamp: float, resources: Sequence[int],
state_id: int) -> None:
if not resources:
raise ValueError('Expected `resources` argument to have at least '
'one resource, got {}.'.format(resources))
super().__init__(timestamp, BatsimRequestType.SET_RESOURCE_STATE)
self.resources = ProcSet(*resources)
self.state = state_id
def add_resource(self, resource_id: int) -> None:
""" Add a resource id to the request.
Args:
resource_id: The resource id.
"""
self.resources.update(ProcSet(resource_id))
def _get_data_dict(self) -> dict:
""" Batsim data dict.
Returns:
A dict with the properties following Batsim format.
"""
return {"resources": str(self.resources), "state": str(self.state)}
def get_notify_machine_available(timestamp: float = 0,
resources: Sequence[int] = (1, )) -> dict:
d = {
"timestamp": timestamp,
"type": "NOTIFY",
"data": {
"type": "EVENT_MACHINE_AVAILABLE",
"resources": str(ProcSet(*resources)),
"timestamp": timestamp
},
}
return d
def get_resource_state_changed(timestamp: float = 0,
resources: Sequence[int] = (1, 2, 3),
state: int = 0) -> dict:
d = {
"timestamp": timestamp,
"type": "RESOURCE_STATE_CHANGED",
"data": {
"resources": str(ProcSet(*resources)),
"state": str(state)
}
}
return d
def get_set_resource_state(timestamp: float = 0,
resources: Sequence[int] = (1, 2, 3),
state: str = "1") -> dict:
d = {
"timestamp": timestamp,
"type": "SET_RESOURCE_STATE",
"data": {
"resources": str(ProcSet(*resources)),
"state": state
}
}
return d
def onNoMoreJobsInWorkloads(self):
pstates_to_change = []
self.end_Workload = True
for r in self.idle_machines:
self.idle_machines.remove(r)
self.machine_wait[r] = -1
self.switching_OFF_machines.add(r)
self.machines_states[r] = State.SwitchingOFF.value
pstates_to_change.append((PState.Sleep.value, (r, r)))
for (val, (r1, r2)) in pstates_to_change:
self.bs.set_resource_state(ProcSet(r1), val)
def change_resource_states(self) -> None:
"""Sends resource state changes to Batsim.
This alters the Cores P-states in the simulation, thus affecting computational capability and power
consumption.
"""
for pstate in (0, 1, 2, 3):
resources = [
i for i, s in self.resource_manager.state_changes.items()
if s == pstate
]
if resources:
self.bs.set_resource_state(ProcSet(*resources), pstate)
# Reset for next decision step
self.resource_manager.state_changes = {}
def __init__(self,
timestamp: float,
job_id: str,
alloc: Sequence[int],
storage_mapping: Dict[str, int] = None) -> None:
super().__init__(timestamp, BatsimRequestType.EXECUTE_JOB)
if storage_mapping:
assert isinstance(storage_mapping, dict)
assert all(
isinstance(k, str) and isinstance(v, int)
for k, v in storage_mapping.items())
self.job_id = job_id
self.storage_mapping = storage_mapping
self.alloc = ProcSet(*alloc)
