def test_generator(self):
"""
The configuration object should have some sane defaults
"""
# If all of the required arguments are supplied, this should result in a valid job
ts_complete_set = {
tsk: TimeSignal.from_values(tsk, [0., 0.1], [1., 999.])
for tsk in time_signal_names
}
valid_args = {
'time_start': 0.1,
'duration': 0.2,
'ncpus': 1,
'nnodes': 1,
'timesignals': ts_complete_set
}
ts_complete_set_2 = {
tsk: TimeSignal.from_values(tsk, [0., 0.1], [1., 444.])
for tsk in time_signal_names
}
valid_args_2 = {
'time_start': 0.1,
'duration': 0.2,
'ncpus': 1,
'nnodes': 1,
'timesignals': ts_complete_set_2
}
# check that it is a valid job
job1 = ModelJob(**valid_args)
job1.label = "job1"
job2 = ModelJob(**valid_args_2)
job2.label = "job2"
job3 = ModelJob(**valid_args)
job3.label = "job3"
job4 = ModelJob(**valid_args_2)
job4.label = "job4"
job5 = ModelJob(**valid_args)
job5.label = "job5"
input_jobs = [job1, job2, job3, job4, job5]
# diversify the time start..
for jj, job in enumerate(input_jobs):
job.time_start += jj * 0.1
for job in input_jobs:
self.assertTrue(job.is_valid())
config_generator = {
"type": "cluster_and_spawn",
"job_clustering": {
"type": "Kmeans",
"rseed": 0,
"apply_to": ["test_wl_0"],
"ok_if_low_rank": True,
"max_iter": 100,
"max_num_clusters": 3,
"delta_num_clusters": 1,
"num_timesignal_bins": 1,
"user_does_not_check": True
},
"job_submission_strategy": {
"type": "match_job_pdf_exact",
"n_bins_for_pdf": 20,
"submit_rate_factor": 8,
"total_submit_interval": 60,
"random_seed": 0
}
}
# select the appropriate workload_filling strategy
workloads = [
Workload(jobs=input_jobs, tag='test_wl_0'),
Workload(jobs=input_jobs, tag='test_wl_1'),
Workload(jobs=input_jobs, tag='test_wl_2')
]
workload_modeller = workload_modelling_types[config_generator["type"]](
workloads)
workload_modeller.apply(config_generator)
# get the newly created set of (modelled) workloads
workload_set = workload_modeller.get_workload_set()
# make sure that we are creating only one workload
self.assertEqual(len(workload_set.workloads), 1)
# ---- check that all the jobs are correctly formed.. ----
# check that each job has time-signals as expected..
for job in workload_set.workloads[0].jobs:
self.assertTrue(hasattr(job, "timesignals"))
# check that each job has all the time-signals at this point..
for job in workload_set.workloads[0].jobs:
self.assertTrue(
all([k in job.timesignals.keys() for k in time_signal_names]))
def test_splitter(self):
# -------------- prepare a couple of dummy jobs ---------------
# If all of the required arguments are supplied, this should result in a valid job
ts_complete_set = {
tsk: TimeSignal.from_values(tsk, [0., 0.1], [1., 999.])
for tsk in time_signal_names
}
ts_complete_set_2 = {
tsk: TimeSignal.from_values(tsk, [0., 0.1], [1., 444.])
for tsk in time_signal_names
}
valid_args = {
'time_start': 0.1,
'duration': 0.2,
'ncpus': 1,
'nnodes': 1,
'timesignals': ts_complete_set,
'job_name': "job_name_1"
}
valid_args_2 = {
'time_start': 0.2,
'duration': 0.4,
'ncpus': 2,
'nnodes': 2,
'timesignals': ts_complete_set_2,
'job_name': "job_name_2"
}
# a model job that WILL NOT be picked by the algorithm..
job1 = ModelJob(**valid_args)
job1.label = "label_nottobepicked"
# a model job that WILL be picked by the algorithm..
job2 = ModelJob(**valid_args_2)
job2.label = "label_includeme"
# dummy workload with 20 jobs
np.random.seed(0)
jobs_all = []
for i in range(20):
# spawn a new job from either job1 or job2
if np.random.rand() < 0.5:
new_job = copy.deepcopy(job1)
else:
new_job = copy.deepcopy(job2)
# assign it a new label
jobs_all.append(new_job)
# create a workload out of all the jobs..
workload = Workload(jobs=jobs_all, tag="testing_workload")
# configure the splitter from user config
config_splitting = {
"type": "split",
"keywords_in": ["includeme"],
"keywords_out": ["excludeme"],
"split_by": "label",
"apply_to": ["testing_workload"],
"create_workload": "spawn_workload"
}
workloads = [workload]
splitter = WorkloadSplit(workloads)
splitter.apply(config_splitting)
wl_out = None
for wl in workloads:
if wl.tag == config_splitting["create_workload"]:
wl_out = wl
break
# make sure that we have created a workload as expected
self.assertTrue(wl_out is not None)
self.assertEqual(wl_out.tag, config_splitting["create_workload"])
# make sure that all the jobs have a label consistent with the filter
for j in wl_out.jobs:
self.assertTrue("includeme" in j.label
and "excludeme" not in j.label)