def setUp(self):
unittest.TestCase.setUp(self)
# This _gerenator will allways produce "3" as a number
self._generator = ProbabilityMap(probabilities=[1.0],
value_ranges=[(3, 4)],
interval_policy="low")
self._controller = TimeController(self._generator)
def set_alarm_list_date(self, datetime_list):
""" Sets a list of future timestamps when alarm should be triggered.
Args:
- alarm_list: growing numeric datetime when the alarm should
be triggered. Raises ValueError if any value is smaller than
current timestamp or values not ordered.
"""
self.set_alarm_list(
[TimeController.get_epoch(x) for x in datetime_list])
def test_is_time_to_stop(self):
new_date = int(
TimeController.get_epoch(datetime.datetime(2015, 0o1, 0o1)))
self._controller.reset_time_counter(datetime.datetime(2015, 0o1, 0o1))
self._controller.set_run_limit(6)
self.assertEqual(new_date + 3,
self._controller.get_next_job_create_time())
self.assertFalse(self._controller.is_time_to_stop())
self.assertEqual(new_date + 6,
self._controller.get_next_job_create_time())
self._controller.get_next_job_create_time()
self.assertTrue(self._controller.is_time_to_stop())
def test_set_alarm_list_date(self):
base_time = TimeController.get_epoch(da(1970, 1, 1, 00, 00, 00))
ma = MultiAlarmTimer(None,
register_datetime=datetime.datetime(
1970, 1, 1, 00, 00, 10))
ma.set_alarm_list_date(
[da(1970, 1, 1, 00, 00, 12),
da(1970, 1, 1, 00, 00, 20)])
self.assertFalse(ma.is_it_time(base_time + 11))
self.assertTrue(ma.is_it_time(base_time + 12))
self.assertFalse(ma.is_it_time(base_time + 15))
self.assertTrue(ma.is_it_time(base_time + 20))
def test_init(self):
ma = MultiAlarmTimer(None)
self.assertEqual(ma._alarm_list, [])
ma.register_time(100)
self.assertEqual(ma._current_timestamp, 100)
self.assertEqual(ma._register_timestamp, 100)
ma = MultiAlarmTimer(None, 200)
self.assertEqual(ma._current_timestamp, 200)
self.assertEqual(ma._register_timestamp, 200)
base_time = TimeController.get_epoch(da(1970, 1, 1, 00, 00, 00))
ma = MultiAlarmTimer(None,
register_datetime=datetime.datetime(
1970, 1, 1, 00, 00, 10))
self.assertEqual(ma._current_timestamp, base_time + 10)
self.assertEqual(ma._register_timestamp, base_time + 10)
def test_do_trigger(self):
gen = FakeGen(None)
base_time = TimeController.get_epoch(da(1970, 1, 1, 00, 00, 00))
ma = MultiAlarmTimer(gen,
register_datetime=datetime.datetime(
1970, 1, 1, 00, 00, 10))
ma.set_alarm_list_date([
da(1970, 1, 1, 00, 00, 12),
da(1970, 1, 1, 00, 00, 12),
da(1970, 1, 1, 00, 00, 20)
])
self.assertEqual(ma.do_trigger(base_time + 11), 0)
self.assertEqual(ma.do_trigger(base_time + 12), 2)
self.assertEqual(ma.do_trigger(base_time + 15), 0)
self.assertEqual(ma.do_trigger(base_time + 20), 1)
self.assertEqual(gen._count, 3)
def regiter_datetime(self, the_datetime):
"""Sets the current time stamp and time of registration of the generator
to the datetime object the_datetime."""
self.register_time(TimeController.get_epoch(the_datetime))
class TestTimeController(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
# This _gerenator will allways produce "3" as a number
self._generator = ProbabilityMap(probabilities=[1.0],
value_ranges=[(3, 4)],
interval_policy="low")
self._controller = TimeController(self._generator)
def test_creation(self):
self.assertEqual(self._controller._time_counter, 0)
self.assertEqual(self._controller._start_date, 0)
self.assertEqual(self._controller._run_limit, 0)
def test_reset_time_counter(self):
right_now = int(datetime.datetime.now().strftime('%s'))
self._controller.reset_time_counter()
self.assertLess(self._controller._time_counter - right_now, 2)
self.assertLess(self._controller._start_date - right_now, 2)
new_date = int(datetime.datetime(2015, 0o1, 0o1).strftime('%s'))
self._controller.reset_time_counter(datetime.datetime(2015, 0o1, 0o1))
self.assertEqual(self._controller._time_counter, new_date)
self.assertEqual(self._controller._start_date, new_date)
def test_get_next_job_create_time(self):
new_date = int(datetime.datetime(2015, 0o1, 0o1).strftime('%s'))
self._controller.reset_time_counter(datetime.datetime(2015, 0o1, 0o1))
self.assertEqual(new_date + 3,
self._controller.get_next_job_create_time())
self.assertEqual(new_date + 6,
self._controller.get_next_job_create_time())
def test_is_time_to_stop(self):
new_date = int(
TimeController.get_epoch(datetime.datetime(2015, 0o1, 0o1)))
self._controller.reset_time_counter(datetime.datetime(2015, 0o1, 0o1))
self._controller.set_run_limit(6)
self.assertEqual(new_date + 3,
self._controller.get_next_job_create_time())
self.assertFalse(self._controller.is_time_to_stop())
self.assertEqual(new_date + 6,
self._controller.get_next_job_create_time())
self._controller.get_next_job_create_time()
self.assertTrue(self._controller.is_time_to_stop())
def get_end_epoch(self):
"""
Returns the ending date of the experiment in epoch format (int).
"""
return (TimeController.get_epoch(self._start_date) +
self._workload_duration_s)
def get_start_epoch(self):
"""
Returns the start date of the experiment in epoch format (int).
"""
return TimeController.get_epoch(self._start_date)
def _generate_trace_files(self, definition, trace_generator=None):
"""Creates the workload files according an Experiment definition.
Args:
- definition: Definition object defining the experiment.
"""
if trace_generator is None:
trace_generator = TraceGenerator()
print(("This is the seed to be used:", definition._seed))
random_control.set_global_random_gen(seed=definition._seed)
machine = definition.get_machine()
(filter_cores, filter_runtime,
filter_core_hours) = machine.get_filter_values()
wg = WorkloadGenerator(machine=definition.get_machine(),
trace_generator=trace_generator,
user_list=definition.get_user_list(),
qos_list=definition.get_qos_list(),
partition_list=definition.get_partition_list(),
account_list=definition.get_account_list())
if definition._workflow_policy.split("-")[0] == "sp":
special_gen = SpecialGenerators.get_generator(
definition._workflow_policy,
wg,
register_datetime=(definition._start_date -
timedelta(0, definition._preload_time_s)))
wg.register_pattern_generator_timer(special_gen)
else:
wg.config_filter_func(machine.job_can_be_submitted)
wg.set_max_interarrival(machine.get_max_interarrival())
if definition._trace_type != "single":
raise ValueError("Only 'single' experiments require trace "
"generation")
if definition.get_overload_factor() > 0.0:
print(("doing overload:", definition.get_overload_factor()))
max_cores = machine.get_total_cores()
single_job_gen = PatternGenerator(wg)
overload_time = OverloadTimeController(
single_job_gen,
register_datetime=(
definition._start_date -
timedelta(0, definition._preload_time_s)))
overload_time.configure_overload(
trace_generator,
max_cores,
overload_target=definition.get_overload_factor())
print(("about to register", wg, overload_time))
wg.register_pattern_generator_timer(overload_time)
manifest_list = [m["manifest"] for m in definition._manifest_list]
share_list = [m["share"] for m in definition._manifest_list]
if (definition._workflow_handling == "single"
or definition._workflow_handling == "manifest"):
flow = WorkflowGeneratorSingleJob(manifest_list, share_list,
wg)
else:
flow = WorkflowGeneratorMultijobs(manifest_list, share_list,
wg)
if definition._workflow_policy == "period":
alarm = RepeatingAlarmTimer(
flow, register_datetime=definition._start_date)
alarm.set_alarm_period(definition._workflow_period_s)
wg.register_pattern_generator_timer(alarm)
elif definition._workflow_policy == "percentage":
wg.register_pattern_generator_share(
flow, definition._workflow_share / 100)
target_wait = definition.get_forced_initial_wait()
if target_wait:
default_job_separation = 10
separation = int(
os.getenv("FW_JOB_SEPARATION", default_job_separation))
filler(wg,
start_time=TimeController.get_epoch(
(definition._start_date -
timedelta(0, definition._preload_time_s))),
target_wait=target_wait,
max_cores=machine.get_total_cores(),
cores_per_node=machine._cores_per_node,
job_separation=separation)
trace_generator.reset_work()
wg.generate_trace(
(definition._start_date -
timedelta(0, definition._preload_time_s)),
(definition._preload_time_s + definition._workload_duration_s))
max_cores = machine.get_total_cores()
total_submitted_core_s = trace_generator.get_total_submitted_core_s()
job_pressure = (float(total_submitted_core_s) / float(
(definition._preload_time_s + definition._workload_duration_s) *
max_cores))
print(("Observed job pressure (bound): {0}".format(job_pressure)))
trace_generator.dump_trace(
path.join(ExperimentRunner._trace_generation_folder,
definition.get_trace_file_name()))
trace_generator.dump_qos(
path.join(ExperimentRunner._trace_generation_folder,
definition.get_qos_file_name()))
trace_generator.dump_users(
path.join(ExperimentRunner._trace_generation_folder,
definition.get_users_file_name()),
extra_users=definition.get_system_user_list())
trace_generator.free_mem()
return [
definition.get_trace_file_name(),
definition.get_qos_file_name(),
definition.get_users_file_name()
]
def test_generate_trace_files_first_job(self):
ExperimentRunner.configure("tmp/trace_folder",
"tmp",
True,
"myhost",
"myUser",
drain_time=0)
self.assertEqual(ExperimentRunner._trace_folder, "tmp/trace_folder")
self.assertEqual(ExperimentRunner._trace_generation_folder, "tmp")
self.assertEqual(ExperimentRunner._local, True)
ed = ExperimentDefinition(seed="seeeed",
machine="edison",
trace_type="single",
manifest_list=[{
"share": 1.0,
"manifest": "manifestSim.json"
}],
workflow_policy="period",
workflow_period_s=5,
workflow_handling="single",
preload_time_s=20,
start_date=datetime(2016, 1, 1),
workload_duration_s=400,
overload_target=3600000)
er = ExperimentRunner(ed)
er._generate_trace_files(ed)
self.assertTrue(
os.path.exists("tmp/edison-single-m1.0manifestSim.json"
"-period-p5-0.0-single-t-0d-0d-O3600000"
"-sseeeed.trace"))
self.assertTrue(
os.path.exists("tmp/edison-single-m1.0manifestSim.json"
"-period-p5-0.0-single-t-0d-0d-O3600000"
"-sseeeed.qos"))
self.assertTrue(
os.path.exists("tmp/edison-single-m1.0manifestSim.json"
"-period-p5-0.0-single-t-0d-0d-O3600000"
"-sseeeed.users"))
records = trace_gen.extract_records(
file_name="tmp/edison-single-m1.0manifestSim.json"
"-period-p5-0.0-single-t-0d-0d-O3600000"
"-sseeeed.trace",
list_trace_location="../bin/list_trace")
man_count = 0
self.assertGreater(
int(records[-1]["SUBMIT"]) - int(records[0]["SUBMIT"]), 320)
self.assertLess(
int(records[-1]["SUBMIT"]) - int(records[0]["SUBMIT"]),
1500 + 3720)
for rec in records:
if rec["WF"].split("-")[0] == "manifestSim.json":
man_count += 1
self.assertGreaterEqual(
man_count, 64, "There should be at least 80"
" workflows in the "
"trace, found: {0}".format(man_count))
self.assertLessEqual(
man_count, 104, "There should be at least 80"
" workflows in the "
"trace, found: {0}".format(man_count))
first_submit = TimeController.get_epoch(datetime(2016, 1,
1)) - 20 - 3600 - 120
for i in range(360):
self.assertEqual(int(records[i]["NUM_TASKS"]), 16 * 24)
self.assertEqual(int(records[i]["DURATION"]), 7320)
self.assertEqual(int(records[i]["WCLIMIT"]), 123)
self.assertEqual(int(records[i]["SUBMIT"]), first_submit)
first_submit += 10
self.assertGreaterEqual(
int(records[360]["SUBMIT"]),
TimeController.get_epoch(datetime(2016, 1, 1)) - 20)
self.assertNotEqual(int(records[360]["DURATION"]), 3600)