def cool_down(self, temp_threshold=80, stress_present=False):
"""
If the temperature is above a certain threshold, this function will delay
the next experiment until the chip has cooled down.
:param temp_threshold: The maximum temperature allowed
:param stress_present: If true, we need to kill the stress before cooldown
:return: If it was forced to kill stress
"""
killed_stress = False
temp = get_temp()
if temp:
while temp > temp_threshold:
print("Temperature " + str(temp) +
" is too high! Cooling down")
if stress_present:
self._processes.kill_stress()
killed_stress = True
sleep(10)
temp = get_temp()
print("Temperature " + str(temp) + " is ok. Running experiment")
else:
print("\n\tWARNING: Using default cooldown time of 30 s\n")
sleep(30)
return killed_stress
def run_sut_stress(self, sut, stress, cores):
"""
:param sut: System under stress
:param stress: Enemy process
:param cores: Number of cores to start the enemy on
:param style: Run the SUT with perf or some similar instrument
"""
# start up the stress on cores 1-cores+1
if cores > 0:
for i in range(1, cores + 1):
self.start_stress(stress, i)
# Run the program on core 0
s_out, s_err = self.run_program_single(sut, 0)
self._check_error(s_err)
if cores > 0:
self._processes.kill_stress()
if get_event(s_out, "total time(us): "):
ex_time = get_event(s_out, "total time(us): ")
elif get_event(s_out, "Total time (secs): "):
ex_time = get_event(s_out, "Total time (secs): ")
elif get_event(s_out, "Max: "):
ex_time = get_event(s_out, "Max: ")
elif get_event(s_out, "average = "):
ex_time = get_event(s_out, "average = ")
else:
print("Unable find execution time or maximum latency")
sys.exit(0)
ex_temp = get_temp()
return ex_time, ex_temp
def main():
citizen_rules_cfn = get_temp(RULES_TEMP_BASE).replace(
"{{citizen_rules}}",
get_rules_cf(get_rules())
# get_rules_cf([{'description': 'Placeholder', 'name': 'check_root_access_keys' }])
)
generate_file(TEMP_DESTINATION, citizen_rules_cfn)
def main():
roles_cf = get_temp(
TEMP_BASE).replace( # Update Citizen roles with Citizen account info
"{{citizen_roles}}", get_citizen_roles(get_accounts())
).replace( # Update CitizenUpdate roles with Citizen account info
"{{citizen_update_roles}}",
get_citizen_update_roles(get_accounts()))
generate_file(TEMP_DESTINATION, roles_cf) # Creates the deployable CF file
def main():
citizen_update_cf = get_temp(
TEMP_BASE
).replace( # Update bucket policy with child account information
"{{bucket_policy_accounts}}", get_bucket_policy_cf(get_accounts())
).replace( # Update CitizenUpdate lambda function with checksum details
"{{update_citizen_stacks}}", get_checksum_zip("update_citizen_stacks"))
generate_file(TEMP_DESTINATION,
citizen_update_cf) # Creates the deployable CF file
def main():
reporting_cf = get_temp(
TEMP_BASE
).replace( # Update CF template with all CloudWatch events
"{{event_rule}}", get_event_rule_cf(get_accounts())
).replace( # Update generate_compliance_report lambda function with checksum details
"{{generate_compliance_report}}",
get_checksum_zip("generate_compliance_report"))
generate_file(TEMP_DESTINATION,
reporting_cf) # Creates the deployable CF file
def main():
elasticsearch_cf = get_temp(
TEMP_BASE
).replace( # Update logs_to_elastic_search lambda function with checksum details
"{{logs_to_elastic_search}}",
get_checksum_zip("logs_to_elastic_search")
).replace( # Update roll_indexes lambda function with checksum details
"{{roll_indexes}}",
get_checksum_zip("roll_indexes")).replace("{{external_cidr}}",
get_external_cidr())
generate_file(TEMP_DESTINATION,
elasticsearch_cf) # Creates the deployable CF file
def main():
verification_rule_cf = get_temp(
RULES_TEMP_BASE
).replace( # Update cf with each rule stack
"{{verification_rules}}", get_rules_cf(get_rules())
).replace( # Update S3 bucket policy to only allow access from company's IP address range
"{{external_cidr}}",
get_external_cidr()).replace("{{notifications_slack}}",
get_notification_slack()).replace(
"{{slack_channel_hook_url}}",
get_slack_channel_hook_url()).replace(
"{{notifications_email}}",
get_notification_email())
generate_file(TEMP_DESTINATION, verification_rule_cf)
def run_mapping(self, experiment_info, mapping, iteration_name=None):
"""
Run a mapping described by a mapping object
:param experiment_info: An ExperimentInfo object
:param mapping: A dict of core mappings
:param iteration_name: For tuning, we can store the exact param
:return:
"""
assert isinstance(experiment_info, ExperimentInfo)
# Make sure the governor is correctly
cmd = "echo " + experiment_info.governor + \
" | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor"
self._processes.system_call(cmd)
delta_temp = 5
total_times = []
total_temps = []
perf_results = dict()
voluntary_switches = []
involuntary_switches = []
# start from 95 and decrease to 50 by 1
candidate_quantiles = [x / 100.0 for x in range(95, 49, -1)]
if iteration_name is None:
iteration_name = mapping
result = MappingResult(iteration_name)
# Initialise a perf result list
if self._instrument_cmd:
perf_results = []
while len(total_temps) < experiment_info.measurement_iterations_max:
it = 0
# start up the stress in accordance with the mapping
for core in mapping:
self.start_stress(mapping[core], core)
while it < experiment_info.measurement_iterations_step:
if self.cool_down(experiment_info.max_temperature - delta_temp,
mapping):
for core in mapping:
self.start_stress(mapping[core], core)
# Clear the cache first
cmd = "sync; echo 1 > /proc/sys/vm/drop_caches"
s_out, s_err = self._processes.system_call(cmd)
self._check_error(s_err)
# For perf, I need to think if we need to log all values, take an average...
# For the moment, an average should be fine
# Run the program on core 0
s_out, s_err = self.run_program_single(experiment_info.sut, 0)
if self._instrument_cmd:
perf_results.append(get_perf_event(s_err))
if self.get_switches(s_out) is not None:
(voluntary, involuntary) = self.get_switches(s_out)
voluntary_switches.append(voluntary)
involuntary_switches.append(involuntary)
final_temp = get_temp()
if final_temp < experiment_info.max_temperature:
total_times.append(self.get_metric(s_out))
total_temps.append(final_temp)
it = it + 1
else:
print(
"The final temperature was to high, redoing experiment"
)
delta_temp += 5
if delta_temp > 25:
print(
"The test heats up the processor more than 25 degrees, I o not know what to do"
)
exit(1)
if len(mapping) > 0:
self._processes.kill_stress()
# This part runs if we have variable iterations based on confidence interval
# and can stop early
if experiment_info.stopping == "no_decrease" or experiment_info.stopping == "optimistic":
(conf_var, conf_min, conf_max) = \
confidence_variation(times=total_times,
quantile=experiment_info.quantile,
confidence_interval=experiment_info.confidence_interval)
print("The confidence variation is ", conf_var)
if conf_var < experiment_info.max_confidence_variation:
result.log_result(
perf_results=perf_results,
total_times=total_times,
total_temps=total_temps,
quantile=experiment_info.quantile,
conf_min=conf_min,
conf_max=conf_max,
voluntary_switches=voluntary_switches,
involuntary_switches=involuntary_switches,
success=True)
print("The q value is", result.q_value)
self._processes.kill_stress()
return result
elif experiment_info.stopping == "pessimistic":
for q in candidate_quantiles:
(conf_var, conf_min, conf_max) = \
confidence_variation(times=total_times,
quantile=q,
confidence_interval=experiment_info.confidence_interval)
if conf_var < experiment_info.max_confidence_variation:
result.log_result(
perf_results=perf_results,
total_times=total_times,
total_temps=total_temps,
quantile=q,
conf_min=conf_min,
conf_max=conf_max,
voluntary_switches=voluntary_switches,
involuntary_switches=involuntary_switches,
success=True)
print("The q value is", result.q_value)
self._processes.kill_stress()
return result
# At this point we know that we have hit max iterations
if experiment_info.stopping == "optimistic":
for q in candidate_quantiles:
(conf_var, conf_min, conf_max) = \
confidence_variation(times=total_times,
quantile=q,
confidence_interval=experiment_info.confidence_interval)
if conf_var < experiment_info.max_confidence_variation:
result.log_result(
perf_results=perf_results,
total_times=total_times,
total_temps=total_temps,
quantile=q,
conf_min=conf_min,
conf_max=conf_max,
voluntary_switches=voluntary_switches,
involuntary_switches=involuntary_switches,
success=True)
print("The q value is", result.q_value)
self._processes.kill_stress()
return result
# If we hit this and we did not intend to (not using "fixed"), we failed
# to get a stable quantile basically
(conf_var, conf_min, conf_max) = \
confidence_variation(times=total_times,
quantile=experiment_info.quantile,
confidence_interval=experiment_info.confidence_interval)
result.log_result(
perf_results=perf_results,
total_times=total_times,
total_temps=total_temps,
quantile=experiment_info.quantile,
conf_min=conf_min,
conf_max=conf_max,
voluntary_switches=voluntary_switches,
involuntary_switches=involuntary_switches,
success=True
if conf_var < experiment_info.max_confidence_variation else False)
print("The q value is", result.q_value)
self._processes.kill_stress()
return result
def main():
verification_rule_cf = get_temp(
RULE_TEMP_BASE).replace( # Update cf with with resources for each rule
"{{verification_rule}}", get_rule_cf(get_accounts()))
generate_file(TEMP_DESTINATION, verification_rule_cf)
def main():
subscriptions_cf = get_temp(
RULES_TEMP_BASE).replace( # Update cf with each rule subscription
"{{rules-subscriptions}}", get_subscriptions_cf(get_rules()))
generate_file(TEMP_DESTINATION,
subscriptions_cf) # Creates the deployable CF file