def run_filter(raw, filter_function):
match = re.search(filter_function, raw)
if match:
return match.group(1)
else:
warning_msg("Filter returned None! Filter was: " + filter_function.pattern)
raise Exception("Filter failed!", FILTER_FAILED_ERROR)
def load_app_config(self):
"""Load the app config from file.
If it doesn't exist or is faulty, it will be reset to the default app config.
"""
app_config_path = self.get_app_config_path()
self.create_dir(app_config_path)
if app_config_path.is_file():
debug.debug_msg("app_config.json found, loading into app_data.config...")
try:
with open(app_config_path, "r") as file:
self.config = json.load(file)
except:
debug.warning_msg("app_config.json not valid, resetting app config...")
with open(app_config_path, "w") as file:
default_app_config = self.get_default_config()
json.dump(default_app_config, file, indent=4)
debug.debug_msg("app_config.json has been set up")
self.config = default_app_config
finally:
debug.debug_msg("app_data.config loaded!")
else:
debug.debug_msg(
"app_config.json has not been set up, initializing default config..."
)
with open(app_config_path, "w") as file:
default_app_config = self.get_default_config()
json.dump(default_app_config, file, indent=4)
debug.debug_msg("app_config.json has been set up")
self.config = default_app_config
debug.debug_msg("app_data.config loaded!")
def parent(self, cost_group):
if self.parent_allowed(cost_group):
self._parent = cost_group
else:
self._parent = None
debug.warning_msg(
f"Setting parent of {self.id} / {self.name} to {cost_group.id} / {cost_group.name} failed, due to recursion! Setting parent to None."
)
def run_filter(raw, filter_function):
match = re.search(filter_function, raw)
if match:
return match.group(1)
else:
warning_msg("Filter returned None! Filter was: " +
filter_function.pattern)
raise Exception("Filter failed!", FILTER_FAILED_ERROR)
def execute_and_capture_output_with_timeout(program, timeout):
try:
output_file = tempfile.SpooledTemporaryFile()
start = datetime.datetime.now()
process = subprocess.Popen(program, shell=True, stdout=output_file, stderr=subprocess.STDOUT)
while process.poll() is None:
time.sleep(0.5)
now = datetime.datetime.now()
if (now - start).seconds > timeout:
warning_msg("Timing out! (" + str((now-start).seconds) + "s)")
os.kill(process.pid, signal.SIGKILL)
os.waitpid(-1, os.WNOHANG)
raise Exception("Run timed out!", TIMEOUT_ERROR)
output_file.seek(0)
raw = output_file.read()
output_file.close()
return str(raw)
except subprocess.CalledProcessError as e:
# TODO: Work out how on earth I get information about the error!
raise Exception("Run failed!", FAILURE_ERROR)
def send_email(address, results, mailserver='localhost', formatter=json.dumps):
me = "marky@" + gethostname()
you = address
timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime())
msg = MIMEText(formatter(results))
msg['Subject'] = "marky: Experiment completed at " + timestamp
msg['From'] = me
msg['To'] = you
try:
s = smtplib.SMTP(mailserver)
s.sendmail(me, [you], msg.as_string())
s.quit()
except Exception:
warning_msg("Failed to send email!")
return
debug_msg(3, "Sent e-mail to " + address + " at " + timestamp + ".")
def execute_and_capture_output_with_timeout(program, timeout):
try:
output_file = tempfile.SpooledTemporaryFile()
start = datetime.datetime.now()
process = subprocess.Popen(program,
shell=True,
stdout=output_file,
stderr=subprocess.STDOUT)
while process.poll() is None:
time.sleep(0.5)
now = datetime.datetime.now()
if (now - start).seconds > timeout:
warning_msg("Timing out! (" + str((now - start).seconds) +
"s)")
os.kill(process.pid, signal.SIGKILL)
os.waitpid(-1, os.WNOHANG)
raise Exception("Run timed out!", TIMEOUT_ERROR)
output_file.seek(0)
raw = output_file.read()
output_file.close()
return str(raw)
except subprocess.CalledProcessError as e:
# TODO: Work out how on earth I get information about the error!
raise Exception("Run failed!", FAILURE_ERROR)
def calculate_mean(results, exp_name, column_name):
mean_table = {}
exp = results["experiments"][exp_name]
benchmarks = exp["benchmarks"].keys()
for bm_name in benchmarks:
if "runs" not in exp["benchmarks"][bm_name]:
warning_msg(exp_name)
warning_msg(bm_name)
warning_msg("No successful runs occurred for experiment.")
mean_table[bm_name] = (0, 0, 0)
continue
data = gather_data_points(exp["benchmarks"][bm_name]["runs"], column_name)
(lower, mean, upper) = calculate_CI(data)
mean_table[bm_name] = (lower, mean, upper)
return mean_table
def check_ecd(suite):
debug.debug_msg(3, "Sanity checking the provided ECD...")
visible_vars = suite.__dict__.keys()
for (var_name, t, cannot_be_empty) in rules:
if var_name not in visible_vars:
debug.warning_msg("'" + var_name + "' variable must be present!")
else:
var = suite.__dict__[var_name]
if type(var) is not t:
debug.warning_msg("'" + var_name + "' must be of type: " + str(t))
if cannot_be_empty and len(var) == 0:
debug.warning_msg("'" + var_name + "' cannot be empty!")
if debug.seen_warnings():
debug.error_msg("Execution Configuration Description was invalid!")
debug.reset_warnings()
def check_ecd(suite):
debug.debug_msg(3, "Sanity checking the provided ECD...")
visible_vars = suite.__dict__.keys()
for (var_name, t, cannot_be_empty) in rules:
if var_name not in visible_vars:
debug.warning_msg("'" + var_name + "' variable must be present!")
else:
var = suite.__dict__[var_name]
if type(var) is not t:
debug.warning_msg("'" + var_name + "' must be of type: " +
str(t))
if cannot_be_empty and len(var) == 0:
debug.warning_msg("'" + var_name + "' cannot be empty!")
if debug.seen_warnings():
debug.error_msg("Execution Configuration Description was invalid!")
debug.reset_warnings()
def calculate_speedup(results, exp_a_name, exp_b_name, column_name):
speedup_table = {}
exp_a = results["experiments"][exp_a_name]
exp_b = results["experiments"][exp_b_name]
benchmarks = exp_a["benchmarks"].keys()
for bm_name in benchmarks:
if "runs" not in exp_a["benchmarks"][bm_name]:
warning_msg(exp_a_name)
warning_msg(bm_name)
warning_msg("No successful runs occurred for experiment.")
speedup_table[bm_name] = (1.0, 1.0, 1.0)
continue
if "runs" not in exp_b["benchmarks"][bm_name]:
warning_msg(exp_b_name)
warning_msg(bm_name)
warning_msg("No successful runs occurred for experiment.")
speedup_table[bm_name] = (1.0, 1.0, 1.0)
continue
data_a = gather_data_points(exp_a["benchmarks"][bm_name]["runs"], column_name)
data_b = gather_data_points(exp_b["benchmarks"][bm_name]["runs"], column_name)
average = numpy.mean(data_a)
speedups = []
for v in data_b:
speedup = float(average) / v
speedups.append(speedup)
(lower, mean, upper) = calculate_CI(speedups)
speedup_table[bm_name] = (lower, mean, upper)
return speedup_table
