def save_raw_output(name, iteration, raw): directory = config["saveraw_dir"] if not os.path.exists(directory): debug_msg(3, "Raw output directory doesn't exist! Creating...") os.mkdir(directory) filename = get_raw_filename(name, iteration) save_location = directory + "/" + filename f = open(save_location, "w") f.write(raw) f.close() debug_msg(3, "Saved raw output to " + save_location)
def run(suite): # The amorphous monstrosity of a table total_result_table = {} total_result_table["experiments"] = {} # Go through the programs... for (program_alias, program) in suite.programs.items(): # Check if there's argument variables that will require iterating over if ((len(suite.argument_variables) + len(suite.file_argument_variables)) == 0): exp_name = program_alias # There were none, so simply run this program with no extra arguments debug_msg(1, "BEGIN EXPERIMENT: " + exp_name) total_result_table["experiments"][exp_name] = run_experiment( suite, program, program_alias, exp_name) else: # There are some, so use args.get_experiment_arguments to get a list of all combos, iterate over them. for experiment_arguments in args.get_experiment_arguments( suite.argument_variables, suite.file_argument_variables): exp_params = "" experiment_arguments_string = "" for exp_arg in experiment_arguments: (name, value, is_file_arg_var) = exp_arg if is_file_arg_var: exp_params += (name + ":" + str(value) + " ") # change a config file (temp_file, output_file, pattern, values) = suite.file_argument_variables[name] update_based_on_file_argument(temp_file, output_file, pattern, value) else: exp_params += args.emit_argument(name, value) experiment_arguments_string += args.emit_argument( name, value) exp_name = program_alias + " " + exp_params debug_msg(1, "BEGIN EXPERIMENT: " + exp_name) total_result_table["experiments"][exp_name] = run_experiment( suite, program, program_alias, exp_name, experiment_arguments_string) return total_result_table
def run(suite): # The amorphous monstrosity of a table total_result_table = {} total_result_table["experiments"] = {} # Go through the programs... for (program_alias, program) in suite.programs.items(): # Check if there's argument variables that will require iterating over if ((len(suite.argument_variables) + len( suite.file_argument_variables)) == 0): exp_name = program_alias # There were none, so simply run this program with no extra arguments debug_msg(1, "BEGIN EXPERIMENT: " + exp_name) total_result_table["experiments"][exp_name] = run_experiment( suite, program, program_alias, exp_name) else: # There are some, so use args.get_experiment_arguments to get a list of all combos, iterate over them. for experiment_arguments in args.get_experiment_arguments( suite.argument_variables, suite.file_argument_variables): exp_params = "" experiment_arguments_string = "" for exp_arg in experiment_arguments: (name, value, is_file_arg_var) = exp_arg if is_file_arg_var: exp_params += (name + ":" + str(value) + " ") # change a config file (temp_file, output_file, pattern, values) = suite.file_argument_variables[name] update_based_on_file_argument(temp_file, output_file, pattern, value) else: exp_params += args.emit_argument(name, value) experiment_arguments_string += args.emit_argument( name, value) exp_name = program_alias + " " + exp_params debug_msg(1, "BEGIN EXPERIMENT: " + exp_name) total_result_table["experiments"][exp_name] = run_experiment( suite, program, program_alias, exp_name, experiment_arguments_string) return total_result_table
def check_ecd(suite): debug.debug_msg(3, "Sanity checking the provided ECD...") visible_vars = suite.__dict__.keys() for (var_name, t, cannot_be_empty) in rules: if var_name not in visible_vars: debug.warning_msg("'" + var_name + "' variable must be present!") else: var = suite.__dict__[var_name] if type(var) is not t: debug.warning_msg("'" + var_name + "' must be of type: " + str(t)) if cannot_be_empty and len(var) == 0: debug.warning_msg("'" + var_name + "' cannot be empty!") if debug.seen_warnings(): debug.error_msg("Execution Configuration Description was invalid!") debug.reset_warnings()
def send_email(address, results, mailserver='localhost', formatter=json.dumps): me = "marky@" + gethostname() you = address timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime()) msg = MIMEText(formatter(results)) msg['Subject'] = "marky: Experiment completed at " + timestamp msg['From'] = me msg['To'] = you try: s = smtplib.SMTP(mailserver) s.sendmail(me, [you], msg.as_string()) s.quit() except Exception: warning_msg("Failed to send email!") return debug_msg(3, "Sent e-mail to " + address + " at " + timestamp + ".")
def delete_old_autosaves(self): """Delete autosaves of loaded project, if the number of autosaves exceeds the max_autosaves value defined in the config. """ max_autosaves = self.config["max_autosaves"] debug.debug_msg(f"deleting old autosaves (max_autosaves={max_autosaves})...") autosave_dir_path = self.get_autosave_dir() file_suffix = self.get_autosave_filename_suffix() if autosave_dir_path.is_dir(): autosaves = [ f for f in autosave_dir_path.iterdir() if f.suffix == file_suffix and (autosave_dir_path / f).is_file() ] autosaves.sort(reverse=True) # reversed, so the oldest save(s) gets deleted for autosave in autosaves[max_autosaves:]: (autosave_dir_path / autosave).unlink() debug.debug_msg(f"...{len(autosaves[max_autosaves:])} file(s) deleted...") else: debug.debug_msg("autosave dir does not exist. Nothing to delete...")
def load_app_config(self): """Load the app config from file. If it doesn't exist or is faulty, it will be reset to the default app config. """ app_config_path = self.get_app_config_path() self.create_dir(app_config_path) if app_config_path.is_file(): debug.debug_msg("app_config.json found, loading into app_data.config...") try: with open(app_config_path, "r") as file: self.config = json.load(file) except: debug.warning_msg("app_config.json not valid, resetting app config...") with open(app_config_path, "w") as file: default_app_config = self.get_default_config() json.dump(default_app_config, file, indent=4) debug.debug_msg("app_config.json has been set up") self.config = default_app_config finally: debug.debug_msg("app_data.config loaded!") else: debug.debug_msg( "app_config.json has not been set up, initializing default config..." ) with open(app_config_path, "w") as file: default_app_config = self.get_default_config() json.dump(default_app_config, file, indent=4) debug.debug_msg("app_config.json has been set up") self.config = default_app_config debug.debug_msg("app_data.config loaded!")
def main(): parser = argparse.ArgumentParser( description = "marky - a benchmark execution and statistics gathering framework") parser.add_argument('file', type=str, nargs=1, metavar='FILE', help='A file containing an execution configuration. (or ECD) (Minus the .py)') parser.add_argument('--disable-aggregation', '-a', dest='disable_agg', action='store_true', help='Turn off aggregation calculation for this session.') parser.add_argument('--warmup', '-w', dest='should_warmup', action='store_true', help='Perform a warmup run of each benchmark that is not recorded.') parser.add_argument('--time', '-t', dest='should_time', action='store_true', help='Use marky to measure the runtime of any experiments.') parser.add_argument('--explain', '-e', dest='should_explain', action='store_true', help='Explain the experiments that will be run by the provided ECD.') parser.add_argument('--print', '-p', dest='should_print', action='store_true', help='"Pretty-print" the results.') parser.add_argument('--print-format', '-pf', dest='printfmt', nargs=1, choices=formats, help='Choose which format to print the data in. (default: json)') parser.add_argument('--load', '-l', dest='should_load', nargs=1, metavar='FILE', help='Load previous results from a file. (supports JSON only)') parser.add_argument('--load-raw', '-lr', dest='should_loadraw', nargs=1, metavar='DIR', help='Load the raw output from each run from the given directory.') parser.add_argument('--save', '-s', dest='should_save', nargs=1, metavar='FILE', help='Output the results into a file.') parser.add_argument('--save-format', '-sf', dest='savefmt', nargs=1, choices=formats, help='Choose which format to save the data in. (default: json)') parser.add_argument('--email', '-m', dest='should_email', nargs=1, metavar='ADDRESS', help='Send an email to the address once complete. (Uses localhost unless --mailserver is given.)') parser.add_argument('--email-format', '-mf', dest='emailfmt', nargs=1, choices=formats, help='Choose which format to email the data in. (default: json)') parser.add_argument('--mailserver', '-ms', dest='mailserver', nargs=1, metavar='HOST', help='Use the provided host as a mailserver.') parser.add_argument('--save-raw', '-r', dest='should_saveraw', nargs=1, metavar='DIR', help='Save the raw output from each run into the given directory.') parser.add_argument('--debug', '-d', dest='debuglevel', nargs=1, type=int, metavar='LEVEL', help='Set debug info level. 1 = Announce each benchmark invocation. 2 = Include time taken. 3 = Everything else. (Default = 1) (Set to 0 for quiet, or use --quiet.)') parser.add_argument('--quiet', '-q', dest='quiet', action='store_true', help='Hide all output (apart from errors.)') parser.add_argument('--speedups', '-cs', dest='should_calculate_speedups', action='store_true', help='Assuming only two experiments will be run, calculate the speedups.') args = parser.parse_args() config["debuglevel"] = 2 if args.debuglevel: config["debuglevel"] = args.debuglevel[0] if args.quiet: config["debuglevel"] = 0 suite = None if args.file: # (ecd = Execution Configuration Description) ecd_name = args.file[0] ecd_name = ecd_name.replace(".py", "") suite = __import__(ecd_name) ecd.check_ecd(suite) if args.should_explain: ecd.explain_ecd(suite) exit(0) config["original_dir"] = os.getcwd() config["saveraw"] = False if args.should_saveraw: config["saveraw"] = True config["saveraw_dir"] = config["original_dir"] + "/" + args.should_saveraw[0] config["loadraw"] = False if args.should_loadraw: config["loadraw"] = True config["loadraw_dir"] = config["original_dir"] + "/" + args.should_loadraw[0] if not os.path.exists(config["loadraw_dir"]): error_msg("Raw results directory required for loading doesn't exist!") debug_msg(1, "Will load output from " + config["loadraw_dir"]) config["should_warmup"] = False if args.should_warmup: config["should_warmup"] = True config["should_time"] = False if args.should_time: config["should_time"] = True results = None if args.should_load: debug_msg(1, "Loading previous results table from " + args.should_load[0]) json_file = open(args.should_load[0], "r") results = json.load(json_file) json_file.close() else: debug_msg(1, "Running experiment to obtain results!") os.chdir(suite.benchmark_root) results = run(suite) os.chdir(config["original_dir"]) results["description"] = ecd.convert_ecd_to_description(suite) if not args.disable_agg: stats.perform_aggregation(suite, results) #if args.should_calculate_speedups: # speedup.calculate(results) if args.should_print: formatter_name = default_print_format if args.printfmt: formatter_name = args.printfmt[0] formatter = formatters[formatter_name] print_results(results, formatter=formatter) if args.should_save: formatter_name = default_save_format if args.savefmt: formatter_name = args.savefmt[0] formatter = formatters[formatter_name] save_results(args.should_save[0], results, formatter=formatter) if args.should_email: mailserver = 'localhost' if args.mailserver: mailserver = args.mailserver[0] formatter_name = default_email_format if args.emailfmt: formatter_name = args.emailfmt[0] formatter = formatters[formatter_name] email_results(args.should_email[0], results, mailserver=mailserver, formatter=formatter)
def run_experiment(suite, program, program_alias, exp_name, experiment_arguments = ""): experiment_table = {} experiment_table["benchmarks"] = {} # Convert the dict of benchmark groups, to a list of benchmark tuples called actual_benchmarks # We will then iterate over that. actual_benchmarks = [] for (group_name, group_benchmarks) in suite.benchmarks.items(): for (name, directory, executescript, timeout) in group_benchmarks: actual_benchmarks.append((group_name, name, directory, executescript, timeout)) # Go through the benchmarks... for (group_name, benchmark, directory, executescript, timeout) in actual_benchmarks: benchmark_name = group_name + " ++ " + benchmark enter_directory(directory) debug_msg(3, "Entered into " + os.getcwd()) if (executescript): benchmark = get_arguments_from_file(executescript) # Construct the command used to execute the benchmark invocation = " ".join([suite.pre_arguments, program, suite.core_arguments, experiment_arguments, suite.benchmark_argument, benchmark]) if config["should_warmup"]: debug_msg(1, "RUN: '" + invocation + "' (WARMUP)") try: if timeout: execute_and_capture_output_with_timeout(invocation, timeout) else: execute_and_capture_output(invocation) except Exception as e: if e.args[1] == TIMEOUT_ERROR: debug_msg(1, "Run timed out... (timeout is " + str(timeout) + "s)") timedout_iterations += 1 failed_iterations += 1 if e.args[1] == FAILURE_ERROR: debug_msg(1, "Run failed...") failed_iterations += 1 # We store all the runs in here. run_table = [] failed_iterations = 0 timedout_iterations = 0 # Go through the iterations... for i in range(suite.iterations): # This stores the fields collected for this run. run = {} debug_msg(1, "RUN: '" + invocation + "' (ITER: " + str(i+1) + "/" + str(suite.iterations) + ")") # string containing the raw output from the benchmark raw = "" try: raw_output_name = exp_name + experiment_arguments + benchmark start = datetime.datetime.now() timestamp = time.time() * 1000 # Actually execute the benchmark if config["loadraw"] and check_raw_output_exists(raw_output_name, i): raw = load_raw_output(raw_output_name, i) debug_msg(1, "(loaded from file...)") elif timeout: raw = execute_and_capture_output_with_timeout(invocation, timeout) else: raw = execute_and_capture_output(invocation) end = datetime.datetime.now() # Save the output, if required if config["saveraw"]: save_raw_output(raw_output_name, i, raw) # Now collect the fields using our provided filters. for (field, field_filter) in suite.filters.items(): run[field] = run_filter(raw, field_filter) if not config["loadraw"]: delta = end - start runtime = float(delta.seconds) runtime += ( float((delta.microseconds)) / 1000000.0 ) debug_msg(2, "Took " + str(runtime) + " seconds.") if config["should_time"] and not config["loadraw"]: run["marky runtime"] = str(runtime) # Collected data is all strings currently; convert to the correct types. run = cleanup_run(run) # Save this run run_table.append(run) except Exception as e: if len(e.args) < 2: raise e if e.args[1] == TIMEOUT_ERROR: debug_msg(1, "Run timed out... (timeout is " + str(timeout) + "s)") timedout_iterations += 1 failed_iterations += 1 if e.args[1] == FAILURE_ERROR: debug_msg(1, "Run failed...") failed_iterations += 1 if e.args[1] == FILTER_FAILED_ERROR: debug_msg(1, "Filter failed...") failed_iterations += 1 if ("post_function" in suite.__dict__ and "post_function_arguments" in suite.__dict__): execute_post_function(suite.post_function, suite.post_function_arguments, raw_output_name, i, timestamp) # Finished running this benchmark for X iterations... benchmark_result = {} benchmark_result["successes"] = len(run_table) benchmark_result["failures"] = failed_iterations benchmark_result["timeouts"] = timedout_iterations benchmark_result["attempts"] = len(run_table) + failed_iterations # Collect results from runs if len(run_table) > 0: benchmark_result["runs"] = run_table leave_directory() debug_msg(3, "Exited back to " + os.getcwd()) # Save this benchmark in the benchmark table experiment_table["benchmarks"][benchmark_name] = benchmark_result return experiment_table
def main(): parser = argparse.ArgumentParser( description= "marky - a benchmark execution and statistics gathering framework") parser.add_argument( 'file', type=str, nargs=1, metavar='FILE', help= 'A file containing an execution configuration. (or ECD) (Minus the .py)' ) parser.add_argument( '--disable-aggregation', '-a', dest='disable_agg', action='store_true', help='Turn off aggregation calculation for this session.') parser.add_argument( '--warmup', '-w', dest='should_warmup', action='store_true', help='Perform a warmup run of each benchmark that is not recorded.') parser.add_argument( '--time', '-t', dest='should_time', action='store_true', help='Use marky to measure the runtime of any experiments.') parser.add_argument( '--explain', '-e', dest='should_explain', action='store_true', help='Explain the experiments that will be run by the provided ECD.') parser.add_argument('--print', '-p', dest='should_print', action='store_true', help='"Pretty-print" the results.') parser.add_argument( '--print-format', '-pf', dest='printfmt', nargs=1, choices=formats, help='Choose which format to print the data in. (default: json)') parser.add_argument( '--load', '-l', dest='should_load', nargs=1, metavar='FILE', help='Load previous results from a file. (supports JSON only)') parser.add_argument( '--load-raw', '-lr', dest='should_loadraw', nargs=1, metavar='DIR', help='Load the raw output from each run from the given directory.') parser.add_argument('--save', '-s', dest='should_save', nargs=1, metavar='FILE', help='Output the results into a file.') parser.add_argument( '--save-format', '-sf', dest='savefmt', nargs=1, choices=formats, help='Choose which format to save the data in. (default: json)') parser.add_argument( '--email', '-m', dest='should_email', nargs=1, metavar='ADDRESS', help= 'Send an email to the address once complete. (Uses localhost unless --mailserver is given.)' ) parser.add_argument( '--email-format', '-mf', dest='emailfmt', nargs=1, choices=formats, help='Choose which format to email the data in. (default: json)') parser.add_argument('--mailserver', '-ms', dest='mailserver', nargs=1, metavar='HOST', help='Use the provided host as a mailserver.') parser.add_argument( '--save-raw', '-r', dest='should_saveraw', nargs=1, metavar='DIR', help='Save the raw output from each run into the given directory.') parser.add_argument( '--debug', '-d', dest='debuglevel', nargs=1, type=int, metavar='LEVEL', help= 'Set debug info level. 1 = Announce each benchmark invocation. 2 = Include time taken. 3 = Everything else. (Default = 1) (Set to 0 for quiet, or use --quiet.)' ) parser.add_argument('--quiet', '-q', dest='quiet', action='store_true', help='Hide all output (apart from errors.)') parser.add_argument( '--speedups', '-cs', dest='should_calculate_speedups', action='store_true', help= 'Assuming only two experiments will be run, calculate the speedups.') args = parser.parse_args() config["debuglevel"] = 2 if args.debuglevel: config["debuglevel"] = args.debuglevel[0] if args.quiet: config["debuglevel"] = 0 suite = None if args.file: # (ecd = Execution Configuration Description) ecd_name = args.file[0] ecd_name = ecd_name.replace(".py", "") suite = __import__(ecd_name) ecd.check_ecd(suite) if args.should_explain: ecd.explain_ecd(suite) exit(0) config["original_dir"] = os.getcwd() config["saveraw"] = False if args.should_saveraw: config["saveraw"] = True config["saveraw_dir"] = config[ "original_dir"] + "/" + args.should_saveraw[0] config["loadraw"] = False if args.should_loadraw: config["loadraw"] = True config["loadraw_dir"] = config[ "original_dir"] + "/" + args.should_loadraw[0] if not os.path.exists(config["loadraw_dir"]): error_msg( "Raw results directory required for loading doesn't exist!") debug_msg(1, "Will load output from " + config["loadraw_dir"]) config["should_warmup"] = False if args.should_warmup: config["should_warmup"] = True config["should_time"] = False if args.should_time: config["should_time"] = True results = None if args.should_load: debug_msg(1, "Loading previous results table from " + args.should_load[0]) json_file = open(args.should_load[0], "r") results = json.load(json_file) json_file.close() else: debug_msg(1, "Running experiment to obtain results!") os.chdir(suite.benchmark_root) results = run(suite) os.chdir(config["original_dir"]) results["description"] = ecd.convert_ecd_to_description(suite) if not args.disable_agg: stats.perform_aggregation(suite, results) #if args.should_calculate_speedups: # speedup.calculate(results) if args.should_print: formatter_name = default_print_format if args.printfmt: formatter_name = args.printfmt[0] formatter = formatters[formatter_name] print_results(results, formatter=formatter) if args.should_save: formatter_name = default_save_format if args.savefmt: formatter_name = args.savefmt[0] formatter = formatters[formatter_name] save_results(args.should_save[0], results, formatter=formatter) if args.should_email: mailserver = 'localhost' if args.mailserver: mailserver = args.mailserver[0] formatter_name = default_email_format if args.emailfmt: formatter_name = args.emailfmt[0] formatter = formatters[formatter_name] email_results(args.should_email[0], results, mailserver=mailserver, formatter=formatter)
def run_experiment(suite, program, program_alias, exp_name, experiment_arguments=""): experiment_table = {} experiment_table["benchmarks"] = {} # Convert the dict of benchmark groups, to a list of benchmark tuples called actual_benchmarks # We will then iterate over that. actual_benchmarks = [] for (group_name, group_benchmarks) in suite.benchmarks.items(): for (name, directory, executescript, timeout) in group_benchmarks: actual_benchmarks.append( (group_name, name, directory, executescript, timeout)) # Go through the benchmarks... for (group_name, benchmark, directory, executescript, timeout) in actual_benchmarks: benchmark_name = group_name + " ++ " + benchmark enter_directory(directory) debug_msg(3, "Entered into " + os.getcwd()) if (executescript): benchmark = get_arguments_from_file(executescript) # Construct the command used to execute the benchmark invocation = " ".join([ suite.pre_arguments, program, suite.core_arguments, experiment_arguments, suite.benchmark_argument, benchmark ]) if config["should_warmup"]: debug_msg(1, "RUN: '" + invocation + "' (WARMUP)") try: if timeout: execute_and_capture_output_with_timeout( invocation, timeout) else: execute_and_capture_output(invocation) except Exception as e: if e.args[1] == TIMEOUT_ERROR: debug_msg( 1, "Run timed out... (timeout is " + str(timeout) + "s)") timedout_iterations += 1 failed_iterations += 1 if e.args[1] == FAILURE_ERROR: debug_msg(1, "Run failed...") failed_iterations += 1 # We store all the runs in here. run_table = [] failed_iterations = 0 timedout_iterations = 0 # Go through the iterations... for i in range(suite.iterations): # This stores the fields collected for this run. run = {} debug_msg( 1, "RUN: '" + invocation + "' (ITER: " + str(i + 1) + "/" + str(suite.iterations) + ")") # string containing the raw output from the benchmark raw = "" try: raw_output_name = exp_name + experiment_arguments + benchmark start = datetime.datetime.now() timestamp = time.time() * 1000 # Actually execute the benchmark if config["loadraw"] and check_raw_output_exists( raw_output_name, i): raw = load_raw_output(raw_output_name, i) debug_msg(1, "(loaded from file...)") elif timeout: raw = execute_and_capture_output_with_timeout( invocation, timeout) else: raw = execute_and_capture_output(invocation) end = datetime.datetime.now() # Save the output, if required if config["saveraw"]: save_raw_output(raw_output_name, i, raw) # Now collect the fields using our provided filters. for (field, field_filter) in suite.filters.items(): run[field] = run_filter(raw, field_filter) if not config["loadraw"]: delta = end - start runtime = float(delta.seconds) runtime += (float((delta.microseconds)) / 1000000.0) debug_msg(2, "Took " + str(runtime) + " seconds.") if config["should_time"] and not config["loadraw"]: run["marky runtime"] = str(runtime) # Collected data is all strings currently; convert to the correct types. run = cleanup_run(run) # Save this run run_table.append(run) except Exception as e: if len(e.args) < 2: raise e if e.args[1] == TIMEOUT_ERROR: debug_msg( 1, "Run timed out... (timeout is " + str(timeout) + "s)") timedout_iterations += 1 failed_iterations += 1 if e.args[1] == FAILURE_ERROR: debug_msg(1, "Run failed...") failed_iterations += 1 if e.args[1] == FILTER_FAILED_ERROR: debug_msg(1, "Filter failed...") failed_iterations += 1 if ("post_function" in suite.__dict__ and "post_function_arguments" in suite.__dict__): execute_post_function(suite.post_function, suite.post_function_arguments, raw_output_name, i, timestamp) # Finished running this benchmark for X iterations... benchmark_result = {} benchmark_result["successes"] = len(run_table) benchmark_result["failures"] = failed_iterations benchmark_result["timeouts"] = timedout_iterations benchmark_result["attempts"] = len(run_table) + failed_iterations # Collect results from runs if len(run_table) > 0: benchmark_result["runs"] = run_table leave_directory() debug_msg(3, "Exited back to " + os.getcwd()) # Save this benchmark in the benchmark table experiment_table["benchmarks"][benchmark_name] = benchmark_result return experiment_table