def test_configuration_file_correct_experiment_name(correct_arguments,
correct_experiment_name,
tmpdir):
"""Checks that the configuration file was saved to the directory"""
parsed_arguments = arguments.parse(correct_arguments)
directory_prefix = str(tmpdir) + "/"
configuration.save(directory_prefix + constants.CONFIGURATION,
vars(parsed_arguments))
assert len(tmpdir.listdir()) == 1
tada_configuration_dict = configuration.read(directory_prefix +
constants.CONFIGURATION)
assert (configuration.get_experiment_name(tada_configuration_dict,
2) == correct_experiment_name)
def tada(tada_arguments):
start_time = time.time()
# read and verify the command-line arguments
did_verify_arguments = arguments.verify(tada_arguments)
resultstable = PrettyTable()
resultstable.field_names = ["Size", "Mean", "Median", "Ratio"]
meanlastround = 0
result = {}
indicator = 0
steps = constants.STEP_START
last_last_size = 0
current_size = tada_arguments.startsize
total_loop_list = []
sum_of_loops = 0
use_backfill = tada_arguments.backfill
# display debug output
to_print = tada_arguments.log
# incorrect arguments, exit program
if did_verify_arguments is False:
dis.output_message("Incorrect command-line arguments.", to_print)
sys.exit(constants.INCORRECT_ARGUMENTS)
# correct arguments, run doubling experiment
else:
# add the directory to the sys.path
package.add_sys_path(tada_arguments.directory)
# create and save a configuration dictionary from the arguments
configuration.save(constants.CONFIGURATION, vars(tada_arguments))
# save the size of the experiment in the constants.file
save.save_experiment_size(constants.SIZE, current_size)
# save the directory containing functions to be analyzed
save.save_directory(constants.DIRECTORY, tada_arguments.directory)
# perform the small doubling experiment
while True:
if use_backfill:
# run the benchmark by using it through python
# backfill checker
if last_last_size == current_size:
dis.output_message(
f"lastlast: {last_last_size} \ncurrent: {current_size}",
to_print,
)
constants.BACKFILL_TIMES = constants.BACKFILL_TIMES + 1
dis.output_message(
dis.green("Backfill Count: ") +
str(constants.BACKFILL_TIMES),
to_print,
)
if constants.BACKFILL_TIMES == 2:
constants.QUIT_BY_BACKFILL = 1
dis.output_message("\nQuit due to backfill twice",
to_print)
break
if current_size >= tada_arguments.maxsize:
constants.QUIT_BY_MAX_SIZE = 1
dis.output_message(
"\nQuit due to reaching max size: " +
str(tada_arguments.maxsize),
to_print,
)
break
print(dis.start_message(current_size, tada_arguments.function))
current_output, current_error = run.run_command(
constants.PYTHON_EXEC + constants.SPACE +
constants.PERF_BENCHMARK + constants.PYTHON_EXT)
# display the standard output and error
dis.output_message(current_output.decode(constants.UTF8), to_print)
dis.output_message(current_error.decode(constants.UTF8),
to_print=True)
# read the JSON file containing the results
current_benchmark = pyperf.Benchmark.load(
constants.RESULTS +
constants.SEPARATOR + configuration.get_experiment_name(
vars(tada_arguments), current_size) + constants.JSON_EXT)
# Numbers of benchmark run which include one warmup and twenty excution
# print(current_benchmark.get_nrun())
# Total Runtime of the benchmark excution
# print(current_benchmark.get_total_duration())
total_loop_list.append(current_benchmark.get_total_loops())
# perform additional analysis of the results
# reminder: print('Values {0}'.format(current_benchmark.get_values()))
mean = current_benchmark.mean()
median = current_benchmark.median()
result.update({current_size: [mean, median]})
dis.output_message(
dis.green("Mean: ") + str(mean),
to_print,
)
dis.output_message(
dis.green("Median: ") + str(median),
to_print,
)
if meanlastround == 0:
ratio = 0
indicator = tada_arguments.indicator
end_time = mean
last_size = 0
last_end_time = end_time
last_end_time_rate = 1
end_time_rate = 1
else:
if current_size > last_size:
ratio = mean / meanlastround
avg = (mean + meanlastround) / 2
std = abs(mean - avg)
indicator = std / abs(avg)
end_time = (mean - 0.01 * meanlastround) / 0.99
last_end_time_rate = end_time_rate
end_time_rate = (end_time - last_end_time) / last_end_time
else: # backfill
ratio = meanlastround / mean
avg = (mean + meanlastround) / 2
std = abs(meanlastround - avg)
indicator = std / abs(avg)
end_time = (meanlastround - 0.01 * mean) / 0.99
last_end_time_rate = end_time_rate
end_time_rate = (end_time - last_end_time) / last_end_time
dis.output_message(
dis.green("Last end time: ") + str(last_end_time),
to_print,
)
last_end_time = end_time
dis.output_message(
dis.green("Current indicator: ") + str(indicator), to_print)
dis.output_message(
dis.green("Expected end time: ") + str(end_time), to_print)
results.add_resultstable(resultstable, current_size, mean, median,
ratio)
# show that we are done running for a size
print(dis.end_message(current_size, tada_arguments.function))
# go to the next size for the doubling experiment
last_last_size = last_size
last_size = current_size
dis.output_message(
dis.green("End time rate: ") + str(end_time_rate), to_print)
dis.output_message(
dis.green("Last end time rate: ") + str(last_end_time_rate),
to_print,
)
if last_end_time_rate > end_time_rate and use_backfill:
current_size = int(current_size / constants.FACTOR)
else:
current_size = current_size * constants.FACTOR
# check indicator and quit if smaller than decided indicator
if indicator < tada_arguments.indicator:
dis.output_message(f"\nQuit due to indicator: {indicator}",
to_print)
break
save.save_experiment_size(constants.SIZE, current_size)
meanlastround = mean
current_runningtime = time.time() - start_time
if current_runningtime > tada_arguments.runningtime:
dis.output_message(
"\nQuit due to exceeding the max time limit: " +
current_runningtime,
to_print,
)
constants.QUIT_BY_MAX_RUNTIME = 1
break
steps += 1
if steps > tada_arguments.steps:
dis.output_message(f"\nQuit due to end of rounds: {steps}",
to_print)
constants.QUIT_BY_STEPS = 1
break
big_oh = analysis.analyze_big_oh(ratio)
if tada_arguments.expect is not None:
if indicator < tada_arguments.indicator:
constants.QUIT_BY_INDICATOR = 1
# store indicator
constants.INDICATOR_VALUE = tada_arguments.indicator
# store runningtime
constants.TOTAL_RUNNING_TIME = time.time() - start_time
constants.AVG_RUN_TIME = constants.TOTAL_RUNNING_TIME / steps
last_bench_meta = current_benchmark.get_metadata()
name = last_bench_meta.get("name")
# store benchmark metadata
constants.NAME_OF_EXPERIMENT = configuration.get_experiment_info(
vars(tada_arguments))
if "cpu_model_name" in last_bench_meta:
constants.CPU_TYPE = last_bench_meta.get("cpu_model_name")
constants.CPU_COUNT = last_bench_meta.get("cpu_count")
constants.OS = last_bench_meta.get("platform")
constants.PYTHON_VERSION = last_bench_meta.get("python_version")
# store run metadata
with open(
constants.RESULTS + constants.SEPARATOR + name +
constants.JSON_EXT,
"r",
) as f:
readlastjson = json.load(f)
last_exp_run_metadata = readlastjson["benchmarks"][0]["runs"][0][
"metadata"]
constants.CPU_TEMP = last_exp_run_metadata.get("cpu_temp")
if "mem_max_rss" in last_exp_run_metadata:
constants.MEM_MAX_RSS = last_exp_run_metadata["mem_max_rss"]
else:
constants.MEM_PEAK_PAGEFILE_USAGE = last_exp_run_metadata[
"mem_peak_pagefile_usage"]
for item in total_loop_list:
sum_of_loops += item
# calculate avg total loops
constants.PYPERF_AVG_EXPERIMENT_ROUNDS = sum_of_loops / len(
total_loop_list)
# calculate last two loop growth ratio
if len(total_loop_list) >= 2:
constants.PYPERF_LAST_TWO_EXPERIMENT_ROUNDS = (
total_loop_list[-1] / total_loop_list[-2])
# check if result is expected
if tada_arguments.expect in analysis.analyze_big_oh(ratio):
constants.RESULT = 1
constants.DATA_GEN_STRATEGY = tada_arguments.types
constants.START_SIZE = tada_arguments.startsize
constants.INDICATOR_VALUE = tada_arguments.indicator
# set numerical value to backfill for result storing
use_backfill = 1 if use_backfill else 0
# EXPERIMENT_RELIABILITY, CPU_TYPE, CPU_TEMP, TOTAL_RUNNING_TIME, QUIT_BY_MAX_RUNTIME, QUIT_BY_INDICATOR, QUIT_BY_BACKFILL, MEM_MAX_RSS, OS, INDICATOR_VALUE, BACKFILL_TIMES, PYPERF_AVG_EXPERIMENT_ROUNDS, NAME_OF_EXPERIMENT
df_new = pd.DataFrame(
{
"EXPERIMENT_RELIABILITY": constants.RESULT,
"CPU_TYPE": constants.CPU_TYPE,
"CPU_TEMP": constants.CPU_TEMP,
"CPU_COUNT": constants.CPU_COUNT,
"TOTAL_RUNNING_TIME": constants.TOTAL_RUNNING_TIME,
"QUIT_BY_MAX_RUNTIME": constants.QUIT_BY_MAX_RUNTIME,
"QUIT_BY_INDICATOR": constants.QUIT_BY_INDICATOR,
"QUIT_BY_BACKFILL": constants.QUIT_BY_BACKFILL,
"QUIT_BY_STEPS": constants.QUIT_BY_STEPS,
"QUIT_BY_MAX_SIZE": constants.QUIT_BY_MAX_SIZE,
"MEM_MAX_RSS": constants.MEM_MAX_RSS,
"MEM_PEAK_PAGEFILE_USAGE":
constants.MEM_PEAK_PAGEFILE_USAGE,
"OS": constants.OS,
"INDICATOR_VALUE": constants.INDICATOR_VALUE,
"BACKFILL_TIMES": constants.BACKFILL_TIMES,
"PYPERF_AVG_EXPERIMENT_ROUNDS":
constants.PYPERF_AVG_EXPERIMENT_ROUNDS,
"PYPERF_LAST_TWO_EXPERIMENT_ROUNDS_RATIO":
constants.PYPERF_LAST_TWO_EXPERIMENT_ROUNDS_RATIO,
"NAME_OF_EXPERIMENT": constants.NAME_OF_EXPERIMENT,
"PYTHON_VERSION": constants.PYTHON_VERSION,
"DATA_GEN_STRATEGY": constants.DATA_GEN_STRATEGY,
"START_SIZE": constants.START_SIZE,
"USED_BACKFILL": use_backfill,
"AVG_RUN_TIME": constants.AVG_RUN_TIME,
"STEPS": steps,
},
index=[1],
)
# store to csv
df_new.to_csv(constants.EXPERIMENT,
index=False,
header=False,
mode="a")
resultstable.title = dis.blue(f"{tada_arguments.function}: ") + big_oh
return resultstable, {tada_arguments.function: result}
# read the configuration file to access the configuration dictionary
tada_configuration_dict = configuration.read(constants.CONFIGURATION)
# add the specified directory to the system path
package.add_sys_path(configuration.get_directory(tada_configuration_dict))
# reflectively import the chosen module
analyzed_module = importlib.import_module(
configuration.get_module(tada_configuration_dict))
# reflectively access the chosen function
analyzed_function = getattr(
analyzed_module, configuration.get_function(tada_configuration_dict))
# read the chosen_size
chosen_size = read.read_experiment_size()
# configure perf
runner = pyperf.Runner()
# give a by-configuration name to the experiment
current_experiment_name = configuration.get_experiment_name(
tada_configuration_dict, chosen_size)
# set the name of the experiment for perf
runner.metadata[constants.DESCRIPTION_METANAME] = current_experiment_name
# read the chosen types
func_type = configuration.get_types(tada_configuration_dict)
# initialize path for schema
path = None
gen_func = None
# using hypothesis to generate experiment data
if func_type[0] == "hypothesis":
# read path from arguments
path = configuration.get_schema_path(tada_configuration_dict)
if func_type[0] == "custom":
data_directory = configuration.get_data_directory(
tada_configuration_dict)
if data_directory != "":
configuration.save(constants.CONFIGURATION, vars(tada_arguments))
# save the size of the experiment in the constants.file
save.save_experiment_size(constants.SIZE, current_size)
# save the directory containing functions to be analyzed
save.save_directory(constants.DIRECTORY, tada_arguments.directory)
# perform the small doubling experiment
while current_size <= constants.SIZE_STOP:
# run the benchmark by using it through python
display.display_start_message(current_size)
current_output, current_error = run.run_command(
constants.PYTHON_EXEC + constants.SPACE +
constants.PERF_BENCHMARK + constants.PYTHON_EXT)
# display the standard output and error
display.display_output(current_output.decode(constants.UTF8))
display.display_output(current_error.decode(constants.UTF8))
# read the JSON file containing the results
current_benchmark = perf.Benchmark.load(
constants.RESULTS +
constants.SEPARATOR + configuration.get_experiment_name(
vars(tada_arguments), current_size) + constants.JSON_EXT)
# perform additional analysis of the results
# reminder: print('Values {0}'.format(current_benchmark.get_values()))
print("Mean {0}".format(current_benchmark.mean()))
print("Median {0}".format(current_benchmark.median()))
# show that we are done running for a size
display.display_end_message(current_size)
# go to the next size for the doubling experiment
current_size = current_size * constants.FACTOR
# write the next doubling experiment size to the file
save.save_experiment_size(constants.SIZE, current_size)