def test_experiment_perform(): procedure = TmpProcedure() path = "./tmpexperiment" experiment = TmpExperimentCorrect(procedure, path) # Test if provided function for performing experiment needs to be a # TestFunction (which it should) with pytest.raises(Exception): experiment._perform_experiment("not-a-testfunction") # Check if function can be tested for validity by child class through the # .check_testfunction method with pytest.raises(Exception): experiment._perform_experiment(func.GaussianShells()) # Perform experiment and check if all folders and files are created procedure.allow_function = True experiment.run(func.GaussianShells()) assert experiment.finish_line == 1000 assert os.path.exists(path + os.sep + 'gaussianshells') is True assert os.path.exists(path + os.sep + 'gaussianshells/samples.csv') is True assert os.path.exists(path + os.sep + 'gaussianshells/functioncalls.csv') is True assert os.path.exists(path + os.sep + 'gaussianshells/procedurecalls.csv') is True assert os.path.exists(path + os.sep + 'benchmarks.yaml') is True # Repeat experiment, but now disable logging of datapoints shutil.rmtree(path) experiment = TmpExperimentCorrect(procedure, path) experiment.run(func.GaussianShells(), finish_line=90, log_data=False) assert experiment.finish_line == 90 assert os.path.getsize(path + os.sep + 'gaussianshells/samples.csv') < 100 shutil.rmtree(path)
def test_results_submethods(): # Create experiment procedure = TmpProcedure() path = "./tmpresultsexp2" experiment = TmpExperimentCorrect(procedure, path) experiment.run(func.GaussianShells(), finish_line=1000) r = results.Result(path) # Check get_function_information assert r.get_function_information('testname') == ('testname', 0) assert r.get_function_information('testname_1') == ('testname', 1) assert r.get_function_information('test_name_3') == ('test_name', 3) # Check read_procedure_calls procedure_calls = r.read_procedure_calls('gaussianshells') assert isinstance(procedure_calls, pd.DataFrame) is True assert len(procedure_calls) == 10 # Check procedure_information time = r.extract_procedure_information(procedure_calls) assert time == procedure_calls['dt'].sum() # Check read_experiment_yaml exp = r.read_experiment_yaml('gaussianshells') assert isinstance(exp, dict) assert len(exp) > 0 assert 'meta' in exp # Check extract_result_information res = r.extract_result_information(exp) assert isinstance(res, dict) assert 'experiment_type' in res assert 'x' in res assert res['procedure_name'] == 'TmpProcedure' assert res['best_value'] == 3 # Remove folder shutil.rmtree(path)
def test_experiment_logyaml(): procedure = TmpProcedure() procedure.allow_function = True path = "./tmpexperiment" experiment = TmpExperimentCorrect(procedure, path) # Test if the logbook contains all entries function = func.GaussianShells() experiment.run(function) log = {} with open(path + "/gaussianshells/experiment.yaml", 'r') as stream: log = yaml.safe_load(stream) assert 'experiment' in log assert 'type' in log['experiment'] assert 'function' in log assert 'name' in log['function'] assert 'properties' in log['function'] assert 'ranges' in log['function']['properties'] assert log['function']['properties']['ranges'] == function.ranges assert 'testfunction' in log['function'] assert 'meta' in log assert 'datetime' in log['meta'] assert 'timestamp' in log['meta'] assert 'user' in log['meta'] assert 'procedure' in log assert 'name' in log['procedure'] assert 'properties' in log['procedure'] assert log['procedure']['properties'] == {'a': 10} shutil.rmtree(path)
def test_logger_functioncalls(): basepath = '.' subfolder = 'tmplog' log = exp.Logger(basepath, subfolder) # Fake function calls log.procedure_calls = 10 function = func.GaussianShells() function.counter = [[10, 3, 1], [9, 2, 0]] log.log_function_calls(function) # Read function call log call_log = pd.read_csv(log.path + os.sep + 'functioncalls.csv') reference = np.array([[10, 10, 3, 1], [10, 9, 2, 0]]) assert np.array_equal(call_log.values[:, :-1], reference[:, :-1]) assert np.array_equal(call_log['asked_for_derivative'] * 1, reference[:, -1]) shutil.rmtree(log.path)
def test_testfunction_gaussianshells(): function = func.GaussianShells() # Validate the default configuration assert function.get_dimensionality() == 2 assert np.array_equal(function.c_1, np.array([2.5, 0])) assert function.r_1 == 2.0 assert function.w_1 == 0.1 assert np.array_equal(function.c_2, np.array([-2.5, 0])) assert function.r_2 == 2.0 assert function.w_2 == 0.1 # Validate function properties assert function.is_bounded() is True assert function.is_differentiable() is False # Validate output shape x = np.random.rand(10000, function.get_dimensionality()) y = function(x) assert y.shape == (10000, 1) with pytest.raises(func.NoDerivativeError): function(x, True)
def test_results_result(): path = "./tmpresultsexp1" # Test that if path does not exist, exception is raised with pytest.raises(Exception): _ = results.Result(path) # Create experiment procedure = TmpProcedure() experiment = TmpExperimentCorrect(procedure, path) experiment.run(func.GaussianShells()) # Test that results can now be properly be initialised, regardless of # directory seperator at end of path r = results.Result(path) _ = results.Result(path + '/') assert r.path == path + os.sep # Test that if benchmarks file is missing, results cannot be read os.remove(path + '/benchmarks.yaml') with pytest.raises(Exception): _ = results.Result(path) shutil.rmtree(path)
def test_results_df(): # Create experiment procedure = TmpProcedure() path = "./tmpresultsexp3" experiment = TmpExperimentCorrect2(procedure, path) experiment.run(func.GaussianShells(), finish_line=1000) experiment.run(func.Sphere(), finish_line=1000) r = results.Result(path) # Get df df = r.get_results() assert isinstance(df, pd.DataFrame) assert len(df) == 2 # Create df through function df2 = results.make_dataframe({'standard': path}) print(df2) del (df2['experiment']) assert df.equals(df2) # Check if indicating multiple folders works _ = results.make_dataframe({'standard': path, 'other': path}) # Remove folder shutil.rmtree(path)
def test_results_plots(): # Create experiment procedure = TmpProcedure() path = "./tmpresultsexp5" experiment = TmpExperimentCorrect(procedure, path) experiment.run(func.GaussianShells(), finish_line=1000) experiment.run(func.Sphere(), finish_line=1000) # Get results df df = results.make_dataframe({'standard': path, 'other': path}) # Boxplot experiment with pytest.raises(Exception): results.boxplot_experiment(df, 'time', 'not-an-experiment') with pytest.raises(Exception): results.boxplot_experiment(df, 'not-a-metric', 'standard') results.boxplot_experiment(df, 'time', 'standard', logarithmic=True, path='./plot.png') assert os.path.exists('./plot.png') is True results.boxplot_experiment(df, 'time', 'standard', logarithmic=False, path='./plot.png') os.remove('./plot.png') # Boxplot function with pytest.raises(Exception): results.boxplot_function(df, 'time', 'not-a-function') with pytest.raises(Exception): results.boxplot_function(df, 'not-a-metric', 'sphere') results.boxplot_function(df, 'time', 'sphere', logarithmic=True, path='./plot.png') assert os.path.exists('./plot.png') is True results.boxplot_function(df, 'time', 'sphere', logarithmic=False, path='./plot.png') os.remove('./plot.png') # Histogram experiment with pytest.raises(Exception): results.histogram_experiment(df, 'time', 'not-an-experiment') with pytest.raises(Exception): results.histogram_experiment(df, 'not-a-metric', 'standard') results.histogram_experiment(df, 'time', 'standard', logarithmic=True, path='./plot.png') assert os.path.exists('./plot.png') is True results.histogram_experiment(df, 'time', 'standard', logarithmic=False, path='./plot.png') os.remove('./plot.png') # Histogram function with pytest.raises(Exception): results.histogram_function(df, 'time', 'not-a-function') with pytest.raises(Exception): results.histogram_function(df, 'not-a-metric', 'sphere') results.histogram_function(df, 'time', 'sphere', logarithmic=True, path='./plot.png') assert os.path.exists('./plot.png') is True results.histogram_function(df, 'time', 'sphere', logarithmic=False, path='./plot.png') os.remove('./plot.png') shutil.rmtree(path)
def test_results_tables(): # Create experiment procedure = TmpProcedure() path = "./tmpresultsexp4" experiment = TmpExperimentCorrect(procedure, path) experiment.run(func.GaussianShells(), finish_line=1000) experiment.run(func.Sphere(), finish_line=1000) experiment.run(func.GaussianShells(), finish_line=1000) experiment.run(func.Sphere(), finish_line=1000) # Get results df df = results.make_dataframe({'standard': path, 'other': path}) # Check that exceptions are raised correctly with pytest.raises(Exception): _, _ = results.tabulate_result(df, 'x', 'not-an-experiment') with pytest.raises(Exception): _, _ = results.tabulate_result(df, 'y', 'standard') # Check that the method works in general content, rows = results.tabulate_result(df, 'x', 'other') # Check content and rows format assert rows.tolist() == ['gaussianshells', 'sphere'] assert isinstance(content, list) # Check that the method works with functions specified _, _ = results.tabulate_result(df, 'x', 'other', ['Sphere']) # Check that results can be stored to csv and tex (validate csv) _, _ = results.tabulate_result(df, 'x', 'other', path='./tmp_out.csv') _, _ = results.tabulate_result(df, 'x', 'other', path='./tmp_out.tex') _ = pd.read_csv('./tmp_out.csv') # Validate that allowed table output formats are only csv and tex _ = results.create_table_string(content, rows, ['x'], 'csv', '') _ = results.create_table_string(content, rows, ['x'], 'tex', '') with pytest.raises(Exception): _ = results.create_table_string(content, rows, ['None'], 'txt', '') # Remove folder and output files os.remove('./tmp_out.csv') os.remove('./tmp_out.tex') # Check that exceptions are raised correctly print(df) with pytest.raises(Exception): _, _, _ = results.tabulate_all_aggregated( df, 'time', 'mean', experiment_names=['not-an-experiment']) with pytest.raises(Exception): _, _, _ = results.tabulate_all_aggregated(df, 'y', 'mean') _, _, _ = results.tabulate_all_aggregated(df, 'time', 'mean', functions=['sphere']) _, _, _ = results.tabulate_all_aggregated(df, 'time', 'mean', experiment_names=['standard']) _, _, _ = results.tabulate_all_aggregated(df, 'time', 'mean', path='./tmp_out.csv') _ = pd.read_csv('./tmp_out.csv') content, rows, cols = results.tabulate_all_aggregated(df, 'time', 'mean', path='./tmp_out.tex') assert isinstance(rows, list) is True assert isinstance(cols, list) is True assert isinstance(content, list) # Remove folder and output files os.remove('./tmp_out.csv') os.remove('./tmp_out.tex') shutil.rmtree(path)