def test_experiment_perform():
procedure = TmpProcedure()
path = "./tmpexperiment"
experiment = TmpExperimentCorrect(procedure, path)
# Test if provided function for performing experiment needs to be a
# TestFunction (which it should)
with pytest.raises(Exception):
experiment._perform_experiment("not-a-testfunction")
# Check if function can be tested for validity by child class through the
# .check_testfunction method
with pytest.raises(Exception):
experiment._perform_experiment(func.GaussianShells())
# Perform experiment and check if all folders and files are created
procedure.allow_function = True
experiment.run(func.GaussianShells())
assert experiment.finish_line == 1000
assert os.path.exists(path + os.sep + 'gaussianshells') is True
assert os.path.exists(path + os.sep + 'gaussianshells/samples.csv') is True
assert os.path.exists(path + os.sep +
'gaussianshells/functioncalls.csv') is True
assert os.path.exists(path + os.sep +
'gaussianshells/procedurecalls.csv') is True
assert os.path.exists(path + os.sep + 'benchmarks.yaml') is True
# Repeat experiment, but now disable logging of datapoints
shutil.rmtree(path)
experiment = TmpExperimentCorrect(procedure, path)
experiment.run(func.GaussianShells(), finish_line=90, log_data=False)
assert experiment.finish_line == 90
assert os.path.getsize(path + os.sep + 'gaussianshells/samples.csv') < 100
shutil.rmtree(path)
def test_results_submethods():
# Create experiment
procedure = TmpProcedure()
path = "./tmpresultsexp2"
experiment = TmpExperimentCorrect(procedure, path)
experiment.run(func.GaussianShells(), finish_line=1000)
r = results.Result(path)
# Check get_function_information
assert r.get_function_information('testname') == ('testname', 0)
assert r.get_function_information('testname_1') == ('testname', 1)
assert r.get_function_information('test_name_3') == ('test_name', 3)
# Check read_procedure_calls
procedure_calls = r.read_procedure_calls('gaussianshells')
assert isinstance(procedure_calls, pd.DataFrame) is True
assert len(procedure_calls) == 10
# Check procedure_information
time = r.extract_procedure_information(procedure_calls)
assert time == procedure_calls['dt'].sum()
# Check read_experiment_yaml
exp = r.read_experiment_yaml('gaussianshells')
assert isinstance(exp, dict)
assert len(exp) > 0
assert 'meta' in exp
# Check extract_result_information
res = r.extract_result_information(exp)
assert isinstance(res, dict)
assert 'experiment_type' in res
assert 'x' in res
assert res['procedure_name'] == 'TmpProcedure'
assert res['best_value'] == 3
# Remove folder
shutil.rmtree(path)
def test_experiment_logyaml():
procedure = TmpProcedure()
procedure.allow_function = True
path = "./tmpexperiment"
experiment = TmpExperimentCorrect(procedure, path)
# Test if the logbook contains all entries
function = func.GaussianShells()
experiment.run(function)
log = {}
with open(path + "/gaussianshells/experiment.yaml", 'r') as stream:
log = yaml.safe_load(stream)
assert 'experiment' in log
assert 'type' in log['experiment']
assert 'function' in log
assert 'name' in log['function']
assert 'properties' in log['function']
assert 'ranges' in log['function']['properties']
assert log['function']['properties']['ranges'] == function.ranges
assert 'testfunction' in log['function']
assert 'meta' in log
assert 'datetime' in log['meta']
assert 'timestamp' in log['meta']
assert 'user' in log['meta']
assert 'procedure' in log
assert 'name' in log['procedure']
assert 'properties' in log['procedure']
assert log['procedure']['properties'] == {'a': 10}
shutil.rmtree(path)
def test_logger_functioncalls():
basepath = '.'
subfolder = 'tmplog'
log = exp.Logger(basepath, subfolder)
# Fake function calls
log.procedure_calls = 10
function = func.GaussianShells()
function.counter = [[10, 3, 1], [9, 2, 0]]
log.log_function_calls(function)
# Read function call log
call_log = pd.read_csv(log.path + os.sep + 'functioncalls.csv')
reference = np.array([[10, 10, 3, 1], [10, 9, 2, 0]])
assert np.array_equal(call_log.values[:, :-1], reference[:, :-1])
assert np.array_equal(call_log['asked_for_derivative'] * 1, reference[:,
-1])
shutil.rmtree(log.path)
def test_testfunction_gaussianshells():
function = func.GaussianShells()
# Validate the default configuration
assert function.get_dimensionality() == 2
assert np.array_equal(function.c_1, np.array([2.5, 0]))
assert function.r_1 == 2.0
assert function.w_1 == 0.1
assert np.array_equal(function.c_2, np.array([-2.5, 0]))
assert function.r_2 == 2.0
assert function.w_2 == 0.1
# Validate function properties
assert function.is_bounded() is True
assert function.is_differentiable() is False
# Validate output shape
x = np.random.rand(10000, function.get_dimensionality())
y = function(x)
assert y.shape == (10000, 1)
with pytest.raises(func.NoDerivativeError):
function(x, True)
def test_results_result():
path = "./tmpresultsexp1"
# Test that if path does not exist, exception is raised
with pytest.raises(Exception):
_ = results.Result(path)
# Create experiment
procedure = TmpProcedure()
experiment = TmpExperimentCorrect(procedure, path)
experiment.run(func.GaussianShells())
# Test that results can now be properly be initialised, regardless of
# directory seperator at end of path
r = results.Result(path)
_ = results.Result(path + '/')
assert r.path == path + os.sep
# Test that if benchmarks file is missing, results cannot be read
os.remove(path + '/benchmarks.yaml')
with pytest.raises(Exception):
_ = results.Result(path)
shutil.rmtree(path)
def test_results_df():
# Create experiment
procedure = TmpProcedure()
path = "./tmpresultsexp3"
experiment = TmpExperimentCorrect2(procedure, path)
experiment.run(func.GaussianShells(), finish_line=1000)
experiment.run(func.Sphere(), finish_line=1000)
r = results.Result(path)
# Get df
df = r.get_results()
assert isinstance(df, pd.DataFrame)
assert len(df) == 2
# Create df through function
df2 = results.make_dataframe({'standard': path})
print(df2)
del (df2['experiment'])
assert df.equals(df2)
# Check if indicating multiple folders works
_ = results.make_dataframe({'standard': path, 'other': path})
# Remove folder
shutil.rmtree(path)
def test_results_plots():
# Create experiment
procedure = TmpProcedure()
path = "./tmpresultsexp5"
experiment = TmpExperimentCorrect(procedure, path)
experiment.run(func.GaussianShells(), finish_line=1000)
experiment.run(func.Sphere(), finish_line=1000)
# Get results df
df = results.make_dataframe({'standard': path, 'other': path})
# Boxplot experiment
with pytest.raises(Exception):
results.boxplot_experiment(df, 'time', 'not-an-experiment')
with pytest.raises(Exception):
results.boxplot_experiment(df, 'not-a-metric', 'standard')
results.boxplot_experiment(df,
'time',
'standard',
logarithmic=True,
path='./plot.png')
assert os.path.exists('./plot.png') is True
results.boxplot_experiment(df,
'time',
'standard',
logarithmic=False,
path='./plot.png')
os.remove('./plot.png')
# Boxplot function
with pytest.raises(Exception):
results.boxplot_function(df, 'time', 'not-a-function')
with pytest.raises(Exception):
results.boxplot_function(df, 'not-a-metric', 'sphere')
results.boxplot_function(df,
'time',
'sphere',
logarithmic=True,
path='./plot.png')
assert os.path.exists('./plot.png') is True
results.boxplot_function(df,
'time',
'sphere',
logarithmic=False,
path='./plot.png')
os.remove('./plot.png')
# Histogram experiment
with pytest.raises(Exception):
results.histogram_experiment(df, 'time', 'not-an-experiment')
with pytest.raises(Exception):
results.histogram_experiment(df, 'not-a-metric', 'standard')
results.histogram_experiment(df,
'time',
'standard',
logarithmic=True,
path='./plot.png')
assert os.path.exists('./plot.png') is True
results.histogram_experiment(df,
'time',
'standard',
logarithmic=False,
path='./plot.png')
os.remove('./plot.png')
# Histogram function
with pytest.raises(Exception):
results.histogram_function(df, 'time', 'not-a-function')
with pytest.raises(Exception):
results.histogram_function(df, 'not-a-metric', 'sphere')
results.histogram_function(df,
'time',
'sphere',
logarithmic=True,
path='./plot.png')
assert os.path.exists('./plot.png') is True
results.histogram_function(df,
'time',
'sphere',
logarithmic=False,
path='./plot.png')
os.remove('./plot.png')
shutil.rmtree(path)
def test_results_tables():
# Create experiment
procedure = TmpProcedure()
path = "./tmpresultsexp4"
experiment = TmpExperimentCorrect(procedure, path)
experiment.run(func.GaussianShells(), finish_line=1000)
experiment.run(func.Sphere(), finish_line=1000)
experiment.run(func.GaussianShells(), finish_line=1000)
experiment.run(func.Sphere(), finish_line=1000)
# Get results df
df = results.make_dataframe({'standard': path, 'other': path})
# Check that exceptions are raised correctly
with pytest.raises(Exception):
_, _ = results.tabulate_result(df, 'x', 'not-an-experiment')
with pytest.raises(Exception):
_, _ = results.tabulate_result(df, 'y', 'standard')
# Check that the method works in general
content, rows = results.tabulate_result(df, 'x', 'other')
# Check content and rows format
assert rows.tolist() == ['gaussianshells', 'sphere']
assert isinstance(content, list)
# Check that the method works with functions specified
_, _ = results.tabulate_result(df, 'x', 'other', ['Sphere'])
# Check that results can be stored to csv and tex (validate csv)
_, _ = results.tabulate_result(df, 'x', 'other', path='./tmp_out.csv')
_, _ = results.tabulate_result(df, 'x', 'other', path='./tmp_out.tex')
_ = pd.read_csv('./tmp_out.csv')
# Validate that allowed table output formats are only csv and tex
_ = results.create_table_string(content, rows, ['x'], 'csv', '')
_ = results.create_table_string(content, rows, ['x'], 'tex', '')
with pytest.raises(Exception):
_ = results.create_table_string(content, rows, ['None'], 'txt', '')
# Remove folder and output files
os.remove('./tmp_out.csv')
os.remove('./tmp_out.tex')
# Check that exceptions are raised correctly
print(df)
with pytest.raises(Exception):
_, _, _ = results.tabulate_all_aggregated(
df, 'time', 'mean', experiment_names=['not-an-experiment'])
with pytest.raises(Exception):
_, _, _ = results.tabulate_all_aggregated(df, 'y', 'mean')
_, _, _ = results.tabulate_all_aggregated(df,
'time',
'mean',
functions=['sphere'])
_, _, _ = results.tabulate_all_aggregated(df,
'time',
'mean',
experiment_names=['standard'])
_, _, _ = results.tabulate_all_aggregated(df,
'time',
'mean',
path='./tmp_out.csv')
_ = pd.read_csv('./tmp_out.csv')
content, rows, cols = results.tabulate_all_aggregated(df,
'time',
'mean',
path='./tmp_out.tex')
assert isinstance(rows, list) is True
assert isinstance(cols, list) is True
assert isinstance(content, list)
# Remove folder and output files
os.remove('./tmp_out.csv')
os.remove('./tmp_out.tex')
shutil.rmtree(path)