def test_multidimensional_correct_size(self):
""" Parameters must have correct size."""
parameters = {
'p1': {
'type': 'float',
'min': 0,
'max': 10.0,
'size': 3
},
'p2': {
'type': 'integer',
'min': 0,
'max': 10,
'size': 5
}
}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=parameters,
outcome=default_outcome)
scientist.update({'p1': [1.2, 2.3, 3.4], 'p2': [4, 2, 5]}, 10.2)
def test_multidimensional_min_max(self):
""" All dimensions must be within min/max bounds."""
parameters = {
'p1': {
'type': 'float',
'min': 0,
'max': 10.0,
'size': 3
},
'p2': {
'type': 'integer',
'min': 0,
'max': 10,
'size': 5
}
}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=parameters,
outcome=default_outcome)
scientist.update({'p1': [1.2, 2.3, 3.4], 'p2': [4, 2, 5, 12, 3]}, 10.2)
def test_same_name(self):
""" Can't create two experiments with same name (when resume is False). """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
# Repeat Experiment creation to raise error, with resume set to False
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description + '2',
parameters=default_parameters,
outcome=default_outcome,
resume=False)
def test_update_by_result_id(self):
""" Update adds and can overwrite a result. """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
jobs = []
for i in range(5):
jobs.append(scientist.suggest())
outcomes = []
for i in range(5):
result_id = scientist.get_id(jobs[i])
outcomes.append(npr.randn())
scientist.update_by_result_id(result_id, outcomes[-1])
# Make sure result was added
scientist._sync_with_server()
for i in range(5):
result_id = scientist.get_id(jobs[i])
assert_equals(scientist._ids_to_outcome_values[result_id],
outcomes[i])
def test_cancel_by_result_id(self):
""" Cancel removes a result. """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
jobs = []
for i in range(5):
jobs.append(scientist.suggest())
for i in range(5):
result_id = scientist.get_id(jobs[i])
scientist.update_by_result_id(result_id, npr.randn())
for i in range(5):
result_id = scientist.get_id(jobs[i])
scientist.cancel_by_result_id(result_id)
# Make sure result was removed
scientist._sync_with_server()
assert_equals(len(scientist._ids_to_param_values), 0)
assert_equals(len(scientist._ids_to_outcome_values), 0)
def test_suggest(self):
""" Suggest return a valid job. """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
a = scientist.suggest()
# Check all parameter names are valid in suggestion
for k in a.keys():
assert (k in default_parameters.keys())
# Check if all parameters were assigned a value
for k in default_parameters.keys():
assert (k in a)
# Check parameter values are within the min/max bounds
for k, v in a.items():
assert (v >= default_parameters[k]['min'])
assert (v <= default_parameters[k]['max'])
# Check parameter values are of right type
for k, v in a.items():
if default_parameters[k]['type'] == 'integer':
assert (type(v) == int)
if default_parameters[k]['type'] == 'float':
assert (type(v) == float)
def test_update(self):
""" Update adds and can overwrite a result. """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
scientist.update({'p1': 5.1, 'p2': 5}, 10)
# Make sure result was added
scientist._sync_with_server()
assert list(scientist._ids_to_param_values.values())[0] == {
'p1': 5.1,
'p2': 5
}
assert list(scientist._ids_to_outcome_values.values())[0] == 10
# Make sure result was overwritten
scientist.update({'p1': 5.1, 'p2': 5}, 20)
scientist._sync_with_server()
assert len(scientist._ids_to_param_values.values()) == 1
assert len(scientist._ids_to_outcome_values.values()) == 1
assert list(scientist._ids_to_param_values.values())[0] == {
'p1': 5.1,
'p2': 5
}
assert list(scientist._ids_to_outcome_values.values())[0] == 20
def test_multidimensional_parameters(self):
""" Can use multidimensional parameters."""
parameters = {
'p1': {
'type': 'float',
'min': 0,
'max': 10.0,
'size': 3
},
'p2': {
'type': 'integer',
'min': 0,
'max': 10,
'size': 5
}
}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=parameters,
outcome=default_outcome)
job = scientist.suggest()
assert (len(job['p1']) == 3)
assert (len(job['p2']) == 5)
scientist.update({'p1': [1.2, 2.3, 3.4], 'p2': [4, 2, 5, 2, 1]}, 10.2)
def test_empty_outcome(self):
""" Experiment's outcome can't be empty. """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome={})
def test_access_token(self):
""" Valid access token must be provided. """
scientist = whetlab.Experiment(access_token='',
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
def test_name_too_long(self):
""" Experiment's name must have at most 500 caracters. """
name = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
scientist = whetlab.Experiment(access_token=default_access_token,
name=name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
def test_enum_with_two_options(self):
""" Enums should work with just two options. """
bad_parameters = {'p1': {'type': 'enum', 'options': ['one', 'two']}}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=bad_parameters,
outcome=default_outcome)
def test_resume_false(self):
""" If resume is False and experiment's name is unique, can create an experiment. """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome,
resume=False)
def test_int_instead_of_integer(self):
""" Can use 'int' as type, instead of 'integer'."""
parameters = {'p1': {'type': 'int', 'min': 1, 'max': 4}}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=parameters,
outcome=default_outcome)
def test_bad_enum_options(self):
""" Enum options must take a legal name. """
bad_parameters = {'p1': {'type': 'enum', 'options': ['1', '2', '3']}}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=bad_parameters,
outcome=default_outcome)
def test_update_parameter_not_integer(self):
""" Update should raise error if an integer parameter has a non-integer value. """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
scientist.update({'p1': 5., 'p2': 1.}, 5)
def test_update_parameter_not_float(self):
""" Update should raise error if a float parameter has a non-float value. """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
scientist.update({'p1': 'foo', 'p2': 1}, 5)
def test_update_parameter_too_big(self):
""" Update should raise error if parameter larger than maximum. """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
scientist.update({'p1': 5., 'p2': 50}, 5)
def test_resume(self):
""" Resume correctly loads previous results. """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
scientist.update({'p1': 2.1, 'p2': 1}, 3)
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description)
# Make sure result is still there
assert list(scientist._ids_to_param_values.values())[0] == {
'p1': 2.1,
'p2': 1
}
assert list(scientist._ids_to_outcome_values.values())[0] == 3
def test_enum_update(self):
""" Update supports enum. """
parameters = {'p1': {'type': 'enum', 'options': ['one', 'two']}}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=parameters,
outcome=default_outcome)
scientist.update({'p1': 'one'}, 10)
def test_good_enum_options(self):
""" Enum options with a legal name. """
bad_parameters = {
'p1': {
'type': 'enum',
'options': ['one', 'two', 'three']
}
}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=bad_parameters,
outcome=default_outcome)
def test_suggest_twice(self):
""" Calling suggest twice returns two different jobs. """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
a = scientist.suggest()
sleep(2)
b = scientist.suggest()
# Two suggested jobs are different
assert a != b
def test_delete_experiment():
""" Delete experiment should remove the experiment from the server. """
name = 'test ' + str(time())
scientist = whetlab.Experiment(access_token=default_access_token,
name=name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
scientist.update({'p1': 5., 'p2': 1}, 5)
# Delete experiment
whetlab.delete_experiment(name, default_access_token)
# Should now be possible to create an experiment with the same name
scientist = whetlab.Experiment(access_token=default_access_token,
name=name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
# Re-deleting it
whetlab.delete_experiment(name, default_access_token)
def test_bad_enum_update(self):
""" Enum can't update with value not in options. """
bad_parameters = {
'p1': {
'type': 'enum',
'options': ['one', 'two', 'three']
}
}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=bad_parameters,
outcome=default_outcome)
scientist.update({'p1': 'four'}, 10.)
def test_update_as_failed(self):
""" Update of failed results have -np.inf as outcome. """
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=default_parameters,
outcome=default_outcome)
scientist.update_as_failed({'p1': 5.1, 'p2': 5})
# Make sure result was added
scientist._sync_with_server()
assert float(list(
scientist._ids_to_outcome_values.values())[0]) == -np.inf
def test_float_for_int_bounds(self):
""" Parameter properties 'min' and 'max' must be integers if the parameter is an integer. """
bad_parameters = {
'p1': {
'type': 'integer',
'min': 0.0,
'max': 0.5,
'size': 1
}
}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=bad_parameters,
outcome=default_outcome)
def test_enum_not_supported(self):
""" Parameter type 'enum' not yet supported. """
bad_parameters = {
'p1': {
'type': 'enum',
'min': 1.,
'max': 10.,
'size': 1
}
}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=bad_parameters,
outcome=default_outcome)
def test_legal_property_value(self):
""" Parameter property must take a legal value. """
bad_parameters = {
'p1': {
'type': 'BAD_VALUE',
'min': 1.,
'max': 10.,
'size': 1
}
}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=bad_parameters,
outcome=default_outcome)
def test_vector_bounds(self):
""" Parameter properties 'min' and 'max' must be finite numbers. """
bad_parameters = {
'p1': {
'type': 'float',
'min': [0.1, 0.2],
'max': 0.5,
'size': 1
}
}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=bad_parameters,
outcome=default_outcome)
def test_large_bounds(self):
""" Parameter properties 'min' and 'max' must be less than 1e32. """
bad_parameters = {
'p1': {
'type': 'integer',
'min': 1.0,
'max': 1e33,
'size': 1
}
}
scientist = whetlab.Experiment(access_token=default_access_token,
name=self.name,
description=default_description,
parameters=bad_parameters,
outcome=default_outcome)
