def test_multidimensional_correct_size(self): """ Parameters must have correct size.""" parameters = { 'p1': { 'type': 'float', 'min': 0, 'max': 10.0, 'size': 3 }, 'p2': { 'type': 'integer', 'min': 0, 'max': 10, 'size': 5 } } scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=parameters, outcome=default_outcome) scientist.update({'p1': [1.2, 2.3, 3.4], 'p2': [4, 2, 5]}, 10.2)
def test_multidimensional_min_max(self): """ All dimensions must be within min/max bounds.""" parameters = { 'p1': { 'type': 'float', 'min': 0, 'max': 10.0, 'size': 3 }, 'p2': { 'type': 'integer', 'min': 0, 'max': 10, 'size': 5 } } scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=parameters, outcome=default_outcome) scientist.update({'p1': [1.2, 2.3, 3.4], 'p2': [4, 2, 5, 12, 3]}, 10.2)
def test_same_name(self): """ Can't create two experiments with same name (when resume is False). """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome) # Repeat Experiment creation to raise error, with resume set to False scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description + '2', parameters=default_parameters, outcome=default_outcome, resume=False)
def test_update_by_result_id(self): """ Update adds and can overwrite a result. """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome) jobs = [] for i in range(5): jobs.append(scientist.suggest()) outcomes = [] for i in range(5): result_id = scientist.get_id(jobs[i]) outcomes.append(npr.randn()) scientist.update_by_result_id(result_id, outcomes[-1]) # Make sure result was added scientist._sync_with_server() for i in range(5): result_id = scientist.get_id(jobs[i]) assert_equals(scientist._ids_to_outcome_values[result_id], outcomes[i])
def test_cancel_by_result_id(self): """ Cancel removes a result. """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome) jobs = [] for i in range(5): jobs.append(scientist.suggest()) for i in range(5): result_id = scientist.get_id(jobs[i]) scientist.update_by_result_id(result_id, npr.randn()) for i in range(5): result_id = scientist.get_id(jobs[i]) scientist.cancel_by_result_id(result_id) # Make sure result was removed scientist._sync_with_server() assert_equals(len(scientist._ids_to_param_values), 0) assert_equals(len(scientist._ids_to_outcome_values), 0)
def test_suggest(self): """ Suggest return a valid job. """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome) a = scientist.suggest() # Check all parameter names are valid in suggestion for k in a.keys(): assert (k in default_parameters.keys()) # Check if all parameters were assigned a value for k in default_parameters.keys(): assert (k in a) # Check parameter values are within the min/max bounds for k, v in a.items(): assert (v >= default_parameters[k]['min']) assert (v <= default_parameters[k]['max']) # Check parameter values are of right type for k, v in a.items(): if default_parameters[k]['type'] == 'integer': assert (type(v) == int) if default_parameters[k]['type'] == 'float': assert (type(v) == float)
def test_update(self): """ Update adds and can overwrite a result. """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome) scientist.update({'p1': 5.1, 'p2': 5}, 10) # Make sure result was added scientist._sync_with_server() assert list(scientist._ids_to_param_values.values())[0] == { 'p1': 5.1, 'p2': 5 } assert list(scientist._ids_to_outcome_values.values())[0] == 10 # Make sure result was overwritten scientist.update({'p1': 5.1, 'p2': 5}, 20) scientist._sync_with_server() assert len(scientist._ids_to_param_values.values()) == 1 assert len(scientist._ids_to_outcome_values.values()) == 1 assert list(scientist._ids_to_param_values.values())[0] == { 'p1': 5.1, 'p2': 5 } assert list(scientist._ids_to_outcome_values.values())[0] == 20
def test_multidimensional_parameters(self): """ Can use multidimensional parameters.""" parameters = { 'p1': { 'type': 'float', 'min': 0, 'max': 10.0, 'size': 3 }, 'p2': { 'type': 'integer', 'min': 0, 'max': 10, 'size': 5 } } scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=parameters, outcome=default_outcome) job = scientist.suggest() assert (len(job['p1']) == 3) assert (len(job['p2']) == 5) scientist.update({'p1': [1.2, 2.3, 3.4], 'p2': [4, 2, 5, 2, 1]}, 10.2)
def test_empty_outcome(self): """ Experiment's outcome can't be empty. """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome={})
def test_access_token(self): """ Valid access token must be provided. """ scientist = whetlab.Experiment(access_token='', name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome)
def test_name_too_long(self): """ Experiment's name must have at most 500 caracters. """ name = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" scientist = whetlab.Experiment(access_token=default_access_token, name=name, description=default_description, parameters=default_parameters, outcome=default_outcome)
def test_enum_with_two_options(self): """ Enums should work with just two options. """ bad_parameters = {'p1': {'type': 'enum', 'options': ['one', 'two']}} scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=bad_parameters, outcome=default_outcome)
def test_resume_false(self): """ If resume is False and experiment's name is unique, can create an experiment. """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome, resume=False)
def test_int_instead_of_integer(self): """ Can use 'int' as type, instead of 'integer'.""" parameters = {'p1': {'type': 'int', 'min': 1, 'max': 4}} scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=parameters, outcome=default_outcome)
def test_bad_enum_options(self): """ Enum options must take a legal name. """ bad_parameters = {'p1': {'type': 'enum', 'options': ['1', '2', '3']}} scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=bad_parameters, outcome=default_outcome)
def test_update_parameter_not_integer(self): """ Update should raise error if an integer parameter has a non-integer value. """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome) scientist.update({'p1': 5., 'p2': 1.}, 5)
def test_update_parameter_not_float(self): """ Update should raise error if a float parameter has a non-float value. """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome) scientist.update({'p1': 'foo', 'p2': 1}, 5)
def test_update_parameter_too_big(self): """ Update should raise error if parameter larger than maximum. """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome) scientist.update({'p1': 5., 'p2': 50}, 5)
def test_resume(self): """ Resume correctly loads previous results. """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome) scientist.update({'p1': 2.1, 'p2': 1}, 3) scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description) # Make sure result is still there assert list(scientist._ids_to_param_values.values())[0] == { 'p1': 2.1, 'p2': 1 } assert list(scientist._ids_to_outcome_values.values())[0] == 3
def test_enum_update(self): """ Update supports enum. """ parameters = {'p1': {'type': 'enum', 'options': ['one', 'two']}} scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=parameters, outcome=default_outcome) scientist.update({'p1': 'one'}, 10)
def test_good_enum_options(self): """ Enum options with a legal name. """ bad_parameters = { 'p1': { 'type': 'enum', 'options': ['one', 'two', 'three'] } } scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=bad_parameters, outcome=default_outcome)
def test_suggest_twice(self): """ Calling suggest twice returns two different jobs. """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome) a = scientist.suggest() sleep(2) b = scientist.suggest() # Two suggested jobs are different assert a != b
def test_delete_experiment(): """ Delete experiment should remove the experiment from the server. """ name = 'test ' + str(time()) scientist = whetlab.Experiment(access_token=default_access_token, name=name, description=default_description, parameters=default_parameters, outcome=default_outcome) scientist.update({'p1': 5., 'p2': 1}, 5) # Delete experiment whetlab.delete_experiment(name, default_access_token) # Should now be possible to create an experiment with the same name scientist = whetlab.Experiment(access_token=default_access_token, name=name, description=default_description, parameters=default_parameters, outcome=default_outcome) # Re-deleting it whetlab.delete_experiment(name, default_access_token)
def test_bad_enum_update(self): """ Enum can't update with value not in options. """ bad_parameters = { 'p1': { 'type': 'enum', 'options': ['one', 'two', 'three'] } } scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=bad_parameters, outcome=default_outcome) scientist.update({'p1': 'four'}, 10.)
def test_update_as_failed(self): """ Update of failed results have -np.inf as outcome. """ scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=default_parameters, outcome=default_outcome) scientist.update_as_failed({'p1': 5.1, 'p2': 5}) # Make sure result was added scientist._sync_with_server() assert float(list( scientist._ids_to_outcome_values.values())[0]) == -np.inf
def test_float_for_int_bounds(self): """ Parameter properties 'min' and 'max' must be integers if the parameter is an integer. """ bad_parameters = { 'p1': { 'type': 'integer', 'min': 0.0, 'max': 0.5, 'size': 1 } } scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=bad_parameters, outcome=default_outcome)
def test_enum_not_supported(self): """ Parameter type 'enum' not yet supported. """ bad_parameters = { 'p1': { 'type': 'enum', 'min': 1., 'max': 10., 'size': 1 } } scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=bad_parameters, outcome=default_outcome)
def test_legal_property_value(self): """ Parameter property must take a legal value. """ bad_parameters = { 'p1': { 'type': 'BAD_VALUE', 'min': 1., 'max': 10., 'size': 1 } } scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=bad_parameters, outcome=default_outcome)
def test_vector_bounds(self): """ Parameter properties 'min' and 'max' must be finite numbers. """ bad_parameters = { 'p1': { 'type': 'float', 'min': [0.1, 0.2], 'max': 0.5, 'size': 1 } } scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=bad_parameters, outcome=default_outcome)
def test_large_bounds(self): """ Parameter properties 'min' and 'max' must be less than 1e32. """ bad_parameters = { 'p1': { 'type': 'integer', 'min': 1.0, 'max': 1e33, 'size': 1 } } scientist = whetlab.Experiment(access_token=default_access_token, name=self.name, description=default_description, parameters=bad_parameters, outcome=default_outcome)