def test_make_bandit_historical_info_from_params_variance_passed_through(self): """Test that the variance of a given sample arm got passed through.""" historical_info = self.three_arms_with_variance_no_unsampled_arm_test_case T.assert_dicts_equal( _make_bandit_historical_info_from_params(self.make_params_from_bandit_historical_info(historical_info)).json_payload(), historical_info.json_payload() )
def test_list_path_with_non_leaf(self): """ Given a path that contains a single non-leaf child node When path is listed Then return a single non-leaf node """ with patch_urlopen() as mock_urlopen: mock_response = Mock() mock_response.read.return_value = """ [ { "leaf": 0, "context": {}, "text" : "servers", "expandable": 1, "id": "servers", "allowChildren": 1 } ] """ mock_urlopen.return_value = mock_response result = self.ds.list_path(path=['a','b','c']) T.assert_equal(1, len(result)) expected_servers_dict = {'type': 'dir', 'name': u'servers', 'children': None} T.assert_dicts_equal(expected_servers_dict, result[0]) mock_urlopen.assert_called_with(match(starts_with('http://dontcare.com:8080/metrics/find'))) mock_urlopen.assert_called_with(match(contains_string('query=a.b.c.*')))
def test_hyperparameters_passed_through(self): """Test that the hyperparameters get passed through to the endpoint.""" test_case = self.gp_test_environments[0] python_domain, python_gp = test_case python_cov, historical_data = python_gp.get_core_data_copy() # Test default test parameters get passed through json_payload = json.loads(self._build_json_payload(python_domain, python_cov, historical_data)) request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = GpHyperOptView(request) params = view.get_params_from_request() T.assert_dicts_equal(params['hyperparameter_domain_info'], json_payload['hyperparameter_domain_info']) # Test arbitrary parameters get passed through json_payload['hyperparameter_domain_info']['domain_bounds'] = [] for i in range(json_payload['hyperparameter_domain_info']['dim']): json_payload['hyperparameter_domain_info']['domain_bounds'].append({ 'min': 0.2 * i, 'max': 0.5 * i, }) request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = GpHyperOptView(request) params = view.get_params_from_request() T.assert_dicts_equal(params['hyperparameter_domain_info'], json_payload['hyperparameter_domain_info'])
def test_two_arms_epsilon_zero(self): """Check that the two-arms case with zero epsilon always allocate arm1:1.0 and arm2:0.0 when average payoffs are arm1:1.0 and arm2:0.0.""" epsilon = 0.0 bandit = self.bandit_class(self.two_arms_test_case, epsilon) arms_to_allocations = bandit.allocate_arms() T.assert_dicts_equal(arms_to_allocations, {"arm1": 1.0, "arm2": 0.0}) T.assert_equal(bandit.choose_arm(arms_to_allocations), "arm1")
def test_epsilon_greedy_hyperparameters_passed_through(self): """Test that the hyperparameters get passed through to the epsilon-greedy endpoint.""" historical_info = self.one_arm_test_case # Test default test parameters get passed through json_payload = json.loads( self._build_json_payload( EPSILON_SUBTYPE_GREEDY, historical_info, EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[ EPSILON_SUBTYPE_GREEDY])) request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = BanditEpsilonView(request) params = view.get_params_from_request() T.assert_dicts_equal(params['hyperparameter_info'], json_payload['hyperparameter_info']) # Test arbitrary epsilons get passed through json_payload['hyperparameter_info']['epsilon'] = 1.0 request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = BanditEpsilonView(request) params = view.get_params_from_request() T.assert_dicts_equal(params['hyperparameter_info'], json_payload['hyperparameter_info'])
def test_two_arms_epsilon_zero(self): """Check that the two-arms case with zero epsilon (always exploit) always allocate arm1:1.0 and arm2:0.0 when average payoffs are arm1:1.0 and arm2:0.0.""" epsilon = 0.0 bandit = self.bandit_class(self.two_arms_test_case, epsilon) arms_to_allocations = bandit.allocate_arms() T.assert_dicts_equal(arms_to_allocations, {"arm1": 1.0, "arm2": 0.0}) T.assert_equal(bandit.choose_arm(arms_to_allocations), "arm1")
def test_epsilon_first_hyperparameters_passed_through(self): """Test that the hyperparameters get passed through to the epsilon-first endpoint.""" historical_info = self.one_arm_test_case # Test default test parameters get passed through json_payload = json.loads( self._build_json_payload( EPSILON_SUBTYPE_FIRST, historical_info, EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[ EPSILON_SUBTYPE_FIRST])) request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = self._view(request) params = view.get_params_from_request() T.assert_dicts_equal(params['hyperparameter_info'], json_payload['hyperparameter_info']) # Test an arbitrary epsilon and total_tamples get passed through json_payload['hyperparameter_info']['epsilon'] = 1.0 json_payload['hyperparameter_info']['total_samples'] = 20000 request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = self._view(request) params = view.get_params_from_request() T.assert_dicts_equal(params['hyperparameter_info'], json_payload['hyperparameter_info'])
def test_one_arm(self): """Check that the one-arm case always returns the given arm as the winning arm and the allocation is 1.0.""" for epsilon in self.epsilons_to_test: for total_samples in self.total_samples_to_test: bandit = self.bandit_class(self.one_arm_test_case, epsilon, total_samples) T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 1.0}) T.assert_equal(bandit.choose_arm(), "arm1")
def test_three_arms_exploit(self): """Check that the three-arms cases with integer and float payoffs in exploitation phase return the expected arm allocations.""" epsilon = 0.5 total_samples = 10 for historical_info in [self.three_arms_test_case, self.three_arms_float_payoffs_test_case]: bandit = self.bandit_class(historical_info, epsilon, total_samples) T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 1.0, "arm2": 0.0, "arm3": 0.0})
def test_list_path_with_non_leaf(self): """ Given a path that contains a single non-leaf child node When path is listed Then return a single non-leaf node """ with patch_urlopen() as mock_urlopen: mock_response = Mock() mock_response.read.return_value = """ [ { "leaf": 0, "context": {}, "text" : "servers", "expandable": 1, "id": "servers", "allowChildren": 1 } ] """ mock_urlopen.return_value = mock_response result = self.ds.list_path(path=["a", "b", "c"]) T.assert_equal(1, len(result)) expected_servers_dict = {"type": "dir", "name": u"servers", "children": None} T.assert_dicts_equal(expected_servers_dict, result[0]) mock_urlopen.assert_called_with(match(starts_with("http://dontcare.com:8080/metrics/find"))) mock_urlopen.assert_called_with(match(contains_string("query=a.b.c.*")))
def test_get_equal_arm_allocations_no_winner(self): """Test allocations split among all sample arms when there is no winner.""" T.assert_dicts_equal( get_equal_arm_allocations( self.two_unsampled_arms_test_case.arms_sampled), { "arm1": 0.5, "arm2": 0.5 })
def test_historical_data_append_unsampled_arm(self): """Test that adding an unsampled arm (already exists in historical info) to HistoricalData does not change anything.""" historical_info = self.two_unsampled_arms_test_case historical_info.append_sample_arms(self.one_arm_test_case.arms_sampled) T.assert_dicts_equal( historical_info.json_payload(), self.two_unsampled_arms_test_case.json_payload() )
def test_two_arms_epsilon_one(self): """Check that the two-arms case with one epsilon (always explore) always allocate arm1:0.5 and arm2:0.5 when average payoffs are arm1:1.0 and arm2:0.0.""" epsilon = 1.0 bandit = self.bandit_class(self.two_arms_test_case, epsilon) T.assert_dicts_equal(bandit.allocate_arms(), { "arm1": 0.5, "arm2": 0.5 })
def test_make_bandit_historical_info_from_params_make_bernoulli_arms(self): """Test that the function can make historical infos with Bernoulli arms.""" historical_info = self.three_arms_with_variance_no_unsampled_arm_test_case for historical_info in self.bernoulli_historical_infos_to_test: T.assert_dicts_equal( _make_bandit_historical_info_from_params(self.make_params_from_bandit_historical_info(historical_info), BernoulliArm).json_payload(), historical_info.json_payload() )
def test_make_bandit_historical_info_from_params_make_bernoulli_arms(self): """Test that the function can make historical infos with Bernoulli arms.""" historical_info = self.three_arms_with_variance_no_unsampled_arm_test_case for historical_info in self.bernoulli_historical_infos_to_test: T.assert_dicts_equal( _make_bandit_historical_info_from_params( self.make_params_from_bandit_historical_info( historical_info), BernoulliArm).json_payload(), historical_info.json_payload())
def test_make_bandit_historical_info_from_params_variance_passed_through( self): """Test that the variance of a given sample arm got passed through.""" historical_info = self.three_arms_with_variance_no_unsampled_arm_test_case T.assert_dicts_equal( _make_bandit_historical_info_from_params( self.make_params_from_bandit_historical_info( historical_info)).json_payload(), historical_info.json_payload())
def test_two_unsampled_arms(self): """Check that the two-unsampled-arms case always allocate each arm equally (the allocation is 0.5 for both arms). This tests num_winning_arms == num_arms > 1.""" for epsilon in self.epsilons_to_test: bandit = self.bandit_class(self.two_unsampled_arms_test_case, epsilon) T.assert_dicts_equal(bandit.allocate_arms(), { "arm1": 0.5, "arm2": 0.5 })
def test_two_arms_one_winner(self): """Check that the two-arms case with random seed 0 always allocate arm1:1.0 and arm2:0.0.""" old_state = numpy.random.get_state() numpy.random.seed(0) bandit = self.bandit_class(self.two_arms_test_case, DEFAULT_BLA_SUBTYPE) arms_to_allocations = bandit.allocate_arms() T.assert_dicts_equal(arms_to_allocations, {"arm1": 1.0, "arm2": 0.0}) T.assert_equal(bandit.choose_arm(arms_to_allocations), "arm1") numpy.random.set_state(old_state)
def test_get_equal_arm_allocations_one_winner(self): """Test all allocation given to the winning arm.""" T.assert_dicts_equal( get_equal_arm_allocations(self.three_arms_test_case.arms_sampled, frozenset(["arm1"])), { "arm1": 1.0, "arm2": 0.0, "arm3": 0.0 })
def test_data_multiple_metrics(self): sources = [ ['servers', 'admin1', 'loadavg', '01'], ['servers', 'admin2', 'loadavg', '01'], ] start = 1391047920 end = 1391048100 with patch_urlopen() as mock_urlopen: mock_admin1_response = Mock() mock_admin1_response.read.return_value = """ [ { "target": "servers.admin1.loadavg.01", "datapoints": [ [2.0, 1391047920], [6.0, 1391047980], [9.0, 1391048040], [null,1391048100] ] } ] """ mock_admin2_response = Mock() mock_admin2_response.read.return_value = """ [ { "target": "servers.admin2.loadavg.01", "datapoints": [ [1.0, 1391047920], [7.0, 1391047980], [10.0, 1391048040], [null,1391048100] ] } ] """ mock_urlopen.side_effect = [mock_admin1_response, mock_admin2_response] result_json = self.ds.data(sources, start, end, width=100) # [null, 1391048100] expected_results = [ { 't': 1391047920, 'v': [2.0, 1.0]}, { 't': 1391047980, 'v': [6.0, 7.0]}, { 't': 1391048040, 'v': [9.0, 10.0]}, { 't': 1391048100, 'v': [None, None]} ] result_list = json.loads(result_json) T.assert_equal(4, len(result_list)) for i,expected_result in enumerate(expected_results): T.assert_dicts_equal(expected_result, result_list[i]) T.assert_equal(2, mock_urlopen.call_count)
def test_get_equal_arm_allocations_two_winners(self): """Test allocations split between two winning arms.""" T.assert_dicts_equal( get_equal_arm_allocations( self.three_arms_two_winners_test_case.arms_sampled, frozenset(["arm1", "arm2"])), { "arm1": 0.5, "arm2": 0.5, "arm3": 0.0 })
def test_three_arms_two_winners(self): """Check that the three-arms cases with two winners return the expected arm allocations. This tests num_arms > num_winning_arms > 1.""" epsilon = 0.03 bandit = self.bandit_class(self.three_arms_two_winners_test_case, epsilon) T.assert_dicts_equal(bandit.allocate_arms(), { "arm1": 0.495, "arm2": 0.495, "arm3": 0.01 })
def test_three_arms_exploit_two_winners(self): """Check that the three-arms cases with two winners in exploitation phase return the expected arm allocations. This tests num_arms > num_winning_arms > 1.""" epsilon = 0.5 total_samples = 10 bandit = self.bandit_class(self.three_arms_two_winners_test_case, epsilon, total_samples) T.assert_dicts_equal(bandit.allocate_arms(), { "arm1": 0.5, "arm2": 0.5, "arm3": 0.0 })
def test_data_multiple_metrics(self): sources = [["servers", "admin1", "loadavg", "01"], ["servers", "admin2", "loadavg", "01"]] start = 1391047920 end = 1391048100 with patch_urlopen() as mock_urlopen: mock_admin1_response = Mock() mock_admin1_response.read.return_value = """ [ { "target": "servers.admin1.loadavg.01", "datapoints": [ [2.0, 1391047920], [6.0, 1391047980], [9.0, 1391048040], [null,1391048100] ] } ] """ mock_admin2_response = Mock() mock_admin2_response.read.return_value = """ [ { "target": "servers.admin2.loadavg.01", "datapoints": [ [1.0, 1391047920], [7.0, 1391047980], [10.0, 1391048040], [null,1391048100] ] } ] """ mock_urlopen.side_effect = [mock_admin1_response, mock_admin2_response] result_json = self.ds.data(sources, start, end, width=100) # [null, 1391048100] expected_results = [ {"t": 1391047920, "v": [2.0, 1.0]}, {"t": 1391047980, "v": [6.0, 7.0]}, {"t": 1391048040, "v": [9.0, 10.0]}, {"t": 1391048100, "v": [None, None]}, ] result_list = json.loads(result_json) T.assert_equal(4, len(result_list)) for i, expected_result in enumerate(expected_results): T.assert_dicts_equal(expected_result, result_list[i]) T.assert_equal(2, mock_urlopen.call_count)
def test_three_arms(self): """Check that the three-arms cases with integer and float payoffs return the expected arm allocations.""" epsilon = 0.03 for historical_info in [ self.three_arms_test_case, self.three_arms_float_payoffs_test_case ]: bandit = self.bandit_class(historical_info, epsilon) T.assert_dicts_equal(bandit.allocate_arms(), { "arm1": 0.98, "arm2": 0.01, "arm3": 0.01 })
def _test_historical_info_passed_through(self): """Test that the historical infos get passed through to the endpoint.""" for subtype in BANDIT_ENDPOINTS_TO_SUBTYPES[self._endpoint]: for historical_info in self._historical_infos: # Test default test parameters get passed through json_payload = json.loads(self._build_json_payload(subtype, historical_info)) request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = self._view(request) params = view.get_params_from_request() T.assert_dicts_equal(params['historical_info'], json_payload['historical_info'])
def test_historical_info_passed_through(self): """Test that the historical info get passed through to the endpoint.""" for subtype in EPSILON_SUBTYPES: for historical_info in self.historical_infos_to_test: # Test default test parameters get passed through json_payload = json.loads(self._build_json_payload(subtype, historical_info, EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[subtype])) request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = BanditEpsilonView(request) params = view.get_params_from_request() T.assert_dicts_equal(params['historical_info'], json_payload['historical_info'])
def test_no_hang(self, process_mock): """Test that nothing hangs if the processes are never used. The mechanics of this test are tricky: We make it such that a process pool is deleted, but this process pool has an overridden method that tells our test class that it actually cleaned up workers. """ test_passes = {} processes = vimap.pool.fork(worker_proc.init_args(init=i) for i in [1, 2, 3]) processes.finish_workers = lambda: test_passes.setdefault("result", True) del processes # will happen if it falls out of scope # gc.collect() -- doesn't seem necessary T.assert_dicts_equal(test_passes, {"result": True})
def test_historical_data_append_arms(self): """Test that appending arms to HistoricalData updates historical info correctly.""" historical_info = copy.deepcopy(self.three_arms_test_case) historical_info.append_sample_arms( self.three_arms_two_winners_test_case.arms_sampled) expected_historical_info = HistoricalData( sample_arms={ "arm1": SampleArm(win=4, loss=2, total=6), "arm2": SampleArm(win=3, loss=2, total=5), "arm3": SampleArm(win=0, loss=0, total=0), }) T.assert_dicts_equal(historical_info.json_payload(), expected_historical_info.json_payload())
def test_three_arms_exploit(self): """Check that the three-arms cases with integer and float payoffs in exploitation phase return the expected arm allocations.""" epsilon = 0.5 total_samples = 10 for historical_info in [ self.three_arms_test_case, self.three_arms_float_payoffs_test_case ]: bandit = self.bandit_class(historical_info, epsilon, total_samples) T.assert_dicts_equal(bandit.allocate_arms(), { "arm1": 1.0, "arm2": 0.0, "arm3": 0.0 })
def test_historical_data_append_arms(self): """Test that appending arms to HistoricalData updates historical info correctly.""" historical_info = copy.deepcopy(self.three_arms_test_case) historical_info.append_sample_arms(self.three_arms_two_winners_test_case.arms_sampled) expected_historical_info = HistoricalData( sample_arms={ "arm1": SampleArm(win=4, loss=2, total=6), "arm2": SampleArm(win=3, loss=2, total=5), "arm3": SampleArm(win=0, loss=0, total=0), } ) T.assert_dicts_equal( historical_info.json_payload(), expected_historical_info.json_payload() )
def _test_historical_info_passed_through(self): """Test that the historical infos get passed through to the endpoint.""" for subtype in BANDIT_ENDPOINTS_TO_SUBTYPES[self._endpoint]: for historical_info in self._historical_infos: # Test default test parameters get passed through json_payload = json.loads( self._build_json_payload(subtype, historical_info)) request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = self._view(request) params = view.get_params_from_request() T.assert_dicts_equal(params['historical_info'], json_payload['historical_info'])
def test_no_hang(self, process_mock): '''Test that nothing hangs if the processes are never used. The mechanics of this test are tricky: We make it such that a process pool is deleted, but this process pool has an overridden method that tells our test class that it actually cleaned up workers. ''' test_passes = {} processes = vimap.pool.fork( worker_proc.init_args(init=i) for i in [1, 2, 3]) processes.finish_workers = lambda: test_passes.setdefault( 'result', True) del processes # will happen if it falls out of scope # gc.collect() -- doesn't seem necessary T.assert_dicts_equal(test_passes, {'result': True})
def test_data_single_metric(self): sources = [['servers', 'admin1', 'loadavg', '01']] start = 1391047920 end = 1391048100 with patch_urlopen() as mock_urlopen: mock_response = Mock() mock_response.read.return_value = """ [ { "target": "servers.admin1.loadavg.01", "datapoints": [ [2.0, 1391047920], [6.0, 1391047980], [9.0, 1391048040], [null,1391048100] ] } ] """ mock_urlopen.return_value = mock_response result_json = self.ds.data(sources, start, end, width=100) expected_results = [ { 't': 1391047920, 'v': [2.0]}, { 't': 1391047980, 'v': [6.0]}, { 't': 1391048040, 'v': [9.0]}, { 't': 1391048100, 'v': [None]} ] result_list = json.loads(result_json) T.assert_equal(4, len(result_list)) for i,expected_result in enumerate(expected_results): T.assert_dicts_equal(expected_result, result_list[i]) mock_urlopen.assert_called_with(match(starts_with('http://dontcare.com:8080/render'))) mock_urlopen.assert_called_with(match(contains_string('.'.join(sources[0]))))
def test_data_single_metric(self): sources = [["servers", "admin1", "loadavg", "01"]] start = 1391047920 end = 1391048100 with patch_urlopen() as mock_urlopen: mock_response = Mock() mock_response.read.return_value = """ [ { "target": "servers.admin1.loadavg.01", "datapoints": [ [2.0, 1391047920], [6.0, 1391047980], [9.0, 1391048040], [null,1391048100] ] } ] """ mock_urlopen.return_value = mock_response result_json = self.ds.data(sources, start, end, width=100) expected_results = [ {"t": 1391047920, "v": [2.0]}, {"t": 1391047980, "v": [6.0]}, {"t": 1391048040, "v": [9.0]}, {"t": 1391048100, "v": [None]}, ] result_list = json.loads(result_json) T.assert_equal(4, len(result_list)) for i, expected_result in enumerate(expected_results): T.assert_dicts_equal(expected_result, result_list[i]) mock_urlopen.assert_called_with(match(starts_with("http://dontcare.com:8080/render"))) mock_urlopen.assert_called_with(match(contains_string(".".join(sources[0]))))
def test_epsilon_greedy_hyperparameters_passed_through(self): """Test that the hyperparameters get passed through to the epsilon-greedy endpoint.""" historical_info = self.one_arm_test_case # Test default test parameters get passed through json_payload = json.loads(self._build_json_payload(EPSILON_SUBTYPE_GREEDY, historical_info, EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[EPSILON_SUBTYPE_GREEDY])) request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = BanditEpsilonView(request) params = view.get_params_from_request() T.assert_dicts_equal(params['hyperparameter_info'], json_payload['hyperparameter_info']) # Test arbitrary epsilons get passed through json_payload['hyperparameter_info']['epsilon'] = 1.0 request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = BanditEpsilonView(request) params = view.get_params_from_request() T.assert_dicts_equal(params['hyperparameter_info'], json_payload['hyperparameter_info'])
def test_hyperparameters_passed_through(self): """Test that the hyperparameters get passed through to the endpoint.""" test_case = self.gp_test_environments[0] python_domain, python_gp = test_case python_cov, historical_data = python_gp.get_core_data_copy() # Test default test parameters get passed through json_payload = json.loads( self._build_json_payload(python_domain, python_cov, historical_data)) request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = GpHyperOptView(request) params = view.get_params_from_request() T.assert_dicts_equal(params['hyperparameter_domain_info'], json_payload['hyperparameter_domain_info']) # Test arbitrary parameters get passed through json_payload['hyperparameter_domain_info']['domain_bounds'] = [] for i in range(json_payload['hyperparameter_domain_info']['dim']): json_payload['hyperparameter_domain_info']['domain_bounds'].append( { 'min': 0.2 * i, 'max': 0.5 * i, }) request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = GpHyperOptView(request) params = view.get_params_from_request() T.assert_dicts_equal(params['hyperparameter_domain_info'], json_payload['hyperparameter_domain_info'])
def test_epsilon_first_hyperparameters_passed_through(self): """Test that the hyperparameters get passed through to the epsilon-first endpoint.""" historical_info = self.one_arm_test_case # Test default test parameters get passed through json_payload = json.loads(self._build_json_payload(EPSILON_SUBTYPE_FIRST, historical_info, EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[EPSILON_SUBTYPE_FIRST])) request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = self._view(request) params = view.get_params_from_request() T.assert_dicts_equal(params['hyperparameter_info'], json_payload['hyperparameter_info']) # Test an arbitrary epsilon and total_tamples get passed through json_payload['hyperparameter_info']['epsilon'] = 1.0 json_payload['hyperparameter_info']['total_samples'] = 20000 request = pyramid.testing.DummyRequest(post=json_payload) request.json_body = json_payload view = self._view(request) params = view.get_params_from_request() T.assert_dicts_equal(params['hyperparameter_info'], json_payload['hyperparameter_info'])
def _test_three_arms_two_winners(self): """Check that the three-arms cases with two winners return the expected arm allocations. This tests num_arms > num_winning_arms > 1.""" bandit = self.bandit_class(self.three_arms_two_winners_no_unsampled_arm_test_case) T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.5, "arm2": 0.5, "arm3": 0.0})
def test_three_arms_two_winners(self): """Check that the three-arms cases with two winners return the expected arm allocations. This tests num_arms > num_winning_arms > 1.""" epsilon = 0.03 bandit = self.bandit_class(self.three_arms_two_winners_test_case, epsilon) T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.495, "arm2": 0.495, "arm3": 0.01})
def test_three_arms(self): """Check that the three-arms cases with integer and float payoffs return the expected arm allocations.""" epsilon = 0.03 for historical_info in [self.three_arms_test_case, self.three_arms_float_payoffs_test_case]: bandit = self.bandit_class(historical_info, epsilon) T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.98, "arm2": 0.01, "arm3": 0.01})
def test_two_arms_epsilon_one(self): """Check that the two-arms case with one epsilon always allocate arm1:0.5 and arm2:0.5 when average payoffs are arm1:1.0 and arm2:0.0.""" epsilon = 1.0 bandit = self.bandit_class(self.two_arms_test_case, epsilon) T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.5, "arm2": 0.5})
def test_three_arms_exploit_two_winners(self): """Check that the three-arms cases with two winners in exploitation phase return the expected arm allocations. This tests num_arms > num_winning_arms > 1.""" epsilon = 0.5 total_samples = 10 bandit = self.bandit_class(self.three_arms_two_winners_test_case, epsilon, total_samples) T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.5, "arm2": 0.5, "arm3": 0.0})
def assert_push_info_attributes(self, ul_attributes, expected): T.assert_dicts_equal(ul_attributes, expected)
def test_get_equal_arm_allocations_no_winner(self): """Test allocations split among all sample arms when there is no winner.""" T.assert_dicts_equal( get_equal_arm_allocations(self.two_unsampled_arms_test_case.arms_sampled), {"arm1": 0.5, "arm2": 0.5} )
def test_get_equal_arm_allocations_one_winner(self): """Test all allocation given to the winning arm.""" T.assert_dicts_equal( get_equal_arm_allocations(self.three_arms_test_case.arms_sampled, frozenset(["arm1"])), {"arm1": 1.0, "arm2": 0.0, "arm3": 0.0}, )
def test_get_equal_arm_allocations_two_winners(self): """Test allocations split between two winning arms.""" T.assert_dicts_equal( get_equal_arm_allocations(self.three_arms_two_winners_test_case.arms_sampled, frozenset(["arm1", "arm2"])), {"arm1": 0.5, "arm2": 0.5, "arm3": 0.0}, )
def _test_one_arm(self, bandit): """Check that the one-arm case always returns the given arm as the winning arm and the allocation is 1.0.""" bandit = self.bandit_class(self.one_arm_test_case) arms_to_allocations = bandit.allocate_arms() T.assert_dicts_equal(arms_to_allocations, {"arm1": 1.0}) T.assert_equal(bandit.choose_arm(arms_to_allocations), "arm1")
def _test_three_arms_one_unsampled_arm(self): """Check that the three-arms cases with integer and float payoffs return the expected arm allocations. When arm3 is the only unsampled arm, we expect all allocation is given to arm3.""" for historical_info in [self.three_arms_test_case, self.three_arms_float_payoffs_test_case, self.three_arms_two_winners_test_case]: bandit = self.bandit_class(historical_info) T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.0, "arm2": 0.0, "arm3": 1.0})
def test_two_new_arms(self): """Check that the two-new-arms case always allocate each arm equally (the allocation is 0.5 for both arms). This tests num_winning_arms == num_arms > 1.""" for epsilon in self.epsilons_to_test: bandit = self.bandit_class(self.two_new_arms_test_case, epsilon) T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.5, "arm2": 0.5})
def test_three_arms_diferent_variance(self): """Check that the three-arms cases with different variance (same average payoff) return the expected arm allocations. The highest variance wins.""" bandit = self.bandit_class(self.three_arms_with_variance_no_unsampled_arm_test_case) T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 1.0, "arm2": 0.0, "arm3": 0.0})
def test_historical_data_append_unsampled_arm(self): """Test that adding an unsampled arm (already exists in historical info) to HistoricalData does not change anything.""" historical_info = self.two_unsampled_arms_test_case historical_info.append_sample_arms(self.one_arm_test_case.arms_sampled) T.assert_dicts_equal(historical_info.json_payload(), self.two_unsampled_arms_test_case.json_payload())