def test_make_bandit_historical_info_from_params_variance_passed_through(self):
"""Test that the variance of a given sample arm got passed through."""
historical_info = self.three_arms_with_variance_no_unsampled_arm_test_case
T.assert_dicts_equal(
_make_bandit_historical_info_from_params(self.make_params_from_bandit_historical_info(historical_info)).json_payload(),
historical_info.json_payload()
)
def test_list_path_with_non_leaf(self):
"""
Given
a path that contains a single non-leaf child node
When
path is listed
Then
return a single non-leaf node
"""
with patch_urlopen() as mock_urlopen:
mock_response = Mock()
mock_response.read.return_value = """
[
{
"leaf": 0,
"context": {},
"text" : "servers",
"expandable": 1,
"id": "servers",
"allowChildren": 1
}
]
"""
mock_urlopen.return_value = mock_response
result = self.ds.list_path(path=['a','b','c'])
T.assert_equal(1, len(result))
expected_servers_dict = {'type': 'dir', 'name': u'servers', 'children': None}
T.assert_dicts_equal(expected_servers_dict, result[0])
mock_urlopen.assert_called_with(match(starts_with('http://dontcare.com:8080/metrics/find')))
mock_urlopen.assert_called_with(match(contains_string('query=a.b.c.*')))
def test_hyperparameters_passed_through(self):
"""Test that the hyperparameters get passed through to the endpoint."""
test_case = self.gp_test_environments[0]
python_domain, python_gp = test_case
python_cov, historical_data = python_gp.get_core_data_copy()
# Test default test parameters get passed through
json_payload = json.loads(self._build_json_payload(python_domain, python_cov, historical_data))
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = GpHyperOptView(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['hyperparameter_domain_info'], json_payload['hyperparameter_domain_info'])
# Test arbitrary parameters get passed through
json_payload['hyperparameter_domain_info']['domain_bounds'] = []
for i in range(json_payload['hyperparameter_domain_info']['dim']):
json_payload['hyperparameter_domain_info']['domain_bounds'].append({
'min': 0.2 * i,
'max': 0.5 * i,
})
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = GpHyperOptView(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['hyperparameter_domain_info'], json_payload['hyperparameter_domain_info'])
def test_two_arms_epsilon_zero(self):
"""Check that the two-arms case with zero epsilon always allocate arm1:1.0 and arm2:0.0 when average payoffs are arm1:1.0 and arm2:0.0."""
epsilon = 0.0
bandit = self.bandit_class(self.two_arms_test_case, epsilon)
arms_to_allocations = bandit.allocate_arms()
T.assert_dicts_equal(arms_to_allocations, {"arm1": 1.0, "arm2": 0.0})
T.assert_equal(bandit.choose_arm(arms_to_allocations), "arm1")
def test_epsilon_greedy_hyperparameters_passed_through(self):
"""Test that the hyperparameters get passed through to the epsilon-greedy endpoint."""
historical_info = self.one_arm_test_case
# Test default test parameters get passed through
json_payload = json.loads(
self._build_json_payload(
EPSILON_SUBTYPE_GREEDY, historical_info,
EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[
EPSILON_SUBTYPE_GREEDY]))
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = BanditEpsilonView(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['hyperparameter_info'],
json_payload['hyperparameter_info'])
# Test arbitrary epsilons get passed through
json_payload['hyperparameter_info']['epsilon'] = 1.0
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = BanditEpsilonView(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['hyperparameter_info'],
json_payload['hyperparameter_info'])
def test_two_arms_epsilon_zero(self):
"""Check that the two-arms case with zero epsilon (always exploit) always allocate arm1:1.0 and arm2:0.0 when average payoffs are arm1:1.0 and arm2:0.0."""
epsilon = 0.0
bandit = self.bandit_class(self.two_arms_test_case, epsilon)
arms_to_allocations = bandit.allocate_arms()
T.assert_dicts_equal(arms_to_allocations, {"arm1": 1.0, "arm2": 0.0})
T.assert_equal(bandit.choose_arm(arms_to_allocations), "arm1")
def test_epsilon_first_hyperparameters_passed_through(self):
"""Test that the hyperparameters get passed through to the epsilon-first endpoint."""
historical_info = self.one_arm_test_case
# Test default test parameters get passed through
json_payload = json.loads(
self._build_json_payload(
EPSILON_SUBTYPE_FIRST, historical_info,
EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[
EPSILON_SUBTYPE_FIRST]))
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = self._view(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['hyperparameter_info'],
json_payload['hyperparameter_info'])
# Test an arbitrary epsilon and total_tamples get passed through
json_payload['hyperparameter_info']['epsilon'] = 1.0
json_payload['hyperparameter_info']['total_samples'] = 20000
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = self._view(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['hyperparameter_info'],
json_payload['hyperparameter_info'])
def test_one_arm(self):
"""Check that the one-arm case always returns the given arm as the winning arm and the allocation is 1.0."""
for epsilon in self.epsilons_to_test:
for total_samples in self.total_samples_to_test:
bandit = self.bandit_class(self.one_arm_test_case, epsilon, total_samples)
T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 1.0})
T.assert_equal(bandit.choose_arm(), "arm1")
def test_three_arms_exploit(self):
"""Check that the three-arms cases with integer and float payoffs in exploitation phase return the expected arm allocations."""
epsilon = 0.5
total_samples = 10
for historical_info in [self.three_arms_test_case, self.three_arms_float_payoffs_test_case]:
bandit = self.bandit_class(historical_info, epsilon, total_samples)
T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 1.0, "arm2": 0.0, "arm3": 0.0})
def test_list_path_with_non_leaf(self):
"""
Given
a path that contains a single non-leaf child node
When
path is listed
Then
return a single non-leaf node
"""
with patch_urlopen() as mock_urlopen:
mock_response = Mock()
mock_response.read.return_value = """
[
{
"leaf": 0,
"context": {},
"text" : "servers",
"expandable": 1,
"id": "servers",
"allowChildren": 1
}
]
"""
mock_urlopen.return_value = mock_response
result = self.ds.list_path(path=["a", "b", "c"])
T.assert_equal(1, len(result))
expected_servers_dict = {"type": "dir", "name": u"servers", "children": None}
T.assert_dicts_equal(expected_servers_dict, result[0])
mock_urlopen.assert_called_with(match(starts_with("http://dontcare.com:8080/metrics/find")))
mock_urlopen.assert_called_with(match(contains_string("query=a.b.c.*")))
def test_get_equal_arm_allocations_no_winner(self):
"""Test allocations split among all sample arms when there is no winner."""
T.assert_dicts_equal(
get_equal_arm_allocations(
self.two_unsampled_arms_test_case.arms_sampled), {
"arm1": 0.5,
"arm2": 0.5
})
def test_historical_data_append_unsampled_arm(self):
"""Test that adding an unsampled arm (already exists in historical info) to HistoricalData does not change anything."""
historical_info = self.two_unsampled_arms_test_case
historical_info.append_sample_arms(self.one_arm_test_case.arms_sampled)
T.assert_dicts_equal(
historical_info.json_payload(),
self.two_unsampled_arms_test_case.json_payload()
)
def test_two_arms_epsilon_one(self):
"""Check that the two-arms case with one epsilon (always explore) always allocate arm1:0.5 and arm2:0.5 when average payoffs are arm1:1.0 and arm2:0.0."""
epsilon = 1.0
bandit = self.bandit_class(self.two_arms_test_case, epsilon)
T.assert_dicts_equal(bandit.allocate_arms(), {
"arm1": 0.5,
"arm2": 0.5
})
def test_make_bandit_historical_info_from_params_make_bernoulli_arms(self):
"""Test that the function can make historical infos with Bernoulli arms."""
historical_info = self.three_arms_with_variance_no_unsampled_arm_test_case
for historical_info in self.bernoulli_historical_infos_to_test:
T.assert_dicts_equal(
_make_bandit_historical_info_from_params(self.make_params_from_bandit_historical_info(historical_info), BernoulliArm).json_payload(),
historical_info.json_payload()
)
def test_make_bandit_historical_info_from_params_make_bernoulli_arms(self):
"""Test that the function can make historical infos with Bernoulli arms."""
historical_info = self.three_arms_with_variance_no_unsampled_arm_test_case
for historical_info in self.bernoulli_historical_infos_to_test:
T.assert_dicts_equal(
_make_bandit_historical_info_from_params(
self.make_params_from_bandit_historical_info(
historical_info), BernoulliArm).json_payload(),
historical_info.json_payload())
def test_make_bandit_historical_info_from_params_variance_passed_through(
self):
"""Test that the variance of a given sample arm got passed through."""
historical_info = self.three_arms_with_variance_no_unsampled_arm_test_case
T.assert_dicts_equal(
_make_bandit_historical_info_from_params(
self.make_params_from_bandit_historical_info(
historical_info)).json_payload(),
historical_info.json_payload())
def test_two_unsampled_arms(self):
"""Check that the two-unsampled-arms case always allocate each arm equally (the allocation is 0.5 for both arms). This tests num_winning_arms == num_arms > 1."""
for epsilon in self.epsilons_to_test:
bandit = self.bandit_class(self.two_unsampled_arms_test_case,
epsilon)
T.assert_dicts_equal(bandit.allocate_arms(), {
"arm1": 0.5,
"arm2": 0.5
})
def test_two_arms_one_winner(self):
"""Check that the two-arms case with random seed 0 always allocate arm1:1.0 and arm2:0.0."""
old_state = numpy.random.get_state()
numpy.random.seed(0)
bandit = self.bandit_class(self.two_arms_test_case, DEFAULT_BLA_SUBTYPE)
arms_to_allocations = bandit.allocate_arms()
T.assert_dicts_equal(arms_to_allocations, {"arm1": 1.0, "arm2": 0.0})
T.assert_equal(bandit.choose_arm(arms_to_allocations), "arm1")
numpy.random.set_state(old_state)
def test_get_equal_arm_allocations_one_winner(self):
"""Test all allocation given to the winning arm."""
T.assert_dicts_equal(
get_equal_arm_allocations(self.three_arms_test_case.arms_sampled,
frozenset(["arm1"])), {
"arm1": 1.0,
"arm2": 0.0,
"arm3": 0.0
})
def test_data_multiple_metrics(self):
sources = [
['servers', 'admin1', 'loadavg', '01'],
['servers', 'admin2', 'loadavg', '01'],
]
start = 1391047920
end = 1391048100
with patch_urlopen() as mock_urlopen:
mock_admin1_response = Mock()
mock_admin1_response.read.return_value = """
[
{
"target": "servers.admin1.loadavg.01",
"datapoints": [
[2.0, 1391047920],
[6.0, 1391047980],
[9.0, 1391048040],
[null,1391048100]
]
}
]
"""
mock_admin2_response = Mock()
mock_admin2_response.read.return_value = """
[
{
"target": "servers.admin2.loadavg.01",
"datapoints": [
[1.0, 1391047920],
[7.0, 1391047980],
[10.0, 1391048040],
[null,1391048100]
]
}
]
"""
mock_urlopen.side_effect = [mock_admin1_response, mock_admin2_response]
result_json = self.ds.data(sources, start, end, width=100)
# [null, 1391048100]
expected_results = [
{ 't': 1391047920, 'v': [2.0, 1.0]},
{ 't': 1391047980, 'v': [6.0, 7.0]},
{ 't': 1391048040, 'v': [9.0, 10.0]},
{ 't': 1391048100, 'v': [None, None]}
]
result_list = json.loads(result_json)
T.assert_equal(4, len(result_list))
for i,expected_result in enumerate(expected_results):
T.assert_dicts_equal(expected_result, result_list[i])
T.assert_equal(2, mock_urlopen.call_count)
def test_get_equal_arm_allocations_two_winners(self):
"""Test allocations split between two winning arms."""
T.assert_dicts_equal(
get_equal_arm_allocations(
self.three_arms_two_winners_test_case.arms_sampled,
frozenset(["arm1", "arm2"])), {
"arm1": 0.5,
"arm2": 0.5,
"arm3": 0.0
})
def test_three_arms_two_winners(self):
"""Check that the three-arms cases with two winners return the expected arm allocations. This tests num_arms > num_winning_arms > 1."""
epsilon = 0.03
bandit = self.bandit_class(self.three_arms_two_winners_test_case,
epsilon)
T.assert_dicts_equal(bandit.allocate_arms(), {
"arm1": 0.495,
"arm2": 0.495,
"arm3": 0.01
})
def test_two_arms_one_winner(self):
"""Check that the two-arms case with random seed 0 always allocate arm1:1.0 and arm2:0.0."""
old_state = numpy.random.get_state()
numpy.random.seed(0)
bandit = self.bandit_class(self.two_arms_test_case,
DEFAULT_BLA_SUBTYPE)
arms_to_allocations = bandit.allocate_arms()
T.assert_dicts_equal(arms_to_allocations, {"arm1": 1.0, "arm2": 0.0})
T.assert_equal(bandit.choose_arm(arms_to_allocations), "arm1")
numpy.random.set_state(old_state)
def test_three_arms_exploit_two_winners(self):
"""Check that the three-arms cases with two winners in exploitation phase return the expected arm allocations. This tests num_arms > num_winning_arms > 1."""
epsilon = 0.5
total_samples = 10
bandit = self.bandit_class(self.three_arms_two_winners_test_case,
epsilon, total_samples)
T.assert_dicts_equal(bandit.allocate_arms(), {
"arm1": 0.5,
"arm2": 0.5,
"arm3": 0.0
})
def test_data_multiple_metrics(self):
sources = [["servers", "admin1", "loadavg", "01"], ["servers", "admin2", "loadavg", "01"]]
start = 1391047920
end = 1391048100
with patch_urlopen() as mock_urlopen:
mock_admin1_response = Mock()
mock_admin1_response.read.return_value = """
[
{
"target": "servers.admin1.loadavg.01",
"datapoints": [
[2.0, 1391047920],
[6.0, 1391047980],
[9.0, 1391048040],
[null,1391048100]
]
}
]
"""
mock_admin2_response = Mock()
mock_admin2_response.read.return_value = """
[
{
"target": "servers.admin2.loadavg.01",
"datapoints": [
[1.0, 1391047920],
[7.0, 1391047980],
[10.0, 1391048040],
[null,1391048100]
]
}
]
"""
mock_urlopen.side_effect = [mock_admin1_response, mock_admin2_response]
result_json = self.ds.data(sources, start, end, width=100)
# [null, 1391048100]
expected_results = [
{"t": 1391047920, "v": [2.0, 1.0]},
{"t": 1391047980, "v": [6.0, 7.0]},
{"t": 1391048040, "v": [9.0, 10.0]},
{"t": 1391048100, "v": [None, None]},
]
result_list = json.loads(result_json)
T.assert_equal(4, len(result_list))
for i, expected_result in enumerate(expected_results):
T.assert_dicts_equal(expected_result, result_list[i])
T.assert_equal(2, mock_urlopen.call_count)
def test_three_arms(self):
"""Check that the three-arms cases with integer and float payoffs return the expected arm allocations."""
epsilon = 0.03
for historical_info in [
self.three_arms_test_case,
self.three_arms_float_payoffs_test_case
]:
bandit = self.bandit_class(historical_info, epsilon)
T.assert_dicts_equal(bandit.allocate_arms(), {
"arm1": 0.98,
"arm2": 0.01,
"arm3": 0.01
})
def _test_historical_info_passed_through(self):
"""Test that the historical infos get passed through to the endpoint."""
for subtype in BANDIT_ENDPOINTS_TO_SUBTYPES[self._endpoint]:
for historical_info in self._historical_infos:
# Test default test parameters get passed through
json_payload = json.loads(self._build_json_payload(subtype, historical_info))
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = self._view(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['historical_info'], json_payload['historical_info'])
def test_historical_info_passed_through(self):
"""Test that the historical info get passed through to the endpoint."""
for subtype in EPSILON_SUBTYPES:
for historical_info in self.historical_infos_to_test:
# Test default test parameters get passed through
json_payload = json.loads(self._build_json_payload(subtype, historical_info, EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[subtype]))
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = BanditEpsilonView(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['historical_info'], json_payload['historical_info'])
def test_no_hang(self, process_mock):
"""Test that nothing hangs if the processes are never used.
The mechanics of this test are tricky: We make it such that a process
pool is deleted, but this process pool has an overridden method that
tells our test class that it actually cleaned up workers.
"""
test_passes = {}
processes = vimap.pool.fork(worker_proc.init_args(init=i) for i in [1, 2, 3])
processes.finish_workers = lambda: test_passes.setdefault("result", True)
del processes # will happen if it falls out of scope
# gc.collect() -- doesn't seem necessary
T.assert_dicts_equal(test_passes, {"result": True})
def test_historical_data_append_arms(self):
"""Test that appending arms to HistoricalData updates historical info correctly."""
historical_info = copy.deepcopy(self.three_arms_test_case)
historical_info.append_sample_arms(
self.three_arms_two_winners_test_case.arms_sampled)
expected_historical_info = HistoricalData(
sample_arms={
"arm1": SampleArm(win=4, loss=2, total=6),
"arm2": SampleArm(win=3, loss=2, total=5),
"arm3": SampleArm(win=0, loss=0, total=0),
})
T.assert_dicts_equal(historical_info.json_payload(),
expected_historical_info.json_payload())
def test_three_arms_exploit(self):
"""Check that the three-arms cases with integer and float payoffs in exploitation phase return the expected arm allocations."""
epsilon = 0.5
total_samples = 10
for historical_info in [
self.three_arms_test_case,
self.three_arms_float_payoffs_test_case
]:
bandit = self.bandit_class(historical_info, epsilon, total_samples)
T.assert_dicts_equal(bandit.allocate_arms(), {
"arm1": 1.0,
"arm2": 0.0,
"arm3": 0.0
})
def test_historical_data_append_arms(self):
"""Test that appending arms to HistoricalData updates historical info correctly."""
historical_info = copy.deepcopy(self.three_arms_test_case)
historical_info.append_sample_arms(self.three_arms_two_winners_test_case.arms_sampled)
expected_historical_info = HistoricalData(
sample_arms={
"arm1": SampleArm(win=4, loss=2, total=6),
"arm2": SampleArm(win=3, loss=2, total=5),
"arm3": SampleArm(win=0, loss=0, total=0),
}
)
T.assert_dicts_equal(
historical_info.json_payload(),
expected_historical_info.json_payload()
)
def _test_historical_info_passed_through(self):
"""Test that the historical infos get passed through to the endpoint."""
for subtype in BANDIT_ENDPOINTS_TO_SUBTYPES[self._endpoint]:
for historical_info in self._historical_infos:
# Test default test parameters get passed through
json_payload = json.loads(
self._build_json_payload(subtype, historical_info))
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = self._view(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['historical_info'],
json_payload['historical_info'])
def test_no_hang(self, process_mock):
'''Test that nothing hangs if the processes are never used.
The mechanics of this test are tricky: We make it such that a process
pool is deleted, but this process pool has an overridden method that
tells our test class that it actually cleaned up workers.
'''
test_passes = {}
processes = vimap.pool.fork(
worker_proc.init_args(init=i) for i in [1, 2, 3])
processes.finish_workers = lambda: test_passes.setdefault(
'result', True)
del processes # will happen if it falls out of scope
# gc.collect() -- doesn't seem necessary
T.assert_dicts_equal(test_passes, {'result': True})
def test_data_single_metric(self):
sources = [['servers', 'admin1', 'loadavg', '01']]
start = 1391047920
end = 1391048100
with patch_urlopen() as mock_urlopen:
mock_response = Mock()
mock_response.read.return_value = """
[
{
"target": "servers.admin1.loadavg.01",
"datapoints": [
[2.0, 1391047920],
[6.0, 1391047980],
[9.0, 1391048040],
[null,1391048100]
]
}
]
"""
mock_urlopen.return_value = mock_response
result_json = self.ds.data(sources, start, end, width=100)
expected_results = [
{ 't': 1391047920, 'v': [2.0]},
{ 't': 1391047980, 'v': [6.0]},
{ 't': 1391048040, 'v': [9.0]},
{ 't': 1391048100, 'v': [None]}
]
result_list = json.loads(result_json)
T.assert_equal(4, len(result_list))
for i,expected_result in enumerate(expected_results):
T.assert_dicts_equal(expected_result, result_list[i])
mock_urlopen.assert_called_with(match(starts_with('http://dontcare.com:8080/render')))
mock_urlopen.assert_called_with(match(contains_string('.'.join(sources[0]))))
def test_data_single_metric(self):
sources = [["servers", "admin1", "loadavg", "01"]]
start = 1391047920
end = 1391048100
with patch_urlopen() as mock_urlopen:
mock_response = Mock()
mock_response.read.return_value = """
[
{
"target": "servers.admin1.loadavg.01",
"datapoints": [
[2.0, 1391047920],
[6.0, 1391047980],
[9.0, 1391048040],
[null,1391048100]
]
}
]
"""
mock_urlopen.return_value = mock_response
result_json = self.ds.data(sources, start, end, width=100)
expected_results = [
{"t": 1391047920, "v": [2.0]},
{"t": 1391047980, "v": [6.0]},
{"t": 1391048040, "v": [9.0]},
{"t": 1391048100, "v": [None]},
]
result_list = json.loads(result_json)
T.assert_equal(4, len(result_list))
for i, expected_result in enumerate(expected_results):
T.assert_dicts_equal(expected_result, result_list[i])
mock_urlopen.assert_called_with(match(starts_with("http://dontcare.com:8080/render")))
mock_urlopen.assert_called_with(match(contains_string(".".join(sources[0]))))
def test_epsilon_greedy_hyperparameters_passed_through(self):
"""Test that the hyperparameters get passed through to the epsilon-greedy endpoint."""
historical_info = self.one_arm_test_case
# Test default test parameters get passed through
json_payload = json.loads(self._build_json_payload(EPSILON_SUBTYPE_GREEDY, historical_info, EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[EPSILON_SUBTYPE_GREEDY]))
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = BanditEpsilonView(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['hyperparameter_info'], json_payload['hyperparameter_info'])
# Test arbitrary epsilons get passed through
json_payload['hyperparameter_info']['epsilon'] = 1.0
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = BanditEpsilonView(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['hyperparameter_info'], json_payload['hyperparameter_info'])
def test_hyperparameters_passed_through(self):
"""Test that the hyperparameters get passed through to the endpoint."""
test_case = self.gp_test_environments[0]
python_domain, python_gp = test_case
python_cov, historical_data = python_gp.get_core_data_copy()
# Test default test parameters get passed through
json_payload = json.loads(
self._build_json_payload(python_domain, python_cov,
historical_data))
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = GpHyperOptView(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['hyperparameter_domain_info'],
json_payload['hyperparameter_domain_info'])
# Test arbitrary parameters get passed through
json_payload['hyperparameter_domain_info']['domain_bounds'] = []
for i in range(json_payload['hyperparameter_domain_info']['dim']):
json_payload['hyperparameter_domain_info']['domain_bounds'].append(
{
'min': 0.2 * i,
'max': 0.5 * i,
})
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = GpHyperOptView(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['hyperparameter_domain_info'],
json_payload['hyperparameter_domain_info'])
def test_epsilon_first_hyperparameters_passed_through(self):
"""Test that the hyperparameters get passed through to the epsilon-first endpoint."""
historical_info = self.one_arm_test_case
# Test default test parameters get passed through
json_payload = json.loads(self._build_json_payload(EPSILON_SUBTYPE_FIRST, historical_info, EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[EPSILON_SUBTYPE_FIRST]))
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = self._view(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['hyperparameter_info'], json_payload['hyperparameter_info'])
# Test an arbitrary epsilon and total_tamples get passed through
json_payload['hyperparameter_info']['epsilon'] = 1.0
json_payload['hyperparameter_info']['total_samples'] = 20000
request = pyramid.testing.DummyRequest(post=json_payload)
request.json_body = json_payload
view = self._view(request)
params = view.get_params_from_request()
T.assert_dicts_equal(params['hyperparameter_info'], json_payload['hyperparameter_info'])
def _test_three_arms_two_winners(self):
"""Check that the three-arms cases with two winners return the expected arm allocations. This tests num_arms > num_winning_arms > 1."""
bandit = self.bandit_class(self.three_arms_two_winners_no_unsampled_arm_test_case)
T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.5, "arm2": 0.5, "arm3": 0.0})
def test_three_arms_two_winners(self):
"""Check that the three-arms cases with two winners return the expected arm allocations. This tests num_arms > num_winning_arms > 1."""
epsilon = 0.03
bandit = self.bandit_class(self.three_arms_two_winners_test_case, epsilon)
T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.495, "arm2": 0.495, "arm3": 0.01})
def test_three_arms(self):
"""Check that the three-arms cases with integer and float payoffs return the expected arm allocations."""
epsilon = 0.03
for historical_info in [self.three_arms_test_case, self.three_arms_float_payoffs_test_case]:
bandit = self.bandit_class(historical_info, epsilon)
T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.98, "arm2": 0.01, "arm3": 0.01})
def test_two_arms_epsilon_one(self):
"""Check that the two-arms case with one epsilon always allocate arm1:0.5 and arm2:0.5 when average payoffs are arm1:1.0 and arm2:0.0."""
epsilon = 1.0
bandit = self.bandit_class(self.two_arms_test_case, epsilon)
T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.5, "arm2": 0.5})
def test_three_arms_exploit_two_winners(self):
"""Check that the three-arms cases with two winners in exploitation phase return the expected arm allocations. This tests num_arms > num_winning_arms > 1."""
epsilon = 0.5
total_samples = 10
bandit = self.bandit_class(self.three_arms_two_winners_test_case, epsilon, total_samples)
T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.5, "arm2": 0.5, "arm3": 0.0})
def assert_push_info_attributes(self, ul_attributes, expected):
T.assert_dicts_equal(ul_attributes, expected)
def test_get_equal_arm_allocations_no_winner(self):
"""Test allocations split among all sample arms when there is no winner."""
T.assert_dicts_equal(
get_equal_arm_allocations(self.two_unsampled_arms_test_case.arms_sampled), {"arm1": 0.5, "arm2": 0.5}
)
def test_get_equal_arm_allocations_one_winner(self):
"""Test all allocation given to the winning arm."""
T.assert_dicts_equal(
get_equal_arm_allocations(self.three_arms_test_case.arms_sampled, frozenset(["arm1"])),
{"arm1": 1.0, "arm2": 0.0, "arm3": 0.0},
)
def test_get_equal_arm_allocations_two_winners(self):
"""Test allocations split between two winning arms."""
T.assert_dicts_equal(
get_equal_arm_allocations(self.three_arms_two_winners_test_case.arms_sampled, frozenset(["arm1", "arm2"])),
{"arm1": 0.5, "arm2": 0.5, "arm3": 0.0},
)
def _test_one_arm(self, bandit):
"""Check that the one-arm case always returns the given arm as the winning arm and the allocation is 1.0."""
bandit = self.bandit_class(self.one_arm_test_case)
arms_to_allocations = bandit.allocate_arms()
T.assert_dicts_equal(arms_to_allocations, {"arm1": 1.0})
T.assert_equal(bandit.choose_arm(arms_to_allocations), "arm1")
def _test_three_arms_one_unsampled_arm(self):
"""Check that the three-arms cases with integer and float payoffs return the expected arm allocations. When arm3 is the only unsampled arm, we expect all allocation is given to arm3."""
for historical_info in [self.three_arms_test_case, self.three_arms_float_payoffs_test_case, self.three_arms_two_winners_test_case]:
bandit = self.bandit_class(historical_info)
T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.0, "arm2": 0.0, "arm3": 1.0})
def test_two_new_arms(self):
"""Check that the two-new-arms case always allocate each arm equally (the allocation is 0.5 for both arms). This tests num_winning_arms == num_arms > 1."""
for epsilon in self.epsilons_to_test:
bandit = self.bandit_class(self.two_new_arms_test_case, epsilon)
T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.5, "arm2": 0.5})
def test_three_arms_diferent_variance(self):
"""Check that the three-arms cases with different variance (same average payoff) return the expected arm allocations. The highest variance wins."""
bandit = self.bandit_class(self.three_arms_with_variance_no_unsampled_arm_test_case)
T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 1.0, "arm2": 0.0, "arm3": 0.0})
def test_historical_data_append_unsampled_arm(self):
"""Test that adding an unsampled arm (already exists in historical info) to HistoricalData does not change anything."""
historical_info = self.two_unsampled_arms_test_case
historical_info.append_sample_arms(self.one_arm_test_case.arms_sampled)
T.assert_dicts_equal(historical_info.json_payload(),
self.two_unsampled_arms_test_case.json_payload())
