def _test_interface_returns_as_expected(self):
"""Integration test for the bandit endpoints."""
for subtype in BANDIT_ENDPOINTS_TO_SUBTYPES[self._endpoint]:
for historical_info in self._historical_infos:
json_payload = self._build_json_payload(
subtype, historical_info)
arm_names = set([
arm_name
for arm_name in historical_info.arms_sampled.iterkeys()
])
resp = self.testapp.post(self._moe_route.endpoint,
json_payload)
resp_schema = BanditResponse()
resp_dict = resp_schema.deserialize(json.loads(resp.body))
resp_arm_names = set([
arm_name
for arm_name in resp_dict['arm_allocations'].iterkeys()
])
T.assert_sets_equal(arm_names, resp_arm_names)
# The allocations should be in range [0, 1]
# The sum of all allocations should be 1.0.
total_allocation = 0
for allocation in resp_dict['arm_allocations'].itervalues():
T.assert_gte(allocation, 0)
T.assert_lte(allocation, 1)
total_allocation += allocation
T.assert_equal(total_allocation, 1.0)
def test_get_winning_arm_names_from_payoff_arm_name_list_two_winners(self):
"""Test winning arm names match the winners."""
T.assert_sets_equal(
get_winning_arm_names_from_payoff_arm_name_list([(0.5, "arm1"),
(0.5, "arm2"),
(0.4, "arm3")]),
frozenset(["arm1", "arm2"]))
def test_fuzz(self):
for n in xrange(1, 100):
processes = self.fork_pool(
worker_proc.init_args(init=i) for i in [1, 1, 1])
res = list(processes.imap(list(range(1, n))).zip_in_out())
T.assert_sets_equal(set(out for in_, out in res),
set(range(2, n + 1)))
def test_fuzz(self):
for n in xrange(1, 100):
processes = self.fork_pool(
worker_proc.init_args(init=i) for i in [1, 1, 1])
res = list(processes.imap(list(range(1, n))).zip_in_out())
T.assert_sets_equal(set(out for in_, out in res),
set(range(2, n + 1)))
def test_zip_in_out(self):
processes = vimap.pool.fork(worker_proc.init_args(init=i) for i in [1, 2, 3])
results = list(processes.imap([4, 4, 4]).zip_in_out(close_if_done=False))
T.assert_sets_equal(set(results), set([(4, 5), (4, 6), (4, 7)]))
results = list(processes.imap([4, 4, 4]).zip_in_out(close_if_done=True))
T.assert_sets_equal(set(results), set([(4, 5), (4, 6), (4, 7)]))
def test_zip_in_out(self):
processes = vimap.pool.fork(worker_proc.init_args(init=i) for i in [1, 2, 3])
results = list(processes.imap([4, 4, 4]).zip_in_out(close_if_done=False))
T.assert_sets_equal(set(results), set([(4, 5), (4, 6), (4, 7)]))
results = list(processes.imap([4, 4, 4]).zip_in_out(close_if_done=True))
T.assert_sets_equal(set(results), set([(4, 5), (4, 6), (4, 7)]))
def assert_position_sets_equal(left, right):
left = map(Position.make, left)
right = map(Position.make, right)
if not isinstance(left, set):
left = set(left)
if not isinstance(right, set):
right = set(right)
T.assert_sets_equal(left, right)
def _test_interface_returns_as_expected(self):
"""Integration test for the bandit endpoints."""
for subtype in BANDIT_ENDPOINTS_TO_SUBTYPES[self._endpoint]:
for historical_info in self._historical_infos:
json_payload = self._build_json_payload(subtype, historical_info)
arm_names = set([arm_name for arm_name in historical_info.arms_sampled.iterkeys()])
resp = self.testapp.post(self._moe_route.endpoint, json_payload)
resp_schema = BanditResponse()
resp_dict = resp_schema.deserialize(json.loads(resp.body))
resp_arm_names = set([arm_name for arm_name in resp_dict['arm_allocations'].iterkeys()])
T.assert_sets_equal(arm_names, resp_arm_names)
# The allocations should be in range [0, 1]
# The sum of all allocations should be 1.0.
total_allocation = 0
for allocation in resp_dict['arm_allocations'].itervalues():
T.assert_gte(allocation, 0)
T.assert_lte(allocation, 1)
total_allocation += allocation
T.assert_equal(total_allocation, 1.0)
def test_interface_returns_as_expected(self):
"""Integration test for the /bandit/epsilon endpoint."""
moe_route = BANDIT_EPSILON_MOE_ROUTE
for subtype in EPSILON_SUBTYPES:
for historical_info in self.historical_infos_to_test:
json_payload = self._build_json_payload(subtype, historical_info, EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[subtype])
arm_names = set([arm_name for arm_name in historical_info.arms_sampled.iterkeys()])
resp = self.testapp.post(moe_route.endpoint, json_payload)
resp_schema = BanditEpsilonResponse()
resp_dict = resp_schema.deserialize(json.loads(resp.body))
resp_arm_names = set([arm_name for arm_name in resp_dict['arm_allocations'].iterkeys()])
T.assert_sets_equal(arm_names, resp_arm_names)
# The allocations should be in range [0, 1]
# The sum of all allocations should be 1.0.
total_allocation = 0
for allocation in resp_dict['arm_allocations'].itervalues():
T.assert_gte(allocation, 0)
T.assert_lte(allocation, 1)
total_allocation += allocation
T.assert_equal(total_allocation, 1.0)
def test_three_arms_two_winners(self):
"""Check that the three-arms cases with two winners return the expected winning arms. This tests num_arms > num_winning_arms > 1."""
T.assert_sets_equal(EpsilonInterface.get_winning_arm_names(self.three_arms_two_winners_test_case.arms_sampled), frozenset(["arm1", "arm2"]))
def test_two_unsampled_arms(self):
"""Check that the two-unsampled-arms case always returns both arms as winning arms. This tests num_winning_arms == num_arms > 1."""
T.assert_sets_equal(EpsilonInterface.get_winning_arm_names(self.two_unsampled_arms_test_case.arms_sampled), frozenset(["arm1", "arm2"]))
def test_get_winning_arm_names_from_payoff_arm_name_list_two_winners(self):
"""Test winning arm names match the winners."""
T.assert_sets_equal(
get_winning_arm_names_from_payoff_arm_name_list([(0.5, "arm1"), (0.5, "arm2"), (0.4, "arm3")]),
frozenset(["arm1", "arm2"]),
)
def test_get_winning_arm_names_from_payoff_arm_name_list_one_winner(self):
"""Test winning arm name matches the winner."""
T.assert_sets_equal(
get_winning_arm_names_from_payoff_arm_name_list([(0.5, "arm1"), (0.0, "arm2")]), frozenset(["arm1"])
)
def test_three_arms_two_winners(self):
"""Check that the three-arms cases with two winners return the expected winning arms. This tests num_arms > num_winning_arms > 1."""
T.assert_sets_equal(
EpsilonInterface.get_winning_arm_names(
self.three_arms_two_winners_test_case.arms_sampled),
frozenset(["arm1", "arm2"]))
def test_two_unsampled_arms(self):
"""Check that the two-unsampled-arms case always returns both arms as winning arms. This tests num_winning_arms == num_arms > 1."""
T.assert_sets_equal(
EpsilonInterface.get_winning_arm_names(
self.two_unsampled_arms_test_case.arms_sampled),
frozenset(["arm1", "arm2"]))
def test_get_winning_arm_names_from_payoff_arm_name_list_one_winner(self):
"""Test winning arm name matches the winner."""
T.assert_sets_equal(
get_winning_arm_names_from_payoff_arm_name_list([(0.5, "arm1"),
(0.0, "arm2")]),
frozenset(["arm1"]))
