def test_countable_outcmoes_user_can_elicit_bisection(self, neg, user):
strategy = EStrategy(strategy="bisection", resolution=1e-4)
strategy.on_enter(neg.ami)
elicited = []
estimated = []
total_cost = 0.0
while True:
e = strategy.utility_estimate((0,))
# assert user.total_cost <= total_cost + cost, 'incorrect total cost'
u, reply = strategy.apply(user=user, outcome=(0,))
if isinstance(u, float) or u.scale < cost:
assert abs(u - utility) < 1e-2
break
total_cost += cost
elicited.append(u)
estimated.append(e)
assert elicited[0].loc == 0.0, "first loc is incorrect in elicitation"
assert elicited[0].scale == 0.5, "first scale is incorrect in elicitation"
assert estimated[0].loc == 0.0, "first loc is incorrect in estimation"
assert estimated[0].scale == 1.0, "first scale is incorrect in estimation"
for u, e in zip(elicited, estimated):
assert u.scale == 0.5 * e.scale, "uncertainty is not decreasing as expected"
for i in range(len(elicited) - 1):
assert (
elicited[i + 1].scale == 0.5 * elicited[i].scale
), "uncertainty is not decreasing"
for i in range(len(estimated) - 1):
assert (
estimated[i + 1].scale == 0.5 * estimated[i].scale
), "uncertainty is not decreasing"
def test_countable_outcmoes_user_can_elicit_dtitration_down(self, neg, user):
step = -0.05
strategy = EStrategy(strategy=f"dtitration{step}", resolution=1e-4)
strategy.on_enter(neg.ami)
elicited = []
estimated = []
total_cost = 0.0
while True:
e = strategy.utility_estimate((0,))
# assert user.total_cost <= total_cost + cost, 'incorrect total cost'
u, reply = strategy.apply(user=user, outcome=(0,))
if isinstance(u, float) or u.scale < cost:
assert abs(u - utility) < -step * 2
break
total_cost += cost
elicited.append(u)
estimated.append(e)
step = -step
assert (
elicited[0].loc + elicited[0].scale == 1.0 - step
), "first loc is incorrect in elicitation"
assert (
elicited[0].scale == 1.0 - step
), "first scale is incorrect in elicitation"
assert estimated[0].loc == 0.0, "first loc is incorrect in estimation"
assert estimated[0].scale == 1.0, "first scale is incorrect in estimation"
def test_countable_outcmoes_user_can_elicit_titration_up(self, neg, user):
step = 0.05
strategy = EStrategy(strategy=f"titration+{step}", resolution=1e-4)
strategy.on_enter(neg.ami)
elicited = []
estimated = []
total_cost = 0.0
while True:
e = strategy.utility_estimate((0, ))
assert user.total_cost == total_cost, "incorrect total cost"
u, reply = strategy.apply(user=user, outcome=(0, ))
if isinstance(u, float) or u.scale < cost:
assert abs(u - utility) < step * 2
break
total_cost += cost
elicited.append(u)
estimated.append(e)
assert elicited[0].loc == step, "first loc is incorrect in elicitation"
assert (elicited[0].scale == 1.0 -
step), "first scale is incorrect in elicitation"
assert estimated[0].loc == 0.0, "first loc is incorrect in estimation"
assert estimated[
0].scale == 1.0, "first scale is incorrect in estimation"
for u, e in zip(elicited[:-1], estimated[:-1]):
assert (abs(u.scale - e.scale + step) <
1e-3), "uncertainty is not decreasing as expected"
for i in range(len(elicited) - 2):
assert (abs(elicited[i + 1].scale - elicited[i].scale + step) <
1e-3), "uncertainty is not decreasing"
for i in range(len(estimated) - 2):
assert (estimated[i + 1].scale - estimated[i].scale +
step) < 1e-3, "uncertainty is not decreasing"
def test_countable_outcmoes_user_can_elicit_titration_up_no_step(self, neg, user):
step = 0.01
strategy = EStrategy(strategy=f"titration", resolution=step)
strategy.on_enter(neg.ami)
elicited = []
estimated = []
total_cost = 0.0
for _ in range(int(0.5 + 1.0 / step) + 2):
e = strategy.utility_estimate((0,))
# assert user.total_cost <= total_cost + cost, 'incorrect total cost'
u, reply = strategy.apply(user=user, outcome=(0,))
if isinstance(u, float) or u.scale < cost:
u = float(u)
assert abs(u - utility) < 2 * step
break
total_cost += cost
elicited.append(u)
estimated.append(e)
else:
# print(elicited)
assert False, "did not end in expected time"
assert elicited[0].loc == step, "first loc is incorrect in elicitation"
assert (
elicited[0].scale == 1.0 - step
), "first scale is incorrect in elicitation"
assert estimated[0].loc == 0.0, "first loc is incorrect in estimation"
assert estimated[0].scale == 1.0, "first scale is incorrect in estimation"
for u, e in zip(elicited[:-1], estimated[:-1]):
assert (
abs(u.scale - e.scale + step) < 1e-3
), "uncertainty is not decreasing as expected"
for i in range(len(elicited) - 2):
assert (
abs(elicited[i + 1].scale - elicited[i].scale + step) < 1e-3
), "uncertainty is not decreasing"
for i in range(len(estimated) - 2):
assert (
estimated[i + 1].scale - estimated[i].scale + step
) < 1e-3, "uncertainty is not decreasing"
