def test_meta_estimation_format(synthetic_bandit_feedback: BanditFeedback,
random_action_dist: np.ndarray) -> None:
"""
Test the response format of OffPolicyEvaluation
"""
# single ope estimator
ope_ = OffPolicyEvaluation(bandit_feedback=synthetic_bandit_feedback,
ope_estimators=[dm])
assert ope_.estimate_policy_values(random_action_dist) == {
"dm": mock_policy_value
}, "OffPolicyEvaluation.estimate_policy_values ([DirectMethod]) returns a wrong value"
assert ope_.estimate_intervals(random_action_dist) == {
"dm": mock_confidence_interval
}, "OffPolicyEvaluation.estimate_intervals ([DirectMethod]) returns a wrong value"
with pytest.raises(AssertionError,
match=r"action_dist must be 3-dimensional.*"):
ope_.estimate_policy_values(
random_action_dist[:, :, 0]
), "action_dist must be 3-dimensional when using OffPolicyEvaluation"
# multiple ope estimators
ope_ = OffPolicyEvaluation(bandit_feedback=synthetic_bandit_feedback,
ope_estimators=[dm, ipw])
assert ope_.estimate_policy_values(random_action_dist) == {
"dm": mock_policy_value,
"ipw": mock_policy_value + ipw.eps,
}, "OffPolicyEvaluation.estimate_policy_values ([DirectMethod, IPW]) returns a wrong value"
assert ope_.estimate_intervals(random_action_dist) == {
"dm": mock_confidence_interval,
"ipw": {k: v + ipw.eps
for k, v in mock_confidence_interval.items()},
}, "OffPolicyEvaluation.estimate_intervals ([DirectMethod]) returns a wrong value"
def test_meta_estimate_policy_values_using_valid_input_data(
action_dist,
estimated_rewards_by_reg_model,
description: str,
synthetic_bandit_feedback: BanditFeedback,
) -> None:
"""
Test the response of estimate_policy_values using valid data
"""
# single ope estimator
ope_ = OffPolicyEvaluation(bandit_feedback=synthetic_bandit_feedback,
ope_estimators=[dm])
assert ope_.estimate_policy_values(
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
) == {
"dm": mock_policy_value
}, "OffPolicyEvaluation.estimate_policy_values ([DirectMethod]) returns a wrong value"
# multiple ope estimators
ope_ = OffPolicyEvaluation(bandit_feedback=synthetic_bandit_feedback,
ope_estimators=[dm, ipw])
assert ope_.estimate_policy_values(
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
) == {
"dm": mock_policy_value,
"ipw": mock_policy_value + ipw.eps,
}, "OffPolicyEvaluation.estimate_policy_values ([DirectMethod, IPW]) returns a wrong value"
def test_meta_create_estimator_inputs_using_valid_input_data(
action_dist,
estimated_rewards_by_reg_model,
description: str,
synthetic_bandit_feedback: BanditFeedback,
) -> None:
"""
Test the _create_estimator_inputs using invalid data
"""
ope_ = OffPolicyEvaluation(
bandit_feedback=synthetic_bandit_feedback, ope_estimators=[ipw]
)
estimator_inputs = ope_._create_estimator_inputs(
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
)
assert set(estimator_inputs.keys()) == set(["ipw"])
assert set(estimator_inputs["ipw"].keys()) == set(
[
"reward",
"action",
"pscore",
"position",
"action_dist",
"estimated_rewards_by_reg_model",
"estimated_pscore",
"estimated_importance_weights",
"p_e_a",
"pi_b",
"context",
"action_embed",
]
), f"Invalid response of _create_estimator_inputs (test case: {description})"
# _create_estimator_inputs function is called in the following functions
_ = ope_.estimate_policy_values(
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
)
_ = ope_.estimate_intervals(
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
)
_ = ope_.summarize_off_policy_estimates(
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
)
_ = ope_.evaluate_performance_of_estimators(
ground_truth_policy_value=0.1,
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
)
_ = ope_.summarize_estimators_comparison(
ground_truth_policy_value=0.1,
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
)
def test_meta_estimated_rewards_by_reg_model_inputs(
synthetic_bandit_feedback: BanditFeedback,
) -> None:
"""
Test the estimate_policy_values/estimate_intervals functions wrt estimated_rewards_by_reg_model
"""
ope_ = OffPolicyEvaluation(
bandit_feedback=synthetic_bandit_feedback, ope_estimators=[DirectMethod()]
)
action_dist = np.zeros(
(synthetic_bandit_feedback["n_rounds"], synthetic_bandit_feedback["n_actions"])
)
with pytest.raises(ValueError):
ope_.estimate_policy_values(
action_dist=action_dist,
estimated_rewards_by_reg_model=None,
)
with pytest.raises(ValueError):
ope_.estimate_intervals(
action_dist=action_dist,
estimated_rewards_by_reg_model=None,
)
def test_response_format_of_ope_estimators_using_random_evaluation_policy(
synthetic_bandit_feedback: BanditFeedback,
random_action_dist: np.ndarray) -> None:
"""
Test the response format of ope estimators using synthetic bandit data and random evaluation policy
"""
expected_reward = synthetic_bandit_feedback["expected_reward"][:, :,
np.newaxis]
action_dist = random_action_dist
# test all estimators
all_estimators = ope.__all_estimators__
estimators = [
getattr(ope.estimators, estimator_name)()
for estimator_name in all_estimators
]
# conduct OPE
ope_instance = OffPolicyEvaluation(
bandit_feedback=synthetic_bandit_feedback, ope_estimators=estimators)
estimated_policy_value = ope_instance.estimate_policy_values(
action_dist=action_dist,
estimated_rewards_by_reg_model=expected_reward)
estimated_intervals = ope_instance.estimate_intervals(
action_dist=action_dist,
estimated_rewards_by_reg_model=expected_reward,
random_state=12345,
)
# check the format of OPE
for key in estimated_policy_value:
# check the keys of the output dictionary of the estimate_intervals method
assert set(estimated_intervals[key].keys()) == set([
"mean", "95.0% CI (lower)", "95.0% CI (upper)"
]), f"Confidence interval of {key} has invalid keys"
# check the relationship between the means and the confidence bounds estimated by OPE estimators
assert (
estimated_intervals[key]["95.0% CI (lower)"] <=
estimated_policy_value[key]
) and (
estimated_intervals[key]["95.0% CI (upper)"] >=
estimated_policy_value[key]
), f"Estimated policy value of {key} is not included in estimated intervals of that estimator"
assert (estimated_intervals[key]["mean"] >=
estimated_intervals[key]["95.0% CI (lower)"]
), f"Invalid confidence interval of {key}: lower bound > mean"
assert (estimated_intervals[key]["mean"] <=
estimated_intervals[key]["95.0% CI (upper)"]
), f"Invalid confidence interval of {key}: upper bound < mean"
def test_meta_create_estimator_inputs_using_invalid_input_data(
action_dist,
estimated_rewards_by_reg_model,
description: str,
synthetic_bandit_feedback: BanditFeedback,
) -> None:
"""
Test the _create_estimator_inputs using valid data
"""
ope_ = OffPolicyEvaluation(
bandit_feedback=synthetic_bandit_feedback, ope_estimators=[ipw]
)
# raise ValueError when the shape of two arrays are different
with pytest.raises(ValueError, match=f"{description}*"):
_ = ope_._create_estimator_inputs(
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
)
# _create_estimator_inputs function is called in the following functions
with pytest.raises(ValueError, match=f"{description}*"):
_ = ope_.estimate_policy_values(
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
)
with pytest.raises(ValueError, match=f"{description}*"):
_ = ope_.estimate_intervals(
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
)
with pytest.raises(ValueError, match=f"{description}*"):
_ = ope_.summarize_off_policy_estimates(
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
)
with pytest.raises(ValueError, match=f"{description}*"):
_ = ope_.evaluate_performance_of_estimators(
ground_truth_policy_value=0.1,
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
)
with pytest.raises(ValueError, match=f"{description}*"):
_ = ope_.summarize_estimators_comparison(
ground_truth_policy_value=0.1,
action_dist=action_dist,
estimated_rewards_by_reg_model=estimated_rewards_by_reg_model,
)
def test_performance_of_ope_estimators_using_random_evaluation_policy(
synthetic_bandit_feedback: BanditFeedback,
random_action_dist: np.ndarray) -> None:
"""
Test the performance of ope estimators using synthetic bandit data and random evaluation policy
"""
expected_reward = synthetic_bandit_feedback["expected_reward"][:, :,
np.newaxis]
action_dist = random_action_dist
# compute ground truth policy value using expected reward
q_pi_e = np.average(expected_reward[:, :, 0],
weights=action_dist[:, :, 0],
axis=1)
# compute statistics of ground truth policy value
gt_mean = q_pi_e.mean()
gt_std = q_pi_e.std(ddof=1)
# test most of the estimators (ReplayMethod is not tested because it is out of scope)
all_estimators = ope.__all_estimators__
estimators = [
getattr(ope.estimators, estimator_name)()
for estimator_name in all_estimators
if estimator_name not in ["ReplayMethod"]
]
# conduct OPE
ope_instance = OffPolicyEvaluation(
bandit_feedback=synthetic_bandit_feedback, ope_estimators=estimators)
estimated_policy_value = ope_instance.estimate_policy_values(
action_dist=action_dist,
estimated_rewards_by_reg_model=expected_reward)
# check the performance of OPE
ci_bound = gt_std * 3 / np.sqrt(q_pi_e.shape[0])
print(f"gt_mean: {gt_mean}, 3 * gt_std / sqrt(n): {ci_bound}")
for key in estimated_policy_value:
print(
f"estimated_value: {estimated_policy_value[key]} ------ estimator: {key}, "
)
# test the performance of each estimator
assert (
np.abs(gt_mean - estimated_policy_value[key]) <= ci_bound
), f"OPE of {key} did not work well (absolute error is greater than 3*sigma)"
def test_response_format_of_ope_estimators_using_random_evaluation_policy(
synthetic_bandit_feedback: BanditFeedback, random_action_dist: np.ndarray
) -> None:
"""
Test the response format of ope estimators using synthetic bandit data and random evaluation policy
"""
expected_reward = synthetic_bandit_feedback["expected_reward"][:, :, np.newaxis]
action_dist = random_action_dist
# test all estimators
all_estimators = ope.__all_estimators__
estimators_standard = [
getattr(ope.estimators, estimator_name)() for estimator_name in all_estimators
]
all_estimators_tuning = ope.__all_estimators_tuning__
estimators_tuning = [
getattr(ope.estimators_tuning, estimator_name)(
lambdas=[1, 100, 10000, np.inf],
tuning_method=tuning_method,
)
for estimator_name in all_estimators_tuning
for tuning_method in ["slope", "mse"]
]
all_estimators_tuning_sg = ope.__all_estimators_tuning_sg__
estimators_tuning_sg = [
getattr(ope.estimators_tuning, estimator_name)(
lambdas=[0.001, 0.01, 0.1, 1.0],
tuning_method=tuning_method,
)
for estimator_name in all_estimators_tuning_sg
for tuning_method in ["slope", "mse"]
]
estimators = estimators_standard + estimators_tuning + estimators_tuning_sg
# skip estimation
estimated_pscore = None
estimated_importance_weights = (
random_action_dist[
np.arange(synthetic_bandit_feedback["action"].shape[0]),
synthetic_bandit_feedback["action"],
np.zeros(
synthetic_bandit_feedback["action"].shape[0], dtype=int
), # position is None
]
/ synthetic_bandit_feedback["pscore"]
)
# conduct OPE
ope_instance = OffPolicyEvaluation(
bandit_feedback=synthetic_bandit_feedback, ope_estimators=estimators
)
estimated_policy_value = ope_instance.estimate_policy_values(
action_dist=action_dist,
estimated_rewards_by_reg_model=expected_reward,
estimated_pscore=estimated_pscore,
estimated_importance_weights=estimated_importance_weights,
)
estimated_intervals = ope_instance.estimate_intervals(
action_dist=action_dist,
estimated_rewards_by_reg_model=expected_reward,
estimated_pscore=estimated_pscore,
estimated_importance_weights=estimated_importance_weights,
random_state=12345,
)
# check the format of OPE
for key in estimated_policy_value:
# check the keys of the output dictionary of the estimate_intervals method
assert set(estimated_intervals[key].keys()) == set(
["mean", "95.0% CI (lower)", "95.0% CI (upper)"]
), f"Confidence interval of {key} has invalid keys"
# check the relationship between the means and the confidence bounds estimated by OPE estimators
assert (
estimated_intervals[key]["95.0% CI (lower)"] <= estimated_policy_value[key]
) and (
estimated_intervals[key]["95.0% CI (upper)"] >= estimated_policy_value[key]
), f"Estimated policy value of {key} is not included in estimated intervals of that estimator"
assert (
estimated_intervals[key]["mean"]
>= estimated_intervals[key]["95.0% CI (lower)"]
), f"Invalid confidence interval of {key}: lower bound > mean"
assert (
estimated_intervals[key]["mean"]
<= estimated_intervals[key]["95.0% CI (upper)"]
), f"Invalid confidence interval of {key}: upper bound < mean"
def test_performance_of_ope_estimators_using_random_evaluation_policy(
synthetic_bandit_feedback: BanditFeedback, random_action_dist: np.ndarray
) -> None:
"""
Test the performance of ope estimators using synthetic bandit data and random evaluation policy
"""
expected_reward = synthetic_bandit_feedback["expected_reward"][:, :, np.newaxis]
action_dist = random_action_dist
# compute ground truth policy value using expected reward
q_pi_e = np.average(expected_reward[:, :, 0], weights=action_dist[:, :, 0], axis=1)
# compute statistics of ground truth policy value
gt_mean = q_pi_e.mean()
# test most of the estimators (ReplayMethod is not tested because it is out of scope)
all_estimators = ope.__all_estimators__
estimators_standard = [
getattr(ope.estimators, estimator_name)()
for estimator_name in all_estimators
if estimator_name not in ["ReplayMethod"]
]
all_estimators_tuning = ope.__all_estimators_tuning__
estimators_tuning = [
getattr(ope.estimators_tuning, estimator_name)(
lambdas=[1, 100, 10000, np.inf],
tuning_method=tuning_method,
)
for estimator_name in all_estimators_tuning
for tuning_method in ["slope", "mse"]
]
all_estimators_tuning_sg = ope.__all_estimators_tuning_sg__
estimators_tuning_sg = [
getattr(ope.estimators_tuning, estimator_name)(
lambdas=[0.001, 0.01, 0.1, 1.0],
tuning_method=tuning_method,
)
for estimator_name in all_estimators_tuning_sg
for tuning_method in ["slope", "mse"]
]
estimators = estimators_standard + estimators_tuning + estimators_tuning_sg
# skip estimation
estimated_pscore = None
estimated_importance_weights = (
random_action_dist[
np.arange(synthetic_bandit_feedback["action"].shape[0]),
synthetic_bandit_feedback["action"],
np.zeros(
synthetic_bandit_feedback["action"].shape[0], dtype=int
), # position is None
]
/ synthetic_bandit_feedback["pscore"]
)
# conduct OPE
ope_instance = OffPolicyEvaluation(
bandit_feedback=synthetic_bandit_feedback, ope_estimators=estimators
)
estimated_policy_value = ope_instance.estimate_policy_values(
action_dist=action_dist,
estimated_rewards_by_reg_model=expected_reward,
estimated_pscore=estimated_pscore,
estimated_importance_weights=estimated_importance_weights,
)
# check the performance of OPE
print(f"gt_mean: {gt_mean}")
for key in estimated_policy_value:
print(
f"estimated_value: {estimated_policy_value[key]} ------ estimator: {key}, "
)
# test the performance of each estimator
assert (
np.abs(gt_mean - estimated_policy_value[key]) / gt_mean <= 0.1
), f"OPE of {key} did not work well (relative absolute error is greater than 10%)"