def test_synthetic_slate_obtain_batch_bandit_feedback_using_uniform_random_behavior_policy_largescale( ): # set parameters n_unique_action = 100 len_list = 10 dim_context = 2 reward_type = "binary" random_state = 12345 n_rounds = 10000 dataset = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, random_state=random_state, ) # obtain feedback bandit_feedback = dataset.obtain_batch_bandit_feedback(n_rounds=n_rounds) # check slate bandit feedback (common test) check_slate_bandit_feedback(bandit_feedback=bandit_feedback) # check pscore marginal pscore_item_position = 1 / n_unique_action assert np.allclose( np.unique(bandit_feedback["pscore_item_position"]), pscore_item_position ), f"pscore_item_position must be [{pscore_item_position}], but {np.unique(bandit_feedback['pscore_item_position'])}"
def test_synthetic_slate_using_valid_inputs( n_unique_action, len_list, dim_context, reward_type, random_state, n_rounds, reward_structure, click_model, behavior_policy_function, reward_function, return_pscore_item_position, description, ): dataset = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, reward_structure=reward_structure, click_model=click_model, random_state=random_state, behavior_policy_function=behavior_policy_function, base_reward_function=reward_function, ) # obtain feedback bandit_feedback = dataset.obtain_batch_bandit_feedback( n_rounds=n_rounds, return_pscore_item_position=return_pscore_item_position) # check slate bandit feedback (common test) check_slate_bandit_feedback(bandit_feedback=bandit_feedback) pscore_columns = [ "pscore_cascade", "pscore", "pscore_item_position", ] bandit_feedback_df = pd.DataFrame() for column in [ "slate_id", "position", "action", "reward", "expected_reward_factual", ] + pscore_columns: bandit_feedback_df[column] = bandit_feedback[column] print(f"-------{description}--------") print(bandit_feedback_df.groupby("position")["reward"].describe()) if reward_type == "binary": assert set(np.unique(bandit_feedback["reward"])) == set([0, 1])
def test_synthetic_slate_obtain_batch_bandit_feedback_using_uniform_random_behavior_policy( ): # set parameters n_unique_action = 10 len_list = 3 dim_context = 2 reward_type = "binary" random_state = 12345 n_rounds = 100 dataset = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, random_state=random_state, ) # obtain feedback bandit_feedback = dataset.obtain_batch_bandit_feedback(n_rounds=n_rounds) # check slate bandit feedback (common test) check_slate_bandit_feedback(bandit_feedback=bandit_feedback) pscore_columns = [ "pscore_cascade", "pscore", "pscore_item_position", ] bandit_feedback_df = pd.DataFrame() for column in ["slate_id", "position", "action"] + pscore_columns: bandit_feedback_df[column] = bandit_feedback[column] # check pscore marginal pscore_item_position = 1 / n_unique_action assert np.allclose( bandit_feedback_df["pscore_item_position"].unique(), pscore_item_position ), f"pscore_item_position must be [{pscore_item_position}], but {bandit_feedback_df['pscore_item_position'].unique()}" # check pscore joint pscore_cascade = [] pscore_above = 1.0 for position_ in np.arange(len_list): pscore_above = pscore_above * 1.0 / (n_unique_action - position_) pscore_cascade.append(pscore_above) assert np.allclose( bandit_feedback_df["pscore_cascade"], np.tile(pscore_cascade, n_rounds) ), f"pscore_cascade must be {pscore_cascade} for all impresessions" assert np.allclose( bandit_feedback_df["pscore"].unique(), [pscore_above]), f"pscore must be {pscore_above} for all slates"
def synthetic_slate_bandit_feedback() -> BanditFeedback: # set parameters n_unique_action = 10 len_list = 3 dim_context = 2 reward_type = "binary" random_state = 12345 n_rounds = 100 dataset = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, random_state=random_state, ) # obtain feedback bandit_feedback = dataset.obtain_batch_bandit_feedback(n_rounds=n_rounds) return bandit_feedback
def test_synthetic_slate_obtain_batch_bandit_feedback_using_linear_behavior_policy( ): # set parameters n_unique_action = 10 len_list = 3 dim_context = 2 reward_type = "binary" random_state = 12345 n_rounds = 100 dataset = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, random_state=random_state, behavior_policy_function=linear_behavior_policy_logit, ) with pytest.raises(ValueError): _ = dataset.obtain_batch_bandit_feedback(n_rounds=-1) with pytest.raises(ValueError): _ = dataset.obtain_batch_bandit_feedback(n_rounds="a") # obtain feedback bandit_feedback = dataset.obtain_batch_bandit_feedback(n_rounds=n_rounds) # check slate bandit feedback (common test) check_slate_bandit_feedback(bandit_feedback=bandit_feedback) # print reward pscore_columns = [ "pscore_cascade", "pscore", "pscore_item_position", ] bandit_feedback_df = pd.DataFrame() for column in ["slate_id", "position", "action", "reward" ] + pscore_columns: bandit_feedback_df[column] = bandit_feedback[column] print(bandit_feedback_df.groupby("position")["reward"].describe()) if reward_type == "binary": assert set(np.unique(bandit_feedback["reward"])) == set([0, 1])
def test_synthetic_slate_init_using_invalid_inputs( n_unique_action, len_list, dim_context, reward_type, reward_structure, click_model, random_state, description, ): with pytest.raises(ValueError, match=f"{description}*"): _ = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, reward_structure=reward_structure, click_model=click_model, random_state=random_state, )
def test_synthetic_slate_obtain_batch_bandit_feedback_using_linear_behavior_policy_without_pscore_item_position( ): # set parameters n_unique_action = 80 len_list = 3 dim_context = 2 reward_type = "binary" random_state = 12345 n_rounds = 100 dataset = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, random_state=random_state, behavior_policy_function=linear_behavior_policy_logit, ) # obtain feedback bandit_feedback = dataset.obtain_batch_bandit_feedback( n_rounds=n_rounds, return_pscore_item_position=False) # check slate bandit feedback (common test) check_slate_bandit_feedback(bandit_feedback=bandit_feedback) assert ( bandit_feedback["pscore_item_position"] is None ), f"pscore marginal must be None, but {bandit_feedback['pscore_item_position']}" # random seed should be fixed dataset2 = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, random_state=random_state, behavior_policy_function=linear_behavior_policy_logit, ) # obtain feedback bandit_feedback2 = dataset2.obtain_batch_bandit_feedback( n_rounds=n_rounds, return_pscore_item_position=False) # check slate bandit feedback (common test) check_slate_bandit_feedback(bandit_feedback=bandit_feedback2) # check random seed effect assert np.allclose( bandit_feedback["expected_reward_factual"], bandit_feedback2["expected_reward_factual"], ) if reward_type == "binary": assert set(np.unique(bandit_feedback["reward"])) == set([0, 1])
def test_slate_ope_performance_using_standard_additive_log(): # set parameters n_unique_action = 10 len_list = 3 dim_context = 2 reward_type = "binary" random_state = 12345 n_rounds = 1000 reward_structure = "standard_additive" click_model = None behavior_policy_function = linear_behavior_policy_logit reward_function = logistic_reward_function dataset = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, reward_structure=reward_structure, click_model=click_model, random_state=random_state, behavior_policy_function=behavior_policy_function, base_reward_function=reward_function, ) random_behavior_dataset = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, reward_structure=reward_structure, click_model=click_model, random_state=random_state, behavior_policy_function=None, base_reward_function=reward_function, ) # obtain feedback bandit_feedback = dataset.obtain_batch_bandit_feedback(n_rounds=n_rounds) slate_id = bandit_feedback["slate_id"] reward = bandit_feedback["reward"] pscore = bandit_feedback["pscore"] pscore_item_position = bandit_feedback["pscore_item_position"] pscore_cascade = bandit_feedback["pscore_cascade"] position = bandit_feedback["position"] # obtain random behavior feedback random_behavior_feedback = random_behavior_dataset.obtain_batch_bandit_feedback( n_rounds=n_rounds) sips_estimated_policy_value = sips.estimate_policy_value( slate_id=slate_id, reward=reward, pscore=pscore, position=position, evaluation_policy_pscore=random_behavior_feedback["pscore"], ) iips_estimated_policy_value = iips.estimate_policy_value( slate_id=slate_id, reward=reward, pscore_item_position=pscore_item_position, position=position, evaluation_policy_pscore_item_position=random_behavior_feedback[ "pscore_item_position"], ) rips_estimated_policy_value = rips.estimate_policy_value( slate_id=slate_id, reward=reward, pscore_cascade=pscore_cascade, position=position, evaluation_policy_pscore_cascade=random_behavior_feedback[ "pscore_cascade"], ) # compute statistics of ground truth policy value q_pi_e = (random_behavior_feedback["reward"].reshape( (n_rounds, dataset.len_list)).sum(axis=1)) gt_mean = q_pi_e.mean() gt_std = q_pi_e.std(ddof=1) print("Standard additive") # check the performance of OPE ci_bound = gt_std * 3 / np.sqrt(q_pi_e.shape[0]) print(f"gt_mean: {gt_mean}, 3 * gt_std / sqrt(n): {ci_bound}") estimated_policy_value = { "sips": sips_estimated_policy_value, "iips": iips_estimated_policy_value, "rips": rips_estimated_policy_value, } for key in estimated_policy_value: print( f"estimated_value: {estimated_policy_value[key]} ------ estimator: {key}, " ) # test the performance of each estimator assert ( np.abs(gt_mean - estimated_policy_value[key]) <= ci_bound ), f"OPE of {key} did not work well (absolute error is greater than 3*sigma)"
def test_slate_ope_performance_using_standard_additive_log(): # set parameters n_unique_action = 10 len_list = 3 dim_context = 2 reward_type = "binary" random_state = 12345 n_rounds = 1000 reward_structure = "standard_additive" click_model = None behavior_policy_function = linear_behavior_policy_logit reward_function = logistic_reward_function dataset = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, reward_structure=reward_structure, click_model=click_model, random_state=random_state, behavior_policy_function=behavior_policy_function, base_reward_function=reward_function, ) random_behavior_dataset = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, reward_structure=reward_structure, click_model=click_model, random_state=random_state, behavior_policy_function=None, base_reward_function=reward_function, ) # obtain feedback bandit_feedback = dataset.obtain_batch_bandit_feedback(n_rounds=n_rounds) slate_id = bandit_feedback["slate_id"] context = bandit_feedback["context"] action = bandit_feedback["action"] reward = bandit_feedback["reward"] pscore = bandit_feedback["pscore_cascade"] position = bandit_feedback["position"] # obtain random behavior feedback random_behavior_feedback = random_behavior_dataset.obtain_batch_bandit_feedback( n_rounds=n_rounds) evaluation_policy_logit_ = np.ones( (n_rounds, n_unique_action)) / n_unique_action evaluation_policy_action_dist = ( np.ones(n_rounds * len_list * n_unique_action) / n_unique_action) ( _, _, evaluation_policy_pscore, ) = dataset.obtain_pscore_given_evaluation_policy_logit( action=action, evaluation_policy_logit_=evaluation_policy_logit_, return_pscore_item_position=False, ) evaluation_policy_action_dist = dataset.calc_evaluation_policy_action_dist( action=action, evaluation_policy_logit_=evaluation_policy_logit_, ) # obtain q_hat base_regression_model = SlateRegressionModel( base_model=DecisionTreeRegressor(max_depth=3, random_state=12345), len_list=len_list, n_unique_action=n_unique_action, fitting_method="iw", ) q_hat = base_regression_model.fit_predict( context=context, action=action, reward=reward, pscore_cascade=pscore, evaluation_policy_pscore_cascade=evaluation_policy_pscore, evaluation_policy_action_dist=evaluation_policy_action_dist, ) # check if q_hat=0 case coincides with rips cascade_dr_estimated_policy_value = dr.estimate_policy_value( slate_id=slate_id, action=action, reward=reward, pscore_cascade=pscore, position=position, evaluation_policy_pscore_cascade=evaluation_policy_pscore, q_hat=q_hat, evaluation_policy_action_dist=evaluation_policy_action_dist, ) # compute statistics of ground truth policy value q_pi_e = (random_behavior_feedback["reward"].reshape( (n_rounds, dataset.len_list)).sum(axis=1)) gt_mean = q_pi_e.mean() gt_std = q_pi_e.std(ddof=1) print("Cascade additive") # check the performance of OPE ci_bound = gt_std * 3 / np.sqrt(q_pi_e.shape[0]) print(f"gt_mean: {gt_mean}, 3 * gt_std / sqrt(n): {ci_bound}") estimated_policy_value = { "cascade-dr": cascade_dr_estimated_policy_value, } for key in estimated_policy_value: print( f"estimated_value: {estimated_policy_value[key]} ------ estimator: {key}, " ) # test the performance of each estimator assert ( np.abs(gt_mean - estimated_policy_value[key]) <= ci_bound ), f"OPE of {key} did not work well (absolute error is greater than 3*sigma)" # check if q_hat = 0 case of cascade-dr coincides with rips cascade_dr_estimated_policy_value_ = dr.estimate_policy_value( slate_id=slate_id, action=action, reward=reward, pscore_cascade=pscore, position=position, evaluation_policy_pscore_cascade=evaluation_policy_pscore, q_hat=np.zeros_like(q_hat), evaluation_policy_action_dist=evaluation_policy_action_dist, ) rips_estimated_policy_value = rips.estimate_policy_value( slate_id=slate_id, reward=reward, pscore_cascade=pscore, position=position, evaluation_policy_pscore_cascade=evaluation_policy_pscore, ) assert np.allclose( np.array([cascade_dr_estimated_policy_value_]), np.array([rips_estimated_policy_value]), )
def test_cascade_dr_criterion_using_standard_additive_log(): # set parameters n_unique_action = 3 len_list = 3 dim_context = 2 reward_type = "binary" random_state = 12345 n_rounds = 1000 reward_structure = "standard_additive" click_model = None behavior_policy_function = linear_behavior_policy_logit reward_function = logistic_reward_function dataset = SyntheticSlateBanditDataset( n_unique_action=n_unique_action, len_list=len_list, dim_context=dim_context, reward_type=reward_type, reward_structure=reward_structure, click_model=click_model, random_state=random_state, behavior_policy_function=behavior_policy_function, base_reward_function=reward_function, ) # obtain feedback bandit_feedback = dataset.obtain_batch_bandit_feedback(n_rounds=n_rounds) context = bandit_feedback["context"] action = bandit_feedback["action"] reward = bandit_feedback["reward"] pscore = bandit_feedback["pscore_cascade"] # random evaluation policy evaluation_policy_logit_ = np.ones( (n_rounds, n_unique_action)) / n_unique_action evaluation_policy_action_dist = ( np.ones(n_rounds * len_list * n_unique_action) / n_unique_action) ( _, _, evaluation_policy_pscore, ) = dataset.obtain_pscore_given_evaluation_policy_logit( action=action, evaluation_policy_logit_=evaluation_policy_logit_, return_pscore_item_position=False, ) evaluation_policy_action_dist = dataset.calc_evaluation_policy_action_dist( action=action, evaluation_policy_logit_=evaluation_policy_logit_, ) q_expected = calc_ground_truth_mean_reward_function( dataset=dataset, context=context, action=action, evaluation_policy_logit_=evaluation_policy_logit_, ) # obtain q_hat and check if q_hat is effective cascade_dr_criterion_pass_rate = 0.7 for fitting_method in ["normal", "iw"]: for model_name, model in model_dict.items(): base_regression_model = SlateRegressionModel( base_model=model(**hyperparams[model_name]), len_list=len_list, n_unique_action=n_unique_action, fitting_method=fitting_method, ) q_hat = base_regression_model.fit_predict( context=context, action=action, reward=reward, pscore_cascade=pscore, evaluation_policy_pscore_cascade=evaluation_policy_pscore, evaluation_policy_action_dist=evaluation_policy_action_dist, ) # compare dr criterion cascade_dr_criterion = np.abs((q_expected - q_hat)) - np.abs(q_hat) print( f"Dr criterion is satisfied with probability {np.mean(cascade_dr_criterion <= 0)} ------ model: {model_name} ({fitting_method})," ) assert ( np.mean(cascade_dr_criterion <= 0) >= cascade_dr_criterion_pass_rate ), f" should be satisfied with a probability at least {cascade_dr_criterion_pass_rate}"