def _optimize(cls, campaign: Campaign) -> (np.ndarray, np.ndarray): """ Optimize the combinatorial problem of the advertising campaign by using a dynamic programming algorithm :param campaign: the campaign to be optimized :return: - the optimization matrix (N+1) x M containing, for each pair (budget, set of sub-campaign), the maximum optimized value achievable (value might be, for instance, the number of clicks, or the profit) - the maximum indices (N+1) x M related to the optimization matrix containing, for each pair (budget, set of sub-campaign), the index of the best budget for the new added sub-campaign w.r.t. previous set of sub-campaign (i.e. row) where N is the number of sub-campaign and M is the number of discrete budgets """ optimization_matrix = np.zeros(shape=(campaign.get_n_sub_campaigns() + 1, len(campaign.get_budgets()))) max_idx_matrix = np.full_like(optimization_matrix, fill_value=-1, dtype=np.int) prev_row = 0 clicks_matrix = campaign.get_sub_campaigns() for row in range(1, optimization_matrix.shape[0]): temp_clicks = clicks_matrix[row - 1][::-1] for col in range(optimization_matrix.shape[1]): cum_sum_clicks = temp_clicks[optimization_matrix.shape[1] - col - 1:] + optimization_matrix[ prev_row, :col + 1] idx_max = np.argmax(cum_sum_clicks) optimization_matrix[row, col] = cum_sum_clicks[idx_max] max_idx_matrix[row, col] = col - idx_max prev_row = row return optimization_matrix, max_idx_matrix
def __init__(self, campaign: Campaign, model_list: List[DiscreteRegressor]): """ :param campaign: campaign that will be optimized :param model_list: regressors that will be used to estimate the quantity in each campaign. There is a regressor for each campaign """ assert len(model_list) == campaign.get_n_sub_campaigns() self.campaign: Campaign = campaign self.t: int = 0 self.collected_rewards: List[float] = [] self.pulled_superarm_list: List[List] = [] self.model_list: List[DiscreteRegressor] = model_list self.collected_rewards_sub_campaign: List[List] = [ [] for _ in range(campaign.get_n_sub_campaigns()) ] self.pulled_arm_sub_campaign: List[List] = [ [] for _ in range(campaign.get_n_sub_campaigns()) ] for sub_index, model in enumerate(self.model_list): sub_campaign_values = self.model_list[ sub_index].sample_distribution() self.campaign.set_sub_campaign_values(sub_index, sub_campaign_values)
def main(args): scenario = EnvironmentManager.load_scenario(args.scenario_name) env = PricingAdvertisingJointEnvironment(scenario) campaign = Campaign(scenario.get_n_subcampaigns(), args.cum_budget, args.n_arms) bandit = build_combinatorial_bandit(bandit_name=args.bandit_name, campaign=campaign, init_std=args.init_std, args=args) budget_allocation = [0, 0, 0] for t in range(0, args.n_rounds): # Choose arm budget_allocation_indexes = bandit.pull_arm() budget_allocation = [ int(campaign.get_budgets()[i]) for i in budget_allocation_indexes ] # Observe reward env.set_budget_allocation(budget_allocation=budget_allocation) env.next_day() rewards = env.get_daily_visits_per_sub_campaign() # Update bandit bandit.update(pulled_arm=budget_allocation_indexes, observed_reward=rewards) return bandit.collected_rewards, budget_allocation
def test_find_best_budgets_3(self): campaign = Campaign(4, 100, 5) campaign.set_sub_campaign_values(0, [0, 3, 12, 20, 15]) campaign.set_sub_campaign_values(1, [0, 2, 7, 10, 9]) campaign.set_sub_campaign_values(2, [0, 5, 8, 12, 18]) campaign.set_sub_campaign_values(3, [0, 9, 9, 10, 7]) max_clicks, best_budgets = CampaignOptimizer.find_best_budgets(campaign) self.assertEqual(max_clicks, 29) self.assertTrue((best_budgets == np.array([75, 0, 0, 25])).all())
def __init__(self, campaign: Campaign, arm_values: np.ndarray, price_learner_class: DiscreteBandit.__class__, price_learner_kwargs, number_of_visit_model_list: List[DiscreteRegressor], ad_value_strategy: AdValueStrategy): super().__init__(campaign) assert len( number_of_visit_model_list) == campaign.get_n_sub_campaigns() # General problem data self.arm_values: np.ndarray = arm_values # Data structure to save overall data self.collected_rewards_sub_campaign: List[List] = [ [] for _ in range(campaign.get_n_sub_campaigns()) ] self.pulled_arm_sub_campaign: List[List] = [ [] for _ in range(campaign.get_n_sub_campaigns()) ] self.value_per_clicks_per_price_idx: Dict[int, List[float]] = {} for arm_idx in range(len(self.arm_values)): rewards_per_subcampaign = [ [] for _ in range(self.campaign.get_n_sub_campaigns()) ] self.value_per_clicks_per_price_idx[arm_idx] = ad_value_strategy. \ get_estimated_value_per_clicks(rewards_per_subcampaign, self.day_t) self.rewards_per_arm_per_user_class: Dict[ int, List[List[float]]] = defaultdict(list) for i in range(self.campaign.get_n_sub_campaigns()): for j in range(len(self.arm_values)): self.rewards_per_arm_per_user_class[i].append([]) # Learners self.price_bandit_class = price_learner_class self.price_bandit_kwargs = price_learner_kwargs self.unique_price_learner: DiscreteBandit = self.price_bandit_class( **self.price_bandit_kwargs) self.number_of_visit_model_list: List[ DiscreteRegressor] = number_of_visit_model_list self.ad_value_strategy: AdValueStrategy = ad_value_strategy
def __init__(self, campaign: Campaign, ads_learner: CombinatorialBandit, price_learner: List[DiscreteBandit], ad_value_strategy: AdValueStrategy): assert len(price_learner) == campaign.get_n_sub_campaigns() super().__init__(campaign=campaign) self.ads_learner: CombinatorialBandit = ads_learner self.price_learner: List[DiscreteBandit] = price_learner self.n_arms = price_learner[0].n_arms self.ad_value_strategy = ad_value_strategy self.estimated_ad_value = None
def __init__(self, campaign: Campaign, number_of_visit_model_list: List[DiscreteRegressor], n_arms_profit: int, arm_profit: np.array, min_std: float): assert len(number_of_visit_model_list) == campaign.get_n_sub_campaigns() super().__init__(campaign=campaign) # Number of visit data structure self.number_of_visit_model_list: List[DiscreteRegressor] = number_of_visit_model_list self.collected_rewards_sub_campaign: List[List] = [[] for _ in range(campaign.get_n_sub_campaigns())] self.pulled_arm_sub_campaign: List[List] = [[] for _ in range(campaign.get_n_sub_campaigns())] # Pricing data structure self.n_arms_price: int = n_arms_profit self.arm_profit = arm_profit self.profit_arm_reward_list = [[[] for _ in range(n_arms_profit)] for _ in range(campaign.get_n_sub_campaigns())] self.min_std = min_std # Current data structure self.current_pricing_arm_idx = np.random.randint(low=0, high=n_arms_profit, size=1) self.curr_best_budget_idx: List[int] = [0 for _ in range(campaign.get_n_sub_campaigns())] self.current_budget_allocation = [0 for _ in range(campaign.get_n_sub_campaigns())] # Initializing randomly budget values for sub_index, model in enumerate(self.number_of_visit_model_list): sub_campaign_values = self.number_of_visit_model_list[sub_index].sample_distribution() self.campaign.set_sub_campaign_values(sub_index, sub_campaign_values) _, best_budgets = CampaignOptimizer.find_best_budgets(self.campaign) self.curr_best_budget_idx = [np.where(self.campaign.get_budgets() == budget)[0][0] for budget in best_budgets]
def test_optimize_3(self): campaign = Campaign(4, 100, 5) campaign.set_sub_campaign_values(0, [0, 3, 12, 20, 15]) campaign.set_sub_campaign_values(1, [0, 2, 7, 10, 9]) campaign.set_sub_campaign_values(2, [0, 5, 8, 12, 18]) campaign.set_sub_campaign_values(3, [0, 9, 9, 10, 7]) opt_matrix, max_idx_matrix = CampaignOptimizer._optimize(campaign) true_opt_matrix = np.array([[0, 0, 0, 0, 0], [0, 3, 12, 20, 20], [0, 3, 12, 20, 22], [0, 5, 12, 20, 25], [0, 9, 14, 21, 29]]) true_max_idx_matrix = np.array([[-1, -1, -1, -1, -1], [0, 1, 2, 3, 3], [0, 0, 0, 0, 1], [0, 1, 0, 0, 1], [0, 1, 1, 1, 1]]) self.assertTrue((true_opt_matrix == opt_matrix).all()) self.assertTrue((true_max_idx_matrix == max_idx_matrix).all())
def test_find_best_budgets_4(self): campaign = Campaign(3, 100, 11) # 100(0.0+0.3)*b = 30*x1 = max 3000 # 100(0.2+0.2)*b = 40*x2 = max 4000 # 100(0.4+0.1)*b = 50*x3 = max 5000 campaign.set_sub_campaign_values(0, [0, 300, 600, 900, 1200, 1500, 1800, 2100, 2400, 2700, 3000]) campaign.set_sub_campaign_values(1, [0, 400, 800, 1200, 1600, 2000, 2000, 2000, 2000, 2000, 2000]) campaign.set_sub_campaign_values(2, [0, 500, 1000, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500]) max_clicks, best_budgets = CampaignOptimizer.find_best_budgets(campaign) print(max_clicks) print(best_budgets) my_best_budgets = np.array([20.0, 50.0, 30.0]) self.assertTrue((best_budgets == my_best_budgets).all())
def test_find_best_budgets_1(self): campaign = Campaign(2, 100, 3) campaign.set_sub_campaign_values(0, [3, 7, 14]) campaign.set_sub_campaign_values(1, [2, 5, 7]) max_clicks, best_budgets = CampaignOptimizer.find_best_budgets(campaign) self.assertEqual(max_clicks, 16) self.assertTrue((best_budgets == np.array([100, 0])).all())
def test_find_best_budgets_2(self): campaign = Campaign(3, 90, 4) campaign.set_sub_campaign_values(0, [0, 3, 12, 20]) campaign.set_sub_campaign_values(1, [0, 2, 7, 10]) campaign.set_sub_campaign_values(2, [0, 5, 8, 12]) max_clicks, best_budgets = CampaignOptimizer.find_best_budgets(campaign) self.assertEqual(max_clicks, 20) self.assertTrue((best_budgets == np.array([90, 0, 0])).all())
def test_optimize_1(self): campaign = Campaign(2, 100, 3) campaign.set_sub_campaign_values(0, [3, 7, 14]) campaign.set_sub_campaign_values(1, [2, 5, 7]) opt_matrix, max_idx_matrix = CampaignOptimizer._optimize(campaign) true_opt_matrix = np.array([[0, 0, 0], [3, 7, 14], [5, 9, 16]]) true_max_idx_matrix = np.array([[-1, -1, -1], [0, 1, 2], [0, 0, 0]]) self.assertTrue((true_opt_matrix == opt_matrix).all()) self.assertTrue((true_max_idx_matrix == max_idx_matrix).all())
def main(args): # Retrieve scenario scenario = EnvironmentManager.load_scenario(args.scenario_name) # Retrieve bandit and basic information campaign = Campaign(scenario.get_n_subcampaigns(), args.cum_budget, args.n_arms_ads) prices = get_prices(args=args) arm_profit = prices - args.unit_cost bandit = get_bandit(args=args, arm_values=arm_profit, campaign=campaign) # Create environment env = PricingAdvertisingJointEnvironment(scenario=scenario) # choose whether to pull one arm a day for each class, or to pull one arm for each user if args.daily_price: learn_per_day(bandit, env, campaign, prices, arm_profit, scenario.get_n_subcampaigns()) else: learn_per_user(bandit, env, campaign, prices, arm_profit) return bandit.get_daily_reward(), env.day_breakpoints
def test_optimize_2(self): campaign = Campaign(3, 90, 4) campaign.set_sub_campaign_values(0, [0, 3, 12, 20]) campaign.set_sub_campaign_values(1, [0, 2, 7, 10]) campaign.set_sub_campaign_values(2, [0, 5, 8, 12]) opt_matrix, max_idx_matrix = CampaignOptimizer._optimize(campaign) true_opt_matrix = np.array([[0, 0, 0, 0], [0, 3, 12, 20], [0, 3, 12, 20], [0, 5, 12, 20]]) true_max_idx_matrix = np.array([[-1, -1, -1, -1], [0, 1, 2, 3], [0, 0, 0, 0], [0, 1, 0, 0]]) self.assertTrue((true_opt_matrix == opt_matrix).all()) self.assertTrue((true_max_idx_matrix == max_idx_matrix).all())
def get_bandit(args, arm_values: np.array, campaign: Campaign) -> IJointBandit: bandit_name: str = args.joint_bandit_name ads_bandit = build_combinatorial_bandit(bandit_name=args.ads_bandit_name, campaign=campaign, init_std=args.init_std, args=args) price_bandit_class, price_bandit_kwargs = get_bandit_class_and_kwargs( bandit_name=args.pricing_bandit_name, n_arms=len(arm_values), arm_values=arm_values, args=args) price_bandit_list = [ price_bandit_class(**price_bandit_kwargs) for _ in range(campaign.get_n_sub_campaigns()) ] ad_value_strategy = ExpectationAdValueStrategy(np.max(arm_values)) \ if bandit_name.find("Exp") >= 0 else QuantileAdValueStrategy(np.max(arm_values), args.min_std_q) is_learn_visits = True if bandit_name[-1] == 'V' else False if bandit_name in ["JBExp", "JBExpV", "JBQV", "JBQ"]: bandit = JointBanditDiscriminatory(ads_learner=ads_bandit, price_learner=price_bandit_list, campaign=campaign, ad_value_strategy=ad_value_strategy, is_learn_visits=is_learn_visits) elif bandit_name in ["JBIExpV", "JBIQV"]: bandit = JointBanditDiscriminatoryImproved( ads_learner=ads_bandit, price_learner=price_bandit_list, campaign=campaign, ad_value_strategy=ad_value_strategy) elif bandit_name in ["JBBQ", "JBBExp"]: model_list: List[DiscreteRegressor] = [ DiscreteGPRegressor(list(campaign.get_budgets()), args.init_std, args.alpha, args.n_restart_opt, normalized=True) for _ in range(campaign.get_n_sub_campaigns()) ] bandit = JointBanditBalanced(campaign=campaign, arm_values=arm_values, price_learner_class=price_bandit_class, price_learner_kwargs=price_bandit_kwargs, number_of_visit_model_list=model_list, ad_value_strategy=ad_value_strategy) elif bandit_name == "JBFQ": assert args.daily_price, "This joint bandit requires to run in a daily manner" model_list: List[DiscreteRegressor] = [ DiscreteGPRegressor(list(campaign.get_budgets()), args.init_std, args.alpha, args.n_restart_opt, normalized=True) for _ in range(campaign.get_n_sub_campaigns()) ] bandit = JointBanditFixedDailyPriceQuantile( campaign=campaign, number_of_visit_model_list=model_list, min_std=args.min_std_q, arm_profit=arm_values, n_arms_profit=len(arm_values)) elif bandit_name == "JBFTS": assert args.daily_price, "This joint bandit requires to run in a daily manner" model_list: List[DiscreteRegressor] = [ DiscreteGPRegressor(list(campaign.get_budgets()), args.init_std, args.alpha, args.n_restart_opt, normalized=True) for _ in range(campaign.get_n_sub_campaigns()) ] bandit = JointBanditFixedDailyPriceTS( campaign=campaign, number_of_visit_model_list=model_list, arm_profit=arm_values, n_arms_profit=len(arm_values)) else: raise argparse.ArgumentParser( "The name of the bandit to be used is not in the available ones") return bandit
std_df.rename(columns={n_bandit: "day"}, inplace=True) total_df = mean_df.merge(std_df, left_on="day", right_on="day") total_df.to_csv("{}instant_reward.csv".format(folder_path_with_date), index=False) # Something mean_scenario: Scenario = EnvironmentManager.load_scenario( SCENARIO_NAME, get_mean_function=True) click_function_list: List[IStochasticFunction] = mean_scenario.get_phases( )[0].get_n_clicks_function() # Optimal point computation campaign = Campaign(n_sub_campaigns=mean_scenario.get_n_subcampaigns(), cum_budget=BUDGET, n_arms=N_ARMS_ADV) for i in range(campaign.get_n_sub_campaigns()): sub_campaign_values = [ click_function_list[i].draw_sample(b) for b in np.linspace(0, BUDGET, N_ARMS_ADV) ] campaign.set_sub_campaign_values(i, sub_campaign_values) max_clicks, best_budgets = CampaignOptimizer.find_best_budgets(campaign) # # Compute regret # if CSV_CUM_REGRET: # mean_regret_data = np.zeros(shape=(n_bandit + 1, n_days)) # std_regret_data = np.zeros(shape=(n_bandit + 1, n_days)) # mean_regret_data[-1] = np.arange(n_days) + 1 # std_regret_data[-1] = np.arange(n_days) + 1
def pull_budget(self) -> List[int]: max_best_clicks = 0 max_best_budgets = None for arm_idx in range(len(self.arm_values)): subcampaign_values = self.campaign.get_sub_campaigns() arm_value_campaign = Campaign(self.campaign.get_n_sub_campaigns(), self.campaign.get_cum_budget(), len(self.campaign.get_budgets())) for sub_idx in range(len(subcampaign_values)): arm_value_campaign.set_sub_campaign_values( sub_idx, subcampaign_values[sub_idx] * self.value_per_clicks_per_price_idx[arm_idx][sub_idx]) max_clicks, best_budgets = CampaignOptimizer.find_best_budgets( self.campaign) if max_best_clicks < max_clicks: max_best_clicks = max_clicks max_best_budgets = best_budgets # Balance price learner with number of clicks distribution estimation budget_value_to_index = { value: i for i, value in enumerate(self.campaign.get_budgets()) } estimated_clicks = np.array([ self.campaign.get_sub_campaigns()[sub_idx, budget_value_to_index[budget]] for sub_idx, budget in enumerate(max_best_budgets) ]) user_probabilities = estimated_clicks / np.sum(estimated_clicks) self.unique_price_learner: DiscreteBandit = self.price_bandit_class( **self.price_bandit_kwargs) for arm_idx in range(len(self.arm_values)): rewards_per_subcampaign = [] for sub_idx in range(self.campaign.get_n_sub_campaigns()): rewards_per_subcampaign.append( self.rewards_per_arm_per_user_class[sub_idx][arm_idx]) rewards_len = np.array( [len(rewards) for rewards in rewards_per_subcampaign]) solution = np.min(rewards_len / user_probabilities) balanced_rewards_len = np.array(np.floor(solution * user_probabilities), dtype=int) # Clip values of balanced rewards length between 0 and original rewards length balanced_rewards_len = np.maximum(balanced_rewards_len, 0) balanced_rewards_len = np.minimum(balanced_rewards_len, rewards_len) for sub_idx in range(self.campaign.get_n_sub_campaigns()): sampled_rewards = np.random.choice( rewards_per_subcampaign[sub_idx], size=balanced_rewards_len[sub_idx], replace=False) for reward in sampled_rewards: self.unique_price_learner.update(arm_idx, reward) return [budget_value_to_index[budget] for budget in max_best_budgets]