def make_stats_row_from_df(cur_data, include_power, effect_size = None, alpha = None): '''Calculates output statistics given the data frame cur_data. If include_power, includes the power calculation. efffect_size and alpha are only required/used if power is calculated ''' sample_sizes = np.array([np.sum(cur_data[action_header] == i) for i in range(1,3)]) successes = np.array([np.sum(cur_data[cur_data[action_header] == 1][obs_reward_header]), np.sum(cur_data[cur_data[action_header] == 2][obs_reward_header])]) #calculate sample size and mean cur_row = {} sample_size_1 = sample_sizes[0] sample_size_2 = sample_sizes[1] cur_row['sample_size_1'] = sample_size_1 cur_row['sample_size_2'] = sample_size_2 mean_1 = np.mean(cur_data[cur_data[action_header] == 1][obs_reward_header])# JN mean for arm 1 mean_2 = np.mean(cur_data[cur_data[action_header] == 2][obs_reward_header])#JN mean for arm 2 cur_row['mean_1'] = mean_1 cur_row['mean_2'] = mean_2 #SE = sqrt[(P^hat_A*(1-P^hat_A)/N_A + (P^hat_B*(1-P^hat_B)/N_B] SE = np.sqrt(mean_1*(1 - mean_1)/sample_size_1 + mean_2*(1 - mean_2)/sample_size_2) wald_type_stat = (mean_1 - mean_2)/SE #(P^hat_A - P^hat_b)/SE #print('wald_type_stat:', wald_type_stat) wald_pval = (1 - scipy.stats.norm.cdf(np.abs(wald_type_stat)))*2 #Two sided, symetric, so compare to 0.05 cur_row['wald_type_stat'] = wald_type_stat #TODO add in Delta!!!! cur_row['wald_pval'] = wald_pval #print("wald_pval", wald_pval) #calculate total reward cur_row['total_reward'] = np.sum(cur_data[obs_reward_header]) #calculate power cur_row['ratio'] = sample_sizes[0] / sample_sizes[1] if include_power: cur_row['power'] = smp.GofChisquarePower().solve_power(effect_size, nobs = sum(sample_sizes), n_bins=(2-1)*(2-1) + 1, alpha = alpha) cur_row['actual_es'] = calculate_effect_size(sample_sizes, successes) #calculate chi squared contingency test table = sms.Table(np.stack((successes,sample_sizes - successes)).T) rslt = table.test_nominal_association() cur_row['stat'] = rslt.statistic cur_row['pvalue'] = rslt.pvalue cur_row['df'] = rslt.df # Added to match normal rewards cur_row['statUnequalVar'],cur_row['pvalueUnequalVar'], cur_row['dfUnequalVar'] = cur_row['stat'],cur_row['pvalue'],cur_row['df'] return cur_row
def calc_chisquared_sample_size( baseline_conversion_rate_percentage: np.float64, expected_uplift_percentage: np.float64, power_percentage: np.float64 = 80, confidence_level_percentage: np.float64 = 95) -> np.float64: """Estimates the minimum sample size when the KPI is conversion rate. Estimated sample size using the Chi-squared test of proportions is the minimum required for either a Test or a Control group in an A/B test. Args: baseline_conversion_rate_percentage: Baseline conversion rate as a percentage. expected_uplift_percentage: Expected uplift of the media experiment on the baseline conversion rate as a percentage. power_percentage: Statistical power of the Chi-squared test as a percentage. confidence_level_percentage: Statistical confidence level of the Chi-squared test as a percentage. Returns: sample_size: Estimated minimum sample size required for either a Test or a Control group. """ null_probability = baseline_conversion_rate_percentage / 100 alternative_probability = (null_probability * (100 + expected_uplift_percentage) / 100) alpha_proportion = (100 - confidence_level_percentage) / 100 power_proportion = power_percentage / 100 effect_size = gof.chisquare_effectsize( probs0=[null_probability, 1 - null_probability], probs1=[alternative_probability, 1 - alternative_probability], correction=None, cohen=True, axis=0) power_test = power.GofChisquarePower() sample_size = power_test.solve_power(effect_size=effect_size, nobs=None, alpha=alpha_proportion, power=power_proportion, n_bins=2) return np.ceil(sample_size)
def cont_table_power(self): rows = 5 cols = 2 df = (rows - 1) * (cols - 1) nbins = df + 1 alpha = 0.05 power = 0.8 st1_scores = self.get_scores('ST1') st2_scores = self.get_scores('ST2') col1 = self.bin_three(st1_scores) col2 = self.bin_three(st2_scores) n = sum(col1) + sum(col2) print(n) ct = np.array([col1, col2]).T print(ct) chi2, p, dof, ex = chi2_contingency(ct, correction=False) es = np.sqrt(chi2 / n * df) # cramer's v print(es) # medium effect sample_size = smp.GofChisquarePower().solve_power(es, n_bins=nbins, alpha=alpha, power=power) print(sample_size)
norm_pow = smp.NormalIndPower().power(0.01, nobs, 0.05, alternative="larger") norm_pow_R = 0.056869534873146124 #value from R: >pwr.2p.test(h=0.01,n=80,sig.level=0.05,alternative="greater") print('norm_pow', norm_pow, norm_pow - norm_pow_R) # Note: negative effect size is same as switching one-sided alternative # TODO: should I switch to larger/smaller instead of "one-sided" options norm_pow = smp.NormalIndPower().power(-0.01, nobs, 0.05, alternative="larger") norm_pow_R = 0.0438089705093578 #value from R: >pwr.2p.test(h=0.01,n=80,sig.level=0.05,alternative="less") print('norm_pow', norm_pow, norm_pow - norm_pow_R) #Note: I use n_bins and ddof instead of df # pwr.chisq.test(w=0.289,df=(4-1),N=100,sig.level=0.05) chi2_pow = smp.GofChisquarePower().power(0.289, 100, 4, 0.05) chi2_pow_R = 0.675077657003721 print('chi2_pow', chi2_pow, chi2_pow - chi2_pow_R) chi2_pow = smp.GofChisquarePower().power(0.01, 100, 4, 0.05) chi2_pow_R = 0.0505845519208533 print('chi2_pow', chi2_pow, chi2_pow - chi2_pow_R) chi2_pow = smp.GofChisquarePower().power(2, 100, 4, 0.05) chi2_pow_R = 1 print('chi2_pow', chi2_pow, chi2_pow - chi2_pow_R) chi2_pow = smp.GofChisquarePower().power(0.9, 100, 4, 0.05) chi2_pow_R = 0.999999999919477 print('chi2_pow', chi2_pow, chi2_pow - chi2_pow_R, 'lower precision ?')
def main(): recalculate_bandits = True #batch_size = 1.0 num_sims = int(sys.argv[2]) outfile_directory = sys.argv[3] burn_in_size, batch_size = int( outfile_directory.split("=")[-1].split('-')[0]), int( outfile_directory.split("=")[-1].split('-')[1]) print("burn_in_size, batch_size", burn_in_size, batch_size) num_arms = 2 # if sys.argv[1] has a comma, just use the result as probability per arm if "," in sys.argv[1]: if sys.argv[1].count(",") == 1: # specifying probability per arm but not effect size prob_per_arm = [ float(armProb) for armProb in sys.argv[1].split(",") ] effect_size = 0 # Note: This will be wrong if arm probs aren't equal! else: # specifying probability per arm as first two arguments, and then effect size numeric_arguments = [ float(armProb) for armProb in sys.argv[1].split(",") ] prob_per_arm = numeric_arguments[: 2] # first two are arm probabilities effect_size = numeric_arguments[2] # final is effect size # We also need to specify n in this case for deciding on step sizes n = int(sys.argv[6]) else: # We just need effect size for this calculation effect_size = float(sys.argv[1].split("-")[0]) center = float(sys.argv[1].split("-")[1]) prob_per_arm = get_prob_per_arm_from_effect_size(effect_size, center) # Assumes we have two arms nobs_total = smp.GofChisquarePower().solve_power(effect_size, n_bins=(2 - 1) * (2 - 1) + 1, alpha=DESIRED_ALPHA, power=DESIRED_POWER) # print("Calculated nobs for effect size:", nobs_total) n = math.ceil(nobs_total) print("center", center) #step_sizes = [math.ceil(n/2), n, 2*n] # These differ from the version for normal because in normal, n represented size for one cond rather than overall size step_sizes = [ math.ceil(n / 2), n, 2 * n, 4 * n ] # These differ from the version for normal because in normal, n represented size for one cond rather than overall size print("prob_per_arm", prob_per_arm) if len(sys.argv) > 7 and sys.argv[7].startswith("forceActions"): run_effect_size_simulations.FORCE_ACTIONS = True num_to_force = float(sys.argv[7].split(",")[1]) else: num_to_force = 0 bandit_type = "Thompson" bandit_type_prefix = 'BB' if len(sys.argv) > 4: bandit_type = sys.argv[4] if bandit_type == "uniform": bandit_type_prefix = "BU" # Bernoulli rewards, uniform policy reorder_rewards = False softmax_beta = None reordering_fn = None if len(sys.argv) > 7 and not sys.argv[7].startswith("forceActions"): # softmax beta for how to reorder rewards reorder_rewards = True softmax_beta = float(sys.argv[7]) reordering_fn = reorder_samples_in_rewards.order_by_named_column( 'Action1OracleActualReward') if len(sys.argv) > 8: reordering_fn_specifier = sys.argv[8] reordering_fn = reorder_samples_in_rewards.get_reordering_fn( reordering_fn_specifier) prior_params = None if recalculate_bandits: if bandit_type == "uniform": run_simulations_uniform_random(num_sims, prob_per_arm, step_sizes, outfile_directory, forceActions=num_to_force) else: if len(sys.argv) > 5: if sys.argv[5] == "armsHigh": # Arms should be higher than the prior priorProportionOnSuccess = min( prob_per_arm) * PRIOR_PROPORTION_DIFFERENCE elif sys.argv[5] == "armsLow": # Arms should be lower than the prior priorProportionOnSuccess = 1 - ( 1 - max(prob_per_arm)) * PRIOR_PROPORTION_DIFFERENCE else: # Prior should be uniform (in between arms) priorProportionOnSuccess = .5 # Make sure the prior sums to 2, mirroring the successes/failures of uniform prior prior_params = [ priorProportionOnSuccess * 2, 2 - priorProportionOnSuccess * 2 ] print("Prior params: ", prior_params) run_simulations(num_sims, prob_per_arm, step_sizes, outfile_directory, prior_params[0], prior_params[1], softmax_beta=softmax_beta, reordering_fn=reordering_fn, forceActions=num_to_force, batch_size=batch_size, burn_in_size=burn_in_size) else: run_simulations(num_sims, prob_per_arm, step_sizes, outfile_directory, forceActions = num_to_force, batch_size = batch_size, \ burn_in_size = burn_in_size) outfile_prefix = outfile_directory + bandit_type_prefix + str(effect_size) if effect_size == 0: # Then include the n in the prefix outfile_prefix += "N" + str(n) df = calculate_statistics_from_sims(outfile_directory, num_sims, step_sizes, effect_size, DESIRED_ALPHA) df.to_pickle(outfile_prefix + 'Df.pkl') df_by_trial = calculate_by_trial_statistics_from_sims( outfile_directory, num_sims, step_sizes, effect_size, DESIRED_ALPHA) df_by_trial.to_pickle(outfile_prefix + 'DfByTrial.pkl') # Print various stats summary_text = effect_size_sim_output_viz.print_output_stats( df, prob_per_arm, False, prior_params=prior_params, reordering_info=softmax_beta) with open(outfile_prefix + 'SummaryText.txt', 'w', newline='') as outf: outf.write(summary_text) overall_stats_df = effect_size_sim_output_viz.make_overall_stats_df( df, prob_per_arm, False, effect_size) overall_stats_df.to_pickle(outfile_prefix + 'OverallStatsDf.pkl') # Make histogram hist_figure = effect_size_sim_output_viz.make_hist_of_trials(df) hist_figure.savefig(outfile_prefix + 'HistOfConditionProportions.pdf', bbox_inches='tight') # Make line plot test_stat_figure = effect_size_sim_output_viz.make_by_trial_graph_of_column( df_by_trial, 'stat') test_stat_figure.savefig(outfile_prefix + 'TestStatOverTime.pdf', bbox_inches='tight') pvalue_figure = effect_size_sim_output_viz.make_by_trial_graph_of_column( df_by_trial, 'pvalue') pvalue_figure.savefig(outfile_prefix + 'PValueOverTime.pdf', bbox_inches='tight') # Plot power power_figure = effect_size_sim_output_viz.plot_power_by_steps( df_by_trial, DESIRED_ALPHA, DESIRED_POWER) power_figure.savefig(outfile_prefix + 'PowerOverTime.pdf', bbox_inches='tight') #Plot reward reward_figure = effect_size_sim_output_viz.make_by_trial_graph_of_column( df_by_trial, 'total_reward') reward_figure = effect_size_sim_output_viz.add_expected_reward_to_figure( reward_figure, prob_per_arm, step_sizes) reward_figure.savefig(outfile_prefix + 'RewardOverTime.pdf', bbox_inches='tight') # Plot arm statistics arm_df_by_trial = create_arm_stats_by_step(outfile_directory, num_sims, step_sizes[-1], num_arms) arm_stats_figure = effect_size_sim_output_viz.make_by_trial_arm_statistics( arm_df_by_trial, num_arms) arm_stats_figure.savefig(outfile_prefix + 'ArmStats.pdf', bbox_inches='tight')
def make_stats_row_from_df(simulations_df, include_power, action_count, effect_size=None, alpha=None): '''Calculates output statistics given the data frame simulations_df. If include_power, includes the power calculation. efffect_size and alpha are only required/used if power is calculated ''' #REPLACE cur_data with simulations_df #number of actions! action_count step_size = max(simulations_df.index) + 1 #will be 4n for instance n = step_size / 4 #assumes set to 4n n_size_list = [math.ceil(n / 2), int(n), int(2 * n), int(4 * n)] print("step_size", step_size) print("n_size_list", n_size_list) trials_count = len(simulations_df) print("trials_count", trials_count) #will go to end, so 4n*num_sims print(simulations_df.columns) all_rows = [] for n_size in n_size_list: sim_num = 0 for idx in range(0, trials_count, step_size): one_sim_df = simulations_df[idx:idx + n_size].copy() assert (len(one_sim_df) == n_size) cur_row = {} prop_exploring_ppd_cuml = np.sum( one_sim_df[ppd_exp_header]) / n_size #cumulative # print(one_sim_df["SampleNumber"]) # print("n_size", n_size, exploring_this_n) exploring_this_n = one_sim_df[ one_sim_df["SampleNumber"] == n_size - 1][ppd_exp_header].iloc[0] #snap shot, conver to idx sample_sizes = np.array([]) successes = np.array([]) means = np.array([]) np.array([]) np.array([]) for i in range(1, action_count + 1): sample_sizes = np.append( sample_sizes, np.sum(one_sim_df[action_header] == i)) successes = np.append( successes, np.sum(one_sim_df[one_sim_df[action_header] == i] [obs_reward_header])) means = np.append( means, np.mean(one_sim_df[one_sim_df[action_header] == i] [obs_reward_header])) #calculate sample size and mean for i in range(action_count): cur_row['sample_size_{}'.format(i + 1)] = sample_sizes[i] cur_row['mean_{}'.format(i + 1)] = means[i] if action_count == 2: #SE = sqrt[(P^hat_A*(1-P^hat_A)/N_A + (P^hat_B*(1-P^hat_B)/N_B] SE = np.sqrt(means[0] * (1 - means[0]) / sample_sizes[0] + means[1] * (1 - means[1]) / sample_sizes[1]) wald_type_stat = (means[0] - means[1]) / SE #(P^hat_A - P^hat_b)/SE #print('wald_type_stat:', wald_type_stat) #Two sided, symetric, so compare to 0.05 wald_pval = (1 - scipy.stats.norm.cdf(np.abs(wald_type_stat))) * 2 cur_row['wald_type_stat'] = wald_type_stat cur_row['wald_pval'] = wald_pval #calculate total reward cur_row['total_reward'] = np.sum(one_sim_df[obs_reward_header]) #calculate power cur_row['ratio'] = sample_sizes[0] / sample_sizes[1] if include_power: cur_row['power'] = smp.GofChisquarePower().solve_power( effect_size, nobs=sum(sample_sizes), n_bins=(2 - 1) * (2 - 1) + 1, alpha=alpha) cur_row['actual_es'] = calculate_effect_size( sample_sizes, successes) #calculate chi squared contingency test table = sms.Table( np.stack((successes, sample_sizes - successes)).T) rslt = table.test_nominal_association() cur_row['stat'] = rslt.statistic cur_row['pvalue'] = rslt.pvalue cur_row['df'] = rslt.df # Added to match normal rewards cur_row['statUnequalVar'] = cur_row['stat'] cur_row['pvalueUnequalVar'] = cur_row['pvalue'] cur_row['dfUnequalVar'] = cur_row['df'] cur_row['num_steps'] = max(one_sim_df.index) + 1 # cur_row['num_steps'] = n_size cur_row['sim'] = sim_num cur_row["prop_exploring_ppd_cuml"] = prop_exploring_ppd_cuml cur_row["exploring_ppd_at_this_n"] = exploring_this_n all_rows.append(cur_row) sim_num += 1 return all_rows
def main(): start_time = time.time() outfile_directory = sys.argv[3] random_dur_m = 0 random_start_r = 0 recalculate_bandits = True num_arms = 2 #batch_size = 1.0 # if len(sys.argv) > 5: # epsilon = float(sys.argv[5]) # print("epsilon", epsilon) if "epsilon=" in outfile_directory: print(outfile_directory) epsilon = float( outfile_directory.split("epsilon=")[-1].split("/")[0].strip("=")) #c = float(outfile_directory.split("=c=")[-1]) print("epsilon", epsilon) num_sims = int(sys.argv[2]) burn_in_size, batch_size = int( outfile_directory.split("=")[-1].split('-')[0]), int( outfile_directory.split("=")[-1].split('-')[1]) print("burn_in_size, batch_size", burn_in_size, batch_size) # if sys.argv[1] has a comma, just use the result as probability per arm if "," in sys.argv[1]: if sys.argv[1].count(",") == 1: # specifying probability per arm but not effect size prob_per_arm = [ float(armProb) for armProb in sys.argv[1].split(",") ] effect_size = 0 # Note: This will be wrong if arm probs aren't equal! else: # specifying probability per arm as first two arguments, and then effect size numeric_arguments = [ float(armProb) for armProb in sys.argv[1].split(",") ] prob_per_arm = numeric_arguments[: 2] # first two are arm probabilities effect_size = numeric_arguments[2] # final is effect size # We also need to specify n in this case for deciding on step sizes n = int(sys.argv[6]) else: # We just need effect size for this calculation effect_size = float(sys.argv[1].split("-")[0]) center = float(sys.argv[1].split("-")[1]) prob_per_arm = get_prob_per_arm_from_effect_size(effect_size, center) # Assumes we have two arms nobs_total = smp.GofChisquarePower().solve_power(effect_size, n_bins=(2 - 1) * (2 - 1) + 1, alpha=DESIRED_ALPHA, power=DESIRED_POWER) #print("Calculated nobs for effect size:", nobs_total) n = math.ceil(nobs_total) print("center", center) ''' These differ from the version for normal because in normal, n represented size for one cond rather than overall size step_sizes = [math.ceil(n/2), n, 2*n, 4*n] ''' #Arghavan: Just run simulation for n step_sizes = [4 * n] print("prob_per_arm", prob_per_arm) if len(sys.argv) > 7 and sys.argv[7].startswith("forceActions"): run_effect_size_simulations.FORCE_ACTIONS = True num_to_force = float(sys.argv[7].split(",")[1]) else: num_to_force = 1 #force one action from each arm to avoid nan means bandit_type = "Thompson" bandit_type_prefix = 'BB' if len(sys.argv) > 4: bandit_type = sys.argv[4] if bandit_type == "uniform": bandit_type_prefix = "BU" # Bernoulli rewards, uniform policy reorder_rewards = False softmax_beta = None reordering_fn = None if len(sys.argv) > 7 and not sys.argv[7].startswith("forceActions"): # softmax beta for how to reorder rewards reorder_rewards = True softmax_beta = float(sys.argv[7]) reordering_fn = reorder_samples_in_rewards.order_by_named_column( 'Action1OracleActualReward') if len(sys.argv) > 8: reordering_fn_specifier = sys.argv[8] reordering_fn = reorder_samples_in_rewards.get_reordering_fn( reordering_fn_specifier) prior_params = None if recalculate_bandits: if bandit_type == "uniform": results_dfs_list, results_output_names = run_simulations( num_sims, prob_per_arm, step_sizes, outfile_directory, forceActions=num_to_force, mode='uniform') else: if len(sys.argv) > 5: if sys.argv[5] == "armsHigh": # Arms should be higher than the prior priorProportionOnSuccess = min( prob_per_arm) * PRIOR_PROPORTION_DIFFERENCE elif sys.argv[5] == "armsLow": # Arms should be lower than the prior priorProportionOnSuccess = 1 - ( 1 - max(prob_per_arm)) * PRIOR_PROPORTION_DIFFERENCE else: # Prior should be uniform (in between arms) priorProportionOnSuccess = .5 # Make sure the prior sums to 2, mirroring the successes/failures of uniform prior prior_params = [ priorProportionOnSuccess * 2, 2 - priorProportionOnSuccess * 2 ] print("Prior params: ", prior_params) results_dfs_list, results_output_names = run_simulations( num_sims, prob_per_arm, step_sizes, outfile_directory, prior_params[0], prior_params[1], softmax_beta=softmax_beta, reordering_fn=reordering_fn, forceActions=num_to_force, batch_size=batch_size, burn_in_size=burn_in_size, random_dur=random_dur_m, random_start=random_start_r, epsilon=epsilon) else: results_dfs_list, results_output_names = run_simulations( num_sims, prob_per_arm, step_sizes, outfile_directory, forceActions=num_to_force, batch_size=batch_size, burn_in_size=burn_in_size, epsilon=epsilon) outfile_prefix = outfile_directory + bandit_type_prefix + str(effect_size) if effect_size == 0: # Then include the n in the prefix outfile_prefix += "N" + str(n) for results_df, results_output_name in zip(results_dfs_list, results_output_names): results_df['SampleNumber'] = results_df.index if num_sims <= 2: results_df.to_csv('{}_sims={}_m={}.csv'.format( results_output_name, num_sims, random_dur_m), index=False) #Not saving for now # results_df.to_csv('{}_sims={}_m={}.csv.gz'.format(results_output_name, num_sims, random_dur_m), compression = "gzip", index=False) stats_df = calculate_statistics_from_sims(results_dfs_list, effect_size, num_arms, alpha=0.05) stats_df.to_pickle(outfile_prefix + 'Df_sim={}_m={}_r={}.pkl'.format( num_sims, random_dur_m, random_start_r)) end_time = time.time() print('Execution time = %.6f seconds' % (end_time - start_time))
def calculate_assgn_prob_by_step_size(actions_root, num_samples, num_actions = 2, cached_probs={}, prior = [1,1], binary_rewards = True, \ config = {}, n = None,\ num_sims = None, batch_size = None, no_effect = True, effect_size = None): """ Computes assignment probabilities for ts, sets these to column 'ProbAction{}IsBest' Draws num_samples from a given model to determine assignment probabilties Some unused args from original code """ assert num_actions == 2 read_config.apply_defaults(config) match_num_samples = config[read_config.MATCH_NUM_SAMPLES_HEADER] smoothing_adder = config[read_config.SMOOTHING_ADDER_HEADER] max_weight = config[read_config.MAX_WEIGHT_HEADER] if no_effect: step_sizes = [int(np.ceil(n / 2)), int(n), int(2 * n), int(4 * n)] else: nobs_total = smp.GofChisquarePower().solve_power( effect_size=effect_size, nobs=None, n_bins=(2 - 1) * (2 - 1) + 1, alpha=DESIRED_ALPHA, power=DESIRED_POWER) # print("Calculated nobs for effect size:", nobs_total) n = np.ceil(nobs_total) step_sizes = [np.ceil(n / 2), n, 2 * n, 4 * n] # fig, ax = plt.subplots(1,4) #ax = ax.ravel() i = 0 if os.path.isfile(actions_root + "-ThompsonSamplingAP.csv"): probs_df = pd.read_csv(actions_root + "-ThompsonSamplingAP.csv") print("TS AP dict save found, loading..") return probs_df else: print("no AP save found, creating..") # print("assign prob cache exists at", actions_root + "/withprob") # else: # print("no assing prob cache found, computing assing prob for TS...") probs_dict = {} for num_steps in step_sizes: num_steps = int(num_steps) probs_per_sim_action_1 = [] for sim_count in range(num_sims): # print(sim_count) actions_infile = actions_root + "/tbb_actions_{}_{}.csv".format( int(num_steps), sim_count) actions_df = pd.read_csv(actions_infile, skiprows=1) max_weights = 0 # print(actions_df) if binary_rewards: all_models, cache_keys = create_models_binary( actions_df, prior, num_actions) else: all_models, cache_keys = create_models_normal( actions_df, prior, num_actions) final_model_idx = len(all_models[0]) - 1 - 1 #extra -1 for idx final_models = [models[final_model_idx] for models in all_models] #plural for arms #print("final_model_idx", final_model_idx) if os.path.isdir(actions_root + "/withprob") and 0: # print("assign prob cache exists at", actions_root + "/withprob") actions_df_withprob = pd.read_csv( actions_root + "/withprob/tbb_actions_{}_{}_withprob.csv".format( num_steps, sim_count)) prob = actions_df_withprob[H_ALGO_PROB_BEST_ACTION.format( 1)][0] # print(actions_root + "/withprob/tbb_actions_{}_{}_withprob.csv".format(num_steps, sim_count)) # print("prob", prob) else: # print("no assing prob cache found, computing assing prob for TS...") counts = thompson_policy.estimate_probability_condition_assignment( None, num_samples, num_actions, final_models) probs = [count / num_samples for count in counts] #condition_assigned = int(actions_df.iloc[final_model_idx].loc[H_ALGO_ACTION]) prob = probs[0] # map back to 0 indexing, choose Action1 #print("prob:", prob) actions_df[H_ALGO_PROB_BEST_ACTION.format(1)] = prob actions_df[H_ALGO_PROB_BEST_ACTION.format(2)] = 1 - prob if not os.path.isdir(actions_root + "/withprob"): os.mkdir(actions_root + "/withprob") actions_df.to_csv( actions_root + "/withprob/tbb_actions_{}_{}_withprob.csv".format( num_steps, sim_count), index=False) probs_per_sim_action_1.append(prob) #probs_per_sim_action_1 = np.array(probs_per_sim_action_1) #np.save(, probs_per_sim_action_1) # y, x, _ = ax[i].hist(probs_per_sim_action_1) # ax[i].clear probs_dict[str(num_steps)] = probs_per_sim_action_1 # ax[i].hist(probs_per_sim_action_1) # ax[i].set_xlabel(str(num_steps)) # #print(np.max(np.bincount(np.array(probs_per_sim_action_1)))) # ax[i].set_ylim(0.0, 255) # prop_sims = np.round((np.array(probs_per_sim_action_1) > 0.90).sum()/num_sims + (np.array(probs_per_sim_action_1) < 1-0.90).sum()/num_sims, 3) # ax[i].set_title("prop. > 0.90 \n or < 0.10 = {}".format(str(prop_sims)), fontsize = 8.0) #ax.text(0, 0.1*max_plot_val, 'Mean = %s' % np.round(mean_outcomes[0],3), ha='center', va='bottom', fontweight='bold', fontsize = 16) i = i + 1 #print("proportion of simulations exceeding 0.95 = ", (np.array(probs_per_sim_action_1) > 0.95).sum()/num_sims) #print("proportion of simulations below 0.05 = ", (np.array(probs_per_sim_action_1) < 1-0.95).sum()/num_sims) #print("proportion of simulations exceeding 0.90 or below 0.10", ) # fig.tight_layout() # if no_effect: # title = "Distrubtion of Assignment Probability for Acton 1 Across {} Sims \n Batch Size = {}".format(num_sims, batch_size) # fig.suptitle(title, y = 1.07) # fig.text(0.5, 0.0, "number of participants = n/2, n, 2*n, 4*n for n = {}".format(n), ha='center') # # else: # title = "Distrubtion of Assignment Probability for Acton 1 Across {} Sims \n Batch Size = {} \n Effect Size = {}".format(num_sims, batch_size, effect_size) # fig.suptitle(title, y = 1.16) # fig.text(0.5, 0.0, "number of participants = n/2, n, 2*n, 4*n for n = {} (n is required for 0.8 power)".format(n), ha='center') # #fig.tight_layout() # # # # #fig.tight_layout() # #fig.subplots_adjust(wspace = 0.90) # #fig.tight_layout() # title = title + "n = {}".format(n) # print("saving debug plots!!!") # fig.savefig("plots/" + title + ".png") # #fig.clf() df_ap = pd.DataFrame(probs_dict) print(df_ap) df_ap.to_csv(actions_root + "-ThompsonSamplingAP.csv") return df_ap
def make_stats_row_from_df(simulations_df, include_power, action_count, effect_size=None, alpha=None): '''Calculates output statistics given the data frame simulations_df. If include_power, includes the power calculation. efffect_size and alpha are only required/used if power is calculated ''' #REPLACE cur_data with simulations_df #number of actions! action_count step_size = max(simulations_df.index) + 1 trials_count = len(simulations_df) sim_num = 0 all_rows = [] for idx in range(0, trials_count, step_size): one_sim_df = simulations_df[idx:idx + step_size].copy() sample_sizes = np.array([]) successes = np.array([]) means = np.array([]) np.array([]) np.array([]) for i in range(1, action_count + 1): sample_sizes = np.append(sample_sizes, np.sum(one_sim_df[action_header] == i)) successes = np.append( successes, np.sum(one_sim_df[one_sim_df[action_header] == i] [obs_reward_header])) means = np.append( means, np.mean(one_sim_df[one_sim_df[action_header] == i] [obs_reward_header])) #calculate sample size and mean cur_row = {} for i in range(action_count): cur_row['sample_size_{}'.format(i + 1)] = sample_sizes[i] cur_row['mean_{}'.format(i + 1)] = means[i] if action_count == 2: #SE = sqrt[(P^hat_A*(1-P^hat_A)/N_A + (P^hat_B*(1-P^hat_B)/N_B] SE = np.sqrt(means[0] * (1 - means[0]) / sample_sizes[0] + means[1] * (1 - means[1]) / sample_sizes[1]) wald_type_stat = (means[0] - means[1]) / SE #(P^hat_A - P^hat_b)/SE #print('wald_type_stat:', wald_type_stat) #Two sided, symetric, so compare to 0.05 wald_pval = (1 - scipy.stats.norm.cdf(np.abs(wald_type_stat))) * 2 cur_row['wald_type_stat'] = wald_type_stat cur_row['wald_pval'] = wald_pval #calculate total reward cur_row['total_reward'] = np.sum(one_sim_df[obs_reward_header]) #calculate power cur_row['ratio'] = sample_sizes[0] / sample_sizes[1] if include_power: cur_row['power'] = smp.GofChisquarePower().solve_power( effect_size, nobs=sum(sample_sizes), n_bins=(2 - 1) * (2 - 1) + 1, alpha=alpha) cur_row['actual_es'] = calculate_effect_size(sample_sizes, successes) #calculate chi squared contingency test table = sms.Table(np.stack((successes, sample_sizes - successes)).T) rslt = table.test_nominal_association() cur_row['stat'] = rslt.statistic cur_row['pvalue'] = rslt.pvalue cur_row['df'] = rslt.df # Added to match normal rewards cur_row['statUnequalVar'] = cur_row['stat'] cur_row['pvalueUnequalVar'] = cur_row['pvalue'] cur_row['dfUnequalVar'] = cur_row['df'] cur_row['num_steps'] = max(one_sim_df.index) + 1 cur_row['sim'] = sim_num sim_num += 1 all_rows.append(cur_row) return all_rows
def mainBinaryRewards(): print("Running binary") recalculate_bandits = True num_sims = int(sys.argv[2]) outfile_directory = sys.argv[3] num_arms = 2 # if sys.argv[1] has a comma, just use the result as probability per arm if "," in sys.argv[1]: if sys.argv[1].count(",") == 1: # specifying probability per arm but not effect size prob_per_arm = [ float(armProb) for armProb in sys.argv[1].split(",") ] effect_size = 0 # Note: This will be wrong if arm probs aren't equal! else: # specifying probability per arm as first two arguments, and then effect size numeric_arguments = [ float(armProb) for armProb in sys.argv[1].split(",") ] prob_per_arm = numeric_arguments[: 2] # first two are arm probabilities effect_size = numeric_arguments[2] # final is effect size # We also need to specify n in this case for deciding on step sizes n = int(sys.argv[6]) else: # We just need effect size for this calculation effect_size = float(sys.argv[1]) prob_per_arm = run_effect_size_simulations_beta.get_prob_per_arm_from_effect_size( effect_size) # Assumes we have two arms nobs_total = smp.GofChisquarePower().solve_power(effect_size, n_bins=(2 - 1) * (2 - 1) + 1, alpha=DESIRED_ALPHA, power=DESIRED_POWER) # print("Calculated nobs for effect size:", nobs_total) n = math.ceil(nobs_total) # step_sizes = [math.ceil(n/2), n, 2*n, 4*n] # These differ from the version for normal because in normal, n represented size for one cond rather than overall size if len(sys.argv) > 7 and sys.argv[7].startswith("forceActions"): FORCE_ACTIONS = True num_to_force = float(sys.argv[7].split(",")[1]) else: num_to_force = 0 bandit_type = "Thompson" bandit_type_prefix = 'BB' if len(sys.argv) > 4: bandit_type = sys.argv[4] if bandit_type == "uniform": bandit_type_prefix = "BU" # Bernoulli rewards, uniform policy reorder_rewards = False softmax_beta = None reordering_fn = None if len(sys.argv) > 7 and not sys.argv[7].startswith("forceActions"): # softmax beta for how to reorder rewards reorder_rewards = True try: softmax_beta = float(sys.argv[7]) reordering_fn = reorder_samples_in_rewards.order_by_named_column( 'Action1OracleActualReward') if len(sys.argv) > 8: reordering_fn_specifier = sys.argv[8] reordering_fn = reorder_samples_in_rewards.get_reordering_fn( reordering_fn_specifier) except: print("Parsing error:", sys.exc_info()[0]) # different kind of argument num_samples_before_switch = -1 if len(sys.argv) > 8 and sys.argv[8].startswith("numSamples:"): num_samples_array = sys.argv[8].split(":")[1:] num_samples_before_switch = int(num_samples_array[0]) num_samples_after_switch = int(num_samples_array[1]) if len(sys.argv) > 9 and sys.argv[9].startswith("switchIfNonSig:"): switch_to_best_if_nonsignificant = sys.argv[9].split( ":")[1].lower() == "true" else: switch_to_best_if_nonsignificant = False # n here is what's required for .8 power (number in both conditions) step_sizes_before_switch = [int(round(0.25 * n)), int(round(0.5 * n)), n] #, 2*n] if len(sys.argv) > 10 and sys.argv[10].startswith("multiplier:"): multiplier = int(sys.argv[10].split(":")[1]) print("multiplier:", multiplier) else: multiplier = 5 step_sizes = [(multiplier + 1) * step_size for step_size in step_sizes_before_switch] prior_params = None if recalculate_bandits: if bandit_type == "uniform": if num_samples_before_switch > 0: step_sizes = [ num_samples_before_switch + num_samples_after_switch ] run_simulations_uniform_random_binary( num_sims, prob_per_arm, num_samples_before_switch, num_samples_after_switch, outfile_directory, forceActions=num_to_force, switch_to_best_if_nonsignificant= switch_to_best_if_nonsignificant) else: for num_steps in step_sizes_before_switch: run_simulations_uniform_random_binary( num_sims, prob_per_arm, num_steps, num_steps * multiplier, outfile_directory, forceActions=num_to_force, switch_to_best_if_nonsignificant= switch_to_best_if_nonsignificant) else: if len(sys.argv) > 5: if sys.argv[5] == "armsHigh": # Arms should be higher than the prior priorProportionOnSuccess = min( prob_per_arm ) * run_effect_size_simulations_beta.PRIOR_PROPORTION_DIFFERENCE elif sys.argv[5] == "armsLow": # Arms should be lower than the prior priorProportionOnSuccess = 1 - ( 1 - max(prob_per_arm) ) * run_effect_size_simulations_beta.PRIOR_PROPORTION_DIFFERENCE else: # Prior should be uniform (in between arms) priorProportionOnSuccess = .5 # Make sure the prior sums to 2, mirroring the successes/failures of uniform prior prior_params = [ priorProportionOnSuccess * 2, 2 - priorProportionOnSuccess * 2 ] print("Prior params: ", prior_params) run_effect_size_simulations_beta.run_simulations( num_sims, prob_per_arm, step_sizes, outfile_directory, prior_params[0], prior_params[1], softmax_beta=softmax_beta, reordering_fn=reordering_fn, forceActions=num_to_force) else: run_effect_size_simulations_beta.run_simulations( num_sims, prob_per_arm, step_sizes, outfile_directory, forceActions=num_to_force) outfile_prefix = outfile_directory + bandit_type_prefix + str(effect_size) if effect_size == 0: # Then include the n in the prefix outfile_prefix += "N" + str(n) df = calculate_statistics_from_sims(outfile_directory, num_sims, step_sizes, effect_size, switch_to_best_if_nonsignificant, step_sizes_before_switch, DESIRED_ALPHA, is_binary=True) df.to_pickle(outfile_prefix + 'Df.pkl') df_by_trial = calculate_by_trial_statistics_from_sims( outfile_directory, num_sims, step_sizes, effect_size, DESIRED_ALPHA)