def make_stats_row_from_df(cur_data, include_power, effect_size = None, alpha = None):
'''Calculates output statistics given the data frame cur_data. If include_power, includes the power calculation.
efffect_size and alpha are only required/used if power is calculated
'''
sample_sizes = np.array([np.sum(cur_data[action_header] == i) for i in range(1,3)])
successes = np.array([np.sum(cur_data[cur_data[action_header] == 1][obs_reward_header]),
np.sum(cur_data[cur_data[action_header] == 2][obs_reward_header])])
#calculate sample size and mean
cur_row = {}
sample_size_1 = sample_sizes[0]
sample_size_2 = sample_sizes[1]
cur_row['sample_size_1'] = sample_size_1
cur_row['sample_size_2'] = sample_size_2
mean_1 = np.mean(cur_data[cur_data[action_header] == 1][obs_reward_header])# JN mean for arm 1
mean_2 = np.mean(cur_data[cur_data[action_header] == 2][obs_reward_header])#JN mean for arm 2
cur_row['mean_1'] = mean_1
cur_row['mean_2'] = mean_2
#SE = sqrt[(P^hat_A*(1-P^hat_A)/N_A + (P^hat_B*(1-P^hat_B)/N_B]
SE = np.sqrt(mean_1*(1 - mean_1)/sample_size_1 + mean_2*(1 - mean_2)/sample_size_2)
wald_type_stat = (mean_1 - mean_2)/SE #(P^hat_A - P^hat_b)/SE
#print('wald_type_stat:', wald_type_stat)
wald_pval = (1 - scipy.stats.norm.cdf(np.abs(wald_type_stat)))*2 #Two sided, symetric, so compare to 0.05
cur_row['wald_type_stat'] = wald_type_stat #TODO add in Delta!!!!
cur_row['wald_pval'] = wald_pval
#print("wald_pval", wald_pval)
#calculate total reward
cur_row['total_reward'] = np.sum(cur_data[obs_reward_header])
#calculate power
cur_row['ratio'] = sample_sizes[0] / sample_sizes[1]
if include_power:
cur_row['power'] = smp.GofChisquarePower().solve_power(effect_size, nobs = sum(sample_sizes), n_bins=(2-1)*(2-1) + 1, alpha = alpha)
cur_row['actual_es'] = calculate_effect_size(sample_sizes, successes)
#calculate chi squared contingency test
table = sms.Table(np.stack((successes,sample_sizes - successes)).T)
rslt = table.test_nominal_association()
cur_row['stat'] = rslt.statistic
cur_row['pvalue'] = rslt.pvalue
cur_row['df'] = rslt.df
# Added to match normal rewards
cur_row['statUnequalVar'],cur_row['pvalueUnequalVar'], cur_row['dfUnequalVar'] = cur_row['stat'],cur_row['pvalue'],cur_row['df']
return cur_row
def calc_chisquared_sample_size(
baseline_conversion_rate_percentage: np.float64,
expected_uplift_percentage: np.float64,
power_percentage: np.float64 = 80,
confidence_level_percentage: np.float64 = 95) -> np.float64:
"""Estimates the minimum sample size when the KPI is conversion rate.
Estimated sample size using the Chi-squared test of proportions is the
minimum required for either a Test or a Control group in an A/B test.
Args:
baseline_conversion_rate_percentage: Baseline conversion rate as a
percentage.
expected_uplift_percentage: Expected uplift of the media experiment on the
baseline conversion rate as a percentage.
power_percentage: Statistical power of the Chi-squared test as a percentage.
confidence_level_percentage: Statistical confidence level of the Chi-squared
test as a percentage.
Returns:
sample_size: Estimated minimum sample size required for either a Test or
a Control group.
"""
null_probability = baseline_conversion_rate_percentage / 100
alternative_probability = (null_probability *
(100 + expected_uplift_percentage) / 100)
alpha_proportion = (100 - confidence_level_percentage) / 100
power_proportion = power_percentage / 100
effect_size = gof.chisquare_effectsize(
probs0=[null_probability, 1 - null_probability],
probs1=[alternative_probability, 1 - alternative_probability],
correction=None,
cohen=True,
axis=0)
power_test = power.GofChisquarePower()
sample_size = power_test.solve_power(effect_size=effect_size,
nobs=None,
alpha=alpha_proportion,
power=power_proportion,
n_bins=2)
return np.ceil(sample_size)
def cont_table_power(self):
rows = 5
cols = 2
df = (rows - 1) * (cols - 1)
nbins = df + 1
alpha = 0.05
power = 0.8
st1_scores = self.get_scores('ST1')
st2_scores = self.get_scores('ST2')
col1 = self.bin_three(st1_scores)
col2 = self.bin_three(st2_scores)
n = sum(col1) + sum(col2)
print(n)
ct = np.array([col1, col2]).T
print(ct)
chi2, p, dof, ex = chi2_contingency(ct, correction=False)
es = np.sqrt(chi2 / n * df) # cramer's v
print(es) # medium effect
sample_size = smp.GofChisquarePower().solve_power(es,
n_bins=nbins,
alpha=alpha,
power=power)
print(sample_size)
norm_pow = smp.NormalIndPower().power(0.01, nobs, 0.05, alternative="larger")
norm_pow_R = 0.056869534873146124
#value from R: >pwr.2p.test(h=0.01,n=80,sig.level=0.05,alternative="greater")
print('norm_pow', norm_pow, norm_pow - norm_pow_R)
# Note: negative effect size is same as switching one-sided alternative
# TODO: should I switch to larger/smaller instead of "one-sided" options
norm_pow = smp.NormalIndPower().power(-0.01, nobs, 0.05, alternative="larger")
norm_pow_R = 0.0438089705093578
#value from R: >pwr.2p.test(h=0.01,n=80,sig.level=0.05,alternative="less")
print('norm_pow', norm_pow, norm_pow - norm_pow_R)
#Note: I use n_bins and ddof instead of df
# pwr.chisq.test(w=0.289,df=(4-1),N=100,sig.level=0.05)
chi2_pow = smp.GofChisquarePower().power(0.289, 100, 4, 0.05)
chi2_pow_R = 0.675077657003721
print('chi2_pow', chi2_pow, chi2_pow - chi2_pow_R)
chi2_pow = smp.GofChisquarePower().power(0.01, 100, 4, 0.05)
chi2_pow_R = 0.0505845519208533
print('chi2_pow', chi2_pow, chi2_pow - chi2_pow_R)
chi2_pow = smp.GofChisquarePower().power(2, 100, 4, 0.05)
chi2_pow_R = 1
print('chi2_pow', chi2_pow, chi2_pow - chi2_pow_R)
chi2_pow = smp.GofChisquarePower().power(0.9, 100, 4, 0.05)
chi2_pow_R = 0.999999999919477
print('chi2_pow', chi2_pow, chi2_pow - chi2_pow_R, 'lower precision ?')
def main():
recalculate_bandits = True
#batch_size = 1.0
num_sims = int(sys.argv[2])
outfile_directory = sys.argv[3]
burn_in_size, batch_size = int(
outfile_directory.split("=")[-1].split('-')[0]), int(
outfile_directory.split("=")[-1].split('-')[1])
print("burn_in_size, batch_size", burn_in_size, batch_size)
num_arms = 2
# if sys.argv[1] has a comma, just use the result as probability per arm
if "," in sys.argv[1]:
if sys.argv[1].count(",") == 1:
# specifying probability per arm but not effect size
prob_per_arm = [
float(armProb) for armProb in sys.argv[1].split(",")
]
effect_size = 0 # Note: This will be wrong if arm probs aren't equal!
else:
# specifying probability per arm as first two arguments, and then effect size
numeric_arguments = [
float(armProb) for armProb in sys.argv[1].split(",")
]
prob_per_arm = numeric_arguments[:
2] # first two are arm probabilities
effect_size = numeric_arguments[2] # final is effect size
# We also need to specify n in this case for deciding on step sizes
n = int(sys.argv[6])
else:
# We just need effect size for this calculation
effect_size = float(sys.argv[1].split("-")[0])
center = float(sys.argv[1].split("-")[1])
prob_per_arm = get_prob_per_arm_from_effect_size(effect_size, center)
# Assumes we have two arms
nobs_total = smp.GofChisquarePower().solve_power(effect_size,
n_bins=(2 - 1) *
(2 - 1) + 1,
alpha=DESIRED_ALPHA,
power=DESIRED_POWER)
# print("Calculated nobs for effect size:", nobs_total)
n = math.ceil(nobs_total)
print("center", center)
#step_sizes = [math.ceil(n/2), n, 2*n] # These differ from the version for normal because in normal, n represented size for one cond rather than overall size
step_sizes = [
math.ceil(n / 2), n, 2 * n, 4 * n
] # These differ from the version for normal because in normal, n represented size for one cond rather than overall size
print("prob_per_arm", prob_per_arm)
if len(sys.argv) > 7 and sys.argv[7].startswith("forceActions"):
run_effect_size_simulations.FORCE_ACTIONS = True
num_to_force = float(sys.argv[7].split(",")[1])
else:
num_to_force = 0
bandit_type = "Thompson"
bandit_type_prefix = 'BB'
if len(sys.argv) > 4:
bandit_type = sys.argv[4]
if bandit_type == "uniform":
bandit_type_prefix = "BU" # Bernoulli rewards, uniform policy
reorder_rewards = False
softmax_beta = None
reordering_fn = None
if len(sys.argv) > 7 and not sys.argv[7].startswith("forceActions"):
# softmax beta for how to reorder rewards
reorder_rewards = True
softmax_beta = float(sys.argv[7])
reordering_fn = reorder_samples_in_rewards.order_by_named_column(
'Action1OracleActualReward')
if len(sys.argv) > 8:
reordering_fn_specifier = sys.argv[8]
reordering_fn = reorder_samples_in_rewards.get_reordering_fn(
reordering_fn_specifier)
prior_params = None
if recalculate_bandits:
if bandit_type == "uniform":
run_simulations_uniform_random(num_sims,
prob_per_arm,
step_sizes,
outfile_directory,
forceActions=num_to_force)
else:
if len(sys.argv) > 5:
if sys.argv[5] == "armsHigh":
# Arms should be higher than the prior
priorProportionOnSuccess = min(
prob_per_arm) * PRIOR_PROPORTION_DIFFERENCE
elif sys.argv[5] == "armsLow":
# Arms should be lower than the prior
priorProportionOnSuccess = 1 - (
1 - max(prob_per_arm)) * PRIOR_PROPORTION_DIFFERENCE
else:
# Prior should be uniform (in between arms)
priorProportionOnSuccess = .5
# Make sure the prior sums to 2, mirroring the successes/failures of uniform prior
prior_params = [
priorProportionOnSuccess * 2,
2 - priorProportionOnSuccess * 2
]
print("Prior params: ", prior_params)
run_simulations(num_sims,
prob_per_arm,
step_sizes,
outfile_directory,
prior_params[0],
prior_params[1],
softmax_beta=softmax_beta,
reordering_fn=reordering_fn,
forceActions=num_to_force,
batch_size=batch_size,
burn_in_size=burn_in_size)
else:
run_simulations(num_sims, prob_per_arm, step_sizes, outfile_directory, forceActions = num_to_force, batch_size = batch_size, \
burn_in_size = burn_in_size)
outfile_prefix = outfile_directory + bandit_type_prefix + str(effect_size)
if effect_size == 0:
# Then include the n in the prefix
outfile_prefix += "N" + str(n)
df = calculate_statistics_from_sims(outfile_directory, num_sims,
step_sizes, effect_size, DESIRED_ALPHA)
df.to_pickle(outfile_prefix + 'Df.pkl')
df_by_trial = calculate_by_trial_statistics_from_sims(
outfile_directory, num_sims, step_sizes, effect_size, DESIRED_ALPHA)
df_by_trial.to_pickle(outfile_prefix + 'DfByTrial.pkl')
# Print various stats
summary_text = effect_size_sim_output_viz.print_output_stats(
df,
prob_per_arm,
False,
prior_params=prior_params,
reordering_info=softmax_beta)
with open(outfile_prefix + 'SummaryText.txt', 'w', newline='') as outf:
outf.write(summary_text)
overall_stats_df = effect_size_sim_output_viz.make_overall_stats_df(
df, prob_per_arm, False, effect_size)
overall_stats_df.to_pickle(outfile_prefix + 'OverallStatsDf.pkl')
# Make histogram
hist_figure = effect_size_sim_output_viz.make_hist_of_trials(df)
hist_figure.savefig(outfile_prefix + 'HistOfConditionProportions.pdf',
bbox_inches='tight')
# Make line plot
test_stat_figure = effect_size_sim_output_viz.make_by_trial_graph_of_column(
df_by_trial, 'stat')
test_stat_figure.savefig(outfile_prefix + 'TestStatOverTime.pdf',
bbox_inches='tight')
pvalue_figure = effect_size_sim_output_viz.make_by_trial_graph_of_column(
df_by_trial, 'pvalue')
pvalue_figure.savefig(outfile_prefix + 'PValueOverTime.pdf',
bbox_inches='tight')
# Plot power
power_figure = effect_size_sim_output_viz.plot_power_by_steps(
df_by_trial, DESIRED_ALPHA, DESIRED_POWER)
power_figure.savefig(outfile_prefix + 'PowerOverTime.pdf',
bbox_inches='tight')
#Plot reward
reward_figure = effect_size_sim_output_viz.make_by_trial_graph_of_column(
df_by_trial, 'total_reward')
reward_figure = effect_size_sim_output_viz.add_expected_reward_to_figure(
reward_figure, prob_per_arm, step_sizes)
reward_figure.savefig(outfile_prefix + 'RewardOverTime.pdf',
bbox_inches='tight')
# Plot arm statistics
arm_df_by_trial = create_arm_stats_by_step(outfile_directory, num_sims,
step_sizes[-1], num_arms)
arm_stats_figure = effect_size_sim_output_viz.make_by_trial_arm_statistics(
arm_df_by_trial, num_arms)
arm_stats_figure.savefig(outfile_prefix + 'ArmStats.pdf',
bbox_inches='tight')
def make_stats_row_from_df(simulations_df,
include_power,
action_count,
effect_size=None,
alpha=None):
'''Calculates output statistics given the data frame simulations_df. If include_power, includes the power calculation.
efffect_size and alpha are only required/used if power is calculated
'''
#REPLACE cur_data with simulations_df
#number of actions! action_count
step_size = max(simulations_df.index) + 1 #will be 4n for instance
n = step_size / 4 #assumes set to 4n
n_size_list = [math.ceil(n / 2), int(n), int(2 * n), int(4 * n)]
print("step_size", step_size)
print("n_size_list", n_size_list)
trials_count = len(simulations_df)
print("trials_count", trials_count) #will go to end, so 4n*num_sims
print(simulations_df.columns)
all_rows = []
for n_size in n_size_list:
sim_num = 0
for idx in range(0, trials_count, step_size):
one_sim_df = simulations_df[idx:idx + n_size].copy()
assert (len(one_sim_df) == n_size)
cur_row = {}
prop_exploring_ppd_cuml = np.sum(
one_sim_df[ppd_exp_header]) / n_size #cumulative
# print(one_sim_df["SampleNumber"])
# print("n_size", n_size, exploring_this_n)
exploring_this_n = one_sim_df[
one_sim_df["SampleNumber"] == n_size -
1][ppd_exp_header].iloc[0] #snap shot, conver to idx
sample_sizes = np.array([])
successes = np.array([])
means = np.array([])
np.array([])
np.array([])
for i in range(1, action_count + 1):
sample_sizes = np.append(
sample_sizes, np.sum(one_sim_df[action_header] == i))
successes = np.append(
successes,
np.sum(one_sim_df[one_sim_df[action_header] == i]
[obs_reward_header]))
means = np.append(
means,
np.mean(one_sim_df[one_sim_df[action_header] == i]
[obs_reward_header]))
#calculate sample size and mean
for i in range(action_count):
cur_row['sample_size_{}'.format(i + 1)] = sample_sizes[i]
cur_row['mean_{}'.format(i + 1)] = means[i]
if action_count == 2:
#SE = sqrt[(P^hat_A*(1-P^hat_A)/N_A + (P^hat_B*(1-P^hat_B)/N_B]
SE = np.sqrt(means[0] * (1 - means[0]) / sample_sizes[0] +
means[1] * (1 - means[1]) / sample_sizes[1])
wald_type_stat = (means[0] -
means[1]) / SE #(P^hat_A - P^hat_b)/SE
#print('wald_type_stat:', wald_type_stat)
#Two sided, symetric, so compare to 0.05
wald_pval = (1 -
scipy.stats.norm.cdf(np.abs(wald_type_stat))) * 2
cur_row['wald_type_stat'] = wald_type_stat
cur_row['wald_pval'] = wald_pval
#calculate total reward
cur_row['total_reward'] = np.sum(one_sim_df[obs_reward_header])
#calculate power
cur_row['ratio'] = sample_sizes[0] / sample_sizes[1]
if include_power:
cur_row['power'] = smp.GofChisquarePower().solve_power(
effect_size,
nobs=sum(sample_sizes),
n_bins=(2 - 1) * (2 - 1) + 1,
alpha=alpha)
cur_row['actual_es'] = calculate_effect_size(
sample_sizes, successes)
#calculate chi squared contingency test
table = sms.Table(
np.stack((successes, sample_sizes - successes)).T)
rslt = table.test_nominal_association()
cur_row['stat'] = rslt.statistic
cur_row['pvalue'] = rslt.pvalue
cur_row['df'] = rslt.df
# Added to match normal rewards
cur_row['statUnequalVar'] = cur_row['stat']
cur_row['pvalueUnequalVar'] = cur_row['pvalue']
cur_row['dfUnequalVar'] = cur_row['df']
cur_row['num_steps'] = max(one_sim_df.index) + 1
# cur_row['num_steps'] = n_size
cur_row['sim'] = sim_num
cur_row["prop_exploring_ppd_cuml"] = prop_exploring_ppd_cuml
cur_row["exploring_ppd_at_this_n"] = exploring_this_n
all_rows.append(cur_row)
sim_num += 1
return all_rows
def main():
start_time = time.time()
outfile_directory = sys.argv[3]
random_dur_m = 0
random_start_r = 0
recalculate_bandits = True
num_arms = 2
#batch_size = 1.0
# if len(sys.argv) > 5:
# epsilon = float(sys.argv[5])
# print("epsilon", epsilon)
if "epsilon=" in outfile_directory:
print(outfile_directory)
epsilon = float(
outfile_directory.split("epsilon=")[-1].split("/")[0].strip("="))
#c = float(outfile_directory.split("=c=")[-1])
print("epsilon", epsilon)
num_sims = int(sys.argv[2])
burn_in_size, batch_size = int(
outfile_directory.split("=")[-1].split('-')[0]), int(
outfile_directory.split("=")[-1].split('-')[1])
print("burn_in_size, batch_size", burn_in_size, batch_size)
# if sys.argv[1] has a comma, just use the result as probability per arm
if "," in sys.argv[1]:
if sys.argv[1].count(",") == 1:
# specifying probability per arm but not effect size
prob_per_arm = [
float(armProb) for armProb in sys.argv[1].split(",")
]
effect_size = 0 # Note: This will be wrong if arm probs aren't equal!
else:
# specifying probability per arm as first two arguments, and then effect size
numeric_arguments = [
float(armProb) for armProb in sys.argv[1].split(",")
]
prob_per_arm = numeric_arguments[:
2] # first two are arm probabilities
effect_size = numeric_arguments[2] # final is effect size
# We also need to specify n in this case for deciding on step sizes
n = int(sys.argv[6])
else:
# We just need effect size for this calculation
effect_size = float(sys.argv[1].split("-")[0])
center = float(sys.argv[1].split("-")[1])
prob_per_arm = get_prob_per_arm_from_effect_size(effect_size, center)
# Assumes we have two arms
nobs_total = smp.GofChisquarePower().solve_power(effect_size,
n_bins=(2 - 1) *
(2 - 1) + 1,
alpha=DESIRED_ALPHA,
power=DESIRED_POWER)
#print("Calculated nobs for effect size:", nobs_total)
n = math.ceil(nobs_total)
print("center", center)
'''
These differ from the version for normal because in normal,
n represented size for one cond rather than overall size
step_sizes = [math.ceil(n/2), n, 2*n, 4*n]
'''
#Arghavan: Just run simulation for n
step_sizes = [4 * n]
print("prob_per_arm", prob_per_arm)
if len(sys.argv) > 7 and sys.argv[7].startswith("forceActions"):
run_effect_size_simulations.FORCE_ACTIONS = True
num_to_force = float(sys.argv[7].split(",")[1])
else:
num_to_force = 1 #force one action from each arm to avoid nan means
bandit_type = "Thompson"
bandit_type_prefix = 'BB'
if len(sys.argv) > 4:
bandit_type = sys.argv[4]
if bandit_type == "uniform":
bandit_type_prefix = "BU" # Bernoulli rewards, uniform policy
reorder_rewards = False
softmax_beta = None
reordering_fn = None
if len(sys.argv) > 7 and not sys.argv[7].startswith("forceActions"):
# softmax beta for how to reorder rewards
reorder_rewards = True
softmax_beta = float(sys.argv[7])
reordering_fn = reorder_samples_in_rewards.order_by_named_column(
'Action1OracleActualReward')
if len(sys.argv) > 8:
reordering_fn_specifier = sys.argv[8]
reordering_fn = reorder_samples_in_rewards.get_reordering_fn(
reordering_fn_specifier)
prior_params = None
if recalculate_bandits:
if bandit_type == "uniform":
results_dfs_list, results_output_names = run_simulations(
num_sims,
prob_per_arm,
step_sizes,
outfile_directory,
forceActions=num_to_force,
mode='uniform')
else:
if len(sys.argv) > 5:
if sys.argv[5] == "armsHigh":
# Arms should be higher than the prior
priorProportionOnSuccess = min(
prob_per_arm) * PRIOR_PROPORTION_DIFFERENCE
elif sys.argv[5] == "armsLow":
# Arms should be lower than the prior
priorProportionOnSuccess = 1 - (
1 - max(prob_per_arm)) * PRIOR_PROPORTION_DIFFERENCE
else:
# Prior should be uniform (in between arms)
priorProportionOnSuccess = .5
# Make sure the prior sums to 2, mirroring the successes/failures of uniform prior
prior_params = [
priorProportionOnSuccess * 2,
2 - priorProportionOnSuccess * 2
]
print("Prior params: ", prior_params)
results_dfs_list, results_output_names = run_simulations(
num_sims,
prob_per_arm,
step_sizes,
outfile_directory,
prior_params[0],
prior_params[1],
softmax_beta=softmax_beta,
reordering_fn=reordering_fn,
forceActions=num_to_force,
batch_size=batch_size,
burn_in_size=burn_in_size,
random_dur=random_dur_m,
random_start=random_start_r,
epsilon=epsilon)
else:
results_dfs_list, results_output_names = run_simulations(
num_sims,
prob_per_arm,
step_sizes,
outfile_directory,
forceActions=num_to_force,
batch_size=batch_size,
burn_in_size=burn_in_size,
epsilon=epsilon)
outfile_prefix = outfile_directory + bandit_type_prefix + str(effect_size)
if effect_size == 0:
# Then include the n in the prefix
outfile_prefix += "N" + str(n)
for results_df, results_output_name in zip(results_dfs_list,
results_output_names):
results_df['SampleNumber'] = results_df.index
if num_sims <= 2:
results_df.to_csv('{}_sims={}_m={}.csv'.format(
results_output_name, num_sims, random_dur_m),
index=False)
#Not saving for now
# results_df.to_csv('{}_sims={}_m={}.csv.gz'.format(results_output_name, num_sims, random_dur_m), compression = "gzip", index=False)
stats_df = calculate_statistics_from_sims(results_dfs_list,
effect_size,
num_arms,
alpha=0.05)
stats_df.to_pickle(outfile_prefix + 'Df_sim={}_m={}_r={}.pkl'.format(
num_sims, random_dur_m, random_start_r))
end_time = time.time()
print('Execution time = %.6f seconds' % (end_time - start_time))
def calculate_assgn_prob_by_step_size(actions_root, num_samples, num_actions = 2, cached_probs={},
prior = [1,1], binary_rewards = True, \
config = {}, n = None,\
num_sims = None, batch_size = None, no_effect = True, effect_size = None):
"""
Computes assignment probabilities for ts, sets these to column 'ProbAction{}IsBest'
Draws num_samples from a given model to determine assignment probabilties
Some unused args from original code
"""
assert num_actions == 2
read_config.apply_defaults(config)
match_num_samples = config[read_config.MATCH_NUM_SAMPLES_HEADER]
smoothing_adder = config[read_config.SMOOTHING_ADDER_HEADER]
max_weight = config[read_config.MAX_WEIGHT_HEADER]
if no_effect:
step_sizes = [int(np.ceil(n / 2)), int(n), int(2 * n), int(4 * n)]
else:
nobs_total = smp.GofChisquarePower().solve_power(
effect_size=effect_size,
nobs=None,
n_bins=(2 - 1) * (2 - 1) + 1,
alpha=DESIRED_ALPHA,
power=DESIRED_POWER)
# print("Calculated nobs for effect size:", nobs_total)
n = np.ceil(nobs_total)
step_sizes = [np.ceil(n / 2), n, 2 * n, 4 * n]
# fig, ax = plt.subplots(1,4)
#ax = ax.ravel()
i = 0
if os.path.isfile(actions_root + "-ThompsonSamplingAP.csv"):
probs_df = pd.read_csv(actions_root + "-ThompsonSamplingAP.csv")
print("TS AP dict save found, loading..")
return probs_df
else:
print("no AP save found, creating..")
# print("assign prob cache exists at", actions_root + "/withprob")
# else:
# print("no assing prob cache found, computing assing prob for TS...")
probs_dict = {}
for num_steps in step_sizes:
num_steps = int(num_steps)
probs_per_sim_action_1 = []
for sim_count in range(num_sims):
# print(sim_count)
actions_infile = actions_root + "/tbb_actions_{}_{}.csv".format(
int(num_steps), sim_count)
actions_df = pd.read_csv(actions_infile, skiprows=1)
max_weights = 0
# print(actions_df)
if binary_rewards:
all_models, cache_keys = create_models_binary(
actions_df, prior, num_actions)
else:
all_models, cache_keys = create_models_normal(
actions_df, prior, num_actions)
final_model_idx = len(all_models[0]) - 1 - 1 #extra -1 for idx
final_models = [models[final_model_idx]
for models in all_models] #plural for arms
#print("final_model_idx", final_model_idx)
if os.path.isdir(actions_root + "/withprob") and 0:
# print("assign prob cache exists at", actions_root + "/withprob")
actions_df_withprob = pd.read_csv(
actions_root +
"/withprob/tbb_actions_{}_{}_withprob.csv".format(
num_steps, sim_count))
prob = actions_df_withprob[H_ALGO_PROB_BEST_ACTION.format(
1)][0]
# print(actions_root + "/withprob/tbb_actions_{}_{}_withprob.csv".format(num_steps, sim_count))
# print("prob", prob)
else:
# print("no assing prob cache found, computing assing prob for TS...")
counts = thompson_policy.estimate_probability_condition_assignment(
None, num_samples, num_actions, final_models)
probs = [count / num_samples for count in counts]
#condition_assigned = int(actions_df.iloc[final_model_idx].loc[H_ALGO_ACTION])
prob = probs[0] # map back to 0 indexing, choose Action1
#print("prob:", prob)
actions_df[H_ALGO_PROB_BEST_ACTION.format(1)] = prob
actions_df[H_ALGO_PROB_BEST_ACTION.format(2)] = 1 - prob
if not os.path.isdir(actions_root + "/withprob"):
os.mkdir(actions_root + "/withprob")
actions_df.to_csv(
actions_root +
"/withprob/tbb_actions_{}_{}_withprob.csv".format(
num_steps, sim_count),
index=False)
probs_per_sim_action_1.append(prob)
#probs_per_sim_action_1 = np.array(probs_per_sim_action_1)
#np.save(, probs_per_sim_action_1)
# y, x, _ = ax[i].hist(probs_per_sim_action_1)
# ax[i].clear
probs_dict[str(num_steps)] = probs_per_sim_action_1
# ax[i].hist(probs_per_sim_action_1)
# ax[i].set_xlabel(str(num_steps))
# #print(np.max(np.bincount(np.array(probs_per_sim_action_1))))
# ax[i].set_ylim(0.0, 255)
# prop_sims = np.round((np.array(probs_per_sim_action_1) > 0.90).sum()/num_sims + (np.array(probs_per_sim_action_1) < 1-0.90).sum()/num_sims, 3)
# ax[i].set_title("prop. > 0.90 \n or < 0.10 = {}".format(str(prop_sims)), fontsize = 8.0)
#ax.text(0, 0.1*max_plot_val, 'Mean = %s' % np.round(mean_outcomes[0],3), ha='center', va='bottom', fontweight='bold', fontsize = 16)
i = i + 1
#print("proportion of simulations exceeding 0.95 = ", (np.array(probs_per_sim_action_1) > 0.95).sum()/num_sims)
#print("proportion of simulations below 0.05 = ", (np.array(probs_per_sim_action_1) < 1-0.95).sum()/num_sims)
#print("proportion of simulations exceeding 0.90 or below 0.10", )
# fig.tight_layout()
# if no_effect:
# title = "Distrubtion of Assignment Probability for Acton 1 Across {} Sims \n Batch Size = {}".format(num_sims, batch_size)
# fig.suptitle(title, y = 1.07)
# fig.text(0.5, 0.0, "number of participants = n/2, n, 2*n, 4*n for n = {}".format(n), ha='center')
#
# else:
# title = "Distrubtion of Assignment Probability for Acton 1 Across {} Sims \n Batch Size = {} \n Effect Size = {}".format(num_sims, batch_size, effect_size)
# fig.suptitle(title, y = 1.16)
# fig.text(0.5, 0.0, "number of participants = n/2, n, 2*n, 4*n for n = {} (n is required for 0.8 power)".format(n), ha='center')
# #fig.tight_layout()
#
#
#
# #fig.tight_layout()
# #fig.subplots_adjust(wspace = 0.90)
# #fig.tight_layout()
# title = title + "n = {}".format(n)
# print("saving debug plots!!!")
# fig.savefig("plots/" + title + ".png")
#
#fig.clf()
df_ap = pd.DataFrame(probs_dict)
print(df_ap)
df_ap.to_csv(actions_root + "-ThompsonSamplingAP.csv")
return df_ap
def make_stats_row_from_df(simulations_df,
include_power,
action_count,
effect_size=None,
alpha=None):
'''Calculates output statistics given the data frame simulations_df. If include_power, includes the power calculation.
efffect_size and alpha are only required/used if power is calculated
'''
#REPLACE cur_data with simulations_df
#number of actions! action_count
step_size = max(simulations_df.index) + 1
trials_count = len(simulations_df)
sim_num = 0
all_rows = []
for idx in range(0, trials_count, step_size):
one_sim_df = simulations_df[idx:idx + step_size].copy()
sample_sizes = np.array([])
successes = np.array([])
means = np.array([])
np.array([])
np.array([])
for i in range(1, action_count + 1):
sample_sizes = np.append(sample_sizes,
np.sum(one_sim_df[action_header] == i))
successes = np.append(
successes,
np.sum(one_sim_df[one_sim_df[action_header] == i]
[obs_reward_header]))
means = np.append(
means,
np.mean(one_sim_df[one_sim_df[action_header] == i]
[obs_reward_header]))
#calculate sample size and mean
cur_row = {}
for i in range(action_count):
cur_row['sample_size_{}'.format(i + 1)] = sample_sizes[i]
cur_row['mean_{}'.format(i + 1)] = means[i]
if action_count == 2:
#SE = sqrt[(P^hat_A*(1-P^hat_A)/N_A + (P^hat_B*(1-P^hat_B)/N_B]
SE = np.sqrt(means[0] * (1 - means[0]) / sample_sizes[0] +
means[1] * (1 - means[1]) / sample_sizes[1])
wald_type_stat = (means[0] -
means[1]) / SE #(P^hat_A - P^hat_b)/SE
#print('wald_type_stat:', wald_type_stat)
#Two sided, symetric, so compare to 0.05
wald_pval = (1 - scipy.stats.norm.cdf(np.abs(wald_type_stat))) * 2
cur_row['wald_type_stat'] = wald_type_stat
cur_row['wald_pval'] = wald_pval
#calculate total reward
cur_row['total_reward'] = np.sum(one_sim_df[obs_reward_header])
#calculate power
cur_row['ratio'] = sample_sizes[0] / sample_sizes[1]
if include_power:
cur_row['power'] = smp.GofChisquarePower().solve_power(
effect_size,
nobs=sum(sample_sizes),
n_bins=(2 - 1) * (2 - 1) + 1,
alpha=alpha)
cur_row['actual_es'] = calculate_effect_size(sample_sizes, successes)
#calculate chi squared contingency test
table = sms.Table(np.stack((successes, sample_sizes - successes)).T)
rslt = table.test_nominal_association()
cur_row['stat'] = rslt.statistic
cur_row['pvalue'] = rslt.pvalue
cur_row['df'] = rslt.df
# Added to match normal rewards
cur_row['statUnequalVar'] = cur_row['stat']
cur_row['pvalueUnequalVar'] = cur_row['pvalue']
cur_row['dfUnequalVar'] = cur_row['df']
cur_row['num_steps'] = max(one_sim_df.index) + 1
cur_row['sim'] = sim_num
sim_num += 1
all_rows.append(cur_row)
return all_rows
def mainBinaryRewards():
print("Running binary")
recalculate_bandits = True
num_sims = int(sys.argv[2])
outfile_directory = sys.argv[3]
num_arms = 2
# if sys.argv[1] has a comma, just use the result as probability per arm
if "," in sys.argv[1]:
if sys.argv[1].count(",") == 1:
# specifying probability per arm but not effect size
prob_per_arm = [
float(armProb) for armProb in sys.argv[1].split(",")
]
effect_size = 0 # Note: This will be wrong if arm probs aren't equal!
else:
# specifying probability per arm as first two arguments, and then effect size
numeric_arguments = [
float(armProb) for armProb in sys.argv[1].split(",")
]
prob_per_arm = numeric_arguments[:
2] # first two are arm probabilities
effect_size = numeric_arguments[2] # final is effect size
# We also need to specify n in this case for deciding on step sizes
n = int(sys.argv[6])
else:
# We just need effect size for this calculation
effect_size = float(sys.argv[1])
prob_per_arm = run_effect_size_simulations_beta.get_prob_per_arm_from_effect_size(
effect_size)
# Assumes we have two arms
nobs_total = smp.GofChisquarePower().solve_power(effect_size,
n_bins=(2 - 1) *
(2 - 1) + 1,
alpha=DESIRED_ALPHA,
power=DESIRED_POWER)
# print("Calculated nobs for effect size:", nobs_total)
n = math.ceil(nobs_total)
# step_sizes = [math.ceil(n/2), n, 2*n, 4*n] # These differ from the version for normal because in normal, n represented size for one cond rather than overall size
if len(sys.argv) > 7 and sys.argv[7].startswith("forceActions"):
FORCE_ACTIONS = True
num_to_force = float(sys.argv[7].split(",")[1])
else:
num_to_force = 0
bandit_type = "Thompson"
bandit_type_prefix = 'BB'
if len(sys.argv) > 4:
bandit_type = sys.argv[4]
if bandit_type == "uniform":
bandit_type_prefix = "BU" # Bernoulli rewards, uniform policy
reorder_rewards = False
softmax_beta = None
reordering_fn = None
if len(sys.argv) > 7 and not sys.argv[7].startswith("forceActions"):
# softmax beta for how to reorder rewards
reorder_rewards = True
try:
softmax_beta = float(sys.argv[7])
reordering_fn = reorder_samples_in_rewards.order_by_named_column(
'Action1OracleActualReward')
if len(sys.argv) > 8:
reordering_fn_specifier = sys.argv[8]
reordering_fn = reorder_samples_in_rewards.get_reordering_fn(
reordering_fn_specifier)
except:
print("Parsing error:",
sys.exc_info()[0]) # different kind of argument
num_samples_before_switch = -1
if len(sys.argv) > 8 and sys.argv[8].startswith("numSamples:"):
num_samples_array = sys.argv[8].split(":")[1:]
num_samples_before_switch = int(num_samples_array[0])
num_samples_after_switch = int(num_samples_array[1])
if len(sys.argv) > 9 and sys.argv[9].startswith("switchIfNonSig:"):
switch_to_best_if_nonsignificant = sys.argv[9].split(
":")[1].lower() == "true"
else:
switch_to_best_if_nonsignificant = False
# n here is what's required for .8 power (number in both conditions)
step_sizes_before_switch = [int(round(0.25 * n)),
int(round(0.5 * n)), n] #, 2*n]
if len(sys.argv) > 10 and sys.argv[10].startswith("multiplier:"):
multiplier = int(sys.argv[10].split(":")[1])
print("multiplier:", multiplier)
else:
multiplier = 5
step_sizes = [(multiplier + 1) * step_size
for step_size in step_sizes_before_switch]
prior_params = None
if recalculate_bandits:
if bandit_type == "uniform":
if num_samples_before_switch > 0:
step_sizes = [
num_samples_before_switch + num_samples_after_switch
]
run_simulations_uniform_random_binary(
num_sims,
prob_per_arm,
num_samples_before_switch,
num_samples_after_switch,
outfile_directory,
forceActions=num_to_force,
switch_to_best_if_nonsignificant=
switch_to_best_if_nonsignificant)
else:
for num_steps in step_sizes_before_switch:
run_simulations_uniform_random_binary(
num_sims,
prob_per_arm,
num_steps,
num_steps * multiplier,
outfile_directory,
forceActions=num_to_force,
switch_to_best_if_nonsignificant=
switch_to_best_if_nonsignificant)
else:
if len(sys.argv) > 5:
if sys.argv[5] == "armsHigh":
# Arms should be higher than the prior
priorProportionOnSuccess = min(
prob_per_arm
) * run_effect_size_simulations_beta.PRIOR_PROPORTION_DIFFERENCE
elif sys.argv[5] == "armsLow":
# Arms should be lower than the prior
priorProportionOnSuccess = 1 - (
1 - max(prob_per_arm)
) * run_effect_size_simulations_beta.PRIOR_PROPORTION_DIFFERENCE
else:
# Prior should be uniform (in between arms)
priorProportionOnSuccess = .5
# Make sure the prior sums to 2, mirroring the successes/failures of uniform prior
prior_params = [
priorProportionOnSuccess * 2,
2 - priorProportionOnSuccess * 2
]
print("Prior params: ", prior_params)
run_effect_size_simulations_beta.run_simulations(
num_sims,
prob_per_arm,
step_sizes,
outfile_directory,
prior_params[0],
prior_params[1],
softmax_beta=softmax_beta,
reordering_fn=reordering_fn,
forceActions=num_to_force)
else:
run_effect_size_simulations_beta.run_simulations(
num_sims,
prob_per_arm,
step_sizes,
outfile_directory,
forceActions=num_to_force)
outfile_prefix = outfile_directory + bandit_type_prefix + str(effect_size)
if effect_size == 0:
# Then include the n in the prefix
outfile_prefix += "N" + str(n)
df = calculate_statistics_from_sims(outfile_directory,
num_sims,
step_sizes,
effect_size,
switch_to_best_if_nonsignificant,
step_sizes_before_switch,
DESIRED_ALPHA,
is_binary=True)
df.to_pickle(outfile_prefix + 'Df.pkl')
df_by_trial = calculate_by_trial_statistics_from_sims(
outfile_directory, num_sims, step_sizes, effect_size, DESIRED_ALPHA)
