def solve(problem, cloning_param, mutation):
final_data = []
final_problem = problem(dim)
final_mutation = mutation(mut_pb, 20)
for x in range(repetitions):
algorithm = CloneAlg(
problem=final_problem,
population_size=100,
offspring_population_size=100,
mutation=final_mutation,
cloning_param=cloning_param,
termination_criterion=StoppingByEvaluations(max_evaluations=5000))
data = []
dataobserver = DataObserver(1.0, data)
algorithm.observable.register(observer=dataobserver)
algorithm.run()
final_data.append(data)
trans_list = np.array(final_data).T.tolist()
fig = plt.figure(figsize=(10, 7))
ax = fig.add_axes([0, 0, 1, 1])
bp = ax.boxplot(trans_list)
plt.title(
"Problem: {0} benchmark, dim: {1}, cloning_param: {2}, mutation: {3}".
format(final_problem.get_name(), dim, algorithm.get_cloning_param(),
final_mutation.get_name()))
plt.show()
# Kruskal-Wallis and Dunn tests
print(stats.kruskal(trans_list[0], trans_list[1], trans_list[-1]))
sp.posthoc_dunn([trans_list[0], trans_list[1], trans_list[-1]],
p_adjust='holm')
def plot_statistical_test(df, column, group_names, title, savefile):
"""
Performs a dunn test, plots and saves results
:param df: the dataframe with the results, we assume the existence of column 'config'
:param column: column in the dataframe representing the random variable to be tested
:param group_names: map or function mapping configs to strings (group names), can be a pd.DataSeries
:param title: title for the plot
:param savefile: pdf or png file to save the results to
:return:
"""
dunnRes = sp.posthoc_dunn(df,
group_col='config',
val_col=column,
p_adjust='holm')
# reorder configs
# dunnRes = dunnRes.loc[cols, cols] # cols must be a permutation of the group names
dunnRes = dunnRes.rename(group_names, axis=0).rename(group_names, axis=1)
columnOrder = dunnRes.columns
# melt for plotting
dunnRes = dunnRes.reset_index().melt(id_vars='index',
value_vars=dunnRes.columns)
dunnRes['stat'] = dunnRes.value.map(stat_range_values)
# plot
p = ggplot(dunnRes, aes('index', 'variable', fill='factor(stat)'))\
+ geom_tile(aes(width=.95, height=.95)) + ggtitle(title)\
+ scale_fill_manual(values=stat_colors, name='p value') \
+ scale_x_discrete(limits=columnOrder) \
+ scale_y_discrete(limits=columnOrder) \
+ labs(x='', y='', title=title) \
+ theme(axis_text_x=element_text(rotation=45, hjust=1))
ggsave(p, savefile, dpi=300)
def kruskal_wallis(df, cat_col, num_col, notebook=True):
"""
Perform kruskal wallis test between the selected columns of the given dataframe.
Columns need to be continuous
:param df:
:param cat_group:
:param num_col:
:return:
"""
variables = []
for idx, cat_group in df.groupby(cat_col):
# NAN values not included in computation
variables.append(cat_group[num_col][cat_group[num_col].notnull()])
kruskal_h, kruskal_p = ss.kruskal(*variables)
if print:
print(f"H-Value: {kruskal_h}, p-Value: {kruskal_p}")
output = f"\tTest: Kruskal-Wallis\n"
output += f"\tH-Value: {kruskal_h}, p-Value: {kruskal_p}\n"
if kruskal_p <= 0.05:
output += "\tSignificance found \n"
output += "\tPost-Hoc Tests: Dunns with Bonferonni Correction\n"
# Remove nan values
selector = df[cat_col].notnull() & df[num_col].notnull()
posthoc_data = df[selector]
posthoc_result = sp.posthoc_dunn(posthoc_data,
num_col,
cat_col,
p_adjust="bonferroni")
if notebook:
print(posthoc_result)
output += str(posthoc_result)
output += "\n"
return output
def my_kruskal_3samp(x1, x2, x3, tag1, tag2, tag3):
n1 = len(x1)
n2 = len(x2)
n3 = len(x3)
sample_sizes = (n1, n2, n3)
mdn1, mdn2, mdn3 = np.median(x1), np.median(x2), np.median(x3)
lqr1, hqr1 = np.quantile(x1, 0.25), np.quantile(x1, 0.75)
lqr2, hqr2 = np.quantile(x2, 0.25), np.quantile(x2, 0.75)
lqr3, hqr3 = np.quantile(x3, 0.25), np.quantile(x3, 0.75)
kruskal_results = kruskal(x1, x2, x3)
Hstat, kruskal_p = kruskal_results
dunntable = posthoc_tests.posthoc_dunn([x1, x2, x3],
p_adjust='fdr_bh').to_numpy()
p12 = dunntable[0, 1]
p23 = dunntable[1, 2]
p13 = dunntable[0, 2]
dunn_pvals = (p12, p23, p13)
ktxt = r'Kruskal-Wallis, %s $(n=%d)$ vs %s $(n=%d)$ vs %s $(n=%d)$, $H_{(2)}=%0.2f, p=%s$'% \
(tag1, n1, tag2, n2, tag3, n3, Hstat, p2str(kruskal_p))
dunntxt12 = r'%s vs %s, $p=%s$' % (tag1, tag2, p2str(p12))
dunntxt23 = r'%s vs %s, $p=%s$' % (tag2, tag3, p2str(p23))
dunntxt13 = r'%s vs %s, $p=%s$' % (tag1, tag3, p2str(p13))
dunntxt = r'\textit{post hoc} Dunn’s test with Benjamini-Hochberg correction: %s, %s, %s' % (
dunntxt12, dunntxt23, dunntxt13)
txt = ktxt + '; ' + dunntxt
descrips = ((mdn1, lqr1, hqr1), (mdn2, lqr2, hqr2), (mdn3, lqr3, hqr3))
return kruskal_p, dunn_pvals, sample_sizes, descrips, txt
def dunn_posthoc_test(df, dependent_variable, between):
"""dunn_posthoc tests with bonferroni multiple correction"""
return sp.posthoc_dunn(
df,
val_col=dependent_variable,
group_col=between,
p_adjust="bonferroni",
)
def create_var_stats(data,
var_list,
type,
test_func,
out_prefix,
force_unique=True,
plot_dunn=True):
out_csv = config.OUT_EVALS_DIR + "/" + out_prefix
out_png = config.OUT_PLOT_DIR + "/" + out_prefix
res = {}
stat = ['s', 'p']
res_posthoc = None
plot_list = []
statistics_dct = {}
statistics_idx_col = "var"
statistics_dct[statistics_idx_col] = []
statistics_dct['n'] = []
column_names = [statistics_idx_col, 'n'] + stat
for var in var_list:
res[var] = {}
dct = data.to_single_type_dict(type, var, force_unique)
lst = data.to_single_type_list(type, var, force_unique)
# statistic test
res[var][stat[0]], res[var][stat[1]] = test_func(
*lst) # '*' splits list into list of arguments
if 'n' not in res[var]:
res[var]['n'] = len(lst[0])
statistics_dct[statistics_idx_col].append(var)
statistics_dct['n'].append(len(lst[0]))
for st in stat:
if st not in statistics_dct:
statistics_dct[st] = []
val = utils.format_number(res[var][st], config.PRINT_PRECISION)
statistics_dct[st].append(val)
# dunn
if plot_dunn:
frame = pd.DataFrame.from_dict(dct)
frame = frame.melt(var_name='groups', value_name='values')
res_posthoc = sp.posthoc_dunn(frame,
val_col='values',
group_col='groups',
p_adjust='bonferroni')
path = out_png + "heat_" + var + "_" + sp.posthoc_dunn.__name__ + "." + config.OUT_PNG_EXT
plot_list.append(plots.saveHeatMapPlot(res_posthoc, path))
# statistics to df
statistics_df = pd.DataFrame(statistics_dct, columns=column_names)
statistics_df.set_index(statistics_idx_col)
out_var_path = out_csv + "question_groups_" + type.name + "_" + test_func.__name__ + "." + config.OUT_CSV_EXT
statistics_df.to_csv(out_var_path, index=False)
return res, plot_list
def compare_languages(list_lang_results, feat_name, list_corpus_names, p_threshold=0.05, dict_path="feature_dict_checked.json"):
list_lang_no_nan = list()
corpus_names = OrderedDict()
for lang_values, corpus_name in zip(list_lang_results, list_corpus_names):
no_nans = lang_values[lang_values.notnull()]
if len(no_nans) > 0:
list_lang_no_nan.append(no_nans)
corpus_names[corpus_name] = len(no_nans)
if len(list_lang_no_nan) == 0:
return 0,0
# scale_of_measurement = check_scale(list_lang_no_nan[0])
scale_of_measurement = check_scale_from_dict(dict_path, "paired", feat_name)
# # 0: nominal, 1: ordinal, 2: interval, 3: ratio
normal_distribution = check_distribution(list_lang_no_nan, p_threshold=0.05)
variance_homogeneity = check_variance_homogeneity(list_lang_no_nan, p_threshold=0.05)
if scale_of_measurement >= 2 and normal_distribution and variance_homogeneity:
# does the language affect the value of the feature? Does simplifications for each langauge work similar?
t_value, p_value = stats.f_oneway(*list_lang_no_nan)
return ("ANOVA", p_value)
#if p_value <= p_threshold:
# posthoc: which langauges are different?
# stats.multicomp.pairwise_tukeyhsd
# if two different ones found, use pearson to get effect size
#effect_size = stats.pearsonr(complex_values, simple_values)[0]
# effec_size = cohend(complex_values, simple_values)
elif scale_of_measurement >= 1:
try:
h_statistic, p_value = stats.kruskal(*list_lang_no_nan)
except ValueError:
return 0,0
if 0 < p_value <= p_threshold:
if p_value <= 0.01:
p_value = "p<=.01"
elif p_value <= 0.05:
p_value = "p<=.05"
else:
p_value = "p>0.05"
output_list = list()
posthoc_frame = scikit_posthocs.posthoc_dunn(list_lang_no_nan, p_adjust="holm")
posthoc_frame_z = posthoc_dunn_z(list_lang_no_nan)
for i, name_corpus_col in zip(posthoc_frame.columns.values, corpus_names.keys()):
for n, name_corpus_row in zip(range(0, len(posthoc_frame)), corpus_names.keys()):
if p_threshold >= posthoc_frame.iloc[n][i] > 0:
effect_size = abs(posthoc_frame_z.iloc[n][i]/math.sqrt(corpus_names[name_corpus_col]+corpus_names[name_corpus_row]))
if effect_size >= 0.1:
output_list.append(["Kruskal ", p_value, "effectsize", str(round(effect_size, 4)),
"h", str(round(h_statistic, 4)), "z", str(round(posthoc_frame_z.iloc[n][i],4)), name_corpus_col, name_corpus_row])
#pos_col = list(corpus_names.keys()).index(name_corpus_col)
#pos_row = list(corpus_names.keys()).index(name_corpus_row)
#effect_size_pearson = stats.pearsonr(list_lang_no_nan[pos_col], list_lang_no_nan[pos_row])[0]
# print(len(list_lang_no_nan[pos_col]), len(list_lang_no_nan[pos_row]))
# effect_size_cohen = cohend(list_lang_no_nan[pos_col], list_lang_no_nan[pos_row])
return output_list
else:
return 0, 0
else:
return 0, 0
def posthoc_dunn(self, data_df, predictor, response):
data_df[response] = pd.to_numeric(data_df[response], errors='coerce')
dunn_df = scikit.posthoc_dunn(data_df,
val_col=response,
group_col=predictor,
p_adjust='bonferroni')
dunn_df = dunn_df.applymap(lambda x: np.nan if x < 0 else x)
dunn_df = dunn_df.round(2)
dunn_df = dunn_df.applymap(lambda x: "< 0.001" if x < 0.001 else x)
return dunn_df
def evaluate(crosssover_algo, problem):
alldata = []
series = []
for x in range(10):
algorithm = GeneticAlgorithm(
problem=problem,
population_size=100,
offspring_population_size=100,
mutation=PolynomialMutation(1.0 / problem.number_of_variables, 20.0),
crossover=crosssover_algo,
selection=BinaryTournamentSelection(),
termination_criterion=StoppingByEvaluations(max_evaluations=500000)
)
data = []
dataobserver = DataObserver(1.0, data)
algorithm.observable.register(observer=dataobserver)
algorithm.run()
result = algorithm.get_result().objectives[0]
series.append(result)
alldata.append(data)
numpy_array = np.array(alldata)
transpose = numpy_array.T
transpose_list = transpose.tolist()
fig = plt.figure(figsize =(60, 42))
ax = fig.add_axes([0, 0, 1, 1])
bp = ax.boxplot(transpose_list)
plt.show()
print(stats.kruskal(transpose_list[0],transpose_list[1],transpose_list[-1]))
series = [series]
print(np.average(series))
sp.posthoc_dunn([transpose_list[0],transpose_list[1],transpose_list[-1]], p_adjust = 'holm')
def post_hoc_df(df, Y_col, X_col, posthoc="tukey", alpha=0.05):
"""
Returns a df with pairwise comparisons with reject column calculated according to alpha
TODO: Add more posthoc tests to this function
"""
if posthoc == "Statsmodels_tukey":
comp = multi.MultiComparison(df[Y_col], df['comb'])
results = comp.tukeyhsd(alpha=alpha)
results = pd.DataFrame(data=results._results_table.data[1:],
columns=results._results_table.data[0])
if posthoc == "dunn":
results = scikit_results_munger(
sp.posthoc_dunn(df,
val_col=Y_col,
group_col=X_col,
p_adjust='holm'), alpha)
if posthoc == "tukey":
results = scikit_results_munger(
sp.posthoc_tukey(df, val_col=Y_col, group_col=X_col), alpha)
return results
def process_instance(list_files):
##Load all the algoritms in a different class..
Wins = [0] * len(list_files)
Losts = [0] * len(list_files)
Ties = [0] * len(list_files)
data = list()
for i, val1 in enumerate(list_files):
data.append(loadtxt(val1))
##Run kruskal test, a p-value > 0.05 accepts the null hypothesis
T, p = ss.kruskal(*data)
##null hypothesis is rejected, try a post-hoc analyzes with a correction procedure
if p < alpha:
ptable = sp.posthoc_dunn(data, p_adjust='hommel')
for i in range(0, len(list_files)):
val1 = list_files[i]
for j in range(0, i):
val2 = list_files[j]
if ptable.iat[i, j] > alpha:
Ties[i] += 1
Ties[j] += 1
else:
if check_means(loadtxt(val1), loadtxt(val2)) == 1:
Wins[i] += 1
Losts[j] += 1
elif check_means(loadtxt(val1), loadtxt(val2)) == -1:
Wins[j] += 1
Losts[i] += 1
else:
Ties[i] += 1
Ties[j] += 1
else:
Ties[0:len(list_files)] += ones(
len(list_files)) * (len(list_files) - 1)
for i, val1 in enumerate(list_files):
sys.stdout.write(" " + str(Wins[i]) + " " + str(Losts[i]) + " " +
str(Ties[i]))
sys.stdout.flush()
print("")
def kruskal_posthoc_tests(benchmark_snapshot_df):
"""Returns p-value tables for various Kruskal posthoc tests.
Results should considered only if Kruskal test rejects null hypothesis.
"""
common_args = {
'a': benchmark_snapshot_df,
'group_col': 'fuzzer',
'val_col': 'edges_covered',
'sort': True
}
p_adjust = 'holm'
posthoc_tests = {}
posthoc_tests['mann_whitney'] = sp.posthoc_mannwhitney(**common_args,
p_adjust=p_adjust)
posthoc_tests['conover'] = sp.posthoc_conover(**common_args,
p_adjust=p_adjust)
posthoc_tests['wilcoxon'] = sp.posthoc_wilcoxon(**common_args,
p_adjust=p_adjust)
posthoc_tests['dunn'] = sp.posthoc_dunn(**common_args, p_adjust=p_adjust)
posthoc_tests['nemenyi'] = sp.posthoc_nemenyi(**common_args)
return posthoc_tests
stats_tests = pd.DataFrame(columns=['variable', 'test_type', 'p_value'])
posthoc_tests = {}
test_df = learned_2.loc[learned_2['lab_number'].isin(
['Lab 1', 'Lab 2', 'Lab 3', 'Lab 4', 'Lab 5', 'Lab 6', 'Lab 7'])]
for i, var in enumerate(['perf_easy', 'reaction_time', 'threshold', 'bias']):
_, normal = stats.normaltest(test_df[var])
if normal < 0.05:
test_type = 'kruskal'
test = stats.kruskal(*[
group[var].values for name, group in test_df.groupby('lab_number')
])
if test[1] < 0.05: # Proceed to posthocs
posthoc = sp.posthoc_dunn(test_df,
val_col=var,
group_col='lab_number')
else:
posthoc = np.nan
else:
test_type = 'anova'
test = stats.f_oneway(*[
group[var].values for name, group in test_df.groupby('lab_number')
])
if test[1] < 0.05:
posthoc = sp.posthoc_tukey(test_df,
val_col=var,
group_col='lab_number')
else:
posthoc = np.nan
for df_col in df_cols_of_interest:
lst_lst_data = [
df[df_col][df['logo_class'] == c_of_i] for c_of_i in lst_ces_of_int
]
print(f'Stat test {df_col}:')
print(stats.kruskal(*lst_lst_data))
# results all seem to have significantly different distributions
# %%
# Conduct Dunn-Bonferroni post-hoc
import scikit_posthocs as sp
dunn_b_p_val = pd.Series()
for df_col in df_cols_of_interest:
dunn_b_p_val[df_col] = sp.posthoc_dunn(df,
val_col=df_col,
group_col='logo_class',
p_adjust='bonferroni')
print(f'Dunn-Bonferroni results for {df_col}:')
print(dunn_b_p_val[df_col]['jp_morgan'])
# get means and std for reporting
df_img_results = pd.DataFrame()
for df_col in df_cols_of_interest:
print(f'{df_col}:')
dunn_b_p_val[df_col]
(df_img_results[f'mean_{df_col}'], df_img_results[f'std_{df_col}'], _, _,
_) = calc_class_stats(df, df_col, lst_ces_of_int, lst_ces_of_int_dis)
print(df_img_results)
df_img_results
# Notes:
cd = pd.read_csv("/Users/ameya/ICSE2020Data/CommonSuperType.csv")
c = pd.read_csv("/Users/ameya/ICSE2020Data/Composition.csv")
# nr = pd.read_csv("/Users/ameya/NoRelationship.csv")
# plt.figure(figsize=(11,2))
h_c = stats.ranksums(h.iloc[:, 0], c.iloc[:, 0]) #,alternative='less')
h_cd = stats.ranksums(h.iloc[:, 0], cd.iloc[:, 0]) #,alternative='less')
c_cd = stats.ranksums(cd.iloc[:, 0], c.iloc[:, 0]) #,alternative='less')
print("p value for hierarchy and composition", h_c)
print("p value for hierarchy and common st", h_cd)
print("p value for common st and composition", c_cd)
print(stats.f_oneway(h.iloc[:, 0], cd.iloc[:, 0], c.iloc[:, 0]))
f = (stats.kruskal(h.iloc[:, 0], cd.iloc[:, 0], c.iloc[:, 0]))
f1 = posthocs.posthoc_dunn([h.iloc[:, 0], c.iloc[:, 0], cd.iloc[:, 0]])
print("H: ", h.iloc[:, 0].mean(), h.iloc[:, 0].median())
print("CD: ", cd.iloc[:, 0].mean(), cd.iloc[:, 0].median())
print("C: ", c.iloc[:, 0].mean(), c.iloc[:, 0].median())
sns.set(font_scale=1.5)
fig, axes = plt.subplots(figsize=(8, 3))
#, sns.color_pallete("muted")
r = axes.violinplot(dataset=[h.iloc[:, 0], cd.iloc[:, 0], c.iloc[:, 0]],
showmeans=True,
showmedians=True)
r['cmeans'].set_color('r')
r['cmedians'].set_color('g')
for i in range(len(H)):
F.append([np.mean(T[i]), np.std(T[i]), np.median(T[i]), np.min(T[i]), np.max(T[i])])
F = pd.DataFrame(F, index=H)
F.to_csv('stats_analysis/resume/resume_'+ids[c_ids], sep='\t', header=['mean', 'std', 'median', 'min', 'max'], float_format='%.6f')
## P_VALUE
if not os.path.exists('stats_analysis/p_value/'):
os.makedirs('stats_analysis/p_value/')
kruskal_values = []
for c_ids in range(len(ids)):
T = pd.read_csv(folder+ids[c_ids], sep='\t')
H = T.columns.values
T = np.array(T).T
dunn_test = sp.posthoc_dunn(T)
kruskal_values.append(stats.kruskal(*T)[1])
#for i in range(len(H)):
#dunn_test[i][i] = 1
dunn_test = pd.DataFrame(dunn_test, index=H)
dunn_test.to_csv('stats_analysis/p_value/dunn_test_'+ids[c_ids], sep='\t', header=H, float_format='%.6f')
kruskal_values = pd.DataFrame(np.array(kruskal_values).T, index=ids)
kruskal_values.to_csv('stats_analysis/p_value/kruskal.txt', sep='\t', header=['p_value'], float_format='%.6f')
## BOXPLOT
if not os.path.exists('stats_analysis/boxplot/'):
os.makedirs('stats_analysis/boxplot/')
def significance(T, P):
def plot_consistency_within_day(res, start, end, shuffle, pretraining,
figure_path):
d = list(zip(start, end))
res_temp = filter.filter_days_per_mouse(res, d)
if pretraining:
res_temp = filter.filter(res_temp, {'odor_valence': ['PT CS+']})
else:
res_temp = filter.filter(res_temp, {'odor_valence': ['CS+', 'CS-']})
corr_res = _correlation(res_temp)
corr_res.pop('data')
analysis.add_naive_learned(corr_res, start, end, '0', '1')
res_ = reduce.new_filter_reduce(
corr_res,
filter_keys=['mouse', 'odor_standard', 'training_day'],
reduce_key='consistency_corrcoef')
res_.pop('consistency_corrcoef_sem')
filter.assign_composite(res_, loop_keys=['training_day', 'odor_valence'])
if shuffle:
s = '_shuffled'
else:
s = ''
ax_args_copy = ax_args.copy()
ax_args_copy.update({
'xlim': [-.5, 2.5],
'ylim': [0, .55],
'yticks': np.arange(0, 1.1, .1)
})
swarm_args_copy = swarm_args.copy()
if pretraining:
swarm_args_copy.update({'palette': ['gray', 'orange', 'green', 'red']})
else:
swarm_args_copy.update({'palette': ['gray', 'gray', 'green', 'red']})
ix = res_['training_day_odor_valence'] == '1_PT CS+'
res_['training_day_odor_valence'][ix] = '1_APT CS+'
plot.plot_results(res_,
x_key='training_day_odor_valence',
y_key='consistency_corrcoef',
path=figure_path,
plot_args=swarm_args_copy,
plot_function=sns.stripplot,
ax_args=ax_args_copy,
reuse=False,
save=False,
sort=True,
name_str=s)
summary = reduce.new_filter_reduce(res_,
filter_keys='training_day_odor_valence',
reduce_key='consistency_corrcoef')
plot.plot_results(summary,
x_key='training_day_odor_valence',
y_key='consistency_corrcoef',
error_key='consistency_corrcoef_sem',
colors='black',
path=figure_path,
plot_args=error_args,
plot_function=plt.errorbar,
save=True,
reuse=True,
legend=False,
name_str=s)
print(summary['consistency_corrcoef'])
ix_a = res_['training_day_odor_valence'] == '0_CS+'
ix_b = res_['training_day_odor_valence'] == '0_CS-'
ix_c = res_['training_day_odor_valence'] == '1_CS+'
ix_d = res_['training_day_odor_valence'] == '1_CS-'
a = res_['consistency_corrcoef'][ix_a]
b = res_['consistency_corrcoef'][ix_b]
c = res_['consistency_corrcoef'][ix_c]
d = res_['consistency_corrcoef'][ix_d]
from scipy.stats import ranksums, wilcoxon, kruskal
import scikit_posthocs
print(kruskal(a, b, c))
x = scikit_posthocs.posthoc_dunn(a=[a, b, c, d], p_adjust=None)
print(x)
'perf_easy', 'threshold_l', 'threshold_r', 'threshold_n', 'bias_l',
'bias_r', 'bias_n'
]):
# Remove any animals with NaNs
test_fits = biased_fits[biased_fits[var].notnull()]
# Test for normality
_, normal = stats.normaltest(test_fits[var])
if normal < 0.05:
test_type = 'kruskal'
test = stats.kruskal(
*[group[var].values for name, group in test_fits.groupby('lab')])
if test[1] < 0.05: # Proceed to posthocs
posthoc = sp.posthoc_dunn(test_fits, val_col=var, group_col='lab')
else:
posthoc = np.nan
else:
test_type = 'anova'
test = stats.f_oneway(
*[group[var].values for name, group in test_fits.groupby('lab')])
if test[1] < 0.05:
posthoc = sp.posthoc_tukey(test_fits, val_col=var, group_col='lab')
else:
posthoc = np.nan
posthoc_tests['posthoc_' + str(var)] = posthoc
stats_tests.loc[i, 'variable'] = var
stats_tests.loc[i, 'test_type'] = test_type
stats_tests.loc[i, 'p_value'] = test[1]
def rest():
df = q1_median_q3_rep_wide
pops = ["pdc", "dc-cd11b", "dc-cd8a"]
stats_l = []
for stat, (popa, popb) in product(["Q1", "median", "Q3"],
product(pops, pops)):
print(stat, popa, popb)
popa = "hsc"
popb = "pdc"
stat = "median"
mw_u, pvalue = scipy.stats.mannwhitneyu(
[0.8, 0.81, 0.79],
[0.4, 0.39, 0.41],
# df.query("Population == @popa")[stat].to_numpy(),
# df.query("Population == @popb")[stat].to_numpy(),
use_continuity=True,
alternative="two-sided",
)
pvalue
stats_l.append([stat, popa, popb, mw_u, pvalue])
stats_df = pd.DataFrame(stats_l).set_axis(
["stat", "popA", "popB", "U", "pvalue"], axis=1)
kruskal_format_means = pd.pivot(
q1_median_q3_rep_wide.query("Population in @pops"),
index="Population",
columns="Replicate",
values="mean",
)
import scikit_posthocs
stat, p_value = scipy.stats.kruskal(
*[kruskal_format_means.loc[pop].to_numpy() for pop in pops], )
dunn_res_df = scikit_posthocs.posthoc_dunn(
kruskal_format_means.to_numpy(),
p_adjust='fdr_bh',
sort=True,
)
stat, pvalue = scipy.stats.f_oneway(
*[kruskal_format_means.loc[pop].to_numpy() for pop in pops], )
import statsmodels
df = kruskal_format_means.stack().reset_index()
kruskal_format_means
res = statsmodels.stats.multicomp.pairwise_tukeyhsd(
df[0], df['Population'].to_numpy(), alpha=0.05)
res.pvalues
res.summary()
# wilcox.test(c(0.8, 0.79, 0.81), c(0.4, 0.39, 0.41), paired=F, exact=F)
plot_pops = ["pdc", "dc-cd8a", "dc-cd11b"]
results_dir = "/icgc/dkfzlsdf/analysis/hs_ontogeny/notebook-data/gNs4xcMJscaLLwlt"
point_plot_quartiles_png = results_dir + "/point-plot-quartiles.png"
q1_median_q3_rep_wide
ggplot_data = (
q1_median_q3_rep_long.query("Population in @plot_pops").sort_values(
"value",
ascending=False,
).groupby(["Population", "stat"]).apply(
lambda df: df.assign(group_order=np.arange(1, df.shape[0] + 1))))
g = (gg.ggplot(ggplot_data) + gg.aes_string(
x="Population", y="value", group="group_order", color="stat") +
gg.geom_point(position=gg.position_dodge(width=0.5), size=1) +
mh_rpy2_styling.gg_paper_theme + gg.labs(y='Methylation (%)', x=''))
a = 3
rpy2_utils.image_png2(g, (ut.cm(6), ut.cm(6)))
ut.save_and_display(
g,
png_path=point_plot_quartiles_png,
# additional_formats=tuple(),
height=ut.cm(6),
width=ut.cm(6),
)
q1_median_q3_rep_wide
g = (
gg.ggplot(
q1_median_q3_rep_wide.query("Population in @plot_pops").assign(
sample=lambda df: df["Population"].astype(str) + df[
"Replicate"].astype(str))) + gg.geom_boxplot(
gg.aes_string(
x="Population",
fill="Population",
group="sample",
lower="Q1",
upper="Q3",
middle="median",
ymin="min1",
ymax="max99",
# position=gg.position_dodge(width=0.5),
),
stat="identity",
)
# + mh_rpy2_styling.gg_paper_theme
+ gg.theme(axis_text_x=gg.element_text(angle=90, hjust=1)) +
gg.scale_fill_brewer(guide=False))
a = 3
ut.save_and_display(
g,
png_path=point_plot_quartiles_png,
additional_formats=tuple(),
height=ut.cm(6),
width=ut.cm(7),
)
# image_png2(g, (ut.cm(12), ut.cm(12)))
beta_values.loc[:, ("hsc", "1")]
# %% Statistics
stats_tests = pd.DataFrame(columns=['variable', 'test_type', 'p_value'])
posthoc_tests = {}
for i, var in enumerate(['threshold_l', 'threshold_r', 'lapselow_l', 'lapselow_r', 'lapsehigh_l',
'lapsehigh_r', 'bias_l', 'bias_r']):
_, normal = stats.normaltest(biased_fits[var])
if normal < 0.05:
test_type = 'kruskal'
test = stats.kruskal(*[group[var].values
for name, group in biased_fits.groupby('lab')])
if test[1] < 0.05: # Proceed to posthocs
posthoc = sp.posthoc_dunn(biased_fits, val_col=var, group_col='lab')
else:
posthoc = np.nan
else:
test_type = 'anova'
test = stats.f_oneway(*[group[var].values
for name, group in biased_fits.groupby('lab')])
if test[1] < 0.05:
posthoc = sp.posthoc_tukey(biased_fits, val_col=var, group_col='lab')
else:
posthoc = np.nan
posthoc_tests['posthoc_'+str(var)] = posthoc
stats_tests.loc[i, 'variable'] = var
stats_tests.loc[i, 'test_type'] = test_type
stats_tests.loc[i, 'p_value'] = test[1]
# nr = pd.read_csv("/Users/ameya/NoRelationship.csv")
# plt.figure(figsize=(11,2))
sns.set(font_scale=1.5)
print("PRmax", pr.iloc[:, 0].median(), pr.iloc[:, 0].mean())
print("Pckgmax", pck.iloc[:, 0].median(), pck.iloc[:, 0].mean())
print("Clsmax", c.iloc[:, 0].median(), c.iloc[:, 0].mean())
# print("MMean", m.median(), m.mean())
print("Smean", s.iloc[:, 0].median(), s.iloc[:, 0].mean())
print(c.values.shape)
ii = np.concatenate([c.values, pr.values], axis=0)
print(stats.kruskal(pr.iloc[:, 0], pck.iloc[:, 0], c.iloc[:, 0], s.iloc[:, 0]))
f = posthocs.posthoc_dunn(
[pr.iloc[:, 0], c.iloc[:, 0], s.iloc[:, 0], pck.iloc[:, 0]])
# f.compare_dunn(0,1)
print(f.shape)
print(f.values)
print("Project-Package",
stats.mannwhitneyu(pr.iloc[:, 0], pck.iloc[:, 0], alternative='greater'))
print("Project-class",
stats.mannwhitneyu(pr.iloc[:, 0], c.iloc[:, 0], alternative='greater'))
print("project-selective",
stats.mannwhitneyu(pr.iloc[:, 0], s.iloc[:, 0], alternative='greater'))
print("package-selective",
stats.mannwhitneyu(pck.iloc[:, 0], s.iloc[:, 0], alternative='greater'))
print("class-selective",
stats.mannwhitneyu(s.iloc[:, 0], c.iloc[:, 0], alternative='greater'))
#print(mannwhitneyu(m_l, u_l)[1])
"""
all_data["Gaussian"] = ga_l
all_data["Hessian"] = h_l
all_data["Laplacian"] = la_l
all_data["Ilastik"] = il_l
all_data["MitoSegNet"] = m_l
all_data["Finetuned\nFiji U-Net"] = u_l_pt
"""
p_val = kruskal(ga_l, h_l, la_l, il_l, m_l, u_l_pt)
print(p_val)
dt = posthoc_dunn([ga_l, h_l, la_l, il_l, m_l, u_l_pt])
#dt = posthoc_dunn(all_data, val_col="MitoSegNet", group_col="Ilastik")
print(dt)
dt.to_excel("dc_posthoc.xlsx")
print("\n")
print(mannwhitneyu(u_l_pt, h_l)[1])
print(mannwhitneyu(u_l_pt, la_l)[1])
print(mannwhitneyu(u_l_pt, ga_l)[1])
print(mannwhitneyu(u_l_pt, il_l)[1])
print("\n")
# pooled standard deviation for calculation of effect size (cohen's d)
def peakPosKruskal(features,
csv="top100-high-level-complete.csv",
popSize=10,
plot=False,
dateRange=(1950, 2010)):
features.append(" peakPos")
features = list(set(features))
data = getTimeSeries(features, csv)
data = data[pd.notnull(data[' peakPos'])]
alpha = 0.001
data[' peakPos'] = data[' peakPos'].map(
lambda a: math.floor(a / popSize) * popSize)
print("\n")
count = 0
for i in features:
# mod = sm.OLS(i+" ~ tags:originaldate", data=data)
print("| ", i, ": ")
clean_data = data[pd.notnull(data[i])]
unique_pos = pd.unique(clean_data[' peakPos'])
unique_pos.sort()
l = pd.DataFrame()
l2 = []
for j in unique_pos:
t = clean_data[clean_data[' peakPos'] == j]
t = t[i]
l[j] = t
l2.append(t)
kruskal = stats.kruskal(*l2)
post_hoc = ph.posthoc_dunn(clean_data, group_col=" peakPos", val_col=i)
dunn_significant = set()
for y1 in range(post_hoc.shape[0]):
for y2 in range(y1):
if post_hoc.iloc[y1, y2] < alpha and y1 != y2:
dunn_significant.add((post_hoc.index.values[y1],
post_hoc.columns.values[y2]))
print("| ", kruskal)
print("| significant pairs: ", dunn_significant)
print("| ", post_hoc < alpha)
print("|___________________\n")
if plot:
f, axes = mpl.subplots(nrows=1, ncols=1, constrained_layout=True)
for j in unique_pos:
clean_data.boxplot(i, by=" peakPos", ax=axes)
# clean_data.boxplot(i,by=" peakPos")
axes.set_title("")
axes.set_xlabel("")
# for j in unique_pos:
# sns.distplot(clean_data[clean_data[' peakPos']==j].loc[:,i],label=str(j),bins=40,norm_hist=True,ax=axes[0])
# axes[0].legend(loc="upper right")
# axes[0].set_xlabel("")
mpl.suptitle(i)
mpl.gcf().subplots_adjust(bottom=0.25)
dunnStr = "P<" + str(alpha) + " (Dunn post-hoc): "
c = len(dunnStr)
for i in str(dunn_significant):
c += 1
dunnStr += i
if (c % 55 == 0):
dunnStr += "\n"
mpl.xlabel(dunnStr, fontsize=14)
count += 1
mpl.show()
# Stats
stats_tests = pd.DataFrame(columns=['variable', 'test_type', 'p_value'])
posthoc_tests = {}
for i, var in enumerate(
['perf_easy', 'reaction_time', 'threshold', 'bias', 'n_trials']):
_, normal = stats.normaltest(learned[var])
if normal < 0.05:
test_type = 'kruskal'
test = stats.kruskal(*[
group[var].values for name, group in learned.groupby('lab_number')
])
if test[1] < 0.05: # Proceed to posthocs
posthoc = sp.posthoc_dunn(learned,
val_col=var,
group_col='lab_number')
else:
posthoc = np.nan
else:
test_type = 'anova'
test = stats.f_oneway(*[
group[var].values for name, group in learned.groupby('lab_number')
])
if test[1] < 0.05:
posthoc = sp.posthoc_tukey(learned,
val_col=var,
group_col='lab_number')
else:
posthoc = np.nan
def kruskal_wallis(self):
dir_name = self.dir_name_stats
file_name = dir_name + 'kruskal wallis results 0'
i = 0
while True:
if not (os.path.isfile(file_name)):
f = open(file_name, 'w')
break
else:
i += 1
file_name = file_name[:-1] + str(i)
lst_groups = [
self.df_for_box[col].dropna().values
for col in self.treatments_to_plot
]
k_w_res = stats.kruskal(*lst_groups)
h_val, p_val = k_w_res
f.write('h value = {0}\n'.format(h_val))
f.write('p value = {0}\n'.format(p_val))
print(
"\npreformed kruskal wallis test, results in stats -> kruskal_wallis_results "
+ str(i))
self.signature = self.get_signature()
if p_val < 0.05:
print(
'the results are significant, preform post hoc tests? (y for yes)'
)
# inp_krus = 'y'
inp_krus = input()
if inp_krus == 'y':
post_hoc_df = sp.posthoc_dunn(lst_groups, p_adjust='holm')
cols = post_hoc_df.columns
post_hoc_df = post_hoc_df.rename(columns={
cols[i]: self.treatments_to_plot[i]
for i in range(len(cols))
})
post_hoc_df = post_hoc_df.rename(index={
cols[i]: self.treatments_to_plot[i]
for i in range(len(cols))
})
post_hoc_df.to_excel(dir_name + "post hoc res " + str(i) +
'.xlsx')
groups = {}
critical_val = 0.05
means = [np.nanmean(l) for l in lst_groups]
keys = list(self.treatments_to_plot)
keys_sorted = [
x for _, x in sorted(zip(means, keys), reverse=True)
]
seen_groups = []
for i in range(len(keys_sorted)):
g = keys_sorted[i]
seen_groups.append(g)
groups[g] = []
groups_to_check = list(set(keys_sorted) - set(seen_groups))
values = post_hoc_df[g][groups_to_check]
for col in values.index:
if values[
col] > critical_val and col not in seen_groups:
groups[g].append(col)
letters = {}
c = 'a'
seen_groups = []
for k in keys_sorted:
if (len(groups[k]) > 0 and not already_contained(groups[k], letters)) or\
(len(groups[k]) == 0 and k not in seen_groups):
letters[c] = [k] + groups[k]
c = chr(ord(c) + 1)
seen_groups += groups[k]
groups_letter = {}
for t in self.treatments_to_plot:
groups_letter[t] = [c for c in letters if t in letters[c]]
print(
"\npreformed dunn test, results in stats -> post hoc res")
print(
'treatments are in the same group if they are not statistically different (p val is more than 0.05)'
)
print(
'\nshow results from post hoc test on graph? (y for yes)')
f.write("results are significant!!\n\n")
f.write(
'post hoc results (treatments are in the same group are not statistically different):\n'
)
f.write("\n".join([
"{0:<5s}:{1}".format(key, str(val))
for key, val in letters.items()
]))
inp = input()
# inp = 'y'
if inp == 'y':
self.groups_letter_without = groups_letter
self.kruskal = True
else:
print('results are not significant')
f.close()
with open('accuracies.json') as json_file:
accuracies_json = json.load(json_file)
accs = [accuracies_json[k] for k in accuracies_json]
len(accs)
from scipy.stats import friedmanchisquare
friedmanchisquare(*accs)
#get_ipython().system(' pip install scikit-posthocs')
from scikit_posthocs import posthoc_dunn
p_values = posthoc_dunn(a=accs, p_adjust='holm', sort=True)
print('Post hocs dunn p-values: ')
p_values.head()
# In[141]:
pvalues = p_values.values
classifiers = accs.copy()
best = 0
best_acc = np.mean(accs[0])
for i in range(96):
acc_i = np.mean(accs[i])
rejected = np.where(pvalues[i,:]<0.05)[0]
rejected = [x for x in rejected if x != i]
def getKWmultiComp(data, labels, verbose=False):
pVals = sp.posthoc_dunn(data, p_adjust='bonferroni')
if verbose:
print np.hstack((np.transpose([0]+labels).reshape(4,1),np.vstack((labels,pVals))))
return [pVals[1,0], pVals[2,0], pVals[2,1]]
def dunn(df: pd.DataFrame, group_variables, variable):
df = prepare_data_frame(df, group_variables, variable)
return sp.posthoc_dunn(df, val_col='values', group_col='groups', p_adjust='holm')
training_time = pd.DataFrame(columns=['sessions'], data=ses.groupby('subject_nickname').size())
training_time['trials'] = ses.groupby('subject_nickname').sum()
training_time['lab'] = ses.groupby('subject_nickname')['institution_short'].apply(list).str[0]
# Change lab name into lab number
training_time['lab_number'] = training_time.lab.map(institution_map()[0])
training_time = training_time.sort_values('lab_number')
# statistics
# Test normality
_, normal = stats.normaltest(training_time['sessions'])
if normal < 0.05:
kruskal = stats.kruskal(*[group['sessions'].values
for name, group in training_time.groupby('lab')])
if kruskal[1] < 0.05: # Proceed to posthocs
posthoc = sp.posthoc_dunn(training_time, val_col='sessions',
group_col='lab_number')
else:
anova = stats.f_oneway(*[group['sessions'].values
for name, group in training_time.groupby('lab')])
if anova[1] < 0.05:
posthoc = sp.posthoc_tukey(training_time, val_col='sessions',
group_col='lab_number')
# %% PLOT
# Set figure style and color palette
use_palette = [[0.6, 0.6, 0.6]] * len(np.unique(training_time['lab']))
use_palette = use_palette + [[1, 1, 0.2]]
lab_colors = group_colors()
print(
mannwhitneyu(all_data["Laplacian"], all_data["Finetuned\nFiji U-Net"])[1])
print(mannwhitneyu(all_data["Ilastik"], all_data["Finetuned\nFiji U-Net"])[1])
#print(np.median(all_data["Hessian"]), np.median(all_data["Ilastik"]), np.median(all_data["MitoSegNet"]))
#print(np.average(all_data["Hessian"]), np.average(all_data["Ilastik"]), np.average(all_data["MitoSegNet"]))
print(
kruskal(all_data["Gaussian"].tolist(), all_data["Hessian"].tolist(),
all_data["Laplacian"].tolist(), all_data["Ilastik"].tolist(),
all_data["MitoSegNet"].tolist(),
all_data["Finetuned\nFiji U-Net"].tolist()))
dt = posthoc_dunn([
all_data["Gaussian"].tolist(), all_data["Hessian"].tolist(),
all_data["Laplacian"].tolist(), all_data["Ilastik"].tolist(),
all_data["MitoSegNet"].tolist(),
all_data["Finetuned\nFiji U-Net"].tolist()
])
dt.to_excel("ed_posthoc.xlsx")
# pooled standard deviation for calculation of effect size (cohen's d)
def cohens_d(data1, data2):
p_std = np.sqrt(
((len(data1) - 1) * np.var(data1) +
(len(data2) - 1) * np.var(data2)) / (len(data1) + len(data2) - 2))
cohens_d = np.abs(np.average(data1) - np.average(data2)) / p_std
