def experiment_analysis(path, quants, trials=list(range(30)), plots=True,
threshold=0.95, filename=None, size=None):
"""Prints statistical tests and makes plots for experiment one.
Args:
path: where the trials in CSV are
plots: whether to make plots or not
"""
# read the data in
data = util.read_trials_from_csv(path, trials)
# FILTER OUT TRIALS WHERE RNN DID NOT LEARN
remove_bad_trials(data, quants, threshold=threshold)
# get convergence points per quantifier
convergence_points = get_convergence_points(data, quants, threshold)
if plots:
# make plots
# make_boxplots(convergence_points, quants)
# make_barplots(convergence_points, quants)
make_plot(data, quants, ylim=(0.8, 1), threshold=threshold,
filename=filename, size=size)
print(stats.ttest_rel(convergence_points[quants[0]],
convergence_points[quants[1]]))
final_n = 50
final_means = [[forward_means(data[trial][quant + '_accuracy'].values,
window_size=final_n)[-final_n]
for quant in quants] for trial in data]
print('final means: {} - {}'.format(quants[0], quants[1]))
print(stats.ttest_rel(
[means[0] for means in final_means],
[means[1] for means in final_means]))
def experiment_analysis(path, quants, path_tosave, title, trials=range(30), plots=True):
"""Prints statistical tests and makes plots for experiment one.
Args:
path: where the trials in CSV are
plots: whether to make plots or not
"""
print(path.split("/")[-1], path.split("/")[1])
# read the data in
data = util.read_trials_from_csv(path, trials)
# print("Data read!")
# FILTER OUT TRIALS WHERE RNN DID NOT LEARN
remove_bad_trials(data)
# get convergence points per quantifier
convergence_points = get_convergence_points(data, quants)
if plots:
# make plots
# make_boxplots(convergence_points, quants)
# make_barplots(convergence_points, quants)
make_plot(data, quants, path_tosave, title, ylim=(0.4, 1))
print(stats.ttest_rel(convergence_points[quants[0]],
convergence_points[quants[1]]))
print()
def experiment_analysis(
path,
verbs,
trials=range(60),
plots=True,
confusion=True,
filename=None,
inset={
"zoom": 3.25,
"xlim": (9000, 11200),
"ylim": (0.93, 0.9575)
},
ylim=(0.8, 0.96),
threshold=None,
):
"""Prints statistical tests and makes plots for experiment one.
Args:
path: where the trials in CSV are
plots: whether to make plots or not
"""
threshold = threshold or 0.925
# read the data in
data = util.read_trials_from_csv(path, trials)
# FILTER OUT TRIALS WHERE RNN DID NOT LEARN
remove_bad_trials(data, threshold=threshold)
# get convergence points per quantifier
convergence_points = get_convergence_points(data, verbs, threshold)
# TODO: no convergence points for this experiment? just final?
# TODO: mean over last N=20 training steps?
final_n = 5
final_points = {
verb: [
np.mean(data[trial][verb + "_accuracy"].values[-final_n:])
for trial in data
]
for verb in verbs
}
if confusion:
conf_mats = defaultdict(dict)
all_dict = defaultdict(float)
conf_dists = defaultdict(dict)
for stat in ["tp", "tn", "fp", "fn"]:
fig, ax = plt.subplots()
for verb in verbs:
name = verb
conf_mats[name][stat] = np.mean([
data[trial][name + "_" + stat].values[-1]
for trial in trials
])
conf_dists[name][stat] = [
data[trial][name + "_" + stat].values[-1]
for trial in trials
]
sns.distplot(conf_dists[name][stat],
ax=ax,
label=name,
axlabel=stat)
plt.legend()
plt.show()
all_dict[stat] = sum([conf_mats[key][stat] for key in conf_mats])
conf_mats["all"] = all_dict
print(conf_mats)
if plots:
"""
# TODO: refactor this into its own method
reshaped = pd.DataFrame()
for trial in data:
for verb in verbs:
new_data = pd.DataFrame(
{'steps': data[trial]['global_step'],
'verb': verb.__name__,
'accuracy': smooth_data(
data[trial][verb.__name__ + '_accuracy'],
smooth_weight=0.7),
'trial': trial})
reshaped = reshaped.append(new_data)
sns.tsplot(reshaped, time='steps', value='accuracy',
condition='verb', unit='trial', err_style='unit_traces',
estimator=np.median)
plt.ylim((0.8, 0.96))
# plt.ylim((0.93, 0.96))
# plt.xlim((10000, 11200))
plt.show()
"""
# make plots
make_boxplots(convergence_points, verbs)
make_boxplots(final_points, verbs)
pairs = list(it.combinations(verbs, 2))
final_data = {}
for pair in pairs:
print()
print("{} vs. {}".format(pair[0], pair[1]))
print(stats.ttest_rel(final_points[pair[0]], final_points[pair[1]]))
print(
stats.ttest_rel(convergence_points[pair[0]],
convergence_points[pair[1]]))
if confusion:
for stat in ["tp", "tn", "fp", "fn"]:
print(stat)
print(
stats.ttest_rel(
conf_dists[pair[0]][stat],
conf_dists[pair[1]][stat],
))
pair_name = "{} - {}".format(pair[0], pair[1])
final_data[pair_name] = np.array(final_points[pair[0]]) - np.array(
final_points[pair[1]])
if plots:
# TODO: re-factor combo_plot into new method
plt.figure(figsize=(18, 12))
gs = gridspec.GridSpec(2, 3)
ax_acc = plt.subplot(gs[:, :-1])
make_plot(
data,
verbs,
ylim=ylim,
threshold=None,
inset=inset,
ax=ax_acc,
)
ax_dists1 = plt.subplot(gs[0, -1])
for pair in pairs:
pair_name = "{} - {}".format(pair[0], pair[1])
if pair[0] == verbs[0]:
sns.distplot(final_data[pair_name],
rug=True,
label=pair_name,
ax=ax_dists1)
plt.legend()
ax_dists2 = plt.subplot(gs[1, -1])
for pair in pairs:
pair_name = "{} - {}".format(pair[0], pair[1])
if pair[0] == verbs[1]:
sns.distplot(final_data[pair_name],
rug=True,
label=pair_name,
ax=ax_dists2)
plt.legend()
plt.tight_layout()
if filename:
plt.savefig(filename, bbox_inches="tight")
else:
plt.show()
plt.close()
sns.barplot(data=pd.DataFrame(final_data))
plt.show()
import pandas as pd
import plotnine as pn
import scipy.stats as stats
import analysis
import util
verbs = ["AllOpen", "BeCertain", "BelPart", "WondowLess"]
trials = range(30)
long_data = pd.DataFrame()
final_acc = pd.DataFrame()
for verb in verbs:
verb_data = util.read_trials_from_csv(
f"../single/{verb.lower()}/data", trials=trials
)
for trial in trials:
accuracies = verb_data[trial][f"{verb}_accuracy"].values
long_data = long_data.append(
pd.DataFrame(
{
"verb": verb,
"trial": trial,
"accuracy": analysis.smooth_data(accuracies, smooth_weight=0.5),
"step": verb_data[trial]["global_step"],
}
),
ignore_index=True,
)
final_acc = final_acc.append(
{"verb": verb, "trial": trial, "accuracy": sum(accuracies[-5:]) / 5},