def compare_more_models_final(experiments, eval_data, labels=None, difficulties=True, runs=1):
labels = sorted(experiments.keys()) if labels is None else labels
df = pd.DataFrame(columns=["labels", "rmse"])
for label in labels:
r = Evaluator(experiments[label][0](label), experiments[label][1](label)).get_report()
df.loc[len(df)] = (label, r["rmse"])
plt.subplot(131)
compare_models([experiments[label][0](label) for label in labels],
[experiments[label][1](label) for label in labels],
names=labels,
palette=sns.color_palette()[:4],
metric="rmse", force_evaluate=False, answer_filters={
"binary": response_as_binary(),
}, runs=runs, hue_order=False, with_all=False)
plt.subplot(132)
compare_models([experiments[label][0](label) for label in labels],
[experiments[label][1](label) for label in labels],
names=labels,
palette=sns.color_palette()[:4],
metric="rmse", force_evaluate=False, answer_filters={
# "response >7s-0.5": transform_response_by_time(((7, 0.5),), binarize_before=True),
"linear 14": transform_response_by_time_linear(14),
}, runs=runs, hue_order=False, with_all=False)
plt.subplot(133)
compare_models([experiments[label][0](label) for label in labels],
[experiments[label][1](label) for label in labels],
names=labels,
palette=sns.color_palette()[:4],
metric="AUC", force_evaluate=False, runs=runs, hue_order=False)
def compare_more_models(experiments, eval_data, labels=None, difficulties=True, runs=1):
labels = sorted(experiments.keys()) if labels is None else labels
results = pd.DataFrame(index=labels, columns=labels, dtype=float)
for label in labels: results[label][label] = 1
for i, label1 in enumerate(labels):
for label2 in labels[i+1:]:
print(label1, label2)
for run in range(runs):
data1 = experiments[label1][0](label1)
data2 = experiments[label2][0](label2)
data1.set_seed(run)
data2.set_seed(run)
compare_fce = compare_model_difficulties if difficulties else compare_model_predictions
c = compare_fce(data1, data2,
experiments[label1][1](label1), experiments[label2][1](label2), plot=False)
results[label1][label2] = c
results[label2][label1] = c
df = pd.DataFrame(columns=["labels", "rmse"])
for label in labels:
r = Evaluator(experiments[label][0](label), experiments[label][1](label)).get_report()
df.loc[len(df)] = (label, r["rmse"])
plt.subplot(221)
plt.title("Correlations of " + "difficulties" if difficulties else "predictions")
sns.heatmap(results, vmax=1, vmin=.4)
# plt.yticks(rotation=0)
# plt.xticks(rotation=90)
plt.subplot(222)
compare_models(eval_data, [experiments[label][1](label) for label in labels], answer_filters={
# "response >7s-0.5": transform_response_by_time(((7, 0.5),)),
"long (30) students": filter_students_with_many_answers(number_of_answers=30),
}, runs=runs, hue_order=False)
plt.subplot(223)
compare_models([experiments[label][0](label) for label in labels],
[experiments[label][1](label) for label in labels],
names=labels,
metric="rmse", force_evaluate=False, answer_filters={
"binary": response_as_binary(),
"response >7s-0.5": transform_response_by_time(((7, 0.5),), binarize_before=True),
}, runs=runs, hue_order=False)
plt.subplot(224)
compare_models([experiments[label][0](label) for label in labels],
[experiments[label][1](label) for label in labels],
names=labels,
metric="AUC", force_evaluate=False, runs=runs, hue_order=False)
return results
