def compare_more_models_final(experiments, eval_data, labels=None, difficulties=True, runs=1):
labels = sorted(experiments.keys()) if labels is None else labels
df = pd.DataFrame(columns=["labels", "rmse"])
for label in labels:
r = Evaluator(experiments[label][0](label), experiments[label][1](label)).get_report()
df.loc[len(df)] = (label, r["rmse"])
plt.subplot(131)
compare_models([experiments[label][0](label) for label in labels],
[experiments[label][1](label) for label in labels],
names=labels,
palette=sns.color_palette()[:4],
metric="rmse", force_evaluate=False, answer_filters={
"binary": response_as_binary(),
}, runs=runs, hue_order=False, with_all=False)
plt.subplot(132)
compare_models([experiments[label][0](label) for label in labels],
[experiments[label][1](label) for label in labels],
names=labels,
palette=sns.color_palette()[:4],
metric="rmse", force_evaluate=False, answer_filters={
# "response >7s-0.5": transform_response_by_time(((7, 0.5),), binarize_before=True),
"linear 14": transform_response_by_time_linear(14),
}, runs=runs, hue_order=False, with_all=False)
plt.subplot(133)
compare_models([experiments[label][0](label) for label in labels],
[experiments[label][1](label) for label in labels],
names=labels,
palette=sns.color_palette()[:4],
metric="AUC", force_evaluate=False, runs=runs, hue_order=False)
def compare_more_models(experiments, eval_data, labels=None, difficulties=True, runs=1):
labels = sorted(experiments.keys()) if labels is None else labels
results = pd.DataFrame(index=labels, columns=labels, dtype=float)
for label in labels: results[label][label] = 1
for i, label1 in enumerate(labels):
for label2 in labels[i+1:]:
print(label1, label2)
for run in range(runs):
data1 = experiments[label1][0](label1)
data2 = experiments[label2][0](label2)
data1.set_seed(run)
data2.set_seed(run)
compare_fce = compare_model_difficulties if difficulties else compare_model_predictions
c = compare_fce(data1, data2,
experiments[label1][1](label1), experiments[label2][1](label2), plot=False)
results[label1][label2] = c
results[label2][label1] = c
df = pd.DataFrame(columns=["labels", "rmse"])
for label in labels:
r = Evaluator(experiments[label][0](label), experiments[label][1](label)).get_report()
df.loc[len(df)] = (label, r["rmse"])
plt.subplot(221)
plt.title("Correlations of " + "difficulties" if difficulties else "predictions")
sns.heatmap(results, vmax=1, vmin=.4)
# plt.yticks(rotation=0)
# plt.xticks(rotation=90)
plt.subplot(222)
compare_models(eval_data, [experiments[label][1](label) for label in labels], answer_filters={
# "response >7s-0.5": transform_response_by_time(((7, 0.5),)),
"long (30) students": filter_students_with_many_answers(number_of_answers=30),
}, runs=runs, hue_order=False)
plt.subplot(223)
compare_models([experiments[label][0](label) for label in labels],
[experiments[label][1](label) for label in labels],
names=labels,
metric="rmse", force_evaluate=False, answer_filters={
"binary": response_as_binary(),
"response >7s-0.5": transform_response_by_time(((7, 0.5),), binarize_before=True),
}, runs=runs, hue_order=False)
plt.subplot(224)
compare_models([experiments[label][0](label) for label in labels],
[experiments[label][1](label) for label in labels],
names=labels,
metric="AUC", force_evaluate=False, runs=runs, hue_order=False)
return results
skills = pd.read_csv('../data/matmat/2016-06-27/skills.csv', index_col='id')
items = pd.read_csv('../data/matmat/2016-06-27/items.csv')
items = items.join(skills, on='skill_lvl_1')
concepts = {}
for concept in items['name'].unique():
concepts[concept] = list(items.loc[items['name'] == concept, 'id'])
filters = {
k: lambda df, v=v: df[df['item'].isin(v)] for k, v in concepts.items()
}
compare_models(data, [
TimeCombiner(AvgModel(), TimeAvgModel()),
TimeCombiner(AvgModel(), TimeItemAvgModel()),
TimeCombiner(AvgModel(), TimeStudentAvgModel()),
TimeCombiner(AvgModel(), BasicTimeModel(alpha=0.6, beta=0.1, K=0.25)),
TimeCombiner(AvgModel(), TimePriorCurrentModel(alpha=0.4, beta=0.04, KC=0.3, KI=0.3)),
TimeCombiner(AvgModel(), TimePriorCurrentModel(alpha=0.4, beta=0.04, KC=0.4, KI=0.4, first_level=False)),
TimeCombiner(AvgModel(), TimeConcepts(alpha=0.4, beta=0.05, K=0.25, concepts=concepts)),
TimeCombiner(AvgModel(), TimeEloHierarchicalModel()),
], dont=0, answer_filters=filters)
# learning(data.get_dataframe_all().join(pd.read_csv('../data/matmat/2016-06-27/items.csv'), on='item', rsuffix='_item'),
# measures=['correct', 'response_time', 'response_time_log'])
# skill_vs_speed(
# EloHierarchicalModel(KC=1, KI=0.75, alpha=0.8, beta=0.02),
# BasicTimeModel(alpha=0.6, beta=0.1, K=0.25),
# data
# )
d = data.Data("../data/matmat/2016-06-27/answers.pd", train_size=0.5, only_first=True)
# d = data.Data("../data/matmat/2015-11-20/answers.pd", response_modification=data.TimeLimitResponseModificator([(5, 0.5)]))
# d = data.Data("../data/matmat/2015-11-20/answers.pd", response_modification=data.TimeLimitResponseModificator([(5, 0.66), (10, 0.33)]))
concepts = d.get_concepts()
compare_models(d, [
# AvgModel(),
ItemAvgModel(),
SkipHandler(ItemAvgModel()),
# EloPriorCurrentModel(),
EloPriorCurrentModel(KC=2, KI=0.5),
SkipHandler(EloPriorCurrentModel(KC=2, KI=0.5)),
# EloHierarchicalModel(),
# EloHierarchicalModel(KC=1, KI=0.75),
EloConcepts(concepts=concepts),
SkipHandler(EloConcepts(concepts=concepts)),
EloHierarchicalModel(KC=1, KI=0.75, alpha=0.8, beta=0.02),
SkipHandler(EloHierarchicalModel(KC=1, KI=0.75, alpha=0.8, beta=0.02)),
# EloHierarchicalModel(alpha=0.25, beta=0.02),
# EloConcepts(),
], dont=0, force_evaluate=0, force_run=0, runs=5, hue_order=False, answer_filters={
"long (50) student": data.filter_students_with_many_answers(),
"long (30) student": data.filter_students_with_many_answers(number_of_answers=30),
"long (11) student": data.filter_students_with_many_answers(number_of_answers=11),
"response >5s-0.5": data.transform_response_by_time(((5, 0.5),))
},
# palette=sns.color_palette()[:2] * 4
)
# evaluator.Evaluator(d, EloHierarchicalModel(alpha=0.25, beta=0.02)).brier_graphs()
"alpha": np.arange(0.1, 0.5, 0.1),
"beta": np.arange(0.02, 0.2, 0.02),
}, plot_axes=['alpha', 'beta'], time=True,
)
def item_answers_hist():
df = data.get_dataframe_all()
counts = df.groupby(['item', 'student']).apply(len)
counts[counts > 20] = 20
sns.distplot(counts)
compare_models(data_all, [
TimeCombiner(AvgModel(), TimeAvgModel()),
TimeCombiner(ItemAvgModel(), TimeItemAvgModel()),
TimeCombiner(StudentAvgModel(), TimeStudentAvgModel()),
TimeCombiner(EloPriorCurrentModel(KC=2, KI=0.5), BasicTimeModel(alpha=0.6, beta=0.1, K=0.25)),
TimeCombiner(EloPriorCurrentModel(KC=2, KI=0.5), BasicTimeModel(alpha=0.6, beta=0.18, K=0.2)),
TimeCombiner(EloPriorCurrentModel(KC=2, KI=0.5), TimePriorCurrentModel(alpha=0.2, beta=0.08, KC=0.4, KI=0.3, first_level=False)),
# TimeCombiner(EloHierarchicalModel(KC=1, KI=0.75, alpha=0.8, beta=0.02), TimeEloHierarchicalModel()),
TimeCombiner(EloHierarchicalModel(KC=1, KI=0.75, alpha=0.8, beta=0.02), TimeEloHierarchicalModel(alpha=0.8, beta=0.08, KC=0.275, KI=0.225)),
], dont=0)
# grid_search_Ks()
# grid_search_K()
# grid_search_AB()
# grid_search_AB2()
plt.show()
