import numpy as np
from pyBKT.generate import synthetic_data, random_model_uni
from pyBKT.fit import EM_fit
from utils import crossvalidate, accuracy, rmse, auc, check_data, data_helper, ktidem_skills
import copy
np.seterr(divide='ignore', invalid='ignore')
num_fit_initializations = 20
seed, folds = 2020, 5 #can customize to anything, keep same seed and # folds over all trials
results = {} #create dictionary to store accuracy and rmse results
df, skill_list, student_count, data_count, template_count = ktidem_skills.find_skills()
for i in range(10):
skill_name = skill_list[i]
results[skill_name]=[student_count[i], data_count[i], template_count[i]]
data = data_helper.convert_data(df, skill_name)
check_data.check_data(data)
results[skill_name].append((np.sum(data["data"][0]) - len(data["data"][0]))/len(data["data"][0]))
print("creating simple model")
results[skill_name].append(crossvalidate.crossvalidate(data, folds=folds, seed=seed)[2])
data_multiguess = data_helper.convert_data(df, skill_name, multiguess=True)
check_data.check_data(data_multiguess)
print("creating kt_idem model")
results[skill_name].append(crossvalidate.crossvalidate(data_multiguess, folds=folds, seed=seed)[2])
#print(results)
print("Model\tNum Students\tNum Data\tNum Templates\tCorrect Percent\tSimple AUC\tKT_IDEM AUC")
for k, v in results.items():
print("%s\t%d\t%d\t%d\t%.5f\t%.5f\t%.5f" % (k, v[0], v[1], v[2], v[3], v[4], v[5]))
from pyBKT.generate import synthetic_data, random_model_uni
from pyBKT.fit import EM_fit
from utils import crossvalidate, accuracy, rmse, auc, check_data, data_helper
import copy
np.seterr(divide='ignore', invalid='ignore')
num_fit_initializations = 20
skill_name = "Box and Whisker"
seed, folds = 2020, 5 #can customize to anything, keep same seed and # folds over all trials
results = {} #create dictionary to store accuracy and rmse results
#data!
print("starting simple model data collection")
data, df = data_helper.convert_data("as.csv", skill_name, return_df=True)#save dataframe for further trials
check_data.check_data(data)
print("creating simple model")
results["Simple Model"] = crossvalidate.crossvalidate(data, folds=folds, seed=seed)
print("starting majority class calculation")
majority = 0
if np.sum(data["data"][0]) - len(data["data"][0]) > len(data["data"][0]) - (np.sum(data["data"][0]) - len(data["data"][0])):
majority = 1
pred_values = np.zeros((len(data["data"][0]),))
pred_values.fill(majority)
true_values = data["data"][0].tolist()
pred_values = pred_values.tolist()
results["Majority Class"] = (accuracy.compute_acc(true_values,pred_values), rmse.compute_rmse(true_values,pred_values), auc.compute_auc(true_values, pred_values))
print("starting item_learning_effect data collection")
data_multilearn = data_helper.convert_data(df, skill_name, multilearn=True)
check_data.check_data(data_multilearn)
import sys
sys.path.append('../')
import numpy as np
from pyBKT.generate import synthetic_data, random_model_uni
from pyBKT.fit import EM_fit
from utils import crossvalidate, data_helper, check_data
from copy import deepcopy
np.seterr(divide='ignore', invalid='ignore')
num_fit_initializations = 20
skill_name = "Range"
#data!
data = data_helper.convert_data("as.csv", skill_name)
check_data.check_data(data)
#specifying verbose allows data from all iterations of crossvalidation to be printed out
crossvalidate.crossvalidate(data, verbose=True)
'user_id': 'Anon Student Id',
'multiguess': 'Problem Name',
}
for i in range(12):
skill_name = skill_list[i]
results[skill_name] = [student_count[i], data_count[i], template_count[i]]
data = data_helper.convert_data(df, skill_name, defaults=ct_default)
check_data.check_data(data)
results[skill_name].append(
(np.sum(data["data"][0]) - len(data["data"][0])) /
len(data["data"][0]))
print("creating simple model")
results[skill_name].append(
crossvalidate.crossvalidate(data, folds=folds, seed=seed)[2])
data_multiguess = data_helper.convert_data(df,
skill_name,
defaults=ct_default,
multiguess=True)
check_data.check_data(data_multiguess)
print("creating kt_idem model")
results[skill_name].append(
crossvalidate.crossvalidate(data_multiguess, folds=folds,
seed=seed)[2])
#print(results)
print(
"Model\tNum Students\tNum Data\tNum Problems\tCorrect Percent\tSimple AUC\tKT_IDEM AUC"
)
skill_count = 124 #hardcoded for nips data set
#data!
Data = nips_data_helper.convert_data("builder_train.csv",
url2="builder_test.csv")
print("Data preprocessing finished")
for i in range(skill_count):
check_data.check_data(Data[i])
print("All data okay")
all_true = []
all_pred = []
for skill in range(skill_count):
if len(Data[skill]["resources"]
) < 5: #auc only calculated when there are 2+ classifiers
print("Not enough data for skill %s" % skill)
continue
temp = crossvalidate.crossvalidate(Data[skill],
verbose=False,
return_arrays=True)
print("Skill %s of %s calculation completed" % (skill, skill_count - 1))
all_true.extend(temp[0])
all_pred.extend(temp[1])
total_auc = auc.compute_auc(all_true, all_pred)
print("Overall AUC:", total_auc)