import numpy as np from pyBKT.generate import synthetic_data, random_model_uni from pyBKT.fit import EM_fit from utils import crossvalidate, accuracy, rmse, auc, check_data, data_helper, ktidem_skills import copy np.seterr(divide='ignore', invalid='ignore') num_fit_initializations = 20 seed, folds = 2020, 5 #can customize to anything, keep same seed and # folds over all trials results = {} #create dictionary to store accuracy and rmse results df, skill_list, student_count, data_count, template_count = ktidem_skills.find_skills() for i in range(10): skill_name = skill_list[i] results[skill_name]=[student_count[i], data_count[i], template_count[i]] data = data_helper.convert_data(df, skill_name) check_data.check_data(data) results[skill_name].append((np.sum(data["data"][0]) - len(data["data"][0]))/len(data["data"][0])) print("creating simple model") results[skill_name].append(crossvalidate.crossvalidate(data, folds=folds, seed=seed)[2]) data_multiguess = data_helper.convert_data(df, skill_name, multiguess=True) check_data.check_data(data_multiguess) print("creating kt_idem model") results[skill_name].append(crossvalidate.crossvalidate(data_multiguess, folds=folds, seed=seed)[2]) #print(results) print("Model\tNum Students\tNum Data\tNum Templates\tCorrect Percent\tSimple AUC\tKT_IDEM AUC") for k, v in results.items(): print("%s\t%d\t%d\t%d\t%.5f\t%.5f\t%.5f" % (k, v[0], v[1], v[2], v[3], v[4], v[5]))
from pyBKT.generate import synthetic_data, random_model_uni from pyBKT.fit import EM_fit from utils import crossvalidate, accuracy, rmse, auc, check_data, data_helper import copy np.seterr(divide='ignore', invalid='ignore') num_fit_initializations = 20 skill_name = "Box and Whisker" seed, folds = 2020, 5 #can customize to anything, keep same seed and # folds over all trials results = {} #create dictionary to store accuracy and rmse results #data! print("starting simple model data collection") data, df = data_helper.convert_data("as.csv", skill_name, return_df=True)#save dataframe for further trials check_data.check_data(data) print("creating simple model") results["Simple Model"] = crossvalidate.crossvalidate(data, folds=folds, seed=seed) print("starting majority class calculation") majority = 0 if np.sum(data["data"][0]) - len(data["data"][0]) > len(data["data"][0]) - (np.sum(data["data"][0]) - len(data["data"][0])): majority = 1 pred_values = np.zeros((len(data["data"][0]),)) pred_values.fill(majority) true_values = data["data"][0].tolist() pred_values = pred_values.tolist() results["Majority Class"] = (accuracy.compute_acc(true_values,pred_values), rmse.compute_rmse(true_values,pred_values), auc.compute_auc(true_values, pred_values)) print("starting item_learning_effect data collection") data_multilearn = data_helper.convert_data(df, skill_name, multilearn=True) check_data.check_data(data_multilearn)
import sys sys.path.append('../') import numpy as np from pyBKT.generate import synthetic_data, random_model_uni from pyBKT.fit import EM_fit from utils import crossvalidate, data_helper, check_data from copy import deepcopy np.seterr(divide='ignore', invalid='ignore') num_fit_initializations = 20 skill_name = "Range" #data! data = data_helper.convert_data("as.csv", skill_name) check_data.check_data(data) #specifying verbose allows data from all iterations of crossvalidation to be printed out crossvalidate.crossvalidate(data, verbose=True)
'user_id': 'Anon Student Id', 'multiguess': 'Problem Name', } for i in range(12): skill_name = skill_list[i] results[skill_name] = [student_count[i], data_count[i], template_count[i]] data = data_helper.convert_data(df, skill_name, defaults=ct_default) check_data.check_data(data) results[skill_name].append( (np.sum(data["data"][0]) - len(data["data"][0])) / len(data["data"][0])) print("creating simple model") results[skill_name].append( crossvalidate.crossvalidate(data, folds=folds, seed=seed)[2]) data_multiguess = data_helper.convert_data(df, skill_name, defaults=ct_default, multiguess=True) check_data.check_data(data_multiguess) print("creating kt_idem model") results[skill_name].append( crossvalidate.crossvalidate(data_multiguess, folds=folds, seed=seed)[2]) #print(results) print( "Model\tNum Students\tNum Data\tNum Problems\tCorrect Percent\tSimple AUC\tKT_IDEM AUC" )
skill_count = 124 #hardcoded for nips data set #data! Data = nips_data_helper.convert_data("builder_train.csv", url2="builder_test.csv") print("Data preprocessing finished") for i in range(skill_count): check_data.check_data(Data[i]) print("All data okay") all_true = [] all_pred = [] for skill in range(skill_count): if len(Data[skill]["resources"] ) < 5: #auc only calculated when there are 2+ classifiers print("Not enough data for skill %s" % skill) continue temp = crossvalidate.crossvalidate(Data[skill], verbose=False, return_arrays=True) print("Skill %s of %s calculation completed" % (skill, skill_count - 1)) all_true.extend(temp[0]) all_pred.extend(temp[1]) total_auc = auc.compute_auc(all_true, all_pred) print("Overall AUC:", total_auc)