def create_model(dataset_name, model, model_name, overwrite=False,parallel=True): args_string = "_".join(["%s=%s" % (k,v) for k,v in model.get_params().items()]) args_hash = get_md5_hash(args_string) filename = MODELS_PATH + model_name + "_" + dataset_name + "_" + args_hash print "Creating", filename if os.path.exists(filename) and not overwrite: print "It exists so skipping..." return # touch file so that other processes don't recreate it touch(filename) dataset = joblib.load(DATASETS_PATH + dataset_name) all_pred = {} n_datasets = len(dataset[0].keys()) # for each essay for i, key in enumerate(dataset[0]): print "Creating %d of %d" % (i+1, n_datasets) predictions = np.zeros((dataset[0][key].shape[0],2)) trainset = np.where(dataset[0][key]["META_LABEL"] == "TRAINING")[0] testset = np.where(dataset[0][key]["META_LABEL"] == "VALIDATION")[0] print "Training #", len(trainset), "Testing #", len(testset) # 2 datasets - each for 1 scorer for scorer in [1,2]: ds = dataset[scorer - 1][key] kf = cross_validation.KFold(len(trainset), n_folds=7) kf = [(trainset[tr], trainset[te]) for tr, te in kf] + [(trainset,testset)] pred = np.zeros(ds.shape[0]) if parallel: pool = Pool(processes=8) essay_sets = pool.map(cv, [[kf, ds, model, n, scorer] for n in range(8)]) pool.close() for n, essay_set in enumerate(essay_sets): pred[kf[n][1]] = essay_set else: for n in range(8): pred[kf[n][1]] = cv([kf,ds,model,n,scorer]) predictions[:,scorer - 1] = pred # DEBUG pred_goc = grade_on_a_curve(pred[trainset], ds["META_SCORE_%d" % scorer].fillna(0)[trainset].map(int)) print key, scorer, kappa.quadratic_weighted_kappa(pred_goc, ds["META_SCORE_%d" % (scorer)].fillna(0)[trainset].map(int)) #DEBUG merged_pred = predictions.mean(axis=1) pred_goc = grade_on_a_curve(merged_pred[trainset], ds["META_SCORE_%d" % scorer].fillna(0)[trainset].map(int)) print key, "Merged", kappa.quadratic_weighted_kappa(pred_goc, ds["META_SCORE_%d" % (scorer)].fillna(0)[trainset].map(int)) print all_pred[key] = predictions data = {"predictions":all_pred ,"model_name":model_name ,"dataset":dataset_name ,"parameters":model.get_params()} pickle.dump(data, open(filename,"w"))
def model_generic_1ofK_clas(pipeline,model_name,model_f,essays_paths,parallel = False): print model_name for essaypath in sorted(essays_paths): print essaypath, essayname = essaypath.split("/")[-1].split(".")[0] save_as = MODEL_PATHS + essayname + "_" + pipeline["name"] + "_" + model_name if os.path.exists(save_as): print "Skipping" continue essays = EssayCollection(essaypath,essayname) essays.apply_datasteps(pipeline["steps"]) essays.create_feature_matrix(min_sparsity=5) predictions = pd.DataFrame({'id':range(essays.meta_data().shape[0])}) trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0] testset = np.where(essays.meta_data().essay_type == "")[0] X_all = np.array(essays.feature_matrix.todense(),dtype=np.float32) y_all = np.array(essays.meta_data()["score3"].map(int),dtype=np.int32) non_duplicated = get_nonduplicate_columns(pd.DataFrame(X_all)) print "orig dimensions", X_all.shape[1], X_all = X_all[:,non_duplicated] print "reduced dimensions", X_all.shape[1], predictions = pd.DataFrame({'id':range(essays.meta_data().shape[0])}) trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0] testset = np.where(essays.meta_data().essay_type == "VALIDATION")[0] for scorer in [3]: kf = cross_validation.KFold(len(trainset), n_folds=7, random_state=0) kf = [(trainset[tr], trainset[te]) for tr, te in kf] + [(trainset,testset)] for grade in sorted(essays.meta_data()["score%d" % (scorer)].unique()): y_all = np.array(essays.meta_data()["score%d" % (scorer)].map(lambda y: 1 if int(y)==int(grade) else 0)) pred_name = "scorer_%d_grade_%d" % (scorer,grade) predictions[pred_name] = 0 if parallel: pool = Pool(processes=4) essay_sets = pool.map(cv, [[kf, X_all, y_all, model_f, n] for n in range(8)]) pool.close() for n, essay_set in enumerate(essay_sets): te_ind = kf[n][1] predictions.ix[te_ind,pred_name] = essay_set[:,1] else: for n,(tr_ind,te_ind) in enumerate(kf): predictions.ix[te_ind,pred_name] = model_f(X_all[tr_ind,:], y_all[tr_ind], X_all[te_ind,:], feature_names=essays.feature_names) predictions = predictions.ix[:,sorted(predictions.columns)] predictions["pred_scorer_3"] = np.array(predictions.ix[:,[c for c in predictions.columns if c.startswith("scorer_3")]]).argmax(axis=1) predictions.to_csv(save_as,index=False) print kappa.quadratic_weighted_kappa(essays.meta_data()["score3"][trainset],predictions["pred_scorer_3"][trainset])
def report_results(path): all_pred = pickle.load(open(path))["predictions"] for ds in dataset: for key in ds: predictions = all_pred[key] merged_pred = predictions.mean(axis=1) pred_goc = grade_on_a_curve(merged_pred, ds[key]["META_SCORE_%d" % scorer].fillna(0)).astype(np.int) print key, "FINAL", kappa.quadratic_weighted_kappa(pred_goc, ds[key]["META_SCORE_FINAL"].fillna(0).astype(np.int)) print
def get_human_results(): dataset = joblib.load(DATASETS_PATH + "dataset_2_sample_size_1") result = {} for key in dataset[0]: filter_records = dataset[0][key]["META_LABEL"]=="TRAINING" dataset[0][key] = dataset[0][key].ix[filter_records] for key in dataset[0]: result[key] = kappa.quadratic_weighted_kappa(dataset[0][key]["META_SCORE_1"].fillna(0).astype(np.int), dataset[0][key]["META_SCORE_2"].fillna(0).astype(np.int)) return result
def report_results(path): all_pred = pickle.load(open(path))["predictions"] for ds in dataset: for key in ds: predictions = all_pred[key] merged_pred = predictions.mean(axis=1) pred_goc = grade_on_a_curve( merged_pred, ds[key]["META_SCORE_%d" % scorer].fillna(0)).astype(np.int) print key, "FINAL", kappa.quadratic_weighted_kappa( pred_goc, ds[key]["META_SCORE_FINAL"].fillna(0).astype(np.int)) print
def model_generic_DBN(pipeline,model_name,model_f,essays_paths,parallel = False): print model_name for essaypath in sorted(essays_paths): print essaypath, essayname = essaypath.split("/")[-1].split(".")[0] save_as = MODEL_PATHS + essayname + "_" + pipeline["name"] + "_" + model_name if os.path.exists(save_as): print "Skipping" continue essays = EssayCollection(essaypath,essayname) essays.apply_datasteps(pipeline["steps"]) essays.create_feature_matrix(min_sparsity=5) predictions = pd.DataFrame({'id':range(essays.meta_data().shape[0])}) trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0] testset = np.where(essays.meta_data().essay_type == "")[0] X_all = np.array(essays.feature_matrix.todense(),dtype=np.float32) y_all = np.array(essays.meta_data()["score3"].map(int),dtype=np.int32) non_duplicated = get_nonduplicate_columns(pd.DataFrame(X_all)) print "orig dimensions", X_all.shape[1], X_all = X_all[:,non_duplicated] print "reduced dimensions", X_all.shape[1], predictions = pd.DataFrame({'id':range(essays.meta_data().shape[0])}) trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0] testset = np.where(essays.meta_data().essay_type == "VALIDATION")[0] scorer = 3 kf = cross_validation.KFold(len(trainset), n_folds=7, random_state=0) kf = [(trainset[tr], trainset[te]) for tr, te in kf] + [(trainset,testset)] scores = sorted(essays.meta_data()["score%d" % (scorer)].unique()) predictions = np.zeros((essays.meta_data().shape[0],len(scores))) model_f.layer_sizes[0] = X_all.shape[1] model_f.layer_sizes[2] = len(scores) try: for n,(tr_ind,te_ind) in enumerate(kf): print n scaler = StandardScaler() _ = scaler.fit(X_all[tr_ind,:]) X_tr = scaler.transform(X_all[tr_ind,:]) / 50.0 X_te = scaler.transform(X_all[te_ind,:]) / 50.0 model_f.fit(X_tr, y_all[tr_ind]) print kappa.quadratic_weighted_kappa(essays.meta_data()["score3"][tr_ind],model_f.predict(X_tr)) print kappa.quadratic_weighted_kappa(essays.meta_data()["score3"][te_ind],model_f.predict(X_te)) predictions[te_ind,:] = model_f.predict_proba(X_te) except: pass predictions = pd.DataFrame(predictions) predictions.columns = ["scorer_%d_grade_%d" % (scorer,grade) for grade in scores] predictions["pred_scorer_3"] = np.array(predictions).argmax(axis=1) predictions.to_csv(save_as,index=False) print kappa.quadratic_weighted_kappa(essays.meta_data()["score3"][trainset],predictions["pred_scorer_3"][trainset])
def report(path): dataset = joblib.load(DATASETS_PATH + "dataset_2_sample_size_1") all_pred = pickle.load(open(path))["predictions"] result = {} """ for key in dataset[0]: wrong_records = dataset[0][key]["META_LABEL"]=="label" assert np.all(all_pred[key][wrong_records,:]==np.zeros((1,2))), "0 prediction for bad records" assert dataset[0][key].shape[0] == all_pred[key].shape[0], "all records wrong" dataset[0][key] = dataset[0][key].ix[~wrong_records] all_pred[key] = all_pred[key][~wrong_records,:] """ for key in dataset[0]: validation_set = np.where(dataset[0][key]["META_LABEL"]=="VALIDATION")[0] train_set = np.where(dataset[0][key]["META_LABEL"]=="TRAINING")[0] predictions = np.zeros((len(validation_set)+len(train_set),2)) for scorer in [1,2]: predictions[:,scorer - 1] = all_pred[key][:,scorer-1] merged_pred = predictions.mean(axis=1) predictions_scores_df = pd.DataFrame({'response':np.array(dataset[0][key]["META_SCORE_1"])[train_set], 'predictions': merged_pred[train_set]}) #predictions_scores_df = predictions_scores_df.ix[~predictions_scores_df["response"].isnull(),:] predictions_scores_df["response"] = predictions_scores_df["response"].fillna(0).map(int) filter_valid_responses = ~predictions_scores_df["response"].isnull() train_goc = grade_on_a_curve(merged_pred[train_set], dataset[0][key]["META_SCORE_%d" % scorer].fillna(0)[train_set].map(int)) kappa_goc = kappa.quadratic_weighted_kappa(train_goc, predictions_scores_df["response"].fillna(0)) result[key] = kappa_goc return result
def create_model(dataset_name, model, model_name, overwrite=False, parallel=True): args_string = "_".join( ["%s=%s" % (k, v) for k, v in model.get_params().items()]) args_hash = get_md5_hash(args_string) filename = MODELS_PATH + model_name + "_" + dataset_name + "_" + args_hash print "Creating", filename if os.path.exists(filename) and not overwrite: print "It exists so skipping..." return # touch file so that other processes don't recreate it touch(filename) dataset = joblib.load(DATASETS_PATH + dataset_name) all_pred = {} n_datasets = len(dataset[0].keys()) # for each essay for i, key in enumerate(dataset[0]): print "Creating %d of %d" % (i + 1, n_datasets) predictions = np.zeros((dataset[0][key].shape[0], 2)) trainset = np.where(dataset[0][key]["META_LABEL"] == "TRAINING")[0] testset = np.where(dataset[0][key]["META_LABEL"] == "VALIDATION")[0] print "Training #", len(trainset), "Testing #", len(testset) # 2 datasets - each for 1 scorer for scorer in [1, 2]: ds = dataset[scorer - 1][key] kf = cross_validation.KFold(len(trainset), n_folds=7) kf = [(trainset[tr], trainset[te]) for tr, te in kf] + [(trainset, testset)] pred = np.zeros(ds.shape[0]) if parallel: pool = Pool(processes=8) essay_sets = pool.map(cv, [[kf, ds, model, n, scorer] for n in range(8)]) pool.close() for n, essay_set in enumerate(essay_sets): pred[kf[n][1]] = essay_set else: for n in range(8): pred[kf[n][1]] = cv([kf, ds, model, n, scorer]) predictions[:, scorer - 1] = pred # DEBUG pred_goc = grade_on_a_curve( pred[trainset], ds["META_SCORE_%d" % scorer].fillna(0)[trainset].map(int)) print key, scorer, kappa.quadratic_weighted_kappa( pred_goc, ds["META_SCORE_%d" % (scorer)].fillna(0)[trainset].map(int)) #DEBUG merged_pred = predictions.mean(axis=1) pred_goc = grade_on_a_curve( merged_pred[trainset], ds["META_SCORE_%d" % scorer].fillna(0)[trainset].map(int)) print key, "Merged", kappa.quadratic_weighted_kappa( pred_goc, ds["META_SCORE_%d" % (scorer)].fillna(0)[trainset].map(int)) print all_pred[key] = predictions data = { "predictions": all_pred, "model_name": model_name, "dataset": dataset_name, "parameters": model.get_params() } pickle.dump(data, open(filename, "w"))
def opt_fn(coef): predictions_vector = np.dot(predictions_matrix[selected_obs], coef) predictions_grade = np.argmax(predictions_vector, axis=1) weighted_kappa = kappa.quadratic_weighted_kappa( real_scores[selected_obs], predictions_grade) return -weighted_kappa
results.write("essay,model,score\n") for essaypath in sorted(glob.glob("data/csv/*.csv")): print essaypath, essays = EssayCollection(essaypath) essayname = os.path.split(essaypath)[-1][:-4] predictions_all_list = [] trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0] for modelpath in glob.glob("models/*" + essayname + "*"): modelname = os.path.split(modelpath)[-1] predictions = pd.read_csv(modelpath) predictions["modelname"] = modelname results.write("%s,%s,%.6f\n" % (essayname, modelname[8:], kappa.quadratic_weighted_kappa( essays.meta_data()["score3"][trainset], predictions["pred_scorer_3"][trainset]))) predictions_all_list.append(predictions.copy()) results.write("%s,%s,%.6f\n" % (essayname, "HUMAN", kappa.quadratic_weighted_kappa( essays.meta_data()["score1"][trainset], essays.meta_data()["score2"][trainset]))) predictions_all = pd.concat(predictions_all_list) scores = [col for col in predictions_all.columns if col.find("grade") > 0] for score_col in scores: predictions_all.ix[:, score_col] = predictions_all.ix[:, score_col].map(
def opt_fn(coef): predictions_vector = np.dot(predictions_matrix[selected_obs],coef) predictions_grade = np.argmax(predictions_vector,axis=1) weighted_kappa = kappa.quadratic_weighted_kappa(real_scores[selected_obs], predictions_grade) return -weighted_kappa
from sklearn.linear_model import ElasticNet results = open("results.csv","w") results.write("essay,model,score\n") for essaypath in sorted(glob.glob("data/csv/*.csv")): print essaypath, essays = EssayCollection(essaypath) essayname = os.path.split(essaypath)[-1][:-4] predictions_all_list = [] trainset = np.where(essays.meta_data().essay_type=="TRAINING")[0] for modelpath in glob.glob("models/*" + essayname + "*"): modelname = os.path.split(modelpath)[-1] predictions = pd.read_csv(modelpath) predictions["modelname"] = modelname results.write("%s,%s,%.6f\n" % (essayname, modelname[8:], kappa.quadratic_weighted_kappa(essays.meta_data()["score3"][trainset],predictions["pred_scorer_3"][trainset]))) predictions_all_list.append(predictions.copy()) results.write("%s,%s,%.6f\n" % (essayname, "HUMAN", kappa.quadratic_weighted_kappa(essays.meta_data()["score1"][trainset],essays.meta_data()["score2"][trainset]))) predictions_all = pd.concat(predictions_all_list) scores = [col for col in predictions_all.columns if col.find("grade") > 0] for score_col in scores: predictions_all.ix[:,score_col] = predictions_all.ix[:,score_col].map(logit) unique_models = predictions_all.modelname.unique() n_obs = essays.meta_data().shape[0] predictions_matrix = np.zeros((len(scores), unique_models.shape[0], n_obs)) for n_score, score in enumerate(scores): for n_model, model in enumerate(unique_models):
np.array(dataset[0][key]["META_SCORE_1"])[train_set], 'predictions': merged_pred[train_set] }) predictions_scores_df["response"][ predictions_scores_df["response"].isnull()] = 0 predictions_scores_df["response"] = predictions_scores_df["response"].map( int) #predictions_scores_df["response"] = predictions_scores_df["response"].map(int) #predictions_scores_df["response"] = predictions_scores_df["response"].fillna(0) train_goc = grade_on_a_curve(predictions_scores_df["predictions"], predictions_scores_df["response"]).astype( np.int) kappa_goc = kappa.quadratic_weighted_kappa( train_goc, predictions_scores_df["response"]) kappa_raw = kappa.quadratic_weighted_kappa( predictions_scores_df["predictions"].round().map(int), predictions_scores_df["response"]) print key, kappa_goc, kappa_raw pred_goc = grade_on_a_curve(merged_pred[validation_set], predictions_scores_df["response"]).astype( np.int) print set(pred_goc), set(predictions_scores_df["response"]) assert set(pred_goc) == set( predictions_scores_df["response"]), "wrong set of values" result[ key] = pred_goc #kappa.quadratic_weighted_kappa(pred_goc, dataset[0][key]["META_SCORE_FINAL"].fillna(0).astype(np.int))
def model_generic_1ofK_clas(pipeline, model_name, model_f, essays_paths, parallel=False): print model_name for essaypath in sorted(essays_paths): print essaypath, essayname = essaypath.split("/")[-1].split(".")[0] save_as = MODEL_PATHS + essayname + "_" + pipeline[ "name"] + "_" + model_name if os.path.exists(save_as): print "Skipping" continue essays = EssayCollection(essaypath, essayname) essays.apply_datasteps(pipeline["steps"]) essays.create_feature_matrix(min_sparsity=5) predictions = pd.DataFrame({'id': range(essays.meta_data().shape[0])}) trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0] testset = np.where(essays.meta_data().essay_type == "")[0] X_all = np.array(essays.feature_matrix.todense(), dtype=np.float32) y_all = np.array(essays.meta_data()["score3"].map(int), dtype=np.int32) non_duplicated = get_nonduplicate_columns(pd.DataFrame(X_all)) print "orig dimensions", X_all.shape[1], X_all = X_all[:, non_duplicated] print "reduced dimensions", X_all.shape[1], predictions = pd.DataFrame({'id': range(essays.meta_data().shape[0])}) trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0] testset = np.where(essays.meta_data().essay_type == "VALIDATION")[0] for scorer in [3]: kf = cross_validation.KFold(len(trainset), n_folds=7, random_state=0) kf = [(trainset[tr], trainset[te]) for tr, te in kf] + [(trainset, testset)] for grade in sorted(essays.meta_data()["score%d" % (scorer)].unique()): y_all = np.array(essays.meta_data()["score%d" % (scorer)].map( lambda y: 1 if int(y) == int(grade) else 0)) pred_name = "scorer_%d_grade_%d" % (scorer, grade) predictions[pred_name] = 0 if parallel: pool = Pool(processes=4) essay_sets = pool.map(cv, [[kf, X_all, y_all, model_f, n] for n in range(8)]) pool.close() for n, essay_set in enumerate(essay_sets): te_ind = kf[n][1] predictions.ix[te_ind, pred_name] = essay_set[:, 1] else: for n, (tr_ind, te_ind) in enumerate(kf): predictions.ix[te_ind, pred_name] = model_f( X_all[tr_ind, :], y_all[tr_ind], X_all[te_ind, :], feature_names=essays.feature_names) predictions = predictions.ix[:, sorted(predictions.columns)] predictions["pred_scorer_3"] = np.array( predictions. ix[:, [c for c in predictions.columns if c.startswith("scorer_3")]]).argmax(axis=1) predictions.to_csv(save_as, index=False) print kappa.quadratic_weighted_kappa( essays.meta_data()["score3"][trainset], predictions["pred_scorer_3"][trainset])
def model_generic_DBN(pipeline, model_name, model_f, essays_paths, parallel=False): print model_name for essaypath in sorted(essays_paths): print essaypath, essayname = essaypath.split("/")[-1].split(".")[0] save_as = MODEL_PATHS + essayname + "_" + pipeline[ "name"] + "_" + model_name if os.path.exists(save_as): print "Skipping" continue essays = EssayCollection(essaypath, essayname) essays.apply_datasteps(pipeline["steps"]) essays.create_feature_matrix(min_sparsity=5) predictions = pd.DataFrame({'id': range(essays.meta_data().shape[0])}) trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0] testset = np.where(essays.meta_data().essay_type == "")[0] X_all = np.array(essays.feature_matrix.todense(), dtype=np.float32) y_all = np.array(essays.meta_data()["score3"].map(int), dtype=np.int32) non_duplicated = get_nonduplicate_columns(pd.DataFrame(X_all)) print "orig dimensions", X_all.shape[1], X_all = X_all[:, non_duplicated] print "reduced dimensions", X_all.shape[1], predictions = pd.DataFrame({'id': range(essays.meta_data().shape[0])}) trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0] testset = np.where(essays.meta_data().essay_type == "VALIDATION")[0] scorer = 3 kf = cross_validation.KFold(len(trainset), n_folds=7, random_state=0) kf = [(trainset[tr], trainset[te]) for tr, te in kf] + [(trainset, testset)] scores = sorted(essays.meta_data()["score%d" % (scorer)].unique()) predictions = np.zeros((essays.meta_data().shape[0], len(scores))) model_f.layer_sizes[0] = X_all.shape[1] model_f.layer_sizes[2] = len(scores) try: for n, (tr_ind, te_ind) in enumerate(kf): print n scaler = StandardScaler() _ = scaler.fit(X_all[tr_ind, :]) X_tr = scaler.transform(X_all[tr_ind, :]) / 50.0 X_te = scaler.transform(X_all[te_ind, :]) / 50.0 model_f.fit(X_tr, y_all[tr_ind]) print kappa.quadratic_weighted_kappa( essays.meta_data()["score3"][tr_ind], model_f.predict(X_tr)) print kappa.quadratic_weighted_kappa( essays.meta_data()["score3"][te_ind], model_f.predict(X_te)) predictions[te_ind, :] = model_f.predict_proba(X_te) except: pass predictions = pd.DataFrame(predictions) predictions.columns = [ "scorer_%d_grade_%d" % (scorer, grade) for grade in scores ] predictions["pred_scorer_3"] = np.array(predictions).argmax(axis=1) predictions.to_csv(save_as, index=False) print kappa.quadratic_weighted_kappa( essays.meta_data()["score3"][trainset], predictions["pred_scorer_3"][trainset])
train_set = np.where(dataset[0][key]["META_LABEL"] == "TRAINING")[0] predictions = np.zeros((len(validation_set) + len(train_set), 2)) for scorer in [1, 2]: predictions[:, scorer - 1] = all_pred[key][:, scorer - 1] merged_pred = predictions.mean(axis=1) predictions_scores_df = pd.DataFrame( {"response": np.array(dataset[0][key]["META_SCORE_1"])[train_set], "predictions": merged_pred[train_set]} ) predictions_scores_df["response"][predictions_scores_df["response"].isnull()] = 0 predictions_scores_df["response"] = predictions_scores_df["response"].map(int) train_goc = grade_on_a_curve(predictions_scores_df["predictions"], predictions_scores_df["response"]).astype(np.int) kappa_goc = kappa.quadratic_weighted_kappa(train_goc, predictions_scores_df["response"]) kappa_raw = kappa.quadratic_weighted_kappa( predictions_scores_df["predictions"].round().map(int), predictions_scores_df["response"] ) print key, kappa_goc, kappa_raw pred_goc = grade_on_a_curve(merged_pred[validation_set], predictions_scores_df["response"]).astype(np.int) print set(pred_goc), set(predictions_scores_df["response"]) assert set(pred_goc) == set(predictions_scores_df["response"]), "wrong set of values" result[ key ] = ( pred_goc ) # kappa.quadratic_weighted_kappa(pred_goc, dataset[0][key]["META_SCORE_FINAL"].fillna(0).astype(np.int))