def create_model(dataset_name, model, model_name, overwrite=False,parallel=True):
args_string = "_".join(["%s=%s" % (k,v) for k,v in model.get_params().items()])
args_hash = get_md5_hash(args_string)
filename = MODELS_PATH + model_name + "_" + dataset_name + "_" + args_hash
print "Creating", filename
if os.path.exists(filename) and not overwrite:
print "It exists so skipping..."
return
# touch file so that other processes don't recreate it
touch(filename)
dataset = joblib.load(DATASETS_PATH + dataset_name)
all_pred = {}
n_datasets = len(dataset[0].keys())
# for each essay
for i, key in enumerate(dataset[0]):
print "Creating %d of %d" % (i+1, n_datasets)
predictions = np.zeros((dataset[0][key].shape[0],2))
trainset = np.where(dataset[0][key]["META_LABEL"] == "TRAINING")[0]
testset = np.where(dataset[0][key]["META_LABEL"] == "VALIDATION")[0]
print "Training #", len(trainset), "Testing #", len(testset)
# 2 datasets - each for 1 scorer
for scorer in [1,2]:
ds = dataset[scorer - 1][key]
kf = cross_validation.KFold(len(trainset), n_folds=7)
kf = [(trainset[tr], trainset[te]) for tr, te in kf] + [(trainset,testset)]
pred = np.zeros(ds.shape[0])
if parallel:
pool = Pool(processes=8)
essay_sets = pool.map(cv, [[kf, ds, model, n, scorer] for n in range(8)])
pool.close()
for n, essay_set in enumerate(essay_sets):
pred[kf[n][1]] = essay_set
else:
for n in range(8):
pred[kf[n][1]] = cv([kf,ds,model,n,scorer])
predictions[:,scorer - 1] = pred
# DEBUG
pred_goc = grade_on_a_curve(pred[trainset], ds["META_SCORE_%d" % scorer].fillna(0)[trainset].map(int))
print key, scorer, kappa.quadratic_weighted_kappa(pred_goc, ds["META_SCORE_%d" % (scorer)].fillna(0)[trainset].map(int))
#DEBUG
merged_pred = predictions.mean(axis=1)
pred_goc = grade_on_a_curve(merged_pred[trainset], ds["META_SCORE_%d" % scorer].fillna(0)[trainset].map(int))
print key, "Merged", kappa.quadratic_weighted_kappa(pred_goc, ds["META_SCORE_%d" % (scorer)].fillna(0)[trainset].map(int))
print
all_pred[key] = predictions
data = {"predictions":all_pred
,"model_name":model_name
,"dataset":dataset_name
,"parameters":model.get_params()}
pickle.dump(data, open(filename,"w"))
def model_generic_1ofK_clas(pipeline,model_name,model_f,essays_paths,parallel = False):
print model_name
for essaypath in sorted(essays_paths):
print essaypath,
essayname = essaypath.split("/")[-1].split(".")[0]
save_as = MODEL_PATHS + essayname + "_" + pipeline["name"] + "_" + model_name
if os.path.exists(save_as):
print "Skipping"
continue
essays = EssayCollection(essaypath,essayname)
essays.apply_datasteps(pipeline["steps"])
essays.create_feature_matrix(min_sparsity=5)
predictions = pd.DataFrame({'id':range(essays.meta_data().shape[0])})
trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0]
testset = np.where(essays.meta_data().essay_type == "")[0]
X_all = np.array(essays.feature_matrix.todense(),dtype=np.float32)
y_all = np.array(essays.meta_data()["score3"].map(int),dtype=np.int32)
non_duplicated = get_nonduplicate_columns(pd.DataFrame(X_all))
print "orig dimensions", X_all.shape[1],
X_all = X_all[:,non_duplicated]
print "reduced dimensions", X_all.shape[1],
predictions = pd.DataFrame({'id':range(essays.meta_data().shape[0])})
trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0]
testset = np.where(essays.meta_data().essay_type == "VALIDATION")[0]
for scorer in [3]:
kf = cross_validation.KFold(len(trainset), n_folds=7, random_state=0)
kf = [(trainset[tr], trainset[te]) for tr, te in kf] + [(trainset,testset)]
for grade in sorted(essays.meta_data()["score%d" % (scorer)].unique()):
y_all = np.array(essays.meta_data()["score%d" % (scorer)].map(lambda y: 1 if int(y)==int(grade) else 0))
pred_name = "scorer_%d_grade_%d" % (scorer,grade)
predictions[pred_name] = 0
if parallel:
pool = Pool(processes=4)
essay_sets = pool.map(cv, [[kf, X_all, y_all, model_f, n] for n in range(8)])
pool.close()
for n, essay_set in enumerate(essay_sets):
te_ind = kf[n][1]
predictions.ix[te_ind,pred_name] = essay_set[:,1]
else:
for n,(tr_ind,te_ind) in enumerate(kf):
predictions.ix[te_ind,pred_name] = model_f(X_all[tr_ind,:],
y_all[tr_ind],
X_all[te_ind,:],
feature_names=essays.feature_names)
predictions = predictions.ix[:,sorted(predictions.columns)]
predictions["pred_scorer_3"] = np.array(predictions.ix[:,[c for c in predictions.columns if c.startswith("scorer_3")]]).argmax(axis=1)
predictions.to_csv(save_as,index=False)
print kappa.quadratic_weighted_kappa(essays.meta_data()["score3"][trainset],predictions["pred_scorer_3"][trainset])
def report_results(path):
all_pred = pickle.load(open(path))["predictions"]
for ds in dataset:
for key in ds:
predictions = all_pred[key]
merged_pred = predictions.mean(axis=1)
pred_goc = grade_on_a_curve(merged_pred, ds[key]["META_SCORE_%d" % scorer].fillna(0)).astype(np.int)
print key, "FINAL", kappa.quadratic_weighted_kappa(pred_goc, ds[key]["META_SCORE_FINAL"].fillna(0).astype(np.int))
print
def get_human_results():
dataset = joblib.load(DATASETS_PATH + "dataset_2_sample_size_1")
result = {}
for key in dataset[0]:
filter_records = dataset[0][key]["META_LABEL"]=="TRAINING"
dataset[0][key] = dataset[0][key].ix[filter_records]
for key in dataset[0]:
result[key] = kappa.quadratic_weighted_kappa(dataset[0][key]["META_SCORE_1"].fillna(0).astype(np.int),
dataset[0][key]["META_SCORE_2"].fillna(0).astype(np.int))
return result
def report_results(path):
all_pred = pickle.load(open(path))["predictions"]
for ds in dataset:
for key in ds:
predictions = all_pred[key]
merged_pred = predictions.mean(axis=1)
pred_goc = grade_on_a_curve(
merged_pred,
ds[key]["META_SCORE_%d" % scorer].fillna(0)).astype(np.int)
print key, "FINAL", kappa.quadratic_weighted_kappa(
pred_goc, ds[key]["META_SCORE_FINAL"].fillna(0).astype(np.int))
print
def model_generic_DBN(pipeline,model_name,model_f,essays_paths,parallel = False):
print model_name
for essaypath in sorted(essays_paths):
print essaypath,
essayname = essaypath.split("/")[-1].split(".")[0]
save_as = MODEL_PATHS + essayname + "_" + pipeline["name"] + "_" + model_name
if os.path.exists(save_as):
print "Skipping"
continue
essays = EssayCollection(essaypath,essayname)
essays.apply_datasteps(pipeline["steps"])
essays.create_feature_matrix(min_sparsity=5)
predictions = pd.DataFrame({'id':range(essays.meta_data().shape[0])})
trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0]
testset = np.where(essays.meta_data().essay_type == "")[0]
X_all = np.array(essays.feature_matrix.todense(),dtype=np.float32)
y_all = np.array(essays.meta_data()["score3"].map(int),dtype=np.int32)
non_duplicated = get_nonduplicate_columns(pd.DataFrame(X_all))
print "orig dimensions", X_all.shape[1],
X_all = X_all[:,non_duplicated]
print "reduced dimensions", X_all.shape[1],
predictions = pd.DataFrame({'id':range(essays.meta_data().shape[0])})
trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0]
testset = np.where(essays.meta_data().essay_type == "VALIDATION")[0]
scorer = 3
kf = cross_validation.KFold(len(trainset), n_folds=7, random_state=0)
kf = [(trainset[tr], trainset[te]) for tr, te in kf] + [(trainset,testset)]
scores = sorted(essays.meta_data()["score%d" % (scorer)].unique())
predictions = np.zeros((essays.meta_data().shape[0],len(scores)))
model_f.layer_sizes[0] = X_all.shape[1]
model_f.layer_sizes[2] = len(scores)
try:
for n,(tr_ind,te_ind) in enumerate(kf):
print n
scaler = StandardScaler()
_ = scaler.fit(X_all[tr_ind,:])
X_tr = scaler.transform(X_all[tr_ind,:]) / 50.0
X_te = scaler.transform(X_all[te_ind,:]) / 50.0
model_f.fit(X_tr, y_all[tr_ind])
print kappa.quadratic_weighted_kappa(essays.meta_data()["score3"][tr_ind],model_f.predict(X_tr))
print kappa.quadratic_weighted_kappa(essays.meta_data()["score3"][te_ind],model_f.predict(X_te))
predictions[te_ind,:] = model_f.predict_proba(X_te)
except:
pass
predictions = pd.DataFrame(predictions)
predictions.columns = ["scorer_%d_grade_%d" % (scorer,grade) for grade in scores]
predictions["pred_scorer_3"] = np.array(predictions).argmax(axis=1)
predictions.to_csv(save_as,index=False)
print kappa.quadratic_weighted_kappa(essays.meta_data()["score3"][trainset],predictions["pred_scorer_3"][trainset])
def report(path):
dataset = joblib.load(DATASETS_PATH + "dataset_2_sample_size_1")
all_pred = pickle.load(open(path))["predictions"]
result = {}
"""
for key in dataset[0]:
wrong_records = dataset[0][key]["META_LABEL"]=="label"
assert np.all(all_pred[key][wrong_records,:]==np.zeros((1,2))), "0 prediction for bad records"
assert dataset[0][key].shape[0] == all_pred[key].shape[0], "all records wrong"
dataset[0][key] = dataset[0][key].ix[~wrong_records]
all_pred[key] = all_pred[key][~wrong_records,:]
"""
for key in dataset[0]:
validation_set = np.where(dataset[0][key]["META_LABEL"]=="VALIDATION")[0]
train_set = np.where(dataset[0][key]["META_LABEL"]=="TRAINING")[0]
predictions = np.zeros((len(validation_set)+len(train_set),2))
for scorer in [1,2]:
predictions[:,scorer - 1] = all_pred[key][:,scorer-1]
merged_pred = predictions.mean(axis=1)
predictions_scores_df = pd.DataFrame({'response':np.array(dataset[0][key]["META_SCORE_1"])[train_set],
'predictions': merged_pred[train_set]})
#predictions_scores_df = predictions_scores_df.ix[~predictions_scores_df["response"].isnull(),:]
predictions_scores_df["response"] = predictions_scores_df["response"].fillna(0).map(int)
filter_valid_responses = ~predictions_scores_df["response"].isnull()
train_goc = grade_on_a_curve(merged_pred[train_set], dataset[0][key]["META_SCORE_%d" % scorer].fillna(0)[train_set].map(int))
kappa_goc = kappa.quadratic_weighted_kappa(train_goc, predictions_scores_df["response"].fillna(0))
result[key] = kappa_goc
return result
def create_model(dataset_name,
model,
model_name,
overwrite=False,
parallel=True):
args_string = "_".join(
["%s=%s" % (k, v) for k, v in model.get_params().items()])
args_hash = get_md5_hash(args_string)
filename = MODELS_PATH + model_name + "_" + dataset_name + "_" + args_hash
print "Creating", filename
if os.path.exists(filename) and not overwrite:
print "It exists so skipping..."
return
# touch file so that other processes don't recreate it
touch(filename)
dataset = joblib.load(DATASETS_PATH + dataset_name)
all_pred = {}
n_datasets = len(dataset[0].keys())
# for each essay
for i, key in enumerate(dataset[0]):
print "Creating %d of %d" % (i + 1, n_datasets)
predictions = np.zeros((dataset[0][key].shape[0], 2))
trainset = np.where(dataset[0][key]["META_LABEL"] == "TRAINING")[0]
testset = np.where(dataset[0][key]["META_LABEL"] == "VALIDATION")[0]
print "Training #", len(trainset), "Testing #", len(testset)
# 2 datasets - each for 1 scorer
for scorer in [1, 2]:
ds = dataset[scorer - 1][key]
kf = cross_validation.KFold(len(trainset), n_folds=7)
kf = [(trainset[tr], trainset[te])
for tr, te in kf] + [(trainset, testset)]
pred = np.zeros(ds.shape[0])
if parallel:
pool = Pool(processes=8)
essay_sets = pool.map(cv, [[kf, ds, model, n, scorer]
for n in range(8)])
pool.close()
for n, essay_set in enumerate(essay_sets):
pred[kf[n][1]] = essay_set
else:
for n in range(8):
pred[kf[n][1]] = cv([kf, ds, model, n, scorer])
predictions[:, scorer - 1] = pred
# DEBUG
pred_goc = grade_on_a_curve(
pred[trainset],
ds["META_SCORE_%d" % scorer].fillna(0)[trainset].map(int))
print key, scorer, kappa.quadratic_weighted_kappa(
pred_goc,
ds["META_SCORE_%d" % (scorer)].fillna(0)[trainset].map(int))
#DEBUG
merged_pred = predictions.mean(axis=1)
pred_goc = grade_on_a_curve(
merged_pred[trainset],
ds["META_SCORE_%d" % scorer].fillna(0)[trainset].map(int))
print key, "Merged", kappa.quadratic_weighted_kappa(
pred_goc,
ds["META_SCORE_%d" % (scorer)].fillna(0)[trainset].map(int))
print
all_pred[key] = predictions
data = {
"predictions": all_pred,
"model_name": model_name,
"dataset": dataset_name,
"parameters": model.get_params()
}
pickle.dump(data, open(filename, "w"))
def opt_fn(coef):
predictions_vector = np.dot(predictions_matrix[selected_obs], coef)
predictions_grade = np.argmax(predictions_vector, axis=1)
weighted_kappa = kappa.quadratic_weighted_kappa(
real_scores[selected_obs], predictions_grade)
return -weighted_kappa
results.write("essay,model,score\n")
for essaypath in sorted(glob.glob("data/csv/*.csv")):
print essaypath,
essays = EssayCollection(essaypath)
essayname = os.path.split(essaypath)[-1][:-4]
predictions_all_list = []
trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0]
for modelpath in glob.glob("models/*" + essayname + "*"):
modelname = os.path.split(modelpath)[-1]
predictions = pd.read_csv(modelpath)
predictions["modelname"] = modelname
results.write("%s,%s,%.6f\n" %
(essayname, modelname[8:],
kappa.quadratic_weighted_kappa(
essays.meta_data()["score3"][trainset],
predictions["pred_scorer_3"][trainset])))
predictions_all_list.append(predictions.copy())
results.write("%s,%s,%.6f\n" %
(essayname, "HUMAN",
kappa.quadratic_weighted_kappa(
essays.meta_data()["score1"][trainset],
essays.meta_data()["score2"][trainset])))
predictions_all = pd.concat(predictions_all_list)
scores = [col for col in predictions_all.columns if col.find("grade") > 0]
for score_col in scores:
predictions_all.ix[:, score_col] = predictions_all.ix[:,
score_col].map(
def opt_fn(coef):
predictions_vector = np.dot(predictions_matrix[selected_obs],coef)
predictions_grade = np.argmax(predictions_vector,axis=1)
weighted_kappa = kappa.quadratic_weighted_kappa(real_scores[selected_obs], predictions_grade)
return -weighted_kappa
from sklearn.linear_model import ElasticNet
results = open("results.csv","w")
results.write("essay,model,score\n")
for essaypath in sorted(glob.glob("data/csv/*.csv")):
print essaypath,
essays = EssayCollection(essaypath)
essayname = os.path.split(essaypath)[-1][:-4]
predictions_all_list = []
trainset = np.where(essays.meta_data().essay_type=="TRAINING")[0]
for modelpath in glob.glob("models/*" + essayname + "*"):
modelname = os.path.split(modelpath)[-1]
predictions = pd.read_csv(modelpath)
predictions["modelname"] = modelname
results.write("%s,%s,%.6f\n" % (essayname, modelname[8:], kappa.quadratic_weighted_kappa(essays.meta_data()["score3"][trainset],predictions["pred_scorer_3"][trainset])))
predictions_all_list.append(predictions.copy())
results.write("%s,%s,%.6f\n" % (essayname, "HUMAN", kappa.quadratic_weighted_kappa(essays.meta_data()["score1"][trainset],essays.meta_data()["score2"][trainset])))
predictions_all = pd.concat(predictions_all_list)
scores = [col for col in predictions_all.columns if col.find("grade") > 0]
for score_col in scores:
predictions_all.ix[:,score_col] = predictions_all.ix[:,score_col].map(logit)
unique_models = predictions_all.modelname.unique()
n_obs = essays.meta_data().shape[0]
predictions_matrix = np.zeros((len(scores), unique_models.shape[0], n_obs))
for n_score, score in enumerate(scores):
for n_model, model in enumerate(unique_models):
np.array(dataset[0][key]["META_SCORE_1"])[train_set],
'predictions':
merged_pred[train_set]
})
predictions_scores_df["response"][
predictions_scores_df["response"].isnull()] = 0
predictions_scores_df["response"] = predictions_scores_df["response"].map(
int)
#predictions_scores_df["response"] = predictions_scores_df["response"].map(int)
#predictions_scores_df["response"] = predictions_scores_df["response"].fillna(0)
train_goc = grade_on_a_curve(predictions_scores_df["predictions"],
predictions_scores_df["response"]).astype(
np.int)
kappa_goc = kappa.quadratic_weighted_kappa(
train_goc, predictions_scores_df["response"])
kappa_raw = kappa.quadratic_weighted_kappa(
predictions_scores_df["predictions"].round().map(int),
predictions_scores_df["response"])
print key, kappa_goc, kappa_raw
pred_goc = grade_on_a_curve(merged_pred[validation_set],
predictions_scores_df["response"]).astype(
np.int)
print set(pred_goc), set(predictions_scores_df["response"])
assert set(pred_goc) == set(
predictions_scores_df["response"]), "wrong set of values"
result[
key] = pred_goc #kappa.quadratic_weighted_kappa(pred_goc, dataset[0][key]["META_SCORE_FINAL"].fillna(0).astype(np.int))
def model_generic_1ofK_clas(pipeline,
model_name,
model_f,
essays_paths,
parallel=False):
print model_name
for essaypath in sorted(essays_paths):
print essaypath,
essayname = essaypath.split("/")[-1].split(".")[0]
save_as = MODEL_PATHS + essayname + "_" + pipeline[
"name"] + "_" + model_name
if os.path.exists(save_as):
print "Skipping"
continue
essays = EssayCollection(essaypath, essayname)
essays.apply_datasteps(pipeline["steps"])
essays.create_feature_matrix(min_sparsity=5)
predictions = pd.DataFrame({'id': range(essays.meta_data().shape[0])})
trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0]
testset = np.where(essays.meta_data().essay_type == "")[0]
X_all = np.array(essays.feature_matrix.todense(), dtype=np.float32)
y_all = np.array(essays.meta_data()["score3"].map(int), dtype=np.int32)
non_duplicated = get_nonduplicate_columns(pd.DataFrame(X_all))
print "orig dimensions", X_all.shape[1],
X_all = X_all[:, non_duplicated]
print "reduced dimensions", X_all.shape[1],
predictions = pd.DataFrame({'id': range(essays.meta_data().shape[0])})
trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0]
testset = np.where(essays.meta_data().essay_type == "VALIDATION")[0]
for scorer in [3]:
kf = cross_validation.KFold(len(trainset),
n_folds=7,
random_state=0)
kf = [(trainset[tr], trainset[te])
for tr, te in kf] + [(trainset, testset)]
for grade in sorted(essays.meta_data()["score%d" %
(scorer)].unique()):
y_all = np.array(essays.meta_data()["score%d" % (scorer)].map(
lambda y: 1 if int(y) == int(grade) else 0))
pred_name = "scorer_%d_grade_%d" % (scorer, grade)
predictions[pred_name] = 0
if parallel:
pool = Pool(processes=4)
essay_sets = pool.map(cv, [[kf, X_all, y_all, model_f, n]
for n in range(8)])
pool.close()
for n, essay_set in enumerate(essay_sets):
te_ind = kf[n][1]
predictions.ix[te_ind, pred_name] = essay_set[:, 1]
else:
for n, (tr_ind, te_ind) in enumerate(kf):
predictions.ix[te_ind, pred_name] = model_f(
X_all[tr_ind, :],
y_all[tr_ind],
X_all[te_ind, :],
feature_names=essays.feature_names)
predictions = predictions.ix[:, sorted(predictions.columns)]
predictions["pred_scorer_3"] = np.array(
predictions.
ix[:, [c for c in predictions.columns
if c.startswith("scorer_3")]]).argmax(axis=1)
predictions.to_csv(save_as, index=False)
print kappa.quadratic_weighted_kappa(
essays.meta_data()["score3"][trainset],
predictions["pred_scorer_3"][trainset])
def model_generic_DBN(pipeline,
model_name,
model_f,
essays_paths,
parallel=False):
print model_name
for essaypath in sorted(essays_paths):
print essaypath,
essayname = essaypath.split("/")[-1].split(".")[0]
save_as = MODEL_PATHS + essayname + "_" + pipeline[
"name"] + "_" + model_name
if os.path.exists(save_as):
print "Skipping"
continue
essays = EssayCollection(essaypath, essayname)
essays.apply_datasteps(pipeline["steps"])
essays.create_feature_matrix(min_sparsity=5)
predictions = pd.DataFrame({'id': range(essays.meta_data().shape[0])})
trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0]
testset = np.where(essays.meta_data().essay_type == "")[0]
X_all = np.array(essays.feature_matrix.todense(), dtype=np.float32)
y_all = np.array(essays.meta_data()["score3"].map(int), dtype=np.int32)
non_duplicated = get_nonduplicate_columns(pd.DataFrame(X_all))
print "orig dimensions", X_all.shape[1],
X_all = X_all[:, non_duplicated]
print "reduced dimensions", X_all.shape[1],
predictions = pd.DataFrame({'id': range(essays.meta_data().shape[0])})
trainset = np.where(essays.meta_data().essay_type == "TRAINING")[0]
testset = np.where(essays.meta_data().essay_type == "VALIDATION")[0]
scorer = 3
kf = cross_validation.KFold(len(trainset), n_folds=7, random_state=0)
kf = [(trainset[tr], trainset[te])
for tr, te in kf] + [(trainset, testset)]
scores = sorted(essays.meta_data()["score%d" % (scorer)].unique())
predictions = np.zeros((essays.meta_data().shape[0], len(scores)))
model_f.layer_sizes[0] = X_all.shape[1]
model_f.layer_sizes[2] = len(scores)
try:
for n, (tr_ind, te_ind) in enumerate(kf):
print n
scaler = StandardScaler()
_ = scaler.fit(X_all[tr_ind, :])
X_tr = scaler.transform(X_all[tr_ind, :]) / 50.0
X_te = scaler.transform(X_all[te_ind, :]) / 50.0
model_f.fit(X_tr, y_all[tr_ind])
print kappa.quadratic_weighted_kappa(
essays.meta_data()["score3"][tr_ind],
model_f.predict(X_tr))
print kappa.quadratic_weighted_kappa(
essays.meta_data()["score3"][te_ind],
model_f.predict(X_te))
predictions[te_ind, :] = model_f.predict_proba(X_te)
except:
pass
predictions = pd.DataFrame(predictions)
predictions.columns = [
"scorer_%d_grade_%d" % (scorer, grade) for grade in scores
]
predictions["pred_scorer_3"] = np.array(predictions).argmax(axis=1)
predictions.to_csv(save_as, index=False)
print kappa.quadratic_weighted_kappa(
essays.meta_data()["score3"][trainset],
predictions["pred_scorer_3"][trainset])
train_set = np.where(dataset[0][key]["META_LABEL"] == "TRAINING")[0]
predictions = np.zeros((len(validation_set) + len(train_set), 2))
for scorer in [1, 2]:
predictions[:, scorer - 1] = all_pred[key][:, scorer - 1]
merged_pred = predictions.mean(axis=1)
predictions_scores_df = pd.DataFrame(
{"response": np.array(dataset[0][key]["META_SCORE_1"])[train_set], "predictions": merged_pred[train_set]}
)
predictions_scores_df["response"][predictions_scores_df["response"].isnull()] = 0
predictions_scores_df["response"] = predictions_scores_df["response"].map(int)
train_goc = grade_on_a_curve(predictions_scores_df["predictions"], predictions_scores_df["response"]).astype(np.int)
kappa_goc = kappa.quadratic_weighted_kappa(train_goc, predictions_scores_df["response"])
kappa_raw = kappa.quadratic_weighted_kappa(
predictions_scores_df["predictions"].round().map(int), predictions_scores_df["response"]
)
print key, kappa_goc, kappa_raw
pred_goc = grade_on_a_curve(merged_pred[validation_set], predictions_scores_df["response"]).astype(np.int)
print set(pred_goc), set(predictions_scores_df["response"])
assert set(pred_goc) == set(predictions_scores_df["response"]), "wrong set of values"
result[
key
] = (
pred_goc
) # kappa.quadratic_weighted_kappa(pred_goc, dataset[0][key]["META_SCORE_FINAL"].fillna(0).astype(np.int))
