def test_median_kappa_on_idrid():
y_preds = []
for index, predictions, in enumerate(idrid_predictions):
if not isinstance(predictions, pd.DataFrame):
predictions = pd.read_pickle(predictions)
y_true = predictions['diagnosis'].values
y_pred = predictions['ordinal'].values
y_preds.append(y_pred)
print(
'Score on Idrid-test', index,
cohen_kappa_score(y_true,
regression_to_class(y_pred),
weights='quadratic'))
y_preds = np.row_stack(y_preds)
y_pred_median = np.median(y_preds, axis=0)
y_pred_avg = np.mean(y_preds, axis=0)
print(
'Score on Idrid-test',
cohen_kappa_score(y_true,
regression_to_class(y_pred_avg),
weights='quadratic'),
cohen_kappa_score(y_true,
regression_to_class(y_pred_median),
weights='quadratic'))
def test_optimize_kappa_on_idrid():
average_predictions = None
for index, predictions, in enumerate(idrid_predictions):
if not isinstance(predictions, pd.DataFrame):
predictions = pd.read_pickle(predictions)
y_true = predictions['diagnosis'].values
y_pred = predictions['ordinal'].values
if average_predictions is None:
average_predictions = y_pred.copy()
else:
average_predictions += y_pred
print(
'Score on Idrid-test', index,
cohen_kappa_score(y_true,
regression_to_class(y_pred),
weights='quadratic'))
average_predictions /= len(idrid_predictions)
rounder = OptimizedRounder()
rounder.fit(average_predictions, y_true)
print(rounder.coefficients())
print(
'Score on Idrid-test',
cohen_kappa_score(y_true,
regression_to_class(average_predictions),
weights='quadratic'),
cohen_kappa_score(y_true,
regression_to_class(
average_predictions,
rounding_coefficients=rounder.coefficients()),
weights='quadratic'))
def test_pseudolabeling_aptos2015_round1(predictions, output_csv):
print('Saving pseudolabels to ', output_csv)
num_models = len(predictions)
ids, x, y_true, y_average = prepare_inference_datasets(
predictions, use_features=False, use_predictions=True)
for i in range(num_models):
print(
fs.id_from_fname(predictions[i]),
cohen_kappa_score(y_true,
regression_to_class(x[:, i]),
weights='quadratic'))
y_round = to_numpy(regression_to_class(x))
y_major = majority_voting(y_round, axis=1)
y_agreement = y_round == np.expand_dims(y_major, -1)
# y_agreement_all = np.all(y_agreement, axis=1)
# y_agreement_all = np.sum(y_agreement, axis=1) >= 16
y_agreement_all = y_major == y_true
print('Agreement', np.mean(y_agreement_all))
print('Distribution', np.bincount(y_major[y_agreement_all]))
y_true[~y_agreement_all] = -100
print(y_round)
df = pd.DataFrame.from_dict({'id_code': ids, 'diagnosis': y_true})
df.to_csv(output_csv, index=None)
def test_optimize_kappa_on_aptos2015_v2():
pl1 = pd.read_csv(
'../data/aptos-2015/test_private_pseudolabel_round_1.csv')
labeled_gt = dict(
(row['id_code'], row['diagnosis']) for i, row in pl1.iterrows())
ids, train_x, train_y, train_y_avg = prepare_inference_datasets(
aptos2015_predictions, use_features=False, use_predictions=True)
mask = np.zeros(len(ids), dtype=np.bool)
for i, id in enumerate(ids):
mask[i] = labeled_gt[id] >= 0
ids = ids[mask]
train_x = train_x[mask]
train_y = train_y[mask]
train_y_avg = train_y_avg[mask]
ids, val_x, val_y, val_y_avg = prepare_inference_datasets(
idrid_predictions, use_features=False, use_predictions=True)
rounder = OptimizedRounderV2()
rounder.fit(train_x, train_y)
print(rounder.coefficients())
print(
'Score on APTOS',
cohen_kappa_score(train_y,
regression_to_class(train_y_avg),
weights='quadratic'),
cohen_kappa_score(train_y,
rounder.predict(train_x),
weights='quadratic'))
print(
'Score on IDRID',
cohen_kappa_score(val_y,
regression_to_class(val_y_avg),
weights='quadratic'),
cohen_kappa_score(val_y, rounder.predict(val_x), weights='quadratic'))
# Vice versa
rounder = OptimizedRounderV2()
rounder.fit(val_x, val_y)
print(rounder.coefficients())
print(
'Score on IDRID',
cohen_kappa_score(val_y,
regression_to_class(val_y_avg),
weights='quadratic'),
cohen_kappa_score(val_y, rounder.predict(val_x), weights='quadratic'))
print(
'Score on APTOS',
cohen_kappa_score(train_y,
regression_to_class(train_y_avg),
weights='quadratic'),
cohen_kappa_score(train_y,
rounder.predict(train_x),
weights='quadratic'))
def test_rf_on_idrid():
ids, train_x, train_y, train_y_avg = prepare_inference_datasets(
aptos2015_predictions, use_features=False, use_predictions=True)
ids, val_x, val_y, val_y_avg = prepare_inference_datasets(
idrid_predictions, use_features=False, use_predictions=True)
# {'criterion': 'gini', 'max_depth': 12, 'n_estimators': 64}
# RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
# max_depth=12, max_features='auto', max_leaf_nodes=None,
# min_impurity_decrease=0.0, min_impurity_split=None,
# min_samples_leaf=1, min_samples_split=2,
# min_weight_fraction_leaf=0.0, n_estimators=64,
# n_jobs=None, oob_score=False, random_state=None,
# verbose=0, warm_start=False)
# LR on Train 0.8058995695509186 0.9164135768322392
# LR on IDRID 0.8260826380611839 0.8900600400266845
# create a dictionary of all values we want to test for n_neighbors
params_rf = {
'n_estimators': [8, 16, 32, 64, 128],
'criterion': ['gini', 'entropy'],
'max_depth': [2, 4, 6, 8, 12],
}
forest_gs = GridSearchCV(RandomForestClassifier(),
params_rf,
cv=5,
verbose=1,
n_jobs=4)
forest_gs.fit(train_x, train_y)
print(forest_gs.best_params_)
print(forest_gs.best_estimator_)
print(
'LR on Train',
cohen_kappa_score(train_y,
regression_to_class(train_y_avg),
weights='quadratic'),
cohen_kappa_score(train_y,
forest_gs.best_estimator_.predict(train_x),
weights='quadratic'))
print(
'LR on IDRID',
cohen_kappa_score(val_y,
regression_to_class(val_y_avg),
weights='quadratic'),
cohen_kappa_score(val_y,
forest_gs.best_estimator_.predict(val_x),
weights='quadratic'))
with open('forest.pkl', 'wb') as f:
pickle.dump(forest_gs.best_estimator_, f)
def test_stack_with_ada_boost():
pl1 = pd.read_csv(
'../data/aptos-2015/test_private_pseudolabel_round_1.csv')
labeled_gt = dict(
(row['id_code'], row['diagnosis']) for i, row in pl1.iterrows())
ids, train_x, train_y, train_y_avg = prepare_inference_datasets(
aptos2015_predictions, use_features=False, use_predictions=True)
mask = np.zeros(len(ids), dtype=np.bool)
for i, id in enumerate(ids):
mask[i] = labeled_gt[id] >= 0
ids = ids[mask]
train_x = train_x[mask]
train_y = train_y[mask]
train_y_avg = train_y_avg[mask]
_, val_x, val_y, val_y_avg = prepare_inference_datasets(
idrid_predictions, use_features=False, use_predictions=True)
from sklearn.ensemble import AdaBoostClassifier
clf = AdaBoostClassifier(n_estimators=100)
scores = cross_val_score(clf, train_x, train_y, cv=5)
print(scores)
print(scores.mean())
clf = AdaBoostClassifier(n_estimators=100)
clf.fit(train_x, train_y)
y_pred = clf.predict(train_x)
df = pd.DataFrame.from_dict({
'id_code': ids,
'y_true': train_y,
'y_pred': y_pred
})
negatives = df[df['y_pred'] != df['y_true']]
negatives.to_csv('aptos_negatives.csv', index=None)
print(
'Score on APTOS',
cohen_kappa_score(train_y,
regression_to_class(train_y_avg),
weights='quadratic'),
cohen_kappa_score(train_y, clf.predict(train_x), weights='quadratic'))
print(
'Score on IDRID',
cohen_kappa_score(val_y,
regression_to_class(val_y_avg),
weights='quadratic'),
cohen_kappa_score(val_y, clf.predict(val_x), weights='quadratic'))
def test_knn_on_idrid():
ids, train_x, train_y, train_y_avg = prepare_inference_datasets(
aptos2015_predictions, use_features=False, use_predictions=True)
ids, val_x, val_y, val_y_avg = prepare_inference_datasets(
idrid_predictions, use_features=False, use_predictions=True)
# {'algorithm': 'ball_tree', 'leaf_size': 8, 'n_neighbors': 64, 'p': 1, 'weights': 'distance'}
# KNeighborsClassifier(algorithm='ball_tree', leaf_size=8, metric='minkowski',
# metric_params=None, n_jobs=None, n_neighbors=64, p=1,
# weights='distance')
# LR on Train 0.8058995695509186 1.0
# LR on IDRID 0.8260826380611839 0.8692778993435448
# create a dictionary of all values we want to test for n_neighbors
params_knn = {
'n_neighbors': [8, 16, 32, 64, 128],
'weights': ['uniform', 'distance'],
'p': [1, 2],
'algorithm': ['ball_tree', 'kd_tree'],
'leaf_size': [8, 16, 32, 64, 128]
}
knn_gs = GridSearchCV(KNeighborsClassifier(),
params_knn,
cv=5,
verbose=1,
n_jobs=4)
knn_gs.fit(train_x, train_y)
print(knn_gs.best_params_)
print(knn_gs.best_estimator_)
print(
'LR on Train',
cohen_kappa_score(train_y,
regression_to_class(train_y_avg),
weights='quadratic'),
cohen_kappa_score(train_y,
knn_gs.best_estimator_.predict(train_x),
weights='quadratic'))
print(
'LR on IDRID',
cohen_kappa_score(val_y,
regression_to_class(val_y_avg),
weights='quadratic'),
cohen_kappa_score(val_y,
knn_gs.best_estimator_.predict(val_x),
weights='quadratic'))
with open('knn.pkl', 'wb') as f:
pickle.dump(knn_gs.best_estimator_, f)
def test_logistic_regression_on_idrid():
ids, train_x, train_y, train_y_avg = prepare_inference_datasets(
aptos2015_predictions, use_features=False, use_predictions=True)
ids, val_x, val_y, val_y_avg = prepare_inference_datasets(
idrid_predictions, use_features=False, use_predictions=True)
params_lr = {
'class_weight': ['balanced', None],
'multi_class': ['multinomial', 'auto', 'ovr'],
'solver': ['newton-cg', 'lbfgs'],
'max_iter': [100, 250, 500, 1000, 2000, 5000],
'fit_intercept': [True, False],
'random_state': [42]
}
# {'class_weight': None, 'fit_intercept': True, 'max_iter': 100, 'multi_class': 'multinomial', 'random_state': 42, 'solver': 'lbfgs'}
# LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,
# intercept_scaling=1, l1_ratio=None, max_iter=100,
# multi_class='multinomial', n_jobs=None, penalty='l2',
# random_state=42, solver='lbfgs', tol=0.0001, verbose=0,
# warm_start=False)
# LR on Train 0.8058995695509186 0.8408521184285193
# LR on IDRID 0.8260826380611839 0.8655784001198091
lr_gs = GridSearchCV(LogisticRegression(), params_lr, cv=5, verbose=1)
lr_gs.fit(train_x, train_y)
print(lr_gs.best_params_)
print(lr_gs.best_estimator_)
print(
'LR on Train',
cohen_kappa_score(train_y,
regression_to_class(train_y_avg),
weights='quadratic'),
cohen_kappa_score(train_y,
lr_gs.best_estimator_.predict(train_x),
weights='quadratic'))
print(
'LR on IDRID',
cohen_kappa_score(val_y,
regression_to_class(val_y_avg),
weights='quadratic'),
cohen_kappa_score(val_y,
lr_gs.best_estimator_.predict(val_x),
weights='quadratic'))
with open('logistic_regression.pkl', 'wb') as f:
pickle.dump(lr_gs.best_estimator_, f)
def test_round():
x = torch.tensor([
-0.9, -0.2, 0.2, 0.5, 0.7, 1.1, 1.4, 1.5, 1.6, 2.4, 2.5, 2.6, 3.3, 3.5,
3.9, 4, 4.5, 5
])
y = regression_to_class(x)
print(x)
print(y)
def test_evaluate_model(predictions):
num_models = len(predictions)
ids, x, y_true, y_average = prepare_inference_datasets(
predictions, use_features=False, use_predictions=True)
for i in range(num_models):
print(
fs.id_from_fname(predictions[i]),
cohen_kappa_score(y_true,
regression_to_class(x),
weights='quadratic'))
def test_pseudolabeling_aptos2019_round1():
ids, x, y_true, y_average = prepare_inference_datasets(
aptos2019_predictions, use_features=False, use_predictions=True)
y_round = to_numpy(regression_to_class(x))
y_major = majority_voting(y_round, axis=1)
y_agreement = y_round == np.expand_dims(y_major, -1)
y_agreement_all = np.all(y_agreement, axis=1)
y_agreement_all = np.sum(y_agreement, axis=1) >= 16
print('Agreement', np.mean(y_agreement_all))
print('Distribution', np.bincount(y_major[y_agreement_all]))
y_true[y_agreement_all] = y_major[y_agreement_all]
print(y_round)
df = pd.DataFrame.from_dict({'id_code': ids, 'diagnosis': y_true})
df.to_csv('../data/aptos-2019/test_pseudolabel_round_1.csv', index=None)
def test_pseudolabeling_messirod_2_round1(predictions, output_csv):
ids, x, y_true, y_average = prepare_inference_datasets(
predictions, use_features=False, use_predictions=True)
y_round = to_numpy(regression_to_class(x))
y_major = majority_voting(y_round, axis=1)
y_agreement = y_round == np.expand_dims(y_major, -1)
num_models = x.shape[1]
y_agreement_most = np.sum(y_agreement, axis=1) >= int(0.75 * num_models)
# y_agreement_all = y_major == y_true
print('Agreement', np.mean(y_agreement_most))
print('Distribution', np.bincount(y_major[y_agreement_most]))
y_major[~y_agreement_most] = -100
print(y_round)
df = pd.DataFrame.from_dict({'id_code': ids, 'diagnosis': y_major})
df.to_csv(output_csv, index=None)
def main():
f0_aptos15 = pd.read_pickle(
fs.auto_file(
'reg_seresnext50_rms_512_medium_mse_aptos2019_fold0_awesome_babbage_on_aptos2019_fold0.pkl'
))
f0_aptos15['fold'] = 0
f0_idrid = pd.read_pickle(
fs.auto_file(
'reg_seresnext50_rms_512_medium_mse_idrid_fold0_heuristic_ptolemy_on_aptos2019_fold0.pkl'
))
f0_idrid['fold'] = 0
f1_aptos15 = pd.read_pickle(
fs.auto_file(
'reg_seresnext50_rms_512_medium_mse_aptos2019_fold1_hopeful_khorana_on_aptos2019_fold1.pkl'
))
f1_aptos15['fold'] = 1
f1_idrid = pd.read_pickle(
fs.auto_file(
'reg_seresnext50_rms_512_medium_mse_idrid_fold1_gifted_visvesvaraya_on_aptos2019_fold1.pkl'
))
f1_idrid['fold'] = 1
f2_aptos15 = pd.read_pickle(
fs.auto_file(
'reg_seresnext50_rms_512_medium_mse_aptos2019_fold2_trusting_nightingale_on_aptos2019_fold2.pkl'
))
f2_aptos15['fold'] = 2
f2_idrid = pd.read_pickle(
fs.auto_file(
'reg_seresnext50_rms_512_medium_mse_idrid_fold2_sharp_brattain_on_aptos2019_fold2.pkl'
))
f2_idrid['fold'] = 2
f3_aptos15 = pd.read_pickle(
fs.auto_file(
'reg_seresnext50_rms_512_medium_mse_aptos2019_fold3_epic_wing_on_aptos2019_fold3.pkl'
))
f3_aptos15['fold'] = 3
f3_idrid = pd.read_pickle(
fs.auto_file(
'reg_seresnext50_rms_512_medium_mse_idrid_fold3_vibrant_minsky_on_aptos2019_fold3.pkl'
))
f3_idrid['fold'] = 3
df_aptos15 = pd.concat([f0_aptos15, f1_aptos15, f2_aptos15, f3_aptos15])
df_idrid = pd.concat([f0_idrid, f1_idrid, f2_idrid, f3_idrid])
print(len(f0_aptos15), len(f1_aptos15), len(f2_aptos15), len(f3_aptos15),
len(df_aptos15))
# logits = np.array(df_aptos15['logits'].values.tolist())
regression = np.array(df_aptos15['regression'].values.tolist())
X = np.hstack((np.array(df_aptos15['features'].values.tolist()),
np.array(df_idrid['features'].values.tolist())))
Y = np.array(df_aptos15['diagnosis_true'].values.tolist())
print(X.shape, Y.shape)
x_train, x_test, y_train, y_test, y_hat_train, y_hat_test = train_test_split(
X,
Y,
to_numpy(regression_to_class(regression)),
stratify=Y,
test_size=0.25,
random_state=0)
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
x_train = sc.fit_transform(x_train)
x_test = sc.transform(x_test)
# d_train = lgb.Dataset(x_train, label=y_train.astype(np.float32))
#
# params = {}
# params['learning_rate'] = 0.003
# params['boosting_type'] = 'gbdt'
# params['objective'] = 'regression'
# params['metric'] = 'mse'
# params['sub_feature'] = 0.5
# params['num_leaves'] = 10
# params['min_data'] = 50
# # params['max_depth'] = 4
#
# clf = lgb.train(params, d_train, 1000)
#
# y_pred = clf.predict(x_test)
# y_pred_class = regression_to_class(y_pred)
cls = LGBMClassifier(n_estimators=64,
max_depth=10,
random_state=0,
num_leaves=256)
cls.fit(x_train, y_train)
y_pred_class = cls.predict(x_test)
raw_score, num, denom = cohen_kappa_score(y_hat_test,
y_test,
weights='quadratic')
print('raw_score', raw_score)
lgb_score, num, denom = cohen_kappa_score(y_pred_class,
y_test,
weights='quadratic')
print('lgb_score', lgb_score)
def test_evaluate_model_v2(train, validation):
num_models = len(train)
ids, train_x, train_y_true, train_y_average = prepare_inference_datasets(
train, use_features=False, use_predictions=True)
ids, valid_x, valid_y_true, valid_y_average = prepare_inference_datasets(
validation, use_features=False, use_predictions=True)
for i in range(num_models):
print(
fs.id_from_fname(train[i]),
cohen_kappa_score(train_y_true,
regression_to_class(train_x[:, i]),
weights='quadratic'),
cohen_kappa_score(train_y_true,
regression_to_class(valid_x[:, i]),
weights='quadratic'),
)
print(
'Averaged',
cohen_kappa_score(train_y_true,
regression_to_class(train_y_average),
weights='quadratic'),
cohen_kappa_score(valid_y_true,
regression_to_class(valid_y_average),
weights='quadratic'))
print(
'Median ',
cohen_kappa_score(train_y_true,
regression_to_class(np.median(train_x, axis=1)),
weights='quadratic'),
cohen_kappa_score(valid_y_true,
regression_to_class(np.median(valid_x, axis=1)),
weights='quadratic'))
print(
'TrimMean',
cohen_kappa_score(train_y_true,
regression_to_class(
trim_mean(train_x, proportiontocut=0.1, axis=1)),
weights='quadratic'),
cohen_kappa_score(valid_y_true,
regression_to_class(
trim_mean(valid_x, proportiontocut=0.1, axis=1)),
weights='quadratic'))
rounder = OptimizedRounder()
rounder.fit(train_y_average, train_y_true)
print(rounder.coefficients())
print(
'Optimized',
cohen_kappa_score(train_y_true,
rounder.predict(train_y_average,
rounder.coefficients()),
weights='quadratic'),
cohen_kappa_score(valid_y_true,
rounder.predict(valid_y_average,
rounder.coefficients()),
weights='quadratic'))