def evaluate(self, predictions, predictions_interpolated):
"""
Function responsible for computing evaluation metrics (C-index, Brier score).
Also responsible for logging said metrics to MLflow using my Logger module.
"""
self.logger.info("Evaluating Model Strength.. \n")
targets = dataset_2(self.dataset_filtered, phase='test',
targets=True)[0][0]
events = dataset_2(self.dataset_filtered, phase='test',
targets=True)[0][1]
survs = dataset_2(self.dataset_filtered, phase='test',
targets=True)[0][2]
censored = []
for surv in events:
if surv == 1:
censored.append(False)
else:
censored.append(True)
self.logger.info("Calculating concordance and brier score\n")
ev = EvalSurv(predictions_interpolated, survs, events, 'km')
concordance = ev.concordance_td()
time_grid = np.linspace(0, survs.max())
integrated_brier_score = ev.integrated_brier_score(time_grid)
self.logger.info(
"Concordance: {}, Integrated Brier Score: {}\n".format(
concordance, integrated_brier_score))
logger = Logger(self.mlflow_url, "CNN_Predictions")
logger.log_predictions(predictions, "{}".format(self.mode),
concordance, integrated_brier_score,
self.params, targets, self.cuts)
self.logger.info("Logged Evaluation metrics to MLFlow\n")
def metric_fn_par(args):
predicted_test_times, predicted_survival_functions, observed_y = args
# predicted survival function dim: n_train * n_test
obs_test_times = observed_y[:, 0].astype('float32')
obs_test_events = observed_y[:, 1].astype('float32')
results = [0, 0, 0, 0, 0]
ev = EvalSurv(predicted_survival_functions, obs_test_times, obs_test_events, censor_surv='km')
results[0] = ev.concordance_td('antolini') # concordance_antolini
results[1] = concordance_index(obs_test_times, predicted_test_times, obs_test_events.astype(np.bool)) # concordance_median
# we ignore brier scores at the highest test times because it becomes unstable
time_grid = np.linspace(obs_test_times.min(), obs_test_times.max(), 100)[:80]
results[2] = ev.integrated_brier_score(time_grid) # integrated_brier
if sum(obs_test_events) > 0:
# only noncensored samples are used for rmse/mae calculation
pred_obs_differences = predicted_test_times[obs_test_events.astype(np.bool)] - obs_test_times[obs_test_events.astype(np.bool)]
results[3] = np.sqrt(np.mean((pred_obs_differences)**2)) # rmse
results[4] = np.mean(np.abs(pred_obs_differences)) # mae
else:
print("[WARNING] All samples are censored.")
results[3] = 0
results[4] = 0
return results
def output_sim_data(model, surv, X_train, df_train, X_test, df_test):
""" Compute the output of the model on the test set
# Arguments
model: neural network model trained with final parameters.
X_train : input variables of the training set
df_train: training dataset
X_val : input variables of the validation set
df_val: validation dataset
# Returns
results_test: Uno C-index at median survival time and Integrated Brier Score
"""
time_grid = np.linspace(np.percentile(df_test['yy'], 10),
np.percentile(df_test['yy'], 90), 100)
median_time = np.percentile(df_test['yy'], 50)
data_train = skSurv.from_arrays(event=df_train['status'],
time=df_train['yy'])
data_test = skSurv.from_arrays(event=df_test['status'], time=df_test['yy'])
c_med = concordance_index_ipcw(
data_train, data_test,
np.array(-determine_surv_prob(surv, median_time)), median_time)[0]
ev = EvalSurv(surv,
np.array(df_test['yy']),
np.array(df_test['status']),
censor_surv='km')
ibs = ev.integrated_brier_score(time_grid)
res = pd.DataFrame([c_med, ibs]).T
res.columns = ['c_median', 'ibs']
return res
def test_quality(t_true,
y_true,
pred,
time_grid=np.linspace(0, 300, 30, dtype=np.int),
concordance_at_t=None,
plot=False):
# get survival proba for time_grid
all_surv_time = pd.DataFrame()
for t in time_grid:
surv_prob = np.exp(-1 * np.power(t / (pred[:, 0] + 1e-6), pred[:, 1]))
all_surv_time = pd.concat([all_surv_time, pd.DataFrame(surv_prob).T])
all_surv_time.index = time_grid
ev = EvalSurv(surv=all_surv_time,
durations=t_true,
events=y_true,
censor_surv='km')
dt_c_index = ev.concordance_td('antolini')
int_brier_score = ev.integrated_brier_score(time_grid)
int_nbill = ev.integrated_nbll(time_grid)
if plot:
fig, ax = plt.subplots(1, 3, figsize=(20, 7))
d = all_surv_time.sample(5, axis=1).loc[1:]
obs = d.columns
for o in obs:
ax[0].plot(d.index, d[o])
ax[0].set_xlabel('Time')
ax[0].set_title("Sample survival curves")
nb = ev.nbll(time_grid)
ax[1].plot(time_grid, nb)
ax[1].set_title('NBLL')
ax[1].set_xlabel('Time')
br = ev.brier_score(time_grid)
ax[2].plot(time_grid, br)
ax[2].set_title('Brier score')
ax[2].set_xlabel('Time')
plt.show()
if concordance_at_t is not None:
harell_c_index = concordance_index(
predicted_scores=all_surv_time.loc[concordance_at_t, :].values,
event_times=t_true,
event_observed=y_true)
return pd.DataFrame([{
'harell_c_index': harell_c_index,
'dt_c_index': dt_c_index,
'int_brier_score': int_brier_score,
'int_nbill': int_nbill
}])
else:
return pd.DataFrame([{
'dt_c_index': dt_c_index,
'int_brier_score': int_brier_score,
'int_nbill': int_nbill
}])
def get_metrics(val_data, surv_pred, time_grid):
ev = EvalSurv(surv_pred, val_data['t'], val_data['y'], censor_surv='km')
return pd.DataFrame([{
'dt_c_index':
ev.concordance_td('antolini'),
'int_brier_score':
ev.integrated_brier_score(time_grid),
'int_nbill':
ev.integrated_nbll(time_grid)
}])
def Bootstrap(self, surv, event: list, duration: list):
np.random.seed(42) # control reproducibility
cindex, brier, nbll = [], [], []
for _ in range(self.bootstrap_n):
sampled_index = choices(range(surv.shape[1]), k=surv.shape[1])
sampled_surv = surv.iloc[:, sampled_index]
sampled_event = [event[i] for i in sampled_index]
sampled_duration = [duration[i] for i in sampled_index]
ev = EvalSurv(sampled_surv, np.array(sampled_duration),
np.array(sampled_event).astype(int), censor_surv='km')
time_grid = np.linspace(min(sampled_duration), max(sampled_duration), 100)
cindex.append(ev.concordance_td('antolini'))
brier.append(ev.integrated_brier_score(time_grid))
nbll.append(ev.integrated_nbll(time_grid))
return cindex, brier, nbll
def run_baseline(runs=10):
concordance = []
ibs = []
for i in tqdm(range(runs)):
df_train,df_test,df_val = load_data("./summaries/survival_data")
x_mapper, labtrans, train, val, x_test, durations_test, events_test, pca = transform_data(
df_train,df_test,df_val,'LogisticHazard', "standard", cols_standardize, log_columns, num_durations=100)
x_train, y_train = train
cols = ['PC'+str(i) for i in range(x_train.shape[1])] + ['duration','event']
pc_col = ['PC'+str(i) for i in range(x_train.shape[1])]
cox_train = pd.DataFrame(x_train,columns = pc_col)
cox_test = pd.DataFrame(x_test,columns=pc_col)
# cox_train.loc[:,pc_col] = x_train
cox_train.loc[:,["duration"]] = y_train[0]
cox_train.loc[:,'event'] = y_train[1]
# cox_train = cox_train.drop(columns=[i for i in list(df_train) if i not in cols])
# cox_test.loc[:,pc_col] = x_test
# cox_test = cox_test.drop(columns=[i for i in list(df_train) if i not in cols])
cox_train = cox_train.dropna()
cox_test = cox_test.dropna()
cph = CoxPHFitter().fit(cox_train, 'duration', 'event')
# cph.print_summary()
surv = cph.predict_survival_function(cox_test)
ev = EvalSurv(surv, durations_test, events_test, censor_surv='km')
concordance.append(ev.concordance_td('antolini'))
time_grid = np.linspace(durations_test.min(), durations_test.max(), 100)
ibs.append(ev.integrated_brier_score(time_grid))
print("Average concordance: %s"%np.mean(concordance))
print("Average IBS: %s"%np.mean(ibs))
plot_survival(cox_train,
pc_col,cph,'./survival/cox',baseline=True)
+ '%f second(s)' % elapsed)
surv_model.sub = 10
surv_df = surv_model.predict_surv_df(fold_X_val_std)
ev = EvalSurv(surv_df, fold_y_val[:, 0], fold_y_val[:, 1],
censor_surv='km')
sorted_fold_y_val = np.sort(np.unique(fold_y_val[:, 0]))
time_grid = np.linspace(sorted_fold_y_val[0],
sorted_fold_y_val[-1], 100)
surv = surv_df.to_numpy().T
cindex_scores.append(ev.concordance_td('antolini'))
integrated_brier_scores.append(
ev.integrated_brier_score(time_grid))
print(' c-index (td):', cindex_scores[-1])
print(' Integrated Brier score:',
integrated_brier_scores[-1])
cross_val_cindex = np.mean(cindex_scores)
cross_val_integrated_brier = np.mean(integrated_brier_scores)
print(hyperparam, ':', cross_val_cindex,
cross_val_integrated_brier, flush=True)
print(hyperparam, ':', cross_val_cindex,
cross_val_integrated_brier, flush=True,
file=train_metrics_file)
if cross_val_cindex > max_cindex:
max_cindex = cross_val_cindex
arg_max_cindex = hyperparam
# Training ======================================================================
epochs = args.epochs
callbacks = [tt.callbacks.EarlyStopping()]
verbose = True
log = model.fit(x_train,
y_train,
batch_size,
epochs,
callbacks,
verbose,
val_data=val,
val_batch_size=batch_size)
# log = model.fit(x_train, y_train_transformed, batch_size, epochs, callbacks, verbose, val_data = val_transformed, val_batch_size = batch_size)
# Evaluation ===================================================================
_ = model.compute_baseline_hazards()
surv = model.predict_surv_df(x_test)
ev = EvalSurv(surv, durations_test, events_test, censor_surv='km')
# ctd = ev.concordance_td()
ctd = ev.concordance_td()
time_grid = np.linspace(durations_test.min(), durations_test.max(), 100)
ibs = ev.integrated_brier_score(time_grid)
nbll = ev.integrated_nbll(time_grid)
val_loss = min(log.monitors['val_'].scores['loss']['score'])
wandb.log({'val_loss': val_loss, 'ctd': ctd, 'ibs': ibs, 'nbll': nbll})
wandb.finish()
model = initialize_model(dim,labtrans,in_features)
model.optimizer.set_lr(0.001)
callbacks = [tt.callbacks.EarlyStopping()]
log = model.fit(x_train, y_train, batch_size, epochs, callbacks, val_data=val)
surv = model.predict_surv_df(x_test)
ev = EvalSurv(surv, durations_test, events_test, censor_surv='km')
result = pd.DataFrame([[0]*8],columns=["random","model","hiddens",
"lr","scalers","c-index","brier","nll"])
result["c-index"] = ev.concordance_td('antolini')
print(ev.concordance_td('antolini') )
time_grid = np.linspace(durations_test.min(), durations_test.max(), 100)
result["brier"] = ev.integrated_brier_score(time_grid)
print(ev.integrated_brier_score(time_grid) )
result["nll"] = ev.integrated_nbll(time_grid)
result["lr"] = lr
result["model"] = mod
result["scaler"] = scale
result["random"] = seed
result["hiddens"] = dim
results = pd.concat([results,result],ignore_index=True)
results.to_csv(os.path.join(outpath,"results"))
pc_col = ['PC'+str(i) for i in range(x_train.shape[1])]
pc_train = pd.DataFrame(x_train,columns = pc_col)
plot_survival(pc_train,pc_col,model,outpath,duration=100)
y_train,
batch_size,
epochs,
callbacks,
verbose,
val_data=val)
_ = log.plot()
# Evaluation
surv = model.predict_surv_df(x_test)
ev = EvalSurv(surv, durations_test, events_test != 0, censor_surv='km')
ev.concordance_td()
ev.integrated_brier_score(np.linspace(0, durations_test.max(), 100))
cif = model.predict_cif(x_test)
cif1 = pd.DataFrame(cif[0], model.duration_index)
cif2 = pd.DataFrame(cif[1], model.duration_index)
ev1 = EvalSurv(1 - cif1, durations_test, events_test == 1, censor_surv='km')
ev2 = EvalSurv(1 - cif2, durations_test, events_test == 2, censor_surv='km')
ev1.concordance_td()
ev2.concordance_td()
sample = np.random.choice(len(durations_test), 6)
fig, axs = plt.subplots(2, 3, True, True, figsize=(10, 5))
for ax, idx in zip(axs.flat, sample):