def test_uno_auc_times_failure(uno_auc_times_failure_data):
y_train, y_test, times, match = uno_auc_times_failure_data
estimate = numpy.random.randn(y_test.shape[0])
with pytest.raises(ValueError,
match=match):
cumulative_dynamic_auc(y_train, y_test, estimate, times)
def train_coxph(data_df, r_splits):
c_index_at = []
c_index_30 = []
time_auc_30 = []
time_auc_60 = []
time_auc_365 = []
for i in range(len(r_splits)):
print("\nIteration %s"%(i))
#DATA PREP
df_train, df_val, df_test, df_test_30 = prepare_datasets(data_df, r_splits[i][2], r_splits[i][1], r_splits[i][0])
(data_x, data_y), (val_x, val_y), (test_x, test_y), (test_30_x, test_30_y) = df2array(data_df, df_train, df_val, df_test, df_test_30)
estimator = CoxPHSurvivalAnalysis(alpha=1e-04)
estimator.fit(data_x, data_y)
c_index_at.append(estimator.score(test_x, test_y))
c_index_30.append(estimator.score(test_30_x, test_30_y))
for time_x in [30, 60, 365]:
t_auc, t_mean_auc = cumulative_dynamic_auc(data_y, test_y, estimator.predict(test_x), time_x)
eval("time_auc_" + str(time_x)).append(t_auc[0])
print("C-index_30:", c_index_30[i])
print("C-index_AT:", c_index_at[i])
print("time_auc_30", time_auc_30[i])
print("time_auc_60", time_auc_60[i])
print("time_auc_365", time_auc_365[i])
return c_index_at, c_index_30, time_auc_30, time_auc_60, time_auc_365
def evaluate_performance(T_train, c_train, T_test, c_test, prediction, time_horizon,
num_causes=2, cause_names=["Cause 1", "Cause 2"]):
Harell_c_index = []
UNO_c_index = []
dynamic_auc = []
for _ in range(num_causes):
y_train = np.array([((c_train.loc[c_train.index[k]]== _ + 1), T_train.loc[T_train.index[k]]) for k in range(len(T_train))], dtype=[('Status', '?'), ('Survival_in_days', '<f8')])
y_test = np.array([((c_test.loc[c_test.index[k]]== _ + 1), T_test.loc[T_test.index[k]]) for k in range(len(T_test))], dtype=[('Status', '?'), ('Survival_in_days', '<f8')])
Harell_c_index.append(concordance_index(T_test, prediction[_ + 1], event_observed=(c_test==(_+1))*1))
tau = max(y_train['Survival_in_days'])
ci_tau = concordance_index_ipcw(y_train, y_test, 1 - prediction[_ + 1], tau=tau)[0]
UNO_c_index.append(ci_tau)
try:
dynamic_auc_val = cumulative_dynamic_auc(y_train, y_test, 1 - prediction[_ + 1], times=[time_horizon])[0][0]
except ValueError:
print('*warning: exception while calculating dynamic_auc, dynamic_auc is not calculated*')
dynamic_auc_val = "-"
dynamic_auc.append(dynamic_auc_val)
print("--- Cause: {} -> [C-index: {:0.4f} ] [Dynamic AUC-ROC: {} ]".format(
cause_names[_],
UNO_c_index[-1],
'{:0.4f}'.format(dynamic_auc[-1]) if dynamic_auc[-1] != "-" else "-"))
def dynamic_auc(y_train, y_test, y_pred, year=3):
"""Dynamic or Time-Dependent AUC
is the average of how often a model says X is greater than Y when,
in the observed data, X is indeed greater than Y
https://lifelines.readthedocs.io/en/latest/lifelines.utils.html#lifelines.utils.concordance_index
Parameters
----------
y_true : pandas.DataFrame
DataFrame with annotation of samples. Two columns are mandatory:
Event (binary labels), Time to event (float time to event).
y_test : pandas.DataFrame
DataFrame with annotation of samples. Two columns are mandatory:
Event (binary labels), Time to event (float time to event).
y_pred : array-like
List of predicted risk scores.
year: float
Timepoint at which to calculate the AUC score
Returns
-------
float [0, 1]
dynamic auc for specified year
"""
structured_y_train = structure_y_to_sksurv(y_train)
structured_y_test = structure_y_to_sksurv(y_test)
return cumulative_dynamic_auc(
structured_y_train,
structured_y_test,
y_pred,
[year],
)[0][0]
def functionToOptimize(**params):
self.counter += 1
print(f"Bayesian Optimization model: {2 ** params['alphas']}; time: {self.counter}")
model = CoxnetSurvivalAnalysis(l1_ratio = 1.0, max_iter = 1000000)
params["alphas"] = [2 ** params["alphas"]]
model.set_params(**params)
cvAucMeans = []
for trainIndex, testIndex in KFold(n_splits = 4).split(self.data.values):
trainX, trainY = self.data.values[trainIndex,], self.data.tags[trainIndex[:, None],]
testX, testY = self.data.values[testIndex,:], self.data.tags[testIndex[:, None],]
trainY = np.reshape(trainY, -1)
testY = np.reshape(testY, -1)
model.fit(trainX, trainY)
times = np.percentile(testY["Time_in_days"], np.linspace(5, 81, 15))
_, meanAuc = cumulative_dynamic_auc(testY, testY,
model.predict(testX),
times)
cvAucMeans.append(meanAuc)
return -np.mean(cvAucMeans)
def test_uno_auc_time_dependent_without_censoring(
uno_auc_time_dependent_without_censoring_data):
y, times, estimate, expected, iauc = uno_auc_time_dependent_without_censoring_data
auc, iauc = cumulative_dynamic_auc(y, y, estimate, times)
assert_array_almost_equal(auc, expected)
assert round(abs(iauc - iauc), 6) == 0
def output_simulations(surv, df_train, x_test, df_test, name):
""" Compute the output of the model on the test set
# Arguments
model: neural network model trained with final parameters.
df_train: training dataset
x_test: 20 simulated input variables
df_test: test dataset
name: name of the model
# Returns
results_test: AUC and Uno C-index at median survival time
"""
data_train = skSurv.from_arrays(event=df_train['status'],
time=df_train['yy'])
data_test = skSurv.from_arrays(event=df_test['status'], time=df_test['yy'])
cens_test = 100. - df_test['status'].sum(
) * 100. / df_test['status'].shape[0]
time_med = np.percentile(data_test['time'], np.linspace(0, 50, 2))
auc_med = float(
cumulative_dynamic_auc(data_train, data_test,
-determine_surv_prob(surv, time_med[1]),
time_med[1])[0])
unoc = float(
concordance_index_ipcw(data_train, data_test,
-determine_surv_prob(surv, time_med[1]),
time_med[1])[0])
results_test = pd.DataFrame({
't_med': time_med[1],
'auc_med': [auc_med],
'unoc': [unoc],
'cens_rate': [cens_test]
})
return results_test
def plot_cumulative_dynamic_auc(risk_score, label, color=None):
auc, mean_auc = cumulative_dynamic_auc(y_train, y_test, risk_score, times)
plt.plot(times, auc, marker="o", color=color, label=label)
plt.xlabel("days from enrollment")
plt.ylabel("time-dependent AUC")
plt.axhline(mean_auc, color=color, linestyle="--")
plt.legend()
def rocInfo(self):
self.upgradeInfo("Calculating model ROC curves")
auc, mean, times = dict(), dict(), dict()
from sksurv.metrics import cumulative_dynamic_auc
for datasetName, dataset in self.model.dataset.items():
serial = list(set(dataset["tags"]["Time_in_days"]))
serial.sort()
times[datasetName] = serial[1:-1]
auc[datasetName], mean[datasetName] = cumulative_dynamic_auc(dataset["tags"],
dataset["tags"],
self.model.model.predict(dataset["values"]),
times[datasetName])
return auc, mean, times
def cox_dynamic_auc(df, ann, n, year=3):
"""Select n features with the highest time-dependent auc on one-factor Cox regression.
Parameters
----------
df : pandas.DataFrame
A pandas DataFrame whose rows represent samples
and columns represent features.
ann : pandas.DataFrame
DataFrame with annotation of samples. Three columns are mandatory:
Class (binary labels), Dataset (dataset identifiers) and
Dataset type (Training, Filtration, Validation).
n : int
Number of features to select.
year: float
Timepoint for which to calculate AUC score
Returns
-------
list
List of n features associated with the highest auc.
"""
ann = ann[['Event', 'Time to event']]
structured_y = structure_y_to_sksurv(ann)
columns = df.columns
scores = []
for j, column in enumerate(columns):
df_j = df[[column]]
model = CoxRegression()
model.fit(df_j, ann)
preds = model.predict(df_j)
auc, _ = cumulative_dynamic_auc(structured_y, structured_y, preds,
[year])
score = auc[0]
scores.append(score)
scores, features = zip(
*sorted(zip(scores, columns), key=lambda x: x[0], reverse=True))
return features[:n]
def plot_cumulative_dynamic_auc(risk_score, label, color=None):
auc, mean_auc = cumulative_dynamic_auc(y_train, y_test, risk_score, times)
plt.plot(times, auc, marker="o", color=color, label=label)
plt.xlabel("days from enrollment")
plt.ylabel("time-dependent AUC")
plt.axhline(mean_auc, color=color, linestyle="--")
plt.legend()
for i, col in enumerate(num_columns):
plot_cumulative_dynamic_auc(x_test[:, i], col, color="C{}".format(i))
ret = concordance_index_ipcw(y_train, y_test, x_test[:, i], tau=times[-1])
from sksurv.datasets import load_veterans_lung_cancer
va_x, va_y = load_veterans_lung_cancer()
cph = make_pipeline(OneHotEncoder(), CoxPHSurvivalAnalysis())
cph.fit(va_x, va_y)
va_times = np.arange(7, 183, 7)
# estimate performance on training data, thus use `va_y` twice.
va_auc, va_mean_auc = cumulative_dynamic_auc(va_y, va_y, cph.predict(va_x),
va_times)
plt.plot(va_times, va_auc, marker="o")
plt.axhline(va_mean_auc, linestyle="--")
plt.xlabel("days from enrollment")
plt.ylabel("time-dependent AUC")
plt.grid(True)
def train_deepsurv(data_df, r_splits):
epochs = 100
verbose = True
num_nodes = [32]
out_features = 1
batch_norm = True
dropout = 0.6
output_bias = False
c_index_at = []
c_index_30 = []
time_auc_30 = []
time_auc_60 = []
time_auc_365 = []
for i in range(len(r_splits)):
print("\nIteration %s"%(i))
#DATA PREP
df_train, df_val, df_test, df_test_30 = prepare_datasets(data_df, r_splits[i][2], r_splits[i][1], r_splits[i][0])
xcols = list(df_train.columns)
for col_name in ["subject_id", "event", "duration"]:
if col_name in xcols:
xcols.remove(col_name)
cols_standardize = xcols
standardize = [([col], StandardScaler()) for col in cols_standardize]
x_mapper = DataFrameMapper(standardize)
x_train = x_mapper.fit_transform(df_train).astype('float32')
x_val = x_mapper.transform(df_val).astype('float32')
x_test = x_mapper.transform(df_test).astype('float32')
x_test_30 = x_mapper.transform(df_test_30).astype('float32')
labtrans = CoxTime.label_transform()
get_target = lambda df: (df['duration'].values, df['event'].values)
y_train = labtrans.fit_transform(*get_target(df_train))
y_val = labtrans.transform(*get_target(df_val))
durations_test, events_test = get_target(df_test)
durations_test_30, events_test_30 = get_target(df_test_30)
val = tt.tuplefy(x_val, y_val)
(train_x, train_y), (val_x, val_y), (test_x, test_y), _ = df2array(data_df, df_train, df_val, df_test, df_test_30)
#MODEL
in_features = x_train.shape[1]
callbacks = [tt.callbacks.EarlyStopping()]
net = tt.practical.MLPVanilla(in_features, num_nodes, out_features, batch_norm, dropout, output_bias=output_bias)
model = CoxPH(net, tt.optim.Adam)
model.optimizer.set_lr(0.0001)
if x_train.shape[0] % 2:
batch_size = 255
else:
batch_size = 256
log = model.fit(x_train, y_train, batch_size, epochs, callbacks, val_data=val, val_batch_size=batch_size)
model.compute_baseline_hazards()
surv = model.predict_surv_df(x_test)
ev = EvalSurv(surv, durations_test, events_test, censor_surv='km')
c_index_at.append(ev.concordance_td())
surv_30 = model.predict_surv_df(x_test_30)
ev_30 = EvalSurv(surv_30, durations_test_30, events_test_30, censor_surv='km')
c_index_30.append(ev_30.concordance_td())
for time_x in [30, 60, 365]:
va_auc, va_mean_auc = cumulative_dynamic_auc(train_y, test_y, model.predict(x_test).flatten(), time_x)
eval("time_auc_" + str(time_x)).append(va_auc[0])
print("C-index_30:", c_index_30[i])
print("C-index_AT:", c_index_at[i])
print("time_auc_30", time_auc_30[i])
print("time_auc_60", time_auc_60[i])
print("time_auc_365", time_auc_365[i])
return c_index_at, c_index_30, time_auc_30, time_auc_60, time_auc_365
def test_uno_auc_whas500(uno_auc_whas500_data):
y_train, y_test, estimate, times, expect_auc, expect_iauc = uno_auc_whas500_data
auc, iauc = cumulative_dynamic_auc(y_train, y_test, estimate, times)
assert_array_almost_equal(auc, expect_auc)
assert_almost_equal(iauc, expect_iauc)
def test_uno_auc_censoring_failure(uno_auc_censoring_failure_data):
y_train, y_test, times, estimate, match = uno_auc_censoring_failure_data
with pytest.raises(ValueError,
match=match):
cumulative_dynamic_auc(y_train, y_test, estimate, times)
print(f"weighted brier score: {weighted_br_score: .4f}")
#%% Time-dependent Area under the ROC
survival_train=np.dtype([('event',data_event_train.dtype),('surv_time',data_time_train.dtype)])
survival_train=np.empty(len(data_event_train),dtype=survival_train)
survival_train['event']=data_event_train
survival_train['surv_time']=data_time_train
survival_test=np.dtype([('event',data_event_test.dtype),('surv_time',data_time_test.dtype)])
survival_test=np.empty(len(data_event_test),dtype=survival_test)
survival_test['event']=data_event_test
survival_test['surv_time']=data_time_test
event_times = np.arange(np.min(data_time_test), np.max(data_time_test)/2, 75)
test_auc, test_mean_auc = cumulative_dynamic_auc(survival_train, survival_test, model_prediction, event_times)
print(f"Time-dependent Area under the ROC: {test_mean_auc: .4f}")
plt.plot(event_times, test_auc, marker="o")
plt.axhline(test_mean_auc, linestyle="--")
plt.xlabel("Days from Enrollment")
plt.ylabel("Time-dependent Area under the ROC")
plt.grid(True)
plt.savefig('typical_auc_batch.png',dpi = 600)
#%% log -partial likelihood
log_lik = log_partial_lik(model_prediction.reshape(-1,1), data_event_test.reshape(-1,1))
print(f"Log partial likelihood: {log_lik: .4f}")
#%% individual fairness measures
def train_LSTMCox(data_df, r_splits):
epochs = 100
verbose = True
in_features = 768
out_features = 1
batch_norm = True
dropout = 0.6
output_bias = False
c_index_at = []
c_index_30 = []
time_auc_30 = []
time_auc_60 = []
time_auc_365 = []
for i in range(len(r_splits)):
print("\nIteration %s"%(i))
#DATA PREP
df_train, df_val, df_test, df_test_30 = prepare_datasets(data_df, r_splits[i][2], r_splits[i][1], r_splits[i][0])
x_train = np.array(df_train["x0"].tolist()).astype("float32")
x_val = np.array(df_val["x0"].tolist()).astype("float32")
x_test = np.array(df_test["x0"].tolist()).astype("float32")
x_test_30 = np.array(df_test_30["x0"].tolist()).astype("float32")
labtrans = CoxTime.label_transform()
get_target = lambda df: (df['duration'].values, df['event'].values)
y_train = labtrans.fit_transform(*get_target(df_train))
y_val = labtrans.transform(*get_target(df_val))
durations_test, events_test = get_target(df_test)
durations_test_30, events_test_30 = get_target(df_test_30)
val = tt.tuplefy(x_val, y_val)
(train_x, train_y), (val_x, val_y), (test_x, test_y), _ = df2array(data_df, df_train, df_val, df_test, df_test_30)
#MODEL
callbacks = [tt.callbacks.EarlyStopping()]
net = LSTMCox(768, 32, 1, 1)
model = CoxPH(net, tt.optim.Adam)
model.optimizer.set_lr(0.0001)
if x_train.shape[0] % 2:
batch_size = 255
else:
batch_size = 256
log = model.fit(x_train, y_train, batch_size, epochs, callbacks, val_data=val, val_batch_size=batch_size)
model.compute_baseline_hazards()
surv = model.predict_surv_df(x_test)
ev = EvalSurv(surv, durations_test, events_test, censor_surv='km')
c_index_at.append(ev.concordance_td())
surv_30 = model.predict_surv_df(x_test_30)
ev_30 = EvalSurv(surv_30, durations_test_30, events_test_30, censor_surv='km')
c_index_30.append(ev_30.concordance_td())
for time_x in [30, 60, 365]:
va_auc, va_mean_auc = cumulative_dynamic_auc(train_y, test_y, model.predict(x_test).flatten(), time_x)
eval("time_auc_" + str(time_x)).append(va_auc[0])
print("C-index_30:", c_index_30[i])
print("C-index_AT:", c_index_at[i])
print("time_auc_30", time_auc_30[i])
print("time_auc_60", time_auc_60[i])
print("time_auc_365", time_auc_365[i])
return c_index_at, c_index_30, time_auc_30, time_auc_60, time_auc_365
cph = CoxPHSurvivalAnalysis()
sss = StratifiedShuffleSplit(n_splits=N, test_size=nTest)
for i, x in enumerate(tx):
x = x.to_numpy()
j = 0
for train, test in sss.split(x, y['bio_rec_6']):
xtrain, ytrain = x[train], y[train]
xtest, ytest = x[test], y[test]
cph.fit(xtrain, ytrain)
ypred = cph.predict(xtrain)
time = np.linspace(
np.min(ytrain['bio_rec_6_delay']) + 1,
np.max(ytrain['bio_rec_6_delay']) - 1, 3)
auc[i, :], meanAuc[j,
i] = cumulative_dynamic_auc(ytrain, ytrain, ypred,
time)
j += 1
meanAuc = pd.DataFrame(data=meanAuc,
columns=[
'4_feat.', "Prediction 1", 'Prediction 2',
'Prediction 3', 'Prediction 4'
])
meanMeanAuc = meanAuc.mean(axis=0)
stdMeanAuc = meanAuc.std(axis=0)
#%%
N = 100
#x = data
#x = x.drop(columns = ['bio_rec_6', 'bio_rec_6_delay'])
tx = [
def output_bootstrap(model, n_iterations, df_train, data_train, y_train,
df_test, name):
""" Compute the output of the model on the bootstraped test set
# Arguments
model: neural network model trained with final parameters.
n_iterations: number of bootstrap iterations
df_train: training dataset
data_train: two columns dataset with survival time and censoring status for training samples
y_train: survival time
df_test: test dataset
name: name of the model
# Returns
results_all: AUC and Uno C-index at 5 and 10 years
"""
if name == "CoxTime" or name == "Cox-CC":
_ = model.compute_baseline_hazards()
results_all = pd.DataFrame(columns=['auc5', 'auc10', 'unoc5', 'unoc10'])
results_final = pd.DataFrame(
columns=['mean', 'ci95_lo', 'ci95_hi', 'std', 'count'])
for i in range(n_iterations):
print(i)
test_boot = resample(df_test, n_samples=len(df_test), replace=True)
x_test_boot = test_boot.drop(['surv_test', 'cen_test'], axis=1)
duration_test_b, event_test_b = test_boot[
'surv_test'].values, test_boot['cen_test'].values
data_test_b = skSurv.from_arrays(event=event_test_b,
time=duration_test_b)
if name == "Cox-CC" or name == "CoxTime" or name == "DeepHit":
surv = model.predict_surv_df(np.array(x_test_boot,
dtype='float32'))
else:
n_picktime = int(y_train[['s']].apply(pd.Series.nunique))
x_test_boot_all = pd.concat([x_test_boot] * n_picktime)
time_test = pd.DataFrame(
np.repeat(np.unique(y_train[['s']]), len(x_test_boot)))
x_test_boot_all.reset_index(inplace=True, drop=True)
x_test_boot_all = pd.concat([x_test_boot_all, time_test], axis=1)
surv = make_predictions_pseudobs(model, y_train, x_test_boot_all,
x_test_boot, name)
time_grid = np.linspace(duration_test_b.min(), duration_test_b.max(),
100)
prob_5_10 = pd.concat([
determine_surv_prob(surv, i)
for i in (duration_test_b.min(), 5, 10)
],
axis=1)
auc5 = float(
cumulative_dynamic_auc(data_train, data_test_b,
-prob_5_10.iloc[:, 1], 5)[0])
auc10 = float(
cumulative_dynamic_auc(data_train, data_test_b,
-prob_5_10.iloc[:, 2], 10)[0])
unoc5 = float(
concordance_index_ipcw(data_train, data_test_b,
-prob_5_10.iloc[:, 1], 5)[0])
unoc10 = float(
concordance_index_ipcw(data_train, data_test_b,
-prob_5_10.iloc[:, 2], 10)[0])
results = pd.DataFrame({
'auc5': [auc5],
'auc10': [auc10],
'unoc5': [unoc5],
'unoc10': [unoc10]
})
results_all = results_all.append(results,
ignore_index=True,
sort=False)
for column in results_all:
stats = results_all[column].agg(['mean', 'count', 'std'])
scores = np.array(results_all[column])
sorted_scores = np.sort(scores, axis=None)
ci95_lo = sorted_scores[int(0.05 * len(sorted_scores))]
ci95_hi = sorted_scores[int(0.95 * len(sorted_scores))]
results_stat = pd.DataFrame({
'mean': [stats[0]],
'ci95_lo': ci95_lo,
'ci95_hi': [ci95_hi],
'std': [stats[2]],
'count': [stats[1]]
})
results_final = results_final.append(results_stat,
ignore_index=False,
sort=False)
results_final.index = results_all.columns.tolist()
return results_final
def test_uno_auc(uno_auc_data):
y_train, y_test, estimate, times, expect_auc, expect_iauc = uno_auc_data
auc, iauc = cumulative_dynamic_auc(y_train, y_test, estimate, times)
assert_array_almost_equal(auc, expect_auc)
assert round(abs(iauc - expect_iauc), 6) == 0
