# Cox Proportional Hazards and Random Survival Forests

# Goal:

# - Cox Proportional Hazards

# - Data Preprocessing for Cox Models.

# - Random Survival Forests

# - Permutation Methods for Interpretation.

# Import Packages

import sklearn # most popular machine learning libraries

import numpy as np # fundamental package for scientific computing in python

import pandas as pd # to manipulate the data

import matplotlib.pyplot as plt # plotting library

from lifelines import CoxPHFitter # open-source survival analysis library

from lifelines.utils import concordance_index as cindex

from sklearn.model_selection import train_test_split

from util import load_data

#

# Run cell to load the data.

df = load_data()

# Be familiarized with the data and the shape of it.

print(df.shape)

# df.head() only outputs the top few rows

df.head()

# Take a minute to examine particular cases.

i = 20

df.iloc[i, :]

# Now, split your dataset into train, validation and test set using 60/20/20 split.

np.random.seed(0)

df_dev, df_test = train_test_split(df, test_size = 0.2)

df_train, df_val = train_test_split(df_dev, test_size = 0.25)

print("Total number of patients:", df.shape[0])

print("Total number of patients in training set:", df_train.shape[0])

print("Total number of patients in validation set:", df_val.shape[0])

print("Total number of patients in test set:", df_test.shape[0])

# Normalize the continuous covariates by using statistics from the train data to make sure they're on the same scale.

continuous_columns = ['age', 'bili', 'chol', 'albumin', 'copper', 'alk.phos', 'ast', 'trig', 'platelet', 'protime']

mean = df_train.loc[:, continuous_columns].mean()

std = df_train.loc[:, continuous_columns].std()

df_train.loc[:, continuous_columns] = (df_train.loc[:, continuous_columns] - mean) / std

df_val.loc[:, continuous_columns] = (df_val.loc[:, continuous_columns] - mean) / std

df_test.loc[:, continuous_columns] = (df_test.loc[:, continuous_columns] - mean) / std

# Check the summary statistics on our training dataset to make sure it's standardized.

df_train.loc[:, continuous_columns].describe()

# Goal is to build a risk score using the survival data obtained and begin by fitting a Cox Proportional Hazards model to the data.

# Implement the `to_one_hot(...)` function.

def to_one_hot(dataframe, columns):

return one_hot_df

# Now, use the function coded to transform the training, validation, and test sets.

# List of categorical columns

to_encode = ['edema', 'stage']

one_hot_train = to_one_hot(df_train, to_encode)

one_hot_val = to_one_hot(df_val, to_encode)

one_hot_test = to_one_hot(df_test, to_encode)

print(one_hot_val.columns.tolist())

print(f"There are {len(one_hot_val.columns)} columns")

# Take a peek at one of the transformed data sets for new features.

print(one_hot_train.shape)

one_hot_train.head()

# Run the following cell to fit your Cox Proportional Hazards model using the `lifelines` package.

cph = CoxPHFitter()

cph.fit(one_hot_train, duration_col = 'time', event_col = 'status', step_size=0.1)

# Use `cph.print_summary()` to view the coefficients associated with each covariate as well as confidence intervals.

cph.print_summary()

# Run the next cell to plot survival curves using the `plot_covariate_groups()` function.

cph.plot_covariate_groups('trt', values=[0, 1]); # Can compare the predicted survival curves for treatment variables.

# Write a function to compute the hazard ratio between two individuals given the cox model's coefficients

def hazard_ratio(case_1, case_2, cox_params):

return hr

# Now, evaluate it on the following pair of indivduals: `i = 1` and `j = 5`

i = 1

case_1 = one_hot_train.iloc[i, :].drop(['time', 'status'])

j = 5

case_2 = one_hot_train.iloc[j, :].drop(['time', 'status'])

print(hazard_ratio(case_1.values, case_2.values, cph.params_.values))

# Inspect different pairs, and figure out which patient is more at risk.

i = 4

case_1 = one_hot_train.iloc[i, :].drop(['time', 'status'])

j = 7

case_2 = one_hot_train.iloc[j, :].drop(['time', 'status'])

print("Case 1\n\n", case_1, "\n")

print("Case 2\n\n", case_2, "\n")

print("Hazard Ratio:", hazard_ratio(case_1.values, case_2.values, cph.params_.values))

# Fill in the function below to compute Harrel's C-index.

def harrell_c(y_true, scores, event):

return result

# Test the function on the following test cases:

y_true = [30, 12, 84, 9]

# Case 1

event = [1, 1, 1, 1]

scores = [0.5, 0.9, 0.1, 1.0]

print("Case 1")

print("Expected: 1.0, Output: {}".format(harrell_c(y_true, scores, event)))

# Case 2

scores = [0.9, 0.5, 1.0, 0.1]

print("\nCase 2")

print("Expected: 0.0, Output: {}".format(harrell_c(y_true, scores, event)))

# Case 3

event = [1, 0, 1, 1]

scores = [0.5, 0.9, 0.1, 1.0]

print("\nCase 3")

print("Expected: 1.0, Output: {}".format(harrell_c(y_true, scores, event)))

# Case 4

y_true = [30, 30, 20, 20]

event = [1, 0, 1, 0]

scores = [10, 5, 15, 20]

print("\nCase 4")

print("Expected: 0.75, Output: {}".format(harrell_c(y_true, scores, event)))

# Case 5

y_true = list(reversed([30, 30, 30, 20, 20]))

event = [0, 1, 0, 1, 0]

scores = list(reversed([15, 10, 5, 15, 20]))

print("\nCase 5")

print("Expected: 0.583, Output: {}".format(harrell_c(y_true, scores, event)))

# Case 6

y_true = [10,10]

event = [0,1]

scores = [4,5]

print("\nCase 6")

print(f"Expected: 1.0 , Output:{harrell_c(y_true, scores, event):.4f}")

# Now use the Harrell's C-index function to evaluate the cox model on our data sets.

# Train

scores = cph.predict_partial_hazard(one_hot_train)

cox_train_scores = harrell_c(one_hot_train['time'].values, scores.values, one_hot_train['status'].values)

# Validation

scores = cph.predict_partial_hazard(one_hot_val)

cox_val_scores = harrell_c(one_hot_val['time'].values, scores.values, one_hot_val['status'].values)

# Test

scores = cph.predict_partial_hazard(one_hot_test)

cox_test_scores = harrell_c(one_hot_test['time'].values, scores.values, one_hot_test['status'].values)

print("Train:", cox_train_scores)

print("Val:", cox_val_scores)

print("Test:", cox_test_scores)

# Use a Random Survival Forest by using the `RandomForestSRC` package in R.

# Run the following cell to import the necessary requirements.

get_ipython().run_line_magic('load_ext', 'rpy2.ipython')

get_ipython().run_line_magic('R', 'require(ggplot2)')

from rpy2.robjects.packages import importr

# import R's "base" package

base = importr('base')

# import R's "utils" package

utils = importr('utils')

# import rpy2's package module

import rpy2.robjects.packages as rpackages

forest = rpackages.importr('randomForestSRC', lib_loc='R')

from rpy2 import robjects as ro

R = ro.r

from rpy2.robjects import pandas2ri

pandas2ri.activate()

#

# Run the code cell below to build the forest.

model = forest.rfsrc(ro.Formula('Surv(time, status) ~ .'), data=df_train, ntree=300, nodedepth=5, seed=-1)

print(model)

# Finally, evaluate on the validation and test sets, and compare it with the Cox model.

result = R.predict(model, newdata=df_val)

scores = np.array(result.rx('predicted')[0])

print("Cox Model Validation Score:", cox_val_scores)

print("Survival Forest Validation Score:", harrell_c(df_val['time'].values, scores, df_val['status'].values))

result = R.predict(model, newdata=df_test)

scores = np.array(result.rx('predicted')[0])

print("Cox Model Test Score:", cox_test_scores)

print("Survival Forest Validation Score:", harrell_c(df_test['time'].values, scores, df_test['status'].values))

# The random forest model should be outperforming the Cox model slightly.

# Note: random surival forests come with their own built in variable importance feature (VIMP).

# The higher absolute value of the VIMP, then the variable generally has a larger effect on the model outcome.

# Run the next cell to compute and plot VIMP for the random survival forest.

vimps = np.array(forest.vimp(model).rx('importance')[0])

y = np.arange(len(vimps))

plt.barh(y, np.abs(vimps))

plt.yticks(y, df_train.drop(['time', 'status'], axis=1).columns)

plt.title("VIMP (absolute value)")

plt.show()