def train(
self,
params: Union[None, dict] = None,
subset: Union[None, pd.core.series.Series] = None,
validation_early_stopping: bool = True,
) -> keras.Model:
"""Train with survival loss function of Gensheimer, Narasimhan (2019).
See https://peerj.com/articles/6257/.
Use the AMSGrad variant of the Adam optimizer. See Reddi, Kale, and
Kumar (2018) at https://openreview.net/forum?id=ryQu7f-RZ.
Train until validation set performance does not improve for the given
number of epochs or the given maximum number of epochs.
Returns:
A trained Keras model.
"""
if params is None:
params = {}
if subset is None:
subset = ~self.data[self.test_col] & ~self.data[self.predict_col]
train_subset = subset & ~self.data[self.validation_col]
valid_subset = subset & self.data[self.validation_col]
x_train = self.format_input_data(subset=train_subset)
x_valid = self.format_input_data(subset=valid_subset)
y_surv = make_surv_array(
self.data[[self.duration_col, self.max_lead_col]].min(axis=1),
self.data[self.event_col],
np.arange(self.n_intervals + 1),
)
model = Model(self.model.inputs, self.model.outputs)
model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
callbacks = []
if validation_early_stopping:
callbacks.append(
EarlyStopping(
monitor="val_loss",
mode="min",
verbose=1,
patience=self.config.get("PATIENCE", 4),
restore_best_weights=True,
))
model.fit(
x_train,
y_surv[train_subset],
batch_size=min(
params.get("BATCH_SIZE", self.config.get("BATCH_SIZE", 512)),
train_subset.sum(),
),
epochs=params.get("MAX_EPOCHS", self.config.get("MAX_EPOCHS",
256)),
validation_data=(x_valid, y_surv[valid_subset]),
verbose=2,
callbacks=callbacks,
)
return model
def train(self) -> keras.Model:
"""Train with survival loss function of Gensheimer, Narasimhan (2019).
See https://peerj.com/articles/6257/.
Use the AMSGrad variant of the Adam optimizer. See Reddi, Kale, and
Kumar (2018) at https://openreview.net/forum?id=ryQu7f-RZ.
Train until validation set performance does not improve for the given
number of epochs or the given maximum number of epochs.
Returns:
A trained Keras model.
"""
train_subset = (~self.data[self.validation_col]
& ~self.data[self.test_col]
& ~self.data[self.predict_col])
valid_subset = (self.data[self.validation_col]
& ~self.data[self.test_col]
& ~self.data[self.predict_col])
x_train = self.format_input_data(subset=train_subset)
x_valid = self.format_input_data(subset=valid_subset)
y_surv = make_surv_array(self.data[self.duration_col],
self.data[self.event_col],
np.arange(self.n_intervals + 1))
model = Model(self.model.inputs, self.model.outputs)
model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
model.fit(x_train,
y_surv[train_subset],
batch_size=min(self.config['BATCH_SIZE'],
train_subset.sum()),
epochs=self.config['MAX_EPOCHS'],
validation_data=(x_valid, y_surv[valid_subset]),
verbose=2,
callbacks=[
EarlyStopping(monitor='val_loss',
mode='min',
verbose=1,
patience=self.config['PATIENCE'],
restore_best_weights=True)
])
return model
def evaluate_params(
trial: optuna.trial.Trial,
x_train: List[np.array],
y_train: np.array,
x_valid: List[np.array],
y_valid: np.array,
max_epochs: int,
) -> Union[None, dict]:
"""Compute out-of-sample performance for a parameter set."""
params = {}
params["BATCH_SIZE"] = trial.suggest_int(
"BATCH_SIZE", min(32, x_train[0].shape[0]),
x_train[0].shape[0])
params["DENSE_LAYERS"] = trial.suggest_int("DENSE_LAYERS", 0, 3)
params["DROPOUT_SHARE"] = trial.suggest_uniform(
"DROPOUT_SHARE", 0, 0.5)
params["EMBED_EXPONENT"] = trial.suggest_uniform(
"EMBED_EXPONENT", 0, 0.25)
params["EMBED_L2_REG"] = trial.suggest_uniform(
"EMBED_L2_REG", 0, 16.0)
if self.categorical_features:
params["POST_FREEZE_EPOCHS"] = trial.suggest_int(
"POST_FREEZE_EPOCHS", 4, max_epochs)
else:
params["POST_FREEZE_EPOCHS"] = 0
max_pre_freeze_epochs = max_epochs
params["PRE_FREEZE_EPOCHS"] = trial.suggest_int(
"PRE_FREEZE_EPOCHS", 4, max_epochs)
params["NODES_PER_DENSE_LAYER"] = trial.suggest_int(
"NODES_PER_DENSE_LAYER", 16, 1024)
construction_args = getfullargspec(
self.construct_embedding_network).args
self.model = self.construct_embedding_network(**{
k.lower(): v
for k, v in params.items() if k in construction_args
})
self.data[self.numeric_features] = self.data[
self.numeric_features].fillna(
self.config.get("NON_CAT_MISSING_VALUE", -1))
model = self.construct_embedding_network(
**{
k.lower(): v
for k, v in self.config.items() if k in construction_args
})
model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
for step in range(params["PRE_FREEZE_EPOCHS"]):
model.fit(x_train,
y_train,
batch_size=params["BATCH_SIZE"],
epochs=1)
validation_loss = model.evaluate(x_valid, y_valid)
trial.report(validation_loss, step)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
model = freeze_embedding_layers(model)
model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
for step in range(params["POST_FREEZE_EPOCHS"]):
model.fit(x_train,
y_train,
batch_size=params["BATCH_SIZE"],
epochs=1)
validation_loss = model.evaluate(x_valid, y_valid)
trial.report(validation_loss, step + max_pre_freeze_epochs)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
return validation_loss
def hyperoptimize(
self,
n_trials: int = 64,
subset: Union[None, pd.core.series.Series] = None,
max_epochs: int = 128,
) -> dict:
"""Search for hyperparameters with greater out-of-sample performance.
Args:
n_trials: The number of hyperparameter sets to evaluate for each
time horizon. Return None if non-positive.
subset: A Boolean Series that is True for observations on which
to train and validate. If None, default to all observations not
flagged by self.test_col or self.predict_col.
Returns:
A dictionary containing the best-performing parameters.
"""
def evaluate_params(
trial: optuna.trial.Trial,
x_train: List[np.array],
y_train: np.array,
x_valid: List[np.array],
y_valid: np.array,
max_epochs: int,
) -> Union[None, dict]:
"""Compute out-of-sample performance for a parameter set."""
params = {}
params["BATCH_SIZE"] = trial.suggest_int(
"BATCH_SIZE", min(32, x_train[0].shape[0]),
x_train[0].shape[0])
params["DENSE_LAYERS"] = trial.suggest_int("DENSE_LAYERS", 0, 3)
params["DROPOUT_SHARE"] = trial.suggest_uniform(
"DROPOUT_SHARE", 0, 0.5)
params["EMBED_EXPONENT"] = trial.suggest_uniform(
"EMBED_EXPONENT", 0, 0.25)
params["EMBED_L2_REG"] = trial.suggest_uniform(
"EMBED_L2_REG", 0, 16.0)
if self.categorical_features:
params["POST_FREEZE_EPOCHS"] = trial.suggest_int(
"POST_FREEZE_EPOCHS", 4, max_epochs)
else:
params["POST_FREEZE_EPOCHS"] = 0
max_pre_freeze_epochs = max_epochs
params["PRE_FREEZE_EPOCHS"] = trial.suggest_int(
"PRE_FREEZE_EPOCHS", 4, max_epochs)
params["NODES_PER_DENSE_LAYER"] = trial.suggest_int(
"NODES_PER_DENSE_LAYER", 16, 1024)
construction_args = getfullargspec(
self.construct_embedding_network).args
self.model = self.construct_embedding_network(**{
k.lower(): v
for k, v in params.items() if k in construction_args
})
self.data[self.numeric_features] = self.data[
self.numeric_features].fillna(
self.config.get("NON_CAT_MISSING_VALUE", -1))
model = self.construct_embedding_network(
**{
k.lower(): v
for k, v in self.config.items() if k in construction_args
})
model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
for step in range(params["PRE_FREEZE_EPOCHS"]):
model.fit(x_train,
y_train,
batch_size=params["BATCH_SIZE"],
epochs=1)
validation_loss = model.evaluate(x_valid, y_valid)
trial.report(validation_loss, step)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
model = freeze_embedding_layers(model)
model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
for step in range(params["POST_FREEZE_EPOCHS"]):
model.fit(x_train,
y_train,
batch_size=params["BATCH_SIZE"],
epochs=1)
validation_loss = model.evaluate(x_valid, y_valid)
trial.report(validation_loss, step + max_pre_freeze_epochs)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
return validation_loss
default_params = {"BATCH_SIZE": 512, "PRE_FREEZE_EPOCHS": 16}
if n_trials <= 0:
return {
time_horizon: default_params
for time_horizon in range(self.n_intervals)
}
params = {}
if subset is None:
subset = ~self.data[self.test_col] & ~self.data[self.predict_col]
x_train = self.format_input_data(subset=subset
& ~self.data[self.validation_col])
x_valid = self.format_input_data(subset=subset
& self.data[self.validation_col])
y_surv = make_surv_array(
self.data[[self.duration_col, self.max_lead_col]].min(axis=1),
self.data[self.event_col],
np.arange(self.n_intervals + 1),
)
y_train = y_surv[[subset & ~self.data[self.validation_col]]]
y_valid = y_surv[[subset & self.data[self.validation_col]]]
study = optuna.create_study(
pruner=optuna.pruners.MedianPruner(),
sampler=optuna.samplers.TPESampler(
seed=self.config.get("SEED", 9999)),
)
study.optimize(
lambda trial: evaluate_params(trial, x_train, y_train, x_valid,
y_valid, max_epochs),
n_trials=n_trials,
)
params = study.best_params
if self.categorical_features:
default_params["POST_FREEZE_EPOCHS"] = 16
else:
default_params["POST_FREEZE_EPOCHS"] = 0
construction_args = getfullargspec(
self.construct_embedding_network).args
default_model = self.construct_embedding_network(
**{
k.lower(): v
for k, v in default_params.items() if k in construction_args
})
default_model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
default_model.fit(
x_train,
y_train,
batch_size=default_params["BATCH_SIZE"],
epochs=default_params["PRE_FREEZE_EPOCHS"],
)
default_model = freeze_embedding_layers(default_model)
default_model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
default_model.fit(
x_train,
y_train,
batch_size=default_params["BATCH_SIZE"],
epochs=default_params["POST_FREEZE_EPOCHS"],
)
default_validation_loss = default_model.evaluate(x_valid, y_valid)
if default_validation_loss <= study.best_value:
params = default_params
return params