def run_new():
val_split = 0.5
X1, Y1, X2, Y2, X3, Y3 = transfer.get_data(split_type='c_topics',
amt=20000)
main, intermediate, shallow = networks.create_dnn()
seconddeep, _, _ = networks.create_dnn()
first_half = (X1, Y1)
second_half = (X2, Y2)
main, history, cbs = transfer.train_and_validate(
main, data=first_half, validation_split=val_split)
intermediate, shallow, shallow_history, shallow_cbs = transfer.transfer_and_repeat(
main,
intermediate,
shallow,
data=second_half,
validation_split=val_split)
seconddeep, second_history, second_cbs = transfer.train_and_validate(
seconddeep, data=second_half, validation_split=val_split)
plotting.plot_metrics()
def main(chosen_device):
exercice_number = 1
print('exercice {}\n=================='.format(exercice_number))
data_inputs, expected_outputs = generate_data(
# See: https://github.com/guillaume-chevalier/seq2seq-signal-prediction/blob/master/datasets.py
exercice_number=exercice_number,
n_samples=None,
window_size_past=None,
window_size_future=None)
print('data_inputs shape: {} => (n_samples, window_size_past, input_dim)'.
format(data_inputs.shape))
print(
'expected_outputs shape: {} => (n_samples, window_size_future, output_dim)'
.format(expected_outputs.shape))
sequence_length = data_inputs.shape[1]
input_dim = data_inputs.shape[2]
output_dim = expected_outputs.shape[2]
batch_size = 100
epochs = 3
validation_size = 0.15
max_plotted_validation_predictions = 10
seq2seq_pipeline_hyperparams = HyperparameterSamples({
'hidden_dim':
100,
'layers_stacked_count':
2,
'lambda_loss_amount':
0.0003,
'learning_rate':
0.006,
'window_size_future':
sequence_length,
'output_dim':
output_dim,
'input_dim':
input_dim
})
pipeline = Pipeline([
MiniBatchSequentialPipeline(
[
ForEachDataInput(MeanStdNormalizer()),
ToNumpy(),
PlotPredictionsWrapper(
Tensorflow2ModelStep(
# See: https://github.com/Neuraxio/Neuraxle-TensorFlow
create_model=create_model,
create_loss=create_loss,
create_optimizer=create_optimizer,
expected_outputs_dtype=tf.dtypes.float32,
data_inputs_dtype=tf.dtypes.float32,
device_name=chosen_device,
print_loss=True).set_hyperparams(
seq2seq_pipeline_hyperparams))
],
batch_size=batch_size),
]).set_name('SignalPrediction')
trainer = Trainer(
epochs=epochs,
validation_splitter=ValidationSplitter(test_size=validation_size),
scoring_callback=ScoringCallback(
metric_function=metric_3d_to_2d_wrapper(mean_squared_error),
higher_score_is_better=False))
trial: Trial = trainer.train(pipeline=pipeline,
data_inputs=data_inputs,
expected_outputs=expected_outputs)
plot_metrics(
metric_name='mse',
train_values=trial.validation_splits[0].metrics_results['main']
['train_values'],
validation_values=trial.validation_splits[0].metrics_results['main']
['validation_values'],
exercice_number=exercice_number)
# Get trained pipeline
pipeline = trial.get_trained_pipeline(split_number=0)
# Get validation set with trainer.validation_split_function.split function.
_, _, data_inputs_validation, expected_outputs_validation = trainer.validation_split_function.split(
data_inputs=data_inputs, expected_outputs=expected_outputs)
# Enable the plotting feature inside the PlotPredictionsWrapper wrapper step.
pipeline.apply('toggle_plotting')
pipeline.apply(method='set_max_plotted_predictions',
max_plotted_predictions=max_plotted_validation_predictions)
# Transform the trained pipeline to plot predictions
pipeline.transform_data_container(
DataContainer(data_inputs=data_inputs_validation[0],
expected_outputs=expected_outputs_validation[0]))
def run(network="dnn",
amount=100,
val_split=0.5,
split_type="simple",
name="experiment",
layer_count=10,
interm_fraction=0.25,
neuron_count_per_layer=256):
data = json.load(open('categories.json'))
# Fetch data and make simple split of data
X1, Y1, X2, Y2, first_ind, second_ind = transfer.get_data(
split_type=split_type, amt=amount * 2)
# split data on specified axes
for inx, elt in enumerate(Y1):
if inx == 0:
firstY1 = np.take(elt, first_ind)
else:
firstY1 = np.vstack((firstY1, np.take(elt, first_ind)))
for inx, elt in enumerate(Y2):
if inx == 0:
secondY2 = np.take(elt, second_ind)
else:
secondY2 = np.vstack((secondY2, np.take(elt, second_ind)))
# Gather models
first_model, intermediate, second_model, latent_model = networks.create(
network=network,
first_output_dim=len(first_ind),
second_output_dim=len(second_ind),
input_dim=X1.shape[1],
layer_count=layer_count,
interm_fraction=interm_fraction,
neuron_count=neuron_count_per_layer)
print(first_model.summary(), intermediate.summary(),
second_model.summary(), latent_model.summary())
# Split data for training/testing for before and after transfer
first_half = (X1[:amount], firstY1[:amount])
second_half = (X2[:amount], secondY2[:amount])
# Train main network on first data split
first_model, first_history, first_cb = transfer.train_and_validate(
epochs=10,
model=first_model,
data=first_half,
validation_split=val_split)
# Train second deep network on second data split
second_model, second_history, second_cb = transfer.train_and_validate(
epochs=10,
model=second_model,
data=second_half,
validation_split=val_split)
intermediate, latent_model, latent_history, latent_cb = transfer.transfer_and_repeat(
epochs=10,
model=first_model,
intermediate=intermediate,
transfer_model=latent_model,
data=second_half,
validation_split=val_split)
# plot and save models
unique_name = '{}-{}-{}-{}'.format(amount, network, split_type, name)
plotting.plot_acc(name=unique_name,
first_history=first_history,
second_history=second_history,
latent_history=latent_history)
plotting.plot_times(name=unique_name,
first_time_cb=first_cb,
second_time_cb=second_cb,
latent_time_cb=latent_cb)
plotting.plot_metrics(name=unique_name,
intermediate=intermediate,
second_model=second_model,
latent_model=latent_model,
x_test=X2[amount:],
y_test=secondY2[amount:],
indices=second_ind,
categories=data)
transfer.save_model(first_model, "first-{}".format(unique_name))
transfer.save_model(second_model, "second-{}".format(unique_name))
transfer.save_model(latent_model, "latent-{}".format(unique_name))
import ROOT as R
import matplotlib.pyplot as plt
import numpy as np
R.gInterpreter.ProcessLine('#include "DataLoader.h"')
from training import training
from mymodel import * #gives us all n_tau, n_pf, ....
from plotting import plot_metrics, plt_conf_dm, plot_p4_resolution
from evaluation import evaluation, accuracy_calc
history = training() # trains the model
plot_metrics(
history
) # Plots the loss and accuracy curves for trainset and validationset
conf_dm_mat, conf_dm_mat_old, h_pt_tot, h_eta_tot, h_phi_tot, h_m2_tot = evaluation(
) # creates the confusion matrices
print('\nEvaluation is finished.\n')
# plt_conf_dm(conf_dm_mat) # Plots the configuration matrix of decay modes (truth vs. predicted)
# plt_conf_dm(conf_dm_mat_old, old = True) # Plots the confusion matrix to compare predicted decay modes to old reconstruction method
# print('\nConfusion matrices finished.')
# accuracy = accuracy_calc(conf_dm_mat) # Accuracy calculation of main decay modes (truth vs. predicted)
# accuracy = accuracy_calc(conf_dm_mat_old, old = True)
def main(chosen_device):
exercice_number = 1
print('exercice {}\n=================='.format(exercice_number))
data_inputs, expected_outputs = generate_data(
# See: https://github.com/guillaume-chevalier/seq2seq-signal-prediction/blob/master/datasets.py
exercice_number=exercice_number,
n_samples=None,
window_size_past=None,
window_size_future=None)
print('data_inputs shape: {} => (n_samples, window_size_past, input_dim)'.
format(data_inputs.shape))
print(
'expected_outputs shape: {} => (n_samples, window_size_future, output_dim)'
.format(expected_outputs.shape))
sequence_length = data_inputs.shape[1]
input_dim = data_inputs.shape[2]
output_dim = expected_outputs.shape[2]
batch_size = 100
epochs = 3
validation_size = 0.15
max_plotted_validation_predictions = 10
seq2seq_pipeline_hyperparams = HyperparameterSamples({
'hidden_dim':
100,
'layers_stacked_count':
2,
'lambda_loss_amount':
0.0003,
'learning_rate':
0.006,
'window_size_future':
sequence_length,
'output_dim':
output_dim,
'input_dim':
input_dim
})
feature_0_metric = metric_3d_to_2d_wrapper(mean_squared_error)
metrics = {'mse': feature_0_metric}
signal_prediction_pipeline = Pipeline([
ForEachDataInput(MeanStdNormalizer()),
ToNumpy(),
PlotPredictionsWrapper(
Tensorflow2ModelStep(
# See: https://github.com/Neuraxio/Neuraxle-TensorFlow
create_model=create_model,
create_loss=create_loss,
create_optimizer=create_optimizer,
expected_outputs_dtype=tf.dtypes.float32,
data_inputs_dtype=tf.dtypes.float32,
print_loss=True).set_hyperparams(seq2seq_pipeline_hyperparams))
]).set_name('SignalPrediction')
pipeline = Pipeline([
EpochRepeater(ValidationSplitWrapper(
MetricsWrapper(Pipeline([
TrainOnlyWrapper(DataShuffler()),
MiniBatchSequentialPipeline([
MetricsWrapper(signal_prediction_pipeline,
metrics=metrics,
name='batch_metrics')
],
batch_size=batch_size)
]),
metrics=metrics,
name='epoch_metrics',
print_metrics=True),
test_size=validation_size,
scoring_function=feature_0_metric),
epochs=epochs)
])
pipeline, outputs = pipeline.fit_transform(data_inputs, expected_outputs)
plot_metrics(pipeline=pipeline, exercice_number=exercice_number)
plot_predictions(data_inputs, expected_outputs, pipeline,
max_plotted_validation_predictions)
def run_hcomb_final(h, ID, hcm, model_dir, INTERMEDIATE_PLOTS, GLOBAL_GRADIENT_NORM_PLOT):
################################################# FINAL EXPERIMENT
start = timer()
ALL_FOLDS = list(range(1, 7))
print(5 * '\n' + 'Starting Final Experiment, training {} epochs...\n'.format(h.MAX_EPOCHS))
model_save_dir = os.path.join(model_dir, 'final')
os.makedirs(model_save_dir, exist_ok=True)
################################################# MODEL DEFINITION
reg = None
adam_kwargs = {'clipnorm': 1.0}
kernel_initializer_dense = 'glorot_uniform'
if 'changbin' in model_dir:
from keras import regularizers
reg = regularizers.l2(0.0000459)
subsample_time_steps = False
if h.TIME_STEPS >= 2000:
subsample_time_steps = True
if h.TIME_STEPS >= 2000:
from keras import regularizers
reg = regularizers.l2(0.001)
adam_kwargs['clipvalue'] = 0.3
adam_kwargs['clipnorm'] = 0.7
kernel_initializer_dense = 'he_uniform' # should prevent exploding grads for ReLU
print('\nBuild model...\n')
time_steps = h.TIME_STEPS if not subsample_time_steps else h.TIME_STEPS // 2
x = Input(batch_shape=(h.BATCH_SIZE, time_steps, h.N_FEATURES), name='Input', dtype='float32')
y = x
# Input dropout
y = Dropout(h.INPUT_DROPOUT, noise_shape=(h.BATCH_SIZE, 1, h.N_FEATURES))(y)
for units in h.UNITS_PER_LAYER_LSTM:
y = CuDNNLSTM(units, return_sequences=True, stateful=True, kernel_regularizer=reg, recurrent_regularizer=reg)(y)
# LSTM Output dropout
y = Dropout(h.LSTM_OUTPUT_DROPOUT, noise_shape=(h.BATCH_SIZE, 1, units))(y)
for units in h.UNITS_PER_LAYER_MLP:
if units != h.N_CLASSES:
y = Dense(units, activation='relu', kernel_regularizer=reg, kernel_initializer=kernel_initializer_dense)(y)
else:
y = Dense(units, activation='linear', kernel_regularizer=reg)(y)
# MLP Output dropout but not last layer
if units != h.N_CLASSES:
y = Dropout(h.MLP_OUTPUT_DROPOUT, noise_shape=(h.BATCH_SIZE, 1, units))(y)
model = Model(x, y)
model.summary()
print(5 * '\n')
my_loss = tensorflow_utils.my_loss_builder(h.MASK_VAL,
tensorflow_utils.get_loss_weights(ALL_FOLDS, h.TRAIN_SCENES,
h.LABEL_MODE))
################################################# LOAD CHECKPOINTED MODEL
model_is_resumed = False
epochs_finished_old = None
# use val_fold = 0 as dummy for finished epochs
val_fold = 1
val_fold_str = 'final_experiment: {} ({} / {})'.format(val_fold, 1, 1)
latest_weights_path, epochs_finished, val_acc, best_epoch_, best_val_acc_, epochs_without_improvement_ \
= tensorflow_utils.latest_training_state(model_save_dir)
if latest_weights_path is not None:
model.load_weights(latest_weights_path)
model_is_resumed = True
if h.epochs_finished[val_fold - 1] != epochs_finished:
epochs_finished_old = h.epochs_finished[val_fold - 1]
print(
'MISMATCH: Latest state in hyperparameter combination list is different to checkpointed state.')
h.epochs_finished[val_fold - 1] = epochs_finished
h.val_acc[val_fold - 1] = val_acc
hcm.replace_at_id(ID, h)
################################################# COMPILE MODEL
adam = Adam(lr=h.LEARNING_RATE, **adam_kwargs)
model.compile(optimizer=adam, loss=my_loss, metrics=None, sample_weight_mode='temporal')
print('\nModel compiled.\n')
################################################# DATA LOADER
use_multithreading = True
BUFFER = utils.get_buffer_size_wrt_time_steps(h.TIME_STEPS)
train_loader = tr_utils.create_train_dataloader(h.LABEL_MODE, ALL_FOLDS, -1, h.BATCH_SIZE, h.TIME_STEPS,
h.MAX_EPOCHS, 160, 13,
BUFFER=BUFFER, use_multithreading=use_multithreading)
################################################# CALLBACKS
model_ckp_last = ModelCheckpoint(os.path.join(model_save_dir,
'model_ckp_epoch_{epoch:02d}-val_acc_{val_final_acc:.3f}.hdf5'),
verbose=1, monitor='val_final_acc')
model_ckp_last.set_model(model)
args = [h.OUTPUT_THRESHOLD, h.MASK_VAL, h.MAX_EPOCHS, val_fold_str, GLOBAL_GRADIENT_NORM_PLOT,
h.RECURRENT_DROPOUT, h.METRIC]
# training phase
train_phase = tr_utils.Phase('train', model, train_loader, BUFFER, *args,
no_new_weighting=True if 'nnw' in model_save_dir else False,
changbin_recurrent_dropout=True if 'changbin' in model_dir else False,
subsample_time_steps=subsample_time_steps)
if model_is_resumed:
try:
old_metrics = utils.load_metrics(model_save_dir, name="metrics_train")
# merge metrics
h.METRIC = old_metrics['metric']
train_phase.metric = h.METRIC
train_iterations_done = old_metrics['train_losses'].shape[0]
epochs_done = old_metrics['train_accs'].shape[0]
if epochs_finished_old is not None:
epochs_done_old = epochs_done
epochs_done = epochs_done if epochs_finished > epochs_done else epochs_finished
train_iterations_done = int(train_iterations_done / epochs_done_old) * epochs_done
train_phase.losses = old_metrics['train_losses'].tolist()[:train_iterations_done]
train_phase.accs = old_metrics['train_accs'].tolist()[:epochs_done]
train_phase.sens_spec_class_scene = old_metrics['train_sens_spec_class_scene'].tolist()[:epochs_done]
if 'global_gradient_norm' in old_metrics:
train_phase.global_gradient_norms = old_metrics['global_gradient_norm'].tolist()[
:train_iterations_done]
except:
pass
train_phase.resume_from_epoch(h.epochs_finished[val_fold - 1] + 1)
for e in range(h.epochs_finished[val_fold - 1], h.MAX_EPOCHS):
train_loss_is_nan, _ = train_phase.run()
if train_loss_is_nan:
print('\n\n\n---------------------------------------\n\n\n')
print("ERROR: Training loss is NaN.")
print('\n\n\n---------------------------------------\n\n\n')
break
tr_utils.update_latest_model_ckp(model_ckp_last, model_save_dir, e, 0.0)
metrics = {
'metric': h.METRIC,
'train_losses': np.array(train_phase.losses),
'train_accs': np.array(train_phase.accs),
'train_sens_spec_class_scene': np.array(train_phase.sens_spec_class_scene),
}
if GLOBAL_GRADIENT_NORM_PLOT:
metrics['global_gradient_norm'] = np.array(train_phase.global_gradient_norms)
utils.pickle_metrics(metrics, model_save_dir, name="metrics_train")
hcm.finish_epoch(ID, h, 0.0, 0.0, 0.0, 0.0, val_fold - 1, e + 1, e + 1, (timer() - start) / 60)
if INTERMEDIATE_PLOTS:
plot.plot_metrics(metrics, model_save_dir)
if GLOBAL_GRADIENT_NORM_PLOT:
plot.plot_global_gradient_norm(np.array(train_phase.global_gradient_norms), model_save_dir,
epochs_done=e + 1)
del model
K.clear_session()
hcm.finish_hcomb(ID, h)
################################################## TESTING
test_loader = tr_utils.create_test_dataloader(h.LABEL_MODE)
################################################# MODEL DEFINITION
print('\nBuild model for testing...\n')
x = Input(batch_shape=(1, None, h.N_FEATURES), name='Input', dtype='float32')
y = x
# Input dropout
y = Dropout(h.INPUT_DROPOUT, noise_shape=(1, 1, h.N_FEATURES))(y)
for units in h.UNITS_PER_LAYER_LSTM:
y = CuDNNLSTM(units, return_sequences=True, stateful=True, kernel_regularizer=reg, recurrent_regularizer=reg)(y)
# LSTM Output dropout
y = Dropout(h.LSTM_OUTPUT_DROPOUT, noise_shape=(1, 1, units))(y)
for units in h.UNITS_PER_LAYER_MLP:
if units != h.N_CLASSES:
y = Dense(units, activation='relu', kernel_regularizer=reg, kernel_initializer=kernel_initializer_dense)(y)
else:
y = Dense(units, activation='linear', kernel_regularizer=reg)(y)
# MLP Output dropout but not last layer
if units != h.N_CLASSES:
y = Dropout(h.MLP_OUTPUT_DROPOUT, noise_shape=(1, 1, units))(y)
model = Model(x, y)
model.summary()
latest_weights_path, _, _, _, _, _ = tensorflow_utils.latest_training_state(model_save_dir)
if latest_weights_path is not None:
model.load_weights(latest_weights_path)
model.compile(optimizer=adam, loss=my_loss, metrics=None)
print('\nModel compiled.\n')
test_phase = tr_utils.TestPhase(model, test_loader, h.OUTPUT_THRESHOLD, h.MASK_VAL, 1, val_fold_str, model_save_dir,
metric=('BAC', 'BAC2'), ret=('final', 'per_class', 'per_class_scene', 'per_scene'))
test_loss_is_nan, _ = test_phase.run()
metrics_test = {
'metric': h.METRIC,
'test_accs': np.array(test_phase.accs),
'test_accs_bac2': np.array(test_phase.accs_bac2),
'test_class_accs': np.array(test_phase.class_accs),
'test_class_accs_bac2': np.array(test_phase.class_accs_bac2),
'test_class_scene_accs': np.array(test_phase.class_scene_accs),
'test_class_scene_accs_bac2': np.array(test_phase.class_scene_accs_bac2),
'test_scene_accs': np.array(test_phase.scene_accs),
'test_scene_accs_bac2': np.array(test_phase.scene_accs_bac2),
'test_sens_spec_class_scene': np.array(test_phase.sens_spec_class_scene),
'test_sens_spec_class': np.array(test_phase.sens_spec_class)
}
utils.pickle_metrics(metrics_test, model_save_dir)
else:
print('\n\n\n---------------------------------------\n\n\n')
print("ERROR: No testing possible, because no trained model saved.")
print('\n\n\n---------------------------------------\n\n\n')
go_to_next_stage = False
return go_to_next_stage
def run_hcomb_cv(h, ID, hcm, model_dir, INTERMEDIATE_PLOTS, GLOBAL_GRADIENT_NORM_PLOT):
################################################# CROSS VALIDATION
start = timer()
NUMBER_OF_CLASSES = 13
# METRICS
ALL_FOLDS = h.ALL_FOLDS if h.ALL_FOLDS != -1 else list(range(1, 7))
best_val_class_accuracies_over_folds = [[0] * NUMBER_OF_CLASSES] * len(ALL_FOLDS)
best_val_acc_over_folds = [0] * len(ALL_FOLDS)
best_val_class_accuracies_over_folds_bac2 = [[0] * NUMBER_OF_CLASSES] * len(ALL_FOLDS)
best_val_acc_over_folds_bac2 = [0] * len(ALL_FOLDS)
go_to_next_stage = False
subsample_time_steps = False
if h.TIME_STEPS >= 2000:
subsample_time_steps = True
print(5 * '\n' + 'Starting Cross Validation STAGE {}...\n'.format(h.STAGE))
for i_val_fold, val_fold in enumerate(h.VAL_FOLDS):
model_save_dir = os.path.join(model_dir, 'val_fold{}'.format(val_fold))
os.makedirs(model_save_dir, exist_ok=True)
TRAIN_FOLDS = list(set(ALL_FOLDS).difference({val_fold}))
val_fold_str = 'val_fold: {} ({} / {})'.format(val_fold, i_val_fold + 1, len(h.VAL_FOLDS))
################################################# MODEL DEFINITION
print('\nBuild model...\n')
time_steps = h.TIME_STEPS if not subsample_time_steps else h.TIME_STEPS // 2
x = Input(batch_shape=(h.BATCH_SIZE, time_steps, h.N_FEATURES), name='Input', dtype='float32')
y = x
# Input dropout
y = Dropout(h.INPUT_DROPOUT, noise_shape=(h.BATCH_SIZE, 1, h.N_FEATURES))(y)
for units in h.UNITS_PER_LAYER_LSTM:
y = CuDNNLSTM(units, return_sequences=True, stateful=True)(y)
# LSTM Output dropout
y = Dropout(h.LSTM_OUTPUT_DROPOUT, noise_shape=(h.BATCH_SIZE, 1, units))(y)
for units in h.UNITS_PER_LAYER_MLP:
if units != h.N_CLASSES:
y = Dense(units, activation='relu')(y)
else:
y = Dense(units, activation='linear')(y)
# MLP Output dropout but not last layer
if units != h.N_CLASSES:
y = Dropout(h.MLP_OUTPUT_DROPOUT, noise_shape=(h.BATCH_SIZE, 1, units))(y)
model = Model(x, y)
model.summary()
print(5 * '\n')
my_loss = tensorflow_utils.my_loss_builder(h.MASK_VAL,
tensorflow_utils.get_loss_weights(TRAIN_FOLDS, h.TRAIN_SCENES,
h.LABEL_MODE))
################################################# LOAD CHECKPOINTED MODEL
model_is_resumed = False
epochs_finished_old = None
latest_weights_path, epochs_finished, val_acc, best_epoch_, best_val_acc_, epochs_without_improvement_ \
= tensorflow_utils.latest_training_state(model_save_dir)
if latest_weights_path is not None:
model.load_weights(latest_weights_path)
model_is_resumed = True
if h.epochs_finished[val_fold - 1] != epochs_finished:
epochs_finished_old = h.epochs_finished[val_fold - 1]
print(
'MISMATCH: Latest state in hyperparameter combination list is different to checkpointed state.')
h.epochs_finished[val_fold - 1] = epochs_finished
h.val_acc[val_fold - 1] = val_acc
hcm.replace_at_id(ID, h)
################################################# COMPILE MODEL
adam = Adam(lr=h.LEARNING_RATE, clipnorm=1.)
model.compile(optimizer=adam, loss=my_loss, metrics=None, sample_weight_mode='temporal')
print('\nModel compiled.\n')
################################################# DATA LOADER
use_multithreading = True
BUFFER = utils.get_buffer_size_wrt_time_steps(h.TIME_STEPS)
train_loader, val_loader = tr_utils.create_dataloaders(h.LABEL_MODE, TRAIN_FOLDS, h.TRAIN_SCENES,
h.BATCH_SIZE,
h.TIME_STEPS, h.MAX_EPOCHS, h.N_FEATURES,
h.N_CLASSES,
[val_fold], h.VAL_STATEFUL,
BUFFER=BUFFER, use_multithreading=use_multithreading,
subsample_time_steps=subsample_time_steps)
################################################# CALLBACKS
model_ckp_last = ModelCheckpoint(os.path.join(model_save_dir,
'model_ckp_epoch_{epoch:02d}-val_acc_{val_final_acc:.3f}.hdf5'),
verbose=1, monitor='val_final_acc')
model_ckp_last.set_model(model)
model_ckp_best = ModelCheckpoint(os.path.join(model_save_dir,
'best_model_ckp_epoch_{epoch:02d}-val_acc_{val_final_acc:.3f}.hdf5'),
verbose=1, monitor='val_final_acc', save_best_only=True)
model_ckp_best.set_model(model)
args = [h.OUTPUT_THRESHOLD, h.MASK_VAL, h.MAX_EPOCHS, val_fold_str, GLOBAL_GRADIENT_NORM_PLOT,
h.RECURRENT_DROPOUT, h.METRIC]
# training phase
train_phase = tr_utils.Phase('train', model, train_loader, BUFFER, *args,
no_new_weighting=True if 'nnw' in model_save_dir else False,
subsample_time_steps=subsample_time_steps)
# validation phase
val_phase = tr_utils.Phase('val', model, val_loader, BUFFER, *args,
no_new_weighting=True if 'nnw' in model_save_dir else False)
# needed for early stopping
best_val_acc = -1 if not model_is_resumed else best_val_acc_
best_val_acc_bac2 = -1
best_epoch = 0 if not model_is_resumed else best_epoch_
epochs_without_improvement = 0 if not model_is_resumed else epochs_without_improvement_
if model_is_resumed:
old_metrics = utils.load_metrics(model_save_dir)
# merge metrics
h.METRIC = old_metrics['metric']
train_phase.metric = h.METRIC
val_phase.metric = h.METRIC
train_iterations_done = old_metrics['train_losses'].shape[0]
val_iterations_done = old_metrics['val_losses'].shape[0]
epochs_done = old_metrics['val_accs'].shape[0]
if epochs_finished_old is not None:
epochs_done_old = epochs_done
epochs_done = epochs_done if epochs_finished > epochs_done else epochs_finished
train_iterations_done = int(train_iterations_done / epochs_done_old) * epochs_done
val_iterations_done = int(val_iterations_done / epochs_done_old) * epochs_done
train_phase.losses = old_metrics['train_losses'].tolist()[:train_iterations_done]
train_phase.accs = old_metrics['train_accs'].tolist()[:epochs_done]
val_phase.losses = old_metrics['val_losses'].tolist()[:val_iterations_done]
val_phase.accs = old_metrics['val_accs'].tolist()[:epochs_done]
val_phase.accs_bac2 = old_metrics['val_accs_bac2'].tolist()[:epochs_done]
val_phase.class_accs = old_metrics['val_class_accs'].tolist()[:epochs_done]
val_phase.class_accs_bac2 = old_metrics['val_class_accs_bac2'].tolist()[:epochs_done]
val_phase.class_scene_accs = old_metrics['val_class_scene_accs'].tolist()[:epochs_done]
val_phase.class_scene_accs_bac2 = old_metrics['val_class_scene_accs_bac2'].tolist()[:epochs_done]
val_phase.scene_accs = old_metrics['val_scene_accs'].tolist()[:epochs_done]
val_phase.scene_accs_bac2 = old_metrics['val_scene_accs_bac2'].tolist()[:epochs_done]
train_phase.sens_spec_class_scene = old_metrics['train_sens_spec_class_scene'].tolist()[:epochs_done]
val_phase.sens_spec_class_scene = old_metrics['val_sens_spec_class_scene'].tolist()[:epochs_done]
val_phase.sens_spec_class = old_metrics['val_sens_spec_class'].tolist()[:epochs_done]
if 'global_gradient_norm' in old_metrics:
train_phase.global_gradient_norms = old_metrics['global_gradient_norm'].tolist()[
:train_iterations_done]
best_val_acc = np.max(val_phase.accs)
best_val_acc_bac2 = old_metrics['val_accs_bac2'][np.argmax(val_phase.accs)]
# set the dataloaders to correct epoch
train_phase.resume_from_epoch(h.epochs_finished[val_fold - 1] + 1)
val_phase.resume_from_epoch(h.epochs_finished[val_fold - 1] + 1)
stage_was_finished = True
loss_is_nan = False
for e in range(h.epochs_finished[val_fold - 1], h.MAX_EPOCHS):
# early stopping
if epochs_without_improvement >= h.PATIENCE_IN_EPOCHS and h.PATIENCE_IN_EPOCHS > 0:
break
else:
stage_was_finished = False
train_loss_is_nan, _ = train_phase.run()
val_loss_is_nan, _ = val_phase.run()
if train_loss_is_nan or val_loss_is_nan:
loss_is_nan = True
print('\n\n\n---------------------------------------\n\n\n')
print("ERROR: Training loss is NaN.")
print('\n\n\n---------------------------------------\n\n\n')
break
tr_utils.update_latest_model_ckp(model_ckp_last, model_save_dir, e, val_phase.accs[-1])
tr_utils.update_best_model_ckp(model_ckp_best, model_save_dir, e, val_phase.accs[-1])
metrics = {
'metric': h.METRIC,
'train_losses': np.array(train_phase.losses),
'train_accs': np.array(train_phase.accs),
'val_losses': np.array(val_phase.losses),
'val_accs': np.array(val_phase.accs),
'val_accs_bac2': np.array(val_phase.accs_bac2),
'val_class_accs': np.array(val_phase.class_accs),
'val_class_accs_bac2': np.array(val_phase.class_accs_bac2),
'val_class_scene_accs': np.array(val_phase.class_scene_accs),
'val_class_scene_accs_bac2': np.array(val_phase.class_scene_accs_bac2),
'val_scene_accs': np.array(val_phase.scene_accs),
'val_scene_accs_bac2': np.array(val_phase.scene_accs_bac2),
'train_sens_spec_class_scene': np.array(train_phase.sens_spec_class_scene),
'val_sens_spec_class_scene': np.array(val_phase.sens_spec_class_scene),
'val_sens_spec_class': np.array(val_phase.sens_spec_class)
}
if GLOBAL_GRADIENT_NORM_PLOT:
metrics['global_gradient_norm'] = np.array(train_phase.global_gradient_norms)
utils.pickle_metrics(metrics, model_save_dir)
if val_phase.accs[-1] > best_val_acc:
best_val_acc = val_phase.accs[-1]
best_val_acc_bac2 = val_phase.accs_bac2[-1]
epochs_without_improvement = 0
best_epoch = e + 1
else:
epochs_without_improvement += 1
hcm.finish_epoch(ID, h, val_phase.accs[-1], best_val_acc, val_phase.accs_bac2[-1], best_val_acc_bac2,
val_fold - 1, e + 1, best_epoch, (timer() - start) / 60)
if INTERMEDIATE_PLOTS:
plot.plot_metrics(metrics, model_save_dir)
if GLOBAL_GRADIENT_NORM_PLOT:
plot.plot_global_gradient_norm(np.array(train_phase.global_gradient_norms), model_save_dir,
epochs_done=e + 1)
del metrics
if not loss_is_nan:
if not stage_was_finished:
best_val_class_accuracies_over_folds[val_fold - 1] = val_phase.class_accs[best_epoch - 1]
best_val_acc_over_folds[val_fold - 1] = val_phase.accs[best_epoch - 1]
best_val_class_accuracies_over_folds_bac2[val_fold - 1] = val_phase.class_accs_bac2[best_epoch - 1]
best_val_acc_over_folds_bac2[val_fold - 1] = val_phase.accs_bac2[best_epoch - 1]
################################################# CROSS VALIDATION: MEAN AND VARIANCE
best_val_class_accs_over_folds = np.array(best_val_class_accuracies_over_folds)
best_val_accs_over_folds = np.array(best_val_acc_over_folds)
best_val_class_accs_over_folds_bac2 = np.array(best_val_class_accuracies_over_folds_bac2)
best_val_accs_over_folds_bac2 = np.array(best_val_acc_over_folds_bac2)
metrics_over_folds = utils.create_metrics_over_folds_dict(best_val_class_accs_over_folds,
best_val_accs_over_folds,
best_val_class_accs_over_folds_bac2,
best_val_accs_over_folds_bac2)
if h.STAGE > 1:
metrics_over_folds_old = utils.load_metrics(model_dir)
best_val_class_accs_over_folds += metrics_over_folds_old['best_val_class_accs_over_folds']
best_val_accs_over_folds += metrics_over_folds_old['best_val_acc_over_folds']
best_val_class_accs_over_folds_bac2 += metrics_over_folds_old['best_val_class_accs_over_folds_bac2']
best_val_accs_over_folds_bac2 += metrics_over_folds_old['best_val_acc_over_folds_bac2']
metrics_over_folds = utils.create_metrics_over_folds_dict(best_val_class_accs_over_folds,
best_val_accs_over_folds,
best_val_class_accs_over_folds_bac2,
best_val_accs_over_folds_bac2)
utils.pickle_metrics(metrics_over_folds, model_dir)
if INTERMEDIATE_PLOTS:
plot.plot_metrics(metrics_over_folds, model_dir)
hcm.finish_stage(ID, h,
metrics_over_folds['best_val_acc_mean_over_folds'],
metrics_over_folds['best_val_acc_std_over_folds'],
metrics_over_folds['best_val_acc_mean_over_folds_bac2'],
metrics_over_folds['best_val_acc_std_over_folds_bac2'],
timer() - start)
else:
metrics_over_folds = utils.load_metrics(model_dir)
# STAGE thresholds
stage_thresholds = {1: 0.81, 2: 0.81, 3: np.inf} # 3 is the last stage
if metrics_over_folds['best_val_acc_mean_over_folds'] >= stage_thresholds[h.STAGE]:
go_to_next_stage = True
if go_to_next_stage:
hcm.next_stage(ID, h)
else:
if h.STAGE == 3 or stage_thresholds[h.STAGE] != np.inf:
hcm.finish_hcomb(ID, h)
return go_to_next_stage
else:
hcm.finish_hcomb(ID, h)
return False
def run(network="dnn",
amount=100,
val_split=0.5,
split_type="simple",
name="experiment"):
# Fetch data and make simple split of data
X1, Y1, X2, Y2, X3, Y3 = transfer.get_data(split_type=split_type,
amt=amount)
if network == "cnn":
# Need to expand dimension for CNN to make sense
X1, X2, X3 = np.expand_dims(X1, axis=2), np.expand_dims(
X2, axis=2), np.expand_dims(X3, axis=2)
main, intermediate, shallow = networks.create_cnn()
comparer, _, _ = networks.create_cnn()
elif network == "dnn":
main, intermediate, shallow = networks.create_dnn()
comparer, _, _ = networks.create_dnn()
elif network == "mlp":
main, intermediate, shallow = networks.create_mlp()
comparer, _, _ = networks.create_mlp()
elif network == "wide_cnn":
X1, X2, X3 = np.expand_dims(X1, axis=2), np.expand_dims(
X2, axis=2), np.expand_dims(X3, axis=2)
main, intermediate, shallow = networks.create_wider_cnn()
comparer, _, _ = networks.create_wider_cnn()
elif network == "large_cnn":
X1, X2, X3 = np.expand_dims(X1, axis=2), np.expand_dims(
X2, axis=2), np.expand_dims(X3, axis=2)
main, intermediate, shallow = networks.create_large_cnn()
comparer, _, _ = networks.create_large_cnn()
else:
main, intermediate, shallow = networks.create_dnn()
# Split data for training/testing for before and after transfer
first_half = (X1, Y1)
second_half = (X2, Y2)
# Train and transfer main and shallow networks
main, history, cbs = transfer.train_and_validate(
main, data=first_half, validation_split=val_split)
intermediate, shallow, shallow_history, shallow_cbs = transfer.transfer_and_repeat(
main,
intermediate,
shallow,
data=second_half,
validation_split=val_split)
# Train comparer deep network to compare against shallow network
comparer, comparer_history, comprarer_cbs = transfer.train_and_validate(
comparer, data=second_half, validation_split=val_split)
unique_name = "{}-{}-{}".format(network, split_type, name)
transfer.save_model(main, "{}-main".format(unique_name))
transfer.save_model(intermediate, "{}-intermediate".format(unique_name))
transfer.save_model(shallow, "{}-shallow".format(unique_name))
transfer.save_model(comparer, "{}-comparer".format(unique_name))
# main_result, shallow_result = plotting.plot_metrics(unique_name, main, intermediate, shallow, X3, Y3)
comparer_res, shallow_res = plotting.plot_metrics(unique_name, comparer,
intermediate, shallow,
X3, Y3)
plotting.plot_acc(history, "main-{}".format(unique_name))
plotting.plot_acc(shallow_history, "shallow-{}".format(unique_name))
plotting.plot_acc(comparer_history, "comparer-{}".format(unique_name))
return (main, history,
cbs), (intermediate, shallow, shallow_history, shallow_cbs,
shallow_res), (comparer, comparer_history, comprarer_cbs,
comparer_res)
#######################################################')
###### Parameters:
if args.mode:
_mode = args.mode
else:
_mode = "dm"
_filename = args.output_filename #"/data/results/run1/" # + "mymodel_{}".format(epoch+1) or + "log0.cvs"
### Create the directory:
try:
os.mkdir(_filename)
except OSError:
print("Creation of the directory %s failed" % _filename)
sys.exit()
else:
print("Successfully created the directory %s " % _filename)
R.DataLoader.Initialize('/data/store/reco_skim_v1/tau_DYJetsToLL_M-50_v2.root')
history = training(mode=_mode, filename=_filename,
parameters=parameters) # trains the model
plot_metrics(
history, mode=_mode, filename=_filename
) # Plots the loss and accuracy curves for trainset and validationset
print('#######################################################\
\n Training finished !!! \n\
#######################################################')
print "========================\nWorking with %s\n========================"%(prot_name)
# Look for the directory and enter it
base_dir = os.path.join(cwd, prot_name)
os.chdir(base_dir)
traj_pdb = "%s.pdb"%prot_name
###############################################
# Generate metrics plot for the whole ensemble
###############################################
plots = datum["plots"]
for metrics_key in datum["plots"]:
records = []
processFile(traj_pdb, records, True)
all_metrics = genMetrics(plots[metrics_key], records)
print "* %d metrics extracted"%(len(all_metrics))
plot_metrics(os.path.join(base_dir, "%s.svg"%metrics_key), all_metrics, plots[metrics_key][0],plots[metrics_key][1])
######################################
# Read the ligand info (only one line)
######################################
ligand_file = open(os.path.join(base_dir, "%s_ligand.txt"%(prot_name)),"r")
ligand_file_description = ligand_file.readlines()[0].split()
ligand_file.close()
ligand = {
"resname":ligand_file_description[0],
"chain":ligand_file_description[1],
"atoms":ligand_file_description[2:]
}
ligand_description = "resname %s and name %s"%(ligand["resname"],"".join( a+" " for a in ligand["atoms"]))
print "* Ligand parsed: ",ligand_description
