def count_variables(self):
if self.model is not None:
trainable = count_params(self.model.trainable_weights)
non_trainable = count_params(self.model.non_trainable_weights)
return ({"trainable": trainable, "non_trainable": non_trainable})
else:
return None
def freeze_model(model, num = -1, trainable = False):
from keras.utils.layer_utils import count_params
if trainable:
for layer in model.layers:
layer.trainable = True
if num==-1:
for layer in model.layers:
layer.trainable = False
else:
for layer in model.layers[:num]:
layer.trainable = False
# recompile model
model.compile(loss = model.loss, optimizer=model.optimizer, metrics = model.metrics)
trainable_count = count_params(model.trainable_weights)
non_trainable_count = count_params(model.non_trainable_weights)
print(f'Trainable Params: {trainable_count:,d} || Non-Trainable Params: {non_trainable_count:,d}')
return model
def count_params(model):
from keras.utils.layer_utils import count_params
model._check_trainable_weights_consistency()
if hasattr(model, '_collected_trainable_weights'):
trainable_count = count_params(model._collected_trainable_weights)
else:
trainable_count = count_params(model.trainable_weights)
return trainable_count
def print_weights(self):
no_trainable = count_params(self.SimCLR_classifier.trainable_weights)
no_nontrainable = count_params(self.SimCLR_classifier.non_trainable_weights)
print(f'trainable counts (1): {round(no_trainable/1e6, 2)} M.')
print(f'trainable counts (2): {len(self.SimCLR_classifier.trainable_weights)}')
print(f'non-trainable counts (1): {round(no_nontrainable/1e6, 2)} M.')
print(f'non-trainable counts (2): {len(self.SimCLR_classifier.non_trainable_weights)}')
def print_weights(self):
""" Function to print (non)-learnable weights
Helps checking unfreezing process
"""
trainable_count = count_params(self.SimCLR_model.trainable_weights)
non_trainable_count = count_params(
self.SimCLR_model.non_trainable_weights)
print(f"trainable parameters: {round(trainable_count/1e6,2)} M.")
print(
f"non-trainable parameters: {round(non_trainable_count/1e6,2)} M.")
def get_structure_info(model: tf.keras.Model) -> StructureInfo:
trainable_count = count_params(model.trainable_weights)
non_trainable_count = count_params(model.non_trainable_weights)
info = StructureInfo()
info.variable_count = trainable_count
info.weights_size = (trainable_count + non_trainable_count) * 4
info.svg = model_to_dot(model,
show_layer_names=True,
show_shapes=True,
dpi=None).create(prog='dot', format='svg')
info.flops = get_flops(model, batch_size=1)
return info
def datadefreezer(self, num_of_unfrozen_layers, prout=False):
for layer in range(0, len(self.classification.layers), 1):
if layer < number_of_unfrozen_layers:
self.classification.layers[layer].trainable = False
else:
self.classification.layers[layer].trainable = True
if prout:
no_trainable = count_params(self.classification.trainable_weights)
no_nontrainable = count_params(self.classification.non_trainable_weights)
print(f'trainable counts (1): {round(no_trainable/1e6, 2)} M.')
print(f'trainable counts (2): {len(self.classification.trainable_weights)}')
print(f'non-trainable counts (1): {round(no_nontrainable/1e6, 2)} M.')
print(f'non-trainable counts (2): {len(self.classification.non_trainable_weights)}')
def get_de_loss(self, net_model, train_hist, last_n_perc=0.1):
"""
Make the differential evolution loss.
:return:
"""
nb_params = count_params(net_model.trainable_weights)
res_loss_fct = self.de_loss['Res_Loss']
param_loss_fct = self.de_loss['Param_Loss']
last_n = int(last_n_perc * self.epochs)
last_n_losses = train_hist.history['loss'][-last_n - 1:]
if self.loss_mode == 'last_n':
res_loss_val = np.mean(last_n_losses)
else:
res_loss_val = res_loss_fct(
self.train_dict['y'],
net_model.predict(self.train_dict['x']).flatten()).numpy()
p_loss_val = param_loss_fct(nb_params)
return res_loss_val, p_loss_val, {
'Residual Loss': res_loss_val,
'Nb Params Loss': p_loss_val
}
def __init__(self,
output_directory,
input_shape,
nb_classes,
verbose=False,
build=True,
batch_size=64,
lr=0.001,
nb_filters=32,
use_residual=True,
use_bottleneck=True,
depth=6,
kernel_size=41,
nb_epochs=2000,
bottleneck_size=32,
class_weight=None):
self.output_directory = output_directory
self.loss = coral.OrdinalCrossEntropy(num_classes=nb_classes,
importance_weights=class_weight)
self.nb_filters = nb_filters
self.use_residual = use_residual
self.use_bottleneck = use_bottleneck
self.depth = depth
self.kernel_size = kernel_size - 1
self.callbacks = None
self.batch_size = batch_size
self.bottleneck_size = bottleneck_size
self.nb_epochs = nb_epochs
self.lr = lr
self.verbose = verbose
if build == True:
self.model = self.build_model(input_shape, nb_classes)
if (verbose == True):
self.model.summary()
self.model.save_weights(self.output_directory + 'model_init.hdf5')
trainable_count = count_params(self.model.trainable_weights)
model_hyper = {
'model': 'masked-inception',
'filters': nb_filters,
'residuals': use_residual,
'bottleneck': use_bottleneck,
'depth': depth,
'kernel_size': kernel_size,
'batch_size': batch_size,
'epochs': nb_epochs,
'bottleneck_size': self.bottleneck_size,
'classes': nb_classes,
'input_shape': input_shape,
'trainable_params': trainable_count
}
f = open(os.path.join(self.output_directory, 'hyperparams.txt'), "w")
f.write(str(model_hyper))
f.close()
def __init__(self, output_directory, input_shape, nb_classes, lr=0.001,
batch_size=16, verbose=2, nb_epochs=2000, depth=1,
filters=16, window=21, decay=False):
input_shape = (None, None, input_shape[-1])
self.output_directory = output_directory
self.batch_size = batch_size
self.verbose = verbose
self.lr = lr
self.nb_epochs = nb_epochs
self.input_shape = input_shape
self.nb_classes = nb_classes
self.depth = depth
self.filters = filters
self.window = window
self.decay = decay
self.model = self.build_model()
trainable_count = count_params(self.model.trainable_weights)
model_hyper = {'model': 'masked-xcm', 'filters': filters,
'depth': depth, 'window_size': window, 'decay': decay,
'batch_size': batch_size, 'classes': nb_classes,
'input_shape': input_shape, 'epochs': nb_epochs,
'trainable_params': trainable_count}
f = open(os.path.join(self.output_directory, 'hyperparams.txt'), "w")
f.write(str(model_hyper))
f.close()
return
def on_epoch_end(self, epoch, logs=None):
super().on_epoch_end(epoch, logs)
trainable_count = layer_utils.count_params(
self.model.trainable_weights)
non_trainable_count = layer_utils.count_params(
self.model.non_trainable_weights)
self._params_logger.log_scalar("trainable_parameters", trainable_count,
epoch)
self._params_logger.log_scalar("non_trainable_parameters",
non_trainable_count, epoch)
if self._verbose > 0:
print("Trainable PARAMS at epoch {0}: {1:,}".format(
epoch, trainable_count))
print("Non trainable PARAMS at epoch {0}: {1:,}".format(
epoch, non_trainable_count))
def construct(self):
inputs = [Input(input_shape) for input_shape in self.input_shapes]
outputs = self.construct_graph_fn(inputs)
self.model = Model(inputs=inputs, outputs=outputs)
self._compile()
if mlflow.active_run():
mlflow.log_metric('num_of_parameters',
count_params(self.model.trainable_weights))
return self.model
def update_from_model(self):
from keras.models import load_model
from keras.utils import plot_model
from keras.utils.layer_utils import count_params
model = load_model(
os.path.join(settings.MEDIA_ROOT, self.model_file.name))
model._check_trainable_weights_consistency()
self.inputs = [str(i) for i in model.inputs]
self.outputs = [str(o) for o in model.outputs]
self.config = model.get_config()
self.nb_non_trainable = count_params(model.non_trainable_weights)
if hasattr(model, '_collected_trainable_weights'):
self.nb_trainable = count_params(
model._collected_trainable_weights)
else:
self.nb_trainable = count_params(model.trainable_weights)
img_path = os.path.join(settings.MEDIA_ROOT, self.plot_file())
plot_model(model, to_file=img_path)
def __init__(self, output_directory, input_shape, nb_classes, lr=0.001,
batch_size=16, verbose=2, nb_epochs=2000, depth=1, mask=True,
filters=16, window=21, decay=False, metric='val_accuracy',
reduce_lr_metric='train_loss'):
input_shape = (None, None, input_shape[-1])
self.output_directory = output_directory
self.batch_size = batch_size
self.verbose = verbose
self.lr = lr
self.nb_epochs = nb_epochs
self.input_shape = input_shape
self.nb_classes = nb_classes
self.depth = depth
self.filters = filters
self.window = window
self.decay = decay
self.mask = mask
if metric == 'train_loss':
self.metric = 0
elif metric == 'train_accuracy':
self.metric = 1
elif metric == 'val_loss':
self.metric = 2
else:
self.metric = 3
if reduce_lr_metric == 'train_accuracy':
self.reduce_lr_metric = 1
elif reduce_lr_metric == 'val_loss':
self.reduce_lr_metric = 2
elif reduce_lr_metric == 'val_accuracy':
self.reduce_lr_metric = 3
else:
self.reduce_lr_metric = 0
self.model = self.build_model()
trainable_count = count_params(self.model.trainable_weights)
model_hyper = {'model': 'masked-xcm', 'filters': filters, 'mask': mask,
'depth': depth, 'window_size': window, 'decay': decay,
'batch_size': batch_size, 'classes': nb_classes,
'input_shape': input_shape, 'epochs': nb_epochs,
'trainable_params': trainable_count, 'metric': metric,
'lr_metric': reduce_lr_metric}
f = open(os.path.join(self.output_directory, 'hyperparams.txt'), "w")
f.write(str(model_hyper))
f.close()
return
def __init__(self,
output_directory,
input_shape,
nb_classes,
verbose=False,
build=True,
load_weights=False,
n_feature_maps=64,
depth=3,
nb_epochs=1500,
batch_size=64):
self.output_directory = output_directory
self.n_feature_maps = n_feature_maps
self.depth = depth
self.nb_epochs = nb_epochs
self.batch_size = batch_size
if build == True:
self.model = self.build_model(input_shape, nb_classes)
if (verbose == True):
self.model.summary()
self.verbose = verbose
if load_weights == True:
self.model.load_weights(
self.output_directory.replace('resnet_augment', 'resnet').
replace('TSC_itr_augment_x_10', 'TSC_itr_10') +
'/model_init.hdf5')
else:
self.model.save_weights(self.output_directory +
'model_init.hdf5')
trainable_count = count_params(self.model.trainable_weights)
model_hyper = {
'model': 'masked-resnet',
'classes': nb_classes,
'input_shape': input_shape,
'depth': depth,
'feature_maps': n_feature_maps,
'epochs': nb_epochs,
'trainable_params': trainable_count,
'batch_size': batch_size
}
f = open(os.path.join(self.output_directory, 'hyperparams.txt'), "w")
f.write(str(model_hyper))
f.close()
return
def build_and_fit_model(x, *args):
train_data_gen, val_data_gen = args
layer_codes = np.array(x).reshape(-1, 2).astype(int)
print(layer_codes)
try:
model = build_model(layer_codes)
except ValueError:
return 0.0
model.compile(loss="categorical_crossentropy",
optimizer=Adadelta(),
metrics=["accuracy"])
print(model.summary())
trainable_count = count_params(model.trainable_weights)
print("trainable_count", trainable_count)
if trainable_count > 0:
try:
history = model.fit_generator(
generator=train_data_gen,
steps_per_epoch=train_data_gen.n // train_data_gen.batch_size,
epochs=10,
validation_data=val_data_gen,
validation_steps=val_data_gen.n // val_data_gen.batch_size,
verbose=1)
except ValueError:
return 0.0
val_acc = history.history["val_acc"][-1]
with open(_RESULTS_FILE_NAME, "a") as f:
f.write("{}\n".format(
json_tricks.dumps(
{
"model_config": model.to_json(),
"val_acc": val_acc,
},
indent=4,
sort_keys=True)))
else:
val_acc = 0.0
return -val_acc
def __init__(self,
output_directory,
input_shape,
nb_classes,
depth=4,
kernel_size=21,
filters=64,
verbose=2,
build=True,
nb_epochs=2000,
batch_size=64):
self.output_directory = output_directory
self.depth = depth
self.kernel_size = kernel_size
self.filters = filters
self.nb_epochs = nb_epochs
self.batch_size = batch_size
if build == True:
self.model = self.build_model(input_shape, nb_classes)
if (verbose == True):
self.model.summary()
self.verbose = verbose
self.model.save_weights(self.output_directory + 'model_init.hdf5')
trainable_count = count_params(self.model.trainable_weights)
model_hyper = {
'model': 'masked-fcn',
'filters': filters,
'depth': depth,
'kernel_size': kernel_size,
'batch_size': batch_size,
'classes': nb_classes,
'input_shape': input_shape,
'epochs': nb_epochs,
'trainable_params': trainable_count
}
f = open(os.path.join(self.output_directory, 'hyperparams.txt'), "w")
f.write(str(model_hyper))
f.close()
return
def test_model_implementation_should_look_like(raw_data, simple_transformer):
transformed_model_data = simple_transformer.transform(raw_data)
train_data, test_data = Splitter.train_test_split(transformed_model_data)
backend.clear_session()
convnet = ChunkyCNN(data_shape=(72, 63),
model_config_dir='./tests/model_dir',
component_name='global')
assert count_params(convnet.model.trainable_weights) == 32414
convnet.train(train_data, n_epochs=1)
layer_0_weights = convnet.model.layers[0].get_weights()
convnet.load_model_weights('global_modelparams.h5')
convnet.train(train_data, n_epochs=1)
assert not np.array_equal(layer_0_weights,
convnet.model.layers[0].get_weights())
convnet.load_model_weights('global_modelparams.h5')
prediction = convnet.predict(test_data)
assert all(np.round(prediction.probabilities) == prediction.classes)
evaluation_stats = convnet.evaluate(test_data, prediction)
assert np.sum(evaluation_stats.support) == len(test_data.labels)
def get_number_of_weights(model):
return count_params(model.trainable_weights) + count_params(
model.non_trainable_weights)
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
self.conv_layers = {c: [] for c in ['i', 'f', 'c', 'o', 'a', 'ahat']}
for l in range(self.nb_layers):
for c in ['i', 'f', 'c', 'o']:
act = self.LSTM_activation if c == 'c' else self.LSTM_inner_activation
self.conv_layers[c].append(
Conv2D(self.R_stack_sizes[l],
self.R_filt_sizes[l],
padding='same',
activation=act,
data_format=self.data_format))
act = 'relu' if l == 0 else self.A_activation
self.conv_layers['ahat'].append(
Conv2D(self.stack_sizes[l],
self.Ahat_filt_sizes[l],
padding='same',
activation=act,
data_format=self.data_format))
if l < self.nb_layers - 1:
self.conv_layers['a'].append(
Conv2D(self.stack_sizes[l + 1],
self.A_filt_sizes[l],
padding='same',
activation=self.A_activation,
data_format=self.data_format))
self.upsample = UpSampling2D(data_format=self.data_format)
self.pool = MaxPooling2D(data_format=self.data_format)
self.trainable_weights = []
print('PredNet trainable layers:', self.trainable_layers)
print('PredNet trainable units:', self.trainable_units)
nb_row, nb_col = (
input_shape[-2],
input_shape[-1]) if self.data_format == 'channels_first' else (
input_shape[-3], input_shape[-2])
trainable_param_count = 0
non_trainable_param_count = 0
for c in sorted(self.conv_layers.keys()):
for l in range(len(self.conv_layers[c])):
ds_factor = 2**l
if c == 'ahat':
nb_channels = self.R_stack_sizes[l]
elif c == 'a':
nb_channels = 2 * self.R_stack_sizes[l]
else:
nb_channels = self.stack_sizes[l] * 2 + self.R_stack_sizes[
l]
if l < self.nb_layers - 1:
nb_channels += self.R_stack_sizes[l + 1]
in_shape = (input_shape[0], nb_channels, nb_row // ds_factor,
nb_col // ds_factor)
if self.data_format == 'channels_last':
in_shape = (in_shape[0], in_shape[2], in_shape[3],
in_shape[1])
trainable = (self.trainable_layers is None or l in self.trainable_layers) and \
(self.trainable_units is None or c in self.trainable_units)
with K.name_scope('layer_' + c + '_' + str(l)):
self.conv_layers[c][l].trainable = trainable
self.conv_layers[c][l].build(in_shape)
if trainable:
layer_param_count = layer_utils.count_params(
self.conv_layers[c][l].trainable_weights)
print(
f'Layer {c}_{l} trainable params: {layer_param_count}')
trainable_param_count += layer_param_count
self.trainable_weights += self.conv_layers[c][
l].trainable_weights
else:
layer_param_count = layer_utils.count_params(
self.conv_layers[c][l].non_trainable_weights)
print(
f'Layer {c}_{l} non-trainable params: {layer_param_count}'
)
non_trainable_param_count += layer_param_count
self.non_trainable_weights += self.conv_layers[c][
l].non_trainable_weights
print('PredNet trainable params:', trainable_param_count)
print('PredNet non-trainable params:', non_trainable_param_count)
self.states = [None] * self.nb_layers * 3
if self.extrap_start_time is not None:
self.t_extrap = K.variable(
self.extrap_start_time,
int if K.backend() != 'tensorflow' else 'int32')
self.states += [None] * 2 # [previous frame prediction, timestep]
if self.stateful:
self.reset_states()
def wrap_train_test(gene):
global x_train, y_train, x_test, y_test
global x_train_shapes, x_test_shapes
runid = "N/A"
print(gene)
with open("tested.log", "a") as f:
f.write(genestr(gene))
f.write("\n")
print("\nWrapping...\n")
strategy = tf.distribute.MirroredStrategy()
print('Number of devices: {}'.format(strategy.num_replicas_in_sync))
# reshaping of the training data
if gene[-1][0] == 2: # reshaping is enabled
desired_size = gene[0][1]
if desired_size not in x_train_shapes:
x_train_shapes[desired_size] = resize(x_train, desired_size)
if desired_size not in x_test_shapes:
x_test_shapes[desired_size] = resize(x_test, desired_size)
x_train_current = x_train_shapes[desired_size]
x_test_current = x_test_shapes[desired_size]
elif gene[-1][0] == 1: # no reshaping
x_train_current = x_train
x_test_current = x_test
else:
print("#### INVALID GENE - last value is not 1 nor 2", gene[-1][0])
return runid, 0
# define model
try:
print("x_train shape: " + str(x_train_current.shape[1:]))
model, eval_model, manipulate_model = CapsNet(
gene=gene,
input_shape=x_train_current.shape[1:],
n_class=len(np.unique(np.argmax(y_train, 1))),
routings=args.routings)
except ValueError as e: # some bug in the chromosome ....
print("#### VALUE error desc ", e)
print("#### VALUE error gene ", gene)
tf.keras.backend.clear_session()
K.clear_session()
return runid, 0
except tf.errors.ResourceExhaustedError as e: # some bug in the chromosome ....
print("#### Out of resources error desc ", e)
print("#### Out of resources error gene ", gene)
tf.keras.backend.clear_session()
K.clear_session()
return runid, 0
model.summary()
trainable_count = count_params(model.trainable_weights)
if args.max_params > 0 and trainable_count > args.max_params:
print(
f"## ERR: number of trainable params {trainable_count} exceeded limit {args.max_params}"
)
tf.keras.backend.clear_session()
K.clear_session()
return runid, 0
# train or test
if args.weights is not None: # init the model weights with provided one
model.load_weights(args.weights)
if not args.testing:
# if gene[len(gene)-1][0]==2:
# x_train = resize(x_train, gene[0][1]) #64
# x_test = resize(x_test, gene[0][1])
# train(model=model, data=((x_train, y_train), (x_test, y_test)), args=args)
# elif gene[len(gene)-1][0]==1:
print("Train shapes:", x_train.shape, y_train.shape)
runid, _ = train(model=model,
data=((x_train_current, y_train), (x_test_current,
y_test)),
args=args)
else: # as long as weights are given, will run testing
if args.weights is None:
print(
'No weights are provided. Will test using random initialized weights.'
)
test_acc = test(model=eval_model, data=(x_test_current, y_test), args=args)
tf.keras.backend.clear_session()
K.clear_session()
return runid, test_acc
def train_iitnet_crossvalid():
"""
Train the iitnet using cross validation. Using only training and validation data
for parameter tuning. Best model will then be evaluated in a separate program.
"""
# log timestamps of relevant stages
start_processing = datetime.datetime.now()
timestamps = {'processstart': start_processing}
# print used devices
print("Using GPU:", K.tensorflow_backend._get_available_gpus())
# get parameters
params = Params()
# get additional parameters for iitnet
plx: dict = pp3.get_parameters()
params.plx.update(plx)
# adjust winslow parameters
if 'WINSLOW_PIPELINE_NAME' in os.environ:
winslow_params(params)
params.plx['subject_batch'] = 1 # !
# NOTE: mdl_architecture has to be set to 'iitnet_cnn_bilstm'
# set local parameters for the cross validation
k = params.plx.get('k_crossval')
train_total = params.plx.get('train_count') + params.plx.get('val_count')
count_per_fold = train_total // k
data_int = DataInt(save_path=params.plx["save_path"],
perform_save_raw=params.plx["save_raw_data"],
key_labels=params.plx["key_labels"],
uuid=params.plx["experiment_uuid"])
# Process data, if not already processed
if not params.plx.get("data_already_processed"):
# Process Data
process_data(params, data_int, params.plx["data_count"])
else:
# recover self.experiment.data_objects_list = List of the subject names
preprocessed_data_path = params.plx["save_path"] + params.plx[
"experiment_uuid"] # "D:/PhysioNet/processed/sa6pr7/"
pickle_object = params.plx["experiment_uuid"] + ".pckl"
subject_folders = [
name for name in os.listdir(preprocessed_data_path)
if not name == pickle_object
]
relevant_subjects = subject_folders[:train_total]
data_int.experiment.recover_data_objectlist(relevant_subjects)
print("Data already processed. Recover", str(len(relevant_subjects)),
"Subjects from", preprocessed_data_path)
num_epochs = params.plx.get('epochs')
apply_oversampling = params.plx.get(
'apply_oversampling') # !only on training data
timestamps['modelstart'] = datetime.datetime.now()
# build model
model, callbacks = choose_model(params, compile=False)
timestamps['modelend'] = datetime.datetime.now()
# save untrained model
if k > 1:
print("Save untrained model ... ", end=" ")
model.save(params.file_path_raw_mdl)
print("done")
timestamps_trainingstart = []
timestamps_trainingend = []
all_val_accs = []
all_val_loss = []
timestamps['crossval_start'] = datetime.datetime.now()
for i in range(k):
print("\n=============================================")
print("=======> Cross Validation - Fold #", i + 1, "<=======")
print("=============================================")
# get raw model
if k > 1:
print("Load untrained model ... ", end=" ")
model = load_model(params.file_path_raw_mdl)
print("done")
# compile model
model = compile_model_iitnet(params=params, model=model)
# set indices for the data to be loaded in this fold
if k == 1:
train_start = 0
train_end = int(train_total * 0.8)
val_start = train_end
train_count = train_end
val_count = train_total - train_count
else:
train_start = i * count_per_fold
train_end = train_start + (count_per_fold * (k - 1))
if train_end >= train_total:
train_end -= train_total
val_start = train_end
if val_start >= train_total:
val_start = 0
# configure the data generators for training and validation
train_count = train_total - count_per_fold
val_count = count_per_fold
train_generator = InterIntraEpochGenerator(
data_int,
params,
train_count,
start_val=train_start,
shuffle=True,
oversampling=apply_oversampling,
crossval_samples=train_total)
validation_generator = InterIntraEpochGenerator(
data_int,
params,
val_count,
start_val=val_start,
crossval_samples=train_total)
# model training
print("####\n\n\nTraining###\n\n")
timestamps_trainingstart.append(datetime.datetime.now())
history = model.fit_generator(generator=train_generator,
epochs=num_epochs,
callbacks=callbacks,
workers=0,
validation_data=validation_generator,
use_multiprocessing=False)
timestamps_trainingend.append(datetime.datetime.now())
print("Model Training done. Save Performance to Log ... ", end=" ")
# log the performance of this fold
val_acc_history = history.history[
'val_accuracy'] # val_accuracy for Winslow, val_acc local
val_loss_history = history.history['val_loss']
all_val_accs.append(val_acc_history)
all_val_loss.append(val_loss_history)
print("done.")
print("=======> Cross Validation - Performance Evaluation <=======")
timestamps['crossval_end'] = datetime.datetime.now()
timestamps['trainstarts'] = timestamps_trainingstart
timestamps['trainends'] = timestamps_trainingend
train_parameters = count_params(model.trainable_weights)
record_performance(all_val_accs, all_val_loss, params, timestamps,
train_parameters)
