def get_model_memory_usage(batch_size, model):
shapes_mem_count = 0
layers = 1
for l in model.layers:
print("Layer {}".format(layers))
layers+=1
single_layer_mem = 1
for s in l.output_shape:
if s is None:
continue
single_layer_mem *= s
print("Shape {} Memory {}".format(l.output_shape, single_layer_mem))
shapes_mem_count += single_layer_mem
print("Total {}".format(shapes_mem_count))
trainable_count = np.sum([K.count_params(p) for p in set(model.trainable_weights)])
print("Trainable weights {}".format(trainable_count))
non_trainable_count = np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])
print("Non Trainable weights {}".format(non_trainable_count))
total_memory = 4.0*batch_size*(shapes_mem_count + trainable_count + non_trainable_count)
gbytes = np.round(total_memory / (1024.0 ** 3), 3)
return gbytes
def model_memory_params(batch_size, model):
shapes_count = int(np.sum([np.prod(np.array([s if isinstance(s, int) else 1 for s in l.output_shape])) for l in model.layers]))
memory = shapes_count * 4 * batch_size
memory_gb = np.round(memory / (1024 ** 3), 3)
print ("Memory {} GB".format(memory_gb))
trainable_count = np.sum([K.count_params(p) for p in set(model.trainable_weights)])
print("Trainable weights {}".format(trainable_count))
non_trainable_count = np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])
print("Non Trainable weights {}".format(non_trainable_count))
print("Total Parameters {}".format(trainable_count + non_trainable_count))
def __init__(self, layers, weights=None, dense_activation='tanh', verbose=0):
self.model = self._build_model( layers, dense_activation)
if weights:
self.model.set_weights( weights )
self.trainable_params = int(np.sum([K.count_params(p) for p in set(self.model.trainable_weights)]))
if verbose:
self.model.summary()
def train(self,
x_df,
y_df,
epochs=100,
validation_split=0.3,
batch_size=10,
loss='mean_squared_error',
shuffle=False):
self.model.compile(loss=loss,
optimizer=self.optimizer)
self.trainable_count = int(np.sum(
[K.count_params(p) for p in set(self.model.trainable_weights)]))
start = time.time()
if self.verbose:
print('Start training (', start, ')')
history = self.model.fit(x_df,
y_df,
batch_size=batch_size,
verbose=self.verbose,
epochs=epochs,
validation_split=validation_split,
callbacks=[self.early_stopping,
self.checkpointer],
shuffle=shuffle)
train_time = time.time() - start
if self.verbose:
print('Finish trainning. Total time: ', train_time)
return {'trainable_vars': self.trainable_count,
'training_time': train_time}
def get_model_memory_usage(batch_size, model):
shapes_mem_count = 0
for l in model.layers:
single_layer_mem = 1
for s in l.output_shape:
if type(s) is tuple:
s = s[1]
if s is None:
continue
single_layer_mem *= s
shapes_mem_count += single_layer_mem
trainable_count = np.sum([K.count_params(p) for p in set(model.trainable_weights)])
non_trainable_count = np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])
total_memory = 4.0*batch_size*(shapes_mem_count + trainable_count + non_trainable_count)
gbytes = np.round(total_memory / (1024.0 ** 3), 3)
print('number of gbytes = ' + str(gbytes))
return gbytes
def __init__(self, rnn_arch=[2,16,32,64,1], drop_out=0.3,
model_file="lstm_model.hdf5", new=True, min_delta = 0.0001, patience = 50, dense_activation='tanh'):
"""Train a RNN using the input data
rnn_arch: list containing the number of neurons per layer (the number of hidden layers
is defined implicitly)
drop_out: drop out used in the RNN
model_file: name of the file where the model is saved
new: if True, a new model is created, otherwise an existent model is used
"""
if new:
self.model = self._build_lstm_model(rnn_arch, drop_out, dense_activation)
adam = Adam(lr = 5e-5)
self.model.compile(loss='mean_squared_error', optimizer=adam)
else:
self.model = load_model(model_file)
self.checkpointer = ModelCheckpoint(filepath=model_file, verbose=0, save_best_only=True)
self.early_stopping = EarlyStopping(monitor='val_loss', min_delta=min_delta, patience=patience, verbose=0, mode='auto')
self.trainable_count = int(np.sum([K.count_params(p) for p in set(self.model.trainable_weights)]))
def build_model(self,
layers,
cell=LSTM,
weights=None,
dense_activation='tanh',
verbose=0,
**kwargs):
# self.hidden_layers=len(layers) - 2
# self.layers=layers
# self.input_dim=layers[0]
# self.output_dim=layers[-1]
self.model = Sequential()
for i in range(len(layers) - 2):
self.model.add(cell(
# Keras API 2
input_shape=(None, layers[i]),
units=layers[i+1],
# Keras API 1
# input_dim=layers[i],
# output_dim=layers[i+1],
kernel_initializer='zeros',
recurrent_initializer='zeros',
bias_initializer='zeros',
# Uncomment to use last batch state to init next training step.
# Specify shuffle=False when calling fit()
# batch_size=batch_size, stateful=True,
return_sequences=True if i < (len(layers) - 3) else False))
self.model.add(Dense(layers[-1],
activation=dense_activation,
kernel_initializer='zeros',
bias_initializer='zeros'))
if weights:
self.model.set_weights(weights)
self.trainable_params = int(np.sum(
[K.count_params(p) for p in set(self.model.trainable_weights)]))
if verbose > 1:
self.model.summary()
return self.model
current_epoch = epoch
print('current epoch ' + str(current_epoch))
def on_epoch_end(self, epoch, logs=None):
open('epoch/' + job_name + '.txt', 'a').close()
my_callback = PrintEpoch()
callbacks = [tensorboard_callback, my_callback]
#[checkpoint, lr_reducer, lr_scheduler, tensorboard_callback]
# Run training
if not args.resume:
trainable_count = int(
np.sum([K.count_params(p) for p in set(model.trainable_weights)]))
param_dict = {}
modify = False
with open('param_lock.json', 'r') as fp:
param_dict = json.load(fp)
if job_name not in param_dict:
param_dict[job_name] = trainable_count
modify = True
elif param_dict[job_name] != trainable_count:
param_dict[job_name] = trainable_count
modify = True
if modify:
json_file = json.dumps(param_dict)
with open('param_lock.json', 'w') as fp:
fp.write(json_file)
os.rename('param_lock.json', 'param.json')
def count_params(weights):
return int(np.sum([K.count_params(p) for p in set(weights)]))
def fine_tune(MODEL,
preprocess,
height,
freeze_till,
lr,
batch,
nb_epoch,
weights=None):
x = Input(shape=(height, height, 3))
x = Lambda(preprocess)(x)
base_model = MODEL(include_top=False,
input_tensor=x,
weights='imagenet',
pooling='avg')
for layer in base_model.layers:
layer.trainable = True
for layer in base_model.layers[:freeze_till]:
layer.trainable = False
y = Dropout(0.2)(base_model.output)
y = Dense(1, activation='sigmoid', kernel_initializer='he_normal')(y)
model = Model(inputs=base_model.input, outputs=y, name='Transfer_Learning')
sgd = SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='binary_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
# print('Trainable: %d, Non-Trainable: %d' % get_params_count(model))
trainable_count = int(
np.sum([K.count_params(p) for p in set(model.trainable_weights)]))
non_trainable_count = int(
np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]))
print('Trainable: %d, Non-Trainable: %d' %
(trainable_count, non_trainable_count))
model.summary()
if weights is not None:
model.load_weights(weights)
# Prepare Callbacks for Model Checkpoint, Early Stopping and Tensorboard.
log_name = '/Differentiation-EP{epoch:02d}-LOSS{val_loss:.4f}.h5'
log_dir = datetime.now().strftime('transfer_model_%Y%m%d_%H%M')
if not os.path.exists(log_dir):
os.mkdir(log_dir)
rc = RocAucMetricCallback(predict_batch_size=32, include_on_batch=True)
es = EarlyStopping(monitor='val_loss', patience=20, verbose=0)
mc = ModelCheckpoint(log_dir + log_name,
monitor='val_loss',
save_best_only=True,
verbose=0)
tb = TensorBoard(log_dir=log_dir)
history = model.fit(x=X_train,
y=y_train,
batch_size=batch,
epochs=nb_epoch,
verbose=2,
validation_data=(X_val, y_val),
callbacks=[es, mc, tb])
with open('log_txt', 'w') as f:
f.write(str(history.history))
# plt_score(history)
plt_aucScore(history)
trainscore = model.evaluate(x=train, y=trainlabel)
testscore = model.evaluate(x=test, y=testlabel)
# train_pre = model.predict(x=train)
# test_pre = model.predict(x=test)
# pd.DataFrame(test_pre).to_excel(output_path)
print(trainscore, testscore)
def train_models(config):
# obtain x_dim
x_dim = config['x_dim']
if not os.path.exists(os.path.join(config['exp_save_dir'], "models",
"vis")):
os.makedirs(os.path.join(config['exp_save_dir'], "models", 'vis'))
logger = set_logger(
os.path.join(
config['exp_save_dir'], "models", 'train_model_architecture_' +
dt.now().strftime("%Y-%m-%dT%H-%M-%SZ") + '.log'))
print_and_log(logger, "x_dim: {}".format(x_dim))
# LTSM model architecture
input_sequences = Input(shape=(config['window_size'], x_dim),
name='input_sequences')
lstm_1 = LSTM(250, return_sequences=True, name="lstm_1")(input_sequences)
lstm_2 = LSTM(150, name="lstm_2")(lstm_1)
# branching event type
et_1 = Dense(128, activation='relu', name="et_1")(lstm_2)
et_2 = Dropout(0.5, name="et_2")(et_1)
et_3 = Dense(64, activation='relu', name="et_3")(et_2)
et_4 = Dropout(0.5, name="et_4")(et_3)
event_type_output = Dense(len(config['eventtype_2_id']),
activation='softmax',
name="event_type_output")(et_4)
# branching time delay
td_1 = Dense(128, activation='relu', name="td_1")(lstm_2)
td_2 = Dropout(0.5, name="td_2")(td_1)
td_3 = Dense(64, activation='relu', name="td_3")(td_2)
td_4 = Dropout(0.5, name="td_4")(td_3)
time_delay_output = Dense(1, activation='linear',
name="time_delay_output")(td_4)
# branching user cluster
uc_1 = Dense(128, activation='relu', name="uc_1")(lstm_2)
uc_2 = Dropout(0.5, name="uc_2")(uc_1)
uc_3 = Dense(64, activation='relu', name="uc_3")(uc_2)
uc_4 = Dropout(0.5, name="uc_4")(uc_3)
user_cluster_output = Dense(config['dim_user_cluster_one_hot'],
activation='softmax',
name="user_cluster_output")(uc_4)
# model
model = Model(
inputs=input_sequences,
outputs=[event_type_output, time_delay_output, user_cluster_output])
model.summary(print_fn=logger.info)
print(model.summary())
model = multi_gpu_model(model, gpus=config['num_gpu'])
# get partition and labels
with open(
os.path.join(config['exp_save_dir'], "dataset", 'partition.json'),
'r') as f:
partition = json.load(f)
with open(os.path.join(config['exp_save_dir'], "dataset", 'labels.json'),
'r') as f:
labels = json.load(f)
# train val generator
training_generator = DataGenerator_ET_TD_UC_one_hot(
os.path.join(config['exp_save_dir'], "dataset"),
partition['train'],
labels,
batch_size=config['batch_size'],
dim=x_dim,
window_size=config['window_size'],
et_classes=len(config['eventtype_2_id']),
uc_classes=config['dim_user_cluster_one_hot'],
shuffle=config['generator_shuffle'])
validation_generator = DataGenerator_ET_TD_UC_one_hot(
os.path.join(config['exp_save_dir'], "dataset"),
partition['validation'],
labels,
batch_size=config['batch_size'],
dim=x_dim,
window_size=config['window_size'],
et_classes=len(config['eventtype_2_id']),
uc_classes=config['dim_user_cluster_one_hot'],
shuffle=config['generator_shuffle'])
# callback
# if not os.path.exists(os.path.join(
# config['exp_save_dir'], "models", "vis")):
# os.makedirs(os.path.join(
# config['exp_save_dir'], "models", 'vis'))
# TensorBoard_callback = TensorBoard(
# log_dir=os.path.join(config['exp_save_dir'], "models", 'vis'),
# histogram_freq=0,
# write_graph=True,
# write_images=True,
# write_grads=False)
"""
Keras TensorBoard Reference:
https://keras.io/callbacks/#tensorboard
launch: tensorboard --logdir=/full_path_to_your_logs
"""
callbacks = [ # EarlyStopping(monitor='val_loss', patience=50),
ModelCheckpoint(filepath=os.path.join(config['exp_save_dir'], "models",
'model.hdf5'),
monitor='val_loss',
verbose=2)
]
# save train confg in case testing need it
with open(
os.path.join(config['exp_save_dir'], "models",
'train_config.pickle'), 'wb') as f:
pickle.dump(config, f)
print("{} saved!".format('train_config.pickle'))
# model_history
model_history = dict()
# start training
for epoch in range(config['num_epochs']):
print("\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!! "
"Epoch {}/{} {}".format(epoch + 1, config['num_epochs'],
"---> train all branches..."))
trainable_count = int(
np.sum([K.count_params(p) for p in set(model.trainable_weights)]))
non_trainable_count = int(
np.sum(
[K.count_params(p) for p in set(model.non_trainable_weights)]))
print('Total params: {:,}'.format(trainable_count +
non_trainable_count))
print('Trainable params: {:,}'.format(trainable_count))
print('Non-trainable params: {:,}'.format(non_trainable_count))
if (config['use_user_cluster_one_hot']) and (
not config['use_user_cluster_minmax']):
if config['et_loss'] == 'focalloss':
model.compile(optimizer=Adam(lr=config['lr'],
decay=config['decay'],
amsgrad=config['amsgrad']),
loss=[
categorical_focal_loss(
gamma=config['focalloss_gamma'],
alpha=config['focalloss_alpha']),
logcosh, binary_crossentropy
],
loss_weights=config['loss_weights'],
metrics={
'event_type_output':
(metrics.categorical_accuracy),
'time_delay_output':
(losses.mean_squared_error),
'user_cluster_output':
(metrics.categorical_accuracy)
})
elif config['et_loss'] == 'bce':
model.compile(
optimizer='adam',
loss=[binary_crossentropy, logcosh, binary_crossentropy],
loss_weights=config['loss_weights'],
metrics={
'event_type_output': (metrics.categorical_accuracy),
'time_delay_output': (losses.mean_squared_error),
'user_cluster_output': (metrics.categorical_accuracy)
})
else:
print("event type loss undefined!")
pdb.set_trace()
else:
model.compile(
optimizer='adam',
loss=[
categorical_focal_loss(gamma=config['focalloss_gamma'],
alpha=config['focalloss_alpha']),
logcosh, logcosh
],
loss_weights=config['loss_weights'],
metrics={
'event_type_output': (metrics.categorical_accuracy),
'time_delay_output': (losses.mean_squared_error),
'user_cluster_output': (losses.mean_squared_error)
})
history = model.fit_generator(generator=training_generator,
epochs=1,
callbacks=callbacks,
validation_data=validation_generator,
use_multiprocessing=True,
workers=config['multiprocessing_cpu'],
shuffle=True)
""" Whether to shuffle the order of the batches
at the beginning of each epoch.
Only used with instances of Sequence (keras.utils.Sequence).
Has no effect when steps_per_epoch is not None.
Basically, no effect here.
https://stackoverflow.com/questions/49027174/
what-does-shuffle-do-in-fit-generator-in-keras
"""
model.save(
os.path.join(config['exp_save_dir'], "models",
'model-{}.hdf5'.format(epoch + 1)))
print("model-{}.hdf5 saved!".format(epoch + 1))
if len(model_history) == 0:
model_history = history.history.copy()
else:
for key in history.history:
model_history[key] += history.history[key]
with open(
os.path.join(config['exp_save_dir'], "models",
'history-{}.pickle'.format(epoch + 1)),
'wb') as f:
pickle.dump(model_history, f)
print("history-{}.pickle saved!".format(epoch + 1))
with open(os.path.join(config['exp_save_dir'], "models", 'history.pickle'),
'wb') as f:
pickle.dump(model_history, f)
print("history.pickle saved!")
return
activation='relu',
W_regularizer=l2(L2) if L2 else None)(joint)
joint = Dropout(DP)(joint)
joint = BatchNormalization()(joint)
prediction = Dense(CLASS_COUNT, activation='softmax')(joint)
model = Model(inputs=[premise, hypothesis], outputs=prediction)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
logging.info(
'Number of trainable parameters: ' +
str(int(np.sum([K.count_params(p)
for p in set(model.trainable_weights)]))))
class LossHistory(keras.callbacks.Callback):
def on_epoch_end(self, batch, logs={}):
print(str(logs))
loss = logs.get('loss')
val_acc = logs.get('val_acc')
acc = logs.get('acc')
logging.info("Epoch: " + str(batch) + "/" + str(EPOCHS) + " ACC: " +
str(acc) + " LOSS: " + str(loss) + " VAL_ACC: " +
str(val_acc))
logging.info('Training')
def print_parameters(self):
from keras.engine.training import collect_trainable_weights
logger.info("total number of parameters: %s" % self.graph.count_params())
trainable_weights = collect_trainable_weights(self.graph)
total = sum([K.count_params(p) for p in trainable_weights])
logger.info("number of trainable parameters: %s" % total)
def main():
start = time.time()
ap = argparse.ArgumentParser()
ap.add_argument("-e",
"--epochs",
required=True,
type=int,
help="Number of epochs",
default=25)
ap.add_argument("-m",
"--model_name",
required=True,
type=str,
help="Imagenet model to train",
default="xception")
ap.add_argument("-b",
"--batch_size",
required=True,
type=int,
help="Batch size",
default=32)
args = ap.parse_args()
# Training dataset loading
train_data = np.load("train_data.npy")
train_label = np.load("train_label.npy")
print("Dataset Loaded...")
# Train and validation split
trainX, valX, trainY, valY = train_test_split(train_data,
train_label,
test_size=0.1,
shuffle=False)
print(trainX.shape, valX.shape, trainY.shape, valY.shape)
# Train nad validation image data generator
trainAug = ImageDataGenerator(
rescale=1.0 / 255.0,
rotation_range=30,
zoom_range=0.15,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.15,
horizontal_flip=True,
fill_mode="nearest",
)
valAug = ImageDataGenerator(rescale=1.0 / 255.0)
model = cnn_model(args.model_name)
# Number of trainable and non-trainable parameters
trainable_count = int(
np.sum([K.count_params(p) for p in set(model.trainable_weights)]))
non_trainable_count = int(
np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]))
print("Total params: {:,}".format(trainable_count + non_trainable_count))
print("Trainable params: {:,}".format(trainable_count))
print("Non-trainable params: {:,}".format(non_trainable_count))
if not exists("./trained_wts"):
makedirs("./trained_wts")
if not exists("./training_logs"):
makedirs("./training_logs")
if not exists("./plots"):
makedirs("./plots")
# Keras backend
model_checkpoint = ModelCheckpoint(
"trained_wts/xception_best.hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
save_weights_only=True,
)
stopping = EarlyStopping(monitor="val_loss", patience=10, verbose=0)
csv_logger = CSVLogger(
"training_logs/xception.log",
separator=",",
append=True,
)
print("Training is going to start in 3... 2... 1... ")
# Model Training
H = model.fit_generator(
trainAug.flow(trainX, trainY, batch_size=args.batch_size),
steps_per_epoch=len(trainX) // args.batch_size,
validation_data=valAug.flow(valX, valY),
validation_steps=len(valX) // args.batch_size,
epochs=args.epochs,
callbacks=[model_checkpoint, stopping, csv_logger],
)
# plot the training loss and accuracy
plt.style.use("ggplot")
plt.figure()
N = args.epochs
plt.plot(np.arange(0, N), H.history["loss"], label="train_loss")
plt.plot(np.arange(0, N), H.history["val_loss"], label="val_loss")
plt.plot(np.arange(0, N), H.history["accuracy"], label="train_acc")
plt.plot(np.arange(0, N), H.history["val_accuracy"], label="val_acc")
plt.title("Training Loss and Accuracy on Santa/Not Santa")
plt.xlabel("Epoch #")
plt.ylabel("Loss/Accuracy")
plt.legend(loc="lower left")
plt.savefig("plots/training_plot.png")
end = time.time()
dur = end - start
if dur < 60:
print("Execution Time:", dur, "seconds")
elif dur > 60 and dur < 3600:
dur = dur / 60
print("Execution Time:", dur, "minutes")
else:
dur = dur / (60 * 60)
print("Execution Time:", dur, "hours")
print("Regularization:", args.reg_value)
print("Trainable embeddings:", args.trainable_embeddings)
print()
model = mpcnn_model.he_model_siamese
siamese_model, encoder = model(
(max_seq_length, ), filters_ga, filters_gb, args.conv2d_type,
args.conv_act, args.dense_act, embedding_dims, embedding_matrices,
word_to_index, max_seq_length, args.reg_value, args.hidden_units,
args.trainable_embeddings, args.use_groupa, args.use_groupb,
args.use_algo1, args.use_algo2, poolings_ga, poolings_gb,
args.use_cos_a1, args.use_cos_a2, args.use_euc_a1, args.use_euc_a2,
args.use_abs_a1, args.use_abs_a2)
trainable_count = int(
np.sum(
[K.count_params(p) for p in set(siamese_model.trainable_weights)]))
non_trainable_count = int(
np.sum([
K.count_params(p) for p in set(siamese_model.non_trainable_weights)
]))
print("Trainable parameters: {:,}".format(trainable_count))
print("Non-trainable parameters: {:,}".format(non_trainable_count))
print("Total parameters: {:,}".format(trainable_count +
non_trainable_count))
print("Done!")
print("\n")
# Printing summaries
if args.print_encoder:
encoder.summary()
if args.print_model:
def get_dimensions(self):
# calculated the number of trianable weights
return int(
np.sum([
K.count_params(p) for p in set(self.model.trainable_weights)
]))
def Emonet(num_classes):
# use sequential model to build a VGG like network
emonet = Sequential()
"""
Convolution and Maxpool layers: Block 1
"""
# Conv Layer 1:48x48x32
emonet.add(
Conv2D(filters=32,
kernel_size=3,
padding='same',
activation='linear',
input_shape=(48, 48, 1)))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 2:48x48x32
emonet.add(
Conv2D(filters=32, kernel_size=3, padding='same', activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# MaxPool layer: 1
emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
emonet.add(Dropout(0.3))
"""
Convolution and Maxpool layers: Block 2
"""
# Conv Layer 3:24x24x64
emonet.add(
Conv2D(filters=64, kernel_size=3, padding='same', activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 4:24x24x64
emonet.add(
Conv2D(filters=64, kernel_size=3, padding='same', activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# MaxPool layer: 2
emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
emonet.add(Dropout(0.3))
"""
Convolution and Maxpool layers: Block 3
"""
# Conv Layer 5:12x12x128
emonet.add(
Conv2D(filters=128, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 6:12x12x128
emonet.add(
Conv2D(filters=128, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# MaxPool layer: 3
emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
emonet.add(Dropout(0.25))
"""
Convolution and Maxpool layers: Block 4
"""
# Conv Layer 7:6x6x256
emonet.add(
Conv2D(filters=256, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 8:6x6x256
emonet.add(
Conv2D(filters=256, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# MaxPool layer: 4
emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
emonet.add(Dropout(0.25))
# Flatten
emonet.add(Flatten())
# Dense layer 1:
emonet.add(Dense(256, activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
emonet.add(Dropout(0.5))
# Dense layer 2:
emonet.add(Dense(256, activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
emonet.add(Dropout(0.5))
# Output layer
emonet.add(Dense(num_classes, activation='softmax'))
trainable_count = int(
np.sum([K.count_params(p) for p in set(emonet.trainable_weights)]))
non_trainable_count = int(
np.sum([K.count_params(p) for p in set(emonet.non_trainable_weights)]))
# network summary
print('\n\n---<summary>---')
print(
'\n Layers: \n\tConvolution2D: {0}\n\tMaxPooling2D: {1}\n\tFully Connected Layers: {2}'
.format(8, 4, 2))
print('\n Total params: {:,}'.format(trainable_count +
non_trainable_count))
print('\n Trainable params: {:,}'.format(trainable_count))
print('\n Non-trainable params: {:,}'.format(non_trainable_count))
print('\n\n---</summary>---')
return emonet
def Emonet_extend(num_classes):
"""
This model is optional and bigger than the previous one. Practically, this model is less useful than the first one
therefore is not called by the application. Use only for experimental purposes
"""
emonet = Sequential()
"""
Convolution and Maxpool layers: Block 1
"""
# Conv Layer 1:48x48x32
emonet.add(
Conv2D(filters=64,
kernel_size=3,
padding='same',
activation='linear',
input_shape=(48, 48, 1)))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 2:48x48x32
emonet.add(
Conv2D(filters=64, kernel_size=3, padding='same', activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# MaxPool layer: 1
emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
emonet.add(Dropout(0.2))
"""
Convolution and Maxpool layers: Block 2
"""
# Conv Layer 3:24x24x64
emonet.add(
Conv2D(filters=128, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 4:24x24x64
emonet.add(
Conv2D(filters=128, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 5:24x24x64
emonet.add(
Conv2D(filters=128, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# MaxPool layer: 2
emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
emonet.add(Dropout(0.2))
"""
Convolution and Maxpool layers: Block 3
"""
# Conv Layer 6:12x12x256
emonet.add(
Conv2D(filters=256, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 7:12x12x256
emonet.add(
Conv2D(filters=256, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 8:12x12x256
emonet.add(
Conv2D(filters=256, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 9:12x12x256
emonet.add(
Conv2D(filters=256, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# MaxPool layer: 3
emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
emonet.add(Dropout(0.3))
"""
Convolution and Maxpool layers: Block 4
"""
# Conv Layer 9:6x6x256
emonet.add(
Conv2D(filters=512, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 10:6x6x256
emonet.add(
Conv2D(filters=512, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 11:6x6x256
emonet.add(
Conv2D(filters=512, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# Conv Layer 12:6x6x256
emonet.add(
Conv2D(filters=512, kernel_size=3, padding='same',
activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
# MaxPool layer: 4
emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
emonet.add(Dropout(0.2))
# Flatten
emonet.add(Flatten())
# Dense layer 1:
emonet.add(Dense(2048, activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
emonet.add(Dropout(0.5))
# Dense layer 2:
emonet.add(Dense(2048, activation='linear'))
emonet.add(LeakyReLU(alpha=0.3))
emonet.add(BatchNormalization(axis=-1))
emonet.add(Dropout(0.5))
# Output layer
emonet.add(Dense(num_classes, activation='softmax'))
trainable_count = int(
np.sum([K.count_params(p) for p in set(emonet.trainable_weights)]))
non_trainable_count = int(
np.sum([K.count_params(p) for p in set(emonet.non_trainable_weights)]))
# network summary
print('\n\n---<summary>---')
print(
'\n Layers: \n\tConvolution2D: {0}\n\tMaxPooling2D: {1}\n\tFully Connected Layers: {2}'
.format(8, 4, 2))
print('\n Total params: {:,}'.format(trainable_count +
non_trainable_count))
print('\n Trainable params: {:,}'.format(trainable_count))
print('\n Non-trainable params: {:,}'.format(non_trainable_count))
print('\n\n---</summary>---')
return emonet
def _document_training(history, model, start_training, end_training, save_dir,
save_models, training_size, validation_size, args):
"""
Write down the training results.
Parameters
----------
history: dic
Training history.
model:
Trained model.
start_training: datetime
Training start time.
end_training: datetime
Training end time.
save_dir: str
Path to the output directory.
save_models: str
Path to the folder for saveing the models.
training_size: int
Number of training samples.
validation_size: int
Number of validation samples.
args: dic
A dictionary containing all of the input parameters.
Returns
--------
./output_name/history.npy: Training history.
./output_name/X_report.txt: A summary of parameters used for the prediction and perfomance.
./output_name/X_learning_curve_f1.png: The learning curve of Fi-scores.
./output_name/X_learning_curve_loss.png: The learning curve of loss.
"""
np.save(save_dir + '/history', history)
model.save(save_dir + '/final_model.h5')
model.to_json()
model.save_weights(save_dir + '/model_weights.h5')
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(history.history['loss'])
ax.plot(history.history['detector_loss'])
ax.plot(history.history['picker_P_loss'])
ax.plot(history.history['picker_S_loss'])
try:
ax.plot(history.history['val_loss'], '--')
ax.plot(history.history['val_detector_loss'], '--')
ax.plot(history.history['val_picker_P_loss'], '--')
ax.plot(history.history['val_picker_S_loss'], '--')
ax.legend([
'loss', 'detector_loss', 'picker_P_loss', 'picker_S_loss',
'val_loss', 'val_detector_loss', 'val_picker_P_loss',
'val_picker_S_loss'
],
loc='upper right')
except Exception:
ax.legend(['loss', 'detector_loss', 'picker_P_loss', 'picker_S_loss'],
loc='upper right')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.grid(b=True, which='major', color='#666666', linestyle='-')
fig.savefig(os.path.join(save_dir, str('X_learning_curve_loss.png')))
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(history.history['detector_f1'])
ax.plot(history.history['picker_P_f1'])
ax.plot(history.history['picker_S_f1'])
try:
ax.plot(history.history['val_detector_f1'], '--')
ax.plot(history.history['val_picker_P_f1'], '--')
ax.plot(history.history['val_picker_S_f1'], '--')
ax.legend([
'detector_f1', 'picker_P_f1', 'picker_S_f1', 'val_detector_f1',
'val_picker_P_f1', 'val_picker_S_f1'
],
loc='lower right')
except Exception:
ax.legend(['detector_f1', 'picker_P_f1', 'picker_S_f1'],
loc='lower right')
plt.ylabel('F1')
plt.xlabel('Epoch')
plt.grid(b=True, which='major', color='#666666', linestyle='-')
fig.savefig(os.path.join(save_dir, str('X_learning_curve_f1.png')))
delta = end_training - start_training
hour = int(delta / 3600)
delta -= hour * 3600
minute = int(delta / 60)
delta -= minute * 60
seconds = delta
trainable_count = int(
np.sum([K.count_params(p) for p in model.trainable_weights]))
non_trainable_count = int(
np.sum([K.count_params(p) for p in model.non_trainable_weights]))
with open(os.path.join(save_dir, 'X_report.txt'), 'a') as the_file:
the_file.write(
'================== Overal Info ==============================' +
'\n')
the_file.write('date of report: ' + str(datetime.datetime.now()) +
'\n')
the_file.write('input_hdf5: ' + str(args['input_hdf5']) + '\n')
the_file.write('input_csv: ' + str(args['input_csv']) + '\n')
the_file.write('output_name: ' +
str(args['output_name'] + '_outputs') + '\n')
the_file.write(
'================== Model Parameters =========================' +
'\n')
the_file.write('input_dimention: ' + str(args['input_dimention']) +
'\n')
the_file.write('cnn_blocks: ' + str(args['cnn_blocks']) + '\n')
the_file.write('lstm_blocks: ' + str(args['lstm_blocks']) + '\n')
the_file.write('padding_type: ' + str(args['padding']) + '\n')
the_file.write('activation_type: ' + str(args['activation']) + '\n')
the_file.write('drop_rate: ' + str(args['drop_rate']) + '\n')
the_file.write(
str('total params: {:,}'.format(trainable_count +
non_trainable_count)) + '\n')
the_file.write(
str('trainable params: {:,}'.format(trainable_count)) + '\n')
the_file.write(
str('non-trainable params: {:,}'.format(non_trainable_count)) +
'\n')
the_file.write(
'================== Training Parameters ======================' +
'\n')
the_file.write('mode of training: ' + str(args['mode']) + '\n')
the_file.write('loss_types: ' + str(args['loss_types']) + '\n')
the_file.write('loss_weights: ' + str(args['loss_weights']) + '\n')
the_file.write('batch_size: ' + str(args['batch_size']) + '\n')
the_file.write('epochs: ' + str(args['epochs']) + '\n')
the_file.write('train_valid_test_split: ' +
str(args['train_valid_test_split']) + '\n')
the_file.write('total number of training: ' + str(training_size) +
'\n')
the_file.write('total number of validation: ' + str(validation_size) +
'\n')
the_file.write('monitor: ' + str(args['monitor']) + '\n')
the_file.write('patience: ' + str(args['patience']) + '\n')
the_file.write('multi_gpu: ' + str(args['multi_gpu']) + '\n')
the_file.write('number_of_gpus: ' + str(args['number_of_gpus']) + '\n')
the_file.write('gpuid: ' + str(args['gpuid']) + '\n')
the_file.write('gpu_limit: ' + str(args['gpu_limit']) + '\n')
the_file.write('use_multiprocessing: ' +
str(args['use_multiprocessing']) + '\n')
the_file.write(
'================== Training Performance =====================' +
'\n')
the_file.write(
'finished the training in: {} hours and {} minutes and {} seconds \n'
.format(hour, minute, round(seconds, 2)))
the_file.write('stoped after epoche: ' +
str(len(history.history['loss'])) + '\n')
the_file.write('last loss: ' + str(history.history['loss'][-1]) + '\n')
the_file.write('last detector_loss: ' +
str(history.history['detector_loss'][-1]) + '\n')
the_file.write('last picker_P_loss: ' +
str(history.history['picker_P_loss'][-1]) + '\n')
the_file.write('last picker_S_loss: ' +
str(history.history['picker_S_loss'][-1]) + '\n')
the_file.write('last detector_f1: ' +
str(history.history['detector_f1'][-1]) + '\n')
the_file.write('last picker_P_f1: ' +
str(history.history['picker_P_f1'][-1]) + '\n')
the_file.write('last picker_S_f1: ' +
str(history.history['picker_S_f1'][-1]) + '\n')
the_file.write(
'================== Other Parameters =========================' +
'\n')
the_file.write('label_type: ' + str(args['label_type']) + '\n')
the_file.write('augmentation: ' + str(args['augmentation']) + '\n')
the_file.write('shuffle: ' + str(args['shuffle']) + '\n')
the_file.write('normalization_mode: ' +
str(args['normalization_mode']) + '\n')
the_file.write('add_event_r: ' + str(args['add_event_r']) + '\n')
the_file.write('add_noise_r: ' + str(args['add_noise_r']) + '\n')
the_file.write('shift_event_r: ' + str(args['shift_event_r']) + '\n')
the_file.write('drop_channel_r: ' + str(args['drop_channel_r']) + '\n')
the_file.write('scale_amplitude_r: ' + str(args['scale_amplitude_r']) +
'\n')
the_file.write('pre_emphasis: ' + str(args['pre_emphasis']) + '\n')
def count_params(model):
trainable_count = np.sum([K.count_params(w) for w in model.trainable_weights])
non_trainable_count = np.sum([K.count_params(w) for w in model.non_trainable_weights])
return trainable_count, non_trainable_count
predictions = Dense(5, activation = "softmax", kernel_initializer = Orthogonal())(x)
model = Model(inputs = base.input, outputs = predictions)
#########
for layer in model.layers:
layer.trainable = True
#########
model.compile(optimizer = "SGD", loss = "categorical_crossentropy", metrics = ["accuracy"])
#########
trainable_params = int(np.sum([K.count_params(p) for p in set(model.trainable_weights)]))
non_trainable_params = int(np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]))
print("\nModel Stats")
print("=" * 30)
print("Total Parameters: {:,}".format((trainable_params + non_trainable_params)))
print("Non-Trainable Parameters: {:,}".format(non_trainable_params))
print("Trainable Parameters: {:,}\n".format(trainable_params))
#########
### Create training data set by oversampling all classes to the size of the majority class ###
### Classes in alphabetical order; CYT = 0, FIB = 1, HOF = 2, SYN = 3, VEN = 4 ###
def get_number_of_parameters(self):
trainable_count = int(
np.sum([
K.count_params(p) for p in set(self.model.trainable_weights)
]))
return trainable_count
def get_trainable_count(net):
"""https://stackoverflow.com/questions/45046525"""
return sum([K.count_params(p) for p in set(net.trainable_weights)])
def main():
start = time.time()
ap = argparse.ArgumentParser()
ap.add_argument("-e",
"--epochs",
required=True,
type=int,
help="Number of epochs",
default=25)
ap.add_argument("-w",
"--weights_save_name",
required=True,
type=str,
help="Model weights name")
ap.add_argument("-b",
"--batch_size",
required=True,
type=int,
help="Batch size",
default=32)
args = ap.parse_args()
# Training dataset loading
train_data = np.load("lstm_40f_data.npy")
train_label = np.load("lstm_40f_labels.npy")
train_label = utils.to_categorical(train_label)
print("Dataset Loaded...")
# Train validation split
trainX, valX, trainY, valY = train_test_split(train_data,
train_label,
shuffle=True,
test_size=0.1)
model = lstm_model(train_data)
trainable_count = int(
np.sum([K.count_params(p) for p in set(model.trainable_weights)]))
non_trainable_count = int(
np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]))
# Number of trainable and non-trainable parameters
print("Total params: {:,}".format(trainable_count + non_trainable_count))
print("Trainable params: {:,}".format(trainable_count))
print("Non-trainable params: {:,}".format(non_trainable_count))
# Keras backend
model_checkpoint = ModelCheckpoint(
"trained_wts/" + args.weights_save_name + ".hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
save_weights_only=True,
)
stopping = EarlyStopping(monitor="val_loss", patience=10, verbose=0)
print("Training is going to start in 3... 2... 1... ")
# Model training
H = model.fit(
trainX,
trainY,
validation_data=(valX, valY),
batch_size=args.batch_size,
epochs=args.epochs,
shuffle=True,
callbacks=[model_checkpoint, stopping],
)
# plot the training loss and accuracy
plt.style.use("ggplot")
plt.figure()
N = stopping.stopped_epoch + 1
plt.plot(np.arange(0, N), H.history["loss"], label="train_loss")
plt.plot(np.arange(0, N), H.history["val_loss"], label="val_loss")
plt.plot(np.arange(0, N), H.history["accuracy"], label="train_acc")
plt.plot(np.arange(0, N), H.history["val_accuracy"], label="val_acc")
plt.title("Training Loss and Accuracy")
plt.xlabel("Epoch #")
plt.ylabel("Loss/Accuracy")
plt.legend(loc="lower left")
plt.savefig("plots/training_plot.png")
end = time.time()
dur = end - start
if dur < 60:
print("Execution Time:", dur, "seconds")
elif dur > 60 and dur < 3600:
dur = dur / 60
print("Execution Time:", dur, "minutes")
else:
dur = dur / (60 * 60)
print("Execution Time:", dur, "hours")
def count_variables(variables):
return int(np.sum([K.count_params(p) for p in set(variables)]))
def Train_frcnn(
train_path, # path to the text file containing the data
network_arch, # the full faster rcnn network architecture object
num_epochs, # num of epochs
output_weight_path, # path to save the model_all.weights as hdf5
preprocessing_function=None,
config_filename="config.pickle",
input_weights_path=None,
train_rpn=True,
train_final_classifier=True,
train_base_nn=True,
losses_to_watch=['rpn_cls', 'rpn_reg', 'final_cls', 'final_reg'],
tb_log_dir="log",
num_rois=32,
horizontal_flips=False,
vertical_flips=False,
rot_90=False,
anchor_box_scales=[128, 256, 512],
anchor_box_ratios=[[1, 1], [1. / math.sqrt(2), 2. / math.sqrt(2)],
[2. / math.sqrt(2), 1. / math.sqrt(2)]],
im_size=600,
rpn_stride=16, # depends on network architecture
visualize_model=None,
verify_trainable=True,
optimizer_rpn=Adam(lr=1e-5),
optimizer_classifier=Adam(lr=1e-5),
validation_interval=3,
rpn_min_overlap=0.3,
rpn_max_overlap=0.7,
classifier_min_overlap=0.1,
classifier_max_overlap=0.5,
rpn_nms_threshold=0.7, # original implementation
seed=5000):
"""
Trains a Faster RCNN for object detection in keras
NOTE: This trains 2 models namely model_rpn and model_classifer with the same shared base_nn (fixed feature extractor)
Keyword Arguments
train_path -- str: path to the text file or pascal_voc (no Default)
network_arch --object: the full faster rcnn network .py file passed as an object (no default)
num_epochs -- int: number of epochs to train (no Default)
output_weight_path --str: path to save the frcnn weights (no Default)
preprocessing_function --function: Optional preprocessing function (must be defined like given in keras docs) (Default None)
config_filename --str: Path to save the config file. Used when testing (Default "config.pickle")
input_weight_path --str: Path to hdf5 file containing weights for the model (Default None)
you can pass path to both classification and detection checkpoints as long as the names dont' change
train_rpn --bool: whether to train the rpn layer (Default True)
train_final_classifier --bool:Whether to train the final_classifier (Fast Rcnn layer) (Default True)
train_base_nn --bool:Whether to train the base_nn/fixed_feature_extractor (Default True)
losses_to_watch --list: A list of losses to watch (Default ['rpn_cls','rpn_reg','final_cls','final_reg']).
The losses in this list are added and then weights are saved wrt to that.
The list can contain any combination of the above 4 only.
tb_log_dir --str: path to log dir for tensorboard logging (Default 'log')
num_rois --int: The number of rois to use at once (Default = 32)
horizontal_flips --bool: augment training data by horizontal flips (Default False)
vertical_flips --bool: augment training data by vertical flips (Default False)
rot_90 --bool: augment training data by 90 deg rotations (Default False)
anchor_box_scales --list: The list of anchor box scales to use (Default [128,256,512])
anchor_box ratios --list of list: The list of anchorbox aspect ratios to use (Default [[1, 1], [1./math.sqrt(2), 2./math.sqrt(2)], [2./math.sqrt(2), 1./math.sqrt(2)]])
im_size --int: The size to resize the image (Default 600). This is the smallest side of Pascal VOC format
rpn_stride --int: The stride for rpn (Default = 16)
visualize_model --str: Path to save the model as .png file
verify_trainable --bool: print layer wise names and prints if it is trainable or not (Default True)
optimizer_rpn --keras.optimizer: The optimizer for rpn (Default Adam(lr=1e-5))
optimizer_classifier --keras.optimizer: The optimizer for classifier (Default Adam(lr=1e-5))
validation_interval --int: The frequency (in epochs) to do validation. supply 0 if no validation
rpn_min_overlap --float: (0,1) The Min IOU in rpn layer (Default 0.3) (original implementation)
rpn_max_overlap --float: (0,1) The max IOU in rpn layer (Default 0.7) (original implementation)
classifier_min_overlap --float: (0,1) same as above but in final classifier (Default 0.1) (original implementation)
classifier_max_overlap --float: (0,1) same as above (Default 0.5) (original implementation)
rpn_nms_threshold --float :(0,1) The threshold above which to supress the bbox using Non max supression in rpn (Default 0.7)(from original implementation)
seed --int: To seed the random shuffling of training data (Default = 5000)
Performing alternating training:
- Use the train_rpn,train_final_classifier and train_base_nn boolean arguments to accomplish
alternating training.
- While using the above arguments change the members of losses_to_watch = ['rpn_cls','rpn_reg','final_cls','final_reg']
accordingly else it will throw error
- for eg if you are training only the base_nn and the rpn set:
train_rpn = True
train_base_nn = True
train_final_classifier = False
losses_to_watch = ['rpn_cls','rpn_reg'] (do not include 'final_cls', 'final_reg')
OUTPUT:
prints the training log. Does not return anything
Save details:
1.saves the weights of the full FRCNN model as .h5
2.saves a tensorboard file
3.saves the history of weights saved in ./saving_log.txt so that it can be known at which epoch the model is saved
4.saves the model configuration as a .pickle file
5.optionally saves the full FRCNN architecture as .png
NOTE:
as of now the batch size = 1
Prints loss = 0 for losses from model which is not being trained
TODO: The training is a bit slow because of the data generation step. Generate_data in multiple threads and queue them for faster training
"""
check_list = ['rpn_cls', 'rpn_reg', 'final_cls', 'final_reg']
for n in losses_to_watch:
if n not in check_list:
raise ValueError(
"unsupported loss the supported losses are: {}".format(
check_list))
if not train_rpn:
if "rpn_cls" in losses_to_watch or "rpn_reg" in losses_to_watch:
raise ValueError(
"Cannot watch rpn_cls and rpn_reg when train_rpn == False")
if not train_final_classifier:
if "final_cls" in losses_to_watch or "final_reg" in losses_to_watch:
raise ValueError(
"cannot watch final_cls and final_reg when train_final_classifier == False"
)
nn = network_arch
random.seed(seed)
np.random.seed(seed)
# pass the settings from the function call, and persist them in the config object
C = config.Config()
C.rpn_max_overlap = rpn_max_overlap
C.rpn_min_overlap = rpn_min_overlap
C.classifier_min_overlap = classifier_min_overlap
C.classifier_max_overlap = classifier_max_overlap
C.anchor_box_scales = anchor_box_scales
C.anchor_box_ratios = anchor_box_ratios
C.im_size = im_size
C.use_horizontal_flips = bool(horizontal_flips)
C.use_vertical_flips = bool(vertical_flips)
C.rot_90 = bool(rot_90)
C.rpn_stride = rpn_stride
C.rpn_nms_threshold = rpn_nms_threshold
C.weights_all_path = output_weight_path
C.num_rois = int(num_rois)
# check if weight path was passed via command line
if input_weights_path:
C.initial_weights = input_weights_path
all_imgs, classes_count, class_mapping = get_data(train_path)
print("The class mapping is:")
print(class_mapping)
if 'bg' not in classes_count:
classes_count['bg'] = 0
class_mapping['bg'] = len(class_mapping)
C.class_mapping = class_mapping
print('Training images per class:')
pprint.pprint(classes_count)
print('Num classes (including bg) = {}'.format(len(classes_count)))
with open(config_filename, 'wb') as config_f:
pickle.dump(C, config_f)
print(
'Config has been written to {}, and can be loaded when testing to ensure correct results'
.format(config_filename))
np.random.shuffle(all_imgs)
train_imgs = [s for s in all_imgs if s['imageset'] == 'train']
val_imgs = [s for s in all_imgs if s['imageset'] == 'valid']
print('Num train samples {}'.format(len(train_imgs)))
print('Num val samples {}'.format(len(val_imgs)))
input_shape_img = (None, None, 3)
img_input = Input(shape=input_shape_img)
roi_input = Input(shape=(None, 4))
# define the base network (resnet here, can be VGG, Inception, etc)
shared_layers = nn.nn_base(img_input, trainable=train_base_nn)
# define the RPN, built on the base layers
num_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios)
rpn = nn.rpn(shared_layers, num_anchors, trainable=train_rpn)
# define the classifier, built on base layers
classifier = nn.classifier(shared_layers,
roi_input,
C.num_rois,
len(classes_count),
trainable=train_final_classifier)
# create models
model_base = Model(img_input,
shared_layers) # for computing the output shape
model_rpn = Model(img_input, rpn[:2]) # used for training
model_classifier = Model([img_input, roi_input],
classifier) # used for training
# this is a model that holds both the RPN and the classifier, used to load/save and freeze/unfreeze weights for the models
model_all = Model([img_input, roi_input], rpn[:2] + classifier)
# tensorboard
tbCallBack = TensorBoard(log_dir=tb_log_dir,
histogram_freq=1,
write_graph=False,
write_images=False)
tbCallBack.set_model(model_all)
#NOTE: both model_rpn and model_classifer contains the base_nn
try:
print('loading weights from {}'.format(C.initial_weights))
model_all.load_weights(C.initial_weights, by_name=True)
except:
print('Could not load pretrained model weights')
# number of trainable parameters
trainable_count = int(
np.sum([K.count_params(p) for p in set(model_all.trainable_weights)]))
non_trainable_count = int(
np.sum(
[K.count_params(p) for p in set(model_all.non_trainable_weights)]))
print('Total params: {:,}'.format(trainable_count + non_trainable_count))
print('Trainable params: {:,}'.format(trainable_count))
print('Non-trainable params: {:,}'.format(non_trainable_count))
if verify_trainable:
for layer in model_all.layers:
print(layer.name, layer.trainable)
model_rpn.compile(optimizer=optimizer_rpn,
loss=[
Losses.rpn_loss_cls(num_anchors),
Losses.rpn_loss_regr(num_anchors)
])
model_classifier.compile(
optimizer=optimizer_classifier,
loss=[
Losses.class_loss_cls,
Losses.class_loss_regr(len(classes_count) - 1)
],
metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'})
model_all.compile(optimizer='sgd', loss='mse')
# save model_all as png for visualization
if visualize_model != None:
plot_model(model=model_all,
to_file=visualize_model,
show_shapes=True,
show_layer_names=True)
epoch_length = len(train_imgs)
validation_epoch_length = len(val_imgs)
num_epochs = int(num_epochs)
iter_num = 0
# train and valid data generator
data_gen_train = data_generators.get_anchor_gt(train_imgs,
classes_count,
C,
model_base,
K.image_dim_ordering(),
preprocessing_function,
mode='train')
data_gen_val = data_generators.get_anchor_gt(val_imgs,
classes_count,
C,
model_base,
K.image_dim_ordering(),
preprocessing_function,
mode='val')
losses_val = np.zeros((validation_epoch_length, 5))
losses = np.zeros((epoch_length, 5))
rpn_accuracy_rpn_monitor = []
rpn_accuracy_for_epoch = []
start_time = time.time()
best_loss = np.Inf
val_best_loss = np.Inf
val_best_loss_epoch = 0
print('Starting training')
def write_log(callback, names, logs, batch_no):
for name, value in zip(names, logs):
summary = tf.Summary()
summary_value = summary.value.add()
summary_value.simple_value = value
summary_value.tag = name
callback.writer.add_summary(summary, batch_no)
callback.writer.flush()
train_names = [
'train_loss_rpn_cls', 'train_loss_rpn_reg', 'train_loss_class_cls',
'train_loss_class_reg', 'train_total_loss', 'train_acc'
]
val_names = [
'val_loss_rpn_cls', 'val_loss_rpn_reg', 'val_loss_class_cls',
'val_loss_class_reg', 'val_total_loss', 'val_acc'
]
for epoch_num in range(num_epochs):
progbar = generic_utils.Progbar(epoch_length)
print('Epoch {}/{}'.format(epoch_num + 1, num_epochs))
while True:
try:
if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose:
mean_overlapping_bboxes = float(
sum(rpn_accuracy_rpn_monitor)) / len(
rpn_accuracy_rpn_monitor)
rpn_accuracy_rpn_monitor = []
print(
'Average number of overlapping bounding boxes from RPN = {} for {} previous iterations'
.format(mean_overlapping_bboxes, epoch_length))
if mean_overlapping_bboxes == 0:
print(
'RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.'
)
X, Y, img_data = next(data_gen_train)
if train_rpn:
loss_rpn = model_rpn.train_on_batch(X, Y)
P_rpn = model_rpn.predict_on_batch(X)
R = roi_helpers.rpn_to_roi(P_rpn[0],
P_rpn[1],
C,
K.image_dim_ordering(),
use_regr=True,
overlap_thresh=C.rpn_nms_threshold,
flag="train")
# note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format
X2, Y1, Y2, IouS = roi_helpers.calc_iou(
R, img_data, C, class_mapping)
if X2 is None:
rpn_accuracy_rpn_monitor.append(0)
rpn_accuracy_for_epoch.append(0)
continue
neg_samples = np.where(Y1[0, :, -1] == 1)
pos_samples = np.where(Y1[0, :, -1] == 0)
if len(neg_samples) > 0:
neg_samples = neg_samples[0]
else:
neg_samples = []
if len(pos_samples) > 0:
pos_samples = pos_samples[0]
else:
pos_samples = []
rpn_accuracy_rpn_monitor.append(len(pos_samples))
rpn_accuracy_for_epoch.append((len(pos_samples)))
if C.num_rois > 1:
if len(pos_samples) < C.num_rois // 2:
selected_pos_samples = pos_samples.tolist()
else:
selected_pos_samples = np.random.choice(
pos_samples, C.num_rois // 2,
replace=False).tolist()
try:
selected_neg_samples = np.random.choice(
neg_samples,
C.num_rois - len(selected_pos_samples),
replace=False).tolist()
except:
selected_neg_samples = np.random.choice(
neg_samples,
C.num_rois - len(selected_pos_samples),
replace=True).tolist()
sel_samples = selected_pos_samples + selected_neg_samples
else:
# in the extreme case where num_rois = 1, we pick a random pos or neg sample
selected_pos_samples = pos_samples.tolist()
selected_neg_samples = neg_samples.tolist()
if np.random.randint(0, 2):
sel_samples = random.choice(neg_samples)
else:
sel_samples = random.choice(pos_samples)
if train_final_classifier:
loss_class = model_classifier.train_on_batch(
[X, X2[:, sel_samples, :]],
[Y1[:, sel_samples, :], Y2[:, sel_samples, :]])
# losses
if train_rpn:
losses[iter_num, 0] = loss_rpn[1]
losses[iter_num, 1] = loss_rpn[2]
else:
losses[iter_num, 0] = 0
losses[iter_num, 1] = 0
if train_final_classifier:
losses[iter_num, 2] = loss_class[1]
losses[iter_num, 3] = loss_class[2]
losses[iter_num, 4] = loss_class[3] # accuracy
else:
losses[iter_num, 2] = 0
losses[iter_num, 3] = 0
losses[iter_num, 4] = 0
iter_num += 1
progbar.update(
iter_num,
[('rpn_cls', np.mean(losses[:iter_num, 0])),
('rpn_regr', np.mean(losses[:iter_num, 1])),
('detector_cls', np.mean(losses[:iter_num, 2])),
('detector_regr', np.mean(losses[:iter_num, 3]))])
if iter_num == epoch_length:
if train_rpn:
loss_rpn_cls = np.mean(losses[:, 0])
loss_rpn_regr = np.mean(losses[:, 1])
else:
loss_rpn_cls = 0
loss_rpn_regr = 0
if train_final_classifier:
loss_class_cls = np.mean(losses[:, 2])
loss_class_regr = np.mean(losses[:, 3])
class_acc = np.mean(losses[:, 4])
else:
loss_class_cls = 0
loss_class_regr = 0
class_acc = 0
mean_overlapping_bboxes = float(sum(
rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch)
rpn_accuracy_for_epoch = []
if C.verbose:
print(
'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'
.format(mean_overlapping_bboxes))
print(
'Classifier accuracy for bounding boxes from RPN: {}'
.format(class_acc))
print('Loss RPN classifier: {}'.format(loss_rpn_cls))
print('Loss RPN regression: {}'.format(loss_rpn_regr))
print('Loss Detector classifier: {}'.format(
loss_class_cls))
print('Loss Detector regression: {}'.format(
loss_class_regr))
print('Elapsed time: {}'.format(time.time() -
start_time))
loss_dict_train = {
"rpn_cls": loss_rpn_cls,
"rpn_reg": loss_rpn_regr,
"final_cls": loss_class_cls,
"final_reg": loss_class_regr
}
curr_loss = 0
for l in losses_to_watch:
curr_loss += loss_dict_train[l]
iter_num = 0
start_time = time.time()
write_log(tbCallBack, train_names, [
loss_rpn_cls, loss_rpn_regr, loss_class_cls,
loss_class_regr, curr_loss, class_acc
], epoch_num)
if curr_loss < best_loss:
if C.verbose:
print(
'Total loss decreased from {} to {} in training, saving weights'
.format(best_loss, curr_loss))
save_log_data = '\nTotal loss decreased from {} to {} in epoch {}/{} in training, saving weights'.format(
best_loss, curr_loss, epoch_num + 1,
num_epochs)
with open("./saving_log.txt", "a") as f:
f.write(save_log_data)
best_loss = curr_loss
model_all.save_weights(C.weights_all_path)
break
except Exception as e:
print('Exception: {}'.format(e))
continue
if validation_interval > 0:
# validation
if (epoch_num + 1) % validation_interval == 0:
progbar = generic_utils.Progbar(validation_epoch_length)
print("Validation... \n")
while True:
try:
X, Y, img_data = next(data_gen_val)
if train_rpn:
val_loss_rpn = model_rpn.test_on_batch(X, Y)
P_rpn = model_rpn.predict_on_batch(X)
R = roi_helpers.rpn_to_roi(
P_rpn[0],
P_rpn[1],
C,
K.image_dim_ordering(),
use_regr=True,
overlap_thresh=C.rpn_nms_threshold,
flag="train")
# note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format
X2, Y1, Y2, IouS = roi_helpers.calc_iou(
R, img_data, C, class_mapping)
neg_samples = np.where(Y1[0, :, -1] == 1)
pos_samples = np.where(Y1[0, :, -1] == 0)
if len(neg_samples) > 0:
neg_samples = neg_samples[0]
else:
neg_samples = []
if len(pos_samples) > 0:
pos_samples = pos_samples[0]
else:
pos_samples = []
rpn_accuracy_rpn_monitor.append(len(pos_samples))
rpn_accuracy_for_epoch.append((len(pos_samples)))
if C.num_rois > 1:
if len(pos_samples) < C.num_rois // 2:
selected_pos_samples = pos_samples.tolist()
else:
selected_pos_samples = np.random.choice(
pos_samples,
C.num_rois // 2,
replace=False).tolist()
try:
selected_neg_samples = np.random.choice(
neg_samples,
C.num_rois - len(selected_pos_samples),
replace=False).tolist()
except:
selected_neg_samples = np.random.choice(
neg_samples,
C.num_rois - len(selected_pos_samples),
replace=True).tolist()
sel_samples = selected_pos_samples + selected_neg_samples
else:
# in the extreme case where num_rois = 1, we pick a random pos or neg sample
selected_pos_samples = pos_samples.tolist()
selected_neg_samples = neg_samples.tolist()
if np.random.randint(0, 2):
sel_samples = random.choice(neg_samples)
else:
sel_samples = random.choice(pos_samples)
if train_final_classifier:
val_loss_class = model_classifier.test_on_batch(
[X, X2[:, sel_samples, :]],
[Y1[:, sel_samples, :], Y2[:, sel_samples, :]])
if train_rpn:
losses_val[iter_num, 0] = val_loss_rpn[1]
losses_val[iter_num, 1] = val_loss_rpn[2]
else:
losses_val[iter_num, 0] = 0
losses_val[iter_num, 1] = 0
if train_final_classifier:
losses_val[iter_num, 2] = val_loss_class[1]
losses_val[iter_num, 3] = val_loss_class[2]
losses_val[iter_num, 4] = val_loss_class[3]
else:
losses_val[iter_num, 2] = 0
losses_val[iter_num, 3] = 0
losses_val[iter_num, 4] = 0
iter_num += 1
progbar.update(
iter_num,
[('rpn_cls', np.mean(losses_val[:iter_num, 0])),
('rpn_regr', np.mean(losses_val[:iter_num, 1])),
('detector_cls', np.mean(losses_val[:iter_num,
2])),
('detector_regr', np.mean(losses_val[:iter_num,
3]))])
if iter_num == validation_epoch_length:
if train_rpn:
val_loss_rpn_cls = np.mean(losses_val[:, 0])
val_loss_rpn_regr = np.mean(losses_val[:, 1])
else:
val_loss_rpn_cls = 0
val_loss_rpn_regr = 0
if train_final_classifier:
val_loss_class_cls = np.mean(losses_val[:, 2])
val_loss_class_regr = np.mean(losses_val[:, 3])
val_class_acc = np.mean(losses_val[:, 4])
else:
val_loss_class_cls = 0
val_loss_class_regr = 0
val_class_acc = 0
mean_overlapping_bboxes = float(
sum(rpn_accuracy_for_epoch)) / len(
rpn_accuracy_for_epoch)
rpn_accuracy_for_epoch = []
loss_dict_valid = {
"rpn_cls": val_loss_rpn_cls,
"rpn_reg": val_loss_rpn_regr,
"final_cls": val_loss_class_cls,
"final_reg": val_loss_class_regr
}
val_curr_loss = 0
for l in losses_to_watch:
val_curr_loss += loss_dict_valid[l]
write_log(tbCallBack, val_names, [
val_loss_rpn_cls, val_loss_rpn_regr,
val_loss_class_cls, val_loss_class_regr,
val_curr_loss, val_class_acc
], epoch_num)
if C.verbose:
print('[INFO VALIDATION]')
print(
'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'
.format(mean_overlapping_bboxes))
print(
'Classifier accuracy for bounding boxes from RPN: {}'
.format(val_class_acc))
print('Loss RPN classifier: {}'.format(
val_loss_rpn_cls))
print('Loss RPN regression: {}'.format(
val_loss_rpn_regr))
print('Loss Detector classifier: {}'.format(
val_loss_class_cls))
print('Loss Detector regression: {}'.format(
val_loss_class_regr))
print(
"current loss: %.2f, best loss: %.2f at epoch: %d"
% (val_curr_loss, val_best_loss,
val_best_loss_epoch))
print('Elapsed time: {}'.format(time.time() -
start_time))
if val_curr_loss < val_best_loss:
if C.verbose:
print(
'Total loss decreased from {} to {}, saving weights'
.format(val_best_loss, val_curr_loss))
save_log_data = '\nTotal loss decreased from {} to {} in epoch {}/{} in validation, saving weights'.format(
val_best_loss, val_curr_loss,
epoch_num + 1, num_epochs)
with open("./saving_log.txt", "a") as f:
f.write(save_log_data)
val_best_loss = val_curr_loss
val_best_loss_epoch = epoch_num
model_all.save_weights(C.weights_all_path)
start_time = time.time()
iter_num = 0
break
except:
pass
print('Training complete, exiting.')
def plot_parameter_statistic(model,
layer_types=['Dense', 'Conv2D'],
trainable=True,
non_trainable=True,
outputs=False):
layer_types = [l.__name__ if type(l) == type else l for l in layer_types]
def get_layers_recursion(model):
layers = []
for l in model.layers:
if l.__class__.__name__ is 'Model':
child_layers = get_layers_recursion(l)
else:
child_layers = [l]
for cl in child_layers:
if cl not in layers:
layers.append(cl)
return layers
layers = get_layers_recursion(model)
layers = [l for l in layers if l.__class__.__name__ in layer_types]
names = [l.name for l in layers]
y = range(len(names))
plt.figure(figsize=[12, max(len(y) // 4, 1)])
offset = np.zeros(len(layers), dtype=int)
legend = []
if trainable:
counts_trainable = [
np.sum([K.count_params(p) for p in set(l.trainable_weights)])
for l in layers
]
plt.barh(y, counts_trainable, align='center', color='#1f77b4')
offset += np.array(counts_trainable, dtype=int)
legend.append('trainable')
if non_trainable:
counts_non_trainable = [
np.sum([K.count_params(p) for p in set(l.non_trainable_weights)])
for l in layers
]
plt.barh(y,
counts_non_trainable,
align='center',
color='#ff7f0e',
left=offset)
offset += np.array(counts_non_trainable, dtype=int)
legend.append('non-trainable')
if outputs:
counts_outputs = [
np.sum([
np.sum([np.prod(s[1:]) for s in n.output_shapes])
for n in l._inbound_nodes
]) for l in layers
]
plt.barh(y,
counts_outputs,
align='center',
color='#2ca02c',
left=offset)
offset += np.array(counts_outputs, dtype=int)
legend.append('outputs')
plt.yticks(y, names)
plt.ylim(y[0] - 1, y[-1] + 1)
ax = plt.gca()
ax.invert_yaxis()
ax.xaxis.tick_top()
plt.legend(legend)
plt.show()
def count_parameters(model):
trainable_count = int(np.sum([K.count_params(p) for p in set(model.trainable_weights)]))
non_trainable_count = int(np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]))
print('trainable %16i' %(trainable_count))
print('non-trainable %16i' %(non_trainable_count))
def customLoss(yTrue,yPred):
if(yTrue.shape[1]!=None):
return (K.mean(K.square((K.log(yTrue) - K.log(yPred))),axis=-1)-(0.5/pow(K.count_params(yTrue),2))*K.sum((K.log(yTrue) - K.log(yPred)),axis=-1))
else:
return K.sum(K.abs(yPred - yTrue), axis=-1)
# top_model.load_weights('./Xception_decay.hdf5')
for layer in top_model.layers:
layer.trainable = True
LearningRate = 0.0001
n_epochs = 30
sgd = optimizers.SGD(lr=config.MIN_LR, momentum=0.9)
adam = optimizers.Adam(lr=LearningRate,
decay=LearningRate / n_epochs,
amsgrad=True)
top_model.compile(optimizer=sgd,
loss="categorical_crossentropy",
metrics=["accuracy"])
trainable_params = int(
np.sum([K.count_params(p) for p in set(top_model.trainable_weights)]))
non_trainable_params = int(
np.sum([K.count_params(p) for p in set(top_model.non_trainable_weights)]))
print("\nModel Status")
print("=" * 40)
print("Total Parameters: {:,}".format(
(trainable_params + non_trainable_params)))
print("Non-Trainable Parameters: {:,}".format(non_trainable_params))
print("Trainable Parameters: {:,}\n".format(trainable_params))
train_path = '/cptjack/totem/barrylee/cut_small_cell/hepat-tri-classification/new-train/96-train'
train_folders = [
train_path + "/ballooning/", train_path + "/normal/",
train_path + "/steatosis/"
] #,"/cptjack/totem/barrylee/cut_small_cell/train/other_cells/"
def main():
start = time.time()
epochs=30
version=2
weights_save_name = "bidir_LSTM_NT_{}".format(str(version))
batch_size=32
# Training dataset loading
train_data1 = np.load("./dataset_npy/lstm_NT_train_data_500.npy", allow_pickle=True)
train_label1 = np.load("./dataset_npy/lstm_NT_train_label_500.npy", allow_pickle=True)
train_data2 = np.load("./dataset_npy/lstm_NT_train_data_1000.npy", allow_pickle=True)
train_label2 = np.load("./dataset_npy/lstm_NT_train_label_1000.npy", allow_pickle=True)
train_data = np.concatenate((train_data1, train_data2), axis=0)
train_label = np.concatenate((train_label1, train_label2), axis=0)
del train_data1, train_data2, train_label1, train_label2
train_data3 = np.load("./dataset_npy/lstm_NT_train_data_1500.npy", allow_pickle=True)
train_label3 = np.load("./dataset_npy/lstm_NT_train_label_1500.npy", allow_pickle=True)
train_data = np.concatenate((train_data, train_data3), axis=0)
train_label = np.concatenate((train_label, train_label3), axis=0)
del train_data3, train_label3
train_data4 = np.load("./dataset_npy/lstm_NT_train_data_1719.npy", allow_pickle=True)
train_label4 = np.load("./dataset_npy/lstm_NT_train_labels_1719.npy", allow_pickle=True)
train_data = np.concatenate((train_data, train_data4), axis=0)
train_label = np.concatenate((train_label, train_label4), axis=0)
del train_data4, train_label4
print(train_data.shape, train_label.shape)
train_label = utils.to_categorical(train_label)
test_data = np.load("./dataset_npy/lstm_NT_test_data.npy", allow_pickle=True)
test_label = np.load("./dataset_npy/lstm_NT_test_labels.npy", allow_pickle=True)
test_label = utils.to_categorical(test_label)
print(test_data.shape, test_label.shape)
print("Dataset Loaded...")
# Train validation split
trainX, valX, trainY, valY = train_test_split(
train_data, train_label, shuffle=True, test_size=0.2
)
model = rnn_models(train_data)
trainable_count = int(
np.sum([K.count_params(p) for p in set(model.trainable_weights)])
)
non_trainable_count = int(
np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])
)
# Number of trainable and non-trainable parameters
print("Total params: {:,}".format(trainable_count + non_trainable_count))
print("Trainable params: {:,}".format(trainable_count))
print("Non-trainable params: {:,}".format(non_trainable_count))
# Keras backend
model_checkpoint = ModelCheckpoint(
"models/" + weights_save_name + ".hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
save_weights_only=True,
)
stopping = EarlyStopping(monitor="val_loss", patience=10, verbose=0)
print("Training is going to start in 3... 2... 1... ")
# Model training
H = model.fit(
trainX,
trainY,
validation_data=(valX, valY),
batch_size=batch_size,
epochs=epochs,
shuffle=True,
callbacks=[model_checkpoint, stopping],
)
# plot the training loss and accuracy
results = model.evaluate(test_data, test_label)
print("Test acc, loss : ", results)
end = time.time()
dur = end - start
if dur < 60:
print("Execution Time:", dur, "seconds")
elif dur > 60 and dur < 3600:
dur = dur / 60
print("Execution Time:", dur, "minutes")
else:
dur = dur / (60 * 60)
print("Execution Time:", dur, "hours")
layer.trainable = True
LearningRate = 0.01
decay = 0.0001
n_epochs = 20
sgd = optimizers.SGD(lr=LearningRate,
decay=LearningRate / n_epochs,
momentum=0.9,
nesterov=True)
parallel_model.compile(optimizer=sgd,
loss="categorical_crossentropy",
metrics=["accuracy"])
trainable_params = int(
np.sum([K.count_params(p) for p in set(parallel_model.trainable_weights)]))
non_trainable_params = int(
np.sum([
K.count_params(p) for p in set(parallel_model.non_trainable_weights)
]))
print("\nModel Stats")
print("=" * 30)
print("Total Parameters: {:,}".format(
(trainable_params + non_trainable_params)))
print("Non-Trainable Parameters: {:,}".format(non_trainable_params))
print("Trainable Parameters: {:,}\n".format(trainable_params))
train_folders = [
"./raw/train/cancer cell/", "./raw/train/lymphocyte/",
def test():
args = cmd.args()
if args.load_generated_images:
print("Loading images...")
input_images, target_images, generated_images, names = load_generated_images(
args.generated_images_dir)
else:
print("Generate images...")
from keras import backend as K
if args.use_dropout_test:
K.set_learning_phase(1)
dataset = PoseHMDataset(test_phase=True, **vars(args))
generator = make_generator(args.image_size, args.use_input_pose,
args.warp_skip, args.disc_type,
args.warp_agg)
trainable_count = int(
np.sum(
[K.count_params(p) for p in set(generator.trainable_weights)]))
non_trainable_count = int(
np.sum([
K.count_params(p) for p in set(generator.non_trainable_weights)
]))
print('Total params: {:,}'.format(trainable_count +
non_trainable_count))
print('Trainable params: {:,}'.format(trainable_count))
print('Non-trainable params: {:,}'.format(non_trainable_count))
assert (args.generator_checkpoint is not None)
generator.load_weights(args.generator_checkpoint)
input_images, target_images, generated_images, names = generate_images(
dataset, generator, args.use_input_pose)
print("Save images to %s..." % (args.generated_images_dir, ))
save_images(input_images, target_images, generated_images, names,
args.generated_images_dir)
print("Compute inception score...")
inception_score = get_inception_score(generated_images)
print("Inception score %s" % inception_score[0])
print("Compute structured similarity score (SSIM)...")
structured_score = ssim_score(generated_images, target_images)
print("SSIM score %s" % structured_score)
print("Compute l1 score...")
norm_score = l1_score(generated_images, target_images)
print("L1 score %s" % norm_score)
print("Compute masked inception score...")
generated_images_masked = create_masked_image(names, generated_images,
args.annotations_file_test)
reference_images_masked = create_masked_image(names, target_images,
args.annotations_file_test)
inception_score_masked = get_inception_score(generated_images_masked)
print("Inception score masked %s" % inception_score_masked[0])
print("Compute masked SSIM...")
structured_score_masked = ssim_score(generated_images_masked,
reference_images_masked)
print("SSIM score masked %s" % structured_score_masked)
print(
"Inception score = %s, masked = %s; SSIM score = %s, masked = %s; l1 score = %s"
% (inception_score, inception_score_masked, structured_score,
structured_score_masked, norm_score))
n_part_param_layers = [
len(model.layers[L].trainable_weights) for L in range(n_layers)
] # how many parts of parameters that each layer has
index_layers_n0 = np.nonzero(n_part_param_layers)[
0] # indices for layers with trainable parameters
n_layers_n0 = len(index_layers_n0)
trainable_count = np.zeros(
n_layers_n0,
dtype=int) # the number of trainable parameters that each layer has
for i in range(n_layers_n0):
layer_id = index_layers_n0[i]
trainable_count[i] = np.sum([
K.count_params(model.layers[layer_id].trainable_weights[p])
for p in range(n_part_param_layers[layer_id])
])
trainable_count_id_start_end = np.insert(np.cumsum(trainable_count), 0, 0)
t = 0
FI_train_all = np.zeros(n_train) # FI for all parameters
grad_K = tf.concat(
axis=1,
values=[
tf.concat(
axis=0,
values=[
K.flatten(b)[..., None]
def get_params_count(model):
trainable = int(
np.sum([K.count_params(p) for p in set(model.trainable_weights)]))
non_trainable = int(
np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]))
return trainable, non_trainable
def evaluate(self, phenotype, load_prev_weights, weights_save_path, parent_weights_path,\
train_time, num_epochs, datagen=None, datagen_test = None, input_size=(32, 32, 3)):
"""
Evaluates the keras model using the keras optimiser
Parameters
----------
phenotype : str
individual phenotype
load_prev_weights : bool
resume training from a previous train or not
weights_save_path : str
path where to save the model weights after training
parent_weights_path : str
path to the weights of the previous training
train_time : float
maximum training time
num_epochs : int
maximum number of epochs
datagen : keras.preprocessing.image.ImageDataGenerator
Data augmentation method image data generator
input_size : tuple
dataset input shape
Returns
-------
score_history : dict
training data: loss and accuracy
"""
model_phenotype, learning_phenotype = phenotype.split('learning:')
learning_phenotype = 'learning:' + learning_phenotype.rstrip().lstrip()
model_phenotype = model_phenotype.rstrip().lstrip().replace(' ', ' ')
keras_layers = self.get_layers(model_phenotype)
keras_learning = self.get_learning(learning_phenotype)
batch_size = int(keras_learning['batch_size'])
if load_prev_weights:
model = keras.models.load_model(
parent_weights_path.replace('.hdf5', '.h5'))
else:
model = self.assemble_network(keras_layers, input_size)
opt = self.assemble_optimiser(keras_learning)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
#early stopping
early_stop = keras.callbacks.EarlyStopping(
monitor='val_loss', patience=int(keras_learning['early_stop']))
#time based stopping
time_stop = TimedStopping(seconds=train_time, verbose=DEBUG)
#save individual with the lowest validation loss
monitor = ModelCheckpoint(weights_save_path,
monitor='val_loss',
verbose=0,
save_best_only=True)
trainable_count = int(
np.sum([
backend.count_params(p) for p in set(model.trainable_weights)
]))
if datagen is not None:
score = model.fit_generator(
datagen.flow(self.dataset['evo_x_train'],
self.dataset['evo_y_train'],
batch_size=batch_size),
steps_per_epoch=(self.dataset['evo_x_train'].shape[0] //
batch_size),
epochs=int(keras_learning['epochs']),
validation_data=(datagen_test.flow(self.dataset['evo_x_val'],
self.dataset['evo_y_val'],
batch_size=batch_size)),
validation_steps=(self.dataset['evo_x_val'].shape[0] //
batch_size),
callbacks=[early_stop, time_stop, monitor],
initial_epoch=num_epochs,
verbose=DEBUG)
else:
score = model.fit(
x=self.dataset['evo_x_train'],
y=self.dataset['evo_y_train'],
batch_size=batch_size,
epochs=int(keras_learning['epochs']),
steps_per_epoch=(self.dataset['evo_x_train'].shape[0] //
batch_size),
validation_data=(self.dataset['evo_x_val'],
self.dataset['evo_y_val']),
callbacks=[early_stop, time_stop, monitor],
initial_epoch=num_epochs,
verbose=DEBUG)
#load weights with the lowest val loss
if os.path.isfile(weights_save_path):
model.load_weights(weights_save_path)
#save final moodel to file
model.save(weights_save_path.replace('.hdf5', '.h5'))
#measure test performance
if datagen_test is None:
y_pred_test = model.predict(self.dataset['evo_x_test'],
batch_size=batch_size,
verbose=0)
else:
y_pred_test = model.predict_generator(
datagen_test.flow(self.dataset['evo_x_test'],
batch_size=100,
shuffle=False),
steps=self.dataset['evo_x_test'].shape[0] // 100,
verbose=DEBUG)
accuracy_test = self.fitness_metric(self.dataset['evo_y_test'],
y_pred_test)
if DEBUG:
print(phenotype, accuracy_test)
score.history['trainable_parameters'] = trainable_count
score.history['accuracy_test'] = accuracy_test
return score.history
def train():
train_dataset = dataset.H5SegmentationDataset(
root=dataDir_,
fn_txt=trainFileText_,
n_class=numClass_,
label_prob_to_cls=labelProbToCls_,
label_one_hot=oneHot_,
transforms=transformer.Compose([
transformer.MinMaxScale(mins=channelMin_, maxs=channelMax_),
transformer.RandomGammaTransform(gamma=0.5, p=0.5),
transformer.RandomBlur(kernel_size=(3, 3), p=0.1),
transformer.RandomHorizontalFlip(p=0.5),
transformer.RandomVerticalFlip(p=0.5),
transformer.RandomRotate(angle_threshold=20, p=0.5),
transformer.RandomScale(scale_range=(0.8, 1.2), p=0.5),
# transformer.Normalize(mean=channelMean_, std=channelStd_)
]))
val_dataset = dataset.H5SegmentationDataset(
root=dataDir_,
fn_txt=valFileText_,
n_class=numClass_,
label_prob_to_cls=labelProbToCls_,
label_one_hot=oneHot_,
transforms=transformer.Compose([
# transformer.Normalize(mean=channelMean_, std=channelStd_)
transformer.MinMaxScale(mins=channelMin_, maxs=channelMax_)
]))
train_generator = datagenerator.data_generator(train_dataset,
batch_size=batchSize_,
shuffle=True)
val_generator = datagenerator.data_generator(val_dataset,
batch_size=batchSize_,
shuffle=False)
stepsPerValEpoch_ = len(val_dataset) // batchSize_
stepsPerEpoch_ = stepsPerValEpoch_ * 5
# for debug
if debug_:
for _ in range(1):
x, y = next(train_generator)
print(x.shape, y.shape)
if oneHot_:
plot_image_label_per_channel(x[0], np.argmax(y[0], -1))
else:
from semantic_segmentation.utils import plot_rgb_image
plot_image_label_per_channel(x[0], y[0, :, :, 0])
plot_rgb_image(x[0], y[0, :, :, 0])
# define model, load weights, compile model
model = pspnet(input_shape=(inputHeight_, inputWidth_, inputChannel_),
n_class=numClass_,
one_hot=oneHot_,
backbone_name=backboneName_,
backbone_weights=backboneWeights_,
output_stride=8)
if os.path.exists(resumeModelPath_):
log('load weights from `{}`'.format(resumeModelPath_))
model.load_weights(resumeModelPath_, by_name=True, skip_mismatch=True)
else:
log('build new model `{}`'.format(saveModelPath_))
plot_model(model, to_file=plotPath_)
with open(plotPath_.replace('png', 'json'), 'w') as f:
f.write(model.to_json())
if modelSummary_:
model.summary()
log('model input: {}, output: {}'.format(model.input_shape,
model.output_shape))
log('trainable parameters: {}'.format(
int(
np.sum([
K.count_params(p) for p in set(model.trainable_weights)
]))))
log('non-trainable parameters: {}'.format(
int(
np.sum([
K.count_params(p) for p in set(model.non_trainable_weights)
]))))
model.compile(optimizer=optimizer_, loss=loss_, metrics=['acc'] + metrics_)
# train
model.fit_generator(generator=train_generator,
steps_per_epoch=stepsPerEpoch_,
validation_data=val_generator,
validation_steps=stepsPerValEpoch_,
epochs=epoch_,
callbacks=callbacks_,
verbose=verbose_,
initial_epoch=0)
model.save_weights(filepath=checkpointDir_ +
'/{}_{}.h5'.format(saveVersion_, epoch_))
log('training success!')
def main():
start = time.time()
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
try: # Currently, memory growth needs to be the same across GPUs
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
logical_gpus = tf.config.experimental.list_logical_devices('GPU')
print(len(gpus), "Physical GPUs,", len(logical_gpus),
"Logical GPUs")
except RuntimeError as e: # Memory growth must be set before GPUs have been initialized
print(e)
epochs = 30
image_size = 299
batch_size = 10
weights = True
version = 4
weights_save_name = "EfficientNetB7_w_NT_" + str(
image_size) + "_batch_" + str(batch_size) + "_v{}".format(str(version))
# Training dataset loading
train_data = np.load("./dataset_npy/train_data_NT_fpv_25_rev_299.npy")
train_label = np.load("./dataset_npy/train_label_NT_fpv_25_rev_299.npy")
print("Dataset Loaded...")
# Train and validation split
trainX, valX, trainY, valY = train_test_split(train_data,
train_label,
test_size=0.2,
shuffle=True)
print(trainX.shape, valX.shape, trainY.shape, valY.shape)
# Train nad validation image data generator
trainAug = ImageDataGenerator(
rescale=1.0 / 255.0,
rotation_range=30,
zoom_range=0.15,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.15,
horizontal_flip=True,
fill_mode="nearest",
)
valAug = ImageDataGenerator(rescale=1.0 / 255.0)
model = cnn_model(img_size=image_size, weights=weights)
# Number of trainable and non-trainable parameters
trainable_count = int(
np.sum([K.count_params(p) for p in set(model.trainable_weights)]))
non_trainable_count = int(
np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]))
print("Total params: {:,}".format(trainable_count + non_trainable_count))
print("Trainable params: {:,}".format(trainable_count))
print("Non-trainable params: {:,}".format(non_trainable_count))
if not exists("./models"):
makedirs("./models")
if not exists("./training_logs"):
makedirs("./training_logs")
if not exists("./plots"):
makedirs("./plots")
# Keras backend
model_checkpoint = ModelCheckpoint(
"models/" + weights_save_name + ".hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
save_weights_only=True,
)
stopping = EarlyStopping(monitor="val_loss",
patience=5,
verbose=1,
mode='min')
csv_logger = CSVLogger(
"training_logs/EfficientNetB7_NT_{}_batch_{}_v{}.log".format(
str(image_size), str(batch_size), str(version)),
separator=",",
append=True,
)
print("Training is going to start in 3... 2... 1... ")
# Model Training
H = model.fit_generator(
trainAug.flow(trainX, trainY, batch_size=batch_size),
steps_per_epoch=len(trainX) // batch_size,
validation_data=valAug.flow(valX, valY),
validation_steps=len(valX) // batch_size,
epochs=epochs,
callbacks=[model_checkpoint, stopping, csv_logger],
)
'''
H = model.fit(
trainX, trainY, batch_size=batch_size,
#steps_per_epoch=len(trainX) // batch_size,
#validation_data=valAug.flow(valX, valY),
validation_data=(valX, valY),
#validation_steps=len(valX) // batch_size,
epochs=epochs,
callbacks=[model_checkpoint, stopping, csv_logger],
)
'''
# plot the training loss and accuracy
plt.style.use("ggplot")
plt.figure()
N = stopping.stopped_epoch + 1
plt.plot(np.arange(0, N), H.history["loss"], label="train_loss")
plt.plot(np.arange(0, N), H.history["val_loss"], label="val_loss")
plt.plot(np.arange(0, N), H.history["accuracy"], label="train_acc")
plt.plot(np.arange(0, N), H.history["val_accuracy"], label="val_acc")
plt.title("Training Loss and Accuracy")
plt.xlabel("Epoch #")
plt.ylabel("Loss/Accuracy")
plt.legend(loc="lower left")
plt.savefig(
"plots/EfficientNetB7_training_plot_NT_{}_batch_{}_v{}.png".format(
str(image_size), str(batch_size), str(version)))
end = time.time()
dur = end - start
if dur < 60:
print("Execution Time:", dur, "seconds")
elif dur > 60 and dur < 3600:
dur = dur / 60
print("Execution Time:", dur, "minutes")
else:
dur = dur / (60 * 60)
print("Execution Time:", dur, "hours")
def get_trainable_count(net):
"""https://stackoverflow.com/questions/45046525"""
return sum([K.count_params(p) for p in set(net.trainable_weights)])
