def train_Keras(train_X, train_y, test_X, test_y, kwargs, cae_model_func=None, n_features=None, epochs=150):
normalization = normalization_func()
num_classes = train_y.shape[-1]
norm_train_X = normalization.fit_transform(train_X)
norm_test_X = normalization.transform(test_X)
batch_size = max(2, len(train_X) // 50)
class_weight = train_y.shape[0] / np.sum(train_y, axis=0)
class_weight = num_classes * class_weight / class_weight.sum()
sample_weight = None
print('mu :', kwargs['mu'], ', batch_size :', batch_size)
print('reps : ', reps, ', weights : ', class_weight)
if num_classes == 2:
sample_weight = np.zeros((len(norm_train_X),))
sample_weight[train_y[:, 1] == 1] = class_weight[1]
sample_weight[train_y[:, 1] == 0] = class_weight[0]
class_weight = None
model_clbks = [
callbacks.LearningRateScheduler(scheduler()),
]
if cae_model_func is not None:
svc_model = LinearSVC(nfeatures=(n_features,), **kwargs)
svc_model.create_keras_model(nclasses=num_classes)
classifier = svc_model.model
cae_model = cae_model_func(output_function=classifier, K=n_features)
start_time = time.process_time()
cae_model.fit(
norm_train_X, train_y, norm_test_X, test_y, num_epochs=800, batch_size=batch_size,
class_weight=class_weight
)
model = cae_model.model
model.indices = cae_model.get_support(True)
model.heatmap = cae_model.probabilities.max(axis=0)
model.fs_time = time.process_time() - start_time
else:
svc_model = LinearSVC(norm_train_X.shape[1:], **kwargs)
svc_model.create_keras_model(nclasses=num_classes)
model = svc_model.model
model.compile(
loss=LinearSVC.loss_function(loss_function, class_weight),
optimizer=optimizer_class(lr=initial_lr),
metrics=[LinearSVC.accuracy]
)
model.fit(
norm_train_X, train_y, batch_size=batch_size,
epochs=epochs,
callbacks=model_clbks,
validation_data=(norm_test_X, test_y),
class_weight=class_weight,
sample_weight=sample_weight,
verbose=verbose
)
model.normalization = normalization
return model
def train_with_lr_decay_and_augmentation(model):
model.compile(loss='sparse_categorical_crossentropy',
metrics=['accuracy'],
optimizer=optimizers.Adam(lr=1e-3))
# Callback for learning rate adjustment (see below)
lr_scheduler = callbacks.LearningRateScheduler(learning_rate_schedule)
# TensorBoard callback
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
logdir = os.path.join(log_root, f'{model.name}_{timestamp}')
tensorboard_callback = callbacks.TensorBoard(logdir, histogram_freq=1)
# YOUR CODE HERE
data_augmentation = ImageDataGenerator(width_shift_range=[-4, 4],
horizontal_flip=True)
data_augmentation.fit(x_train)
model.fit_generator(data_augmentation.flow(x_train,
y_train,
batch_size=128),
steps_per_epoch=len(x_train) / 128,
validation_data=(x_test, y_test),
epochs=150,
verbose=1,
callbacks=[lr_scheduler, tensorboard_callback])
def callbacks(self):
""" set callbacks when model fit """
checkpoint = KC.ModelCheckpoint(filepath=self.config.checkpoint,
monitor='val_loss',
verbose=1,
save_weights_only=False,
period=self.config.ckpt_interval)
def lr_schedule(epoch):
lr = self.config.init_lr
total_epochs = self.config.epochs
check_1 = int(total_epochs * 0.3)
check_2 = int(total_epochs * 0.6)
check_3 = int(total_epochs * 0.8)
if epoch > check_1:
lr *= 3e-1
if epoch > check_2:
lr *= 3e-1
if epoch > check_3:
lr *= 3e-1
return lr
lr_scheduler = KC.LearningRateScheduler(lr_schedule)
tensorboard = KC.TensorBoard(
log_dir=self.config.tb_logs,
batch_size=self.config.batch_size,
histogram_freq=0, # validation data can not be generator
write_graph=False,
write_grads=False)
callbacks = [
checkpoint, lr_scheduler, tensorboard
] # TODO bug：Failed to create a directory: out_model/mobilenet_t1/logs\train;
return callbacks
def train(model, data, args):
"""
Training a CapsuleNet
:param model: the CapsuleNet model
:param data: a tuple containing training and testing data, like `((x_train, y_train), (x_test, y_test))`
:param args: arguments
:return: The trained model
"""
# unpacking the data
(x_train, y_train), (x_test, y_test) = data
# callbacks
log = callbacks.CSVLogger(args.save_dir + '/log.csv')
tb = callbacks.TensorBoard(log_dir=args.save_dir + '/tensorboard-logs',
batch_size=args.batch_size,
histogram_freq=int(args.debug))
checkpoint = callbacks.ModelCheckpoint(args.save_dir +
'/weights-{epoch:02d}.h5',
monitor='val_capsnet_acc',
save_best_only=True,
save_weights_only=True,
verbose=1)
lr_decay = callbacks.LearningRateScheduler(
schedule=lambda epoch: args.lr * (args.lr_decay**epoch))
# compile the model
model.compile(optimizer=optimizers.Adam(lr=args.lr),
loss=[margin_loss, 'mse'],
loss_weights=[1., args.lam_recon],
metrics={'capsnet': 'accuracy'})
"""
# Training without data augmentation:
model.fit([x_train, y_train], [y_train, x_train], batch_size=args.batch_size, epochs=args.epochs,
validation_data=[[x_test, y_test], [y_test, x_test]], callbacks=[log, tb, checkpoint, lr_decay])
"""
# Begin: Training with data augmentation ---------------------------------------------------------------------#
def train_generator(x, y, batch_size, shift_fraction=0.):
train_datagen = ImageDataGenerator(
width_shift_range=shift_fraction,
height_shift_range=shift_fraction) # shift up to 2 pixel for MNIST
generator = train_datagen.flow(x, y, batch_size=batch_size)
while 1:
x_batch, y_batch = generator.next()
yield ([x_batch, y_batch], [y_batch, x_batch])
# Training with data augmentation. If shift_fraction=0., also no augmentation.
model.fit_generator(
generator=train_generator(x_train, y_train, args.batch_size,
args.shift_fraction),
steps_per_epoch=int(y_train.shape[0] / args.batch_size),
epochs=args.epochs,
validation_data=[[x_test, y_test], [y_test, x_test]],
callbacks=[log, tb, checkpoint, lr_decay])
# End: Training with data augmentation -----------------------------------------------------------------------#
model.save_weights(args.save_dir + '/trained_model.h5')
print('Trained model saved to \'%s/trained_model.h5\'' % args.save_dir)
return model
def search(train, val):
# TODO: how to discriminate between trials?
log_dir = './logs/' + datetime.now().strftime('%Y%m%d-%H%M%S')
cp_dir = './checkpoints/' + 'weights.{epoch:02d}-{val_loss:.2f}.hdf5'
os.makedirs(cp_dir, exist_ok=True)
tb_cb = LRTensorBoard(log_dir=log_dir)
es_cb = callbacks.EarlyStopping(patience=2)
mc_cb = callbacks.ModelCheckpoint(cp_dir,
save_weights_only=True,
save_best_only=True)
lrs_cb = callbacks.LearningRateScheduler(scheduler, verbose=0)
cbs = [
#tb_cb,
#es_cb,
#mc_cb,
lrs_cb,
]
time.sleep(1)
x_train, y_train = train
tuner = build_search()
tuner.search(x_train,
y_train,
batch_size=64,
epochs=10,
validation_data=val,
callbacks=cbs,
verbose=0)
return tuner
def train(model, data, args):
# unpacking the data
(x_train, y_train), (x_test, y_test) = data
# callbacks
log = callbacks.CSVLogger(args.save_dir + '/log.csv')
checkpoint = callbacks.ModelCheckpoint(args.save_dir +
'weights-{epoch:02d}.h5',
monitor='val_capsnet_acc',
save_best_only=True,
save_weights_only=True,
verbose=1)
lr_decay = callbacks.LearningRateScheduler(
schedule=lambda epoch: args.lr * (args.lr_decay**epoch))
model.compile(optimizer=optimizers.Adam(lr=args.lr),
loss=[margin_loss, 'mse'],
loss_weights=[1., args.lam_recon],
metrics={'capsnet': 'accuracy'})
# Training without data augmentation:
model.fit((x_train, y_train), (y_train, x_train),
batch_size=args.batch_size,
epochs=args.epochs,
validation_data=((x_test, y_test), (y_test, x_test)),
callbacks=[log, checkpoint, lr_decay])
model.save_weights(args.save_dir + '/trained_model.h5')
print('Trained model saved to \'%s/trained_model.h5\'' % args.save_dir)
return model
def train_model(model: keras.Sequential, data, epochs):
logdir = f'logs/fit/{epochs}/' + \
datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = callbacks.TensorBoard(log_dir=logdir,
histogram_freq=1)
early_stop_callback = callbacks.EarlyStopping(monitor='val_accuracy',
patience=5,
min_delta=0.0001,
restore_best_weights=True,
verbose=0)
learning_rate_callback = callbacks.LearningRateScheduler(
lambda epoch, lr: lr if epoch < 2 else lr * 0.9, verbose=0)
save_callback = callbacks.ModelCheckpoint(
filepath='logs/model' + datetime.now().strftime("%Y%m%d-%H%M%S"),
save_best_only=True)
x_train = to_dict(data['training_data'])
y_train = tf.convert_to_tensor(data['training_labels'])
x_val = to_dict(data['test_data'])
y_val = tf.convert_to_tensor(data['test_labels'])
training_history = model.fit(x_train,
y_train,
epochs=epochs,
validation_data=(x_val, y_val),
callbacks=[
tensorboard_callback, early_stop_callback,
learning_rate_callback, save_callback
])
return training_history
def init_callbacks(self):
def schedule(epoch):
for step in self.config.trainer.lr_schedule:
if epoch < step.until_epoch:
return step.lr
self.callbacks.append(callbacks.LearningRateScheduler(schedule))
if self.config.trainer.tensorboard_enabled:
self.callbacks.append(callbacks.LambdaCallback(
on_epoch_begin=lambda epoch, loss: self.log_lr(epoch)
))
def _create_callbacks(self) -> List[callbacks.Callback]:
checkpoint_path = self._output_path.joinpath("checkpoint/checkpoint")
checkpoint = callbacks.ModelCheckpoint(
checkpoint_path, save_weights_only=True, save_best_only=True
)
log_path = self._output_path.joinpath("logs")
tensorboard = callbacks.TensorBoard(log_dir=log_path, histogram_freq=1)
early_stopping = callbacks.EarlyStopping(
monitor="val_loss", patience=15, mode="min"
)
lr_scheduler = callbacks.LearningRateScheduler(self._scheduler)
return [checkpoint, tensorboard, early_stopping, lr_scheduler]
def init_callbacks(self):
def schedule(epoch):
for step in self.config.trainer.lr_schedule:
if epoch < step.until_epoch:
return step.lr
self.callbacks.append(callbacks.LearningRateScheduler(schedule))
if "model_checkpoint" in self.config.trainer:
self.callbacks.append(
callbacks.ModelCheckpoint(
save_weights_only=True,
**self.config.trainer.model_checkpoint))
self.callbacks.append(
callbacks.LambdaCallback(
on_epoch_begin=lambda epoch, loss: self.log_lr(epoch)))
def get_callbacks(model_path,
save_weights_only=False,
lr_sched=None,
tensorboard_log_dir=None,
reduce_lr_on_plateau=False,
monitor='val_loss',
verbose=1):
"""Returns a list of callbacks used for training
Args:
model_path: (str) path for the h5 model file.
save_weights_only: (bool) if True, then only the model's weights
will be saved.
lr_sched (function): learning rate scheduler per epoch.
from deepcell.utils.train_utils.rate_scheduler.
tensorboard_log_dir (str): log directory for tensorboard.
monitor (str): quantity to monitor.
verbose (int): verbosity mode, 0 or 1.
Returns:
list: a list of callbacks to be passed to model.fit()
"""
cbs = [
callbacks.ModelCheckpoint(model_path,
monitor=monitor,
save_best_only=True,
verbose=verbose,
save_weights_only=save_weights_only),
]
if lr_sched:
cbs.append(callbacks.LearningRateScheduler(lr_sched))
if reduce_lr_on_plateau:
cbs.append(
callbacks.ReduceLROnPlateau(monitor=monitor,
factor=0.1,
patience=10,
verbose=verbose,
mode='auto',
min_delta=0.0001,
cooldown=0,
min_lr=0))
if tensorboard_log_dir:
cbs.append(callbacks.TensorBoard(log_dir=tensorboard_log_dir))
return cbs
def get_callbacks(self,
swa,
monitor="val_my_iou_metric",
mode="max",
model_prefix="Model"):
callback_list = [
callbacks.ModelCheckpoint("./keras.model",
monitor=monitor,
mode=mode,
save_best_only=True,
verbose=1), swa,
callbacks.LearningRateScheduler(
schedule=self._cosine_anneal_schedule, verbose=1)
]
return callback_list
def train_cnn_model(x_train, y_train):
x_train = array(x_train)
x_train = x_train.reshape((len(x_train), 3, int(len(x_train[0])/3), 1))
y_train = array(y_train)
#create model
cnn_model = Sequential()
cnn_model.add(Conv2D(64,
kernel_size=3,
activation='relu',
input_shape=(3,21,1),
padding='same'))
cnn_model.add(layers.BatchNormalization(1))
cnn_model.add(Conv2D(64,
kernel_size=3,
activation='relu',
padding='same'))
cnn_model.add(layers.BatchNormalization(1))
cnn_model.add(MaxPooling2D(2,2))
cnn_model.add(Flatten())
cnn_model.add(Dense(512, activation = 'relu'))
cnn_model.add(Dense(1, activation='sigmoid'))
# compile and fit
cnn_model.compile(optimizer='Adam',
loss='binary_crossentropy',
metrics=['acc',
metrics.AUC(),
metrics.FalseNegatives(),
metrics.Recall(),
metrics.Precision(),
metrics.FalseNegatives(),
metrics.TrueNegatives(),
metrics.FalsePositives(),
metrics.TruePositives()])
cnn_history = cnn_model.fit(x_train, y_train,
epochs=100,
batch_size=16,
validation_split=0.2,
callbacks=[callbacks.EarlyStopping(monitor='val_loss', patience=5),
callbacks.LearningRateScheduler(scheduler)])
print("finish training cnn model")
return cnn_model, cnn_history
def train_Keras(train_X, train_y, test_X, test_y, kwargs):
normalization = normalization_func()
num_classes = train_y.shape[-1]
norm_train_X = normalization.fit_transform(train_X)
norm_test_X = normalization.transform(test_X)
class_weight = train_y.shape[0] / np.sum(train_y, axis=0)
class_weight = num_classes * class_weight / class_weight.sum()
sample_weight = None
batch_size = max(2, len(norm_train_X) // 50)
print('reps : ', reps, ', weights : ', class_weight)
if num_classes == 2:
sample_weight = np.zeros((len(norm_train_X),))
sample_weight[train_y[:, 1] == 1] = class_weight[1]
sample_weight[train_y[:, 1] == 0] = class_weight[0]
class_weight = None
svc_model = LinearSVC(nfeatures=norm_train_X.shape[1:], **kwargs)
svc_model.create_keras_model(nclasses=num_classes, warming_up=False)
model = svc_model.model
optimizer = optimizer_class(lr=initial_lr)
model.compile(
loss=LinearSVC.loss_function('square_hinge', class_weight),
optimizer=optimizer,
metrics=[LinearSVC.accuracy]
)
model.fit(
norm_train_X, train_y, batch_size=batch_size,
epochs=epochs,
callbacks=[
callbacks.LearningRateScheduler(scheduler()),
],
validation_data=(norm_test_X, test_y),
class_weight=class_weight,
sample_weight=sample_weight,
verbose=verbose
)
model.normalization = normalization
return model
def main(input_dir, output_dir):
class_names = [path.basename(s) for s in glob(input_dir + "/*/")]
n_classes = len(class_names)
# image_gen = preprocessing.image.ImageDataGenerator(
# rescale=1.0 / 255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True
# )
image_gen = preprocessing.image.ImageDataGenerator(rescale=1.0 / 255)
train_gen = image_gen.flow_from_directory(
input_dir,
target_size=(224, 224),
batch_size=16,
class_mode="categorical", # "binary"
color_mode="rgb",
)
base_model = applications.mobilenet_v2.MobileNetV2(input_shape=(224, 224,
3),
include_top=False,
weights="imagenet",
pooling="max")
for layer in base_model.layers:
layer.trainable = False
y = base_model.output
y = layers.Dense(n_classes, activation="softmax")(y)
model = models.Model(inputs=base_model.inputs, outputs=y)
lr = 0.05
model.compile(
optimizer=optimizers.Adam(lr),
loss="categorical_crossentropy",
metrics=["accuracy", metrics.AUC()],
)
# print(model.summary())
model.fit(
train_gen,
validation_data=train_gen,
epochs=20,
callbacks=[
callbacks.ModelCheckpoint(output_dir, save_best_only=True),
callbacks.EarlyStopping(patience=2),
callbacks.LearningRateScheduler(
lambda epoch: lr * np.exp(-0.1 * (epoch - 1))),
callbacks.CSVLogger(path.join(output_dir, "metrics.csv")),
],
)
def train_Keras(train_X, train_y, test_X, test_y, kwargs):
normalization = normalization_func()
num_classes = train_y.shape[-1]
norm_train_X = normalization.fit_transform(train_X)
norm_test_X = normalization.transform(test_X)
class_weight = train_y.shape[0] / np.sum(train_y, axis=0)
class_weight = num_classes * class_weight / class_weight.sum()
sample_weight = None
batch_size = max(2, len(norm_train_X) // 50)
print('reps : ', reps, ', weights : ', class_weight)
if num_classes == 2:
sample_weight = np.zeros((len(norm_train_X),))
sample_weight[train_y[:, 1] == 1] = class_weight[1]
sample_weight[train_y[:, 1] == 0] = class_weight[0]
class_weight = None
model = three_layer_nn(nfeatures=norm_train_X.shape[1:], **kwargs)
optimizer = optimizer_class(lr=1e-3) # optimizers.adam(lr=1e-2)
model.compile(
loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['acc']
)
model.fit(
norm_train_X, train_y, batch_size=batch_size,
epochs=epochs,
callbacks=[
callbacks.LearningRateScheduler(scheduler()),
],
validation_data=(norm_test_X, test_y),
class_weight=class_weight,
sample_weight=sample_weight,
verbose=verbose
)
model.normalization = normalization
return model
def train_with_lr_decay(model):
model.compile(loss='sparse_categorical_crossentropy',
metrics=['accuracy'],
optimizer=optimizers.Adam(lr=1e-3))
# Callback for learning rate adjustment (see below)
lr_scheduler = callbacks.LearningRateScheduler(learning_rate_schedule)
# TensorBoard callback
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
logdir = os.path.join(log_root, f'{model.name}_{timestamp}')
tensorboard_callback = callbacks.TensorBoard(logdir, histogram_freq=1)
# Fit the model on the batches generated by datagen.flow()
model.fit(x_train,
y_train,
batch_size=128,
validation_data=(x_test, y_test),
epochs=150,
verbose=1,
callbacks=[lr_scheduler, tensorboard_callback])
def f_train_model(self, train_data, val_data, num_epochs=5, batch_size=64):
# model,inpx,inpy,model_weights):
'''
Train model. Returns just history.history
'''
def f_learnrate_sch(epoch, lr):
''' Module to schedule the learn rate'''
step = 10 ### learn rate is constant up to here
#if epoch>step: lr=lr*np.exp(-0.2*(epoch-10)) # Exponential decay after 10
if (epoch >= step and epoch % step == 0): lr = lr / 2.0
return lr
callbacks_lst = []
callbacks_lst.append(
callbacks.EarlyStopping(monitor='val_loss', patience=15,
verbose=1))
callbacks_lst.append(
callbacks.ModelCheckpoint(self.fname_model_wts,
save_best_only=True,
monitor='val_loss',
mode='min'))
callbacks_lst.append(
callbacks.LearningRateScheduler(f_learnrate_sch, verbose=1))
model = self.cnn_model
history = model.fit(
x=train_data.x,
y=train_data.y,
batch_size=batch_size,
epochs=num_epochs,
verbose=1,
callbacks=callbacks_lst,
#validation_split=cv_fraction,
validation_data=(val_data.x, val_data.y),
shuffle=True)
print("Number of parameters", model.count_params())
self.history = history.history
def train(self, epochs: int, lr: float, steps_per_epoch: int = 1):
"""
This function is used to Train the model, it uses Adam Optimizer to train, and it saves the weights of every
epoch in 'model_weights' dir, training steps_per_epoch=1 and val_steps=5 by default.
You can optionally set the following parameters:
param: epochs (NO of epochs to train the model)
param: lr (learning rate for the model)
param: steps_per_epoch (it defines steps per epoch for training data)
"""
if (self.modelType == 'tinyyolov4'):
self.optimizer = optimizers.Adam(learning_rate=lr)
self.model.compile(optimizer=self.optimizer,
loss_iou_type='ciou',
loss_verbose=0)
def lr_scheduler(epoch, lr):
return lr * tf.math.exp(-0.1)
self.model.fit(self.train_dataset,
epochs=epochs,
callbacks=[
callbacks.LearningRateScheduler(lr_scheduler,
verbose=1),
callbacks.TerminateOnNaN(),
callbacks.TensorBoard(histogram_freq=1,
log_dir="./logs"),
SaveWeightsCallback(yolo=self.model,
dir_path="./model_weights",
weights_type="yolo",
epoch_per_save=1),
],
validation_data=self.val_dataset,
validation_steps=self.val_steps,
steps_per_epoch=steps_per_epoch)
else:
raise RuntimeError('Invalid ModelType: Valid Type is YOLOv4')
def get_callbacks():
def lr_scheduler(epoch, lr):
init_lr = 0.01
cycle = 5
min_lr = 1e-5
if init_lr * (np.math.cos(np.pi / 2 / cycle *
(epoch - cycle *
(epoch // cycle)))) + min_lr < min_lr:
lr = init_lr
else:
lr = init_lr * (np.math.cos(np.pi / 2 / cycle *
(epoch - cycle *
(epoch // cycle)))) + min_lr
return lr
fit_callbacks = [
callbacks.LearningRateScheduler(lr_scheduler),
callbacks.ModelCheckpoint(monitor='val_out_views_acc',
save_best_only=True,
filepath=f'models/model.h5')
]
return fit_callbacks
def train_lstm_model(x_train, y_train):
x_train = array(x_train)
x_train = x_train.reshape((len(x_train), 1, len(x_train[0])))
print("x_train.shape", x_train.shape)
print(x_train[0])
y_train = array(y_train)
print("y_train.shape", y_train.shape)
# imrpove log: use batch size 16 and add one more lstm layer
lstm_model = Sequential()
lstm_model.add(LSTM(16,
input_shape=(1, 63),
return_sequences=True))
lstm_model.add(LSTM(16, ))
lstm_model.add(layers.Dense(1, activation='sigmoid'))
lstm_model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['acc',
metrics.AUC(),
metrics.FalseNegatives(),
metrics.Recall(),
metrics.Precision(),
metrics.FalseNegatives(),
metrics.TrueNegatives(),
metrics.FalsePositives(),
metrics.TruePositives()])
lstm_history = lstm_model.fit(x_train, y_train,
epochs=100,
batch_size=16,
validation_split=0.2,
callbacks=[callbacks.EarlyStopping(monitor='val_loss', patience=5),
callbacks.LearningRateScheduler(scheduler)])
print("finish training lstm model")
return lstm_model, lstm_history
logger.debug(f"{history_cb.dataframe.index.size=}")
logger.debug(f"{history_cb.last_epoch=}")
history_plot_cb = tomo2seg_callbacks.HistoryPlot(
history_callback=history_cb,
save_path=t2s_model.train_history_plot_wip_path
)
logger.debug(f"{history_plot_cb=}")
# [manual-input]
n_epochs = 200
initial_lr = optimizer.lr.numpy()
lr_schedule_cb = keras_callbacks.LearningRateScheduler(
schedule=(
# schedule := tomo2seg_schedule.get_schedule00()
schedule := tomo2seg_schedule.LinSpaceSchedule(
offset_epoch=0, wait=100, start=initial_lr, stop=initial_lr / 10, n_between=100
)
),
verbose=2,
)
# todo plot schedule
logger.info(f"{lr_schedule_cb.schedule.range=}")
callbacks = [
keras_callbacks.TerminateOnNaN(),
keras_callbacks.ModelCheckpoint(
t2s_model.autosaved2_model_path_str,
def main():
dataset = load_dataset()
train_data = np.asarray(dataset['train']['data'])
train_labels = dataset['train']['label']
num_classes = len(np.unique(train_labels))
test_data = np.asarray(dataset['test']['data'])
test_labels = dataset['test']['label']
train_labels = to_categorical(train_labels, num_classes=num_classes)
test_labels = to_categorical(test_labels, num_classes=num_classes)
generator = dataset['generator']
fs_generator = dataset['fs_generator']
generator_kwargs = {
'batch_size': batch_size
}
print('reps : ', reps)
name = 'mnist_' + fs_network + '_r_' + str(regularization)
print(name)
model_kwargs = {
'nclasses': num_classes,
'regularization': regularization
}
total_features = int(np.prod(train_data.shape[1:]))
fs_filename = directory + fs_network + '_trained_model.h5'
classifier_filename = directory + classifier_network + '_trained_model.h5'
if not os.path.isdir(directory):
os.makedirs(directory)
if not os.path.exists(fs_filename) and warming_up:
np.random.seed(1001)
tf.set_random_seed(1001)
model = getattr(network_models, fs_network)(input_shape=train_data.shape[1:], **model_kwargs)
print('training_model')
model.fit_generator(
generator.flow(train_data, train_labels, **generator_kwargs),
steps_per_epoch=train_data.shape[0] // batch_size, epochs=110,
callbacks=[
callbacks.LearningRateScheduler(scheduler())
],
validation_data=(test_data, test_labels),
validation_steps=test_data.shape[0] // batch_size,
verbose=verbose
)
model.save(fs_filename)
del model
K.clear_session()
for e2efs_class in e2efs_classes:
nfeats = []
accuracies = []
times = []
cont_seed = 0
for factor in [.05, .1, .25, .5]:
n_features = int(total_features * factor)
n_accuracies = []
n_times = []
for r in range(reps):
print('factor : ', factor, ' , rep : ', r)
np.random.seed(cont_seed)
tf.set_random_seed(cont_seed)
cont_seed += 1
mask = (np.std(train_data, axis=0) > 1e-3).astype(int).flatten()
classifier = load_model(fs_filename) if warming_up else getattr(network_models, fs_network)(input_shape=train_data.shape[1:], **model_kwargs)
e2efs_layer = e2efs_class(n_features, input_shape=train_data.shape[1:], kernel_initializer=initializers.constant(mask))
model = e2efs_layer.add_to_model(classifier, input_shape=train_data.shape[1:])
optimizer = custom_optimizers.E2EFS_Adam(e2efs_layer=e2efs_layer, lr=1e-3) # optimizers.adam(lr=1e-2)
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc'])
model.fs_layer = e2efs_layer
model.classifier = classifier
model.summary()
start_time = time.time()
model.fit_generator(
fs_generator.flow(train_data, train_labels, **generator_kwargs),
steps_per_epoch=train_data.shape[0] // batch_size, epochs=20000,
callbacks=[
E2EFSCallback(verbose=verbose)
],
validation_data=(test_data, test_labels),
validation_steps=test_data.shape[0] // batch_size,
verbose=verbose
)
fs_rank = np.argsort(K.eval(model.heatmap))[::-1]
mask = np.zeros(train_data.shape[1:])
mask.flat[fs_rank[:n_features]] = 1.
# mask = K.eval(model.fs_kernel).reshape(train_data.shape[1:])
n_times.append(time.time() - start_time)
print('nnz : ', np.count_nonzero(mask))
del model
K.clear_session()
model = load_model(classifier_filename) if warming_up else getattr(network_models, classifier_network)(
input_shape=train_data.shape[1:], **model_kwargs)
optimizer = optimizers.Adam(lr=1e-2) # optimizers.adam(lr=1e-2)
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc'])
model.fit_generator(
generator.flow(mask * train_data, train_labels, **generator_kwargs),
steps_per_epoch=train_data.shape[0] // batch_size, epochs=80,
callbacks=[
callbacks.LearningRateScheduler(scheduler()),
],
validation_data=(mask * test_data, test_labels),
validation_steps=test_data.shape[0] // batch_size,
verbose=verbose
)
n_accuracies.append(model.evaluate(mask * test_data, test_labels, verbose=0)[-1])
del model
K.clear_session()
print(
'n_features : ', n_features, ', acc : ', n_accuracies, ', time : ', n_times
)
accuracies.append(n_accuracies)
nfeats.append(n_features)
times.append(n_times)
output_filename = directory + fs_network + '_' + classifier_network + '_' + e2efs_class.__name__ + \
'_results_warming_' + str(warming_up) + '.json'
try:
with open(output_filename) as outfile:
info_data = json.load(outfile)
except:
info_data = {}
if name not in info_data:
info_data[name] = []
info_data[name].append(
{
'regularization': regularization,
'reps': reps,
'classification': {
'n_features': nfeats,
'accuracy': accuracies,
'times': times
}
}
)
with open(output_filename, 'w') as outfile:
json.dump(info_data, outfile)
def fit(self, generator, X, Y=None, val_X=None, val_Y=None, num_epochs=100, batch_size=None, start_temp=10.0,
min_temp=0.1, tryout_limit=1, class_weight=None):
if Y is None:
Y = X
assert len(X) == len(Y)
validation_data = None
if val_X is not None and val_Y is not None:
assert len(val_X) == len(val_Y)
validation_data = (val_X, val_Y)
if batch_size is None:
batch_size = max(len(X) // 256, 16)
steps_per_epoch = (len(X) + batch_size - 1) // batch_size
for i in range(tryout_limit):
K.set_learning_phase(1)
inputs = layers.Input(shape=X.shape[1:])
x = layers.Flatten()(inputs)
alpha = np.exp(np.log(min_temp / start_temp) / (num_epochs * steps_per_epoch))
self.concrete_select = ConcreteSelect(self.K, start_temp, min_temp, alpha, name='concrete_select')
selected_features = self.concrete_select(x)
outputs = self.output_function(selected_features)
self.model = models.Model(inputs, outputs)
self.model.compile(
loss='categorical_crossentropy',
optimizer=optimizers.Adam(1e-3),
metrics=['acc']
)
print(self.model.summary())
stopper_callback = StopperCallback()
hist = self.model.fit_generator(
generator.flow(X, Y, batch_size=batch_size),
steps_per_epoch=X.shape[0] // batch_size, epochs=80,
callbacks=[StopperCallback(), callbacks.LearningRateScheduler(scheduler(80, .1))],
validation_data=validation_data,
verbose=2
# validation_steps=test_data.shape[0] // batch_size,
) # , validation_freq = 10)
if K.get_value(
K.mean(K.max(K.softmax(self.concrete_select.logits, axis=-1)))) >= stopper_callback.mean_max_target:
break
num_epochs *= 2
self.probabilities = K.get_value(K.softmax(self.model.get_layer('concrete_select').logits))
self.indices = K.get_value(K.argmax(self.model.get_layer('concrete_select').logits))
return self
def main():
dataset = load_dataset()
train_data = np.asarray(dataset['train']['data'])
train_labels = dataset['train']['label']
num_classes = len(np.unique(train_labels))
# mask = (np.std(train_data, axis=0) > 5e-3).astype(int).flatten()
test_data = np.asarray(dataset['test']['data'])
test_labels = dataset['test']['label']
train_labels = to_categorical(train_labels, num_classes=num_classes)
test_labels = to_categorical(test_labels, num_classes=num_classes)
generator = dataset['generator']
generator_fs = dataset['generator_fs']
generator_kwargs = {
'batch_size': batch_size
}
print('reps : ', reps)
name = 'mnist_' + classifier_network + '_r_' + str(regularization)
print(name)
model_kwargs = {
'nclasses': num_classes,
# 'regularization': regularization
}
total_features = int(np.prod(train_data.shape[1:]))
model_filename = directory + classifier_network + '_trained_model.h5'
fs_filename = directory + fs_network + '_trained_model.h5'
for network_name in (fs_network, classifier_network):
filename = directory + network_name + '_trained_model.h5'
if not os.path.isdir(directory):
os.makedirs(directory)
if not os.path.exists(filename) and warming_up:
np.random.seed(1001)
tf.set_random_seed(1001)
model = getattr(network_models, network_name)(input_shape=train_data.shape[1:], **model_kwargs)
print('training_model')
model.fit_generator(
generator.flow(train_data, train_labels, **generator_kwargs),
steps_per_epoch=train_data.shape[0] // batch_size, epochs=80,
callbacks=[
callbacks.LearningRateScheduler(scheduler())
],
validation_data=(test_data, test_labels),
validation_steps=test_data.shape[0] // batch_size,
verbose=verbose
)
model.save(filename)
del model
K.clear_session()
nfeats = []
accuracies = []
times = []
cont_seed = 0
for i, factor in enumerate([.05, .1, .25, .5]):
n_features = int(total_features * factor)
nfeats.append(n_features)
n_accuracies = []
n_times = []
for r in range(reps):
heatmap = np.zeros(np.prod(train_data.shape[1:]))
np.random.seed(cont_seed)
tf.set_random_seed(cont_seed)
cont_seed += 1
classifier = getattr(network_models, fs_network)(input_shape=(n_features, ), **model_kwargs)
cae_model = ConcreteAutoencoderFeatureSelector(K=n_features, output_function=classifier)
start_time = time.time()
cae_model.fit(generator_fs, train_data, train_labels, test_data, test_labels, batch_size=batch_size)
fs_rank = cae_model.indices
n_times.append(time.time() - start_time)
del cae_model.model
K.clear_session()
mask = np.zeros(train_data.shape[1:])
mask.flat[fs_rank] = 1.
np.random.seed(cont_seed)
tf.set_random_seed(cont_seed)
cont_seed += 1
model = load_model(model_filename) if warming_up else getattr(network_models, classifier_network)(input_shape=train_data.shape[1:], **model_kwargs)
optimizer = optimizers.Adam(learning_rate=1e-3) # optimizers.adam(lr=1e-2)
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc'])
model.fit_generator(
generator.flow(mask * train_data, train_labels, **generator_kwargs),
steps_per_epoch=train_data.shape[0] // batch_size, epochs=80,
callbacks=[
callbacks.LearningRateScheduler(scheduler()),
],
validation_data=(mask * test_data, test_labels),
validation_steps=test_data.shape[0] // batch_size,
verbose=verbose
)
acc = model.evaluate(mask * test_data, test_labels, verbose=0)[-1]
n_accuracies.append(acc)
del model
K.clear_session()
print(
'n_features : ', n_features, ', acc : ', acc, ', time : ', n_times[-1]
)
accuracies.append(n_accuracies)
times.append(n_times)
output_filename = directory + fs_network + '_' + classifier_network + \
'_CAE_results_warming_' + str(warming_up) + '.json'
try:
with open(output_filename) as outfile:
info_data = json.load(outfile)
except:
info_data = {}
if name not in info_data:
info_data[name] = []
info_data[name].append(
{
'regularization': regularization,
'reps': reps,
'classification': {
'n_features': nfeats,
'accuracy': accuracies,
'times': times
}
}
)
with open(output_filename, 'w') as outfile:
json.dump(info_data, outfile)
def train_Keras(train_X,
train_y,
test_X,
test_y,
kwargs,
e2efs_class=None,
n_features=None,
e2efs_kwargs=None,
T=300,
extra=300):
normalization = normalization_func()
num_classes = train_y.shape[-1]
norm_train_X = normalization.fit_transform(train_X)
norm_test_X = normalization.transform(test_X)
batch_size = max(2, len(train_X) // 50)
class_weight = train_y.shape[0] / np.sum(train_y, axis=0)
class_weight = num_classes * class_weight / class_weight.sum()
sample_weight = None
print('reps : ', reps, ', weights : ', class_weight)
if num_classes == 2:
sample_weight = np.zeros((len(norm_train_X), ))
sample_weight[train_y[:, 1] == 1] = class_weight[1]
sample_weight[train_y[:, 1] == 0] = class_weight[0]
class_weight = None
classifier = three_layer_nn(nfeatures=norm_train_X.shape[1:], **kwargs)
model_clbks = [
callbacks.LearningRateScheduler(
scheduler(extra=0 if e2efs_class is None else extra)),
]
if e2efs_class is not None:
e2efs_layer = e2efs_class(n_features,
input_shape=norm_train_X.shape[1:],
**e2efs_kwargs)
model = e2efs_layer.add_to_model(classifier,
input_shape=norm_train_X.shape[1:])
model_clbks.append(
clbks.E2EFSCallback(factor_func=e2efs_factor(T),
units_func=None,
verbose=verbose))
else:
model = classifier
e2efs_layer = None
optimizer = optimizer_class(e2efs_layer, lr=1e-3)
model_epochs = epochs
if e2efs_class is not None:
model_epochs += extra_epochs
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['acc'])
if e2efs_class is not None:
model.fs_layer = e2efs_layer
model.heatmap = e2efs_layer.moving_heatmap
model.fit(norm_train_X,
train_y,
batch_size=batch_size,
epochs=model_epochs,
callbacks=model_clbks,
validation_data=(norm_test_X, test_y),
class_weight=class_weight,
sample_weight=sample_weight,
verbose=verbose)
model.normalization = normalization
return model
#evaluation (if the flag -l was used)/training
if args.load:
saved_weights = args.load
model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
print("Loading weights {}".format(saved_weights))
model.load_weights(saved_weights)
history = model.evaluate(test_images, test_labels, batch_size=batch_dim, verbose=2)
else:
model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
epochs = 500
lr_schedule = callbacks.LearningRateScheduler(lambda epoch: learning_rate * (0.5 ** (epoch // 100)), verbose=0)
terminate_on_NaN = callbacks.TerminateOnNaN()
earlystopping = callbacks.EarlyStopping(monitor='val_accuracy', min_delta=0.001, patience=45, verbose=1, mode='max', baseline=None, restore_best_weights=False)
callbacks_list = list(filter(None, [lr_schedule, terminate_on_NaN, earlystopping]))
tic_training = datetime.datetime.now()
history = model.fit(train_images, train_labels, batch_size=batch_dim, epochs=epochs, validation_data=(test_images, test_labels), shuffle=True, verbose=2, callbacks=callbacks_list)
toc_training = datetime.datetime.now()
print("Training, elapsed: {} minutes.".format((toc_training - tic_training).seconds//60))
def get_filename(type):
"""Computes the filename for the outputs of the training
(checks whether the file already exists, in that case adds a number to the filename
to avoid overriding it)
def lr_schedule(epoch, lr):
return lr * decay_rate
model.compile(Adam(init_learn_rate),
'mean_squared_error',
metrics=['mean_absolute_error'])
history = model.fit(
train_loader,
validation_data=val_loader,
epochs=args.num_epochs,
verbose=True,
shuffle=False,
callbacks=[
LRLogger(),
EpochTimeLogger(),
cb.LearningRateScheduler(lr_schedule),
cb.ModelCheckpoint(os.path.join(test_dir, 'best_model.h5'),
save_best_only=True),
cb.EarlyStopping(patience=128, restore_best_weights=True),
cb.CSVLogger(os.path.join(test_dir, 'train_log.csv')),
cb.TerminateOnNaN()
])
# Run on the validation set and assess statistics
y_true = np.hstack([x[1].numpy()[:, 0] for x in iter(test_loader)])
y_pred = np.squeeze(model.predict(test_loader))
pd.DataFrame({
'true': y_true,
'pred': y_pred
}).to_csv(os.path.join(test_dir, 'test_results.csv'), index=False)
plot_history(self.history,
acc_name='plots/char_keras_accuracy.png',
loss_name='plots/char_keras_loss.png')
plotter = PlotterCallback()
def schedule(epoch, lr):
decay = 0.08
if (epoch + 1) % 8 == 0:
return lr * (1 - decay)
return lr
scheduler = callbacks.LearningRateScheduler(schedule)
callbacks_list = [checkpoint, generator, scheduler, plotter]
start = time.time()
history = model.fit(train_x,
train_y,
epochs=480,
batch_size=128,
validation_data=(test_x, test_y),
callbacks=callbacks_list)
end = time.time()
print('#' * 80)
print(f'Elapsed Training Time: {end - start:.02f}s')
print('#' * 80)
# In[ ]:
checkpoint = callbacks.ModelCheckpoint(
"./models/20200102-val_loss.{val_loss:.2f}.h5",
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=False,
mode='auto',
period=1)
# checkpoint = callbacks.ModelCheckpoint("./models/loss.{loss:.2f}-acc.{acc:.2f}-val_loss.{val_loss:.2f}-val_acc.{val_acc:.2f}.h5", monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1)
# early = EarlyStopping(monitor='val_acc', min_delta=0, patience=3, verbose=1, mode='auto')
# reduce_lr = callbacks.ReduceLROnPlateau(monitor='acc', factor=0.1, patience=2, min_lr=0.000001)
learningRateScheduler = callbacks.LearningRateScheduler(lr_decay)
tensorboard = callbacks.TensorBoard(log_dir='./logs')
# In[ ]:
history = model.fit_generator(
generator=train_generator,
steps_per_epoch=ceil(len(train_labels) / batch_size),
validation_data=test_generator,
validation_steps=ceil(len(test_labels) / batch_size),
epochs=nb_epochs,
callbacks=[checkpoint, tensorboard, learningRateScheduler],
use_multiprocessing=True,
workers=6,
verbose=1)