def fit_tabnet(params, X_train, Y_train, X_val, Y_val, model_path):
K.clear_session()
model = TabNetRegressor(num_regressors=n_classes,
num_features=n_features,
**params)
model.compile(
optimizer=AdaBeliefOptimizer(learning_rate=lr),
# loss=tf.keras.losses.MeanSquaredLogarithmicError(), # MSLE
loss=root_mean_squared_logarithmic_error, # RMSLE
)
callbacks = build_callbacks(model_path, factor=factor, patience=patience)
history = model.fit(
X_train,
Y_train,
batch_size=batch_size,
callbacks=callbacks,
validation_data=(X_val, Y_val),
**fit_params,
)
model.load_weights(model_path)
return model
model = model_factory_function(
name=t2s_model.name,
**model_factory_kwargs
)
logger.info("Compiling the model.")
# [manual-input]
# using the avg jaccard is dangerous if one of the classes is too
# underrepresented because it's jaccard will be unstable
# to be verified!
loss = tomo2seg_losses.jaccard2_flat
# optimizer = optimizers.Adam(lr=.003)
optimizer = AdaBeliefOptimizer(learning_rate=1e-3, epsilon=1e-14, rectify=True)
metrics = []
logger.debug(f"{loss=}")
logger.debug(f"{optimizer=}")
logger.debug(f"{metrics=}")
model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
if not args.train_mode.is_continuation:
logger.info(f"Saving the model at {t2s_model.model_path=}.")
model.save(t2s_model.model_path)
logger.info(f"Writing the model summary at {t2s_model.summary_path=}.")
ReLU(),
MaxPooling2D(pool_size=(2, 2), strides=(2, 2)),
Conv2D(256, kernel_size=3, strides=(1, 1)),
BatchNormalization(),
ReLU(),
GlobalAveragePooling2D(),
Dropout(0.5),
Flatten(name="flatten"),
Dense(512),
Dropout(0.2),
Dense(256),
Dropout(0.2),
Dense(2, activation='softmax')
])
optimizer = AdaBeliefOptimizer(learning_rate=1e-3, epsilon=1e-6, rectify=False)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer='adam',
metrics=["accuracy"])
checkpoint = ModelCheckpoint('model-{epoch:03d}.model',
monitor='val_loss',
verbose=0,
save_best_only=True,
mode='auto')
train = model.fit_generator(train_augmentator.flow(out_x, out_y,
batch_size=64),
epochs=40,
validation_data=validation_augmentator.flow(
def objective(trial):
standard_object = EvaluateMNIST(
train_validate_images=train_images,
train_validate_labels=train_labels,
validation_data_proportion=(1-JUN_SHAO_TRAINING_PROPORTION),
early_stopping_significant_delta=EARLY_STOPPING_SIGNIFICANT_DELTA,
early_stopping_patience=EARLY_STOPPING_PATIENCE_PARAMETER,
verbosity=VERBOSITY_LEVEL_FOR_TENSORFLOW,
seed=0,
)
# Generate hyper-parameters
standard_object.batch_size_base_two_logarithm = trial.suggest_int(
'batch_size_base_two_logarithm',
0,
MAXIMUM_BATCH_SIZE_POWER_OF_TWO,
)
standard_object.adam_learning_rate = trial.suggest_uniform(
'adam_learning_rate',
0,
2,
)
standard_object.adam_beta_1 = trial.suggest_uniform(
'adam_beta_1',
0,
1,
)
standard_object.adam_beta_2 = trial.suggest_uniform(
'adam_beta_2',
0,
1,
)
standard_object.adam_epsilon = trial.suggest_uniform(
'adam_epsilon',
0,
1
)
print('\nThis trial will use the following hyper-parameters:')
for key, val in trial.params.items():
print('{}: {}'.format(key, val))
if trial.number > 0:
print('in an attempt to improve on best objective function score of {:0.5f}\n'.format(study.best_trial.value))
print('\n\n')
# Add early stopping callback
standard_object.append_early_stopper_callback()
# Train and validate using hyper-parameters generated above
clear_session()
classifier_model = models.Sequential()
classifier_model.add(layers.Conv2D(
8,
(2, 2),
activation='relu',
input_shape=(28, 28, 1),
))
classifier_model.add(layers.MaxPooling2D((2, 2), strides=2))
classifier_model.add(layers.Conv2D(
8,
(2, 2),
activation='relu',
))
classifier_model.add(layers.MaxPooling2D((2, 2), strides=2))
classifier_model.add(layers.Flatten())
classifier_model.add(layers.Dense(10, activation='softmax'))
standard_object.optimizer = AdaBeliefOptimizer(
learning_rate=standard_object.adam_learning_rate,
beta_1=standard_object.adam_beta_1,
beta_2=standard_object.adam_beta_2,
epsilon=standard_object.adam_epsilon,
rectify=False, # recommended by developer
amsgrad=False, # this was just another attempt to make Adam converge
)
classifier_model.compile(
optimizer=standard_object.optimizer,
loss=CategoricalCrossentropy(),
metrics=[CategoricalAccuracy()],
)
standard_object.set_batch_size(standard_object.batch_size_base_two_logarithm)
standard_object.stratified_split_for_training_and_validation()
set_seed(standard_object.seed)
classifier_model.fit(
standard_object.train_split_images,
standard_object.train_split_labels,
epochs=MAXIMUM_EPOCHS_TO_TRAIN,
validation_data=standard_object.validate_split_data,
verbose=VERBOSITY_LEVEL_FOR_TENSORFLOW,
callbacks=standard_object.callbacks,
batch_size=standard_object.batch_size,
)
# Evaluate performance on test data and report score
test_metrics = classifier_model.evaluate(
test_images,
test_labels,
batch_size=standard_object.batch_size,
)
test_results_dict = {out: test_metrics[i] for i, out in enumerate(classifier_model.metrics_names)}
return(test_results_dict['categorical_accuracy'])
def main(args):
exp = Experiment(args, ignore=('epochs', 'resume'))
print(exp)
np.random.seed(args.seed)
tf.random.set_seed(args.seed)
data = load_datasets(args.data)
# TRAIN/VAL/TEST SPLIT
if args.split == 'subjects': # by SUBJECTS
val_subjects = (6, 9, 11, 13, 16, 28, 30, 48, 49)
test_subjects = (3, 4, 19, 38, 45, 46, 51, 52)
train_data = data[~data['sub'].isin(val_subjects + test_subjects)]
val_data = data[data['sub'].isin(val_subjects)]
test_data = data[data['sub'].isin(test_subjects)]
elif args.split == 'random': # 70-20-10 %
train_data, valtest_data = train_test_split(data,
test_size=.3,
shuffle=True)
val_data, test_data = train_test_split(valtest_data, test_size=.33)
lengths = map(len, (data, train_data, val_data, test_data))
print("Total: {} - Train / Val / Test: {} / {} / {}".format(*lengths))
x_shape = (args.resolution, args.resolution, 1)
y_shape = (args.resolution, args.resolution, 1)
train_gen, _ = get_loader(train_data,
batch_size=args.batch_size,
shuffle=True,
augment=True,
x_shape=x_shape)
val_gen, val_categories = get_loader(val_data,
batch_size=args.batch_size,
x_shape=x_shape)
# test_gen, test_categories = get_loader(test_data, batch_size=1, x_shape=x_shape)
log = exp.path_to('log.csv')
# weights_only checkpoints
best_weights_path = exp.path_to('best_weights.h5')
best_mask_weights_path = exp.path_to('best_weights_mask.h5')
# whole model checkpoints
best_ckpt_path = exp.path_to('best_model.h5')
last_ckpt_path = exp.path_to('last_model.h5')
if args.resume and os.path.exists(last_ckpt_path):
custom_objects = {
'AdaBeliefOptimizer': AdaBeliefOptimizer,
'iou_coef': evaluate.iou_coef,
'dice_coef': evaluate.dice_coef,
'hard_swish': hard_swish
}
model = tf.keras.models.load_model(last_ckpt_path,
custom_objects=custom_objects)
optimizer = model.optimizer
initial_epoch = len(pd.read_csv(log))
else:
config = vars(args)
model = build_model(x_shape, y_shape, config)
optimizer = AdaBeliefOptimizer(learning_rate=args.lr,
print_change_log=False)
initial_epoch = 0
model.compile(optimizer=optimizer,
loss='binary_crossentropy',
metrics={
'mask': [evaluate.iou_coef, evaluate.dice_coef],
'tags': 'binary_accuracy'
})
model_stopped_file = exp.path_to('early_stopped.txt')
need_training = not os.path.exists(
model_stopped_file) and initial_epoch < args.epochs
if need_training:
best_checkpointer = ModelCheckpoint(best_weights_path,
monitor='val_loss',
save_best_only=True,
save_weights_only=True)
best_mask_checkpointer = ModelCheckpoint(best_mask_weights_path,
monitor='val_mask_dice_coef',
mode='max',
save_best_only=True,
save_weights_only=True)
last_checkpointer = ModelCheckpoint(last_ckpt_path,
save_best_only=False,
save_weights_only=False)
logger = CSVLogger(log, append=args.resume)
progress = TqdmCallback(verbose=1,
initial=initial_epoch,
dynamic_ncols=True)
early_stop = tf.keras.callbacks.EarlyStopping(
monitor='val_mask_dice_coef', mode='max', patience=100)
callbacks = [
best_checkpointer, best_mask_checkpointer, last_checkpointer,
logger, progress, early_stop
]
model.fit(train_gen,
epochs=args.epochs,
callbacks=callbacks,
initial_epoch=initial_epoch,
steps_per_epoch=len(train_gen),
validation_data=val_gen,
validation_steps=len(val_gen),
verbose=False)
if model.stop_training:
open(model_stopped_file, 'w').close()
tf.keras.models.save_model(model,
best_ckpt_path,
include_optimizer=False)
# evaluation on test set
evaluate.evaluate(exp, force=need_training)
# save best snapshot in SavedModel format
model.load_weights(best_mask_weights_path)
best_savedmodel_path = exp.path_to('best_savedmodel')
model.save(best_savedmodel_path, save_traces=True)
# export to tfjs (Layers model)
tfjs_model_dir = exp.path_to('tfjs')
tfjs.converters.save_keras_model(model, tfjs_model_dir)
def main(args):
physical_devices = tf.config.list_physical_devices('GPU')
if len(physical_devices) > 0 and args.memory_growth:
for device in physical_devices:
tf.config.experimental.set_memory_growth(device, True)
print('{} memory growth: {}'.format(
device, tf.config.experimental.get_memory_growth(device)))
else:
print("Not enough GPU hardware devices available")
# Create datasets
train_dataset, test_dataset = create_dataset(args)
# Get image shape
source_image, target_image = next(iter(train_dataset))
source_shape = source_image.shape[1:]
target_shape = target_image.shape[1:]
# Create model
cut = CUT_model(source_shape,
target_shape,
cut_mode=args.mode,
impl=args.impl,
norm_layer='instance',
use_antialias=args.use_antialias,
ndf=args.ndf,
ngf=args.ngf,
resnet_blocks=4,
downsample_blocks=3,
netF_units=256,
netF_num_patches=256,
nce_layers=[0, 3, 4, 5, 6],
use_diffaugment=args.use_diffaugment,
gan_mode=args.gan_mode,
vgg_lambda=args.vgg_lambda,
nce_lambda=args.nce_lambda,
use_noise=True)
cut.summary()
# Define learning rate schedule
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate=args.lr,
decay_steps=args.lr_decay_step,
decay_rate=args.lr_decay_rate,
staircase=True)
if args.optimizer == 'adam':
G_optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule,
beta_1=args.beta_1,
beta_2=args.beta_2)
F_optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule,
beta_1=args.beta_1,
beta_2=args.beta_2)
D_optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule,
beta_1=args.beta_1,
beta_2=args.beta_2)
elif args.optimizer == 'adaBelief':
G_optimizer = AdaBeliefOptimizer(learning_rate=lr_schedule,
beta_1=args.beta_1,
beta_2=args.beta_2,
epsilon=1e-12,
rectify=False)
F_optimizer = AdaBeliefOptimizer(learning_rate=lr_schedule,
beta_1=args.beta_1,
beta_2=args.beta_2,
epsilon=1e-12,
rectify=False)
D_optimizer = AdaBeliefOptimizer(learning_rate=lr_schedule,
beta_1=args.beta_1,
beta_2=args.beta_2,
epsilon=1e-12,
rectify=False)
else:
raise Exception('Invalid optimizer')
# Compile model
cut.compile(
G_optimizer=G_optimizer,
F_optimizer=F_optimizer,
D_optimizer=D_optimizer,
)
# Restored from previous checkpoints, or initialize checkpoints from scratch
if args.ckpt is not None:
latest_ckpt = tf.train.latest_checkpoint(args.ckpt)
cut.load_weights(latest_ckpt)
initial_epoch = int(latest_ckpt[-3:])
print(f"Restored from {latest_ckpt}.")
else:
initial_epoch = 0
print("Initializing from scratch...")
# Define the folders to store output information
result_dir = f'{args.out_dir}/images'
checkpoint_dir = f'{args.out_dir}/checkpoints'
log_dir = f'{args.out_dir}/logs/{datetime.datetime.now().strftime("%Y%m%d-%H%M%S")}'
# Create validating callback to generate output image every epoch
plotter_callback = GANMonitor(cut.netG, test_dataset, result_dir)
# Create checkpoint callback to save model's checkpoints every n epoch (default 5)
# Use period to save every n epochs, use save_freq to save every n batches
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_dir + '/{epoch:03d}',
period=args.save_n_epoch,
verbose=1)
# Create tensorboard callback to log losses every epoch
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir)
# Train cut model
cut.fit(
train_dataset,
epochs=args.epochs,
initial_epoch=initial_epoch,
callbacks=[
plotter_callback,
checkpoint_callback,
# tensorboard_callback
],
workers=args.n_workers,
steps_per_epoch=args.steps_per_epoch,
verbose=1)
}
return {'content': content_dict, 'style': style_dict}
extractor = StyleContentModel(style_layers, content_layers)
style_targets = extractor(style_image)['style']
content_targets = extractor(content_image)['content']
image = tf.Variable(content_image)
# optimal: 0.05
#opt = tf.keras.optimizers.Adam(learning_rate=0.05)
opt = AdaBeliefOptimizer(learning_rate=0.05,
rectify=False,
print_change_log=False)
style_weight, epochs = get_parameters(style_weight_select, quality_select)
content_weight = 1.0
total_variation_weight = style_weight / 2000
print('{ ' + '"type": "initialized"' + ' }', flush=True)
start = time.time()
for n in range(epochs):
train_step(image)
print('{ ' + '"type": "progress", "value": "{}"'.format(
round((n + 1) / epochs * 100)) + '}',
flush=True)
