def _create_model(self, inputs: List[Input], outputs: List[Tensor]) -> None:
"""
Creation of model
Parameters
----------
inputs: List[Input]
inputs to the context and target branch
output: List[Reshape]
output(s) of the (context) and target branch
"""
# initilization of keras model
super().__init__(inputs, outputs)
# set losses and loss weigths
losses = (
{
"reshape_target": "categorical_crossentropy",
"reshape_context": "categorical_crossentropy",
}
if self._multi_loss
else {"reshape_target": "categorical_crossentropy"}
)
loss_weights = (
{
"reshape_target": self._loss_weights[0],
"reshape_context": self._loss_weights[1],
}
if self._multi_loss
else {"reshape_target": self._loss_weights[0]}
)
sample_weights_mode = (
["temporal", "temporal"] if self._multi_loss else "temporal"
)
# compile model
self.compile(
optimizer=self._opt(),
loss=losses,
sample_weight_mode=sample_weights_mode,
loss_weights=loss_weights,
weighted_metrics=["accuracy", CategoricalCrossentropy()],
)
def do_training(initial_learning_rate=0.1):
gids = get_gids_from_database("unet")
training_gen, validation_gen = initialize_train_and_validation_generators(
"unet", gids, batch_size=4, label_target_size=388)
steps_per_epoch = next(training_gen)
validation_steps = next(validation_gen)
model = UNet(input_size=(572, 572, 3))
metrics = [
Accuracy(),
CategoricalAccuracy(),
CategoricalCrossentropy(),
ArgmaxMeanIoU(num_classes=6, name="mean_iou")
]
optimizer = SGD(learning_rate=initial_learning_rate,
momentum=0.99,
nesterov=True)
model.compile(optimizer=optimizer,
loss=categorical_crossentropy,
metrics=metrics)
start_time = int(time.time())
os.mkdir(f"weights/{start_time}_{model.name}/")
metrics_to_log = [
"loss", "accuracy", "categorical_accuracy", "mean_iou",
"categorical_crossentropy"
]
model_path = f"weights/{start_time}_{model.name}/"
callbacks = [
save_model_on_epoch_end(model.name, model, model_path),
metrics_to_csv_logger(model_path + "/batch.csv", metrics_to_log),
CSVLogger(model_path + "/epoch.csv", separator=";"),
LearningRateScheduler(lr_schedule(initial_lr=initial_learning_rate)),
]
model.fit(training_gen,
epochs=20,
steps_per_epoch=steps_per_epoch,
validation_data=validation_gen,
validation_steps=validation_steps,
callbacks=callbacks)
def do_training(initial_learning_rate=0.001):
gids = get_gids_from_database("wnet")
training_gen, validation_gen = initialize_train_and_validation_generators(
"wnet",
gids,
batch_size=5,
label_target_size=256,
use_image_as_label=True)
steps_per_epoch = next(training_gen)
validation_steps = next(validation_gen)
full_model, encoder_model = WNet(nb_classes=6)
metrics = [Accuracy(), CategoricalAccuracy(), CategoricalCrossentropy()]
optimizer = Adam(lr=initial_learning_rate)
full_model.compile(optimizer=optimizer,
loss=categorical_crossentropy,
metrics=metrics)
start_time = int(time.time())
model_path = f"weights/{start_time}_{full_model.name}/"
os.mkdir(model_path)
metrics_to_log = [metric.name for metric in metrics]
callbacks = [
save_model_on_epoch_end(full_model.name, full_model, model_path),
save_model_on_epoch_end(encoder_model.name, encoder_model, model_path),
metrics_to_csv_logger(model_path + "/batch.csv",
["loss"] + metrics_to_log),
CSVLogger(model_path + "/epoch.csv", separator=";"),
]
full_model.fit(training_gen,
epochs=1,
steps_per_epoch=steps_per_epoch,
validation_data=validation_gen,
validation_steps=validation_steps,
callbacks=callbacks)
def objective(trial: optuna.Trial):
base_lr = trial.suggest_float("base_lr", 0.001, 0.05, step=0.001)
weight_decay = trial.suggest_loguniform("weight_decay", 1e-5, 1e-3)
ema = trial.suggest_categorical("ema", ["true", "false"])
ema_decay = trial.suggest_loguniform("ema_decay", 0.99,
0.9999) if ema == 'true' else None
@curry
def transform(image, label, training):
image = pad(image, 2)
image, label = to_tensor(image, label)
image = normalize(image, [0.1307], [0.3081])
label = tf.one_hot(label, 10)
return image, label
batch_size = 128
eval_batch_size = 256
ds_train, ds_test, steps_per_epoch, test_steps = make_mnist_dataset(
batch_size, eval_batch_size, transform, sub_ratio=0.01)
model = LeNet5()
model.build((None, 32, 32, 1))
criterion = CrossEntropy()
epochs = 20
lr_shcedule = CosineLR(base_lr, steps_per_epoch, epochs=epochs, min_lr=0)
optimizer = SGD(lr_shcedule,
momentum=0.9,
nesterov=True,
weight_decay=weight_decay)
train_metrics = {
'loss': Mean(),
'acc': CategoricalAccuracy(),
}
eval_metrics = {
'loss': CategoricalCrossentropy(from_logits=True),
'acc': CategoricalAccuracy(),
}
learner = SuperLearner(model,
criterion,
optimizer,
train_metrics=train_metrics,
eval_metrics=eval_metrics,
work_dir=f"./MNIST",
multiple_steps=True)
callbacks = [OptunaReportIntermediateResult('acc', trial)]
# if ema == 'true':
# callbacks.append(EMA(ema_decay))
learner.fit(ds_train,
epochs,
ds_test,
val_freq=2,
steps_per_epoch=steps_per_epoch,
val_steps=test_steps,
callbacks=callbacks)
return learner.metric_history.get_metric('acc', "eval")[-1]
model.build((None, 32, 32, 3))
model.summary()
criterion = CrossEntropy(auxiliary_weight=0.4)
base_lr = 0.025
epochs = 600
lr_schedule = CosineLR(base_lr * mul, steps_per_epoch, epochs=epochs, min_lr=0)
optimizer = SGD(lr_schedule, momentum=0.9, weight_decay=3e-4, nesterov=True)
train_metrics = {
'loss': Mean(),
'acc': CategoricalAccuracy(),
}
eval_metrics = {
'loss': CategoricalCrossentropy(from_logits=True),
'acc': CategoricalAccuracy(),
}
learner = SuperLearner(model,
criterion,
optimizer,
grad_clip_norm=5.0,
jit_compile=False,
train_metrics=train_metrics,
eval_metrics=eval_metrics,
work_dir="./CIFAR10-PC-DARTS")
learner.fit(ds_train,
epochs,
ds_test,
def objective(trial: optuna.Trial):
cutout_prob = trial.suggest_float("cutout_prob", 0, 1.0, step=0.1)
mixup_alpha = trial.suggest_float("mixup_alpha", 0, 0.5, step=0.1)
label_smoothing = trial.suggest_uniform("label_smoothing", 0, 0.2)
base_lr = trial.suggest_float("base_lr", 0.01, 0.2, step=0.01)
weight_decay = trial.suggest_loguniform("weight_decay", 1e-5, 1e-3)
ema = trial.suggest_categorical("ema", ["true", "false"])
ema_decay = trial.suggest_loguniform("ema_decay", 0.995,
0.9999) if ema == 'true' else None
@curry
def transform(image, label, training):
if training:
image = random_crop(image, (32, 32), (4, 4))
image = tf.image.random_flip_left_right(image)
image = autoaugment(image, "CIFAR10")
image, label = to_tensor(image, label)
image = normalize(image, [0.491, 0.482, 0.447], [0.247, 0.243, 0.262])
if training:
image = random_apply(cutout(length=16), cutout_prob, image)
label = tf.one_hot(label, 100)
return image, label
def zip_transform(data1, data2):
return mixup(data1, data2, alpha=mixup_alpha)
batch_size = 128
eval_batch_size = 2048
ds_train, ds_test, steps_per_epoch, test_steps = make_cifar100_dataset(
batch_size, eval_batch_size, transform, zip_transform)
setup_runtime(fp16=True)
ds_train, ds_test = distribute_datasets(ds_train, ds_test)
model = ResNet(depth=16, k=8, num_classes=100)
model.build((None, 32, 32, 3))
model.summary()
criterion = CrossEntropy(label_smoothing=label_smoothing)
epochs = 50
lr_schedule = CosineLR(base_lr, steps_per_epoch, epochs=epochs, min_lr=0)
optimizer = SGD(lr_schedule,
momentum=0.9,
weight_decay=weight_decay,
nesterov=True)
train_metrics = {
'loss': Mean(),
'acc': CategoricalAccuracy(),
}
eval_metrics = {
'loss': CategoricalCrossentropy(from_logits=True),
'acc': CategoricalAccuracy(),
}
learner = SuperLearner(model,
criterion,
optimizer,
train_metrics=train_metrics,
eval_metrics=eval_metrics,
work_dir=f"./CIFAR100-NNI",
multiple_steps=True)
callbacks = [OptunaReportIntermediateResult('acc', trial)]
if ema == 'true':
callbacks.append(EMA(ema_decay))
learner.fit(ds_train,
epochs,
ds_test,
val_freq=1,
steps_per_epoch=steps_per_epoch,
val_steps=test_steps,
callbacks=callbacks)
return learner.metric_history.get_metric('acc', "eval")[-1]