def objective(trial: optuna.Trial): base_lr = trial.suggest_float("base_lr", 0.001, 0.05, step=0.001) weight_decay = trial.suggest_loguniform("weight_decay", 1e-5, 1e-3) ema = trial.suggest_categorical("ema", ["true", "false"]) ema_decay = trial.suggest_loguniform("ema_decay", 0.99, 0.9999) if ema == 'true' else None @curry def transform(image, label, training): image = pad(image, 2) image, label = to_tensor(image, label) image = normalize(image, [0.1307], [0.3081]) label = tf.one_hot(label, 10) return image, label batch_size = 128 eval_batch_size = 256 ds_train, ds_test, steps_per_epoch, test_steps = make_mnist_dataset( batch_size, eval_batch_size, transform, sub_ratio=0.01) model = LeNet5() model.build((None, 32, 32, 1)) criterion = CrossEntropy() epochs = 20 lr_shcedule = CosineLR(base_lr, steps_per_epoch, epochs=epochs, min_lr=0) optimizer = SGD(lr_shcedule, momentum=0.9, nesterov=True, weight_decay=weight_decay) train_metrics = { 'loss': Mean(), 'acc': CategoricalAccuracy(), } eval_metrics = { 'loss': CategoricalCrossentropy(from_logits=True), 'acc': CategoricalAccuracy(), } learner = SuperLearner(model, criterion, optimizer, train_metrics=train_metrics, eval_metrics=eval_metrics, work_dir=f"./MNIST", multiple_steps=True) callbacks = [OptunaReportIntermediateResult('acc', trial)] # if ema == 'true': # callbacks.append(EMA(ema_decay)) learner.fit(ds_train, epochs, ds_test, val_freq=2, steps_per_epoch=steps_per_epoch, val_steps=test_steps, callbacks=callbacks) return learner.metric_history.get_metric('acc', "eval")[-1]
cls_scores, bbox_coder, centerness, nms_pre=1000, iou_threshold=0.6, score_threshold=0.05, use_sigmoid=True) local_eval_metrics = { 'loss': MeanMetricWrapper(criterion), 'mAP': MeanAveragePrecision(output_transform=output_transform), } learner = SuperLearner(model, criterion, optimizer, train_metrics=train_metrics, eval_metrics=eval_metrics, work_dir=f"./models") learner.fit( ds_train, epochs, ds_val, val_freq=1, steps_per_epoch=steps_per_epoch, val_steps=val_steps, local_eval_metrics=local_eval_metrics, local_eval_freq=[(0, 5), (45, 1)], )
momentum=0.9, weight_decay=params['weight_decay'], nesterov=True) train_metrics = { 'loss': Mean(), 'acc': CategoricalAccuracy(), } eval_metrics = { 'loss': CategoricalCrossentropy(from_logits=True), 'acc': CategoricalAccuracy(), } learner = SuperLearner(model, criterion, optimizer, train_metrics=train_metrics, eval_metrics=eval_metrics, work_dir=f"./CIFAR100-NNI", multiple_steps=True) callbacks = [NNIReportIntermediateResult('acc')] if params['ema']['_name'] == 'true': callbacks.append(EMA(params['ema']['decay'])) hist = learner.fit(ds_train, epochs, ds_test, val_freq=1, steps_per_epoch=steps_per_epoch, val_steps=test_steps, callbacks=callbacks)
base_lr = 0.025 epochs = 600 lr_schedule = CosineLR(base_lr * mul, steps_per_epoch, epochs=epochs, min_lr=0) optimizer = SGD(lr_schedule, momentum=0.9, weight_decay=3e-4, nesterov=True) train_metrics = { 'loss': Mean(), 'acc': CategoricalAccuracy(), } eval_metrics = { 'loss': CategoricalCrossentropy(from_logits=True), 'acc': CategoricalAccuracy(), } learner = SuperLearner(model, criterion, optimizer, grad_clip_norm=5.0, jit_compile=False, train_metrics=train_metrics, eval_metrics=eval_metrics, work_dir="./CIFAR10-PC-DARTS") learner.fit(ds_train, epochs, ds_test, val_freq=1, steps_per_epoch=steps_per_epoch, val_steps=test_steps, callbacks=[DropPathRateScheduleV2()])
model.build((None, 32, 32, 3)) model.summary() criterion = CrossEntropy(label_smoothing=0) base_lr = 0.05 epochs = 1800 lr_schedule = CosineLR(base_lr * mul, steps_per_epoch, epochs=epochs, min_lr=0) optimizer = SGD(lr_schedule, momentum=0.9, weight_decay=1e-4, nesterov=True) train_metrics = { 'loss': Mean(), 'acc': CategoricalAccuracy(), } eval_metrics = { 'loss': CategoricalCrossentropy(from_logits=True), 'acc': CategoricalAccuracy(), } learner = SuperLearner(model, criterion, optimizer, train_metrics=train_metrics, eval_metrics=eval_metrics, work_dir="./drive/MyDrive/models/CIFAR100") hist = learner.fit(ds_train, epochs, ds_test, val_freq=1, steps_per_epoch=steps_per_epoch, val_steps=test_steps)
cls_scores, bbox_coder, centerness, iou_threshold=0.6, score_threshold=0.05, use_sigmoid=True) local_eval_metrics = { 'loss': MeanMetricWrapper(criterion), 'mAP': MeanAveragePrecision(output_transform=output_transform), } learner = SuperLearner(model, criterion, optimizer, train_metrics=train_metrics, eval_metrics=eval_metrics, work_dir=f"./models") learner.fit( ds_train, epochs, ds_val, val_freq=1, steps_per_epoch=steps_per_epoch, val_steps=val_steps, local_eval_metrics=local_eval_metrics, local_eval_freq=[(0, 5), (45, 1)], )
} }) backbone = resnet50(output_stride=16, multi_grad=(1, 2, 4)) model = DeepLabV3P(backbone, aspp_ratios=(1, 6, 12, 18), aspp_channels=256, num_classes=19) model.build((None, HEIGHT, WIDTH, 3)) criterion = cross_entropy(ignore_label=255) base_lr = 1e-3 epochs = 400 lr_schedule = CosineLR(base_lr * mul, steps_per_epoch, epochs, min_lr=0, warmup_min_lr=0, warmup_epoch=5) optimizer = SGD(lr_schedule, momentum=0.9, nesterov=True, weight_decay=4e-5) train_metrics = { 'loss': Mean(), } eval_metrics = { 'loss': CrossEntropy(ignore_label=255), 'miou': MeanIoU(num_classes=19), } learner = SuperLearner( model, criterion, optimizer, train_metrics=train_metrics, eval_metrics=eval_metrics, work_dir=f"./models") learner.fit( ds_train_dist, epochs, ds_val_dist, val_freq=1, steps_per_epoch=steps_per_epoch, val_steps=val_steps)
def objective(trial: optuna.Trial): cutout_prob = trial.suggest_float("cutout_prob", 0, 1.0, step=0.1) mixup_alpha = trial.suggest_float("mixup_alpha", 0, 0.5, step=0.1) label_smoothing = trial.suggest_uniform("label_smoothing", 0, 0.2) base_lr = trial.suggest_float("base_lr", 0.01, 0.2, step=0.01) weight_decay = trial.suggest_loguniform("weight_decay", 1e-5, 1e-3) ema = trial.suggest_categorical("ema", ["true", "false"]) ema_decay = trial.suggest_loguniform("ema_decay", 0.995, 0.9999) if ema == 'true' else None @curry def transform(image, label, training): if training: image = random_crop(image, (32, 32), (4, 4)) image = tf.image.random_flip_left_right(image) image = autoaugment(image, "CIFAR10") image, label = to_tensor(image, label) image = normalize(image, [0.491, 0.482, 0.447], [0.247, 0.243, 0.262]) if training: image = random_apply(cutout(length=16), cutout_prob, image) label = tf.one_hot(label, 100) return image, label def zip_transform(data1, data2): return mixup(data1, data2, alpha=mixup_alpha) batch_size = 128 eval_batch_size = 2048 ds_train, ds_test, steps_per_epoch, test_steps = make_cifar100_dataset( batch_size, eval_batch_size, transform, zip_transform) setup_runtime(fp16=True) ds_train, ds_test = distribute_datasets(ds_train, ds_test) model = ResNet(depth=16, k=8, num_classes=100) model.build((None, 32, 32, 3)) model.summary() criterion = CrossEntropy(label_smoothing=label_smoothing) epochs = 50 lr_schedule = CosineLR(base_lr, steps_per_epoch, epochs=epochs, min_lr=0) optimizer = SGD(lr_schedule, momentum=0.9, weight_decay=weight_decay, nesterov=True) train_metrics = { 'loss': Mean(), 'acc': CategoricalAccuracy(), } eval_metrics = { 'loss': CategoricalCrossentropy(from_logits=True), 'acc': CategoricalAccuracy(), } learner = SuperLearner(model, criterion, optimizer, train_metrics=train_metrics, eval_metrics=eval_metrics, work_dir=f"./CIFAR100-NNI", multiple_steps=True) callbacks = [OptunaReportIntermediateResult('acc', trial)] if ema == 'true': callbacks.append(EMA(ema_decay)) learner.fit(ds_train, epochs, ds_test, val_freq=1, steps_per_epoch=steps_per_epoch, val_steps=test_steps, callbacks=callbacks) return learner.metric_history.get_metric('acc', "eval")[-1]
criterion = CrossEntropy() base_lr = 0.1 epochs = 100 lr_schedule = CosineLR(base_lr, steps_per_epoch, epochs=epochs, min_lr=0) optimizer = SGD(lr_schedule, momentum=0.9, weight_decay=5e-4, nesterov=True) train_metrics = { 'loss': Mean(), 'acc': CategoricalAccuracy(), } eval_metrics = { 'loss': CategoricalCrossentropy(from_logits=True), 'acc': CategoricalAccuracy(), } learner = SuperLearner(model, criterion, optimizer, train_metrics=train_metrics, eval_metrics=eval_metrics, work_dir=f"./cifar10") hist = learner.fit(ds_train, epochs, ds_test, val_freq=1, steps_per_epoch=steps_per_epoch, val_steps=test_steps)