def configure_optimizers(self):
optimizer = RMSpropTF(self.parameters(), lr=self.hparams.learning_rate,
weight_decay=self.hparams.weight_decay,
momentum=0.9, alpha=0.9, eps=1.0)
return {
'optimizer': optimizer,
'scheduler': CosineAnnealingLR(optimizer, self.hparams.max_epochs)
}
def create_optimizer(args, model, filter_bias_and_bn=True):
weight_decay = args.weight_decay
if weight_decay and filter_bias_and_bn:
parameters = add_weight_decay(model, weight_decay)
weight_decay = 0.
else:
parameters = model.parameters()
if args.opt.lower() == 'sgd':
optimizer = optim.SGD(parameters,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay,
nesterov=True)
elif args.opt.lower() == 'adam':
optimizer = optim.Adam(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif args.opt.lower() == 'nadam':
optimizer = Nadam(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif args.opt.lower() == 'adadelta':
optimizer = optim.Adadelta(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif args.opt.lower() == 'rmsprop':
optimizer = optim.RMSprop(parameters,
lr=args.lr,
alpha=0.9,
eps=args.opt_eps,
momentum=args.momentum,
weight_decay=weight_decay)
elif args.opt.lower() == 'rmsproptf':
optimizer = RMSpropTF(parameters,
lr=args.lr,
alpha=0.9,
eps=args.opt_eps,
momentum=args.momentum,
weight_decay=weight_decay)
else:
assert False and "Invalid optimizer"
raise ValueError
return optimizer
def create_optimizer(args,
model,
filter_bias_and_bn=True,
classification_layer_name=None):
opt_lower = args.opt.lower()
weight_decay = args.weight_decay
if 'adamw' in opt_lower or 'radam' in opt_lower:
# Compensate for the way current AdamW and RAdam optimizers apply LR to the weight-decay
# I don't believe they follow the paper or original Torch7 impl which schedules weight
# decay based on the ratio of current_lr/initial_lr
weight_decay /= args.lr
if weight_decay and filter_bias_and_bn: # batch norm and bias params
if classification_layer_name is not None:
parameters = set_lr_per_params(args, model,
classification_layer_name,
weight_decay)
else:
parameters = add_weight_decay(model, weight_decay)
weight_decay = 0. # reset to 0
else:
if classification_layer_name is not None:
parameters = set_lr_per_params(args,
model,
classification_layer_name,
weight_decay=0)
else:
parameters = model.parameters()
if 'fused' in opt_lower:
assert has_apex and torch.cuda.is_available(
), 'APEX and CUDA required for fused optimizers'
opt_split = opt_lower.split('_')
opt_lower = opt_split[-1]
if opt_lower == 'sgd' or opt_lower == 'nesterov':
optimizer = optim.SGD(parameters,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay,
nesterov=True)
elif opt_lower == 'momentum':
optimizer = optim.SGD(parameters,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay,
nesterov=False)
elif opt_lower == 'adam':
optimizer = optim.Adam(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'adamw':
optimizer = AdamW(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'nadam':
optimizer = Nadam(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'radam':
optimizer = RAdam(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'adamp':
optimizer = AdamP(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps,
delta=0.1,
wd_ratio=0.01,
nesterov=True)
elif opt_lower == 'sgdp':
optimizer = SGDP(parameters,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay,
eps=args.opt_eps,
nesterov=True)
elif opt_lower == 'adadelta':
optimizer = optim.Adadelta(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'rmsprop':
optimizer = optim.RMSprop(parameters,
lr=args.lr,
alpha=0.9,
eps=args.opt_eps,
momentum=args.momentum,
weight_decay=weight_decay)
elif opt_lower == 'rmsproptf':
optimizer = RMSpropTF(parameters,
lr=args.lr,
alpha=0.9,
eps=args.opt_eps,
momentum=args.momentum,
weight_decay=weight_decay)
elif opt_lower == 'novograd':
optimizer = NovoGrad(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'nvnovograd':
optimizer = NvNovoGrad(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'fusedsgd':
optimizer = FusedSGD(parameters,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay,
nesterov=True)
elif opt_lower == 'fusedmomentum':
optimizer = FusedSGD(parameters,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay,
nesterov=False)
elif opt_lower == 'fusedadam':
optimizer = FusedAdam(parameters,
lr=args.lr,
adam_w_mode=False,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'fusedadamw':
optimizer = FusedAdam(parameters,
lr=args.lr,
adam_w_mode=True,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'fusedlamb':
optimizer = FusedLAMB(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'fusednovograd':
optimizer = FusedNovoGrad(parameters,
lr=args.lr,
betas=(0.95, 0.98),
weight_decay=weight_decay,
eps=args.opt_eps)
else:
assert False and "Invalid optimizer"
raise ValueError
if len(opt_split) > 1:
if opt_split[0] == 'lookahead':
optimizer = Lookahead(optimizer)
return optimizer
def create_optimizer(args, model, filter_bias_and_bn=True):
opt_lower = args.opt.lower()
weight_decay = args.weight_decay
if opt_lower == 'adamw' or opt_lower == 'radam':
# compensate for the way current AdamW and RAdam optimizers
# apply the weight-decay
weight_decay /= args.lr
if weight_decay and filter_bias_and_bn:
parameters = add_weight_decay(model, weight_decay)
weight_decay = 0.
else:
parameters = model.parameters()
opt_split = opt_lower.split('_')
opt_lower = opt_split[-1]
if opt_lower == 'sgd':
optimizer = optim.SGD(parameters,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay,
nesterov=True)
elif opt_lower == 'adam':
optimizer = optim.Adam(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'adamw':
optimizer = AdamW(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'nadam':
optimizer = Nadam(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'radam':
optimizer = RAdam(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'adadelta':
optimizer = optim.Adadelta(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'rmsprop':
optimizer = optim.RMSprop(parameters,
lr=args.lr,
alpha=0.9,
eps=args.opt_eps,
momentum=args.momentum,
weight_decay=weight_decay)
elif opt_lower == 'rmsproptf':
optimizer = RMSpropTF(parameters,
lr=args.lr,
alpha=0.9,
eps=args.opt_eps,
momentum=args.momentum,
weight_decay=weight_decay)
elif opt_lower == 'novograd':
optimizer = NovoGrad(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
else:
assert False and "Invalid optimizer"
raise ValueError
if len(opt_split) > 1:
if opt_split[0] == 'lookahead':
optimizer = Lookahead(optimizer)
return optimizer
def create_optimizer(args, model, filter_bias_and_bn=True):
opt_lower = args.opt.lower()
weight_decay = args.weight_decay
if 'adamw' in opt_lower or 'radam' in opt_lower:
# Compensate for the way current AdamW and RAdam optimizers apply LR to the weight-decay
# I don't believe they follow the paper or original Torch7 impl which schedules weight
# decay based on the ratio of current_lr/initial_lr
weight_decay /= args.lr
if weight_decay and filter_bias_and_bn:
print("has weight decay and filter bias")
parameters = add_weight_decay(model, weight_decay)
weight_decay = 0.
else:
print("Comes here to unfrozen params inside optim")
parameters = unfrozen_params(model)
if 'fused' in opt_lower:
assert has_apex and torch.cuda.is_available(
), 'APEX and CUDA required for fused optimizers'
opt_split = opt_lower.split('_')
opt_lower = opt_split[-1]
if opt_lower == 'sgd' or opt_lower == 'nesterov':
optimizer = optim.SGD(parameters,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay,
nesterov=True)
elif opt_lower == 'momentum':
optimizer = optim.SGD(parameters,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay,
nesterov=False)
elif opt_lower == 'adam':
optimizer = optim.Adam(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'adamw':
optimizer = AdamW(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'nadam':
optimizer = Nadam(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'radam':
optimizer = RAdam(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'adadelta':
optimizer = optim.Adadelta(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'rmsprop':
optimizer = optim.RMSprop(parameters,
lr=args.lr,
alpha=0.9,
eps=args.opt_eps,
momentum=args.momentum,
weight_decay=weight_decay)
elif opt_lower == 'rmsproptf':
optimizer = RMSpropTF(parameters,
lr=args.lr,
alpha=0.9,
eps=args.opt_eps,
momentum=args.momentum,
weight_decay=weight_decay)
elif opt_lower == 'novograd':
optimizer = NovoGrad(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'nvnovograd':
optimizer = NvNovoGrad(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'fusedsgd':
optimizer = FusedSGD(parameters,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay,
nesterov=True)
elif opt_lower == 'fusedmomentum':
print("my optimizer")
optimizer = FusedSGD(parameters,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay,
nesterov=False)
elif opt_lower == 'fusedadam':
optimizer = FusedAdam(parameters,
lr=args.lr,
adam_w_mode=False,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'fusedadamw':
optimizer = FusedAdam(parameters,
lr=args.lr,
adam_w_mode=True,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'fusedlamb':
optimizer = FusedLAMB(parameters,
lr=args.lr,
weight_decay=weight_decay,
eps=args.opt_eps)
elif opt_lower == 'fusednovograd':
optimizer = FusedNovoGrad(parameters,
lr=args.lr,
betas=(0.95, 0.98),
weight_decay=weight_decay,
eps=args.opt_eps)
else:
assert False and "Invalid optimizer"
raise ValueError
if len(opt_split) > 1:
if opt_split[0] == 'lookahead':
optimizer = Lookahead(optimizer)
return optimizer
def __init__(self,
baseline_model,
pretrained=True,
num_classes=None,
lr=6.25e-5,
dropout=0.2,
drop_connect=0.2,
cuda=True,
warmup_ratio=0.1,
num_training_steps=1000,
gamma=0.97,
device_idxs=(),
mixed_precision=False):
super().__init__(cuda=cuda,
warmup_ratio=warmup_ratio,
num_training_steps=num_training_steps,
device_idxs=device_idxs,
mixed_precision=mixed_precision)
self.baseline_model = timm.create_model(baseline_model,
pretrained=pretrained,
num_classes=num_classes,
drop_rate=dropout,
drop_connect_rate=drop_connect,
drop_path_rate=drop_connect)
self.cross_entropy = nn.CrossEntropyLoss()
self.reset()
if self.cuda and len(self.devices) > 1:
self.baseline_model = nn.DataParallel(
self.baseline_model,
device_ids=self.devices,
output_device=self.model_device)
if self.mixed_precision:
self.baseline_model.forward = insert_autocast(
self.baseline_model.forward)
if True:
self.optimizer = RMSpropTF(self.parameters(),
alpha=0.9,
momentum=0.9,
weight_decay=1e-5,
eps=1e-3,
lr=lr)
self.scheduler = StepLRScheduler(
self.optimizer,
decay_t=self.num_training_steps * 2.4 / 450,
decay_rate=gamma,
warmup_lr_init=1e-6,
warmup_t=self.num_training_steps * 3 / 450,
noise_range_t=None,
noise_pct=0.67,
noise_std=1,
noise_seed=42)
else:
self.optimizer = AdamW(self.parameters(), lr=lr)
self.scheduler = StepLR(self.optimizer,
step_size=int(num_training_steps * 2.4 /
450),
gamma=gamma)
self.main_losses = {'im'}