def configure_optimizers(self):
opt_cfg = self.cfg['optimizer']
lr = float(self.cfg['optimizer']['lr'])
if opt_cfg['name'] == 'AdamW':
optimizer = AdamW(self.model.parameters(), lr=lr, )
elif opt_cfg['name'] == 'Adam_GCC':
optimizer = Adam_GCC(self.model.parameters(), lr=lr)
elif opt_cfg['name'] == 'AdamW_GCC2':
optimizer = AdamW_GCC2(self.model.parameters(), lr=lr)
if self.cfg['scheduler']['type'] == 'none':
sched = None
elif self.cfg['scheduler']['type'] == 'CosineAnnealingWarmRestarts':
T_mult = self.cfg['scheduler']['T_mult']
T_0 = self.cfg['scheduler']['T_0']
eta_min = float(self.cfg['scheduler']['eta_min'])
sched = CosineAnnealingWarmRestarts(optimizer, T_0=T_0, T_mult=T_mult, eta_min=eta_min, last_epoch=-1)
elif self.cfg['scheduler']['type'] == 'OneCycleLR':
max_lr = float(self.cfg['scheduler']['max_lr'])
steps_per_epoch = cfg['scheduler']['steps_per_epoch']
epochs = cfg['scheduler']['epochs']
sched = OneCycleLR(optimizer, max_lr=max_lr, steps_per_epoch=steps_per_epoch, epochs=epochs)
else:
raise Exception('scheduler {} not supported'.format(self.cfg['scheduler']['type']))
if sched is not None:
sched = {'scheduler': sched, 'name': format(self.cfg['scheduler']['type'])}
if sched is not None:
return [optimizer], [sched]
else:
return optimizer
return optimizer
def configure_optimizers(self):
lr = float(self.cfg['train_params']['lr'])
if self.cfg['optimizer'] == 'adam':
optimizer = torch.optim.Adam(self.model.parameters(), lr=lr)
elif self.cfg['optimizer'] == 'adamw':
optimizer = torch.optim.AdamW(self.model.parameters(), lr=lr)
elif self.cfg['optimizer'] == 'adamw_gcc2':
optimizer = AdamW_GCC2(self.model.parameters(), lr=lr)
elif self.cfg['optimizer'] == 'radam':
optimizer = RAdam(self.model.parameters(), lr=lr)
else:
raise Exception('optimizer {} not supported'.format(self.cfg['optimizer']))
self.opt = optimizer
if self.cfg['scheduler']['type'] == 'CosineAnnealingWarmRestarts':
T_mult = self.cfg['scheduler']['T_mult']
T_0 = self.cfg['scheduler']['T_0']
eta_min = float(self.cfg['scheduler']['eta_min'])
sched = CosineAnnealingWarmRestarts(optimizer, T_0=T_0, T_mult=T_mult, eta_min=eta_min, last_epoch=-1)
elif self.cfg['scheduler']['type'] == 'OneCycleLR':
max_lr = float(self.cfg['scheduler']['max_lr'])
steps_per_epoch = cfg['scheduler']['steps_per_epoch']
epochs = cfg['scheduler']['epochs']
sched = OneCycleLR(optimizer, max_lr=max_lr, steps_per_epoch=steps_per_epoch, epochs=epochs)
else:
raise Exception('scheduler {} not supported')
sched = {'scheduler': sched, 'name': 'adam+{}'.format(self.cfg['scheduler']['type'])}
return [optimizer], [sched]
def configure_optimizers(self):
lr = float(get_or_default(self.cfg['train_params'], key='lr', default_value=5e-4))
optimizer = AdamW_GCC2(self.model.parameters(), lr=lr)
self.opt = optimizer
T_mult = 2
T_0 = 10
eta_min = 1e-8
sched = CosineAnnealingWarmRestarts(optimizer, T_0=T_0, T_mult=T_mult, eta_min=eta_min, last_epoch=-1)
sched = {'scheduler': sched, 'name': 'CosineAnnealingWarmRestarts'}
return [optimizer], [sched]
def configure_optimizers(self):
lr = float(self.cfg['train_params']['lr'])
optimizer = AdamW_GCC2(self.model.parameters(), lr=lr)
self.opt = optimizer
if self.cfg['scheduler']['type'] == 'CosineAnnealingWarmRestarts':
T_mult = self.cfg['scheduler']['T_mult']
T_0 = self.cfg['scheduler']['T_0']
eta_min = float(self.cfg['scheduler']['eta_min'])
sched = CosineAnnealingWarmRestarts(optimizer,
T_0=T_0,
T_mult=T_mult,
eta_min=eta_min,
last_epoch=-1)
sched = {
'scheduler': sched,
'name': 'adam+{}'.format(self.cfg['scheduler']['type'])
}
return [optimizer], [sched]