def get_scheduler(scheduler_name: str,
optimizer,
lr,
num_epochs,
batches_in_epoch=None):
if scheduler_name is None or scheduler_name.lower() == 'none':
return None
if scheduler_name.lower() in {'1cycle', 'one_cycle', 'onecycle'}:
return OneCycleLR(optimizer,
lr_range=(lr, 1e-6, 1e-5),
num_steps=num_epochs,
warmup_fraction=0.05,
decay_fraction=0.1)
if scheduler_name.lower() == 'exp':
return ExponentialLR(optimizer, gamma=0.95)
if scheduler_name.lower() == 'multistep':
return MultiStepLR(
optimizer,
milestones=[
# int(num_epochs * 0.3),
int(num_epochs * 0.5),
int(num_epochs * 0.7),
int(num_epochs * 0.9),
int(num_epochs * 0.95)
],
gamma=0.3)
if scheduler_name.lower() == 'simple':
# Reduce LR by factor of 10 after 1/3 of training and by factor of 10 more after 2/3 of training
return MultiStepLR(
optimizer,
milestones=[int(num_epochs * 0.33),
int(num_epochs * 0.66)],
gamma=0.1)
raise KeyError(scheduler_name)
epochs = 30
resnet = get_model(model_name, num_classes)
for param in resnet.parameters():
param.requires_grad = False
resnet.last_linear.weight.requires_grad = True
for param in resnet.layer4.parameters():
param.requires_grad = True
loss_fn = nn.CrossEntropyLoss()
opt = torch.optim.SGD(resnet.parameters(), lr=0.01, momentum=0.9)
logdir = '/tmp/protein/logs/'
runner = SupervisedRunner()
sched = OneCycleLR(opt,
num_steps=epochs * len(loaders['train']),
warmup_fraction=0.3,
lr_range=(0.1, 0.0001))
runner.train(model=resnet,
criterion=loss_fn,
optimizer=opt,
loaders=loaders,
logdir=logdir,
num_epochs=epochs,
scheduler=sched,
callbacks=[
AccuracyCallback(num_classes=num_classes),
F1ScoreCallback(input_key="targets_one_hot",
activation="Softmax")
],
verbose=True)
from catalyst.dl.runner import SupervisedRunner
from catalyst.dl.callbacks import EarlyStoppingCallback, AccuracyCallback
from catalyst.contrib.schedulers import OneCycleLR
# experiment setup
num_epochs = NUM_EPOCHS
logdir = "./logs/cifar_simple_notebook_5"
# model, criterion, optimizer, scheduler
model = Net()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())
scheduler = OneCycleLR(optimizer,
num_steps=num_epochs,
lr_range=(0.005, 0.00005),
warmup_steps=2,
momentum_range=(0.85, 0.95))
# model runner
runner = SupervisedRunner()
# model training
runner.train(model=model,
criterion=criterion,
optimizer=optimizer,
scheduler=scheduler,
loaders=loaders,
callbacks=[
AccuracyCallback(accuracy_args=[1, 3, 5]),
EarlyStoppingCallback(patience=2, min_delta=0.01),
callbacks=[
AccuracyMetricCallback(),
F1ScoreMetricCallback(),
QuadraticKappScoreMetricCallback(),
MixupCallback()
],
num_epochs=head_n_epochs,
verbose=True)
print('Train whole net for {} epochs with initial lr {}'.format(full_n_epochs, full_lr))
for p in model.parameters():
p.requires_grad = True
optimizer = torch.optim.Adam(model.parameters(), lr=full_lr)
criterion = nn.CrossEntropyLoss()
#scheduler = ReduceLROnPlateau(optimizer=optimizer, factor=0.5, patience=5)
scheduler = OneCycleLR(optimizer=optimizer, num_steps=full_n_epochs, warmup_fraction=0.5,
momentum_range=(0.9, 0.98),
lr_range=(full_lr*15, full_lr/15))
runner.train(model=model,
criterion=criterion,
optimizer=optimizer,
loaders=loaders,
logdir=logdir,
scheduler=scheduler,
callbacks=[
AccuracyMetricCallback(),
F1ScoreMetricCallback(),
QuadraticKappScoreMetricCallback(),
MixupCallback()
],
num_epochs=full_n_epochs,
verbose=True)