def test_warmup_cosine(self):
p = nn.Parameter(torch.zeros(0))
opt = torch.optim.SGD([p], lr=5)
multiplier = CompositeParamScheduler(
[
LinearParamScheduler(0.001, 1), # warmup
CosineParamScheduler(1, 0),
],
interval_scaling=["rescaled", "fixed"],
lengths=[5 / 30, 25 / 30],
)
sched = LRMultiplier(opt, multiplier, 30)
p.sum().backward()
opt.step()
self.assertEqual(opt.param_groups[0]["lr"], 0.005)
lrs = [0.005]
for _ in range(30):
sched.step()
lrs.append(opt.param_groups[0]["lr"])
for idx, lr in enumerate(lrs):
expected_cosine = 2.5 * (1.0 + math.cos(math.pi * idx / 30))
if idx >= 5:
self.assertAlmostEqual(lr, expected_cosine)
else:
self.assertNotAlmostEqual(lr, expected_cosine)
def build_lr_scheduler(
cfg: CfgNode, optimizer: torch.optim.Optimizer
) -> torch.optim.lr_scheduler._LRScheduler:
"""
Build a LR scheduler from config.
"""
name = cfg.SOLVER.LR_SCHEDULER_NAME
if name == "WarmupMultiStepLR":
steps = [x for x in cfg.SOLVER.STEPS if x <= cfg.SOLVER.MAX_ITER]
if len(steps) != len(cfg.SOLVER.STEPS):
logger = logging.getLogger(__name__)
logger.warning(
"SOLVER.STEPS contains values larger than SOLVER.MAX_ITER. "
"These values will be ignored."
)
sched = MultiStepParamScheduler(
values=[cfg.SOLVER.GAMMA ** k for k in range(len(steps) + 1)],
milestones=steps,
num_updates=cfg.SOLVER.MAX_ITER,
)
elif name == "WarmupCosineLR":
sched = CosineParamScheduler(1, 0)
else:
raise ValueError("Unknown LR scheduler: {}".format(name))
sched = WarmupParamScheduler(
sched,
cfg.SOLVER.WARMUP_FACTOR,
cfg.SOLVER.WARMUP_ITERS / cfg.SOLVER.MAX_ITER,
cfg.SOLVER.WARMUP_METHOD,
)
return LRMultiplier(optimizer, multiplier=sched, max_iter=cfg.SOLVER.MAX_ITER)
def build_lr_scheduler(
cfg: CfgNode, optimizer: torch.optim.Optimizer
) -> torch.optim.lr_scheduler._LRScheduler:
"""
Build a LR scheduler from config.
"""
name = cfg.SOLVER.LR_SCHEDULER_NAME
if name == "WarmupMultiStepLR":
sched = MultiStepParamScheduler(
values=[cfg.SOLVER.GAMMA ** k for k in range(len(cfg.SOLVER.STEPS) + 1)],
milestones=cfg.SOLVER.STEPS,
num_updates=cfg.SOLVER.MAX_ITER,
)
elif name == "WarmupCosineLR":
sched = CosineParamScheduler(1, 0)
else:
raise ValueError("Unknown LR scheduler: {}".format(name))
sched = WarmupParamScheduler(
sched,
cfg.SOLVER.WARMUP_FACTOR,
cfg.SOLVER.WARMUP_ITERS / cfg.SOLVER.MAX_ITER,
cfg.SOLVER.WARMUP_METHOD,
)
return LRMultiplier(optimizer, multiplier=sched, max_iter=cfg.SOLVER.MAX_ITER)
def _get_valid_mixed_config(self):
return {
"schedulers": [
StepParamScheduler(values=[0.1, 0.2, 0.3, 0.4, 0.5], num_updates=10),
CosineParamScheduler(start_value=0.42, end_value=0.0001),
],
"lengths": [0.5, 0.5],
"interval_scaling": ["rescaled", "rescaled"],
}
def test_scheduler_warmup_decay_match(self):
decay_config = self._get_valid_decay_config()
decay_scheduler = CosineParamScheduler(**decay_config)
warmup_config = copy.deepcopy(decay_config)
# Swap start and end lr to change to warmup
tmp = warmup_config["start_value"]
warmup_config["start_value"] = warmup_config["end_value"]
warmup_config["end_value"] = tmp
warmup_scheduler = CosineParamScheduler(**warmup_config)
decay_schedule = [
round(decay_scheduler(epoch_num / 1000), 8)
for epoch_num in range(1, 1000)
]
warmup_schedule = [
round(warmup_scheduler(epoch_num / 1000), 8)
for epoch_num in range(1, 1000)
]
self.assertEqual(decay_schedule, list(reversed(warmup_schedule)))
def test_scheduler_as_decay(self):
config = self._get_valid_decay_config()
scheduler = CosineParamScheduler(**config)
schedule = [
round(scheduler(epoch_num / self._num_epochs), 4)
for epoch_num in range(self._num_epochs)
]
expected_schedule = [
config["start_value"]
] + self._get_valid_decay_config_intermediate_values()
self.assertEqual(schedule, expected_schedule)
def test_scheduler_as_warmup(self):
config = self._get_valid_decay_config()
# Swap start and end lr to change to warmup
tmp = config["start_value"]
config["start_value"] = config["end_value"]
config["end_value"] = tmp
scheduler = CosineParamScheduler(**config)
schedule = [
round(scheduler(epoch_num / self._num_epochs), 4)
for epoch_num in range(self._num_epochs)
]
# Schedule should be decay reversed
expected_schedule = [config["start_value"]] + list(
reversed(self._get_valid_decay_config_intermediate_values()))
self.assertEqual(schedule, expected_schedule)
def build_lr_scheduler(
cfg: CfgNode, optimizer: torch.optim.Optimizer
) -> torch.optim.lr_scheduler._LRScheduler:
"""
Build a LR scheduler from config.
"""
name = cfg.SOLVER.LR_SCHEDULER_NAME
if name == "WarmupMultiStepLR":
sched = MultiStepParamScheduler(
values=[
cfg.SOLVER.GAMMA**k for k in range(len(cfg.SOLVER.STEPS) + 1)
],
milestones=cfg.SOLVER.STEPS,
num_updates=cfg.SOLVER.MAX_ITER,
)
elif name == "WarmupCosineLR":
sched = CosineParamScheduler(1, 0)
else:
raise ValueError("Unknown LR scheduler: {}".format(name))
# Add warmup
warmup_method = cfg.SOLVER.WARMUP_METHOD
if warmup_method == "constant":
warmup = ConstantParamScheduler(cfg.SOLVER.WARMUP_FACTOR)
elif warmup_method == "linear":
warmup = LinearParamScheduler(cfg.SOLVER.WARMUP_FACTOR, 1.0)
else:
raise ValueError("Unknown warmup method: {}".format(warmup_method))
warmup_ratio = cfg.SOLVER.WARMUP_ITERS / cfg.SOLVER.MAX_ITER
sched = CompositeParamScheduler(
[warmup, sched],
interval_scaling=["rescaled", "fixed"],
lengths=[warmup_ratio, 1 - warmup_ratio],
)
return LRMultiplier(optimizer,
multiplier=sched,
max_iter=cfg.SOLVER.MAX_ITER)
def test_warmup_cosine(self):
p = nn.Parameter(torch.zeros(0))
opt = torch.optim.SGD([p], lr=5)
multiplier = WarmupParamScheduler(
CosineParamScheduler(1, 0),
0.001,
5 / 30,
)
sched = LRMultiplier(opt, multiplier, 30)
p.sum().backward()
opt.step()
self.assertEqual(opt.param_groups[0]["lr"], 0.005)
lrs = [0.005]
for _ in range(30):
sched.step()
lrs.append(opt.param_groups[0]["lr"])
for idx, lr in enumerate(lrs):
expected_cosine = 2.5 * (1.0 + math.cos(math.pi * idx / 30))
if idx >= 5:
self.assertAlmostEqual(lr, expected_cosine)
else:
self.assertNotAlmostEqual(lr, expected_cosine)
# resize_and_crop_image in:
# https://github.com/tensorflow/tpu/blob/b24729de804fdb751b06467d3dce0637fa652060/models/official/detection/utils/input_utils.py#L127 # noqa: E501, B950
image_size = 1024
dataloader.train.mapper.augmentations = [
L(T.ResizeScale)(min_scale=0.1,
max_scale=2.0,
target_height=image_size,
target_width=image_size),
L(T.FixedSizeCrop)(crop_size=(image_size, image_size)),
L(T.RandomFlip)(horizontal=True),
]
# recompute boxes due to cropping
dataloader.train.mapper.recompute_boxes = True
# larger batch-size.
dataloader.train.total_batch_size = 64
# Equivalent to 100 epochs.
# 100 ep = 184375 iters * 64 images/iter / 118000 images/ep
train.max_iter = 184375
lr_multiplier = L(WarmupParamScheduler)(
scheduler=CosineParamScheduler(1.0, 0.0),
warmup_length=500 / train.max_iter,
warmup_factor=0.067,
)
optimizer.lr = 0.1
optimizer.weight_decay = 4e-5