def test_invalid_config(self):
config = self._get_valid_mixed_config()
bad_config = copy.deepcopy(config)
# Size of schedulers and lengths doesn't match
bad_config["schedulers"] = copy.deepcopy(config["schedulers"])
bad_config["lengths"] = copy.deepcopy(config["lengths"])
bad_config["schedulers"].append(bad_config["schedulers"][-1])
with self.assertRaises(ValueError):
CompositeParamScheduler(**bad_config)
# Sum of lengths < 1
bad_config["schedulers"] = copy.deepcopy(config["schedulers"])
bad_config["lengths"][-1] -= 0.1
with self.assertRaises(ValueError):
CompositeParamScheduler(**bad_config)
# Sum of lengths > 1
bad_config["lengths"] = copy.deepcopy(config["lengths"])
bad_config["lengths"][-1] += 0.1
with self.assertRaises(ValueError):
CompositeParamScheduler(**bad_config)
# Bad value for composition_mode
bad_config["interval_scaling"] = ["rescaled", "rescaleds"]
with self.assertRaises(ValueError):
CompositeParamScheduler(**bad_config)
# Wrong number composition modes
bad_config["interval_scaling"] = ["rescaled"]
with self.assertRaises(ValueError):
CompositeParamScheduler(**bad_config)
def test_linear_scheduler_no_gaps(self):
config = self._get_valid_linear_config()
# Check rescaled
scheduler = CompositeParamScheduler(**config)
schedule = [
scheduler(epoch_num / self._num_updates)
for epoch_num in range(self._num_updates)
]
expected_schedule = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
self.assertEqual(expected_schedule, schedule)
# Check fixed composition gives same result as only 1 scheduler
config["schedulers"][1] = config["schedulers"][0]
config["interval_scaling"] = ["fixed", "fixed"]
scheduler = CompositeParamScheduler(**config)
linear_scheduler = config["schedulers"][0]
schedule = [
scheduler(epoch_num / self._num_updates)
for epoch_num in range(self._num_updates)
]
expected_schedule = [
linear_scheduler(epoch_num / self._num_updates)
for epoch_num in range(self._num_updates)
]
self.assertEqual(expected_schedule, schedule)
def test_scheduler_with_mixed_types(self):
config = self._get_valid_mixed_config()
scheduler_0 = config["schedulers"][0]
scheduler_1 = config["schedulers"][1]
# Check scaled
config["interval_scaling"] = ["rescaled", "rescaled"]
scheduler = CompositeParamScheduler(**config)
scaled_schedule = [
round(scheduler(epoch_num / self._num_updates), 4)
for epoch_num in range(self._num_updates)
]
expected_schedule = [
round(scheduler_0(epoch_num / self._num_updates), 4)
for epoch_num in range(0, self._num_updates, 2)
] + [
round(scheduler_1(epoch_num / self._num_updates), 4)
for epoch_num in range(0, self._num_updates, 2)
]
self.assertEqual(scaled_schedule, expected_schedule)
# Check fixed
config["interval_scaling"] = ["fixed", "fixed"]
scheduler = CompositeParamScheduler(**config)
fixed_schedule = [
round(scheduler(epoch_num / self._num_updates), 4)
for epoch_num in range(self._num_updates)
]
expected_schedule = [
round(scheduler_0(epoch_num / self._num_updates), 4)
for epoch_num in range(0, int(self._num_updates / 2))
] + [
round(scheduler_1(epoch_num / self._num_updates), 4)
for epoch_num in range(int(self._num_updates / 2), self._num_updates)
]
self.assertEqual(fixed_schedule, expected_schedule)
# Check warmup of rescaled then fixed
config["interval_scaling"] = ["rescaled", "fixed"]
scheduler = CompositeParamScheduler(**config)
fixed_schedule = [
round(scheduler(epoch_num / self._num_updates), 4)
for epoch_num in range(self._num_updates)
]
expected_schedule = [
round(scheduler_0(epoch_num / self._num_updates), 4)
for epoch_num in range(0, int(self._num_updates), 2)
] + [
round(scheduler_1(epoch_num / self._num_updates), 4)
for epoch_num in range(int(self._num_updates / 2), self._num_updates)
]
self.assertEqual(fixed_schedule, expected_schedule)
def test_warmup_cosine(self):
p = nn.Parameter(torch.zeros(0))
opt = torch.optim.SGD([p], lr=5)
multiplier = CompositeParamScheduler(
[
LinearParamScheduler(0.001, 1), # warmup
CosineParamScheduler(1, 0),
],
interval_scaling=["rescaled", "fixed"],
lengths=[5 / 30, 25 / 30],
)
sched = LRMultiplier(opt, multiplier, 30)
p.sum().backward()
opt.step()
self.assertEqual(opt.param_groups[0]["lr"], 0.005)
lrs = [0.005]
for _ in range(30):
sched.step()
lrs.append(opt.param_groups[0]["lr"])
for idx, lr in enumerate(lrs):
expected_cosine = 2.5 * (1.0 + math.cos(math.pi * idx / 30))
if idx >= 5:
self.assertAlmostEqual(lr, expected_cosine)
else:
self.assertNotAlmostEqual(lr, expected_cosine)
def test_warmup_multistep(self):
p = nn.Parameter(torch.zeros(0))
opt = torch.optim.SGD([p], lr=5)
multiplier = CompositeParamScheduler(
[
LinearParamScheduler(0.001, 1), # warmup
MultiStepParamScheduler(
[1, 0.1, 0.01, 0.001],
milestones=[10, 15, 20],
num_updates=30,
),
],
interval_scaling=["rescaled", "fixed"],
lengths=[5 / 30, 25 / 30],
)
sched = LRMultiplier(opt, multiplier, 30)
# This is an equivalent of:
# sched = WarmupMultiStepLR(
# opt, milestones=[10, 15, 20], gamma=0.1, warmup_factor=0.001, warmup_iters=5)
p.sum().backward()
opt.step()
lrs = [0.005]
for _ in range(30):
sched.step()
lrs.append(opt.param_groups[0]["lr"])
self.assertTrue(
np.allclose(lrs[:5], [0.005, 1.004, 2.003, 3.002, 4.001]))
self.assertTrue(np.allclose(lrs[5:10], 5.0))
self.assertTrue(np.allclose(lrs[10:15], 0.5))
self.assertTrue(np.allclose(lrs[15:20], 0.05))
self.assertTrue(np.allclose(lrs[20:], 0.005))
def test_scheduler_lengths_within_epsilon_of_one(self):
config = self._get_lengths_sum_less_one_config()
scheduler = CompositeParamScheduler(**config)
schedule = [
scheduler(epoch_num / self._num_updates)
for epoch_num in range(self._num_updates)
]
expected_schedule = [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.2, 0.2, 0.2]
self.assertEqual(schedule, expected_schedule)
def test_long_scheduler(self):
config = self._get_valid_long_config()
scheduler = CompositeParamScheduler(**config)
schedule = [
scheduler(epoch_num / self._num_updates)
for epoch_num in range(self._num_updates)
]
expected_schedule = [0.1, 0.1, 0.2, 0.2, 0.2, 0.2, 0.3, 0.4, 0.4, 0.4]
self.assertEqual(schedule, expected_schedule)
def build_lr_scheduler(
cfg: CfgNode, optimizer: torch.optim.Optimizer
) -> torch.optim.lr_scheduler._LRScheduler:
"""
Build a LR scheduler from config.
"""
name = cfg.SOLVER.LR_SCHEDULER_NAME
if name == "WarmupMultiStepLR":
sched = MultiStepParamScheduler(
values=[
cfg.SOLVER.GAMMA**k for k in range(len(cfg.SOLVER.STEPS) + 1)
],
milestones=cfg.SOLVER.STEPS,
num_updates=cfg.SOLVER.MAX_ITER,
)
elif name == "WarmupCosineLR":
sched = CosineParamScheduler(1, 0)
else:
raise ValueError("Unknown LR scheduler: {}".format(name))
# Add warmup
warmup_method = cfg.SOLVER.WARMUP_METHOD
if warmup_method == "constant":
warmup = ConstantParamScheduler(cfg.SOLVER.WARMUP_FACTOR)
elif warmup_method == "linear":
warmup = LinearParamScheduler(cfg.SOLVER.WARMUP_FACTOR, 1.0)
else:
raise ValueError("Unknown warmup method: {}".format(warmup_method))
warmup_ratio = cfg.SOLVER.WARMUP_ITERS / cfg.SOLVER.MAX_ITER
sched = CompositeParamScheduler(
[warmup, sched],
interval_scaling=["rescaled", "fixed"],
lengths=[warmup_ratio, 1 - warmup_ratio],
)
return LRMultiplier(optimizer,
multiplier=sched,
max_iter=cfg.SOLVER.MAX_ITER)