def test_scale_fn(self): finder = lrf.CyclicLR(scale_fn=None) self.assertTrue(finder.scale_fn(100) == 1.0) scaler = lambda x: x**2 finder2 = lrf.CyclicLR(scale_fn=scaler) self.assertTrue(finder2.scale_fn(12) == 144)
def test_modes(self): finder = lrf.CyclicLR(mode='triangular') self.assertTrue(finder.scale_mode == 'cycle') self.assertTrue(finder.scale_fn(100) == 1.0) true_scale_fn = lambda x: 1 / (2.**(x - 1)) finder2 = lrf.CyclicLR(mode='triangular2') self.assertTrue(finder.scale_mode == 'cycle') self.assertTrue(finder2.scale_fn(12) == true_scale_fn(12)) gamma = 0.9 true_scale_fn = lambda x: gamma**x finder3 = lrf.CyclicLR(mode='exp_range', gamma=gamma) self.assertTrue(finder3.scale_mode == 'iterations') self.assertTrue(finder3.scale_fn(12) == true_scale_fn(12))
def test_end_to_end(self): import torch.optim import torch.nn model = torch.nn.Linear(10, 10) model2 = torch.nn.Linear(10, 10) optim = torch.optim.SGD([{ 'params': model.parameters() }, { 'params': model2.parameters(), 'lr': 1e-3 }], lr=1e-2, momentum=0.9) clr = lrf.CyclicLR(step_size=75, base_lr=[0.001, 0.0001], max_lr=[0.006, 0.0006]) clr.on_start({torchbearer.OPTIMIZER: optim}) lrs = [] for i in range(100): clr.on_sample({torchbearer.OPTIMIZER: optim}) clr.on_step_training({torchbearer.OPTIMIZER: optim}) for param_group in optim.param_groups: lr = param_group['lr'] lrs.append(lr) for i, param_group in enumerate(optim.param_groups): lr = param_group['lr'] self.assertAlmostEqual(lr, 0.004399999999999999 * 0.1**i)
def test_next_lr_tri(self): finder = lrf.CyclicLR(step_size=100) self.assertAlmostEqual(finder.next_lr(0, 0), 0.001, places=5) self.assertAlmostEqual(finder.next_lr(10, 0), 0.0015000000000000005, places=5) self.assertAlmostEqual(finder.next_lr(100, 0), 0.006, places=5) self.assertAlmostEqual(finder.next_lr(110, 0), 0.006 - 0.0005, places=5) self.assertAlmostEqual(finder.next_lr(200, 0), 0.001, places=5)
def test_next_lr_tri2(self): scale_fn = lambda x: 1 / (2.**(x - 1)) cycle_fn = lambda x: np.floor(1 + x / (2 * 100)) finder = lrf.CyclicLR(mode='triangular2', step_size=100) self.assertAlmostEqual(finder.next_lr(0, 0), 0.001, places=5) self.assertAlmostEqual(finder.next_lr(10, 0), 0.0015000000000000005, places=5) self.assertAlmostEqual(finder.next_lr(100, 0), 0.006, places=5) self.assertAlmostEqual(finder.next_lr(110, 0), (0.006 - 0.0005) * scale_fn(cycle_fn(110)), places=5) self.assertAlmostEqual(finder.next_lr(200, 0), (0.001) * scale_fn(cycle_fn(200)), places=5)
def test_next_lr_exp_range(self): gamma = 0.9 scale_fn = lambda x: gamma**x finder = lrf.CyclicLR(mode='exp_range', step_size=100, gamma=gamma) self.assertAlmostEqual(finder.next_lr(0, 0), 0.001, places=5) finder.iterations = 10 self.assertAlmostEqual(finder.next_lr(10, 0), 0.0015000000000000005 * scale_fn(10), places=5) finder.iterations = 100 self.assertAlmostEqual(finder.next_lr(100, 0), 0.006 * scale_fn(100), places=5) finder.iterations = 110 self.assertAlmostEqual(finder.next_lr(110, 0), (0.006 - 0.0005) * scale_fn(110), places=5) finder.iterations = 200 self.assertAlmostEqual(finder.next_lr(200, 0), (0.001) * scale_fn(200), places=5)