def test_set_wds_with_list_of_values(): lo = LayerOptimizer(FakeOpt, params_("A", "B", "C"), 1e-2, 1e-4) lo.set_wds([9e-3, 8e-3, 7e-3]) check_optimizer_( lo.opt, [("A", 1e-2, 9e-3), ("B", 1e-2, 8e-3), ("C", 1e-2, 7e-3)], )
def test_set_lrs_with_list_of_values(): lo = LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, 1e-4) lo.set_lrs([2e-2, 3e-2, 4e-2]) check_optimizer_( lo.opt, [('A', 2e-2, 1e-4), ('B', 3e-2, 1e-4), ('C', 4e-2, 1e-4)], )
def test_set_lrs_with_list_of_values(): lo = LayerOptimizer(FakeOpt, params_("A", "B", "C"), 1e-2, 1e-4) lo.set_lrs([2e-2, 3e-2, 4e-2]) check_optimizer_( lo.opt, [("A", 2e-2, 1e-4), ("B", 3e-2, 1e-4), ("C", 4e-2, 1e-4)], )
def test_set_wds_with_list_of_values(): lo = LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, 1e-4) lo.set_wds([9e-3, 8e-3, 7e-3]) check_optimizer_( lo.opt, [('A', 1e-2, 9e-3), ('B', 1e-2, 8e-3), ('C', 1e-2, 7e-3)], )
def test_set_wds_list(self): lo = LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, 1e-4) lo.set_wds([9e-3, 8e-3, 7e-3]) self.check_optimizer_( lo.opt, [('A', 1e-2, 9e-3), ('B', 1e-2, 8e-3), ('C', 1e-2, 7e-3)], )
def test_set_lrs_list(self): lo = LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, 1e-4) lo.set_lrs([2e-2, 3e-2, 4e-2]) self.check_optimizer_( lo.opt, [('A', 2e-2, 1e-4), ('B', 3e-2, 1e-4), ('C', 4e-2, 1e-4)], )
def _generate_clr_sched(self, model:nn.Module, use_clr_beta:(int), lrs:[float], cycle_len:int): wds = 1e-7 opt_fn = partial(optim.Adam, betas=(0.0,0.9)) layer_opt = LayerOptimizer(opt_fn, self._get_inner_module(model).get_layer_groups(), lrs, wds) div,pct = use_clr_beta[:2] moms = use_clr_beta[2:] if len(use_clr_beta) > 3 else None cycle_end = None return CircularLR_beta(layer_opt, len(self.md.trn_dl)*cycle_len, on_cycle_end=cycle_end, div=div, pct=pct, momentums=moms)
def test_construction_with_lists_of_lrs_and_wds(): lo = LayerOptimizer( FakeOpt, params_('A', 'B', 'C'), (1e-2, 2e-2, 3e-2), (9e-3, 8e-3, 7e-3), ) check_optimizer_( lo.opt, [('A', 1e-2, 9e-3), ('B', 2e-2, 8e-3), ('C', 3e-2, 7e-3)], )
def test_init_list(self): lo = LayerOptimizer( FakeOpt, params_('A', 'B', 'C'), (1e-2, 2e-2, 3e-2), (9e-3, 8e-3, 7e-3), ) self.check_optimizer_( lo.opt, [('A', 1e-2, 9e-3), ('B', 2e-2, 8e-3), ('C', 3e-2, 7e-3)], )
def test_construction_with_lists_of_lrs_and_wds(): lo = LayerOptimizer( FakeOpt, params_("A", "B", "C"), (1e-2, 2e-2, 3e-2), (9e-3, 8e-3, 7e-3), ) check_optimizer_( lo.opt, [("A", 1e-2, 9e-3), ("B", 2e-2, 8e-3), ("C", 3e-2, 7e-3)], )
def test_set_wds_with_single_value(): lo = LayerOptimizer(FakeOpt, params_("A", "B", "C"), 1e-2, 1e-4) lo.set_wds(1e-5) check_optimizer_(lo.opt, [(nm, 1e-2, 1e-5) for nm in "ABC"])
def test_construction_with_too_few_wds(): with pytest.raises(AssertionError): LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, (9e-3, 8e-3))
def test_construction_with_too_few_lrs(): with pytest.raises(AssertionError): LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), (1e-2, 2e-2), 1e-4)
def test_construction_with_singleton_lr_and_wd(): lo = LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, 1e-4) check_optimizer_(lo.opt, [(nm, 1e-2, 1e-4) for nm in 'ABC'])
def test_layer_optimizer(self): lo = LayerOptimizer(FakeOpt, fastai_params_('A', 'B', 'C'), 1e-2, 1e-4) fast_check_optimizer_(lo.opt, [(nm, 1e-2, 1e-4) for nm in 'ABC'])
def test_set_wds_atomic(self): lo = LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, 1e-4) lo.set_wds(1e-5) self.check_optimizer_(lo.opt, [(nm, 1e-2, 1e-5) for nm in 'ABC'])
def test_construction_with_singleton_lr_and_wd(): lo = LayerOptimizer(FakeOpt, params_("A", "B", "C"), 1e-2, 1e-4) check_optimizer_(lo.opt, [(nm, 1e-2, 1e-4) for nm in "ABC"])
def test_init_malformed_wd(self): with self.assertRaises(AssertionError): LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, (9e-3, 8e-3))
def test_init_malformed_lr(self): with self.assertRaises(AssertionError): LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), (1e-2, 2e-2), 1e-4)
def test_init_atomic(self): lo = LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, 1e-4) self.check_optimizer_(lo.opt, [(nm, 1e-2, 1e-4) for nm in 'ABC'])
def test_set_lrs_with_too_few_values(): lo = LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, 1e-4) with pytest.raises(AssertionError): lo.set_lrs([2e-2, 3e-2]) # Also make sure the optimizer didn't change. check_optimizer_(lo.opt, [(nm, 1e-2, 1e-4) for nm in 'ABC'])
def test_set_lrs_atomic(): lo = LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, 1e-4) lo.set_lrs(1e-3) check_optimizer_(lo.opt, [(nm, 1e-3, 1e-4) for nm in 'ABC'])
def test_set_wds_with_single_value(): lo = LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, 1e-4) lo.set_wds(1e-5) check_optimizer_(lo.opt, [(nm, 1e-2, 1e-5) for nm in 'ABC'])
def test_set_wds_malformed(self): lo = LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, 1e-4) with self.assertRaises(AssertionError): lo.set_wds([9e-3, 8e-3]) self.check_optimizer_(lo.opt, [(nm, 1e-2, 1e-4) for nm in 'ABC'])
def test_set_wds_with_too_few_values(): lo = LayerOptimizer(FakeOpt, params_("A", "B", "C"), 1e-2, 1e-4) with pytest.raises(AssertionError): lo.set_wds([9e-3, 8e-3]) # Also make sure the optimizer didn't change. check_optimizer_(lo.opt, [(nm, 1e-2, 1e-4) for nm in "ABC"])
def test_set_lrs_malformed(): lo = LayerOptimizer(FakeOpt, params_('A', 'B', 'C'), 1e-2, 1e-4) with pytest.raises(AssertionError): lo.set_lrs([2e-2, 3e-2]) check_optimizer_(lo.opt, [(nm, 1e-2, 1e-4) for nm in 'ABC'])