def test_l2_regularization(self): model = torch.nn.Sequential(torch.nn.Linear(5, 10), torch.nn.Linear(10, 5)) initializer = InitializerApplicator( default_initializer=lambda tensor: constant(tensor, 0.5)) initializer(model) value = RegularizerApplicator({"": L2Regularizer(1.0)})(model) assert value.data.numpy() == 28.75
def test_from_params(self): params = Params({"regularizers": {"conv": "l1", "linear": {"type": "l2", "alpha": 10}}}) regularizer_applicator = RegularizerApplicator.from_params(params) regularizers = regularizer_applicator._regularizers # pylint: disable=protected-access assert isinstance(regularizers["conv"], L1Regularizer) assert isinstance(regularizers["linear"], L2Regularizer) assert regularizers["linear"].alpha == 10
def test_l1_regularization(self): model = torch.nn.Sequential(torch.nn.Linear(5, 10), torch.nn.Linear(10, 5)) initializer = InitializerApplicator( default_initializer=lambda tensor: constant(tensor, -1)) initializer(model) value = RegularizerApplicator({"": L1Regularizer(1.0)})(model) # 115 because of biases. assert value.data.numpy() == 115.0
def test_regularizer_applicator_respects_regex_matching(self): model = torch.nn.Sequential( torch.nn.Linear(5, 10), torch.nn.Linear(10, 5) ) initializer = InitializerApplicator(default_initializer=lambda tensor: constant(tensor, 1.)) initializer(model) value = RegularizerApplicator({"weight": L2Regularizer(0.5), "bias": L1Regularizer(1.0)})(model) assert value.data.numpy() == 65.0