def __init__(self, encode_size, hidden_size, num_layers, bs, output_size):
super().__init__()
self.encode_size = encode_size
self.hidden_size = hidden_size
self.num_layers = num_layers
self.bs = bs
self.output_size = output_size
resnet_model = resnet34(pretrained=True)
encoder_layers = list(resnet_model.children())[:8] + [
AdaptiveConcatPool2d(), Flatten()
]
self.encoder = nn.Sequential(*encoder_layers).cuda()
for param in self.encoder.parameters():
param.requires_grad = False
set_trainable(self.encoder, False) # fastai fit bug
self.encoder_linear = nn.Sequential(nn.BatchNorm1d(1024),
nn.Dropout(p=0.25),
nn.Linear(1024, 512), nn.ReLU(),
nn.BatchNorm1d(512),
nn.Dropout(p=0.5),
nn.Linear(512,
encode_size)).cuda()
set_trainable(self.encoder_linear, True) # fastai fit bug
self.lstm = nn.LSTM(encode_size, hidden_size, num_layers).cuda()
self.h, self.c = self.init_hidden()
set_trainable(self.lstm, True) # fastai fit bug
self.linear = nn.Linear(hidden_size, output_size).cuda()
set_trainable(self.linear, True) # fastai fit bug
self.init_weights()
def __init__(self):
super().__init__()
resnet_model = resnet34(pretrained=True)
encoder_layers = list(resnet_model.children())[:8] + [
AdaptiveConcatPool2d(), Flatten()
]
self.encoder = nn.Sequential(*encoder_layers).cuda()
for param in self.encoder.parameters():
param.requires_grad = False
set_trainable(self.encoder, False) # fastai fit bug
self.linear = nn.Sequential(nn.BatchNorm1d(1024), nn.Dropout(p=0.25),
nn.Linear(1024, 512), nn.ReLU(),
nn.BatchNorm1d(512), nn.Dropout(p=0.5),
nn.Linear(512, 14)).cuda()
apply_init(self.linear, kaiming_normal)
set_trainable(self.linear, True) # fastai fit bug
def test_set_trainable():
layer1 = torch.nn.Linear(2, 2)
layer2 = torch.nn.Linear(2, 1)
model = torch.nn.Sequential(layer1, layer2)
params_require_grad_before = list(
filter(lambda param: param.requires_grad == True, model.parameters()))
core.set_trainable(model, False)
params_require_grad_after = list(
filter(lambda param: param.requires_grad == True, model.parameters()))
assert len(params_require_grad_before) == 4
assert len(params_require_grad_after) == 0
assert model.trainable == False
assert layer1.trainable == False
assert layer2.trainable == False
def test_set_trainable():
layer1 = torch.nn.Linear(2, 2)
layer2 = torch.nn.Linear(2, 1)
model = torch.nn.Sequential(layer1, layer2)
params_require_grad_before = list(filter(lambda param: param.requires_grad == True,
model.parameters()))
core.set_trainable(model, False)
params_require_grad_after = list(filter(lambda param: param.requires_grad == True,
model.parameters()))
assert len(params_require_grad_before) == 4
assert len(params_require_grad_after) == 0
assert model.trainable == False
assert layer1.trainable == False
assert layer2.trainable == False
next(iter(trn_dl))[0].size() # In[17]: learner.get_layer_groups() # In[18]: from fastai.core import set_trainable set_trainable(learner.model, False) set_trainable(learner.get_layer_groups()[0], True) set_trainable(learner.get_layer_groups()[-1], True) assert learner.model[0].blocks[0].trainable == False assert learner.model[0].blocks[0].attn.c_proj.weight.requires_grad == False assert learner.model[1].weight.requires_grad == True assert learner.model[1].trainable == True assert learner.model[0].embed.trainable == True assert learner.model[0].embed.weight.requires_grad == True # In[19]: lr=1e-3 lrs = lr
