def question_1i_sanity_check(model): """ Sanity check for highway network """ print("-" * 80) print("Running Sanity Check for Question 1i: Padding") print("-" * 80) vocab = VocabEntry() print("Running test on a list of sentences") sentences = [['Human:', 'What', 'do', 'we', 'want?'], ['Computer:', 'Natural', 'language', 'processing!'], ['Human:', 'When', 'do', 'we', 'want', 'it?'], ['Computer:', 'When', 'do', 'we', 'want', 'what?']] word_ids = vocab.words2charindices(sentences) #padded_sentences = pad_sents_char(word_ids, 0) padded_sentences = vocab.to_input_tensor_char(sentences, model.device) gold_padded_sentences = torch.load( './sanity_check_en_es_data/gold_padded_sentences.pkl') #Test with batch size 4 x = torch.rand(4, 1, 21) e_word = 19 k = 2 max_word_len = 21 cnn = CNN(1, e_word, k) #,max_word_len) cnn.forward(x) #Test with batch size 4 #print(a.size()) #print(padded_sentences.size()) #assert padded_sentences.size() == a.size(), "to_input_tensor size incorrect! is incorrect: it should be:\n {} but is:\n{}".format(a.size(), padded_sentences.size()) print("Sanity Check Passed for Question 1i: Padding!") print("-" * 80)
def test_cnn_k_too_big(self): e_word = 30 m_word = 4 e_char = 20 ones = np.ones((8, e_char, m_word)) x = torch.Tensor(ones) cnn_model = CNN(e_char, e_word, k=m_word + 1) with self.assertRaises(RuntimeError): cnn_model.forward(x)
class ModelCharEmbeddings(nn.Module): """ Class that converts input words to their CNN-based embeddings. """ def __init__(self, embed_size, vocab, dropout_rate=0.3): """ Init the Embedding layer for one language @param embed_size (int): Embedding size (dimensionality) for the output @param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation. """ super(ModelCharEmbeddings, self).__init__() ## A4 code # pad_token_idx = vocab.src['<pad>'] # self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx) ## End A4 code ### YOUR CODE HERE for part 1j #embed_size is e_word #padding_idx=vocab.char2id['<pad>'] #the handout says e_char = 50 self.embed_size = embed_size self.embeddings = nn.Embedding(len(vocab.char2id), 50) self.highway = Highway(embed_size) self.cnn = CNN(50, embed_size) self.dropout = nn.Dropout(p=dropout_rate) ### END YOUR CODE def forward(self, input): """ Looks up character-based CNN embeddings for the words in a batch of sentences. @param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where each integer is an index into the character vocabulary @param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the CNN-based embeddings for each word of the sentences in the batch """ ## A4 code # output = self.embeddings(input) # return output ## End A4 code ### YOUR CODE HERE for part 1j embeddings = self.embeddings(input) # input = input.permute(1,0,2) #now input is ->(batch_size, sentence_length,max_word_length) #now embeddings is ->(batch_size, sentence_length,max_word_length, e_char) #reshape to -> (batch_size x sentence_length,max_word_length, e_char) sentence_length, batch_size, max_word_length, e_char = embeddings.shape embeddings = embeddings.view(batch_size * sentence_length, e_char, max_word_length) #cnn_embeds -> batch_size x sentence_length, e_word cnn_embeds = self.cnn.forward(embeddings) output = self.highway.forward(cnn_embeds) x_wordemb = self.dropout(output) _, e_word = x_wordemb.shape x_wordemb = x_wordemb.view(sentence_length, batch_size, e_word) return x_wordemb
class ModelEmbeddings(nn.Module): """ Class that converts input words to their CNN-based embeddings. """ def __init__(self, embed_size, vocab): """ Init the Embedding layer for one language @param embed_size (int): Embedding size (dimensionality) for the output @param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation. """ super(ModelEmbeddings, self).__init__() ## A4 code # pad_token_idx = vocab.src['<pad>'] # self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx) ## End A4 code ### YOUR CODE HERE for part 1f c_embed = 50 pad_token_id = vocab.char2id['<pad>'] self.embed_size = embed_size self.embeddings = nn.Embedding(len(vocab.char2id), c_embed, padding_idx=pad_token_id) self.cnn = CNN(c_embed=c_embed, w_embed=embed_size, k=5) self.highway = Highway(embed_word=embed_size, dropout_rate=0.3) ### END YOUR CODE def forward(self, input_tensor): """ Looks up character-based CNN embeddings for the words in a batch of sentences. @param input_tensor: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where each integer is an index into the character vocabulary @param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the CNN-based embeddings for each word of the sentences in the batch """ ## A4 code # output = self.embeddings(input) # return output ## End A4 code ### YOUR CODE HERE for part 1f [sentence_length, batch_size, max_word_length] = list(input_tensor.size()) input_batch = input_tensor.contiguous().view(-1, max_word_length) x_emb = self.embeddings(input_batch) x_emb = x_emb.permute(0, 2, 1) x_conv_out = self.cnn.forward(x_emb) x_word_emb = self.highway(x_conv_out) x_word_emb_unbatched = x_word_emb.view(sentence_length, batch_size, -1) return x_word_emb_unbatched
class CNNClassifier(nn.Module): """ Class that converts input words to their CNN-based embeddings. """ def __init__(self, embed_size, kenerl_size=5): """ Init the Embedding layer for one language @param embed_size (int): Embedding size (dimensionality) for the output @param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation. """ super(CNNClassifier, self).__init__() self.embed_size = embed_size self.kenerl_size = kenerl_size self.embeddings = nn.Embedding(len(vocab.char2id), self.embed_char, padding_idx=pad_token_idx) self.cnn = CNN(self.embed_char, self.kenerl_size, self.embed_size) #self.embed_size is number of filter self.highway = Highway(self.embed_size, dropout_rate=0.3) def forward(self, input): """ Looks up character-based CNN embeddings for the words in a batch of sentences. @param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where each integer is an index into the character vocabulary @param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the CNN-based embeddings for each word of the sentences in the batch """ ## A4 code # output = self.embeddings(input) # return output ## End A4 code ### YOUR CODE HERE for part 1f #print(input.shape) output = self.embeddings(input) #print(output.shape) x_reshape = output.permute( 0, 1, 3, 2) ##(sentence_length, batch_size, ebed_char, max_word_length) shape = x_reshape.shape sentence_length = shape[0] batch_size = shape[1] max_word_length = shape[3] x_reshape = x_reshape.view(-1, self.embed_char, max_word_length) #print(x_reshape.shape) x_cnn = self.cnn.forward(x_reshape) #print(x_cnn.shape) x_highway = self.highway.forward( x_cnn.view(sentence_length, batch_size, self.embed_size)) return x_highway # (sentence_length, batch_size, embed_size, max_word_length-k+1) # batch_size, sentence_length, embed_size, max_word_length-k+1 ### END YOUR CODE
def question_1g_sanity_check(): """ Sanity check for CNN Class init and forward methods """ print("-" * 80) print("Running Sanity Check for Question 1g: CNN layer") print("-" * 80) print("Running test on a list of out conv layers") B = 2 m_word = 5 e_char = 3 f = 3 k = 4 ch1, ch2, ch3, ch4 = [1, 1, 1], [1, -1, 1], [1, 0, 1], [0, 3, 1] x_reshape = torch.Tensor([[ch1, ch2, ch3, ch4, ch4], [ch4, ch2, ch4, ch1, ch3]]) cnn_layer = CNN(m_word, f, k) my_conv = cnn_layer.forward(x_reshape) output_expected_size = [B, f] assert list( my_conv.size() ) == output_expected_size, "output shape is incorrect: it should be:\n {} but is:\n{}".format( output_expected_size, list(my_conv.size())) print("Sanity Check Passed for Question 1g: Correct Output Shape!") print("-" * 80)
class ModelEmbeddings(nn.Module): """ Class that converts input words to their CNN-based embeddings. """ def __init__(self, embed_size, vocab): """ Init the Embedding layer for one language @param embed_size (int): Embedding size (dimensionality) for the output @param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation. """ super(ModelEmbeddings, self).__init__() ## A4 code # pad_token_idx = vocab.src['<pad>'] # self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx) ## End A4 code ### YOUR CODE HERE for part 1j pad_token_idx = vocab['<pad>'] self.vocab = vocab self.embed_size = embed_size self.embeddings = nn.Embedding(len(vocab.char2id), embedding_dim=50, padding_idx=pad_token_idx) self.dropout = nn.Dropout(p=0.3) self.Highway = Highway(embed_size) self.CNN = CNN(embed_size, char_embed=50) ### END YOUR CODE def forward(self, input): """ Looks up character-based CNN embeddings for the words in a batch of sentences. @param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where each integer is an index into the character vocabulary @param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the CNN-based embeddings for each word of the sentences in the batch """ ## A4 code # output = self.embeddings(input) # return output ## End A4 code ### YOUR CODE HERE for part 1j l = input.shape[0] batch_size = input.shape[1] x_embed = self.embeddings(input) x_reshaped = x_embed.permute(0, 1, 3, 2) x_reshaped = x_reshaped.contiguous().view( x_reshaped.shape[0] * x_reshaped.shape[1], x_reshaped.shape[2], x_reshaped.shape[3]) x_convout = self.CNN.forward(x_reshaped) x_highway = self.Highway.forward(x_convout) output = self.dropout(x_highway) output = output.view(l, batch_size, output.shape[-1]) return output
class ModelEmbeddings(nn.Module): """ Class that converts input words to their CNN-based embeddings. """ def __init__(self, embed_size, vocab): """ Init the Embedding layer for one language @param embed_size (int): Embedding size (dimensionality) for the output @param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation. """ super(ModelEmbeddings, self).__init__() ## A4 code # pad_token_idx = vocab.src['<pad>'] # self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx) ## End A4 code ### YOUR CODE HERE for part 1j self.embed_size = embed_size self.char_embed_size = 50 self.x_embeddings = None pad_token_idx = vocab['<pad>'] self.embeddings = nn.Embedding(len(vocab.char2id), self.char_embed_size, padding_idx=pad_token_idx) self.cnn = CNN(output_features=self.embed_size, char_embeddings=self.char_embed_size) self.highway = Highway(self.embed_size) ### END YOUR CODE def forward(self, input): """ Looks up character-based CNN embeddings for the words in a batch of sentences. @param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where each integer is an index into the character vocabulary @param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the CNN-based embeddings for each word of the sentences in the batch """ ## A4 code # output = self.embeddings(input) # return output ## End A4 code ### YOUR CODE HERE for part 1j self.x_embeddings = self.embeddings(input) new_embeds = self.x_embeddings.permute(0, 1, 3, 2) new_embeds2 = new_embeds.reshape( new_embeds.size()[0] * new_embeds.size()[1], new_embeds.size()[2], new_embeds.size()[3]) cnn_op = self.cnn.forward(new_embeds2) new_res = torch.squeeze(cnn_op, dim=2) highway_op = self.highway.forward(new_res) new_highway_op = highway_op.reshape(input.size()[0], input.size()[1], highway_op.size()[1]) return new_highway_op
def test_cnn_bigger_eword(self): e_word = 300 m_word = MAX_WORD_LEN e_char = 50 ones = np.ones((8, e_char, m_word)) x = torch.Tensor(ones) cnn_model = CNN(e_char, e_word, MAX_WORD_LEN) cnn_model.conv.weight.shape out = cnn_model.forward(x) self.assertEqual(out.shape, (8, e_word))
class ModelEmbeddings(nn.Module): """ Class that converts input words to their CNN-based embeddings. """ def __init__(self, embed_size, vocab): """ Init the Embedding layer for one language @param embed_size (int): Embedding size (dimensionality) for the output @param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation. """ super(ModelEmbeddings, self).__init__() ## A4 code # pad_token_idx = vocab.src['<pad>'] # self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx) ## End A4 code ### YOUR CODE HERE for part 1j pad_token_idx = vocab.char2id['<pad>'] self.embed_size = embed_size self.embeddings = nn.Embedding(len(vocab.char2id), 50, padding_idx=pad_token_idx) self.highway = Highway(e_word=embed_size) self.CNN = CNN(e_word=embed_size, e_char=50, m_word=21, kernel_size=5) self.dropout = nn.Dropout(p=0.3) ### END YOUR CODE def forward(self, input): """ Looks up character-based CNN embeddings for the words in a batch of sentences. Must map from x_padded to x_word_emb @param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where each integer is an index into the character vocabulary @param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the CNN-based embeddings for each word of the sentences in the batch """ ## A4 code # output = self.embeddings(input) # return output ## End A4 code ### YOUR CODE HERE for part 1j sentence_length = input.size()[0] batch_size = input.size()[1] input = input.reshape(sentence_length * batch_size, -1) x_embedding = self.embeddings(input) x_reshaped = x_embedding.permute(0, 2, 1) x_convout = self.CNN.forward(x_reshaped) x_convout = x_convout.squeeze() x_highway = self.highway.forward(x_convout) x_word_emb = self.dropout(x_highway) x_word_emb = x_word_emb.view(sentence_length, batch_size, -1) return x_word_emb
class ModelEmbeddings(nn.Module): """ Class that converts input words to their CNN-based embeddings. """ def __init__(self, embed_size, vocab): """ Init the Embedding layer for one language @param embed_size (int): Embedding size (dimensionality) for the output @param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation. """ super(ModelEmbeddings, self).__init__() ## A4 code # pad_token_idx = vocab.src['<pad>'] # self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx) ## End A4 code ### YOUR CODE HERE for part 1j char_embed_len = 50 pad_token_idx = vocab.char2id['<pad>'] self.embed_size = embed_size self.embeddings = nn.Embedding(len(vocab.char2id), char_embed_len, padding_idx=pad_token_idx) self.cnn = CNN(char_embed_len, embed_size) self.highway = Highway(embed_size) self.dropout = nn.Dropout(0.3) ### END YOUR CODE def forward(self, input): """ Looks up character-based CNN embeddings for the words in a batch of sentences. @param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where each integer is an index into the character vocabulary @param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the CNN-based embeddings for each word of the sentences in the batch """ ## A4 code # output = self.embeddings(input) # return output ## End A4 code ### YOUR CODE HERE for part 1j output = [] for sent in input: char_embed = self.embeddings(sent) # (b_s, w_l, embed_s) char_embed = char_embed.permute(0, 2, 1) # (b_s, embed_s, w_l) conv_out = self.cnn.forward(char_embed) # (b_s, embed_s) x_highway = self.highway(conv_out) # (b_s, embed_s) x_highway = self.dropout(x_highway) output.append(x_highway) output = torch.stack(output, 0) # (sent_len, batch_size, embed_size) ] return output
def question_1g_sanity_check(): ''' sanity check for conv layer shapes ''' print("-" * 80) print("Running sanity check for 1g: conv1d") print("-" * 80) sentence_length = 10 max_word_length = 21 inpt = torch.zeros(BATCH_SIZE, 100, max_word_length) model = CNN(100, 20, max_word_length) output = model.forward(inpt)
def question_1g_sanity_check(): char_embedding_size = 4 word_embedding_size = 8 sentence_length = 10 batch_size = 64 max_word_length = 15 # Test given inputs, output shape is expected x = torch.Tensor( np.ones((sentence_length, batch_size, char_embedding_size, max_word_length))) for factor in range(char_embedding_size): x[:, :, factor, :] *= factor cnn = CNN(word_embedding_size, char_embedding_size) output = cnn.forward(x) assert output.shape == (sentence_length, batch_size, word_embedding_size) # Test given inputs, when convolution is identity, output values are expected cnn = CNN(word_embedding_size, char_embedding_size) torch.nn.init.constant_(cnn.conv_1.weight, 1) cnn.conv_1.bias.data.fill_(0) output = cnn.forward(x) expected_output = torch.Tensor( np.ones((sentence_length, batch_size, word_embedding_size)) * 30) assert torch.all(torch.eq(output, expected_output)) # Test given inputs, max pooling is correct x = torch.Tensor( np.ones((sentence_length, batch_size, char_embedding_size, max_word_length))) for factor in range(char_embedding_size): x[:, :, factor, 5:] *= factor cnn = CNN(word_embedding_size, char_embedding_size) torch.nn.init.constant_(cnn.conv_1.weight, 1) offset = torch.Tensor([0, 1, 2, 3, 4, 5, 6, 7]) cnn.conv_1.bias = nn.Parameter(offset) output = cnn.forward(x) expected_output = torch.Tensor( np.ones((sentence_length, batch_size, word_embedding_size)) * 30) expected_output += offset assert torch.all(torch.eq(output, expected_output))
def question_1i_sanity_check(): print("-" * 80) print("Running Sanity Check for Question 1i: Model Embedding") print("-" * 80) model = CNN(4, 7, 10) input = torch.randn(1, 4, 10) res = model.forward(input) print(res.shape) assert res.shape == input.shape print("Sanity Check Passed for Question 1h: Highway!") print("-" * 80)
def test_cnn(): print() print("==="*30) print("\nCNN Class test") x_reshaped = torch.tensor( [ [ [[0, 1, 1], [-1, 1, 0]], # sentence a's word 1 [[0, 2, 1], [1, 0, 0]] # sentence b's word 1 ], [ [[0, 1, 1], [-5, 1, 0]], # sentence a's word 2 [[0, 2, 0], [-6, 0, 1]] # sentence b's word 2 ] ], dtype=torch.float, device=device ) sent_len = x_reshaped.shape[0] batch_size = x_reshaped.shape[0] correct_x_conv_out = np.array( [ [ [0.3235975, 0.6727363 ], # sentence a's word 1 [0.21636295, 0.528751 ] # sentence b's word 1 ], [ [1.91349313, 0.71236265], # sentence a's word 2 [2.961207, 0.45990318] # sentence b's word 2 ] ] ) model = CNN(2, 2, kernel_size=2).to(device) obtained_x_conv_out = model.forward(torch.flatten(x_reshaped, 0, 1)) obtained_x_conv_out = torch.stack(torch.split(obtained_x_conv_out, batch_size, dim=0)) obtained_x_conv_out = obtained_x_conv_out.cpu().detach().numpy() assert np.allclose(correct_x_conv_out, obtained_x_conv_out), \ "\n\nIncorrect x_conv_out\n\nCorrect x_conv_out:\n{}\n\nYour x_conv_out:\n{}". \ format(correct_x_conv_out, obtained_x_conv_out) print("\n\nx_conv_out =\n", obtained_x_conv_out, "\n") # # Check the weights # print("\nCNN weights:\n", model.convlayer.weight.cpu().detach().numpy()) # print("\n\nCNN bias:\n", model.convlayer.bias.cpu().detach().numpy()) print("\nCNN Test Passed!\n") print("==="*30)
class ModelEmbeddings(nn.Module): """ Class that converts input words to their CNN-based embeddings. """ def __init__(self, embed_size, vocab, dropout_rate=0.3): """ Init the Embedding layer for one language @param embed_size (int): Embedding size (dimensionality) for the output @param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation. """ super(ModelEmbeddings, self).__init__() ## A4 code # pad_token_idx = vocab.src['<pad>'] # self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx) ## End A4 code ### YOUR CODE HERE for part 1j self.embed_size = embed_size self.e_char = 50 pad_token_idx = vocab.char2id['<pad>'] self.embeddings = nn.Embedding(len(vocab.char2id), self.e_char, padding_idx=pad_token_idx) self.cnn = CNN(e_char=self.e_char, e_word=self.embed_size) self.highway = Highway(e_word=self.embed_size) self.dropout = nn.Dropout(p=dropout_rate) ### END YOUR CODE def forward(self, input): """ Looks up character-based CNN embeddings for the words in a batch of sentences. @param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where each integer is an index into the character vocabulary @param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the CNN-based embeddings for each word of the sentences in the batch """ ## A4 code # output = self.embeddings(input) # return output ## End A4 code ### YOUR CODE HERE for part 1j sentence_length, batch_size, max_word_length = input.shape x_emb = self.embeddings(input) x_reshape = x_emb.view(x_emb.shape[0] * x_emb.shape[1], x_emb.shape[2], x_emb.shape[3]).transpose(1, 2) x_conv_out = self.cnn.forward(x_reshape) x_highway = self.highway.forword(x_conv_out) x_word_emb = self.dropout(x_highway).view(sentence_length, batch_size, self.embed_size) return x_word_emb
class Embedding(nn.Module): """Embedding layer used by BiDAF, without the character-level component. Word-level embeddings are further refined using a 2-layer Highway Encoder (see `HighwayEncoder` class for details). Args: word_vectors (torch.Tensor): Pre-trained word vectors. char_vectors (torch.Tensor): Pre-trained character vectors. hidden_size (int): Size of hidden activations. drop_prob (float): Probability of zero-ing out activations """ def __init__(self, word_vectors, char_vectors, hidden_size, drop_prob): super(Embedding, self).__init__() self.drop_prob = drop_prob self.word_embed = nn.Embedding.from_pretrained(word_vectors) self.char_embed = nn.Embedding.from_pretrained(char_vectors, freeze=False) self.cnn = CNN(word_vectors.size(1), char_vectors.size(1)) self.highway = Highway(word_vectors.size(1)) self.proj = nn.Linear(word_vectors.size(1) * 2, hidden_size, bias=False) self.hwy = HighwayEncoder(2, hidden_size) def forward(self, x, y): input_shape = y.shape # (batch_size, seq_len, 16) = (64, seq_len, 16) input_reshaped = torch.reshape(y, (input_shape[0] * input_shape[1], input_shape[2])) # (64*seq_len, 16) # print('input_reshaped') # print(input_reshaped) x_padded = self.char_embed(input_reshaped) # 64-dimensional # print('x_padded') # print(x_padded) x_reshaped = x_padded.permute(0, 2, 1) # (64*seq_len, 64, 16) x_conv_out = self.cnn.forward(x_reshaped) # (64*seq_len, 300) # print('x_conv_out') # print(x_conv_out) x_highway = self.highway(x_conv_out) # print('x_highway') # print(x_highway) x_highway_reshaped = torch.reshape(x_highway, (input_shape[0], input_shape[1], x_highway.shape[1])) x_word_emb = F.dropout(x_highway_reshaped, self.drop_prob, self.training) word_emb = self.word_embed(x) # (batch_size, seq_len, embed_size) word_emb = F.dropout(word_emb, self.drop_prob, self.training) word_emb = torch.cat((word_emb, x_word_emb), dim=2) # print('word_emb') # print(word_emb) word_emb = self.proj(word_emb) # (batch_size, seq_len, hidden_size) word_emb = self.hwy(word_emb) # (batch_size, seq_len, hidden_size) # print('word_emb2') # print(word_emb) return word_emb # (batch_size, seq_len, 2 * embed_size) = (64, seq_len, 100)
def test_cnn(test_image_name): test_img = mpimg.imread(test_image_name) pkl_file = open('params.pkl', 'rb') type = test_image_name.rsplit(".")[1] params = pickle.load(pkl_file) params1, params2, params3, params4, params5, params6, params7, params8, params9 = params h, w, ip_channels = test_img.shape out_channels = 64 filter_size = 3 padding = 1 layer_1 = Convolution(ip_channels, out_channels, filter_size, padding) layer_1.weights = params1[0] layer_1.bias=params1[1] relu_1 = ReLU() layer_2 = Convolution(out_channels, out_channels, filter_size, padding) layer_2.weights = params2[0] layer_2.bias = params2[1] relu_2 = ReLU() layer_3 = Convolution(out_channels, out_channels, filter_size, padding) layer_3.weights = params3[0] layer_3.bias = params3[1] relu_3 = ReLU() layer_4 = Convolution(out_channels, out_channels, filter_size, padding) layer_4.weights = params4[0] layer_4.bias = params4[1] relu_4 = ReLU() layer_5 = Convolution(out_channels, out_channels, filter_size, padding) layer_5.weights = params5[0] layer_5.bias = params5[1] relu_5 = ReLU() layer_6 = Convolution(out_channels, out_channels, filter_size, padding) layer_6.weights = params6[0] layer_6.bias = params6[1] relu_6 = ReLU() layer_7 = Convolution(out_channels, out_channels, filter_size, padding) layer_7.weights = params7[0] layer_7.bias = params7[1] relu_7 = ReLU() layer_8 = Convolution(out_channels, out_channels, filter_size, padding) layer_8.weights = params8[0] layer_8.bias = params8[1] relu_8 = ReLU() layer_9 = Convolution(out_channels, ip_channels, filter_size, padding) layer_9.weights = params9[0] layer_9.bias = params9[1] layers = [(layer_1, relu_1), (layer_2, relu_2), (layer_3, relu_3), (layer_4, relu_4), (layer_5, relu_5), (layer_6, relu_6), (layer_7, relu_7), (layer_8, relu_8), (layer_9, None)] cnn_obj = CNN(layers) out_img = cnn_obj.forward(test_img) output_file_name = test_image_name + "_sr" + str(h) + "." + type cv2.imwrite(output_file_name, out_img)
def cnn_sanity_check(char_embed_size=1): char_embed_size = 1 f = max_word_length = 21 print("CNN Input") reshaped_embedding = torch.Tensor( [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2] * char_embed_size).view(max_word_length, char_embed_size).unsqueeze(2) print(reshaped_embedding) cnn = CNN(char_embed_size, f, k=1, max_word_length=21, bias=True) result = cnn.forward(reshaped_embedding) print("CNN Output") print(result) print(result.size())
class ModelEmbeddings(nn.Module): """ Class that converts input words to their CNN-based embeddings. """ def __init__(self, word_embed_size, vocab): """ Init the Embedding layer for one language @param word_embed_size (int): Embedding size (dimensionality) for the output word @param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation. Hints: - You may find len(self.vocab.char2id) useful when create the embedding """ super(ModelEmbeddings, self).__init__() ### YOUR CODE HERE for part 1h self.e_char = 50 self.word_embed_size = word_embed_size self.dropout_prob = 0.3 self.vocab = vocab # apparently 21 is an arbitrary value chosen for the sanity tests and also the greedy decoder self.max_word_len = 21 self.embedding = nn.Embedding(len(vocab.char2id), self.e_char, padding_idx=vocab.char2id['∏']) self.cnn = CNN(e_char=self.e_char, filters=self.word_embed_size, kernel_size=5, m_word=self.max_word_len) self.highway = Highway(word_embed_size) self.dropout = nn.Dropout(p=self.dropout_prob) ### END YOUR CODE def forward(self, input): """ Looks up character-based CNN embeddings for the words in a batch of sentences. @param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where each integer is an index into the character vocabulary @param output: Tensor of shape (sentence_length, batch_size, word_embed_size), containing the CNN-based embeddings for each word of the sentences in the batch """ ### YOUR CODE HERE for part 1h embedded = self.embedding.forward(input) max_sent_len, batch_size, samples, channels = embedded.shape reshaped = embedded.view( (max_sent_len * batch_size, samples, channels)).transpose(1, 2) conv_out = self.cnn.forward(reshaped) highway = self.highway.forward(conv_out.squeeze()) dropout = self.dropout.forward(highway) return dropout.view(max_sent_len, batch_size, -1)
class ModelEmbeddings(nn.Module): """ Class that converts input words to their CNN-based embeddings. """ def __init__(self, embed_size, vocab): """ Init the Embedding layer for one language @param embed_size (int): Embedding size (dimensionality) for the output @param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation. """ super(ModelEmbeddings, self).__init__() ## A4 code #pad_token_idx = vocab.src['<pad>'] #self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx) ## End A4 code ### YOUR CODE HERE for part 1j self.embed_size=embed_size self.char_embed_size=50 self.embeddings=nn.Embedding(len(vocab.char2id),self.char_embed_size,padding_idx=vocab.char2id['<pad>']) self.conv=CNN(in_channels=self.char_embed_size,out_channels=self.embed_size) self.highway=Highway(size=self.embed_size) self.dropout=nn.Dropout(p=0.3) ### END YOUR CODE def forward(self, input): """ Looks up character-based CNN embeddings for the words in a batch of sentences. @param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where each integer is an index into the character vocabulary @param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the CNN-based embeddings for each word of the sentences in the batch """ ## A4 code #x_emb = self.embeddings(input) # return output ## End A4 code ### YOUR CODE HERE for part 1j x_embed=self.embeddings(input) x_reshaped=torch.reshape(x_embed,[x_embed.size()[0]*x_embed.size()[1],x_embed.size()[3],x_embed.size()[2]]) x_conv_out=self.conv.forward(x_reshaped) x_highway=self.highway.forward(x_conv_out) x_word_emb=self.dropout(x_highway) output=torch.reshape(x_word_emb,[x_embed.size()[0],x_embed.size()[1],-1]) return output
def test_shape(self): print("-" * 80) print("Running Sanity Check for Question 1e: CNN Shape") print("-" * 80) max_word_len = 15 batch_size, char_embed_size, num_filters, window_size = 64, 20, 80, 5 cnn = CNN(char_embed_size, num_filters, max_word_len, window_size) x_emb = torch.randn([batch_size, char_embed_size, max_word_len]) x_conv_out = cnn.forward(x_emb) self.assertEqual(x_conv_out.shape, (batch_size, num_filters)) print("Sanity Check Passed for Question 1e: CNN Shape!") print("-" * 80)
def question_1i_sanity_check(): print("-" * 80) print("Running Sanity Check for Question 1i: question_1g_sanity_check") N, e_char, m_word, kernel_size, filter_num = 10, 5, 6, 5, 4 x_reshape = torch.randn((N, e_char, m_word)) # (N=10,e_char=5, m_word=6) cnn = CNN(m_word, e_char, filter_num, kernel_size=kernel_size) x_conv_out = cnn.forward(x_reshape) print("x_reshape = " + str(x_reshape)) print("x_conv_out = " + str(x_conv_out)) batch_size, filter_num_out = x_conv_out.shape assert (batch_size == N) assert (filter_num_out == filter_num) print("question_1i_sanity_check: shape passed!") print("-" * 80)
class CRNN(nn.Module): """ C-RNN Module. """ def __init__(self): super(CRNN, self).__init__() self.cnn = CNN() self.rnn = RNN(input_size=2048, hidden_size=256, output_size=4) def forward(self, x, hidden): x_feats = self.cnn.forward(x) output, hidden = self.rnn.forward(x_feats, hidden) return output, hidden def init_hidden(self, batch_size): """ Initialize hidden units to zero. """ return self.rnn.init_hidden(batch_size)
def question_1i_sanity_check(): # Sanity check for cnn.py print("-" * 80) print("Running Sanity Check for Question 1i: CNN") print("-" * 80) m_word = 6 e_char = 3 e_word = 4 inpt = torch.ones(BATCH_SIZE, e_char, m_word) cnn_net = CNN(e_char, e_word) output = cnn_net.forward(inpt) output_expected_size = [BATCH_SIZE, e_word] assert ( list(output.size()) == output_expected_size ), "output shape is incorrect: it should be:\n {} but is:\n{}".format( output_expected_size, list(output.size())) print("Sanity Check Passed for Question 1i: CNN!") print("-" * 80)
def question_1g_sanity_check(): """ Sanity check for cnn.py basic shape check """ print("-" * 80) print("Running Sanity Check for Question 1g: CNN") print("-" * 80) max_word_size = 10 char_embed_size = 50 x = torch.zeros(BATCH_SIZE,char_embed_size,max_word_size) print("input size: "+str(x.shape)) cnn = CNN(num_filters=EMBED_SIZE, kernel_size=5, char_embed_size=char_embed_size) result = cnn.forward(x) print("output size :",str(result.shape) ) assert(result.shape == (BATCH_SIZE, EMBED_SIZE)) print("Sanity Check Passed for Question 1g:cnn")
class TestCNN(unittest.TestCase): """ Sanity check for CNN module """ def setUp(self): self.char_emb_size = EMBED_SIZE-1 self.word_emb_size = EMBED_SIZE self.m_word = 10 self.model = CNN(char_embed_size=self.char_emb_size, word_embed_size=self.word_emb_size, max_len=self.m_word) def test_CNN_shape(self): self.model = initialize_layers(self.model) input = torch.ones((BATCH_SIZE, self.char_emb_size, self.m_word)) with torch.no_grad(): output = self.model.forward(input) self.assertEqual(output.shape, (BATCH_SIZE, self.word_emb_size), "incorrect output shape") self.assertEqual(self.model.conv1d.in_channels, self.char_emb_size, "incorrect input channel size") self.assertEqual(self.model.conv1d.out_channels, self.word_emb_size, "incorrect output channel size")
def question_1g_sanity_check(): """ Sanity check for the class `CNN`. """ print("-" * 80) print("Running Sanity Check for Question 1g: CNN") print("-" * 80) SENTENCE_LENGTH = 20 BATCH_SIZE = 5 E_CHAR = 50 M_WORD = 21 F = 3 # model = CNN(f=F, e_char = E_CHAR, m_word=M_WORD) model = CNN(f=F, e_char=E_CHAR) x_reshaped = torch.randn((SENTENCE_LENGTH, BATCH_SIZE, E_CHAR, M_WORD)) print("Running test on a batch of x_reshaped") x_conv_out = model.forward(x_reshaped) assert list(x_conv_out.size()) == [ SENTENCE_LENGTH, BATCH_SIZE, F ], "Output size should be: {}, but got {}".format( (SENTENCE_LENGTH, BATCH_SIZE, F), x_conv_out.size()) print("Sanity Check Passed for Question 1g: CNN!") print("-" * 80)
class ModelEmbeddings(nn.Module): """ Class that converts input words to their CNN-based embeddings. """ def __init__(self, embed_size, vocab): """ Init the Embedding layer for one language @param embed_size (int): Embedding size (dimensionality) for the output @param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation. """ super(ModelEmbeddings, self).__init__() ## A4 code # pad_token_idx = vocab.src['<pad>'] # self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx) ## End A4 code ### YOUR CODE HERE for part 1j pad_token_idx = vocab.char2id['<pad>'] self.char_embed_size = 50 self.dropout_rate = 0.3 self.max_word_size = 21 self.word_embed_size = embed_size self.embed_size = embed_size self.v_char = len(vocab.char2id) self.v_word = len(vocab.word2id) self.embeddings = nn.Embedding(self.v_char, self.char_embed_size, padding_idx=pad_token_idx) self.Dropout = nn.Dropout(p=self.dropout_rate) self.cnn = CNN(e_char=self.char_embed_size, e_word=self.word_embed_size, m_word=self.max_word_size) self.highway = Highway(embedding_size=self.word_embed_size) ### END YOUR CODE def forward(self, input): """ Looks up character-based CNN embeddings for the words in a batch of sentences. @param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where each integer is an index into the character vocabulary @param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the CNN-based embeddings for each word of the sentences in the batch """ ## A4 code # output = self.embeddings(input) # return output ## End A4 code ### YOUR CODE HERE for part 1j max_word_size = input.size()[-1] assert (max_word_size == self.max_word_size) char_embeddings = self.embeddings( input ) # (max_sent_len, batch_size, max_word_len, char_embedding_size) # conv1d only performs on the last dimension so we have to swap char_embeddings = char_embeddings.permute( 0, 1, 3, 2) # (max_sent_len, batch_size, char_embedding_size, max_word_len) max_sent_len = char_embeddings.size()[0] batch_size = char_embeddings.size()[1] char_embedding_size = char_embeddings.size()[2] max_word_len = char_embeddings.size()[3] # conv1d only accepts 3 dimension array, so any extra dimensions need to be concatenated. char_embeddings = char_embeddings.reshape( max_sent_len * batch_size, char_embedding_size, max_word_len ) # (max_sent_len * batch_size, char_embedding_size, max_word_len) cnn_out = self.cnn.forward( char_embeddings ) # (max_sent_len * batch_size, word_embedding_size) highway_out = self.highway.forward( cnn_out) # (max_sent_len * batch_size, word_embedding_size) dropout_out = self.Dropout(highway_out) output = dropout_out.reshape( max_sent_len, batch_size, dropout_out.size() [-1]) # (max_sent_len, batch_size, word_embedding_size) return output
class ModelEmbeddings(nn.Module): """ Class that converts input words to their CNN-based embeddings. """ def __init__(self, embed_size, vocab): """ Init the Embedding layer for one language @param embed_size (int): Embedding size (dimensionality) for the output @param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation. """ super(ModelEmbeddings, self).__init__() ## A4 code # pad_token_idx = vocab.src['<pad>'] # self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx) ## End A4 code ### YOUR CODE HERE for part 1j self.embed_size = embed_size self.embedding_char_size = 50 self.dropout_rate = 0.3 self.max_word_length = 21 self.embedding_word_size = embed_size pad_token_idx = vocab.char2id['<pad>'] self.charEmbeddings = nn.Embedding(len(vocab.char2id), self.embedding_char_size, padding_idx=pad_token_idx) self.dropout = nn.Dropout(p=self.dropout_rate) #construct CNN self.CNN = CNN(self.embedding_char_size, self.embedding_word_size, self.max_word_length) #construct Highway self.highway = Highway(self.embedding_word_size) ### END YOUR CODE def forward(self, input): """ Looks up character-based CNN embeddings for the words in a batch of sentences. @param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where each integer is an index into the character vocabulary @param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the CNN-based embeddings for each word of the sentences in the batch """ ## A4 code # output = self.embeddings(input) # return output ## End A4 code ### YOUR CODE HERE for part 1j sentence_length, batch_size, _ = input.shape x_emb = self.charEmbeddings(input) #note: x_emb : tensor of (sentence_length, batch_size, max_word_length, e_char) x_emb = x_emb.view((sentence_length * batch_size, self.max_word_length, self.embedding_char_size)).transpose(1, 2) #now x_emb is : tensor of (sentence_length * batch_size, e_char, max_word_length) #cnn needs: input shape: x_reshaped:(batch_size, char_embedding_size(also called e_char), max_word_length) # output shape: (batch_size, e_word) x_conv_out = self.CNN.forward(x_emb) #x_conv_out : (batch_size * sentence_length, e_word=embed_size) x_highway = self.highway.forward(x_conv_out) x_word_emb = self.dropout(x_highway) #now x_word_emb is: tensor of (sentence_length * batch_size, embed_size) output = x_word_emb.view( (sentence_length, batch_size, self.embedding_word_size)) return output