def question_1i_sanity_check(model):
""" Sanity check for highway network
"""
print("-" * 80)
print("Running Sanity Check for Question 1i: Padding")
print("-" * 80)
vocab = VocabEntry()
print("Running test on a list of sentences")
sentences = [['Human:', 'What', 'do', 'we', 'want?'],
['Computer:', 'Natural', 'language', 'processing!'],
['Human:', 'When', 'do', 'we', 'want', 'it?'],
['Computer:', 'When', 'do', 'we', 'want', 'what?']]
word_ids = vocab.words2charindices(sentences)
#padded_sentences = pad_sents_char(word_ids, 0)
padded_sentences = vocab.to_input_tensor_char(sentences, model.device)
gold_padded_sentences = torch.load(
'./sanity_check_en_es_data/gold_padded_sentences.pkl')
#Test with batch size 4
x = torch.rand(4, 1, 21)
e_word = 19
k = 2
max_word_len = 21
cnn = CNN(1, e_word, k) #,max_word_len)
cnn.forward(x)
#Test with batch size 4
#print(a.size())
#print(padded_sentences.size())
#assert padded_sentences.size() == a.size(), "to_input_tensor size incorrect! is incorrect: it should be:\n {} but is:\n{}".format(a.size(), padded_sentences.size())
print("Sanity Check Passed for Question 1i: Padding!")
print("-" * 80)
def test_cnn_k_too_big(self):
e_word = 30
m_word = 4
e_char = 20
ones = np.ones((8, e_char, m_word))
x = torch.Tensor(ones)
cnn_model = CNN(e_char, e_word, k=m_word + 1)
with self.assertRaises(RuntimeError):
cnn_model.forward(x)
class ModelCharEmbeddings(nn.Module):
"""
Class that converts input words to their CNN-based embeddings.
"""
def __init__(self, embed_size, vocab, dropout_rate=0.3):
"""
Init the Embedding layer for one language
@param embed_size (int): Embedding size (dimensionality) for the output
@param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation.
"""
super(ModelCharEmbeddings, self).__init__()
## A4 code
# pad_token_idx = vocab.src['<pad>']
# self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx)
## End A4 code
### YOUR CODE HERE for part 1j
#embed_size is e_word
#padding_idx=vocab.char2id['<pad>']
#the handout says e_char = 50
self.embed_size = embed_size
self.embeddings = nn.Embedding(len(vocab.char2id), 50)
self.highway = Highway(embed_size)
self.cnn = CNN(50, embed_size)
self.dropout = nn.Dropout(p=dropout_rate)
### END YOUR CODE
def forward(self, input):
"""
Looks up character-based CNN embeddings for the words in a batch of sentences.
@param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where
each integer is an index into the character vocabulary
@param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the
CNN-based embeddings for each word of the sentences in the batch
"""
## A4 code
# output = self.embeddings(input)
# return output
## End A4 code
### YOUR CODE HERE for part 1j
embeddings = self.embeddings(input)
# input = input.permute(1,0,2)
#now input is ->(batch_size, sentence_length,max_word_length)
#now embeddings is ->(batch_size, sentence_length,max_word_length, e_char)
#reshape to -> (batch_size x sentence_length,max_word_length, e_char)
sentence_length, batch_size, max_word_length, e_char = embeddings.shape
embeddings = embeddings.view(batch_size * sentence_length, e_char,
max_word_length)
#cnn_embeds -> batch_size x sentence_length, e_word
cnn_embeds = self.cnn.forward(embeddings)
output = self.highway.forward(cnn_embeds)
x_wordemb = self.dropout(output)
_, e_word = x_wordemb.shape
x_wordemb = x_wordemb.view(sentence_length, batch_size, e_word)
return x_wordemb
class ModelEmbeddings(nn.Module):
"""
Class that converts input words to their CNN-based embeddings.
"""
def __init__(self, embed_size, vocab):
"""
Init the Embedding layer for one language
@param embed_size (int): Embedding size (dimensionality) for the output
@param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation.
"""
super(ModelEmbeddings, self).__init__()
## A4 code
# pad_token_idx = vocab.src['<pad>']
# self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx)
## End A4 code
### YOUR CODE HERE for part 1f
c_embed = 50
pad_token_id = vocab.char2id['<pad>']
self.embed_size = embed_size
self.embeddings = nn.Embedding(len(vocab.char2id),
c_embed,
padding_idx=pad_token_id)
self.cnn = CNN(c_embed=c_embed, w_embed=embed_size, k=5)
self.highway = Highway(embed_word=embed_size, dropout_rate=0.3)
### END YOUR CODE
def forward(self, input_tensor):
"""
Looks up character-based CNN embeddings for the words in a batch of sentences.
@param input_tensor: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where
each integer is an index into the character vocabulary
@param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the
CNN-based embeddings for each word of the sentences in the batch
"""
## A4 code
# output = self.embeddings(input)
# return output
## End A4 code
### YOUR CODE HERE for part 1f
[sentence_length, batch_size,
max_word_length] = list(input_tensor.size())
input_batch = input_tensor.contiguous().view(-1, max_word_length)
x_emb = self.embeddings(input_batch)
x_emb = x_emb.permute(0, 2, 1)
x_conv_out = self.cnn.forward(x_emb)
x_word_emb = self.highway(x_conv_out)
x_word_emb_unbatched = x_word_emb.view(sentence_length, batch_size, -1)
return x_word_emb_unbatched
class CNNClassifier(nn.Module):
"""
Class that converts input words to their CNN-based embeddings.
"""
def __init__(self, embed_size, kenerl_size=5):
"""
Init the Embedding layer for one language
@param embed_size (int): Embedding size (dimensionality) for the output
@param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation.
"""
super(CNNClassifier, self).__init__()
self.embed_size = embed_size
self.kenerl_size = kenerl_size
self.embeddings = nn.Embedding(len(vocab.char2id),
self.embed_char,
padding_idx=pad_token_idx)
self.cnn = CNN(self.embed_char, self.kenerl_size,
self.embed_size) #self.embed_size is number of filter
self.highway = Highway(self.embed_size, dropout_rate=0.3)
def forward(self, input):
"""
Looks up character-based CNN embeddings for the words in a batch of sentences.
@param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where
each integer is an index into the character vocabulary
@param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the
CNN-based embeddings for each word of the sentences in the batch
"""
## A4 code
# output = self.embeddings(input)
# return output
## End A4 code
### YOUR CODE HERE for part 1f
#print(input.shape)
output = self.embeddings(input)
#print(output.shape)
x_reshape = output.permute(
0, 1, 3,
2) ##(sentence_length, batch_size, ebed_char, max_word_length)
shape = x_reshape.shape
sentence_length = shape[0]
batch_size = shape[1]
max_word_length = shape[3]
x_reshape = x_reshape.view(-1, self.embed_char, max_word_length)
#print(x_reshape.shape)
x_cnn = self.cnn.forward(x_reshape)
#print(x_cnn.shape)
x_highway = self.highway.forward(
x_cnn.view(sentence_length, batch_size, self.embed_size))
return x_highway
# (sentence_length, batch_size, embed_size, max_word_length-k+1)
# batch_size, sentence_length, embed_size, max_word_length-k+1
### END YOUR CODE
def question_1g_sanity_check():
""" Sanity check for CNN Class init and forward methods
"""
print("-" * 80)
print("Running Sanity Check for Question 1g: CNN layer")
print("-" * 80)
print("Running test on a list of out conv layers")
B = 2
m_word = 5
e_char = 3
f = 3
k = 4
ch1, ch2, ch3, ch4 = [1, 1, 1], [1, -1, 1], [1, 0, 1], [0, 3, 1]
x_reshape = torch.Tensor([[ch1, ch2, ch3, ch4, ch4],
[ch4, ch2, ch4, ch1, ch3]])
cnn_layer = CNN(m_word, f, k)
my_conv = cnn_layer.forward(x_reshape)
output_expected_size = [B, f]
assert list(
my_conv.size()
) == output_expected_size, "output shape is incorrect: it should be:\n {} but is:\n{}".format(
output_expected_size, list(my_conv.size()))
print("Sanity Check Passed for Question 1g: Correct Output Shape!")
print("-" * 80)
class ModelEmbeddings(nn.Module):
"""
Class that converts input words to their CNN-based embeddings.
"""
def __init__(self, embed_size, vocab):
"""
Init the Embedding layer for one language
@param embed_size (int): Embedding size (dimensionality) for the output
@param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation.
"""
super(ModelEmbeddings, self).__init__()
## A4 code
# pad_token_idx = vocab.src['<pad>']
# self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx)
## End A4 code
### YOUR CODE HERE for part 1j
pad_token_idx = vocab['<pad>']
self.vocab = vocab
self.embed_size = embed_size
self.embeddings = nn.Embedding(len(vocab.char2id),
embedding_dim=50,
padding_idx=pad_token_idx)
self.dropout = nn.Dropout(p=0.3)
self.Highway = Highway(embed_size)
self.CNN = CNN(embed_size, char_embed=50)
### END YOUR CODE
def forward(self, input):
"""
Looks up character-based CNN embeddings for the words in a batch of sentences.
@param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where
each integer is an index into the character vocabulary
@param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the
CNN-based embeddings for each word of the sentences in the batch
"""
## A4 code
# output = self.embeddings(input)
# return output
## End A4 code
### YOUR CODE HERE for part 1j
l = input.shape[0]
batch_size = input.shape[1]
x_embed = self.embeddings(input)
x_reshaped = x_embed.permute(0, 1, 3, 2)
x_reshaped = x_reshaped.contiguous().view(
x_reshaped.shape[0] * x_reshaped.shape[1], x_reshaped.shape[2],
x_reshaped.shape[3])
x_convout = self.CNN.forward(x_reshaped)
x_highway = self.Highway.forward(x_convout)
output = self.dropout(x_highway)
output = output.view(l, batch_size, output.shape[-1])
return output
class ModelEmbeddings(nn.Module):
"""
Class that converts input words to their CNN-based embeddings.
"""
def __init__(self, embed_size, vocab):
"""
Init the Embedding layer for one language
@param embed_size (int): Embedding size (dimensionality) for the output
@param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation.
"""
super(ModelEmbeddings, self).__init__()
## A4 code
# pad_token_idx = vocab.src['<pad>']
# self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx)
## End A4 code
### YOUR CODE HERE for part 1j
self.embed_size = embed_size
self.char_embed_size = 50
self.x_embeddings = None
pad_token_idx = vocab['<pad>']
self.embeddings = nn.Embedding(len(vocab.char2id),
self.char_embed_size,
padding_idx=pad_token_idx)
self.cnn = CNN(output_features=self.embed_size,
char_embeddings=self.char_embed_size)
self.highway = Highway(self.embed_size)
### END YOUR CODE
def forward(self, input):
"""
Looks up character-based CNN embeddings for the words in a batch of sentences.
@param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where
each integer is an index into the character vocabulary
@param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the
CNN-based embeddings for each word of the sentences in the batch
"""
## A4 code
# output = self.embeddings(input)
# return output
## End A4 code
### YOUR CODE HERE for part 1j
self.x_embeddings = self.embeddings(input)
new_embeds = self.x_embeddings.permute(0, 1, 3, 2)
new_embeds2 = new_embeds.reshape(
new_embeds.size()[0] * new_embeds.size()[1],
new_embeds.size()[2],
new_embeds.size()[3])
cnn_op = self.cnn.forward(new_embeds2)
new_res = torch.squeeze(cnn_op, dim=2)
highway_op = self.highway.forward(new_res)
new_highway_op = highway_op.reshape(input.size()[0],
input.size()[1],
highway_op.size()[1])
return new_highway_op
def test_cnn_bigger_eword(self):
e_word = 300
m_word = MAX_WORD_LEN
e_char = 50
ones = np.ones((8, e_char, m_word))
x = torch.Tensor(ones)
cnn_model = CNN(e_char, e_word, MAX_WORD_LEN)
cnn_model.conv.weight.shape
out = cnn_model.forward(x)
self.assertEqual(out.shape, (8, e_word))
class ModelEmbeddings(nn.Module):
"""
Class that converts input words to their CNN-based embeddings.
"""
def __init__(self, embed_size, vocab):
"""
Init the Embedding layer for one language
@param embed_size (int): Embedding size (dimensionality) for the output
@param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation.
"""
super(ModelEmbeddings, self).__init__()
## A4 code
# pad_token_idx = vocab.src['<pad>']
# self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx)
## End A4 code
### YOUR CODE HERE for part 1j
pad_token_idx = vocab.char2id['<pad>']
self.embed_size = embed_size
self.embeddings = nn.Embedding(len(vocab.char2id),
50,
padding_idx=pad_token_idx)
self.highway = Highway(e_word=embed_size)
self.CNN = CNN(e_word=embed_size, e_char=50, m_word=21, kernel_size=5)
self.dropout = nn.Dropout(p=0.3)
### END YOUR CODE
def forward(self, input):
"""
Looks up character-based CNN embeddings for the words in a batch of sentences.
Must map from x_padded to x_word_emb
@param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where
each integer is an index into the character vocabulary
@param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the
CNN-based embeddings for each word of the sentences in the batch
"""
## A4 code
# output = self.embeddings(input)
# return output
## End A4 code
### YOUR CODE HERE for part 1j
sentence_length = input.size()[0]
batch_size = input.size()[1]
input = input.reshape(sentence_length * batch_size, -1)
x_embedding = self.embeddings(input)
x_reshaped = x_embedding.permute(0, 2, 1)
x_convout = self.CNN.forward(x_reshaped)
x_convout = x_convout.squeeze()
x_highway = self.highway.forward(x_convout)
x_word_emb = self.dropout(x_highway)
x_word_emb = x_word_emb.view(sentence_length, batch_size, -1)
return x_word_emb
class ModelEmbeddings(nn.Module):
"""
Class that converts input words to their CNN-based embeddings.
"""
def __init__(self, embed_size, vocab):
"""
Init the Embedding layer for one language
@param embed_size (int): Embedding size (dimensionality) for the output
@param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation.
"""
super(ModelEmbeddings, self).__init__()
## A4 code
# pad_token_idx = vocab.src['<pad>']
# self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx)
## End A4 code
### YOUR CODE HERE for part 1j
char_embed_len = 50
pad_token_idx = vocab.char2id['<pad>']
self.embed_size = embed_size
self.embeddings = nn.Embedding(len(vocab.char2id),
char_embed_len,
padding_idx=pad_token_idx)
self.cnn = CNN(char_embed_len, embed_size)
self.highway = Highway(embed_size)
self.dropout = nn.Dropout(0.3)
### END YOUR CODE
def forward(self, input):
"""
Looks up character-based CNN embeddings for the words in a batch of sentences.
@param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where
each integer is an index into the character vocabulary
@param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the
CNN-based embeddings for each word of the sentences in the batch
"""
## A4 code
# output = self.embeddings(input)
# return output
## End A4 code
### YOUR CODE HERE for part 1j
output = []
for sent in input:
char_embed = self.embeddings(sent) # (b_s, w_l, embed_s)
char_embed = char_embed.permute(0, 2, 1) # (b_s, embed_s, w_l)
conv_out = self.cnn.forward(char_embed) # (b_s, embed_s)
x_highway = self.highway(conv_out) # (b_s, embed_s)
x_highway = self.dropout(x_highway)
output.append(x_highway)
output = torch.stack(output, 0) # (sent_len, batch_size, embed_size) ]
return output
def question_1g_sanity_check():
''' sanity check for conv layer shapes
'''
print("-" * 80)
print("Running sanity check for 1g: conv1d")
print("-" * 80)
sentence_length = 10
max_word_length = 21
inpt = torch.zeros(BATCH_SIZE, 100, max_word_length)
model = CNN(100, 20, max_word_length)
output = model.forward(inpt)
def question_1g_sanity_check():
char_embedding_size = 4
word_embedding_size = 8
sentence_length = 10
batch_size = 64
max_word_length = 15
# Test given inputs, output shape is expected
x = torch.Tensor(
np.ones((sentence_length, batch_size, char_embedding_size,
max_word_length)))
for factor in range(char_embedding_size):
x[:, :, factor, :] *= factor
cnn = CNN(word_embedding_size, char_embedding_size)
output = cnn.forward(x)
assert output.shape == (sentence_length, batch_size, word_embedding_size)
# Test given inputs, when convolution is identity, output values are expected
cnn = CNN(word_embedding_size, char_embedding_size)
torch.nn.init.constant_(cnn.conv_1.weight, 1)
cnn.conv_1.bias.data.fill_(0)
output = cnn.forward(x)
expected_output = torch.Tensor(
np.ones((sentence_length, batch_size, word_embedding_size)) * 30)
assert torch.all(torch.eq(output, expected_output))
# Test given inputs, max pooling is correct
x = torch.Tensor(
np.ones((sentence_length, batch_size, char_embedding_size,
max_word_length)))
for factor in range(char_embedding_size):
x[:, :, factor, 5:] *= factor
cnn = CNN(word_embedding_size, char_embedding_size)
torch.nn.init.constant_(cnn.conv_1.weight, 1)
offset = torch.Tensor([0, 1, 2, 3, 4, 5, 6, 7])
cnn.conv_1.bias = nn.Parameter(offset)
output = cnn.forward(x)
expected_output = torch.Tensor(
np.ones((sentence_length, batch_size, word_embedding_size)) * 30)
expected_output += offset
assert torch.all(torch.eq(output, expected_output))
def question_1i_sanity_check():
print("-" * 80)
print("Running Sanity Check for Question 1i: Model Embedding")
print("-" * 80)
model = CNN(4, 7, 10)
input = torch.randn(1, 4, 10)
res = model.forward(input)
print(res.shape)
assert res.shape == input.shape
print("Sanity Check Passed for Question 1h: Highway!")
print("-" * 80)
def test_cnn():
print()
print("==="*30)
print("\nCNN Class test")
x_reshaped = torch.tensor(
[
[
[[0, 1, 1], [-1, 1, 0]], # sentence a's word 1
[[0, 2, 1], [1, 0, 0]] # sentence b's word 1
],
[
[[0, 1, 1], [-5, 1, 0]], # sentence a's word 2
[[0, 2, 0], [-6, 0, 1]] # sentence b's word 2
]
],
dtype=torch.float,
device=device
)
sent_len = x_reshaped.shape[0]
batch_size = x_reshaped.shape[0]
correct_x_conv_out = np.array(
[
[
[0.3235975, 0.6727363 ], # sentence a's word 1
[0.21636295, 0.528751 ] # sentence b's word 1
],
[
[1.91349313, 0.71236265], # sentence a's word 2
[2.961207, 0.45990318] # sentence b's word 2
]
]
)
model = CNN(2, 2, kernel_size=2).to(device)
obtained_x_conv_out = model.forward(torch.flatten(x_reshaped, 0, 1))
obtained_x_conv_out = torch.stack(torch.split(obtained_x_conv_out, batch_size, dim=0))
obtained_x_conv_out = obtained_x_conv_out.cpu().detach().numpy()
assert np.allclose(correct_x_conv_out, obtained_x_conv_out), \
"\n\nIncorrect x_conv_out\n\nCorrect x_conv_out:\n{}\n\nYour x_conv_out:\n{}". \
format(correct_x_conv_out, obtained_x_conv_out)
print("\n\nx_conv_out =\n", obtained_x_conv_out, "\n")
# # Check the weights
# print("\nCNN weights:\n", model.convlayer.weight.cpu().detach().numpy())
# print("\n\nCNN bias:\n", model.convlayer.bias.cpu().detach().numpy())
print("\nCNN Test Passed!\n")
print("==="*30)
class ModelEmbeddings(nn.Module):
"""
Class that converts input words to their CNN-based embeddings.
"""
def __init__(self, embed_size, vocab, dropout_rate=0.3):
"""
Init the Embedding layer for one language
@param embed_size (int): Embedding size (dimensionality) for the output
@param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation.
"""
super(ModelEmbeddings, self).__init__()
## A4 code
# pad_token_idx = vocab.src['<pad>']
# self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx)
## End A4 code
### YOUR CODE HERE for part 1j
self.embed_size = embed_size
self.e_char = 50
pad_token_idx = vocab.char2id['<pad>']
self.embeddings = nn.Embedding(len(vocab.char2id),
self.e_char,
padding_idx=pad_token_idx)
self.cnn = CNN(e_char=self.e_char, e_word=self.embed_size)
self.highway = Highway(e_word=self.embed_size)
self.dropout = nn.Dropout(p=dropout_rate)
### END YOUR CODE
def forward(self, input):
"""
Looks up character-based CNN embeddings for the words in a batch of sentences.
@param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where
each integer is an index into the character vocabulary
@param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the
CNN-based embeddings for each word of the sentences in the batch
"""
## A4 code
# output = self.embeddings(input)
# return output
## End A4 code
### YOUR CODE HERE for part 1j
sentence_length, batch_size, max_word_length = input.shape
x_emb = self.embeddings(input)
x_reshape = x_emb.view(x_emb.shape[0] * x_emb.shape[1], x_emb.shape[2],
x_emb.shape[3]).transpose(1, 2)
x_conv_out = self.cnn.forward(x_reshape)
x_highway = self.highway.forword(x_conv_out)
x_word_emb = self.dropout(x_highway).view(sentence_length, batch_size,
self.embed_size)
return x_word_emb
class Embedding(nn.Module):
"""Embedding layer used by BiDAF, without the character-level component.
Word-level embeddings are further refined using a 2-layer Highway Encoder
(see `HighwayEncoder` class for details).
Args:
word_vectors (torch.Tensor): Pre-trained word vectors.
char_vectors (torch.Tensor): Pre-trained character vectors.
hidden_size (int): Size of hidden activations.
drop_prob (float): Probability of zero-ing out activations
"""
def __init__(self, word_vectors, char_vectors, hidden_size, drop_prob):
super(Embedding, self).__init__()
self.drop_prob = drop_prob
self.word_embed = nn.Embedding.from_pretrained(word_vectors)
self.char_embed = nn.Embedding.from_pretrained(char_vectors, freeze=False)
self.cnn = CNN(word_vectors.size(1), char_vectors.size(1))
self.highway = Highway(word_vectors.size(1))
self.proj = nn.Linear(word_vectors.size(1) * 2, hidden_size, bias=False)
self.hwy = HighwayEncoder(2, hidden_size)
def forward(self, x, y):
input_shape = y.shape # (batch_size, seq_len, 16) = (64, seq_len, 16)
input_reshaped = torch.reshape(y, (input_shape[0] * input_shape[1], input_shape[2])) # (64*seq_len, 16)
# print('input_reshaped')
# print(input_reshaped)
x_padded = self.char_embed(input_reshaped) # 64-dimensional
# print('x_padded')
# print(x_padded)
x_reshaped = x_padded.permute(0, 2, 1) # (64*seq_len, 64, 16)
x_conv_out = self.cnn.forward(x_reshaped) # (64*seq_len, 300)
# print('x_conv_out')
# print(x_conv_out)
x_highway = self.highway(x_conv_out)
# print('x_highway')
# print(x_highway)
x_highway_reshaped = torch.reshape(x_highway, (input_shape[0], input_shape[1], x_highway.shape[1]))
x_word_emb = F.dropout(x_highway_reshaped, self.drop_prob, self.training)
word_emb = self.word_embed(x) # (batch_size, seq_len, embed_size)
word_emb = F.dropout(word_emb, self.drop_prob, self.training)
word_emb = torch.cat((word_emb, x_word_emb), dim=2)
# print('word_emb')
# print(word_emb)
word_emb = self.proj(word_emb) # (batch_size, seq_len, hidden_size)
word_emb = self.hwy(word_emb) # (batch_size, seq_len, hidden_size)
# print('word_emb2')
# print(word_emb)
return word_emb # (batch_size, seq_len, 2 * embed_size) = (64, seq_len, 100)
def test_cnn(test_image_name):
test_img = mpimg.imread(test_image_name)
pkl_file = open('params.pkl', 'rb')
type = test_image_name.rsplit(".")[1]
params = pickle.load(pkl_file)
params1, params2, params3, params4, params5, params6, params7, params8, params9 = params
h, w, ip_channels = test_img.shape
out_channels = 64
filter_size = 3
padding = 1
layer_1 = Convolution(ip_channels, out_channels, filter_size, padding)
layer_1.weights = params1[0]
layer_1.bias=params1[1]
relu_1 = ReLU()
layer_2 = Convolution(out_channels, out_channels, filter_size, padding)
layer_2.weights = params2[0]
layer_2.bias = params2[1]
relu_2 = ReLU()
layer_3 = Convolution(out_channels, out_channels, filter_size, padding)
layer_3.weights = params3[0]
layer_3.bias = params3[1]
relu_3 = ReLU()
layer_4 = Convolution(out_channels, out_channels, filter_size, padding)
layer_4.weights = params4[0]
layer_4.bias = params4[1]
relu_4 = ReLU()
layer_5 = Convolution(out_channels, out_channels, filter_size, padding)
layer_5.weights = params5[0]
layer_5.bias = params5[1]
relu_5 = ReLU()
layer_6 = Convolution(out_channels, out_channels, filter_size, padding)
layer_6.weights = params6[0]
layer_6.bias = params6[1]
relu_6 = ReLU()
layer_7 = Convolution(out_channels, out_channels, filter_size, padding)
layer_7.weights = params7[0]
layer_7.bias = params7[1]
relu_7 = ReLU()
layer_8 = Convolution(out_channels, out_channels, filter_size, padding)
layer_8.weights = params8[0]
layer_8.bias = params8[1]
relu_8 = ReLU()
layer_9 = Convolution(out_channels, ip_channels, filter_size, padding)
layer_9.weights = params9[0]
layer_9.bias = params9[1]
layers = [(layer_1, relu_1), (layer_2, relu_2), (layer_3, relu_3), (layer_4, relu_4), (layer_5, relu_5),
(layer_6, relu_6),
(layer_7, relu_7), (layer_8, relu_8), (layer_9, None)]
cnn_obj = CNN(layers)
out_img = cnn_obj.forward(test_img)
output_file_name = test_image_name + "_sr" + str(h) + "." + type
cv2.imwrite(output_file_name, out_img)
def cnn_sanity_check(char_embed_size=1):
char_embed_size = 1
f = max_word_length = 21
print("CNN Input")
reshaped_embedding = torch.Tensor(
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2] *
char_embed_size).view(max_word_length, char_embed_size).unsqueeze(2)
print(reshaped_embedding)
cnn = CNN(char_embed_size, f, k=1, max_word_length=21, bias=True)
result = cnn.forward(reshaped_embedding)
print("CNN Output")
print(result)
print(result.size())
class ModelEmbeddings(nn.Module):
"""
Class that converts input words to their CNN-based embeddings.
"""
def __init__(self, word_embed_size, vocab):
"""
Init the Embedding layer for one language
@param word_embed_size (int): Embedding size (dimensionality) for the output word
@param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation.
Hints: - You may find len(self.vocab.char2id) useful when create the embedding
"""
super(ModelEmbeddings, self).__init__()
### YOUR CODE HERE for part 1h
self.e_char = 50
self.word_embed_size = word_embed_size
self.dropout_prob = 0.3
self.vocab = vocab
# apparently 21 is an arbitrary value chosen for the sanity tests and also the greedy decoder
self.max_word_len = 21
self.embedding = nn.Embedding(len(vocab.char2id),
self.e_char,
padding_idx=vocab.char2id['∏'])
self.cnn = CNN(e_char=self.e_char,
filters=self.word_embed_size,
kernel_size=5,
m_word=self.max_word_len)
self.highway = Highway(word_embed_size)
self.dropout = nn.Dropout(p=self.dropout_prob)
### END YOUR CODE
def forward(self, input):
"""
Looks up character-based CNN embeddings for the words in a batch of sentences.
@param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where
each integer is an index into the character vocabulary
@param output: Tensor of shape (sentence_length, batch_size, word_embed_size), containing the
CNN-based embeddings for each word of the sentences in the batch
"""
### YOUR CODE HERE for part 1h
embedded = self.embedding.forward(input)
max_sent_len, batch_size, samples, channels = embedded.shape
reshaped = embedded.view(
(max_sent_len * batch_size, samples, channels)).transpose(1, 2)
conv_out = self.cnn.forward(reshaped)
highway = self.highway.forward(conv_out.squeeze())
dropout = self.dropout.forward(highway)
return dropout.view(max_sent_len, batch_size, -1)
class ModelEmbeddings(nn.Module):
"""
Class that converts input words to their CNN-based embeddings.
"""
def __init__(self, embed_size, vocab):
"""
Init the Embedding layer for one language
@param embed_size (int): Embedding size (dimensionality) for the output
@param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation.
"""
super(ModelEmbeddings, self).__init__()
## A4 code
#pad_token_idx = vocab.src['<pad>']
#self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx)
## End A4 code
### YOUR CODE HERE for part 1j
self.embed_size=embed_size
self.char_embed_size=50
self.embeddings=nn.Embedding(len(vocab.char2id),self.char_embed_size,padding_idx=vocab.char2id['<pad>'])
self.conv=CNN(in_channels=self.char_embed_size,out_channels=self.embed_size)
self.highway=Highway(size=self.embed_size)
self.dropout=nn.Dropout(p=0.3)
### END YOUR CODE
def forward(self, input):
"""
Looks up character-based CNN embeddings for the words in a batch of sentences.
@param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where
each integer is an index into the character vocabulary
@param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the
CNN-based embeddings for each word of the sentences in the batch
"""
## A4 code
#x_emb = self.embeddings(input)
# return output
## End A4 code
### YOUR CODE HERE for part 1j
x_embed=self.embeddings(input)
x_reshaped=torch.reshape(x_embed,[x_embed.size()[0]*x_embed.size()[1],x_embed.size()[3],x_embed.size()[2]])
x_conv_out=self.conv.forward(x_reshaped)
x_highway=self.highway.forward(x_conv_out)
x_word_emb=self.dropout(x_highway)
output=torch.reshape(x_word_emb,[x_embed.size()[0],x_embed.size()[1],-1])
return output
def test_shape(self):
print("-" * 80)
print("Running Sanity Check for Question 1e: CNN Shape")
print("-" * 80)
max_word_len = 15
batch_size, char_embed_size, num_filters, window_size = 64, 20, 80, 5
cnn = CNN(char_embed_size, num_filters, max_word_len, window_size)
x_emb = torch.randn([batch_size, char_embed_size, max_word_len])
x_conv_out = cnn.forward(x_emb)
self.assertEqual(x_conv_out.shape, (batch_size, num_filters))
print("Sanity Check Passed for Question 1e: CNN Shape!")
print("-" * 80)
def question_1i_sanity_check():
print("-" * 80)
print("Running Sanity Check for Question 1i: question_1g_sanity_check")
N, e_char, m_word, kernel_size, filter_num = 10, 5, 6, 5, 4
x_reshape = torch.randn((N, e_char, m_word)) # (N=10,e_char=5, m_word=6)
cnn = CNN(m_word, e_char, filter_num, kernel_size=kernel_size)
x_conv_out = cnn.forward(x_reshape)
print("x_reshape = " + str(x_reshape))
print("x_conv_out = " + str(x_conv_out))
batch_size, filter_num_out = x_conv_out.shape
assert (batch_size == N)
assert (filter_num_out == filter_num)
print("question_1i_sanity_check: shape passed!")
print("-" * 80)
class CRNN(nn.Module):
""" C-RNN Module. """
def __init__(self):
super(CRNN, self).__init__()
self.cnn = CNN()
self.rnn = RNN(input_size=2048, hidden_size=256, output_size=4)
def forward(self, x, hidden):
x_feats = self.cnn.forward(x)
output, hidden = self.rnn.forward(x_feats, hidden)
return output, hidden
def init_hidden(self, batch_size):
"""
Initialize hidden units to zero.
"""
return self.rnn.init_hidden(batch_size)
def question_1i_sanity_check():
# Sanity check for cnn.py
print("-" * 80)
print("Running Sanity Check for Question 1i: CNN")
print("-" * 80)
m_word = 6
e_char = 3
e_word = 4
inpt = torch.ones(BATCH_SIZE, e_char, m_word)
cnn_net = CNN(e_char, e_word)
output = cnn_net.forward(inpt)
output_expected_size = [BATCH_SIZE, e_word]
assert (
list(output.size()) == output_expected_size
), "output shape is incorrect: it should be:\n {} but is:\n{}".format(
output_expected_size, list(output.size()))
print("Sanity Check Passed for Question 1i: CNN!")
print("-" * 80)
def question_1g_sanity_check():
""" Sanity check for cnn.py
basic shape check
"""
print("-" * 80)
print("Running Sanity Check for Question 1g: CNN")
print("-" * 80)
max_word_size = 10
char_embed_size = 50
x = torch.zeros(BATCH_SIZE,char_embed_size,max_word_size)
print("input size： "+str(x.shape))
cnn = CNN(num_filters=EMBED_SIZE, kernel_size=5, char_embed_size=char_embed_size)
result = cnn.forward(x)
print("output size :",str(result.shape) )
assert(result.shape == (BATCH_SIZE, EMBED_SIZE))
print("Sanity Check Passed for Question 1g:cnn")
class TestCNN(unittest.TestCase):
"""
Sanity check for CNN module
"""
def setUp(self):
self.char_emb_size = EMBED_SIZE-1
self.word_emb_size = EMBED_SIZE
self.m_word = 10
self.model = CNN(char_embed_size=self.char_emb_size,
word_embed_size=self.word_emb_size,
max_len=self.m_word)
def test_CNN_shape(self):
self.model = initialize_layers(self.model)
input = torch.ones((BATCH_SIZE, self.char_emb_size, self.m_word))
with torch.no_grad():
output = self.model.forward(input)
self.assertEqual(output.shape, (BATCH_SIZE, self.word_emb_size), "incorrect output shape")
self.assertEqual(self.model.conv1d.in_channels, self.char_emb_size, "incorrect input channel size")
self.assertEqual(self.model.conv1d.out_channels, self.word_emb_size, "incorrect output channel size")
def question_1g_sanity_check():
""" Sanity check for the class `CNN`.
"""
print("-" * 80)
print("Running Sanity Check for Question 1g: CNN")
print("-" * 80)
SENTENCE_LENGTH = 20
BATCH_SIZE = 5
E_CHAR = 50
M_WORD = 21
F = 3
# model = CNN(f=F, e_char = E_CHAR, m_word=M_WORD)
model = CNN(f=F, e_char=E_CHAR)
x_reshaped = torch.randn((SENTENCE_LENGTH, BATCH_SIZE, E_CHAR, M_WORD))
print("Running test on a batch of x_reshaped")
x_conv_out = model.forward(x_reshaped)
assert list(x_conv_out.size()) == [
SENTENCE_LENGTH, BATCH_SIZE, F
], "Output size should be: {}, but got {}".format(
(SENTENCE_LENGTH, BATCH_SIZE, F), x_conv_out.size())
print("Sanity Check Passed for Question 1g: CNN!")
print("-" * 80)
class ModelEmbeddings(nn.Module):
"""
Class that converts input words to their CNN-based embeddings.
"""
def __init__(self, embed_size, vocab):
"""
Init the Embedding layer for one language
@param embed_size (int): Embedding size (dimensionality) for the output
@param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation.
"""
super(ModelEmbeddings, self).__init__()
## A4 code
# pad_token_idx = vocab.src['<pad>']
# self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx)
## End A4 code
### YOUR CODE HERE for part 1j
pad_token_idx = vocab.char2id['<pad>']
self.char_embed_size = 50
self.dropout_rate = 0.3
self.max_word_size = 21
self.word_embed_size = embed_size
self.embed_size = embed_size
self.v_char = len(vocab.char2id)
self.v_word = len(vocab.word2id)
self.embeddings = nn.Embedding(self.v_char,
self.char_embed_size,
padding_idx=pad_token_idx)
self.Dropout = nn.Dropout(p=self.dropout_rate)
self.cnn = CNN(e_char=self.char_embed_size,
e_word=self.word_embed_size,
m_word=self.max_word_size)
self.highway = Highway(embedding_size=self.word_embed_size)
### END YOUR CODE
def forward(self, input):
"""
Looks up character-based CNN embeddings for the words in a batch of sentences.
@param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where
each integer is an index into the character vocabulary
@param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the
CNN-based embeddings for each word of the sentences in the batch
"""
## A4 code
# output = self.embeddings(input)
# return output
## End A4 code
### YOUR CODE HERE for part 1j
max_word_size = input.size()[-1]
assert (max_word_size == self.max_word_size)
char_embeddings = self.embeddings(
input
) # (max_sent_len, batch_size, max_word_len, char_embedding_size)
# conv1d only performs on the last dimension so we have to swap
char_embeddings = char_embeddings.permute(
0, 1, 3,
2) # (max_sent_len, batch_size, char_embedding_size, max_word_len)
max_sent_len = char_embeddings.size()[0]
batch_size = char_embeddings.size()[1]
char_embedding_size = char_embeddings.size()[2]
max_word_len = char_embeddings.size()[3]
# conv1d only accepts 3 dimension array, so any extra dimensions need to be concatenated.
char_embeddings = char_embeddings.reshape(
max_sent_len * batch_size, char_embedding_size, max_word_len
) # (max_sent_len * batch_size, char_embedding_size, max_word_len)
cnn_out = self.cnn.forward(
char_embeddings
) # (max_sent_len * batch_size, word_embedding_size)
highway_out = self.highway.forward(
cnn_out) # (max_sent_len * batch_size, word_embedding_size)
dropout_out = self.Dropout(highway_out)
output = dropout_out.reshape(
max_sent_len, batch_size,
dropout_out.size()
[-1]) # (max_sent_len, batch_size, word_embedding_size)
return output
class ModelEmbeddings(nn.Module):
"""
Class that converts input words to their CNN-based embeddings.
"""
def __init__(self, embed_size, vocab):
"""
Init the Embedding layer for one language
@param embed_size (int): Embedding size (dimensionality) for the output
@param vocab (VocabEntry): VocabEntry object. See vocab.py for documentation.
"""
super(ModelEmbeddings, self).__init__()
## A4 code
# pad_token_idx = vocab.src['<pad>']
# self.embeddings = nn.Embedding(len(vocab.src), embed_size, padding_idx=pad_token_idx)
## End A4 code
### YOUR CODE HERE for part 1j
self.embed_size = embed_size
self.embedding_char_size = 50
self.dropout_rate = 0.3
self.max_word_length = 21
self.embedding_word_size = embed_size
pad_token_idx = vocab.char2id['<pad>']
self.charEmbeddings = nn.Embedding(len(vocab.char2id),
self.embedding_char_size,
padding_idx=pad_token_idx)
self.dropout = nn.Dropout(p=self.dropout_rate)
#construct CNN
self.CNN = CNN(self.embedding_char_size, self.embedding_word_size,
self.max_word_length)
#construct Highway
self.highway = Highway(self.embedding_word_size)
### END YOUR CODE
def forward(self, input):
"""
Looks up character-based CNN embeddings for the words in a batch of sentences.
@param input: Tensor of integers of shape (sentence_length, batch_size, max_word_length) where
each integer is an index into the character vocabulary
@param output: Tensor of shape (sentence_length, batch_size, embed_size), containing the
CNN-based embeddings for each word of the sentences in the batch
"""
## A4 code
# output = self.embeddings(input)
# return output
## End A4 code
### YOUR CODE HERE for part 1j
sentence_length, batch_size, _ = input.shape
x_emb = self.charEmbeddings(input)
#note: x_emb : tensor of (sentence_length, batch_size, max_word_length, e_char)
x_emb = x_emb.view((sentence_length * batch_size, self.max_word_length,
self.embedding_char_size)).transpose(1, 2)
#now x_emb is : tensor of (sentence_length * batch_size, e_char, max_word_length)
#cnn needs: input shape: x_reshaped:(batch_size, char_embedding_size(also called e_char), max_word_length)
# output shape: (batch_size, e_word)
x_conv_out = self.CNN.forward(x_emb)
#x_conv_out : (batch_size * sentence_length, e_word=embed_size)
x_highway = self.highway.forward(x_conv_out)
x_word_emb = self.dropout(x_highway)
#now x_word_emb is: tensor of (sentence_length * batch_size, embed_size)
output = x_word_emb.view(
(sentence_length, batch_size, self.embedding_word_size))
return output
