def __init__(self, input_channels=1):
super(Autoencoder, self).__init__()
# some sanity checks
batchNorm_momentum = 0.1
self.encoder = PHOCNet() # conv filters
self.enc_rnn = EncoderRNN(512, 512)
self.decoder = Decoder()
def test_encode_decode(self):
gen_matrix = [
[1, 0, 0, 0, 0, 1],
[0, 1, 0, 1, 0, 1],
[0, 0, 1, 0, 1, 1],
]
msg = [1, 1, 1]
vector = encode_message(msg, gen_matrix)
decoder = Decoder(gen_matrix)
rez = decoder.decode(vector, len(vector))
self.assertEqual(msg, rez)
def test_encode_decode_1_mistake_len_2(self):
gen_matrix = [
[1, 0, 1, 0, 0, 1],
[0, 1, 1, 1, 0, 1],
]
msg = [1, 1]
vector = encode_message(msg, gen_matrix)
decoder = Decoder(gen_matrix)
for idx in range(len(vector)):
vector[idx] ^= 1
rez = decoder.decode(vector, len(vector))
self.assertEqual(
msg, rez, f"not equal after changing bit in position {idx}")
vector[idx] ^= 1
def __init__(self, params, embedding_matrix):
super(RVAE_dilated, self).__init__()
self.params = params
self.word_embeddings = nn.Embedding(params.word_vocab_size, params.word_embed_size)
self.word_embeddings.weight = Parameter(t.from_numpy(embedding_matrix).float(),
requires_grad=False)
self.encoder = Encoder(self.params)
self.context_to_mu = nn.Linear(self.params.encoder_rnn_size * 2, self.params.latent_variable_size)
self.context_to_logvar = nn.Linear(self.params.encoder_rnn_size * 2, self.params.latent_variable_size)
self.decoder = Decoder(self.params)
def make_model(src_vocab,
tgt_vocab,
N=6,
d_model=512,
d_ff=2048,
h=8,
dropout=0.1):
"Helper: Construct a model from hyperparameters."
c = copy.deepcopy
attn = MultiHeadedAttention(h, d_model)
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
position = PositionalEncoding(d_model, dropout)
model = EncoderDecoder(
Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),
Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), N),
nn.Sequential(Embeddings(d_model, src_vocab), c(position)),
nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),
Generator(d_model, tgt_vocab))
# This was important from their code.
# Initialize parameters with Glorot / fan_avg.
for p in model.parameters():
if p.dim() > 1:
nn.init.xavier_uniform(p)
return model
def load_model_from_package(cls, package):
encoder = Encoder(package['d_input'],
package['n_layers_enc'],
package['n_head'],
package['d_k'],
package['d_v'],
package['d_model'],
package['d_inner'],
dropout=package['dropout'],
pe_maxlen=package['pe_maxlen'])
decoder = Decoder(
package['sos_id'],
package['eos_id'],
package['vocab_size'],
package['d_word_vec'],
package['n_layers_dec'],
package['n_head'],
package['d_k'],
package['d_v'],
package['d_model'],
package['d_inner'],
dropout=package['dropout'],
tgt_emb_prj_weight_sharing=package['tgt_emb_prj_weight_sharing'],
pe_maxlen=package['pe_maxlen'],
)
model = cls(encoder, decoder)
model.load_state_dict(package['state_dict'])
LFR_m, LFR_n = package['LFR_m'], package['LFR_n']
return model, LFR_m, LFR_n
def run():
k, n, msg, gen_matrix, error_chance = parse_program_args()
print(f"gen_matrix: {gen_matrix}")
channel = Channel(error_chance)
print(f"message: {msg}")
encoded_msg = encode_message(msg, gen_matrix)
print(f"encoded message: {encoded_msg}")
error_vector, error_count = channel.generate_errors(encoded_msg)
print(f"error count: {error_count}")
print(f"error vector: {error_vector}")
edited_error_vector = eval(read_input("edit mode:", str(error_vector)))
print(f"edited_error vector: {edited_error_vector}")
received = channel.add_errors(encoded_msg, edited_error_vector)
print(f"message received: {received}")
decoder = Decoder(gen_matrix)
decoded = decoder.decode(received, len(msg))
print(f"message decoded: {decoded}")
def run_tests():
e = Encoder()
d = Decoder(coding_polynomial=e.coding_polynomial,
k=e.k,
t=e.r,
gf_index=e.gf.index)
message = "zaqwsxcderfvbgtyhnmjuik,ol.p;/zaqwsxedcrfvtgbyhnujmzaqwsxcderf"
codeword = e.encode(message)
decoded_message = d.decode(codeword, 'basic')
for i in range(2, 28):
codeword.elements[i] = codeword.elements[i].multiplicative_inversion()
print('27 errors occurred...')
print(codeword)
decoded_message = d.decode(codeword, 'basic')
print('Decoded message: ' + decoded_message[:len(message)])
def run_example_program():
e = Encoder()
d = Decoder(coding_polynomial=e.coding_polynomial,
k=e.k,
t=e.r,
gf_index=e.gf.index)
message = "zaqwsxcderfvbgtyhnmjuik,ol.p;/zaqwsxedcrf"
print('Message: ' + message)
codeword = e.encode(message)
print('Codeword: ' + str(codeword))
decoded_message = d.decode(codeword, 'basic')
print('Decoded message: ' + decoded_message[:len(message)])
for i in range(1, 28):
codeword.elements[i] = codeword.elements[i].multiplicative_inversion()
print('27 errors occurred...')
print(codeword)
decoded_message = d.decode(codeword, 'basic')
print('Decoded message: ' + decoded_message[:len(message)])
def test_encoder_fix_errors(ii, k, test_type, message):
e = Encoder()
d = Decoder(coding_polynomial=e.coding_polynomial, k=e.k, t=e.r, gf_index=e.gf.index)
encoded_message = e.encode(message)
if test_type == 'multiple':
random_indexes = random.sample(range(0, len(encoded_message)), k)
else:
random_start = random.randint(0, len(encoded_message)-k-1)
random_indexes = [i for i in range(random_start, random_start + k)]
# print("{}): {}".format(k, random_indexes))
for i in random_indexes:
encoded_message.elements[i] = encoded_message.elements[i].multiplicative_inversion()
try:
start = time.time()
decoded_message = d.decode(encoded_message, 'basic')
stop = time.time()
passed.write("{}, {}, {}, {}, {}\n".format(k, test_type, message, random_indexes, stop-start))
except CannotDetectErrorException as c:
failed.write("{}, {}, {}, {}\n".format(k, test_type, message, random_indexes))
assert False
assert message in decoded_message
def __init__(self, _c: "VAEConfig"):
super().__init__()
self._c = _c
self.image_flatten_dim = _c.image_dim[0] * _c.image_dim[1]
adam_params = {
"lr": _c.init_lr,
"betas": (0.96, 0.999),
"clip_norm": 10.0,
"lrd": 0.99996,
"weight_decay": 2.0
}
self.optimizer = ClippedAdam(adam_params)
self.emitter = Decoder(_c.z_dim,
_c.emitter_channel,
dropout_p=_c.dropout_rate)
self.trans = GatedTransition(_c.z_dim, _c.transition_dim)
self.combiner = Combiner(_c.z_dim, _c.rnn_dim)
self.crnn = ConvRNN(_c.image_dim,
_c.rnn_dim,
_c.rnn_layers,
_c.dropout_rate,
use_lstm=_c.use_lstm,
channels=_c.crnn_channel)
self.iafs = [
affine_autoregressive(_c.z_dim, hidden_dims=[_c.iaf_dim])
for _ in range(_c.num_iafs)
]
self.iafs_modules = nn.ModuleList(self.iafs)
self.z_0 = nn.Parameter(torch.zeros(_c.z_dim))
self.z_q_0 = nn.Parameter(torch.zeros(_c.z_dim))
self.h_0 = nn.Parameter(torch.zeros(1, 1, _c.rnn_dim))
if _c.use_lstm:
self.c_0 = nn.Parameter(torch.zeros(1, 1, _c.rnn_dim))
self.cuda()
from src.preprocessing import import_wsj, rep_rare_corpus
from src.decoder import Decoder
sentences = import_wsj("test")
correct = 0
total = 0
decoder = Decoder("train")
for sentence_and_tags in sentences:
local_correct = 0
decode = decoder.decode(sentence_and_tags)
decoded_tags = decode[1]
tags = sentence_and_tags[1]
print("tags:", tags)
print("decoded tags:", decoded_tags)
for i in range(len(tags)):
if tags[i] == decoded_tags[i]:
local_correct += 1
total += 1
if local_correct == 0:
for tag in tags:
total -= 1
else:
correct += local_correct
accuracy = correct / total * 100
print(accuracy)
class RVAE_dilated(nn.Module):
def __init__(self, params, embedding_matrix):
super(RVAE_dilated, self).__init__()
self.params = params
self.word_embeddings = nn.Embedding(params.word_vocab_size, params.word_embed_size)
self.word_embeddings.weight = Parameter(t.from_numpy(embedding_matrix).float(),
requires_grad=False)
self.encoder = Encoder(self.params)
self.context_to_mu = nn.Linear(self.params.encoder_rnn_size * 2, self.params.latent_variable_size)
self.context_to_logvar = nn.Linear(self.params.encoder_rnn_size * 2, self.params.latent_variable_size)
self.decoder = Decoder(self.params)
def forward(self, drop_prob,
encoder_input_tuple=None,
use_cuda=False,
z=None,
inference=False):
"""
:param encoder_input_tuple: An tensor with shape of [batch_size, seq_len] of Long type
:param drop_prob: probability of an element of decoder input to be zeroed in sense of dropout
:param use_cuda: whether to use gpu
:param z: context if sampling is performing
:return: unnormalized logits of sentence words distribution probabilities
with shape of [batch_size, seq_len, word_vocab_size]
kld loss estimation
"""
encoder_input, lengths = encoder_input_tuple
batch_size = encoder_input.size()[0]
encoder_input = Variable(encoder_input, volatile=inference).cuda() if use_cuda else Variable(encoder_input, volatile=inference)
encoder_input_matrix = self.word_embeddings(encoder_input)
if z is None:
''' Get context from encoder and sample z ~ N(mu, std)
'''
packed_seq = pack_padded_sequence(encoder_input_matrix, lengths, batch_first=True)
context = self.encoder.forward(packed_seq, batch_size)
mu = self.context_to_mu(context)
logvar = self.context_to_logvar(context)
std = t.exp(0.5 * logvar)
z = Variable(t.randn([batch_size, self.params.latent_variable_size]))
if use_cuda:
z = z.cuda()
z = z * std + mu
kld = (-0.5 * t.sum(logvar - t.pow(mu, 2) - t.exp(logvar) + 1, 1)).mean()
else:
kld = None
decoder_input = encoder_input_matrix
out = self.decoder.forward(decoder_input, z, drop_prob)
return encoder_input, out, kld
def learnable_parameters(self):
# word_embedding is constant parameter thus it must be dropped from list of parameters for optimizer
return [p for p in self.parameters() if p.requires_grad]
def trainer(self, optimizer, use_cuda, dropout):
def train_one_batch(data_tuple):
target, logits, kld = self.forward(drop_prob=dropout,
encoder_input_tuple=data_tuple,
use_cuda=use_cuda,
z=None)
batch_size = target.data.size()[0]
sequence_length = target.data.size()[1]
logits = logits.view(-1, self.params.word_vocab_size)
target = target.view(-1)
cross_entropy = F.cross_entropy(logits, target)
loss = sequence_length * cross_entropy + kld
optimizer.zero_grad()
loss.backward()
optimizer.step()
return kld, loss
return train_one_batch
def validater(self, use_cuda, dropout):
def validate(data_tuple):
target, logits, kld = self.forward(drop_prob=dropout,
encoder_input_tuple=data_tuple,
use_cuda=use_cuda,
z=None)
logits = logits.view(-1, self.params.word_vocab_size)
target = target.view(-1)
cross_entropy = F.cross_entropy(logits, target)
return np.exp2(cross_entropy.data.cpu().numpy()[0])
return validate
from src.decoder import Decoder
sentence = "The bill, whose backers include Chairman Dan Rostenkowski -LRB- D., Ill. -RRB-, would prevent the Resolution Trust Corp. from raising temporary working capital by having an RTC-owned bank or thrift issue debt that would n't beB counted on the federal budget."
decoder = Decoder("train")
decode = decoder.decode(sentence)
print(list(zip(decode[0],decode[1])))
class Autoencoder(nn.Module):
'''
Network class for generating PHOCNet and TPP-PHOCNet architectures
'''
def __init__(self, input_channels=1):
super(Autoencoder, self).__init__()
# some sanity checks
batchNorm_momentum = 0.1
self.encoder = PHOCNet() # conv filters
self.enc_rnn = EncoderRNN(512, 512)
self.decoder = Decoder()
#self.decoder = Decoder()
def forward(self, x, embedding, lengths):
y_filters = self.encoder(x)
# print(y_filters.shape)
#y_filters = x
# y_filters = F.max_pool2d(x, kernel_size=2, stride=2, padding=0)
batch_size, filter_size, _, _ = y_filters.shape
y_filters = y_filters.view(batch_size, filter_size, -1)
y_filters = y_filters.permute(0, 2, 1)
# print(y_filters.shape)
# y_enc, hidden = self.enc_rnn(y_filters)
# print(y_enc.shape)
# y_enc = y_enc.transpose(0, 1)
# print(y_enc.shape)
# print(embedding.shape)
out, out_captions, lengths_t, alphas, image_out = self.decoder(
y_filters, embedding, lengths)
# out, out_captions, lengths_t, alphas = self.decoder(y_enc, embedding, lengths)
return out, out_captions, lengths_t, alphas, image_out
def sampler(self, x):
y_filters = self.encoder(x)
# y_filters = x
batch_size, filter_size, _, _ = y_filters.shape
y_filters = y_filters.view(batch_size, filter_size, -1)
y_filters = y_filters.permute(0, 2, 1)
# print(y_filters.shape)
# y_enc, hidden = self.enc_rnn(y_filters)
# print(y_enc.shape)
# y_enc = y_enc.transpose(0, 1)
# print(y_enc.shape)
# print(embedding.shape)
out, alpha = self.decoder.sample(y_filters)
return out
#
#
#
#
# # print(x.shape)
# batch_size = x.shape[0]
# # y = F.relu(self.bn11(self.conv1_1(x)))
# # # print(y.shape)
# # # y = F.relu(self.conv1_2(y))
# # size1 = y.size()
# # y, indices1 = F.max_pool2d(y, kernel_size=2, stride=2, padding=0, return_indices=True)
# # # print(y.shape)
# # y = F.relu(self.bn21(self.conv2_1(y)))
# # # print(y.shape)
# # y = F.relu(self.conv2_2(y))
# # # print(y.shape)
# # size2 = y.size()
# # y, indices2 = F.max_pool2d(y, kernel_size=2, stride=2, padding=0, return_indices=True)
# # # print(y.shape)
# # y = F.relu(self.conv3_1(y))
# # # print(y.shape)
# # y = F.relu(self.bn32(self.conv3_2(y)))
# # # print(y.shape)
# # # y = F.relu(self.conv3_3(y))
# # # y = F.relu(self.conv3_4(y))
# # # y = F.relu(self.conv3_5(y))
# # # y = F.relu(self.conv3_6(y))
# # y = F.relu(self.conv4_1(y))
# # # print(y.shape)
# # y = F.relu(self.bn42(self.conv4_2(y)))
# # y = F.relu(self.model_ft(x))
# # print(y.shape)
#
# # rec_img = self.defc1(att_y)
# # y_img = F.dropout(rec_img.view(batch_size, feat_size, 3, 8), training=self.training)
#
# # print(phoc[0])
# #reconstruction
# # y_img = F.relu(self.bn42_d(self.deconv4_2(att_y)))
# # # print(y.shape)
# # y_img = F.relu(self.deconv4_1(y_img))
# # # print(y.shape)
# # y_img = F.relu(self.bn32_d(self.deconv3_2(y_img)))
# # # print(y.shape)
# # y_img = F.relu(self.deconv3_1(y_img))
# # # print(y.shape)
# # y_img = self.unpool1(y_img, indices2.repeat_interleave(55, dim=0), torch.Size((batch_size*55, 128, 35, 75)))
# # # y_img = self.unpool1(y_img, indices2, size2)
# # # print(y.shape)
# # y_img = F.relu(self.deconv2_2(y_img))
# # # print(y.shape)
# # y_img = F.relu(self.bn21_d(self.deconv2_1(y_img)))
# # # print(y.shape)
# # y_img = self.unpool1(y_img, indices1.repeat_interleave(55, dim=0), torch.Size((batch_size*55, 64, 70, 150)))
# #
# # # y_img = self.unpool2(y_img, indices1, size1)
# # # print(y_img.shape)
# # y_img = F.relu(self.deconv1_1(y_img)) # 55*batch_size x 1 x H x W
# # y_img = y_img.view(batch_size,-1, y_img.shape[2], y_img.shape[3])
# # img, _ = torch.max(y_img, dim=1, keepdim=True)
#
# #
# # phoc = torch.zeros(batch_size, 1980).cuda()
# # img = torch.zeros(batch_size, 55, 70, 150).cuda()
# #
# #
# # for i in range(y_attention.shape[1]): # number of attentions
# # y_attention_hoc = y_attention[:, i, :, :] # batch_size x 1 x H x W
# # # print (y_attention_hoc.shape)
# # y_attention_hoc = y_attention_hoc.unsqueeze(1).expand(-1, 512, -1, -1)
# # att_y = y * y_attention_hoc # batch_size x 512 x H x W
# # att_y_emb = torch.sum(att_y.view(batch_size, 512, -1), dim=2) # batch_size x 512
# # # print(att_y_emb.size())
# # st= (i) * 36
# # en = (i+1) * 36
# # phoc[:, st:en] = self.fc(att_y_emb) # batch_size x 36
# # # reconstruction
# # y_img = F.relu(self.bn42_d(self.deconv4_2(att_y)))
# # # print(y.shape)
# # y_img = F.relu(self.deconv4_1(y_img))
# # # print(y.shape)
# # y_img = F.relu(self.bn32_d(self.deconv3_2(y_img)))
# # # print(y.shape)
# # y_img = F.relu(self.deconv3_1(y_img))
# # # print(y.shape)
# # # y = self.unpool1(y, indices2.expand(55, -1, -1, -1), torch.Size((55, 128, 35, 75)))
# # y_img = self.unpool1(y_img, indices2, size2)
# # # print(y.shape)
# # y_img = F.relu(self.deconv2_2(y_img))
# # # print(y.shape)
# # y_img = F.relu(self.bn21_d(self.deconv2_1(y_img)))
# # # print(y.shape)
# # y_img = self.unpool2(y_img, indices1, size1)
# # # print(y_img.shape)
# # y_img = F.relu(self.deconv1_1(y_img)) # batch_size x 1 x H x W
# # # print(y_img.shape)
# # img[:, i, :, :] = y_img.squeeze()
# # img, _ = torch.max(img, dim=1, keepdim=True)
# #
# #
#
#
# # print(y_attention.shape)
# # y_attention = y_attention.permute(1, 0, 2, 3)
# # print(y_attention.shape)
# # y_attention = y_attention.expand(-1, 512, -1, -1)
# # print(y_attention.shape)
# # y_attention.shape
# # att_y = y.expand(55, -1, -1, -1) * y_attention
# # print(y.shape)
# # phoc = self.fc(torch.sum(att_y.view(55, 512, -1), dim=2))
# # phoc = phoc.view(-1).unsqueeze(0)
# # print(att_y.shape)
#
# # y, _ = torch.max(y, dim=0, keepdim=True)
# # print(y.shape)
# # y = F.relu(self.conv4_3(y))
# return phoc
def init_weights(self):
self.apply(PHOCNet._init_weights_he)
'''
@staticmethod
def _init_weights_he(m):
if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
#nn.init.kaiming_normal(m.weight.data)
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, (2. / n)**(1/2.0))
if hasattr(m, 'bias'):
nn.init.constant(m.bias.data, 0)
'''
@staticmethod
def _init_weights_he(m):
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, (2. / n)**(1 / 2.0))
if isinstance(m, nn.Linear):
n = m.out_features
m.weight.data.normal_(0, (2. / n)**(1 / 2.0))
#nn.init.kaiming_normal(m.weight.data)
nn.init.constant(m.bias.data, 0)
def decoder(encoder):
d = Decoder(coding_polynomial=encoder.coding_polynomial, k=encoder.k, t=encoder.r, gf_index=encoder.gf.index)
return d
def main(args):
# load dictionary and generate char_list, sos_id, eos_id
char_list, sos_id, eos_id = process_dict(args.dict)
vocab_size = len(char_list)
tr_dataset = AudioDataset('train', args.batch_size)
cv_dataset = AudioDataset('dev', args.batch_size)
tr_loader = AudioDataLoader(tr_dataset,
batch_size=1,
num_workers=args.num_workers,
shuffle=args.shuffle,
feature_dim=args.feature_dim,
char_list=char_list,
path_list=tr_dataset.path_lst,
label_list=tr_dataset.han_lst,
LFR_m=args.LFR_m,
LFR_n=args.LFR_n)
cv_loader = AudioDataLoader(cv_dataset,
batch_size=1,
num_workers=args.num_workers,
feature_dim=args.feature_dim,
char_list=char_list,
path_list=cv_dataset.path_lst,
label_list=cv_dataset.han_lst,
LFR_m=args.LFR_m,
LFR_n=args.LFR_n)
data = {'tr_loader': tr_loader, 'cv_loader': cv_loader}
encoder = Encoder(args.d_input * args.LFR_m,
args.d_low_dim,
args.n_layers_enc,
args.n_head,
args.d_k,
args.d_v,
args.d_model,
args.d_inner,
dropout=args.dropout,
pe_maxlen=args.pe_maxlen)
decoder = Decoder(
sos_id,
eos_id,
vocab_size,
args.d_word_vec,
args.n_layers_dec,
args.n_head,
args.d_k,
args.d_v,
args.d_model,
args.d_inner,
dropout=args.dropout,
tgt_emb_prj_weight_sharing=args.tgt_emb_prj_weight_sharing,
pe_maxlen=args.pe_maxlen)
model = Transformer(encoder, decoder)
print(model)
model.cuda()
# optimizer
optimizier = TransformerOptimizer(
torch.optim.Adam(model.parameters(), betas=(0.9, 0.98), eps=1e-09),
args.init_lr, args.d_model, args.warmup_steps)
# solver
solver = Solver(data, model, optimizier, args)
solver.train()
def __init__(self, hidden_encoder_size, z_dim, hidden_decoder_size, output_size, rnn_type, device):
super(GrammarVAE, self).__init__()
self.encoder = Encoder(hidden_encoder_size, z_dim)
self.decoder = Decoder(z_dim, hidden_decoder_size, output_size, device, rnn_type)
self.device = device