# return loss, len(numeric) * args.sequence_length
import time
devLosses = []
#for epoch in range(10000):
if True:
training_data = corpusIteratorWiki.training(args.language)
print("Got data")
training_chars = prepareDataset(training_data, train=True) if args.language == "italian" else prepareDatasetChunks(training_data, train=True)
rnn_drop.train(False)
startTime = time.time()
trainChars = 0
counter = 0
while True:
counter += 1
try:
numeric = [next(training_chars) for _ in range(args.batchSize)]
except StopIteration:
break
printHere = (counter % 50 == 0)
forward(numeric, printHere=printHere, train=True)
#backward(loss, printHere)
if printHere:
print((counter))
print("Dev losses")
def encodeSequenceBaseline(numeric):
assert False
input_tensor = Variable(torch.LongTensor(numeric).transpose(0,1)[1:-1].cuda(), requires_grad=False)
target_tensor_forward = Variable(torch.LongTensor(numeric).transpose(0,1)[2:].cuda(), requires_grad=False).view(args.sequence_length+1, len(numeric), 1, 1)
target_tensor_backward = Variable(torch.LongTensor(numeric).transpose(0,1)[:-2].cuda(), requires_grad=False).view(args.sequence_length+1, len(numeric), 1, 1)
target_tensor = torch.cat([target_tensor_forward, target_tensor_backward], dim=2)
embedded = baseline_char_embeddings(input_tensor)
out, encoded = baseline_rnn_encoder_drop(embedded, None)
out = out.view(args.sequence_length+1, len(numeric), 2, -1)
out1 = out[-1, 0, 0, :] # forward encoding
out2 = out[0, -1, 1, :] # backward encoding
return out1.view(-1), out2.view(-1), encoded[0].view(-1), encoded[1].view(-1)
rnn_drop.train(False)
baseline_rnn_encoder_drop.train(False)
def encodeSequence(numeric):
input_tensor = Variable(torch.LongTensor(numeric).transpose(0,1).cuda(), requires_grad=False)
embedded = char_embeddings(input_tensor)
out, hidden = rnn_drop(embedded, None)
return out[-1].view(-1), hidden[0].view(-1), hidden[1].view(-1)
def encodeSequenceBatch(numeric):
input_tensor = Variable(torch.LongTensor(numeric).transpose(0,1).cuda(), requires_grad=False)
embedded = char_embeddings(input_tensor)
out, hidden = rnn_drop(embedded, None)
return out[-1], hidden[0][0], hidden[1][0]
def encodeBaselineSequenceBatch(numeric):
devLosses = []
for epoch in range(10000):
print(epoch)
training_data = training.iterator()
numericAndBoundaries = prepareDataset(training_data,
train=True) # redo for each epoch
# oldParameters = {}
# for name, param in rnn.named_parameters():
# oldParameters[name] = param
# setattr(rnn, name, torch.nn.functional.dropout(param, p=weight_dropout))
# print(name, param.size())
#
rnn_encoder_drop.train(True)
rnn_decoder_drop.train(True)
startTime = time.time()
trainChars = 0
for offset in range(0, len(numericAndBoundaries), args.batchSize):
printHere = (int(offset / args.batchSize) % 50 == 0)
loss, charCounts = forward(padWords(
numericAndBoundaries[offset:offset + args.batchSize]),
printHere=printHere,
train=True)
backward(loss, printHere)
trainChars += charCounts
if printHere:
print("Dev losses")
print(devLosses)
print("Chars per sec " + str(trainChars /
torch.nn.utils.clip_grad_value_(parameters_cached,
5.0) #, norm_type="inf")
optim.step()
import time
devLosses = []
for epoch in range(10000):
print(epoch)
training_data = corpusIteratorWikiSequences.training(
args.language, args.sequence)
print("Got data")
training_chars = prepareDatasetChunks(training_data, train=True)
rnn_drop.train(True)
startTime = time.time()
trainChars = 0
counter = 0
hidden, beginning = None, None
while True:
counter += 1
try:
numeric = next(training_chars)
except StopIteration:
break
printHere = (counter % 50 == 0)
loss, charCounts = forward(numeric, printHere=printHere, train=True)
backward(loss, printHere)
trainChars += charCounts
if printHere:
import time
devLosses = []
for epoch in range(10000):
print(epoch)
training_data = training.iterator()
numericAndBoundaries = prepareDataset(training_data, train=True)
# oldParameters = {}
# for name, param in rnn.named_parameters():
# oldParameters[name] = param
# setattr(rnn, name, torch.nn.functional.dropout(param, p=weight_dropout))
# print(name, param.size())
#
rnn_forward_drop.train(True)
rnn_backward_drop.train(True)
startTime = time.time()
trainChars = 0
for offset in range(0, len(numericAndBoundaries), args.batchSize):
printHere = (int(offset / args.batchSize) % 50 == 0)
loss, charCounts = forward(numericAndBoundaries[offset:offset +
args.batchSize],
printHere=printHere,
train=True)
backward(loss, printHere)
trainChars += charCounts
if printHere:
print("Dev losses")
print(devLosses)
