# return loss, len(numeric) * args.sequence_length import time devLosses = [] #for epoch in range(10000): if True: training_data = corpusIteratorWiki.training(args.language) print("Got data") training_chars = prepareDataset(training_data, train=True) if args.language == "italian" else prepareDatasetChunks(training_data, train=True) rnn_drop.train(False) startTime = time.time() trainChars = 0 counter = 0 while True: counter += 1 try: numeric = [next(training_chars) for _ in range(args.batchSize)] except StopIteration: break printHere = (counter % 50 == 0) forward(numeric, printHere=printHere, train=True) #backward(loss, printHere) if printHere: print((counter)) print("Dev losses")
def encodeSequenceBaseline(numeric): assert False input_tensor = Variable(torch.LongTensor(numeric).transpose(0,1)[1:-1].cuda(), requires_grad=False) target_tensor_forward = Variable(torch.LongTensor(numeric).transpose(0,1)[2:].cuda(), requires_grad=False).view(args.sequence_length+1, len(numeric), 1, 1) target_tensor_backward = Variable(torch.LongTensor(numeric).transpose(0,1)[:-2].cuda(), requires_grad=False).view(args.sequence_length+1, len(numeric), 1, 1) target_tensor = torch.cat([target_tensor_forward, target_tensor_backward], dim=2) embedded = baseline_char_embeddings(input_tensor) out, encoded = baseline_rnn_encoder_drop(embedded, None) out = out.view(args.sequence_length+1, len(numeric), 2, -1) out1 = out[-1, 0, 0, :] # forward encoding out2 = out[0, -1, 1, :] # backward encoding return out1.view(-1), out2.view(-1), encoded[0].view(-1), encoded[1].view(-1) rnn_drop.train(False) baseline_rnn_encoder_drop.train(False) def encodeSequence(numeric): input_tensor = Variable(torch.LongTensor(numeric).transpose(0,1).cuda(), requires_grad=False) embedded = char_embeddings(input_tensor) out, hidden = rnn_drop(embedded, None) return out[-1].view(-1), hidden[0].view(-1), hidden[1].view(-1) def encodeSequenceBatch(numeric): input_tensor = Variable(torch.LongTensor(numeric).transpose(0,1).cuda(), requires_grad=False) embedded = char_embeddings(input_tensor) out, hidden = rnn_drop(embedded, None) return out[-1], hidden[0][0], hidden[1][0] def encodeBaselineSequenceBatch(numeric):
devLosses = [] for epoch in range(10000): print(epoch) training_data = training.iterator() numericAndBoundaries = prepareDataset(training_data, train=True) # redo for each epoch # oldParameters = {} # for name, param in rnn.named_parameters(): # oldParameters[name] = param # setattr(rnn, name, torch.nn.functional.dropout(param, p=weight_dropout)) # print(name, param.size()) # rnn_encoder_drop.train(True) rnn_decoder_drop.train(True) startTime = time.time() trainChars = 0 for offset in range(0, len(numericAndBoundaries), args.batchSize): printHere = (int(offset / args.batchSize) % 50 == 0) loss, charCounts = forward(padWords( numericAndBoundaries[offset:offset + args.batchSize]), printHere=printHere, train=True) backward(loss, printHere) trainChars += charCounts if printHere: print("Dev losses") print(devLosses) print("Chars per sec " + str(trainChars /
torch.nn.utils.clip_grad_value_(parameters_cached, 5.0) #, norm_type="inf") optim.step() import time devLosses = [] for epoch in range(10000): print(epoch) training_data = corpusIteratorWikiSequences.training( args.language, args.sequence) print("Got data") training_chars = prepareDatasetChunks(training_data, train=True) rnn_drop.train(True) startTime = time.time() trainChars = 0 counter = 0 hidden, beginning = None, None while True: counter += 1 try: numeric = next(training_chars) except StopIteration: break printHere = (counter % 50 == 0) loss, charCounts = forward(numeric, printHere=printHere, train=True) backward(loss, printHere) trainChars += charCounts if printHere:
import time devLosses = [] for epoch in range(10000): print(epoch) training_data = training.iterator() numericAndBoundaries = prepareDataset(training_data, train=True) # oldParameters = {} # for name, param in rnn.named_parameters(): # oldParameters[name] = param # setattr(rnn, name, torch.nn.functional.dropout(param, p=weight_dropout)) # print(name, param.size()) # rnn_forward_drop.train(True) rnn_backward_drop.train(True) startTime = time.time() trainChars = 0 for offset in range(0, len(numericAndBoundaries), args.batchSize): printHere = (int(offset / args.batchSize) % 50 == 0) loss, charCounts = forward(numericAndBoundaries[offset:offset + args.batchSize], printHere=printHere, train=True) backward(loss, printHere) trainChars += charCounts if printHere: print("Dev losses") print(devLosses)