print "Dev examples " + str(devCounter) devSurprisalTableHere = [ surp / (devCounter * args.batchSize) for surp in surprisalTable ] return devLoss / devWords, devSurprisalTableHere, devMemory / devCounter wordsProcessed = 0 startTime = time.time() DEV_PERIOD = 10000 epochCount = 0 while failedDevRuns < 10: epochCount += 1 print "Starting new epoch, permuting corpus" corpus = corpusIteratorWikiWords.training(args.language) # stream = createStream(corpus) stream = prepareDatasetChunks(corpus, train=True) while True: counter += 1 printHere = (counter % 50 == 0) if counter % DEV_PERIOD == 0: hidden = None beginning = zeroBeginning newDevLoss, devSurprisalTableHere, newDevMemory = computeDevLoss() hidden = None beginning = zeroBeginning
loss.backward() torch.nn.utils.clip_grad_value_(parameters_cached, 5.0) #, norm_type="inf") optim.step() lossHasBeenBad = 0 import time totalStartTime = time.time() devLosses = [] for epoch in range(10000): print(epoch) training_data_1 = corpusIteratorWikiWords.training(args.language1) training_data_2 = corpusIteratorWikiWords.training(args.language2) print("Got data") training_chars = prepareDatasetChunksTwo(training_data_1, training_data_2, train=True) rnn_drop.train(True) startTime = time.time() trainChars = 0 counter = 0 hidden, beginning = None, None while True: counter += 1 try:
] #print >> outFile, " ".join(map(str,devLosses)) devSurprisalTable = [ float(x) for x in next(inFile).strip().split(" ") ] # print >> outFile, " ".join(map(str,devSurprisalTable)) devMemories = [float(x) for x in next(inFile).strip().split(" ") ] # print >> outFile, " ".join(map(str, devMemories)) wordsProcessed = 0 startTime = time.time() DEV_PERIOD = 30000 epochCount = 0 while failedDevRuns < 10: epochCount += 1 print "Starting new epoch, permuting corpus" corpus_1 = corpusIteratorWikiWords.training(args.language1) corpus_2 = corpusIteratorWikiWords.training(args.language2) # stream = createStream(corpus) stream = prepareDatasetChunksTwo(corpus_1, corpus_2, train=True) while True: counter += 1 printHere = (counter % 50 == 0) if counter % DEV_PERIOD == 0: hidden = None beginning = zeroBeginning newDevLoss, devSurprisalTableHere, newDevMemory = computeDevLoss()
import corpusIteratorWikiWords for chunk in corpusIteratorWikiWords.training("english"): print(chunk)