def train_iter(): Epoch = 0 while True: if self.reshuffle: os.popen('python shuffle.py ' + positive_data + ' ' + negative_data) os.popen('mv ' + positive_data + '.shuf ' + positive_data) os.popen('mv ' + negative_data + '.shuf ' + negative_data) disTrain = disTextIterator(positive_data, negative_data, self.dictionaries[1], batch=self.batch_size * self.gpu_num, maxlen=self.max_len, n_words_target=self.vocab_size) ExampleNum = 0 print('Epoch :', Epoch) EpochStart = time.time() for x, y in disTrain: if len(x) < self.gpu_num: continue ExampleNum += len(x) yield x, y, Epoch TimeCost = time.time() - EpochStart Epoch += 1 print('Seen ', ExampleNum, ' examples for discriminator. Time Cost : ', TimeCost)
def dis_train_iter(): Epoch = 0 while True: disTrain = disTextIterator( self.dev_positive_data, self.dev_negative_data, self.dictionaries[1], batch=self.batch_size, maxlen=self.max_len, n_words_target=self.vocab_size) ExampleNum = 0 EpochStart = time.time() for x, y in disTrain: ExampleNum += len(x) yield x, y, Epoch TimeCost = time.time() - EpochStart Epoch += 1
def dis_train_iter(dis_positive_data, dis_negative_data, reshuffle, dictionary, n_words_trg, batch_size, maxlen): iter = 0 while True: if reshuffle: os.popen('python shuffle.py ' + dis_positive_data + ' ' + dis_positive_data) os.popen('mv ' + dis_negative_data + '.shuf ' + dis_negtive_data) os.popen('mv ' + dis_negative_data + '.shuf ' + dis_negative_data) disTrain = disTextIterator(dis_positive_data, dis_negative_data, dictionary, batch_size, maxlen, n_words_trg) iter += 1 ExampleNum = 0 iterStart = time.time() for x, y in disTrain: ExampleNum += len(x) yield x, y, iter TimeCost = time.time() - EpochStart print(('Seen ', ExampleNum, ' examples for discriminator. Time cost : ', TimeCost))