def train_iter():
Epoch = 0
while True:
if self.reshuffle:
os.popen('python shuffle.py ' + positive_data + ' ' +
negative_data)
os.popen('mv ' + positive_data + '.shuf ' + positive_data)
os.popen('mv ' + negative_data + '.shuf ' + negative_data)
disTrain = disTextIterator(positive_data,
negative_data,
self.dictionaries[1],
batch=self.batch_size *
self.gpu_num,
maxlen=self.max_len,
n_words_target=self.vocab_size)
ExampleNum = 0
print('Epoch :', Epoch)
EpochStart = time.time()
for x, y in disTrain:
if len(x) < self.gpu_num:
continue
ExampleNum += len(x)
yield x, y, Epoch
TimeCost = time.time() - EpochStart
Epoch += 1
print('Seen ', ExampleNum,
' examples for discriminator. Time Cost : ', TimeCost)
def dis_train_iter():
Epoch = 0
while True:
disTrain = disTextIterator(
self.dev_positive_data,
self.dev_negative_data,
self.dictionaries[1],
batch=self.batch_size,
maxlen=self.max_len,
n_words_target=self.vocab_size)
ExampleNum = 0
EpochStart = time.time()
for x, y in disTrain:
ExampleNum += len(x)
yield x, y, Epoch
TimeCost = time.time() - EpochStart
Epoch += 1
def dis_train_iter(dis_positive_data, dis_negative_data, reshuffle, dictionary,
n_words_trg, batch_size, maxlen):
iter = 0
while True:
if reshuffle:
os.popen('python shuffle.py ' + dis_positive_data + ' ' +
dis_positive_data)
os.popen('mv ' + dis_negative_data + '.shuf ' + dis_negtive_data)
os.popen('mv ' + dis_negative_data + '.shuf ' + dis_negative_data)
disTrain = disTextIterator(dis_positive_data, dis_negative_data,
dictionary, batch_size, maxlen, n_words_trg)
iter += 1
ExampleNum = 0
iterStart = time.time()
for x, y in disTrain:
ExampleNum += len(x)
yield x, y, iter
TimeCost = time.time() - EpochStart
print(('Seen ', ExampleNum, ' examples for discriminator. Time cost : ',
TimeCost))