def data_set_test(test_length):
for input_x, target_y in data_gen.test_set[cnf.task][test_length]:
batch_xs = []
batch_ys = []
for _ in range(ATTEMPTS):
xs, ys = data_gen.add_padding(input_x, target_y, test_length)
batch_xs.append(xs)
batch_ys.append(ys)
yield batch_xs, batch_ys, target_y
def create_batch(generator, batch_size, length, for_training=False):
qna = []
while len(qna) < batch_size:
question, answer = next(generator)
if max(len(question), len(answer)) > length:
continue
question_and_answer = data_gen.add_padding(question, answer, length)
qna.append(question_and_answer)
random.shuffle(qna)
questions, answers = tuple(zip(*qna))
return [questions], [answers]