def test(self):
if self.is_inhouse:
return BatchGenerator(self.device,
self.eval_batch_size,
self.inhouse_test_indexes,
self.train_qids,
self.train_labels,
tensors=self.train_data)
else:
return BatchGenerator(self.device,
self.eval_batch_size,
torch.arange(len(self.test_qids)),
self.test_qids,
self.test_labels,
tensors=self.test_data)
def train(self):
if self.is_inhouse:
n_train = self.inhouse_train_indexes.size(0)
train_indexes = self.inhouse_train_indexes[torch.randperm(n_train)]
else:
train_indexes = torch.randperm(len(self.train_qids))
return BatchGenerator(self.device, self.batch_size, train_indexes, self.train_qids, self.train_labels, tensors=self.train_data)
def dev(self):
return BatchGenerator(self.device,
self.eval_batch_size,
torch.arange(len(self.dev_qids)),
self.dev_qids,
self.dev_labels,
tensors=self.dev_data)
def train(model, x, y, epochs, batch_size=1, validation_data=None):
train_op = model.optimizer.minimize(model.loss)
train_gen = BatchGenerator((x, y), batch_size=batch_size)
steps_per_epoch = np.ceil(train_gen.data_length / batch_size).astype(
np.int32)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(1, epochs + 1):
prog_bar = utils.Progbar(steps_per_epoch)
print('Epoch {}/{}'.format(epoch, epochs))
for step, (inputs, labels,
sequence_length) in enumerate(train_gen.next_batch(),
1):
indices = np.array([labels.row, labels.col]).T
values = np.array(labels.data)
dense_shape = labels.shape
_, loss = sess.run(
[train_op, model.loss],
feed_dict={
model.inputs:
inputs,
model.labels:
tf.SparseTensorValue(indices, values, dense_shape),
model.sequence_length:
sequence_length
})
if steps_per_epoch == step:
update_values = [('loss', loss)]
if validation_data is not None:
val_ler, val_loss = eval(sess, model, *validation_data)
update_values += [('val_loss', val_loss),
('val_ler', val_ler)]
prog_bar.update(step, values=update_values, force=True)
else:
prog_bar.update(step, values=[('loss', loss)])
def eval(sess, model, x, y, batch_size=32):
gen = BatchGenerator((x, y), batch_size=batch_size)
ler, loss = 0, 0
for inputs, labels, sequence_length in gen.next_batch():
indices = np.array([labels.row, labels.col]).T
values = np.array(labels.data)
dense_shape = labels.shape
_labels = tf.SparseTensorValue(indices, values, dense_shape)
decoded, _loss = sess.run(
[model.decoded, model.loss],
feed_dict={
model.inputs: inputs,
model.labels: _labels,
model.sequence_length: sequence_length
})
_ler = edit_distance(decoded[0], _labels)
ler += _ler * len(inputs)
loss += _loss * len(inputs)
return ler / len(x), loss / len(x)