def test(self): if self.is_inhouse: return BatchGenerator(self.device, self.eval_batch_size, self.inhouse_test_indexes, self.train_qids, self.train_labels, tensors=self.train_data) else: return BatchGenerator(self.device, self.eval_batch_size, torch.arange(len(self.test_qids)), self.test_qids, self.test_labels, tensors=self.test_data)
def train(self): if self.is_inhouse: n_train = self.inhouse_train_indexes.size(0) train_indexes = self.inhouse_train_indexes[torch.randperm(n_train)] else: train_indexes = torch.randperm(len(self.train_qids)) return BatchGenerator(self.device, self.batch_size, train_indexes, self.train_qids, self.train_labels, tensors=self.train_data)
def dev(self): return BatchGenerator(self.device, self.eval_batch_size, torch.arange(len(self.dev_qids)), self.dev_qids, self.dev_labels, tensors=self.dev_data)
def train(model, x, y, epochs, batch_size=1, validation_data=None): train_op = model.optimizer.minimize(model.loss) train_gen = BatchGenerator((x, y), batch_size=batch_size) steps_per_epoch = np.ceil(train_gen.data_length / batch_size).astype( np.int32) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for epoch in range(1, epochs + 1): prog_bar = utils.Progbar(steps_per_epoch) print('Epoch {}/{}'.format(epoch, epochs)) for step, (inputs, labels, sequence_length) in enumerate(train_gen.next_batch(), 1): indices = np.array([labels.row, labels.col]).T values = np.array(labels.data) dense_shape = labels.shape _, loss = sess.run( [train_op, model.loss], feed_dict={ model.inputs: inputs, model.labels: tf.SparseTensorValue(indices, values, dense_shape), model.sequence_length: sequence_length }) if steps_per_epoch == step: update_values = [('loss', loss)] if validation_data is not None: val_ler, val_loss = eval(sess, model, *validation_data) update_values += [('val_loss', val_loss), ('val_ler', val_ler)] prog_bar.update(step, values=update_values, force=True) else: prog_bar.update(step, values=[('loss', loss)])
def eval(sess, model, x, y, batch_size=32): gen = BatchGenerator((x, y), batch_size=batch_size) ler, loss = 0, 0 for inputs, labels, sequence_length in gen.next_batch(): indices = np.array([labels.row, labels.col]).T values = np.array(labels.data) dense_shape = labels.shape _labels = tf.SparseTensorValue(indices, values, dense_shape) decoded, _loss = sess.run( [model.decoded, model.loss], feed_dict={ model.inputs: inputs, model.labels: _labels, model.sequence_length: sequence_length }) _ler = edit_distance(decoded[0], _labels) ler += _ler * len(inputs) loss += _loss * len(inputs) return ler / len(x), loss / len(x)