def testCrfLogLikelihood(self): inputs = np.array( [[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=np.float32) transition_params = np.array( [[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=np.float32) sequence_lengths = np.array(3, dtype=np.int32) num_words = inputs.shape[0] num_tags = inputs.shape[1] with self.test_session() as sess: all_sequence_log_likelihoods = [] # Make sure all probabilities sum to 1. for tag_indices in itertools.product( range(num_tags), repeat=sequence_lengths): tag_indices = list(tag_indices) tag_indices.extend([0] * (num_words - sequence_lengths)) sequence_log_likelihood, _ = crf.crf_log_likelihood( inputs=array_ops.expand_dims(inputs, 0), tag_indices=array_ops.expand_dims(tag_indices, 0), sequence_lengths=array_ops.expand_dims(sequence_lengths, 0), transition_params=constant_op.constant(transition_params)) all_sequence_log_likelihoods.append(sequence_log_likelihood) total_log_likelihood = math_ops.reduce_logsumexp( all_sequence_log_likelihoods) tf_total_log_likelihood = sess.run(total_log_likelihood) self.assertAllClose(tf_total_log_likelihood, 0.0)
def testCrfLogLikelihood(self): inputs = np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=np.float32) transition_params = np.array([[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=np.float32) sequence_lengths = np.array(3, dtype=np.int32) num_words = inputs.shape[0] num_tags = inputs.shape[1] with self.test_session() as sess: all_sequence_log_likelihoods = [] # Make sure all probabilities sum to 1. for tag_indices in itertools.product(range(num_tags), repeat=sequence_lengths): tag_indices = list(tag_indices) tag_indices.extend([0] * (num_words - sequence_lengths)) sequence_log_likelihood, _ = crf.crf_log_likelihood( inputs=array_ops.expand_dims(inputs, 0), tag_indices=array_ops.expand_dims(tag_indices, 0), sequence_lengths=array_ops.expand_dims( sequence_lengths, 0), transition_params=constant_op.constant(transition_params)) all_sequence_log_likelihoods.append(sequence_log_likelihood) total_log_likelihood = math_ops.reduce_logsumexp( all_sequence_log_likelihoods) tf_total_log_likelihood = sess.run(total_log_likelihood) self.assertAllClose(tf_total_log_likelihood, 0.0)
def __call__(self, seqs, lengths, states): ''' states: bs x L x d ''' batch_size, L, d = tf.shape(states)[0], tf.shape(states)[1], tf.shape(states)[2] unary = tf.matmul(tf.reshape(states, [-1, d]), self.softmax_w, transpose_b=True) + self.softmax_b unary = tf.reshape(unary, [batch_size, L, self.vocab_size]) unary.set_shape([None, self.max_sequence_length, self.vocab_size]) # TF akwardness ll, _ = crf_log_likelihood(unary, seqs, lengths, self.pairwise) return ll
test_sequence_lengths_t = tf.constant(test_word_lengths, dtype=tf.int32, name='test_sequence_lengths') w_t = tf.get_variable('W', shape=(num_features, num_tags), dtype=tf.float32, regularizer=None, initializer=tf.initializers.zeros()) transition_weights_t = tf.get_variable('T', shape=(num_tags, num_tags), dtype=tf.float32, regularizer=None, initializer=tf.initializers.zeros()) x_t_features = tf.reshape(x_t, [-1, num_features], name='X_flattened') scores = tf.matmul(x_t_features, w_t, name='energies') scores = tf.reshape(scores, [num_train_examples, num_train_words, num_tags]) # Compute the log-likelihood of the gold sequences and keep the transition # params for inference at test time. log_likelihood, transition_weights_t = crf_log_likelihood(scores, y_t, train_sequence_lengths_t, transition_weights_t) x_train_t_features = tf.reshape(x_t, [-1, num_features], name='X_train_flattened') x_test_t_features = tf.reshape(x_test_t, [-1, num_features], name='X_test_flattened') test_scores = tf.matmul(x_test_t_features, w_t, name='test_energies') test_scores = tf.reshape(test_scores, [num_test_examples, num_test_words, num_tags]) # Compute the viterbi sequence and score. viterbi_sequence_train, viterbi_train_scores = crf_decode(scores, transition_weights_t, train_sequence_lengths_t) viterbi_sequence, viterbi_score = crf_decode(test_scores, transition_weights_t, test_sequence_lengths_t) # Add a training op to tune the parameters. loss = -C * tf.reduce_mean(log_likelihood) loss += 1e-2 * tf.nn.l2_loss(w_t) #loss += 1e-2 * 0.5 * tf.reduce_sum(tf.square(transition_weights_t))