def _compute_thought_vectors(self, context_token_ids): # thought_vector is (1 x thought_vector_dim); # context_token_ids is (input_seq_len), but we need to make it 1 x input_seq_len, because theano functions # require input_seq_len dimension to be the second one. thought_vector = get_thought_vectors(self._nn_model, context_token_ids[np.newaxis, :]) # All theano functions process each sequence independently: every input sequence is matched to the corresponding # output sequence. So if we want to have probability of all outputs given the save inputs, we need to repeat # the input <num_outputs> times. <num_outputs> = beam_size here. self._thought_batch = np.repeat(thought_vector, self._beam_size, axis=0)
def _compute_thought_vectors(self, context_token_ids): # thought_vector is (1 x thought_vector_dim); # context_token_ids is (input_seq_len), but we need to make it 1 x input_seq_len, because theano functions # require input_seq_len dimension to be the second one. thought_vector = get_thought_vectors(self._nn_model, context_token_ids[np.newaxis, :]) # All theano functions process each sequence independently: every input sequence is matched to the corresponding # output sequence. So if we want to have probability of all outputs given the save inputs, we need to repeat # the input <num_outputs> times. <num_outputs> = beam_size here. self._thought_batch = np.repeat(thought_vector, self._beam_size, axis=0)
def generate_candidates(self, context_tokens_ids, condition_ids, output_seq_len): """ Predict answers for every sequence token by token until EOS_TOKEN occurred in the sequence using sampling with temperature. During the sampling procedure offensive and <unk> tokens are banned. Probabilities of tokens that have already been used in a response are penalized (divided by REPETITION_PENALIZE_COEFFICIENT). All the rest of the sequence is filled with PAD_TOKENs. """ thought_vectors = get_thought_vectors(self._nn_model, context_tokens_ids) sampled_candidates = [ self._sample_response(thought_vectors, condition_ids, output_seq_len) for _ in xrange(self._samples_num) ] # Transpose the result: candidate_id x batch_size x seq_len -> batch_size x candidate_id x seq_len return np.swapaxes(sampled_candidates, 0, 1)
def generate_candidates(self, context_tokens_ids, condition_ids, output_seq_len): """ Predict answers for every sequence token by token until EOS_TOKEN occurred in the sequence using sampling with temperature. During the sampling procedure offensive and <unk> tokens are banned. Probabilities of tokens that have already been used in a response are penalized (divided by REPETITION_PENALIZE_COEFFICIENT). All the rest of the sequence is filled with PAD_TOKENs. """ thought_vectors = get_thought_vectors(self._nn_model, context_tokens_ids) sampled_candidates = [ self._sample_response(thought_vectors, condition_ids, output_seq_len) for _ in range(self._samples_num) ] # Transpose the result: candidate_id x batch_size x seq_len -> batch_size x candidate_id x seq_len return np.swapaxes(sampled_candidates, 0, 1)
def _compute_candidates_scores(self, context, candidates, condition_id): context = context[np.newaxis, :] # from (seq_len,) to (1 x seq_len) thought_vector = get_thought_vectors(self._nn_model, context) candidates_num_repetitions = self._compute_num_repetitions(candidates) if self._reverse_model_score_weight == 0.0: candidates_scores = self._compute_likelihood_of_output_given_input(thought_vector, candidates, condition_id) elif self._reverse_model_score_weight == 1.0: # Don't compute the likelihood in this case for performance candidates_scores = self._compute_likelihood_of_input_given_output(context, candidates, condition_id) else: candidates_likelihood = self._compute_likelihood_of_output_given_input(thought_vector, candidates, condition_id) candidates_reverse_likelihood = self._compute_likelihood_of_input_given_output( context, candidates, condition_id) candidates_scores = (1 - self._reverse_model_score_weight) * candidates_likelihood + \ self._reverse_model_score_weight * candidates_reverse_likelihood candidates_scores -= self._log_repetition_penalization_coefficient * candidates_num_repetitions return candidates_scores
def _compute_candidates_scores(self, context, candidates, condition_id): _logger.info('Reranking {} candidates...'.format(candidates.shape[0])) context = context[np.newaxis, :] # from (seq_len,) to (1 x seq_len) thought_vector = get_thought_vectors(self._nn_model, context) candidates_num_repetitions = self._compute_num_repetitions(candidates) if self._reverse_model_score_weight == 0.0: candidates_scores = self._compute_likelihood_of_output_given_input(thought_vector, candidates, condition_id) elif self._reverse_model_score_weight == 1.0: # Don't compute the likelihood in this case for performance candidates_scores = self._compute_likelihood_of_input_given_output(context, candidates, condition_id) else: candidates_likelihood = self._compute_likelihood_of_output_given_input(thought_vector, candidates, condition_id) candidates_reverse_likelihood = self._compute_likelihood_of_input_given_output( context, candidates, condition_id) candidates_scores = (1 - self._reverse_model_score_weight) * candidates_likelihood + \ self._reverse_model_score_weight * candidates_reverse_likelihood candidates_scores -= self._log_repetition_penalization_coefficient * candidates_num_repetitions return candidates_scores