def get_surface_form(self, sentence, pos, possible_forms): log_debug("Pos: %d, forms: %s" % (pos, unicode(", ".join(possible_forms)))) # get unnormalized scores for the whole vocabulary if pos >= self.max_sent_len: # don't use whole sentence if it's too long pos -= pos - self.max_sent_len + 1 sentence = sentence[pos - self.max_sent_len + 1:] inputs = np.array([self._sent_to_ids(sentence)[:-1]], dtype=np.int32) logits = self.session.run([self._logits], {self._inputs: inputs}) # pick out scores for possible forms scores = [ logits[0][pos][self.vocab.get(form.lower(), self.vocab.get('<UNK>'))] for form in possible_forms ] probs = softmax(scores) log_debug("Vocab: %s" % unicode(", ".join([ unicode(self.vocab.get(form.lower(), self.vocab.get('<UNK>'))) for f in possible_forms ]))) log_debug("Scores: %s, Probs: %s" % (unicode(", ".join( ["%.3f" % s for s in scores])), unicode(", ".join(["%.3f" % p for p in probs])))) # sample from the prob. dist. if self._sample: return np.random.choice(possible_forms, p=probs) # get just the most probable option max_idx, _ = max(enumerate(probs), key=operator.itemgetter(1)) return possible_forms[max_idx]
def get_surface_form(self, sentence, pos, possible_forms): scores = [self._word_freq.get(possible_form.lower(), 0) + 0.1 for possible_form in possible_forms] if self._sample: probs = softmax(scores) return np.random.choice(possible_forms, p=probs) max_idx, _ = max(enumerate(scores), key=operator.itemgetter(1)) return possible_forms[max_idx]
def _valid_perplexity(self): """Compute perplexity of the RNNLM on validation data.""" perp = 0 n_toks = 0 for inputs, targets in self._valid_batches(): logits = self.session.run([self._logits], {self._inputs: inputs})[0] probs = softmax(logits) # logits combine all sentences behind each other -- dimension # is (self.max_sent_len * self.batch_size, self.vocab_size) for tok_no in range(len(probs)): perp += np.log2(probs[tok_no, targets[old_div(tok_no, self.max_sent_len), tok_no % self.max_sent_len]]) n_toks += np.prod(inputs.shape) # perp = exp( -1/N * sum_i=1^N log p(x_i) ) return np.exp2(- perp / float(n_toks))
def get_surface_form(self, sentence, pos, possible_forms): state, dummy_state = kenlm.State(), kenlm.State() self._lm.BeginSentenceWrite(state) for idx in range(pos): self._lm.BaseScore(state, sentence[idx].encode('utf-8'), state) best_form_idx = 0 best_score = float('-inf') scores = [] for form_idx, possible_form in enumerate(possible_forms): possible_form = possible_form.lower().replace(' ', '^').encode('utf-8') score = self._lm.BaseScore(state, possible_form, dummy_state) scores.append(score) if score > best_score: best_score = score best_form_idx = form_idx if self._sample: probs = softmax(scores) return np.random.choice(possible_forms, p=probs) return possible_forms[best_form_idx]
def _beam_search_step(self, dec_inputs, dec_states): """Run one step of beam search decoding with the given decoder inputs and (previous steps') outputs and states.""" step = len(dec_states) # find the decoder position # fill in all previous path data for i in xrange(step): self._beam_search_feed_dict[self.dec_inputs[i]] = dec_inputs[i] self._beam_search_feed_dict[self.states[i]] = dec_states[i] # the decoder outputs are always one step longer self._beam_search_feed_dict[self.dec_inputs[step]] = dec_inputs[step] # run one step of the decoder output, state = self.session.run( [self.outputs[step], self.states[step]], feed_dict=self._beam_search_feed_dict) # softmax (normalize decoder outputs to obtain prob. distribution), assuming batches size 1 out_probs = softmax(output[0]) return out_probs, state