def parse_answer(self, answer): _, text, _ = ldig.normalize_text(answer) events = self.trie.extract_features(u"\u0001" + answer + u"\0001") sum = numpy.zeros(len(self.labels)) data = [] for id in sorted(events, key=lambda id:self.features[id][0]): phi = self.param[id,] sum += phi * events[id] exp_w = numpy.exp(sum - sum.max()) prob = exp_w / exp_w.sum() return [x for x in prob]
def detect(self, st): label, text, org_text = ldig.normalize_text(st) events = self.trie.extract_features(u"\u0001" + text + u"\u0001") sum = numpy.zeros(len(self.labels)) data = [] for id in sorted(events, key=lambda id:self.features[id][0]): phi = self.param[id,] sum += phi * events[id] data.append({"id":int(id), "feature":self.features[id][0], "phi":["%0.3f" % x for x in phi]}) exp_w = numpy.exp(sum - sum.max()) prob = exp_w / exp_w.sum() return {"labels":self.labels, "data":data, "prob":["%0.3f" % x for x in prob]}
def _detect_normalize(self, text): _, text, _ = ldig.normalize_text(text) return self._detect(text)
def assertNormalize(self, org, norm): self.assertEqual(ldig.normalize_text(org), ("", norm, org))