def get_question_tokens(self, index):
tokens = [t.lower() for t in self._questions_data[index]['tokens']]
if self._p.get("replace.entities", False):
tokens = graph.replace_entities_in_instance(
tokens, [self._questions_data[index]])
if self._p.get("normalize.tokens", False):
tokens = [re.sub(r"\d+", "<n>", t.lower()) for t in tokens]
return tokens
def get_question_tokens(self, index):
tokens = [w for w, _, _ in self._dataset_tagged[index]]
if self._p.get("replace.entities",
False) and len(self._silver_graphs) > 0:
tokens = graph.replace_entities_in_instance(
tokens, [
g[0] for g in self._get_question_positive_silver(index)
if len(g) > 0
])
if self._p.get("normalize.tokens", False):
tokens = [re.sub(r"\d+", "<n>", t.lower()) for t in tokens]
return tokens