示例#1
0
文件: parsers.py 项目: ICCV/mimr
 def gen_content_ne(self):
     content_tokens = self._pure_tokenize_content()
     names = TextPreprocesser.gen_ner(content_tokens)
     return names
示例#2
0
文件: parsers.py 项目: ICCV/mimr
 def gen_title_ne(self):
     title_tokens = self._pure_tokenize_title()
     names = TextPreprocesser.gen_ner(title_tokens)
     return names
示例#3
0
文件: parsers.py 项目: ICCV/mimr
 def gen_tokenized_content(self):
     tokens = self._pure_tokenize_content()
     tokens = TextPreprocesser.filter(tokens)
     return tokens
示例#4
0
文件: parsers.py 项目: ICCV/mimr
 def gen_tokenized_title(self):
     tokens = self._pure_tokenize_title()
     tokens = TextPreprocesser.filter(tokens)
     return tokens
示例#5
0
文件: parsers.py 项目: ICCV/mimr
 def _pure_tokenize_content(self):
     if self._tokenized_content is None:
         self._tokenized_content = TextPreprocesser.tokenizor(self.content)
     return self._tokenized_content
示例#6
0
文件: parsers.py 项目: ICCV/mimr
 def _pure_tokenize_title(self):
     if self._tokenized_title is None:
         self._tokenized_title = TextPreprocesser.tokenizor(self.title)
     return self._tokenized_title