Python Parser.find_tokens示例

示例#1

文件： __init__.py 项目： BarcelonaMedia-ViL/pattern

 def find_tokens(self, tokens, **kwargs):
     # 's in Dutch preceded by a vowel indicates plural ("auto's"): don't replace.
     kwargs.setdefault("abbreviations", ABBREVIATIONS)
     kwargs.setdefault("replace", {"'n": " 'n"})
     s = _Parser.find_tokens(self, tokens, **kwargs)
     s = [re.sub(r"' s (ochtends|morgens|middags|avonds)", "'s \\1", s) for s in s]
     return s

示例#2

 def find_tokens(self, tokens, **kwargs):
     # 's in Dutch preceded by a vowel indicates plural ("auto's"): don't replace.
     kwargs.setdefault("abbreviations", ABBREVIATIONS)
     kwargs.setdefault("replace", {"'n": " 'n"})
     s = _Parser.find_tokens(self, tokens, **kwargs)
     s = [re.sub(r"' s (ochtends|morgens|middags|avonds)", "'s \\1", s) for s in s]
     return s

示例#3

文件： __init__.py 项目： pattern3/pattern

 def find_tokens(self, tokens, **kwargs):
     kwargs.setdefault("abbreviations", ABBREVIATIONS)
     kwargs.setdefault("replace", replacements)
     s = _Parser.find_tokens(self, tokens, **kwargs)
     s = [s.replace("&rsquo ;", u"’") if isinstance(
         s, unicode) else s for s in s]
     return s

示例#4

文件： __init__.py 项目： Afey/pattern

 def find_tokens(self, tokens, **kwargs):
     kwargs.setdefault("abbreviations", ABBREVIATIONS)
     kwargs.setdefault("replace", replacements)
     #return _Parser.find_tokens(self, tokens, **kwargs)
     
     s = _Parser.find_tokens(self, tokens, **kwargs)
     s = [s.replace(" &contraction ;", u"'").replace("XXX -", "-") for s in s]
     return s

示例#5

 def find_tokens(self, tokens, **kwargs):
     kwargs.setdefault("abbreviations", ABBREVIATIONS)
     kwargs.setdefault("replace", replacements)
     #return _Parser.find_tokens(self, tokens, **kwargs)
     
     s = _Parser.find_tokens(self, tokens, **kwargs)
     s = [s.replace(" &contraction ;", u"'").replace("XXX -", "-") for s in s]
     return s

示例#6

 def find_tokens(self, tokens, **kwargs):
     kwargs.setdefault("abbreviations", ABBREVIATIONS)
     kwargs.setdefault("replace", replacements)
     s = _Parser.find_tokens(self, tokens, **kwargs)
     s = [
         s.replace("&rsquo ;", "’") if isinstance(s, str) else s for s in s
     ]
     return s

示例#7

文件： __init__.py 项目： DevKhokhar/pattern

 def find_tokens(self, tokens, **kwargs):
     kwargs.setdefault("abbreviations", ABBREVIATIONS)
     kwargs.setdefault("replace", REPLACEMENTS)
     return _Parser.find_tokens(self, tokens, **kwargs)

示例#8

 def find_tokens(self, tokens, **kwargs):
     kwargs.setdefault("abbreviations", ABBREVIATIONS)
     kwargs.setdefault("replace", REPLACEMENTS)
     return _Parser.find_tokens(self, tokens, **kwargs)

示例#9

文件： Python_Computational_Linguistics_Redefine_Find_Token( ).py 项目： VakinduPhilliam/Python_Computation_Linguistics

    def find_tokens(self, tokens, **kwargs):

        kwargs.setdefault("abbreviations", ABBREVIATIONS)
        kwargs.setdefault("replace", CONTRACTIONS)
        return Parser.find_tokens(self, tokens, **kwargs)