def __init__(self, t): Token.__init__(self, TOK.descr[t[0]], t[1]) self.t0 = t[0] # Token type (TOK.WORD, etc.) self.t1 = t[1] # Token text self.t1_lower = t[1].lower() # Token text, lower case self.t2 = t[2] # Token information, such as part-of-speech annotation, numbers, etc. self.is_upper = self.t1[0] != self.t1_lower[0] # True if starts with upper case self._hash = None # Cached hash # We store a cached check of whether this is an "eo". An "eo" is an adverb (atviksorð) # that cannot also be a preposition ("fs") and is therefore a possible non-ambiguous # prefix to a noun ("einkunn") self._is_eo = None
def __init__(self, parser): Token.__init__(self, parser, parser.tokens['NAME'])
def __init__(self, parser, keywords=None): Token.__init__(self, parser, parser.tokens['NAME']) self.keywords = keywords
def __init__(self, t): Token.__init__(self, TOK.descr[t[0]], t[1]) self.t = t self._hash = None