def __init__(self, t):
Token.__init__(self, TOK.descr[t[0]], t[1])
self.t0 = t[0] # Token type (TOK.WORD, etc.)
self.t1 = t[1] # Token text
self.t1_lower = t[1].lower() # Token text, lower case
self.t2 = t[2] # Token information, such as part-of-speech annotation, numbers, etc.
self.is_upper = self.t1[0] != self.t1_lower[0] # True if starts with upper case
self._hash = None # Cached hash
# We store a cached check of whether this is an "eo". An "eo" is an adverb (atviksorð)
# that cannot also be a preposition ("fs") and is therefore a possible non-ambiguous
# prefix to a noun ("einkunn")
self._is_eo = None
def __init__(self, parser):
Token.__init__(self, parser, parser.tokens['NAME'])
def __init__(self, parser, keywords=None):
Token.__init__(self, parser, parser.tokens['NAME'])
self.keywords = keywords
def __init__(self, t):
Token.__init__(self, TOK.descr[t[0]], t[1])
self.t = t
self._hash = None