Beispiel #1
0
    def __init__(self, t):

        Token.__init__(self, TOK.descr[t[0]], t[1])
        self.t0 = t[0] # Token type (TOK.WORD, etc.)
        self.t1 = t[1] # Token text
        self.t1_lower = t[1].lower() # Token text, lower case
        self.t2 = t[2] # Token information, such as part-of-speech annotation, numbers, etc.
        self.is_upper = self.t1[0] != self.t1_lower[0] # True if starts with upper case
        self._hash = None # Cached hash

        # We store a cached check of whether this is an "eo". An "eo" is an adverb (atviksorð)
        # that cannot also be a preposition ("fs") and is therefore a possible non-ambiguous
        # prefix to a noun ("einkunn")
        self._is_eo = None
Beispiel #2
0
 def __init__(self, parser):
     Token.__init__(self, parser, parser.tokens['NAME'])
Beispiel #3
0
 def __init__(self, parser, keywords=None):
     Token.__init__(self, parser, parser.tokens['NAME'])
     self.keywords = keywords
Beispiel #4
0
    def __init__(self, t):

        Token.__init__(self, TOK.descr[t[0]], t[1])
        self.t = t
        self._hash = None