def __init__(self, value): """ Class initializer. Arguments: value -- the string to be tokenized. """ Token.__init__(self, value) self._token_type_string = "Number"
def __init__(self, value, parser): """ Class initializer. Arguments: value -- the string to be tokenized. parser -- a reference to the parser which instantiated the instance, used to parse the contained tokens. """ Token.__init__(self, value) self._token_type_string = "Container" self._tlist = TokenList() # Push opening container token, parsed contained # tokens, and closing container token onto the list c_open, c_middle, c_close = value[0], value[1:-1], value[-1] self._tlist.push(OpeningToken(c_open)) self._tlist.push(parser.list_tokenize(c_middle)) self._tlist.push(ClosingToken(c_close))