def _run(self, text): self.log_d('_run() text=\"%s\")' % (text)) assert self.EOS not in text self.position = point(1, 1) for c in self._chars_plus_eos(text): cr = self._char_type(c) if cr.ctype == self._char_types.UNKNOWN: raise RuntimeError('unknown character: \"%s\"' % (c)) tokens = self.state.handle_char(cr) for token in tokens: self.log_d('tokenize: new token: %s' % (str(token))) yield token self.position = point(self.position.x + 0, self.position.y) assert self.state == self.STATE_DONE yield lexer_token(self.TOKEN_DONE, None, self.position)
def TPUNCT(s, x=1, y=1): return lexer_token(upstream_version_lexer.TOKEN_PUNCTUATION, s, point(x, y))
def TTEXT(s, x=1, y=1): return lexer_token(upstream_version_lexer.TOKEN_TEXT, s, point(x, y))
def TDONE(x=1, y=1): return lexer_token(upstream_version_lexer.TOKEN_DONE, None, point(x, y))
def TSPACE(s=' ', x=1, y=1): return lexer_token(string_lexer.TOKEN_SPACE, s, point(x, y))
def make_token_number(self): return lexer_token(self.TOKEN_NUMBER, int(self.buffer_value()), self.position)
def TDONE(x = 0, y = 0): return lexer_token(semantic_version_lexer.TOKEN_DONE, None, point(x, y)) def TTEXT(value, x = 0, y = 0): return lexer_token(semantic_version_lexer.TOKEN_TEXT, value, point(x, y))
def assertEqual(self, expected, actual): assert isinstance(expected, list) expected = [ lexer_token(*t) for t in expected ] super(test_sentence_lexer, self).assertEqual(expected, actual)
def make_token_part(self): value = self.buffer_value() offset = len(value) - 1 position = self.position.clone(mutations = { 'x': self.position.x - offset }) return lexer_token(self.TOKEN_PART, int(value), position)
def make_token_punctuation(self): value = self.buffer_value() offset = len(value) - 1 position = self.position.clone(mutations = { 'x': self.position.x - offset }) return lexer_token(self.TOKEN_PUNCTUATION, value, position)
def make_token_part_delimiter(self): return lexer_token(self.TOKEN_PART_DELIMITER, self.buffer_value(), self.position)
def TCOMMENT(s, x=1, y=1): return lexer_token(string_lexer.TOKEN_COMMENT, s, point(x, y))
def TSTRING(s, x=1, y=1): return lexer_token(string_lexer.TOKEN_STRING, s, point(x, y))
def TNUM(s, x=1, y=1): return lexer_token(upstream_version_lexer.TOKEN_NUMBER, s, point(x, y))
def TPUN(value, x = 0, y = 0): return lexer_token(semantic_version_lexer.TOKEN_PUNCTUATION, value, point(x, y)) def TPART(value, x = 0, y = 0): return lexer_token(semantic_version_lexer.TOKEN_PART, value, point(x, y))
def assertEqual(self, expected, actual): assert isinstance(expected, list) expected = [lexer_token(*t) for t in expected] super(test_upstream_version_lexer, self).assertEqual(expected, actual)
def TPART(value, x = 0, y = 0): return lexer_token(semantic_version_lexer.TOKEN_PART, value, point(x, y)) def TPD(value, x = 0, y = 0): return lexer_token(semantic_version_lexer.TOKEN_PART_DELIMITER, value, point(x, y))
def TPUNCT(s, x = 1, y = 1): return lexer_token(L.TOKEN_PUNCTUATION, s, (x, y)) class test_sentence_lexer(unit_test):
def TPD(value, x = 0, y = 0): return lexer_token(semantic_version_lexer.TOKEN_PART_DELIMITER, value, point(x, y)) class test_semantic_version_lexer(unit_test):
def make_token_text(self): return lexer_token(self.TOKEN_TEXT, self.buffer_value(), self.position)
def make_token_delimiter(self): return lexer_token(self.TOKEN_DELIMITER, self.delimiter, self.position)
def make_token_punctuation(self): return lexer_token(self.TOKEN_PUNCTUATION, self.buffer_value(), self.position)
def TDONE(x=1, y=1): return lexer_token(string_lexer.TOKEN_DONE, None, point(x, y))