def next(self):
while True:
if self.idx >= len(self.s):
raise StopIteration
for rule in self.lexer.ignore_rules:
match = rule.matches(self.s, self.idx)
if match:
self._update_pos(match)
break
else:
break
for rule in self.lexer.rules:
match = rule.matches(self.s, self.idx)
if match:
lineno = self._lineno
self._colno = self._update_pos(match)
source_pos = SourcePosition(match.start, lineno, self._colno)
if rule.name == "MISMATCH":
raise LexingError(
"%r unexpected" % self.s[match.start:match.end],
SourcePosition(self.idx, self._lineno, self._colno))
token = Token(rule.name, self.s[match.start:match.end],
source_pos)
return token
def next(self):
while True:
if self.idx >= len(self.s):
raise StopIteration
for rule in self._get_current_state().ignore_rules:
match = rule.matches(self.s, self.idx)
if match:
self._update_pos(match)
self._make_transition(rule)
break
else:
break
for rule in self._get_current_state().rules:
match = rule.matches(self.s, self.idx)
if match:
lineno = self._lineno
colno = self._update_pos(match)
source_pos = SourcePosition(match.start, lineno, colno)
token = Token(rule.name, self.s[match.start:match.end],
source_pos)
self._make_transition(rule)
return token
else:
raise LexingError(None, SourcePosition(self.idx, -1, -1))
def next(self):
if self.idx >= len(self.s):
raise StopIteration
for rule in self.lexer.ignore_rules:
match = rule.matches(self.s, self.idx)
if match:
self.idx = match.end
return self.next()
for rule in self.lexer.rules:
match = rule.matches(self.s, self.idx)
if match:
source_pos = self.__get_position__(match.start)
token = Token(rule.name, self.s[match.start:match.end],
source_pos)
self.idx = match.end
return token
else:
raise LexingError(None, SourcePosition(self.idx, -1, -1))
def next(self):
if self.idx >= len(self.s):
raise StopIteration
for rule in self.lexer.ignore_rules:
match = rule.matches(self.s, self.idx)
if match:
self._update_pos(match)
return self.next()
for rule in self.lexer.rules:
match = rule.matches(self.s, self.idx)
if match:
lineno = self._lineno
colno = self._update_pos(match)
source_pos = SourcePosition(match.start, lineno, colno)
token = Token(rule.name, self.s[match.start:match.end],
source_pos)
return token
else:
raise LexingError(None, SourcePosition(self.idx, -1, -1))
def next(self):
if self.idx >= len(self.s):
return None
for rule in self.lexer.ignore_rules:
match = rule.matches(self.s, self.idx)
if match:
self.idx = match.end
return self.next()
for rule in self.lexer.rules:
match = rule.matches(self.s, self.idx)
if match:
# TODO: lineno and colno
source_pos = SourcePosition(match.start, -1, -1)
token = Token(rule.name, self.s[match.start:match.end],
source_pos)
self.idx = match.end
return token
else:
raise LexingError(None, SourcePosition(self.idx, -1, -1))
def next(self):
while True:
if self.idx >= len(self.s):
raise StopIteration
for rule in self.lexer.ignore_rules:
match = rule.matches(self.s, self.idx)
if match:
self._update_pos(match)
break
else:
break
for rule in self.lexer.rules:
match = rule.matches(self.s, self.idx)
if match:
lineno = self._lineno
colno = self._update_pos(match)
source_pos = SourcePosition(match.start, lineno, colno)
source_str = self.s[match.start:match.end]
name = self.lexer.reserved_dict.get(source_str, rule.name)
token = Token(name, source_str, source_pos)
return token
else:
raise LexingError(None, SourcePosition(self.idx, -1, -1))