def Read(self, lex_mode):
#assert self.line_pos <= len(self.line), (self.line, self.line_pos)
tok_type, end_pos = self.match_func(lex_mode, self.line, self.line_pos)
#assert end_pos <= len(self.line)
if tok_type == Id.Eol_Tok: # Do NOT add a span for this sentinel!
return ast.token(tok_type, '', const.NO_INTEGER)
tok_val = self.line[self.line_pos:end_pos]
# NOTE: tok_val is redundant, but even in osh.asdl we have some separation
# between data needed for formatting and data needed for execution. Could
# revisit this later.
# TODO: Add this back once arena is threaded everywhere
#assert self.line_id != -1
line_span = ast.line_span(self.line_id, self.line_pos, len(tok_val))
# NOTE: We're putting the arena hook in LineLexer and not Lexer because we
# want it to be "low level". The only thing fabricated here is a newline
# added at the last line, so we don't end with \0.
if self.arena_skip:
assert self.last_span_id != const.NO_INTEGER
span_id = self.last_span_id
self.arena_skip = False
else:
span_id = self.arena.AddLineSpan(line_span)
self.last_span_id = span_id
#log('LineLexer.Read() span ID %d for %s', span_id, tok_type)
t = ast.token(tok_type, tok_val, span_id)
self.line_pos = end_pos
return t
def _InitMem():
# empty environment, no arena.
arena = test_lib.MakeArena('<state_test.py>')
line_id = arena.AddLine(1, 'foo')
span = ast.line_span(line_id, 0, 1) # dummy
arena.AddLineSpan(span)
return state.Mem('', [], {}, arena)
def GetSpanIdForEof(self): assert self.arena, self.arena # This is mandatory now? # zero length is special! line_span = ast.line_span(self.line_id, self.line_pos, 0) return self.arena.AddLineSpan(line_span)