def _InitMem():
# empty environment, no arena.
arena = test_lib.MakeArena('<state_test.py>')
line_id = arena.AddLine(1, 'foo')
span = line_span(line_id, 0, 1) # dummy
arena.AddLineSpan(span)
return state.Mem('', [], {}, arena)
def AddLineSpan(self, line_id, col, length): # type: (int, int, int) -> int """Save a line_span and return a new span ID for later retrieval.""" span_id = len(self.spans) # spids are just array indices span = line_span(line_id, col, length) self.spans.append(span) return span_id
def Read(self, lex_mode):
# type: (lex_mode_t) -> token
# Inner loop optimization
line = self.line
line_pos = self.line_pos
tok_type, end_pos = self.match_func(lex_mode, line, line_pos)
if tok_type == Id.Eol_Tok: # Do NOT add a span for this sentinel!
return token(tok_type, '', const.NO_INTEGER)
tok_val = line[line_pos:end_pos]
# NOTE: tok_val is redundant, but even in osh.asdl we have some separation
# between data needed for formatting and data needed for execution. Could
# revisit this later.
# TODO: Add this back once arena is threaded everywhere
span = line_span(self.line_id, line_pos, len(tok_val))
# NOTE: We're putting the arena hook in LineLexer and not Lexer because we
# want it to be "low level". The only thing fabricated here is a newline
# added at the last line, so we don't end with \0.
if self.arena_skip:
assert self.last_span_id != const.NO_INTEGER
span_id = self.last_span_id
self.arena_skip = False
else:
span_id = self.arena.AddLineSpan(span)
self.last_span_id = span_id
#log('LineLexer.Read() span ID %d for %s', span_id, tok_type)
t = token(tok_type, tok_val, span_id)
self.line_pos = end_pos
return t
def GetSpanIdForEof(self):
# type: () -> int
# zero length is special!
span = line_span(self.line_id, self.line_pos, 0)
return self.arena.AddLineSpan(span)