def _ParseHereDocBody(parse_ctx, h, line_reader, arena):
"""Fill in attributes of a pending here doc node."""
# "If any character in word is quoted, the delimiter shall be formed by
# performing quote removal on word, and the here-document lines shall not
# be expanded. Otherwise, the delimiter shall be the word itself."
# NOTE: \EOF counts, or even E\OF
ok, delimiter, delim_quoted = word.StaticEval(h.here_begin)
if not ok:
p_die('Invalid here doc delimiter', word=h.here_begin)
here_lines, last_line = _ReadHereLines(line_reader, h, delimiter)
if delim_quoted: # << 'EOF'
# LiteralPart for each line.
h.stdin_parts = _MakeLiteralHereLines(here_lines, arena)
else:
line_reader = reader.VirtualLineReader(here_lines, arena)
w_parser = parse_ctx.MakeWordParserForHereDoc(line_reader)
w_parser.ReadHereDocBody(h.stdin_parts) # fills this in
end_line_id, end_line, end_pos = last_line
# Create a span with the end terminator. Maintains the invariant that
# the spans "add up".
line_span = syntax_asdl.line_span(end_line_id, end_pos, len(end_line))
h.here_end_span_id = arena.AddLineSpan(line_span)
def _InitMem():
# empty environment, no arena.
arena = test_lib.MakeArena('<state_test.py>')
line_id = arena.AddLine(1, 'foo')
span = syntax_asdl.line_span(line_id, 0, 1) # dummy
arena.AddLineSpan(span)
return state.Mem('', [], {}, arena)
def Read(self, lex_mode):
#assert self.line_pos <= len(self.line), (self.line, self.line_pos)
tok_type, end_pos = self.match_func(lex_mode, self.line, self.line_pos)
#assert end_pos <= len(self.line)
if tok_type == Id.Eol_Tok: # Do NOT add a span for this sentinel!
return syntax.token(tok_type, '', const.NO_INTEGER)
tok_val = self.line[self.line_pos:end_pos]
# NOTE: tok_val is redundant, but even in osh.asdl we have some separation
# between data needed for formatting and data needed for execution. Could
# revisit this later.
# TODO: Add this back once arena is threaded everywhere
#assert self.line_id != -1
line_span = syntax.line_span(self.line_id, self.line_pos, len(tok_val))
# NOTE: We're putting the arena hook in LineLexer and not Lexer because we
# want it to be "low level". The only thing fabricated here is a newline
# added at the last line, so we don't end with \0.
if self.arena_skip:
assert self.last_span_id != const.NO_INTEGER
span_id = self.last_span_id
self.arena_skip = False
else:
span_id = self.arena.AddLineSpan(line_span)
self.last_span_id = span_id
#log('LineLexer.Read() span ID %d for %s', span_id, tok_type)
t = syntax.token(tok_type, tok_val, span_id)
self.line_pos = end_pos
return t
def _MakeLiteralHereLines(here_lines, arena):
"""Create a line_span and a token for each line."""
tokens = []
for line_id, line, start_offset in here_lines:
line_span = syntax_asdl.line_span(line_id, start_offset, len(line))
span_id = arena.AddLineSpan(line_span)
t = syntax_asdl.token(Id.Lit_Chars, line[start_offset:], span_id)
tokens.append(t)
return [word_part.LiteralPart(t) for t in tokens]
def GetSpanIdForEof(self):
# type: () -> int
# zero length is special!
line_span = syntax.line_span(self.line_id, self.line_pos, 0)
return self.arena.AddLineSpan(line_span)
def GetSpanIdForEof(self):
assert self.arena, self.arena # This is mandatory now?
# zero length is special!
line_span = syntax.line_span(self.line_id, self.line_pos, 0)
return self.arena.AddLineSpan(line_span)