def _ParseHereDocBody(parse_ctx, h, line_reader, arena): """Fill in attributes of a pending here doc node.""" # "If any character in word is quoted, the delimiter shall be formed by # performing quote removal on word, and the here-document lines shall not # be expanded. Otherwise, the delimiter shall be the word itself." # NOTE: \EOF counts, or even E\OF ok, delimiter, delim_quoted = word.StaticEval(h.here_begin) if not ok: p_die('Invalid here doc delimiter', word=h.here_begin) here_lines, last_line = _ReadHereLines(line_reader, h, delimiter) if delim_quoted: # << 'EOF' # LiteralPart for each line. h.stdin_parts = _MakeLiteralHereLines(here_lines, arena) else: line_reader = reader.VirtualLineReader(here_lines, arena) w_parser = parse_ctx.MakeWordParserForHereDoc(line_reader) w_parser.ReadHereDocBody(h.stdin_parts) # fills this in end_line_id, end_line, end_pos = last_line # Create a span with the end terminator. Maintains the invariant that # the spans "add up". line_span = syntax_asdl.line_span(end_line_id, end_pos, len(end_line)) h.here_end_span_id = arena.AddLineSpan(line_span)
def _InitMem(): # empty environment, no arena. arena = test_lib.MakeArena('<state_test.py>') line_id = arena.AddLine(1, 'foo') span = syntax_asdl.line_span(line_id, 0, 1) # dummy arena.AddLineSpan(span) return state.Mem('', [], {}, arena)
def Read(self, lex_mode): #assert self.line_pos <= len(self.line), (self.line, self.line_pos) tok_type, end_pos = self.match_func(lex_mode, self.line, self.line_pos) #assert end_pos <= len(self.line) if tok_type == Id.Eol_Tok: # Do NOT add a span for this sentinel! return syntax.token(tok_type, '', const.NO_INTEGER) tok_val = self.line[self.line_pos:end_pos] # NOTE: tok_val is redundant, but even in osh.asdl we have some separation # between data needed for formatting and data needed for execution. Could # revisit this later. # TODO: Add this back once arena is threaded everywhere #assert self.line_id != -1 line_span = syntax.line_span(self.line_id, self.line_pos, len(tok_val)) # NOTE: We're putting the arena hook in LineLexer and not Lexer because we # want it to be "low level". The only thing fabricated here is a newline # added at the last line, so we don't end with \0. if self.arena_skip: assert self.last_span_id != const.NO_INTEGER span_id = self.last_span_id self.arena_skip = False else: span_id = self.arena.AddLineSpan(line_span) self.last_span_id = span_id #log('LineLexer.Read() span ID %d for %s', span_id, tok_type) t = syntax.token(tok_type, tok_val, span_id) self.line_pos = end_pos return t
def _MakeLiteralHereLines(here_lines, arena): """Create a line_span and a token for each line.""" tokens = [] for line_id, line, start_offset in here_lines: line_span = syntax_asdl.line_span(line_id, start_offset, len(line)) span_id = arena.AddLineSpan(line_span) t = syntax_asdl.token(Id.Lit_Chars, line[start_offset:], span_id) tokens.append(t) return [word_part.LiteralPart(t) for t in tokens]
def GetSpanIdForEof(self): # type: () -> int # zero length is special! line_span = syntax.line_span(self.line_id, self.line_pos, 0) return self.arena.AddLineSpan(line_span)
def GetSpanIdForEof(self): assert self.arena, self.arena # This is mandatory now? # zero length is special! line_span = syntax.line_span(self.line_id, self.line_pos, 0) return self.arena.AddLineSpan(line_span)