def _ParseHereDocBody(parse_ctx, h, line_reader, arena): """Fill in attributes of a pending here doc node.""" # "If any character in word is quoted, the delimiter shall be formed by # performing quote removal on word, and the here-document lines shall not # be expanded. Otherwise, the delimiter shall be the word itself." # NOTE: \EOF counts, or even E\OF ok, delimiter, delim_quoted = word.StaticEval(h.here_begin) if not ok: p_die('Invalid here doc delimiter', word=h.here_begin) here_lines, last_line = _ReadHereLines(line_reader, h, delimiter) if delim_quoted: # << 'EOF' # LiteralPart for each line. h.stdin_parts = _MakeLiteralHereLines(here_lines, arena) else: line_reader = reader.VirtualLineReader(here_lines, arena) w_parser = parse_ctx.MakeWordParserForHereDoc(line_reader) w_parser.ReadHereDocBody(h.stdin_parts) # fills this in end_line_id, end_line, end_pos = last_line # Create a span with the end terminator. Maintains the invariant that # the spans "add up". line_span = syntax_asdl.line_span(end_line_id, end_pos, len(end_line)) h.here_end_span_id = arena.AddLineSpan(line_span)
def testLineReadersAreEquivalent(self): a1 = alloc.Arena() r1 = reader.StringLineReader('one\ntwo', a1) a2 = alloc.Arena() f = cStringIO.StringIO('one\ntwo') r2 = reader.FileLineReader(f, a2) a3 = alloc.Arena() lines = [(0, 'one\n', 0), (1, 'two', 0)] r3 = reader.VirtualLineReader(lines, a3) for a in [a1, a2, a3]: a.PushSource(source.MainFile('reader_test.py')) for r in [r1, r2, r3]: print(r) # Lines are added to the arena with a line_id. self.assertEqual((0, 'one\n', 0), r.GetLine()) self.assertEqual((1, 'two', 0), r.GetLine()) self.assertEqual((-1, None, 0), r.GetLine())
def _MaybeExpandAliases(self, words, cur_aliases): """Try to expand aliases. Our implementation of alias has two design choices: - Where to insert it in parsing. We do it at the end of ParseSimpleCommand. - What grammar rule to parse the expanded alias buffer with. In our case it's ParseCommand(). This doesn't quite match what other shells do, but I can't figure out a better places. Most test cases pass, except for ones like: alias LBRACE='{' LBRACE echo one; echo two; } alias MULTILINE='echo 1 echo 2 echo 3' MULTILINE NOTE: dash handles aliases in a totally diferrent way. It has a global variable checkkwd in parser.c. It assigns it all over the grammar, like this: checkkwd = CHKNL | CHKKWD | CHKALIAS; The readtoken() function checks (checkkwd & CHKALIAS) and then calls lookupalias(). This seems to provide a consistent behavior among shells, but it's less modular and testable. Bash also uses a global 'parser_state & PST_ALEXPNEXT'. Returns: A command node if any aliases were expanded, or None otherwise. """ # The last char that we might parse. right_spid = word.RightMostSpanForWord(words[-1]) first_word_str = None # for error message expanded = [] i = 0 n = len(words) while i < n: w = words[i] ok, word_str, quoted = word.StaticEval(w) if not ok or quoted: break alias_exp = self.aliases.get(word_str) if alias_exp is None: break # Prevent infinite loops. This is subtle: we want to prevent infinite # expansion of alias echo='echo x'. But we don't want to prevent # expansion of the second word in 'echo echo', so we add 'i' to # "cur_aliases". if (word_str, i) in cur_aliases: break if i == 0: first_word_str = word_str # for error message #log('%r -> %r', word_str, alias_exp) cur_aliases.append((word_str, i)) expanded.append(alias_exp) i += 1 if not alias_exp.endswith(' '): # alias e='echo [ ' is the same expansion as # alias e='echo [' # The trailing space indicates whether we should continue to expand # aliases; it's not part of it. expanded.append(' ') break # No more expansions if not expanded: # No expansions; caller does parsing. return None # We got some expansion. Now copy the rest of the words. # We need each NON-REDIRECT word separately! For example: # $ echo one >out two # dash/mksh/zsh go beyond the first redirect! while i < n: w = words[i] left_spid = word.LeftMostSpanForWord(w) right_spid = word.RightMostSpanForWord(w) # Adapted from tools/osh2oil.py Cursor.PrintUntil for span_id in xrange(left_spid, right_spid + 1): span = self.arena.GetLineSpan(span_id) line = self.arena.GetLine(span.line_id) piece = line[span.col : span.col + span.length] expanded.append(piece) expanded.append(' ') # Put space back between words. i += 1 code_str = ''.join(expanded) lines = code_str.splitlines(True) # Keep newlines line_info = [] # TODO: Add location information self.arena.PushSource( '<expansion of alias %r at line %d of %s>' % (first_word_str, -1, 'TODO')) try: for i, line in enumerate(lines): line_id = self.arena.AddLine(line, i+1) line_info.append((line_id, line, 0)) finally: self.arena.PopSource() line_reader = reader.VirtualLineReader(line_info, self.arena) cp = self.parse_ctx.MakeOshParser(line_reader) try: node = cp.ParseCommand(cur_aliases=cur_aliases) except util.ParseError as e: # Failure to parse alias expansion is a fatal error # We don't need more handling here/ raise if 0: log('AFTER expansion:') from osh import ast_lib ast_lib.PrettyPrint(node) return node