Beispiel #1
0
 def try_subproc_toks(self, node, strip_expr=False):
     """Tries to parse the line of the node as a subprocess."""
     line = self.lines[node.lineno - 1]
     if self.mode == 'eval':
         mincol = len(line) - len(line.lstrip())
         maxcol = None
     else:
         mincol = min_col(node)
         maxcol = max_col(node)
         if mincol == maxcol:
             maxcol = find_next_break(line,
                                      mincol=mincol,
                                      lexer=self.parser.lexer)
         else:
             maxcol += 1
     spline = subproc_toks(line,
                           mincol=mincol,
                           maxcol=maxcol,
                           returnline=False,
                           lexer=self.parser.lexer)
     if spline is None:
         return node
     try:
         newnode = self.parser.parse(spline, mode=self.mode)
         newnode = newnode.body
         if not isinstance(newnode, AST):
             # take the first (and only) Expr
             newnode = newnode[0]
         newnode.lineno = node.lineno
         newnode.col_offset = node.col_offset
     except SyntaxError:
         newnode = node
     if strip_expr and isinstance(newnode, Expr):
         newnode = newnode.value
     return newnode
Beispiel #2
0
 def try_subproc_toks(self, node, strip_expr=False):
     """Tries to parse the line of the node as a subprocess."""
     line = self.lines[node.lineno - 1]
     if self.mode == 'eval':
         mincol = len(line) - len(line.lstrip())
         maxcol = None
     else:
         mincol = min_col(node)
         maxcol = max_col(node)
         if mincol == maxcol:
             maxcol = find_next_break(line, mincol=mincol,
                                      lexer=self.parser.lexer)
         else:
             maxcol += 1
     spline = subproc_toks(line,
                           mincol=mincol,
                           maxcol=maxcol,
                           returnline=False,
                           lexer=self.parser.lexer)
     if spline is None:
         return node
     try:
         newnode = self.parser.parse(spline, mode=self.mode)
         newnode = newnode.body
         if not isinstance(newnode, AST):
             # take the first (and only) Expr
             newnode = newnode[0]
         increment_lineno(newnode, n=node.lineno - 1)
         newnode.col_offset = node.col_offset
     except SyntaxError:
         newnode = node
     if strip_expr and isinstance(newnode, Expr):
         newnode = newnode.value
     return newnode
Beispiel #3
0
 def try_subproc_toks(self, node, strip_expr=False):
     """Tries to parse the line of the node as a subprocess."""
     line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1)
     if self.mode == "eval":
         mincol = len(line) - len(line.lstrip())
         maxcol = None
     else:
         mincol = max(min_col(node) - 1, 0)
         maxcol = max_col(node)
         if mincol == maxcol:
             maxcol = find_next_break(line, mincol=mincol, lexer=self.parser.lexer)
         elif nlogical > 1:
             maxcol = None
         elif maxcol < len(line) and line[maxcol] == ";":
             pass
         else:
             maxcol += 1
     spline = subproc_toks(
         line,
         mincol=mincol,
         maxcol=maxcol,
         returnline=False,
         lexer=self.parser.lexer,
     )
     if spline is None or spline != "![{}]".format(line[mincol:maxcol].strip()):
         # failed to get something consistent, try greedy wrap
         spline = subproc_toks(
             line,
             mincol=mincol,
             maxcol=maxcol,
             returnline=False,
             lexer=self.parser.lexer,
             greedy=True,
         )
     if spline is None:
         return node
     try:
         newnode = self.parser.parse(
             spline,
             mode=self.mode,
             filename=self.filename,
             debug_level=(self.debug_level > 2),
         )
         newnode = newnode.body
         if not isinstance(newnode, AST):
             # take the first (and only) Expr
             newnode = newnode[0]
         increment_lineno(newnode, n=node.lineno - 1)
         newnode.col_offset = node.col_offset
         if self.debug_level > 1:
             msg = "{0}:{1}:{2}{3} - {4}\n" "{0}:{1}:{2}{3} + {5}"
             mstr = "" if maxcol is None else ":" + str(maxcol)
             msg = msg.format(self.filename, node.lineno, mincol, mstr, line, spline)
             print(msg, file=sys.stderr)
     except SyntaxError:
         newnode = node
     if strip_expr and isinstance(newnode, Expr):
         newnode = newnode.value
     return newnode
Beispiel #4
0
 def try_subproc_toks(self, node, strip_expr=False):
     """Tries to parse the line of the node as a subprocess."""
     line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1)
     if self.mode == 'eval':
         mincol = len(line) - len(line.lstrip())
         maxcol = None
     else:
         mincol = max(min_col(node) - 1, 0)
         maxcol = max_col(node)
         if mincol == maxcol:
             maxcol = find_next_break(line,
                                      mincol=mincol,
                                      lexer=self.parser.lexer)
         elif nlogical > 1:
             maxcol = None
         elif maxcol < len(line) and line[maxcol] == ';':
             pass
         else:
             maxcol += 1
     spline = subproc_toks(line,
                           mincol=mincol,
                           maxcol=maxcol,
                           returnline=False,
                           lexer=self.parser.lexer)
     if spline is None or len(spline) < len(line[mincol:maxcol]) + 2:
         # failed to get something consistent, try greedy wrap
         # The +2 comes from "![]" being length 3, minus 1 since maxcol
         # is one beyond the total length for slicing
         spline = subproc_toks(line,
                               mincol=mincol,
                               maxcol=maxcol,
                               returnline=False,
                               lexer=self.parser.lexer,
                               greedy=True)
     if spline is None:
         return node
     try:
         newnode = self.parser.parse(spline,
                                     mode=self.mode,
                                     filename=self.filename,
                                     debug_level=(self.debug_level > 2))
         newnode = newnode.body
         if not isinstance(newnode, AST):
             # take the first (and only) Expr
             newnode = newnode[0]
         increment_lineno(newnode, n=node.lineno - 1)
         newnode.col_offset = node.col_offset
         if self.debug_level > 1:
             msg = ('{0}:{1}:{2}{3} - {4}\n' '{0}:{1}:{2}{3} + {5}')
             mstr = '' if maxcol is None else ':' + str(maxcol)
             msg = msg.format(self.filename, node.lineno, mincol, mstr,
                              line, spline)
             print(msg, file=sys.stderr)
     except SyntaxError:
         newnode = node
     if strip_expr and isinstance(newnode, Expr):
         newnode = newnode.value
     return newnode
Beispiel #5
0
def test_find_next_break():
    cases = [
        ('ls && echo a', 0, 4),
        ('ls && echo a', 6, None),
        ('ls && echo a || echo b', 6, 14),
        ('(ls) && echo a', 1, 4),
        ('not ls && echo a', 0, 8),
        ('not (ls) && echo a', 0, 8),
        ]
    for line, mincol, exp in cases:
        obs = find_next_break(line, mincol=mincol, lexer=LEXER)
        assert exp == obs
Beispiel #6
0
def test_find_next_break():
    cases = [
        ('ls && echo a', 0, 4),
        ('ls && echo a', 6, None),
        ('ls && echo a || echo b', 6, 14),
        ('(ls) && echo a', 1, 4),
        ('not ls && echo a', 0, 8),
        ('not (ls) && echo a', 0, 8),
    ]
    for line, mincol, exp in cases:
        obs = find_next_break(line, mincol=mincol, lexer=LEXER)
        yield assert_equal, exp, obs
Beispiel #7
0
 def try_subproc_toks(self, node, strip_expr=False):
     """Tries to parse the line of the node as a subprocess."""
     line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1)
     if self.mode == 'eval':
         mincol = len(line) - len(line.lstrip())
         maxcol = None
     else:
         mincol = max(min_col(node) - 1, 0)
         maxcol = max_col(node)
         if mincol == maxcol:
             maxcol = find_next_break(line, mincol=mincol,
                                      lexer=self.parser.lexer)
         elif nlogical > 1:
             maxcol = None
         elif maxcol < len(line) and line[maxcol] == ';':
             pass
         else:
             maxcol += 1
     spline = subproc_toks(line, mincol=mincol, maxcol=maxcol,
                           returnline=False, lexer=self.parser.lexer)
     if spline is None or len(spline) < len(line[mincol:maxcol]) + 2:
         # failed to get something consistent, try greedy wrap
         # The +2 comes from "![]" being length 3, minus 1 since maxcol
         # is one beyond the total length for slicing
         spline = subproc_toks(line, mincol=mincol, maxcol=maxcol,
                               returnline=False, lexer=self.parser.lexer,
                               greedy=True)
     if spline is None:
         return node
     try:
         newnode = self.parser.parse(spline, mode=self.mode,
                                     filename=self.filename,
                                     debug_level=(self.debug_level > 2))
         newnode = newnode.body
         if not isinstance(newnode, AST):
             # take the first (and only) Expr
             newnode = newnode[0]
         increment_lineno(newnode, n=node.lineno - 1)
         newnode.col_offset = node.col_offset
         if self.debug_level > 1:
             msg = ('{0}:{1}:{2}{3} - {4}\n'
                    '{0}:{1}:{2}{3} + {5}')
             mstr = '' if maxcol is None else ':' + str(maxcol)
             msg = msg.format(self.filename, node.lineno,
                              mincol, mstr, line, spline)
             print(msg, file=sys.stderr)
     except SyntaxError:
         newnode = node
     if strip_expr and isinstance(newnode, Expr):
         newnode = newnode.value
     return newnode
Beispiel #8
0
    def _parse_ctx_free(self, input, mode='exec'):
        last_error_line = last_error_col = -1
        parsed = False
        original_error = None
        while not parsed:
            try:
                tree = self.parser.parse(input,
                                         filename=self.filename,
                                         mode=mode,
                                         debug_level=(self.debug_level > 1))
                parsed = True
            except IndentationError as e:
                if original_error is None:
                    raise e
                else:
                    raise original_error
            except SyntaxError as e:
                if original_error is None:
                    original_error = e
                if (e.loc is None) or (last_error_line == e.loc.lineno and
                                       last_error_col in (e.loc.column + 1,
                                                          e.loc.column)):
                    raise original_error
                last_error_col = e.loc.column
                last_error_line = e.loc.lineno
                idx = last_error_line - 1
                lines = input.splitlines()
                line = lines[idx]
                if input.endswith('\n'):
                    lines.append('')
                if len(line.strip()) == 0:
                    # whitespace only lines are not valid syntax in Python's
                    # interactive mode='single', who knew?! Just ignore them.
                    # this might cause actual sytax errors to have bad line
                    # numbers reported, but should only effect interactive mode
                    del lines[idx]
                    last_error_line = last_error_col = -1
                    input = '\n'.join(lines)
                    continue

                if last_error_line > 1 and lines[idx-1].rstrip()[-1:] == ':':
                    # catch non-indented blocks and raise error.
                    prev_indent = len(lines[idx-1]) - len(lines[idx-1].lstrip())
                    curr_indent = len(lines[idx]) - len(lines[idx].lstrip())
                    if prev_indent == curr_indent:
                        raise original_error
                lexer = self.parser.lexer
                maxcol = find_next_break(line, mincol=last_error_col,
                                         lexer=lexer)
                sbpline = subproc_toks(line, returnline=True,
                                       maxcol=maxcol, lexer=lexer)
                if sbpline is None:
                    # subprocess line had no valid tokens,
                    if len(line.partition('#')[0].strip()) == 0:
                        # likely because it only contained a comment.
                        del lines[idx]
                        last_error_line = last_error_col = -1
                        input = '\n'.join(lines)
                        continue
                    else:
                        # or for some other syntax error
                        raise original_error
                elif sbpline[last_error_col:].startswith('![![') or \
                     sbpline.lstrip().startswith('![!['):
                    # if we have already wrapped this in subproc tokens
                    # and it still doesn't work, adding more won't help
                    # anything
                    raise original_error
                else:
                    if self.debug_level:
                        msg = ('{0}:{1}:{2}{3} - {4}\n'
                               '{0}:{1}:{2}{3} + {5}')
                        mstr = '' if maxcol is None else ':' + str(maxcol)
                        msg = msg.format(self.filename, last_error_line,
                                         last_error_col, mstr, line, sbpline)
                        print(msg, file=sys.stderr)
                    lines[idx] = sbpline
                last_error_col += 3
                input = '\n'.join(lines)
        return tree, input
Beispiel #9
0
def test_find_next_break(line, mincol, exp):
    obs = find_next_break(line, mincol=mincol, lexer=LEXER)
    assert exp == obs
Beispiel #10
0
    def _parse_ctx_free(self, input, mode="exec", filename=None, logical_input=False):
        last_error_line = last_error_col = -1
        parsed = False
        original_error = None
        greedy = False
        if filename is None:
            filename = self.filename
        while not parsed:
            try:
                tree = self.parser.parse(
                    input,
                    filename=filename,
                    mode=mode,
                    debug_level=(self.debug_level > 2),
                )
                parsed = True
            except IndentationError as e:
                if original_error is None:
                    raise e
                else:
                    raise original_error
            except SyntaxError as e:
                if original_error is None:
                    original_error = e
                if (e.loc is None) or (
                    last_error_line == e.loc.lineno
                    and last_error_col in (e.loc.column + 1, e.loc.column)
                ):
                    raise original_error from None
                elif last_error_line != e.loc.lineno:
                    original_error = e
                last_error_col = e.loc.column
                last_error_line = e.loc.lineno
                idx = last_error_line - 1
                lines = input.splitlines()
                line, nlogical, idx = get_logical_line(lines, idx)
                if nlogical > 1 and not logical_input:
                    _, sbpline = self._parse_ctx_free(
                        line, mode=mode, filename=filename, logical_input=True
                    )
                    self._print_debug_wrapping(
                        line, sbpline, last_error_line, last_error_col, maxcol=None
                    )
                    replace_logical_line(lines, sbpline, idx, nlogical)
                    last_error_col += 3
                    input = "\n".join(lines)
                    continue
                if input.endswith("\n"):
                    lines.append("")
                if len(line.strip()) == 0:
                    # whitespace only lines are not valid syntax in Python's
                    # interactive mode='single', who knew?! Just ignore them.
                    # this might cause actual syntax errors to have bad line
                    # numbers reported, but should only affect interactive mode
                    del lines[idx]
                    last_error_line = last_error_col = -1
                    input = "\n".join(lines)
                    continue

                if last_error_line > 1 and lines[idx - 1].rstrip()[-1:] == ":":
                    # catch non-indented blocks and raise error.
                    prev_indent = len(lines[idx - 1]) - len(lines[idx - 1].lstrip())
                    curr_indent = len(lines[idx]) - len(lines[idx].lstrip())
                    if prev_indent == curr_indent:
                        raise original_error
                lexer = self.parser.lexer
                maxcol = (
                    None
                    if greedy
                    else find_next_break(line, mincol=last_error_col, lexer=lexer)
                )
                if not greedy and maxcol in (e.loc.column + 1, e.loc.column):
                    # go greedy the first time if the syntax error was because
                    # we hit an end token out of place. This usually indicates
                    # a subshell or maybe a macro.
                    if not balanced_parens(line, maxcol=maxcol):
                        greedy = True
                        maxcol = None
                sbpline = subproc_toks(
                    line, returnline=True, greedy=greedy, maxcol=maxcol, lexer=lexer
                )
                if sbpline is None:
                    # subprocess line had no valid tokens,
                    if len(line.partition("#")[0].strip()) == 0:
                        # likely because it only contained a comment.
                        del lines[idx]
                        last_error_line = last_error_col = -1
                        input = "\n".join(lines)
                        continue
                    elif not greedy:
                        greedy = True
                        continue
                    else:
                        # or for some other syntax error
                        raise original_error
                elif sbpline[last_error_col:].startswith(
                    "![!["
                ) or sbpline.lstrip().startswith("![!["):
                    # if we have already wrapped this in subproc tokens
                    # and it still doesn't work, adding more won't help
                    # anything
                    if not greedy:
                        greedy = True
                        continue
                    else:
                        raise original_error
                # replace the line
                self._print_debug_wrapping(
                    line, sbpline, last_error_line, last_error_col, maxcol=maxcol
                )
                replace_logical_line(lines, sbpline, idx, nlogical)
                last_error_col += 3
                input = "\n".join(lines)
        return tree, input
Beispiel #11
0
    def _parse_ctx_free(self,
                        input,
                        mode="exec",
                        filename=None,
                        logical_input=False):
        last_error_line = last_error_col = -1
        parsed = False
        original_error = None
        greedy = False
        if filename is None:
            filename = self.filename
        if logical_input:
            beg_spaces = starting_whitespace(input)
            input = input[len(beg_spaces):]
        while not parsed:
            try:
                tree = self.parser.parse(
                    input,
                    filename=filename,
                    mode=mode,
                    debug_level=(self.debug_level > 2),
                )
                parsed = True
            except IndentationError as e:
                if original_error is None:
                    raise e
                else:
                    raise original_error
            except SyntaxError as e:
                if original_error is None:
                    original_error = e
                if (e.loc is None) or (last_error_line == e.loc.lineno
                                       and last_error_col
                                       in (e.loc.column + 1, e.loc.column)):
                    raise original_error from None
                elif last_error_line != e.loc.lineno:
                    original_error = e
                last_error_col = e.loc.column
                last_error_line = e.loc.lineno
                idx = last_error_line - 1
                lines = input.splitlines()
                line, nlogical, idx = get_logical_line(lines, idx)
                if nlogical > 1 and not logical_input:
                    _, sbpline = self._parse_ctx_free(line,
                                                      mode=mode,
                                                      filename=filename,
                                                      logical_input=True)
                    self._print_debug_wrapping(line,
                                               sbpline,
                                               last_error_line,
                                               last_error_col,
                                               maxcol=None)
                    replace_logical_line(lines, sbpline, idx, nlogical)
                    last_error_col += 3
                    input = "\n".join(lines)
                    continue
                if input.endswith("\n"):
                    lines.append("")
                if len(line.strip()) == 0:
                    # whitespace only lines are not valid syntax in Python's
                    # interactive mode='single', who knew?! Just ignore them.
                    # this might cause actual syntax errors to have bad line
                    # numbers reported, but should only affect interactive mode
                    del lines[idx]
                    last_error_line = last_error_col = -1
                    input = "\n".join(lines)
                    continue

                if last_error_line > 1 and lines[idx - 1].rstrip()[-1:] == ":":
                    # catch non-indented blocks and raise error.
                    prev_indent = len(lines[idx - 1]) - len(
                        lines[idx - 1].lstrip())
                    curr_indent = len(lines[idx]) - len(lines[idx].lstrip())
                    if prev_indent == curr_indent:
                        raise original_error
                lexer = self.parser.lexer
                maxcol = (None if greedy else find_next_break(
                    line, mincol=last_error_col, lexer=lexer))
                if not greedy and maxcol in (e.loc.column + 1, e.loc.column):
                    # go greedy the first time if the syntax error was because
                    # we hit an end token out of place. This usually indicates
                    # a subshell or maybe a macro.
                    if not balanced_parens(line, maxcol=maxcol):
                        greedy = True
                        maxcol = None
                sbpline = subproc_toks(line,
                                       returnline=True,
                                       greedy=greedy,
                                       maxcol=maxcol,
                                       lexer=lexer)
                if sbpline is None:
                    # subprocess line had no valid tokens,
                    if len(line.partition("#")[0].strip()) == 0:
                        # likely because it only contained a comment.
                        del lines[idx]
                        last_error_line = last_error_col = -1
                        input = "\n".join(lines)
                        continue
                    elif not greedy:
                        greedy = True
                        continue
                    else:
                        # or for some other syntax error
                        raise original_error
                elif sbpline[last_error_col:].startswith(
                        "![![") or sbpline.lstrip().startswith("![!["):
                    # if we have already wrapped this in subproc tokens
                    # and it still doesn't work, adding more won't help
                    # anything
                    if not greedy:
                        greedy = True
                        continue
                    else:
                        raise original_error
                # replace the line
                self._print_debug_wrapping(line,
                                           sbpline,
                                           last_error_line,
                                           last_error_col,
                                           maxcol=maxcol)
                replace_logical_line(lines, sbpline, idx, nlogical)
                last_error_col += 3
                input = "\n".join(lines)
        if logical_input:
            input = beg_spaces + input
        return tree, input
Beispiel #12
0
    def _parse_ctx_free(self, input, mode='exec'):
        last_error_line = last_error_col = -1
        parsed = False
        original_error = None
        while not parsed:
            try:
                tree = self.parser.parse(input,
                                         filename=self.filename,
                                         mode=mode,
                                         debug_level=(self.debug_level > 1))
                parsed = True
            except IndentationError as e:
                if original_error is None:
                    raise e
                else:
                    raise original_error
            except SyntaxError as e:
                if original_error is None:
                    original_error = e
                if (e.loc is None) or (last_error_line == e.loc.lineno
                                       and last_error_col
                                       in (e.loc.column + 1, e.loc.column)):
                    raise original_error
                last_error_col = e.loc.column
                last_error_line = e.loc.lineno
                idx = last_error_line - 1
                lines = input.splitlines()
                line = lines[idx]
                if input.endswith('\n'):
                    lines.append('')
                if len(line.strip()) == 0:
                    # whitespace only lines are not valid syntax in Python's
                    # interactive mode='single', who knew?! Just ignore them.
                    # this might cause actual sytax errors to have bad line
                    # numbers reported, but should only effect interactive mode
                    del lines[idx]
                    last_error_line = last_error_col = -1
                    input = '\n'.join(lines)
                    continue

                if last_error_line > 1 and lines[idx - 1].rstrip()[-1:] == ':':
                    # catch non-indented blocks and raise error.
                    prev_indent = len(lines[idx - 1]) - len(
                        lines[idx - 1].lstrip())
                    curr_indent = len(lines[idx]) - len(lines[idx].lstrip())
                    if prev_indent == curr_indent:
                        raise original_error
                lexer = self.parser.lexer
                maxcol = find_next_break(line,
                                         mincol=last_error_col,
                                         lexer=lexer)
                sbpline = subproc_toks(line,
                                       returnline=True,
                                       maxcol=maxcol,
                                       lexer=lexer)
                if sbpline is None:
                    # subprocess line had no valid tokens,
                    if len(line.partition('#')[0].strip()) == 0:
                        # likely because it only contained a comment.
                        del lines[idx]
                        last_error_line = last_error_col = -1
                        input = '\n'.join(lines)
                        continue
                    else:
                        # or for some other syntax error
                        raise original_error
                elif sbpline[last_error_col:].startswith('![![') or \
                     sbpline.lstrip().startswith('![!['):
                    # if we have already wrapped this in subproc tokens
                    # and it still doesn't work, adding more won't help
                    # anything
                    raise original_error
                else:
                    if self.debug_level:
                        msg = ('{0}:{1}:{2}{3} - {4}\n' '{0}:{1}:{2}{3} + {5}')
                        mstr = '' if maxcol is None else ':' + str(maxcol)
                        msg = msg.format(self.filename, last_error_line,
                                         last_error_col, mstr, line, sbpline)
                        print(msg, file=sys.stderr)
                    lines[idx] = sbpline
                last_error_col += 3
                input = '\n'.join(lines)
        return tree, input
Beispiel #13
0
def test_find_next_break(line, mincol, exp):
    obs = find_next_break(line, mincol=mincol, lexer=LEXER)
    assert exp == obs