def try_subproc_toks(self, node, strip_expr=False): """Tries to parse the line of the node as a subprocess.""" line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1) if self.mode == "eval": mincol = len(line) - len(line.lstrip()) maxcol = None else: mincol = max(min_col(node) - 1, 0) maxcol = max_col(node) if mincol == maxcol: maxcol = find_next_break(line, mincol=mincol, lexer=self.parser.lexer) elif nlogical > 1: maxcol = None elif maxcol < len(line) and line[maxcol] == ";": pass else: maxcol += 1 spline = subproc_toks( line, mincol=mincol, maxcol=maxcol, returnline=False, lexer=self.parser.lexer, ) if spline is None or spline != "![{}]".format(line[mincol:maxcol].strip()): # failed to get something consistent, try greedy wrap spline = subproc_toks( line, mincol=mincol, maxcol=maxcol, returnline=False, lexer=self.parser.lexer, greedy=True, ) if spline is None: return node try: newnode = self.parser.parse( spline, mode=self.mode, filename=self.filename, debug_level=(self.debug_level > 2), ) newnode = newnode.body if not isinstance(newnode, AST): # take the first (and only) Expr newnode = newnode[0] increment_lineno(newnode, n=node.lineno - 1) newnode.col_offset = node.col_offset if self.debug_level > 1: msg = "{0}:{1}:{2}{3} - {4}\n" "{0}:{1}:{2}{3} + {5}" mstr = "" if maxcol is None else ":" + str(maxcol) msg = msg.format(self.filename, node.lineno, mincol, mstr, line, spline) print(msg, file=sys.stderr) except SyntaxError: newnode = node if strip_expr and isinstance(newnode, Expr): newnode = newnode.value return newnode
def try_subproc_toks(self, node, strip_expr=False): """Tries to parse the line of the node as a subprocess.""" line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1) if self.mode == 'eval': mincol = len(line) - len(line.lstrip()) maxcol = None else: mincol = max(min_col(node) - 1, 0) maxcol = max_col(node) if mincol == maxcol: maxcol = find_next_break(line, mincol=mincol, lexer=self.parser.lexer) elif nlogical > 1: maxcol = None elif maxcol < len(line) and line[maxcol] == ';': pass else: maxcol += 1 spline = subproc_toks(line, mincol=mincol, maxcol=maxcol, returnline=False, lexer=self.parser.lexer) if spline is None or len(spline) < len(line[mincol:maxcol]) + 2: # failed to get something consistent, try greedy wrap # The +2 comes from "![]" being length 3, minus 1 since maxcol # is one beyond the total length for slicing spline = subproc_toks(line, mincol=mincol, maxcol=maxcol, returnline=False, lexer=self.parser.lexer, greedy=True) if spline is None: return node try: newnode = self.parser.parse(spline, mode=self.mode, filename=self.filename, debug_level=(self.debug_level > 2)) newnode = newnode.body if not isinstance(newnode, AST): # take the first (and only) Expr newnode = newnode[0] increment_lineno(newnode, n=node.lineno - 1) newnode.col_offset = node.col_offset if self.debug_level > 1: msg = ('{0}:{1}:{2}{3} - {4}\n' '{0}:{1}:{2}{3} + {5}') mstr = '' if maxcol is None else ':' + str(maxcol) msg = msg.format(self.filename, node.lineno, mincol, mstr, line, spline) print(msg, file=sys.stderr) except SyntaxError: newnode = node if strip_expr and isinstance(newnode, Expr): newnode = newnode.value return newnode
def test_subproc_hello_mom_second(): fst = "echo 'hello'" sec = "echo 'mom'" s = '{0}; {1}'.format(fst, sec) exp = '{0}; $[{1}]'.format(fst, sec) obs = subproc_toks(s, lexer=LEXER, mincol=len(fst), returnline=True) assert_equal(exp, obs)
def test_subproc_toks_indent_ls_str(): ind = ' ' s = 'ls "wakka"' com = ' # lets list' exp = '{0}$[{1}]{2}'.format(ind, s, com) obs = subproc_toks(ind + s + com, lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_ls_l_semi_ls_second(): lsdl = 'ls -l' ls = 'ls' s = '{0}; {1}'.format(lsdl, ls) exp = '{0}; $[{1}]'.format(lsdl, ls) obs = subproc_toks(s, lexer=LEXER, mincol=7, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_ls_l_semi_ls_first(): lsdl = 'ls -l' ls = 'ls' s = '{0}; {1}'.format(lsdl, ls) exp = '$[{0}]; {1}'.format(lsdl, ls) obs = subproc_toks(s, lexer=LEXER, maxcol=6, returnline=True) assert_equal(exp, obs)
def test_subproc_hello_mom_first(): fst = "echo 'hello'" sec = "echo 'mom'" s = '{0}; {1}'.format(fst, sec) exp = '$[{0}]; {1}'.format(fst, sec) obs = subproc_toks(s, lexer=LEXER, maxcol=len(fst)+1, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_indent_ls_str(): ind = " " s = 'ls "wakka"' com = " # lets list" exp = "{0}![{1}]{2}".format(ind, s, com) obs = subproc_toks(ind + s + com, lexer=LEXER, returnline=True) assert exp == obs
def test_subproc_toks_indent_ls_nl(): s = "ls -l" exp = INDENT + "![{0}]\n".format(s) obs = subproc_toks( INDENT + s + "\n", mincol=len(INDENT), lexer=LEXER, returnline=True ) assert exp == obs
def test_subproc_toks_ls_l_semi_ls_second(): lsdl = "ls -l" ls = "ls" s = "{0}; {1}".format(lsdl, ls) exp = "{0}; ![{1}]".format(lsdl, ls) obs = subproc_toks(s, lexer=LEXER, mincol=7, returnline=True) assert exp == obs
def test_subproc_toks_ls_l_semi_ls_first(): lsdl = "ls -l" ls = "ls" s = "{0}; {1}".format(lsdl, ls) exp = "![{0}]; {1}".format(lsdl, ls) obs = subproc_toks(s, lexer=LEXER, maxcol=6, returnline=True) assert exp == obs
def test_subproc_toks_hello_mom_first(): fst = "echo 'hello'" sec = "echo 'mom'" s = "{0}; {1}".format(fst, sec) exp = "![{0}]; {1}".format(fst, sec) obs = subproc_toks(s, lexer=LEXER, maxcol=len(fst) + 1, returnline=True) assert exp == obs
def test_subproc_toks_hello_mom_second(): fst = "echo 'hello'" sec = "echo 'mom'" s = "{0}; {1}".format(fst, sec) exp = "{0}; ![{1}]".format(fst, sec) obs = subproc_toks(s, lexer=LEXER, mincol=len(fst), returnline=True) assert exp == obs
def try_subproc_toks(self, node, strip_expr=False): """Tries to parse the line of the node as a subprocess.""" line = self.lines[node.lineno - 1] if self.mode == 'eval': mincol = len(line) - len(line.lstrip()) maxcol = None else: mincol = min_col(node) maxcol = max_col(node) if mincol == maxcol: maxcol = find_next_break(line, mincol=mincol, lexer=self.parser.lexer) else: maxcol += 1 spline = subproc_toks(line, mincol=mincol, maxcol=maxcol, returnline=False, lexer=self.parser.lexer) if spline is None: return node try: newnode = self.parser.parse(spline, mode=self.mode) newnode = newnode.body if not isinstance(newnode, AST): # take the first (and only) Expr newnode = newnode[0] increment_lineno(newnode, n=node.lineno - 1) newnode.col_offset = node.col_offset except SyntaxError: newnode = node if strip_expr and isinstance(newnode, Expr): newnode = newnode.value return newnode
def test_subproc_toks_indent_ls_comment(): ind = ' ' s = 'ls -l' com = ' # lets list' exp = '{0}![{1}]{2}'.format(ind, s, com) obs = subproc_toks(ind + s + com, lexer=LEXER, returnline=True) assert (exp == obs)
def try_subproc_toks(self, node): """Tries to parse the line of the node as a subprocess.""" line = self.lines[node.lineno - 1] if self.mode == 'eval': mincol = len(line) - len(line.lstrip()) maxcol = None else: mincol = min_col(node) maxcol = max_col(node) + 1 spline = subproc_toks(line, mincol=mincol, maxcol=maxcol, returnline=False, lexer=self.parser.lexer) try: newnode = self.parser.parse(spline, mode=self.mode) newnode = newnode.body if not isinstance(newnode, AST): # take the first (and only) Expr newnode = newnode[0] newnode.lineno = node.lineno newnode.col_offset = node.col_offset except SyntaxError: newnode = node return newnode
def _parse_ctx_free(self, input, mode='exec'): last_error_line = last_error_col = -1 parsed = False original_error = None while not parsed: try: tree = self.parser.parse(input, filename=self.filename, mode=mode, debug_level=self.debug_level) parsed = True except IndentationError as e: if original_error is None: raise e else: raise original_error except SyntaxError as e: if original_error is None: original_error = e if (e.loc is None) or (last_error_line == e.loc.lineno and last_error_col in (e.loc.column + 1, e.loc.column)): raise original_error last_error_col = e.loc.column last_error_line = e.loc.lineno idx = last_error_line - 1 lines = input.splitlines() line = lines[idx] if input.endswith('\n'): lines.append('') if len(line.strip()) == 0: # whitespace only lines are not valid syntax in Python's # interactive mode='single', who knew?! Just ignore them. # this might cause actual sytax errors to have bad line # numbers reported, but should only effect interactive mode del lines[idx] last_error_line = last_error_col = -1 input = '\n'.join(lines) continue maxcol = self._find_next_break(line, last_error_col) sbpline = subproc_toks(line, returnline=True, maxcol=maxcol, lexer=self.parser.lexer) if sbpline is None: # subprocess line had no valid tokens, likely because # it only contained a comment. del lines[idx] last_error_line = last_error_col = -1 input = '\n'.join(lines) continue else: lines[idx] = sbpline last_error_col += 3 input = '\n'.join(lines) return tree
def test_subproc_toks_twopyeval(): s = "echo @(1+1) @(40 + 2)" exp = "![{0}]".format(s) obs = subproc_toks(s, lexer=LEXER, returnline=True) assert exp == obs
def test_subproc_toks_twopyeval_parens(): s = 'echo @(1+1) @(40+2)' inp = '({0})'.format(s) exp = '(![{0}])'.format(s) obs = subproc_toks(inp, lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_x(): exp = '$[x]' obs = subproc_toks('x', lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_indent_ls_no_min_semi_nl(): s = 'ls' exp = INDENT + '$[{0}];\n'.format(s) obs = subproc_toks(INDENT + s + ';\n', lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_ls_l(): exp = "![ls -l]" obs = subproc_toks("ls -l", lexer=LEXER, returnline=True) assert exp == obs
def test_subproc_toks_greedy_parens_statements_with_grep(): s = '(echo "abc"; sleep 1; echo "def") | grep' exp = "![{0}]".format(s) obs = subproc_toks(s, lexer=LEXER, returnline=True, greedy=True) assert exp == obs
def test_subproc_toks_pyeval_nested(): s = 'echo @(min(1, 42))' exp = '![{0}]'.format(s) obs = subproc_toks(s, lexer=LEXER, returnline=True) assert_equal(exp, obs)
def try_subproc_toks(self, node, strip_expr=False): """Tries to parse the line of the node as a subprocess.""" line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1) if self.mode == "eval": mincol = len(line) - len(line.lstrip()) maxcol = None else: mincol = max(min_col(node) - 1, 0) maxcol = max_col(node) if mincol == maxcol: maxcol = find_next_break(line, mincol=mincol, lexer=self.parser.lexer) elif nlogical > 1: maxcol = None elif maxcol < len(line) and line[maxcol] == ";": pass else: maxcol += 1 spline = subproc_toks( line, mincol=mincol, maxcol=maxcol, returnline=False, lexer=self.parser.lexer, ) if spline is None or spline != f"![{line[mincol:maxcol].strip()}]": # failed to get something consistent, try greedy wrap spline = subproc_toks( line, mincol=mincol, maxcol=maxcol, returnline=False, lexer=self.parser.lexer, greedy=True, ) if spline is None: return node try: newnode = self.parser.parse( spline, mode=self.mode, filename=self.filename, debug_level=(self.debug_level >= 2), ) newnode = newnode.body if not isinstance(newnode, AST): # take the first (and only) Expr newnode = newnode[0] increment_lineno(newnode, n=node.lineno - 1) newnode.col_offset = node.col_offset if self.debug_level >= 1: msg = "{0}:{1}:{2}{3} - {4}\n" "{0}:{1}:{2}{3} + {5}" mstr = "" if maxcol is None else ":" + str(maxcol) msg = msg.format(self.filename, node.lineno, mincol, mstr, line, spline) print(msg, file=sys.stderr) except SyntaxError: newnode = node if strip_expr and isinstance(newnode, Expr): newnode = newnode.value return newnode
def test_subproc_toks_pyeval_redirect(): s = 'echo @("foo") > bar' inp = '{0}'.format(s) exp = '![{0}]'.format(s) obs = subproc_toks(inp, lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_indent_ls_no_min(): s = 'ls -l' exp = INDENT + '$[{0}]'.format(s) obs = subproc_toks(INDENT + s, lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_git_nl(): s = 'git commit -am "hello doc"' exp = '$[{0}]\n'.format(s) obs = subproc_toks(s + '\n', lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_ls_l(): exp = '$[ls -l]' obs = subproc_toks('ls -l', lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_capproc(): s = 'echo !(echo bat)' exp = '![{0}]'.format(s) obs = subproc_toks(s, lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_pyeval_nested_parens(): s = 'echo @(min(1, 42))' inp = '({0})'.format(s) exp = '(![{0}])'.format(s) obs = subproc_toks(inp, lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_greedy_parens_inp(): s = "(sort) < input.txt" exp = "![{0}]".format(s) obs = subproc_toks(s, lexer=LEXER, returnline=True, greedy=True) assert exp == obs
def _parse_ctx_free(self, input, mode='exec'): last_error_line = last_error_col = -1 parsed = False original_error = None while not parsed: try: tree = self.parser.parse(input, filename=self.filename, mode=mode, debug_level=(self.debug_level > 1)) parsed = True except IndentationError as e: if original_error is None: raise e else: raise original_error except SyntaxError as e: if original_error is None: original_error = e if (e.loc is None) or (last_error_line == e.loc.lineno and last_error_col in (e.loc.column + 1, e.loc.column)): raise original_error last_error_col = e.loc.column last_error_line = e.loc.lineno idx = last_error_line - 1 lines = input.splitlines() line = lines[idx] if input.endswith('\n'): lines.append('') if len(line.strip()) == 0: # whitespace only lines are not valid syntax in Python's # interactive mode='single', who knew?! Just ignore them. # this might cause actual sytax errors to have bad line # numbers reported, but should only effect interactive mode del lines[idx] last_error_line = last_error_col = -1 input = '\n'.join(lines) continue if last_error_line > 1 and lines[idx-1].rstrip()[-1:] == ':': # catch non-indented blocks and raise error. prev_indent = len(lines[idx-1]) - len(lines[idx-1].lstrip()) curr_indent = len(lines[idx]) - len(lines[idx].lstrip()) if prev_indent == curr_indent: raise original_error lexer = self.parser.lexer maxcol = find_next_break(line, mincol=last_error_col, lexer=lexer) sbpline = subproc_toks(line, returnline=True, maxcol=maxcol, lexer=lexer) if sbpline is None: # subprocess line had no valid tokens, if len(line.partition('#')[0].strip()) == 0: # likely because it only contained a comment. del lines[idx] last_error_line = last_error_col = -1 input = '\n'.join(lines) continue else: # or for some other syntax error raise original_error elif sbpline[last_error_col:].startswith('![![') or \ sbpline.lstrip().startswith('![!['): # if we have already wrapped this in subproc tokens # and it still doesn't work, adding more won't help # anything raise original_error else: if self.debug_level: msg = ('{0}:{1}:{2}{3} - {4}\n' '{0}:{1}:{2}{3} + {5}') mstr = '' if maxcol is None else ':' + str(maxcol) msg = msg.format(self.filename, last_error_line, last_error_col, mstr, line, sbpline) print(msg, file=sys.stderr) lines[idx] = sbpline last_error_col += 3 input = '\n'.join(lines) return tree, input
def test_subproc_toks_and_or(phrase): s = "echo " + phrase exp = "![{0}]".format(s) obs = subproc_toks(s, lexer=LEXER, returnline=True) assert exp == obs
def test_subproc_toks_x(): exp = "![x]" obs = subproc_toks("x", lexer=LEXER, returnline=True) assert exp == obs
def test_subproc_toks_indent_ls_nl(): s = 'ls -l' exp = INDENT + '$[{0}]\n'.format(s) obs = subproc_toks(INDENT + s + '\n', mincol=len(INDENT), lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_indent_ls_no_min_semi_nl(): s = 'ls' exp = INDENT + '![{0}];\n'.format(s) obs = subproc_toks(INDENT + s + ';\n', lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_twopyeval_parens(): s = "echo @(1+1) @(40+2)" inp = "({0})".format(s) exp = "(![{0}])".format(s) obs = subproc_toks(inp, lexer=LEXER, returnline=True) assert exp == obs
def test_subproc_toks_ls_str_comment(): s = 'ls "wakka"' com = ' # lets list' exp = '![{0}]{1}'.format(s, com) obs = subproc_toks(s + com, lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_capproc(): s = "echo !(echo bat)" exp = "![{0}]".format(s) obs = subproc_toks(s, lexer=LEXER, returnline=True) assert exp == obs
def test_subproc_toks_pyeval_redirect(): s = 'echo @("foo") > bar' inp = "{0}".format(s) exp = "![{0}]".format(s) obs = subproc_toks(inp, lexer=LEXER, returnline=True) assert exp == obs
def test_subproc_toks_comment(): exp = None obs = subproc_toks('# I am a comment', lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_pyeval_nested_parens(): s = "echo @(min(1, 42))" inp = "({0})".format(s) exp = "(![{0}])".format(s) obs = subproc_toks(inp, lexer=LEXER, returnline=True) assert exp == obs
def test_subproc_toks_not(): exp = 'not ![echo mom]' obs = subproc_toks('not echo mom', lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_pyeval_nested(): s = "echo @(min(1, 42))" exp = "![{0}]".format(s) obs = subproc_toks(s, lexer=LEXER, returnline=True) assert exp == obs
def test_subproc_toks_paren_ws(): exp = '(![echo mom]) ' obs = subproc_toks('(echo mom) ', lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_and_paren(): exp = 'True and (![echo mom])' obs = subproc_toks('True and (echo mom)', lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_semicolon_only(): exp = None obs = subproc_toks(';', lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_ls_str_comment(): s = 'ls "wakka"' com = ' # lets list' exp = '$[{0}]{1}'.format(s, com) obs = subproc_toks(s + com, lexer=LEXER, returnline=True) assert_equal(exp, obs)
def test_subproc_toks_twopyeval(): s = 'echo @(1+1) @(40 + 2)' exp = '![{0}]'.format(s) obs = subproc_toks(s, lexer=LEXER, returnline=True) assert_equal(exp, obs)