def try_subproc_toks(self, node, strip_expr=False):
"""Tries to parse the line of the node as a subprocess."""
line = self.lines[node.lineno - 1]
if self.mode == 'eval':
mincol = len(line) - len(line.lstrip())
maxcol = None
else:
mincol = min_col(node)
maxcol = max_col(node)
if mincol == maxcol:
maxcol = find_next_break(line,
mincol=mincol,
lexer=self.parser.lexer)
else:
maxcol += 1
spline = subproc_toks(line,
mincol=mincol,
maxcol=maxcol,
returnline=False,
lexer=self.parser.lexer)
if spline is None:
return node
try:
newnode = self.parser.parse(spline, mode=self.mode)
newnode = newnode.body
if not isinstance(newnode, AST):
# take the first (and only) Expr
newnode = newnode[0]
newnode.lineno = node.lineno
newnode.col_offset = node.col_offset
except SyntaxError:
newnode = node
if strip_expr and isinstance(newnode, Expr):
newnode = newnode.value
return newnode
def try_subproc_toks(self, node, strip_expr=False):
"""Tries to parse the line of the node as a subprocess."""
line = self.lines[node.lineno - 1]
if self.mode == 'eval':
mincol = len(line) - len(line.lstrip())
maxcol = None
else:
mincol = min_col(node)
maxcol = max_col(node)
if mincol == maxcol:
maxcol = find_next_break(line, mincol=mincol,
lexer=self.parser.lexer)
else:
maxcol += 1
spline = subproc_toks(line,
mincol=mincol,
maxcol=maxcol,
returnline=False,
lexer=self.parser.lexer)
if spline is None:
return node
try:
newnode = self.parser.parse(spline, mode=self.mode)
newnode = newnode.body
if not isinstance(newnode, AST):
# take the first (and only) Expr
newnode = newnode[0]
increment_lineno(newnode, n=node.lineno - 1)
newnode.col_offset = node.col_offset
except SyntaxError:
newnode = node
if strip_expr and isinstance(newnode, Expr):
newnode = newnode.value
return newnode
def try_subproc_toks(self, node, strip_expr=False):
"""Tries to parse the line of the node as a subprocess."""
line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1)
if self.mode == "eval":
mincol = len(line) - len(line.lstrip())
maxcol = None
else:
mincol = max(min_col(node) - 1, 0)
maxcol = max_col(node)
if mincol == maxcol:
maxcol = find_next_break(line, mincol=mincol, lexer=self.parser.lexer)
elif nlogical > 1:
maxcol = None
elif maxcol < len(line) and line[maxcol] == ";":
pass
else:
maxcol += 1
spline = subproc_toks(
line,
mincol=mincol,
maxcol=maxcol,
returnline=False,
lexer=self.parser.lexer,
)
if spline is None or spline != "![{}]".format(line[mincol:maxcol].strip()):
# failed to get something consistent, try greedy wrap
spline = subproc_toks(
line,
mincol=mincol,
maxcol=maxcol,
returnline=False,
lexer=self.parser.lexer,
greedy=True,
)
if spline is None:
return node
try:
newnode = self.parser.parse(
spline,
mode=self.mode,
filename=self.filename,
debug_level=(self.debug_level > 2),
)
newnode = newnode.body
if not isinstance(newnode, AST):
# take the first (and only) Expr
newnode = newnode[0]
increment_lineno(newnode, n=node.lineno - 1)
newnode.col_offset = node.col_offset
if self.debug_level > 1:
msg = "{0}:{1}:{2}{3} - {4}\n" "{0}:{1}:{2}{3} + {5}"
mstr = "" if maxcol is None else ":" + str(maxcol)
msg = msg.format(self.filename, node.lineno, mincol, mstr, line, spline)
print(msg, file=sys.stderr)
except SyntaxError:
newnode = node
if strip_expr and isinstance(newnode, Expr):
newnode = newnode.value
return newnode
def try_subproc_toks(self, node, strip_expr=False):
"""Tries to parse the line of the node as a subprocess."""
line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1)
if self.mode == 'eval':
mincol = len(line) - len(line.lstrip())
maxcol = None
else:
mincol = max(min_col(node) - 1, 0)
maxcol = max_col(node)
if mincol == maxcol:
maxcol = find_next_break(line,
mincol=mincol,
lexer=self.parser.lexer)
elif nlogical > 1:
maxcol = None
elif maxcol < len(line) and line[maxcol] == ';':
pass
else:
maxcol += 1
spline = subproc_toks(line,
mincol=mincol,
maxcol=maxcol,
returnline=False,
lexer=self.parser.lexer)
if spline is None or len(spline) < len(line[mincol:maxcol]) + 2:
# failed to get something consistent, try greedy wrap
# The +2 comes from "![]" being length 3, minus 1 since maxcol
# is one beyond the total length for slicing
spline = subproc_toks(line,
mincol=mincol,
maxcol=maxcol,
returnline=False,
lexer=self.parser.lexer,
greedy=True)
if spline is None:
return node
try:
newnode = self.parser.parse(spline,
mode=self.mode,
filename=self.filename,
debug_level=(self.debug_level > 2))
newnode = newnode.body
if not isinstance(newnode, AST):
# take the first (and only) Expr
newnode = newnode[0]
increment_lineno(newnode, n=node.lineno - 1)
newnode.col_offset = node.col_offset
if self.debug_level > 1:
msg = ('{0}:{1}:{2}{3} - {4}\n' '{0}:{1}:{2}{3} + {5}')
mstr = '' if maxcol is None else ':' + str(maxcol)
msg = msg.format(self.filename, node.lineno, mincol, mstr,
line, spline)
print(msg, file=sys.stderr)
except SyntaxError:
newnode = node
if strip_expr and isinstance(newnode, Expr):
newnode = newnode.value
return newnode
def test_find_next_break():
cases = [
('ls && echo a', 0, 4),
('ls && echo a', 6, None),
('ls && echo a || echo b', 6, 14),
('(ls) && echo a', 1, 4),
('not ls && echo a', 0, 8),
('not (ls) && echo a', 0, 8),
]
for line, mincol, exp in cases:
obs = find_next_break(line, mincol=mincol, lexer=LEXER)
assert exp == obs
def test_find_next_break():
cases = [
('ls && echo a', 0, 4),
('ls && echo a', 6, None),
('ls && echo a || echo b', 6, 14),
('(ls) && echo a', 1, 4),
('not ls && echo a', 0, 8),
('not (ls) && echo a', 0, 8),
]
for line, mincol, exp in cases:
obs = find_next_break(line, mincol=mincol, lexer=LEXER)
yield assert_equal, exp, obs
def try_subproc_toks(self, node, strip_expr=False):
"""Tries to parse the line of the node as a subprocess."""
line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1)
if self.mode == 'eval':
mincol = len(line) - len(line.lstrip())
maxcol = None
else:
mincol = max(min_col(node) - 1, 0)
maxcol = max_col(node)
if mincol == maxcol:
maxcol = find_next_break(line, mincol=mincol,
lexer=self.parser.lexer)
elif nlogical > 1:
maxcol = None
elif maxcol < len(line) and line[maxcol] == ';':
pass
else:
maxcol += 1
spline = subproc_toks(line, mincol=mincol, maxcol=maxcol,
returnline=False, lexer=self.parser.lexer)
if spline is None or len(spline) < len(line[mincol:maxcol]) + 2:
# failed to get something consistent, try greedy wrap
# The +2 comes from "![]" being length 3, minus 1 since maxcol
# is one beyond the total length for slicing
spline = subproc_toks(line, mincol=mincol, maxcol=maxcol,
returnline=False, lexer=self.parser.lexer,
greedy=True)
if spline is None:
return node
try:
newnode = self.parser.parse(spline, mode=self.mode,
filename=self.filename,
debug_level=(self.debug_level > 2))
newnode = newnode.body
if not isinstance(newnode, AST):
# take the first (and only) Expr
newnode = newnode[0]
increment_lineno(newnode, n=node.lineno - 1)
newnode.col_offset = node.col_offset
if self.debug_level > 1:
msg = ('{0}:{1}:{2}{3} - {4}\n'
'{0}:{1}:{2}{3} + {5}')
mstr = '' if maxcol is None else ':' + str(maxcol)
msg = msg.format(self.filename, node.lineno,
mincol, mstr, line, spline)
print(msg, file=sys.stderr)
except SyntaxError:
newnode = node
if strip_expr and isinstance(newnode, Expr):
newnode = newnode.value
return newnode
def _parse_ctx_free(self, input, mode='exec'):
last_error_line = last_error_col = -1
parsed = False
original_error = None
while not parsed:
try:
tree = self.parser.parse(input,
filename=self.filename,
mode=mode,
debug_level=(self.debug_level > 1))
parsed = True
except IndentationError as e:
if original_error is None:
raise e
else:
raise original_error
except SyntaxError as e:
if original_error is None:
original_error = e
if (e.loc is None) or (last_error_line == e.loc.lineno and
last_error_col in (e.loc.column + 1,
e.loc.column)):
raise original_error
last_error_col = e.loc.column
last_error_line = e.loc.lineno
idx = last_error_line - 1
lines = input.splitlines()
line = lines[idx]
if input.endswith('\n'):
lines.append('')
if len(line.strip()) == 0:
# whitespace only lines are not valid syntax in Python's
# interactive mode='single', who knew?! Just ignore them.
# this might cause actual sytax errors to have bad line
# numbers reported, but should only effect interactive mode
del lines[idx]
last_error_line = last_error_col = -1
input = '\n'.join(lines)
continue
if last_error_line > 1 and lines[idx-1].rstrip()[-1:] == ':':
# catch non-indented blocks and raise error.
prev_indent = len(lines[idx-1]) - len(lines[idx-1].lstrip())
curr_indent = len(lines[idx]) - len(lines[idx].lstrip())
if prev_indent == curr_indent:
raise original_error
lexer = self.parser.lexer
maxcol = find_next_break(line, mincol=last_error_col,
lexer=lexer)
sbpline = subproc_toks(line, returnline=True,
maxcol=maxcol, lexer=lexer)
if sbpline is None:
# subprocess line had no valid tokens,
if len(line.partition('#')[0].strip()) == 0:
# likely because it only contained a comment.
del lines[idx]
last_error_line = last_error_col = -1
input = '\n'.join(lines)
continue
else:
# or for some other syntax error
raise original_error
elif sbpline[last_error_col:].startswith('![![') or \
sbpline.lstrip().startswith('![!['):
# if we have already wrapped this in subproc tokens
# and it still doesn't work, adding more won't help
# anything
raise original_error
else:
if self.debug_level:
msg = ('{0}:{1}:{2}{3} - {4}\n'
'{0}:{1}:{2}{3} + {5}')
mstr = '' if maxcol is None else ':' + str(maxcol)
msg = msg.format(self.filename, last_error_line,
last_error_col, mstr, line, sbpline)
print(msg, file=sys.stderr)
lines[idx] = sbpline
last_error_col += 3
input = '\n'.join(lines)
return tree, input
def test_find_next_break(line, mincol, exp):
obs = find_next_break(line, mincol=mincol, lexer=LEXER)
assert exp == obs
def _parse_ctx_free(self, input, mode="exec", filename=None, logical_input=False):
last_error_line = last_error_col = -1
parsed = False
original_error = None
greedy = False
if filename is None:
filename = self.filename
while not parsed:
try:
tree = self.parser.parse(
input,
filename=filename,
mode=mode,
debug_level=(self.debug_level > 2),
)
parsed = True
except IndentationError as e:
if original_error is None:
raise e
else:
raise original_error
except SyntaxError as e:
if original_error is None:
original_error = e
if (e.loc is None) or (
last_error_line == e.loc.lineno
and last_error_col in (e.loc.column + 1, e.loc.column)
):
raise original_error from None
elif last_error_line != e.loc.lineno:
original_error = e
last_error_col = e.loc.column
last_error_line = e.loc.lineno
idx = last_error_line - 1
lines = input.splitlines()
line, nlogical, idx = get_logical_line(lines, idx)
if nlogical > 1 and not logical_input:
_, sbpline = self._parse_ctx_free(
line, mode=mode, filename=filename, logical_input=True
)
self._print_debug_wrapping(
line, sbpline, last_error_line, last_error_col, maxcol=None
)
replace_logical_line(lines, sbpline, idx, nlogical)
last_error_col += 3
input = "\n".join(lines)
continue
if input.endswith("\n"):
lines.append("")
if len(line.strip()) == 0:
# whitespace only lines are not valid syntax in Python's
# interactive mode='single', who knew?! Just ignore them.
# this might cause actual syntax errors to have bad line
# numbers reported, but should only affect interactive mode
del lines[idx]
last_error_line = last_error_col = -1
input = "\n".join(lines)
continue
if last_error_line > 1 and lines[idx - 1].rstrip()[-1:] == ":":
# catch non-indented blocks and raise error.
prev_indent = len(lines[idx - 1]) - len(lines[idx - 1].lstrip())
curr_indent = len(lines[idx]) - len(lines[idx].lstrip())
if prev_indent == curr_indent:
raise original_error
lexer = self.parser.lexer
maxcol = (
None
if greedy
else find_next_break(line, mincol=last_error_col, lexer=lexer)
)
if not greedy and maxcol in (e.loc.column + 1, e.loc.column):
# go greedy the first time if the syntax error was because
# we hit an end token out of place. This usually indicates
# a subshell or maybe a macro.
if not balanced_parens(line, maxcol=maxcol):
greedy = True
maxcol = None
sbpline = subproc_toks(
line, returnline=True, greedy=greedy, maxcol=maxcol, lexer=lexer
)
if sbpline is None:
# subprocess line had no valid tokens,
if len(line.partition("#")[0].strip()) == 0:
# likely because it only contained a comment.
del lines[idx]
last_error_line = last_error_col = -1
input = "\n".join(lines)
continue
elif not greedy:
greedy = True
continue
else:
# or for some other syntax error
raise original_error
elif sbpline[last_error_col:].startswith(
"![!["
) or sbpline.lstrip().startswith("![!["):
# if we have already wrapped this in subproc tokens
# and it still doesn't work, adding more won't help
# anything
if not greedy:
greedy = True
continue
else:
raise original_error
# replace the line
self._print_debug_wrapping(
line, sbpline, last_error_line, last_error_col, maxcol=maxcol
)
replace_logical_line(lines, sbpline, idx, nlogical)
last_error_col += 3
input = "\n".join(lines)
return tree, input
def _parse_ctx_free(self,
input,
mode="exec",
filename=None,
logical_input=False):
last_error_line = last_error_col = -1
parsed = False
original_error = None
greedy = False
if filename is None:
filename = self.filename
if logical_input:
beg_spaces = starting_whitespace(input)
input = input[len(beg_spaces):]
while not parsed:
try:
tree = self.parser.parse(
input,
filename=filename,
mode=mode,
debug_level=(self.debug_level > 2),
)
parsed = True
except IndentationError as e:
if original_error is None:
raise e
else:
raise original_error
except SyntaxError as e:
if original_error is None:
original_error = e
if (e.loc is None) or (last_error_line == e.loc.lineno
and last_error_col
in (e.loc.column + 1, e.loc.column)):
raise original_error from None
elif last_error_line != e.loc.lineno:
original_error = e
last_error_col = e.loc.column
last_error_line = e.loc.lineno
idx = last_error_line - 1
lines = input.splitlines()
line, nlogical, idx = get_logical_line(lines, idx)
if nlogical > 1 and not logical_input:
_, sbpline = self._parse_ctx_free(line,
mode=mode,
filename=filename,
logical_input=True)
self._print_debug_wrapping(line,
sbpline,
last_error_line,
last_error_col,
maxcol=None)
replace_logical_line(lines, sbpline, idx, nlogical)
last_error_col += 3
input = "\n".join(lines)
continue
if input.endswith("\n"):
lines.append("")
if len(line.strip()) == 0:
# whitespace only lines are not valid syntax in Python's
# interactive mode='single', who knew?! Just ignore them.
# this might cause actual syntax errors to have bad line
# numbers reported, but should only affect interactive mode
del lines[idx]
last_error_line = last_error_col = -1
input = "\n".join(lines)
continue
if last_error_line > 1 and lines[idx - 1].rstrip()[-1:] == ":":
# catch non-indented blocks and raise error.
prev_indent = len(lines[idx - 1]) - len(
lines[idx - 1].lstrip())
curr_indent = len(lines[idx]) - len(lines[idx].lstrip())
if prev_indent == curr_indent:
raise original_error
lexer = self.parser.lexer
maxcol = (None if greedy else find_next_break(
line, mincol=last_error_col, lexer=lexer))
if not greedy and maxcol in (e.loc.column + 1, e.loc.column):
# go greedy the first time if the syntax error was because
# we hit an end token out of place. This usually indicates
# a subshell or maybe a macro.
if not balanced_parens(line, maxcol=maxcol):
greedy = True
maxcol = None
sbpline = subproc_toks(line,
returnline=True,
greedy=greedy,
maxcol=maxcol,
lexer=lexer)
if sbpline is None:
# subprocess line had no valid tokens,
if len(line.partition("#")[0].strip()) == 0:
# likely because it only contained a comment.
del lines[idx]
last_error_line = last_error_col = -1
input = "\n".join(lines)
continue
elif not greedy:
greedy = True
continue
else:
# or for some other syntax error
raise original_error
elif sbpline[last_error_col:].startswith(
"![![") or sbpline.lstrip().startswith("![!["):
# if we have already wrapped this in subproc tokens
# and it still doesn't work, adding more won't help
# anything
if not greedy:
greedy = True
continue
else:
raise original_error
# replace the line
self._print_debug_wrapping(line,
sbpline,
last_error_line,
last_error_col,
maxcol=maxcol)
replace_logical_line(lines, sbpline, idx, nlogical)
last_error_col += 3
input = "\n".join(lines)
if logical_input:
input = beg_spaces + input
return tree, input
def _parse_ctx_free(self, input, mode='exec'):
last_error_line = last_error_col = -1
parsed = False
original_error = None
while not parsed:
try:
tree = self.parser.parse(input,
filename=self.filename,
mode=mode,
debug_level=(self.debug_level > 1))
parsed = True
except IndentationError as e:
if original_error is None:
raise e
else:
raise original_error
except SyntaxError as e:
if original_error is None:
original_error = e
if (e.loc is None) or (last_error_line == e.loc.lineno
and last_error_col
in (e.loc.column + 1, e.loc.column)):
raise original_error
last_error_col = e.loc.column
last_error_line = e.loc.lineno
idx = last_error_line - 1
lines = input.splitlines()
line = lines[idx]
if input.endswith('\n'):
lines.append('')
if len(line.strip()) == 0:
# whitespace only lines are not valid syntax in Python's
# interactive mode='single', who knew?! Just ignore them.
# this might cause actual sytax errors to have bad line
# numbers reported, but should only effect interactive mode
del lines[idx]
last_error_line = last_error_col = -1
input = '\n'.join(lines)
continue
if last_error_line > 1 and lines[idx - 1].rstrip()[-1:] == ':':
# catch non-indented blocks and raise error.
prev_indent = len(lines[idx - 1]) - len(
lines[idx - 1].lstrip())
curr_indent = len(lines[idx]) - len(lines[idx].lstrip())
if prev_indent == curr_indent:
raise original_error
lexer = self.parser.lexer
maxcol = find_next_break(line,
mincol=last_error_col,
lexer=lexer)
sbpline = subproc_toks(line,
returnline=True,
maxcol=maxcol,
lexer=lexer)
if sbpline is None:
# subprocess line had no valid tokens,
if len(line.partition('#')[0].strip()) == 0:
# likely because it only contained a comment.
del lines[idx]
last_error_line = last_error_col = -1
input = '\n'.join(lines)
continue
else:
# or for some other syntax error
raise original_error
elif sbpline[last_error_col:].startswith('![![') or \
sbpline.lstrip().startswith('![!['):
# if we have already wrapped this in subproc tokens
# and it still doesn't work, adding more won't help
# anything
raise original_error
else:
if self.debug_level:
msg = ('{0}:{1}:{2}{3} - {4}\n' '{0}:{1}:{2}{3} + {5}')
mstr = '' if maxcol is None else ':' + str(maxcol)
msg = msg.format(self.filename, last_error_line,
last_error_col, mstr, line, sbpline)
print(msg, file=sys.stderr)
lines[idx] = sbpline
last_error_col += 3
input = '\n'.join(lines)
return tree, input
def test_find_next_break(line, mincol, exp):
obs = find_next_break(line, mincol=mincol, lexer=LEXER)
assert exp == obs