def test_multi_bind_first(self):
self.assertEqual(
base_matchers.AnyOf(base_matchers.Bind('foo'),
base_matchers.Bind('bar'),
_NOTHING).match(_FAKE_CONTEXT, 1),
matcher.MatchInfo(
match.ObjectMatch(1),
{'foo': matcher.BoundValue(match.ObjectMatch(1))}))
def test_multi_overlap(self):
# TODO: it'd be nice to give a good error at some point, instead.
self.assertEqual(
base_matchers.AllOf(base_matchers.Bind('foo'),
base_matchers.Bind('foo')).match(
_FAKE_CONTEXT, 1),
matcher.MatchInfo(
match.ObjectMatch(1),
{'foo': matcher.BoundValue(match.ObjectMatch(1))}))
def test_type_filter_ordered(self):
"""Tests that type optimizations don't mess with matcher order."""
m = base_matchers.AnyOf(
base_matchers.Bind('a', base_matchers.Anything()),
base_matchers.Bind('b', base_matchers.TypeIs(int)),
)
self.assertEqual(
m.match(_FAKE_CONTEXT, 4).bindings.keys(),
{'a'},
)
def test_multi_bind(self):
self.assertEqual(
base_matchers.AllOf(base_matchers.Bind('foo'),
base_matchers.Bind('bar')).match(
_FAKE_CONTEXT, 1),
matcher.MatchInfo(
match.ObjectMatch(1), {
'foo': matcher.BoundValue(match.ObjectMatch(1)),
'bar': matcher.BoundValue(match.ObjectMatch(1)),
}))
def test_recursive_bindings(self):
"""Recursive matchers cover both recursive/base cases in .bind_variables.
If this test fails with a RecursionError, that is a problem.
"""
m = base_matchers.RecursivelyWrapped(
base_matchers.Bind('base_case', ast_matchers.Num()),
lambda i: base_matchers.Bind(
'recursive_case',
ast_matchers.UnaryOp(op=ast_matchers.Invert(), operand=i)))
self.assertEqual(m.bind_variables, {'base_case', 'recursive_case'})
def test_contains_binds(self):
items = [1, 2, 3]
m = base_matchers.Contains(base_matchers.Bind('foo', 1))
expected = matcher.MatchInfo(
match.ObjectMatch(items),
{'foo': matcher.BoundValue(match.ObjectMatch(1))})
self.assertEqual(m.match(_FAKE_CONTEXT, items), expected)
def test_bind_2arg(self):
self.assertEqual(
base_matchers.Bind('foo', base_matchers.Anything()).match(
_FAKE_CONTEXT, 1),
matcher.MatchInfo(
match.ObjectMatch(1),
{'foo': matcher.BoundValue(match.ObjectMatch(1))}))
def test_string(self):
self.assertEqual(
base_matchers.HasItem(1, base_matchers.Bind('a')).match(
_FAKE_CONTEXT, 'xy'),
matcher.MatchInfo(
match.StringMatch('xy'),
{'a': matcher.BoundValue(match.StringMatch('y'))}))
def test_noconflict(self):
for on_conflict in matcher.BindConflict:
with self.subTest(on_conflict=on_conflict):
bind = base_matchers.Bind('x', on_conflict=on_conflict)
result = bind.match(_FAKE_CONTEXT, 42)
self.assertIsNotNone(result)
self.assertEqual(result.bindings['x'].value.matched, 42)
def test_match(self):
container = [1]
self.assertEqual(
base_matchers.ItemsAre([base_matchers.Bind('a')
]).match(_FAKE_CONTEXT, container),
matcher.MatchInfo(match.ObjectMatch(container),
{'a': matcher.BoundValue(match.ObjectMatch(1))}))
def test_nonexpr_in_expr_context(self, template):
parsed = matcher.parse_ast('[x]')
m = base_matchers.Bind('bound', ast_matchers.Name())
[matchinfo] = matcher.find_iter(m, parsed)
with self.assertRaises(formatting.RewriteError):
template.substitute_match(parsed, matchinfo.match,
{'bound': matchinfo.match})
def test_bindings(self):
m = base_matchers.Once(base_matchers.Bind('foo'))
self.assertEqual(
m.match(_FAKE_CONTEXT.new(), 1),
matcher.MatchInfo(
match.ObjectMatch(1),
{'foo': matcher.BoundValue(match.ObjectMatch(1))}))
self.assertEqual(m.bind_variables, {'foo'})
def test_invalid_syntax(self):
parsed = matcher.parse_ast('x')
m = base_matchers.Bind('x', ast_matchers.Name())
[matchinfo] = matcher.find_iter(m, parsed)
template = syntactic_template.PythonTemplate('$x')
with self.assertRaises(formatting.RewriteError):
template.substitute_match(parsed, matchinfo.match,
{'x': match.StringMatch('x y')}),
def test_nonpython_dollars_source(self, src):
parsed = matcher.parse_ast(src)
m = base_matchers.Bind('bound', ast_matchers.Call())
[matchinfo] = matcher.find_iter(m, parsed)
self.assertEqual(
src,
syntactic_template.PythonExprTemplate('$bound').substitute_match(
parsed, matchinfo.match, {'bound': matchinfo.match}))
def test_negative_index(self):
container = ['xyz']
self.assertEqual(
base_matchers.HasItem(-1, base_matchers.Bind('a')).match(
_FAKE_CONTEXT, container),
matcher.MatchInfo(
match.ObjectMatch(container),
{'a': matcher.BoundValue(match.StringMatch('xyz'))}))
def test_simple(self):
for nonempty_container in (('x', 'y'), ['x', 'y'], {1: 'y'}):
with self.subTest(nonempty_container=nonempty_container):
self.assertEqual(
base_matchers.HasItem(1, base_matchers.Bind('a')).match(
_FAKE_CONTEXT, nonempty_container),
matcher.MatchInfo(
match.ObjectMatch(nonempty_container),
{'a': matcher.BoundValue(match.StringMatch('y'))}))
def test_autoparen_outer(self):
parsed = matcher.parse_ast('x * 2')
m = base_matchers.Bind('x', ast_matchers.Name())
[matchinfo] = matcher.find_iter(m, parsed)
template = syntactic_template.PythonTemplate('$x')
self.assertEqual(
template.substitute_match(parsed, matchinfo.match,
{'x': match.StringMatch('x + y')}),
'(x + y)',
)
def test_nonpython_dollars_dest(self):
src = 'f'
parsed = matcher.parse_ast(src)
m = base_matchers.Bind('bound', ast_matchers.Name())
[matchinfo] = matcher.find_iter(m, parsed)
self.assertEqual(
'f("$x")',
syntactic_template.PythonExprTemplate(
'$bound("$x")').substitute_match(parsed, matchinfo.match,
{'bound': matchinfo.match}))
def _in_exception_handler(identifier, on_conflict):
"""Returns a matcher for a node in the nearest ancestor `except` & binds `identifier`.
Args:
identifier: Name of variable to bind the identifier in the nearest ancestor
exception handler to
on_conflict: BindConflict strategy for binding the identifier
"""
return syntax_matchers.HasFirstAncestor(
ast_matchers.ExceptHandler(),
ast_matchers.ExceptHandler(name=base_matchers.AnyOf(
# In PY2, the name is a `Name` but in PY3 just a
# string.
# So rather than capturing and merging the Name
# nodes, we capture and merge the actual string
# identifier.
ast_matchers.Name(
id=base_matchers.Bind(identifier, on_conflict=on_conflict)),
base_matchers.Bind(identifier, on_conflict=on_conflict),
)))
def test_assignment(self, template):
template = syntactic_template.PythonStmtTemplate(template)
# Test with different values of `ctx` for the variable being substituted.
for variable_source in 'a = 1', 'a':
with self.subTest(variable_souce=variable_source):
[matchinfo] = matcher.find_iter(
base_matchers.Bind('x', ast_matchers.Name()),
matcher.parse_ast(variable_source))
substituted = template.substitute_match(
None, None, {'x': matchinfo.match})
self.assertEqual(substituted,
template.template.replace('$x', 'a'))
def test_variable_conflict(self):
"""Variables use the default conflict resolution outside of the pattern.
Inside of the pattern, they use MERGE_EQUIVALENT_AST, but this is opaque to
callers.
"""
# The AllOf shouldn't make a difference, because the $x variable is just
# a regular Bind() variable outside of the pattern, and merges via KEEP_LAST
# per normal.
self.assertEqual(
self.get_all_match_strings(
base_matchers.AllOf(syntax_matchers.StmtPattern('$x'),
base_matchers.Bind('x')), '1'), ['1'])
def _get_matcher(self):
"""Override of get_matcher to pull things from a function object."""
# `inspect.getsource` doesn't, say, introspect the code object for its
# source. Python, despite its dyanamism, doesn't support that much magic.
# Instead, it gets the file and line number information from the code
# object, and returns those lines as the source. This leads to a few
# interesting consequences:
# - Functions that exist within a class or closure are by default
# `IndentationError`, the code block must be textwrap-dedented before
# being used.
# - This won't work in interactive modes (shell, ipython, etc.)
# - Functions are normally statements, so treating everything from the
# first line to the last as part of the function is probably fine. There
# are a few cases where this will break, namely
# - A lambda will likely be a syntax error, the tool will see
# `lambda x: x)`, where `)` is the closing paren of the enclosing
# scope.
source = textwrap.dedent(inspect.getsource(self.func))
args = _args(self.func)
try:
parsed = ast.parse(source)
except SyntaxError:
raise ValueError(
'Function {} appears to have invalid syntax. Is it a'
' lambda?'.format(self.func.__name__))
actual_body = parsed.body[0].body
if (isinstance(actual_body[0], ast.Expr)
and isinstance(actual_body[0].value, ast.Str)):
# Strip the docstring, if it exists.
actual_body = actual_body[1:]
if not actual_body:
raise ValueError('Format function must include an actual body, a '
'docstring alone is invalid.')
if isinstance(actual_body[0], ast.Pass):
raise ValueError(
'If you *really* want to rewrite a function whose body '
'is just `pass`, use a regex replacer.')
# Since we don't need to mangle names, we just generate bindings.
bindings = {}
for name in args:
bindings[name] = base_matchers.Bind(
name,
base_matchers.Anything(),
on_conflict=matcher.BindConflict.MERGE_EQUIVALENT_AST)
return base_matchers.Rebind(
_ast_pattern(actual_body[0], bindings),
on_conflict=matcher.BindConflict.MERGE,
on_merge=matcher.BindMerge.KEEP_LAST,
)
def test_nonsingular_py_ok(self, template):
"""Tests non-singular PythonTemplate in a context where it's acceptable.
If it is not being placed into a context where it's expected to parse as
an expression, then '' and even 'a; b' are fine.
Args:
template: the template for this test.
"""
parsed = matcher.parse_ast('x')
m = base_matchers.Bind('bound', ast_matchers.Name())
[matchinfo] = matcher.find_iter(m, parsed)
self.assertEqual(
template,
syntactic_template.PythonTemplate(template).substitute_match(
parsed, matchinfo.match, {'bound': matchinfo.match}))
def test_matches(self):
parsed = matcher.parse_ast('xy = 2', '<string>')
matches = list(
matcher.find_iter(
base_matchers.MatchesRegex(r'^(?P<name>.)(.)$',
base_matchers.Bind('inner')),
parsed))
# There is only one AST node of length >= 2 (which the regex requires): xy.
self.assertEqual(matches, [
matcher.MatchInfo(
mock.ANY, {
'inner': mock.ANY,
'name': matcher.BoundValue(match.SpanMatch('x', (0, 1))),
})
])
[matchinfo] = matches
self.assertEqual(matchinfo.match.span, (0, 2))
self.assertEqual(matchinfo.match, matchinfo.bindings['inner'].value)
def _match(self, context, candidate):
if not isinstance(candidate, ast.AST):
return None
# Walk the AST to collect the answer:
values = []
for node in ast.walk(candidate):
# Every node must either be a Constant/Num or an addition node.
if isinstance(node, ast.Constant):
values.append(node.value)
elif isinstance(node, ast.Num): # older pythons
values.append(node.n)
elif isinstance(node, ast.BinOp) or isinstance(node, ast.Add):
# Binary operator nodes are allowed, but only if they have an Add() op.
pass
else:
return None # not a +, not a constant
# For more complex tasks, or for tasks which integrate into how Refex
# builds results and bindings, it can be helpful to defer work into a
# submatcher, such as by running BinOp(op=Add()).match(context, candidate)
# Having walked the AST, we have determined that the whole tree is addition
# of constants, and have collected all of those constants in a list.
if len(values) <= 1:
# Don't bother emitting a replacement for e.g. 7 with itself.
return None
result = str(sum(values))
# Finally, we want to return the answer to Refex:
# 1) bind the result to a variable
# 2) return the tree itself as the matched value
# We can do this by deferring to a matcher that does the right thing.
# StringMatch() will produce a string literal match, and AllOf will retarget
# the returned binding to the AST node which was passed in.
submatcher = base_matchers.AllOf(
base_matchers.Bind("sum", base_matchers.StringMatch(result)))
return submatcher.match(context, candidate)
def _rewrite_submatchers(pattern, restrictions):
"""Rewrites pattern/restrictions to erase metasyntactic variables.
Args:
pattern: a pattern containing $variables.
restrictions: a dictionary of variables to submatchers. If a variable is
missing, Anything() is used instead.
Returns:
(remapped_pattern, variables, new_submatchers)
* remapped_pattern has all variables replaced with new unique names that are
valid Python syntax.
* variables is the mapping of the original name to the remapped name.
* new_submatchers is a dict from remapped names to submatchers. Every
variable is put in a Bind() node, which has a submatcher taken from
`restrictions`.
Raises:
KeyError: if restrictions has a key that isn't a variable name.
"""
pattern, variables = _remap_macro_variables(pattern)
incorrect_variables = set(restrictions) - set(variables)
if incorrect_variables:
raise KeyError(
'Some variables specified in restrictions were missing. '
'Did you misplace a "$"? Missing variables: %r' %
incorrect_variables)
submatchers = {}
for old_name, new_name in variables.items():
submatchers[new_name] = base_matchers.Bind(
old_name,
restrictions.get(old_name, base_matchers.Anything()),
on_conflict=matcher.BindConflict.MERGE_EQUIVALENT_AST,
)
return pattern, variables, submatchers
def test_matcherror(self):
parsed = matcher.parse_ast('x+y')
bind = base_matchers.Bind('var',
on_conflict=matcher.BindConflict.ERROR)
m = ast_matchers.BinOp(left=bind, right=bind)
self.assertEqual(list(matcher.find_iter(m, parsed)), [])
base_matchers.Bind(identifier, on_conflict=on_conflict),
)))
_LOGGING_FIXERS = (
fixer.SimplePythonFixer(
message=
'Use logging.exception inside an except handler to automatically log the full stack trace of the error',
url=
'https://refex.readthedocs.io/en/latest/guide/fixers/logging_exceptions.html',
significant=True,
category=_LOGGING_EXCEPTION_CATEGORY,
matcher=base_matchers.AllOf(
ast_matchers.Call(
func=base_matchers.Bind(
'logging_error',
syntax_matchers.ExprPattern('logging.error')),
args=base_matchers.Contains(
base_matchers.AllOf(
_in_exception_handler(
'e',
on_conflict=matcher_.BindConflict.MERGE_IDENTICAL),
ast_matchers.Name(id=base_matchers.Bind(
'e',
on_conflict=matcher_.BindConflict.MERGE_IDENTICAL))
)),
keywords=base_matchers.Unless(
base_matchers.Contains(
ast_matchers.keyword(arg='exc_info'))),
), ),
replacement=dict(logging_error=syntactic_template.PythonStmtTemplate(
def test_bind_miss(self):
self.assertIsNone(
base_matchers.Bind('foo', _NOTHING).match(_FAKE_CONTEXT, 1))
def test_ne(self, m):
self.assertIsNone(base_matchers.Bind('a', m).match(_FAKE_CONTEXT, 4))
