def test_restrictions(self):
parsed = matcher.parse_ast('1\n2', '<string>')
expr_match = parsed.tree.body[0].value
expr_nomatch = parsed.tree.body[1].value
m = syntax_matchers.ExprPattern(
'$name', {'name': syntax_matchers.ExprPattern('1')})
self.assertIsNotNone(m.match(matcher.MatchContext(parsed), expr_match))
self.assertIsNone(m.match(matcher.MatchContext(parsed), expr_nomatch))
def test_nonvariable_name_fails(self):
"""Names are only treated as variables, or anything weird, on request."""
parsed = matcher.parse_ast('3', '<string>')
expr = parsed.tree.body[0].value
self.assertIsNone(
syntax_matchers.ExprPattern('name').match(
matcher.MatchContext(parsed), expr))
def test_syntax_matchers(self):
for expr in [
"syntax_matchers.ExprPattern('$bar')", "ExprPattern('$bar')"
]:
with self.subTest(expr=expr):
self.assertEqual(evaluate.compile_matcher(expr),
syntax_matchers.ExprPattern('$bar'))
def test_restrictions(self):
parsed = matcher.parse_ast('a = 1\na = 2', '<string>')
stmt_match, stmt_nomatch = parsed.tree.body
m = syntax_matchers.StmtPattern(
'a = $name', {'name': syntax_matchers.ExprPattern('1')})
self.assertIsNotNone(m.match(matcher.MatchContext(parsed), stmt_match))
self.assertIsNone(m.match(matcher.MatchContext(parsed), stmt_nomatch))
def idiom_fixer(
old_expr,
new_expr,
category,
url='https://refex.readthedocs.io/en/latest/guide/fixers/idiom.html',
):
"""Fixer for making expressions "clearer" / less convoluted.
This also helps normalize them for other fixers to apply.
Args:
old_expr: An ExprPattern string for the expr to match.
new_expr: A string.Template string for the replacement.
category: A category for the fix.
url: An URL describing the fix.
Returns:
A fixer that replaces old_expr with new_expr.
"""
dotdotdot = fixer.ImmutableDefaultDict(lambda _: '...')
return fixer.SimplePythonFixer(
message=('This could be more Pythonic: %s -> %s.' %
((string.Template(old_expr).substitute(dotdotdot),
string.Template(new_expr).substitute(dotdotdot)))),
matcher=syntax_matchers.ExprPattern(old_expr),
replacement=syntactic_template.PythonExprTemplate(new_expr),
url=url,
significant=False,
category=category,
)
def assert_alias_fixer(
old_expr,
new_expr,
url='https://docs.python.org/3/library/unittest.html#deprecated-aliases'):
"""Fixer for deprecated unittest aliases.
Args:
old_expr: A string for an ExprPattern matching the target expr.
new_expr: A string for a PythonExprTemplate to replace it with.
url: The URL documenting the deprecation.
Returns:
A fixer that replaces old_expr with new_expr.
"""
dotdotdot = fixer.ImmutableDefaultDict(lambda _: '...')
return fixer.SimplePythonFixer(
message=('{old} is a deprecated alias for {new} in the unittest module.'
.format(
old=string.Template(old_expr).substitute(dotdotdot),
new=string.Template(new_expr).substitute(dotdotdot))),
matcher=syntax_matchers.ExprPattern(old_expr),
replacement=syntactic_template.PythonExprTemplate(new_expr),
url=url,
significant=False,
category='pylint.g-deprecated-assert',
)
def assert_message_fixer(old_expr, new_expr, method, is_absl=False):
"""Fixer for assertTrue()/assertFalse()/etc.
related error fixes.
assertTrue(...) often produces less readable error information than
alternative methods like assertEqual etc.
Args:
old_expr: a ExprPattern string for the expr to match
new_expr: a template string for the replacement
method: the method to link to in the docs.
is_absl: Whether this is an absl method with absl docs.
Returns:
A fixer that replaces old_expr with new_expr.
"""
if is_absl:
# absl doesn't have docs per se.
url = f'https://github.com/abseil/abseil-py/search?q=%22def+{method}%22'
else:
url = f'https://docs.python.org/3/library/unittest.html#unittest.TestCase.{method}'
dotdotdot = fixer.ImmutableDefaultDict(lambda _: '...')
return fixer.SimplePythonFixer(
message=(
'%s is a more specific assertion, and may give more detailed error information than %s.'
% (string.Template(new_expr).substitute(dotdotdot),
string.Template(old_expr).substitute(dotdotdot))),
matcher=syntax_matchers.ExprPattern(old_expr),
replacement=syntactic_template.PythonExprTemplate(new_expr),
url=url,
category='pylint.g-generic-assert',
)
def main():
cli.run(
runner=cli.RefexRunner(
searcher=search.PyExprRewritingSearcher.from_matcher(
# --mode=py.expr is equivalent to PyExprRewritingSearcher paired
# with an ExprPattern. However, you can pass any matcher, not just
# an ExprPattern.
syntax_matchers.ExprPattern('hello'),
{
# Using ROOT_LABEL as a key is equivalent to --sub=world.
# To get the equivalent of --named-sub=x=world,
# it would 'x' as a key instead.
#
# The value type corresponds to the --sub-mode. While
# refex on the command line defaults to picking the paired
# --sub-mode that matches the --mode, here there are no
# defaults and you must be explicit.
# e.g. for unsafe textual substitutions, as with
# --sub-mode=sh, you would use formatting.ShTemplate.
search.ROOT_LABEL:
syntactic_template.PythonExprTemplate('world')
},
),
dry_run=False,
),
files=sys.argv[1:],
bug_report_url='<project bug report URL goes here>',
)
def test_labeled_replacements_example_fragment(self):
fx = fixer.SimplePythonFixer(
message='',
matcher=syntax_matchers.ExprPattern('$y'),
replacement={'y': syntactic_template.PythonExprTemplate('$y')},
)
with self.assertRaises(TypeError):
fx.example_replacement()
def _dict_iter_fixer(method_name):
return fixer.SimplePythonFixer(
message=('dict.{method} is deprecated and does not exist in Python 3. '
'Instead, import six and use six.{method}').format(
method=method_name),
matcher=with_six(
syntax_matchers.ExprPattern('$x.{}()'.format(method_name), {
'x':
base_matchers.Unless(syntax_matchers.ExprPattern('six'))
}), ),
replacement=syntactic_template.PythonExprTemplate(
'six.{}($x)'.format(method_name)),
url='https://www.python.org/doc/sunset-python-2/',
category='pylint.dict-iter-method',
# Must define manually due to the extra restrictions on the pattern.
example_fragment='import six; x.{}()'.format(method_name),
example_replacement='import six; six.{}(x)'.format(method_name),
)
def test_variable_name(self):
parsed = matcher.parse_ast('3', '<string>')
expr = parsed.tree.body[0].value
expr_match = matcher.LexicalASTMatch(expr, parsed.text,
expr.first_token, expr.last_token)
self.assertEqual(
syntax_matchers.ExprPattern('$name').match(
matcher.MatchContext(parsed), expr),
matcher.MatchInfo(expr_match,
{'name': matcher.BoundValue(expr_match)}))
def test_complex_variable(self):
parsed = matcher.parse_ast('foo + bar', '<string>')
expr = parsed.tree.body[0].value
self.assertEqual(
syntax_matchers.ExprPattern('foo + $name').match(
matcher.MatchContext(parsed), expr),
matcher.MatchInfo(matcher.LexicalASTMatch(expr, parsed.text,
expr.first_token,
expr.last_token),
bindings=mock.ANY))
def test_replace(self):
fix = fixer.SimplePythonFixer(
matcher=syntax_matchers.ExprPattern('$obj.attr'),
replacement=syntactic_template.PythonTemplate(u'$obj.other'),
)
searcher = fixer.CombiningPythonFixer([fix])
source = 'my_obj.attr + other_obj.attr'
self.assertEqual('my_obj.other + other_obj.other',
search.rewrite_string(searcher, source, 'example.py'))
def _search_replace_fixer(search_expr,
replace,
message=None,
url='',
**kwargs):
return fixer.SimplePythonFixer(
message=message if message is not None else search_expr,
matcher=syntax_matchers.ExprPattern(search_expr),
replacement=syntactic_template.PythonExprTemplate(replace),
url=url,
category='TESTONLY',
**kwargs)
def test_discards_unparseable_expr(self):
"""Searching discards unparseable substitutions for expressions.
(Note: this only happens during fixedpoint computation.)
"""
fx = fixer.CombiningPythonFixer([
fixer.SimplePythonFixer(message='',
matcher=syntax_matchers.ExprPattern('a'),
replacement=formatting.ShTemplate('x x x'),
url='')
])
self.assertEqual(
list(search.find_iter(fx, 'a', 'foo.py', max_iterations=10)), [])
def test_variable_conflict(self):
"""Variables use the default conflict resolution outside of the pattern.
Inside of the pattern, they use MERGE_EQUIVALENT_AST, but this is opaque to
callers.
"""
# The AllOf shouldn't make a difference, because the $x variable is just
# a regular Bind() variable outside of the pattern, and merges via KEEP_LAST
# per normal.
self.assertEqual(
self.get_all_match_strings(
base_matchers.AllOf(syntax_matchers.ExprPattern('$x'),
base_matchers.Bind('x')), '1'), ['1'])
def test_identical_patterns(self):
"""Tests that patterns match themselves when not parameterized.
Many cases (e.g. None) are interesting for 2/3 compatibility, because the
AST changes in Python 3. syntax_matchers gives an easy way to get
cross-version compatibility.
"""
for code in ['None', '{}', '[]', '{1:2, 3:4}', 'lambda a: a', '""']:
parsed = matcher.parse_ast(code, '<string>')
expr = parsed.tree.body[0].value
for extra_comment in ['', "# comment doesn't matter"]:
with self.subTest(code=code, extra_comment=extra_comment):
self.assertEqual(
syntax_matchers.ExprPattern(
code + extra_comment).match(
matcher.MatchContext(parsed), expr),
matcher.MatchInfo(
matcher.LexicalASTMatch(expr, parsed.text,
expr.first_token,
expr.last_token)))
from refex.fix import fixer
from refex.python import syntactic_template
from refex.python.matchers import ast_matchers
from refex.python.matchers import base_matchers
from refex.python.matchers import syntax_matchers
SIMPLE_PYTHON_FIXERS = [] # Disabled except when running in Python 2.
def with_six(m):
return syntax_matchers.WithTopLevelImport(m, 'six')
_HAS_KEY_FIXER = fixer.SimplePythonFixer(
message='dict.has_key() was removed in Python 3.',
matcher=syntax_matchers.ExprPattern('$a.has_key($b)'),
replacement=syntactic_template.PythonExprTemplate('$b in $a'),
url='https://docs.python.org/3.1/whatsnew/3.0.html#builtins',
significant=True,
category='refex.modernize.dict_has_key',
)
def _dict_iter_fixer(method_name):
return fixer.SimplePythonFixer(
message=('dict.{method} is deprecated and does not exist in Python 3. '
'Instead, import six and use six.{method}').format(
method=method_name),
matcher=with_six(
syntax_matchers.ExprPattern('$x.{}()'.format(method_name), {
'x':
def from_pattern(
cls, pattern: str, templates: Optional[Dict[str, formatting.Template]]
) -> "PyExprRewritingSearcher":
"""Creates a searcher from a ``--mode=py.expr`` template."""
return cls.from_matcher(syntax_matchers.ExprPattern(pattern),
templates=templates)
def test_no_such_variable(self):
with self.assertRaises(KeyError):
syntax_matchers.ExprPattern('a', {'x': base_matchers.Anything()})
def test_syntaxerror(self):
with self.assertRaises(ValueError):
syntax_matchers.ExprPattern('{')
def test_no_statement(self):
with self.assertRaises(ValueError):
syntax_matchers.ExprPattern('')
def test_no_expr(self):
with self.assertRaises(ValueError):
syntax_matchers.ExprPattern('x = 1')
category=_MUTABLE_CONSTANT_CATEGORY,
matcher=matcher,
replacement=replacement,
**kwargs)
def _function_containing(matcher):
"""Returns a ast_matchers matcher for a function where any statement in the body matches `matcher`."""
return syntax_matchers.NamedFunctionDefinition(body=base_matchers.Contains(
syntax_matchers.IsOrHasDescendant(matcher)))
# Matches any function returning Optional[T] for some T.
_IN_FUNCTION_RETURNING_OPTIONAL = syntax_matchers.InNamedFunction(
syntax_matchers.NamedFunctionDefinition(returns=base_matchers.AnyOf(
syntax_matchers.ExprPattern('Optional[$_]'),
syntax_matchers.ExprPattern('typing.Optional[$_]'),
# TODO: May want to also include Union[None, ...].
# TODO: match type comments as well.
)))
# Matches any returning that's not "return" or "return None" (which are two
# different ast node values: ast.Return(value=None) and
# ast.Return(value=ast.Name(id='None')) respectively)
_NON_NONE_RETURN = matcher_.DebugLabeledMatcher(
'Non-none return',
ast_matchers.Return(value=base_matchers.Unless(
base_matchers.AnyOf(base_matchers.Equals(None),
syntax_matchers.ExprPattern('None')))))
_NONE_RETURNS_FIXERS = [
def test_nonname(self):
with self.assertRaises(ValueError) as cm:
syntax_matchers.ExprPattern('a.$x')
self.assertIn('metavariable', str(cm.exception))
def test_multiple_statements(self):
with self.assertRaises(ValueError):
syntax_matchers.ExprPattern('{}; {}')
def test_dict_wrong_order(self):
parsed = matcher.parse_ast('{1:2, 3:4}', '<string>')
expr = parsed.tree.body[0].value
self.assertIsNone(
syntax_matchers.ExprPattern('{3:4, 1:2}').match(
matcher.MatchContext(parsed), expr))
def test_repeated_variable(self):
self.assertEqual(
self.get_all_match_strings(syntax_matchers.ExprPattern('$x + $x'),
'1 + 1\n1 + 2\na + a\na + b'),
['1 + 1', 'a + a'])
def _pattern_factory(pattern):
return ast_matchers.Module(body=base_matchers.ItemsAre(
[ast_matchers.Expr(value=syntax_matchers.ExprPattern(pattern))]))
