def test_fix_unused_and_missing_imports_compound_statements_1():
input = PythonBlock(
dedent('''
import a, b; import c, d; g
a, c; f
''').lstrip())
db = ImportDB("""
from m1 import f
from m1 import g
""")
output = fix_unused_and_missing_imports(input, db=db)
expected = PythonBlock(
dedent('''
import a
import c
from m1 import f, g
g
a, c; f
''').lstrip())
assert output == expected
def test_PythonBlock_statements_single_preceding_comment_1():
block = PythonBlock(
dedent('''
# first
foo(1,2)
''').lstrip())
expected = (
PythonStatement("# first\n", startpos=(1, 1)),
PythonStatement("foo(1,2)\n", startpos=(2, 1)),
)
assert block.statements == expected
def test_fix_unused_and_missing_imports_insertion_after_docstring_1():
input = PythonBlock(
dedent('''
"""
aaa
"""
f1, f2
''').lstrip())
db = ImportDB("from m1 import f1")
output = fix_unused_and_missing_imports(input, db=db)
expected = PythonBlock(
dedent('''
"""
aaa
"""
from m1 import f1
f1, f2
''').lstrip())
assert output == expected
def test_str_lineno_in_dict_1():
block = PythonBlock(dedent('''
{
1:
"""
foo4
bar5
""",
2: \'\'\'
foo8
bar9
\'\'\'}
''').lstrip(),
startpos=(101, 1))
literals = [(f.s, f.startpos) for f in block.string_literals()]
expected_literals = [
('\n foo4\n bar5\n ', FilePos(103, 9)),
('\n foo8\n bar9\n ', FilePos(107, 9))
]
assert literals == expected_literals
def test_PythonBlock_statements_last_line_comment_continuation_no_newline_1():
block = PythonBlock(dedent(r'''
a
b\
# c''').lstrip(),
startpos=(101, 1))
expected = (
PythonStatement('a\n', startpos=(101, 1)),
PythonStatement('b\\\n# c', startpos=(102, 1)),
)
assert block.statements == expected
def test_PythonBlock_statements_single_trailing_comment_1():
block = PythonBlock(
dedent('''
foo(1,2)
# last
''').lstrip())
expected = (
PythonStatement("foo(1,2)\n", startpos=(1, 1)),
PythonStatement("# last\n", startpos=(2, 1)),
)
assert block.statements == expected
def test_PythonBlock_statements_comment_continuation_to_comment_1():
block = PythonBlock(dedent('''
x
# y \\
# z
''').lstrip())
expected = (
PythonStatement("x\n" ),
PythonStatement("# y \\\n# z\n", startpos=(2,1)),
)
assert block.statements == expected
def test_fix_unused_and_missing_print_function_1():
input = PythonBlock(
dedent(r'''
from __future__ import print_function
from m1 import print, m1a
from m2.print import m2a, m2b
m1a, m2a, m3a
''').lstrip())
db = ImportDB("from __future__ import print_function\n"
"from print.m3 import m3a, m3b")
output = fix_unused_and_missing_imports(input, db=db)
expected = PythonBlock(
dedent(r'''
from __future__ import print_function
from m1 import m1a
from m2.print import m2a
from print.m3 import m3a
m1a, m2a, m3a
''').lstrip())
assert output == expected
def test_PythonBlock_statements_last_line_nested_continuation_1():
block = PythonBlock(dedent(r'''
a
if b:
"c\
# d"''').lstrip(), startpos=(101,1))
expected = (
PythonStatement('a\n' , startpos=(101,1)),
PythonStatement('if b:\n "c\\\n# d"', startpos=(102,1)),
)
assert block.statements == expected
def test_fix_unused_and_missing_imports_midfile_1():
input = PythonBlock(
dedent('''
import m1, m2
m2, m3
import m4, m5
m4, m6
''').lstrip())
db = ImportDB("import m1, m2, m3, m4, m5, m6, m7")
output = fix_unused_and_missing_imports(input, db=db)
expected = PythonBlock(
dedent('''
import m2
import m3
m2, m3
import m4
import m6
m4, m6
''').lstrip())
assert output == expected
def _pyflakes_checker(codeblock):
from pyflakes.checker import Checker
codeblock = PythonBlock(codeblock)
version = _pyflakes_version()
if version <= 0.4:
# pyflakes 0.4 uses the 'compiler' module.
return Checker(codeblock.parse_tree)
elif version >= 0.5:
# pyflakes 0.5 uses the 'ast' module.
return Checker(codeblock.ast_node)
else:
raise Exception("Unknown pyflakes version %r" % (version, ))
def test_str_lineno_strprefix_1():
block = PythonBlock(dedent(r'''
r"aa\nbb"
0
Ur"""cc\n
dd"""
r"x"
''').lstrip(),
startpos=(101, 1))
expected_statements = (
PythonStatement('r"aa\\nbb"\n', startpos=(101, 1)),
PythonStatement('0\n', startpos=(102, 1)),
PythonStatement('Ur"""cc\\n\ndd"""\n', startpos=(103, 1)),
PythonStatement('r"x"\n', startpos=(105, 1)),
)
assert block.statements == expected_statements
literals = [(f.s, f.startpos) for f in block.string_literals()]
expected_literals = [('aa\\nbb', FilePos(101, 1)),
('cc\\n\ndd', FilePos(103, 1)), ('x', FilePos(105,
1))]
assert literals == expected_literals
def test_PythonStatement_flags_1():
block = PythonBlock("from __future__ import unicode_literals\nx\n",
flags="division")
s0, s1 = block.statements
assert s0.block.source_flags == CompilerFlags("unicode_literals")
assert s1.block.source_flags == CompilerFlags(0)
if sys.version_info >= (3, 8):
assert s0.block.flags == CompilerFlags("unicode_literals", "division",)
assert s1.block.flags == CompilerFlags("unicode_literals", "division",)
else:
assert s0.block.flags == CompilerFlags("unicode_literals", "division")
assert s1.block.flags == CompilerFlags("unicode_literals", "division")
def test_PythonBlock_decorator_1():
block = PythonBlock(dedent('''
@foo1
def bar1(): pass
@foo2
def bar2(): pass
''').lstrip(), startpos=(101,1))
expected = (
PythonStatement("@foo1\ndef bar1(): pass\n", startpos=(101,1)),
PythonStatement("@foo2\ndef bar2(): pass\n", startpos=(103,1)),
)
assert block.statements == expected
def insert_new_import_block(self):
"""
Adds a new empty imports block. It is added before the first
non-comment statement. Intended to be used when the input contains no
import blocks (before uses).
"""
block = SourceToSourceImportBlockTransformation("")
sepblock = SourceToSourceTransformation("")
sepblock.output = PythonBlock("\n")
self.insert_new_blocks_after_comments([block, sepblock])
self.import_blocks.insert(0, block)
return block
def test_fix_unused_and_missing_imports_decorator_2():
input = PythonBlock(
dedent('''
from m1 import d1, d2, d3
@d1
def f1(): pass
@d9.d2
def f9(): pass
''').lstrip())
db = ImportDB("from m2 import d8, d9")
output = fix_unused_and_missing_imports(input, db=db)
expected = PythonBlock(
dedent('''
from m1 import d1
from m2 import d9
@d1
def f1(): pass
@d9.d2
def f9(): pass
''').lstrip())
assert output == expected
def test_PythonBlock_auto_flags_pf_flagps_1():
block = PythonBlock(dedent('''
print(42, out=x)
''').lstrip(), auto_flags=True)
if PY2:
assert (block.flags & "print_function")
assert (block.ast_node.input_flags & "print_function")
assert not (block.source_flags & "print_function")
else:
assert not (block.flags & "print_function")
assert not (block.ast_node.input_flags & "print_function")
assert not (block.source_flags & "print_function")
def test_fix_unused_and_missing_imports_multiple_import_used_1():
input = PythonBlock(
dedent('''
from m0 import f, x
import z, x
import y, z
from m1 import h, f
import x
import x
from m1 import f, f, g
x, f
''').lstrip())
db = ImportDB("")
output = fix_unused_and_missing_imports(input, db=db)
expected = PythonBlock(
dedent('''
import x
from m1 import f
x, f
''').lstrip())
assert output == expected
def test_PythonBlock_FileText_1():
text = FileText(dedent('''
foo()
bar()
''').lstrip(),
filename="/foo/test_PythonBlock_1.py",
startpos=(101, 55))
block = PythonBlock(text)
assert text is block.text
assert text is FileText(block)
assert block.filename == Filename("/foo/test_PythonBlock_1.py")
assert block.startpos == FilePos(101, 55)
def test_fix_unused_and_missing_imports_funcall_1():
input = PythonBlock(
dedent('''
from m1 import X1, X3, X9
def F1(X1, X2=x2, X3=x3, *X4, **X5): X1, X5, x6, X9
def F2(X7=x7): X7
def F3(a1): a1
def F4(): a1
''').lstrip())
db = ImportDB("from m2 import x1, x2, x3, x4, x5, x6, x7, a1, a2")
output = fix_unused_and_missing_imports(input, db=db)
expected = PythonBlock(
dedent('''
from m1 import X9
from m2 import a1, x2, x3, x6, x7
def F1(X1, X2=x2, X3=x3, *X4, **X5): X1, X5, x6, X9
def F2(X7=x7): X7
def F3(a1): a1
def F4(): a1
''').lstrip())
assert output == expected
def transform_imports(codeblock, transformations, params=None):
"""
Transform imports as specified by ``transformations``.
transform_imports() perfectly replaces all imports in top-level import
blocks.
For the rest of the code body, transform_imports() does a crude textual
string replacement. This is imperfect but handles most cases. There may
be some false positives, but this is difficult to avoid. Generally we do
want to do replacements even within in strings and comments.
>>> result = transform_imports("from m import x", {"m.x": "m.y.z"})
>>> print(result.text.joined.strip())
from m.y import z as x
:type codeblock:
`PythonBlock` or convertible (``str``)
:type transformations:
``dict`` from ``str`` to ``str``
:param transformations:
A map of import prefixes to replace, e.g. {"aa.bb": "xx.yy"}
:rtype:
`PythonBlock`
"""
codeblock = PythonBlock(codeblock)
params = ImportFormatParams(params)
transformer = SourceToSourceFileImportsTransformation(codeblock)
@memoize
def transform_import(imp):
# Transform a block of imports.
# TODO: optimize
# TODO: handle transformations containing both a.b=>x and a.b.c=>y
for k, v in transformations.items():
imp = imp.replace(k, v)
return imp
def transform_block(block):
# Do a crude string replacement in the PythonBlock.
block = PythonBlock(block)
s = block.text.joined
for k, v in transformations.items():
s = re.sub("\\b%s\\b" % (re.escape(k)), v, s)
return PythonBlock(s, flags=block.flags)
# Loop over transformer blocks.
for block in transformer.blocks:
if isinstance(block, SourceToSourceImportBlockTransformation):
input_imports = block.importset.imports
output_imports = [ transform_import(imp) for imp in input_imports ]
block.importset = ImportSet(output_imports, ignore_shadowed=True)
else:
block.output = transform_block(block.input)
return transformer.output(params=params)
def test_PythonStatement_auto_flags_1():
block = PythonBlock(
"from __future__ import unicode_literals\nprint(1,file=x)\n",
flags="division", auto_flags=True)
s0, s1 = block.statements
assert s0.block.source_flags == CompilerFlags("unicode_literals")
assert s1.block.source_flags == CompilerFlags(0)
if PY2:
expected = CompilerFlags("unicode_literals", "division", "print_function")
else:
expected = CompilerFlags("unicode_literals", "division")
assert s0.block.flags == expected
assert s1.block.flags == expected
def test_PythonBlock_statements_continuation_1():
block = PythonBlock(dedent(r'''
a
b,\
c
d
''').lstrip(), startpos=(101,1))
expected = (
PythonStatement('a\n' , startpos=(101,1)),
PythonStatement('b,\\\nc\n', startpos=(102,1)),
PythonStatement('d\n' , startpos=(104,1)),
)
assert block.statements == expected
def test_fix_unused_and_missing_imports_ClassDef_1():
input = PythonBlock(
dedent('''
@x1
class x2(x3(x4), x5):
@x6
def x7(x8=x9): x10, x8
def x11(): x11
''').lstrip())
db = ImportDB("from m1 import x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12")
output = fix_unused_and_missing_imports(input, db=db)
expected = PythonBlock(
dedent('''
from m1 import x1, x10, x11, x3, x4, x5, x6, x9
@x1
class x2(x3(x4), x5):
@x6
def x7(x8=x9): x10, x8
def x11(): x11
''').lstrip())
assert output == expected
def test_PythonBlock_flags_type_comment_ignore_fails_transform():
"""
See https://github.com/deshaw/pyflyby/issues/174
Type: ignore are custom ast.AST who have no col_offset.
"""
block = PythonBlock(
dedent("""
a = None # type: ignore
"""
))
s = SourceToSourceFileImportsTransformation(block)
assert s.output() == block
def test_fix_unused_and_missing_imports_xref_1():
input = PythonBlock(
dedent('''
from m1 import f1, f2, f3
def foo():
"""
Hello L{f1} C{f3}
"""
return None
''').lstrip())
db = ImportDB("")
output = fix_unused_and_missing_imports(input, db=db)
expected = PythonBlock(
dedent('''
from m1 import f1, f3
def foo():
"""
Hello L{f1} C{f3}
"""
return None
''').lstrip())
assert output == expected
def test_reformat_import_statements_star_1():
input = PythonBlock(
dedent('''
from mod1 import f1b
from mod1 import f1a
from mod2 import *
from mod2 import f2b as F2B
from mod2 import f2a as F2A
from mod3 import f3b
from mod3 import f3a
''').lstrip(),
filename="/foo/test_reformat_import_statements_star_1.py")
output = reformat_import_statements(input)
expected = PythonBlock(
dedent('''
from mod1 import f1a, f1b
from mod2 import *
from mod2 import f2a as F2A, f2b as F2B
from mod3 import f3a, f3b
''').lstrip(),
filename="/foo/test_reformat_import_statements_star_1.py")
assert output == expected
def test_fix_unused_and_missing_imports_doctests_1():
input = PythonBlock(
dedent('''
from m1 import f1, f2, f3
def foo():
"""
>>> f1 + f2 + f9
"""
return None
''').lstrip())
db = ImportDB("")
output = fix_unused_and_missing_imports(input, db=db)
expected = PythonBlock(
dedent('''
from m1 import f1, f2
def foo():
"""
>>> f1 + f2 + f9
"""
return None
''').lstrip())
assert output == expected
def test_fix_unused_and_missing_imports_1():
input = PythonBlock(
dedent('''
from foo import m1, m2, m3, m4
m2, m4, np.foo
''').lstrip(),
filename="/foo/test_fix_unused_and_missing_imports_1.py")
db = ImportDB("""
import numpy as np
__mandatory_imports__ = ["from __future__ import division"]
""")
output = fix_unused_and_missing_imports(input, db=db)
expected = PythonBlock(
dedent('''
from __future__ import division
import numpy as np
from foo import m2, m4
m2, m4, np.foo
''').lstrip(),
filename="/foo/test_fix_unused_and_missing_imports_1.py")
assert output == expected
def test_PythonBlock_flags_deduce_1():
block = PythonBlock(
dedent(
"""
from __future__ import print_function
print("x",
file=None)
"""
).lstrip()
)
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
assert block.flags == print_function_flag