def test_str_lineno_strprefix_1():
code = r'''
r"aa\nbb"
0
r"x"
'''
if PY2:
# ur"" is not valid syntax in Python 3
code += r''' Ur"""cc\n
dd"""
'''
block = PythonBlock(dedent(code).lstrip(), startpos=(101,1))
expected_statements = (
PythonStatement('r"aa\\nbb"\n' , startpos=(101,1)),
PythonStatement('0\n' , startpos=(102,1)),
PythonStatement('r"x"\n' , startpos=(103,1)),
)
if PY2:
expected_statements += (
PythonStatement('Ur"""cc\\n\ndd"""\n', startpos=(104,1)),
)
assert block.statements == expected_statements
literals = [(f.s, f.startpos) for f in block.string_literals()]
expected_literals = [
('aa\\nbb' , FilePos(101,1)),
('x' , FilePos(103,1)),
]
if PY2:
expected_literals += [('cc\\n\ndd', FilePos(104,1))]
assert literals == expected_literals
def test_str_lineno_simple_1():
block = PythonBlock(dedent(r'''
a = [
2,
3,
]
b = 'five-a' + 'five-b'
"""
seven
eight
"""
d = 'ten\n'
''').lstrip(), startpos=(101,1))
expected_statements = (
PythonStatement("a = [\n 2,\n 3,\n]\n" , startpos=(101,1)),
PythonStatement("b = 'five-a' + 'five-b'\n", startpos=(105,1)),
PythonStatement('"""\nseven\neight\n"""\n' , startpos=(106,1)),
PythonStatement("d = 'ten\\n'\n" , startpos=(110,1))
)
assert block.statements == expected_statements
literals = [(f.s, f.startpos) for f in block.string_literals()]
expected = [
("five-a" , FilePos(105,5)),
("five-b" , FilePos(105,16)),
("\nseven\neight\n", FilePos(106,1)),
("ten\n" , FilePos(110,5)),
]
assert literals == expected
def test_str_lineno_multichars_1():
block = PythonBlock(dedent('''
[
2,
3,
]
""" 5 "" " ""
6 "" "
7 "" """, """b "" "
8 " "" """
9
''').lstrip(),
startpos=(101, 1))
expected_statements = (
PythonStatement("[\n 2,\n 3,\n]\n", startpos=(101, 1)),
PythonStatement(
'""" 5 "" " ""\n6 "" "\n'
'7 "" """, """b "" "\n8 " "" """\n',
startpos=(105, 1)),
PythonStatement("9\n", startpos=(109, 1)),
)
assert block.statements == expected_statements
literals = [(f.s, f.startpos) for f in block.string_literals()]
expected_literals = [(' 5 "" " ""\n6 "" "\n7 "" ', FilePos(105, 1)),
('b "" "\n8 " "" ', FilePos(107, 11))]
assert literals == expected_literals
def test_str_lineno_in_dict_1():
block = PythonBlock(dedent('''
{
1:
"""
foo4
bar5
""",
2: \'\'\'
foo8
bar9
\'\'\'}
''').lstrip(), startpos=(101,1))
literals = [(f.s, f.startpos) for f in block.string_literals()]
expected_literals = [('\n foo4\n bar5\n ', FilePos(103,9)),
('\n foo8\n bar9\n ', FilePos(107,9))]
assert literals == expected_literals
def test_str_lineno_escaped_single_1():
block = PythonBlock(dedent(r'''
x = "a\
#b" 'c\
#d' \
'e'
'f' \
#'g'
''').lstrip(), startpos=(101,1))
expected_statements = (
PythonStatement("""x = "a\\\n#b" 'c\\\n#d' \\\n'e'\n""", startpos=(101,1)),
PythonStatement("""'f' \\\n#'g'\n""", startpos=(105,1)),
)
assert block.statements == expected_statements
literals = [(f.s, f.startpos) for f in block.string_literals()]
expected_literals = [('a#bc#de', FilePos(101,5)),
('f' , FilePos(105,1))]
assert literals == expected_literals
def test_str_lineno_trailing_1():
block = PythonBlock(dedent(r'''
one
"""
three
four
""".split()
six
''').lstrip(), startpos=(101,1))
expected_statements = (
PythonStatement('one\n' , startpos=(101,1)),
PythonStatement('"""\nthree\nfour\n""".split()\n', startpos=(102,1)),
PythonStatement('six\n' , startpos=(106,1)),
)
assert block.statements == expected_statements
literals = [(f.s, f.startpos) for f in block.string_literals()]
expected_literals = [("\nthree\nfour\n", FilePos(102,1))]
assert literals == expected_literals
def test_str_lineno_strprefix_1():
block = PythonBlock(dedent(r'''
r"aa\nbb"
0
Ur"""cc\n
dd"""
r"x"
''').lstrip(),
startpos=(101, 1))
expected_statements = (
PythonStatement('r"aa\\nbb"\n', startpos=(101, 1)),
PythonStatement('0\n', startpos=(102, 1)),
PythonStatement('Ur"""cc\\n\ndd"""\n', startpos=(103, 1)),
PythonStatement('r"x"\n', startpos=(105, 1)),
)
assert block.statements == expected_statements
literals = [(f.s, f.startpos) for f in block.string_literals()]
expected_literals = [('aa\\nbb', FilePos(101, 1)),
('cc\\n\ndd', FilePos(103, 1)), ('x', FilePos(105,
1))]
assert literals == expected_literals
def test_str_lineno_concatenated_1():
block = PythonBlock(dedent('''
"A" "a"
"B" 'b'
'C' 'c'
'D' "d"
"""E
e""" 'E'
x = """F""" 'f'
\'G\'\'\'\'g
\'\'\' "G"
x = """H
h
H""" 'h' \'\'\'H
h\'\'\'
"I" 'i'.split()
"J" """j
J""" 'j'.split()
'K'
'L'
r"M" u'm' b"""M""" Ur\'\'\'
m\'\'\'
"N" ''"" "n" """"""\'\'\'\'\'\'\'N\'\'\'
"""
O
"""
"""
P
"""
"Q" "q"""
"R" "r""" + "S""""s""S""""s""""
S"""
''').lstrip(),
startpos=(101, 1))
expected_statements = (
PythonStatement('''"A" "a"\n''', startpos=(101, 1)),
PythonStatement('''"B" 'b'\n''', startpos=(102, 1)),
PythonStatement(''''C' 'c'\n''', startpos=(103, 1)),
PythonStatement(''''D' "d"\n''', startpos=(104, 1)),
PythonStatement('''"""E\ne""" 'E'\n''', startpos=(105, 1)),
PythonStatement('''x = """F""" 'f'\n''', startpos=(107, 1)),
PythonStatement("""'G''''g\n\n''' "G"\n""", startpos=(108, 1)),
PythonStatement('''x = """H\nh\nH""" 'h' \'\'\'H\n\n\nh\'\'\'\n''',
startpos=(111, 1)),
PythonStatement('''"I" 'i'.split()\n''', startpos=(117, 1)),
PythonStatement('''"J" """j\nJ""" 'j'.split()\n''',
startpos=(118, 1)),
PythonStatement("""'K'\n""", startpos=(120, 1)),
PythonStatement("""'L'\n""", startpos=(121, 1)),
PythonStatement('''r"M" u'm' b"""M""" Ur\'\'\'\nm\'\'\'\n''',
startpos=(122, 1)),
PythonStatement('''"N" ''"" "n" """"""\'\'\'\'\'\'\'N\'\'\'\n''',
startpos=(124, 1)),
PythonStatement('''"""\nO\n"""\n''', startpos=(125, 1)),
PythonStatement('''"""\nP\n"""\n''', startpos=(128, 1)),
PythonStatement('''"Q" "q"""\n''', startpos=(131, 1)),
PythonStatement('''"R" "r""" + "S""""s""S""""s""""\nS"""\n''',
startpos=(132, 1)),
)
assert block.statements == expected_statements
literals = [(f.s, f.startpos) for f in block.string_literals()]
expected_literals = [
("Aa", FilePos(101, 1)),
("Bb", FilePos(102, 1)),
("Cc", FilePos(103, 1)),
("Dd", FilePos(104, 1)),
("E\neE", FilePos(105, 1)),
("Ff", FilePos(107, 5)),
("Gg\n\nG", FilePos(108, 1)),
("H\nh\nHhH\n\n\nh", FilePos(111, 5)),
("Ii", FilePos(117, 1)),
("Jj\nJj", FilePos(118, 1)),
("K", FilePos(120, 1)),
("L", FilePos(121, 1)),
("MmM\nm", FilePos(122, 1)),
("NnN", FilePos(124, 1)),
("\nO\n", FilePos(125, 1)),
("\nP\n", FilePos(128, 1)),
("Qq", FilePos(131, 1)),
("Rr", FilePos(132, 1)),
('Ss""Ss\nS', FilePos(132, 13)),
]
assert literals == expected_literals
def test_str_lineno_concatenated_1():
code = '''
"A" "a"
"B" 'b'
'C' 'c'
'D' "d"
"""E
e""" 'E'
x = """F""" 'f'
\'G\'\'\'\'g
\'\'\' "G"
x = """H
h
H""" 'h' \'\'\'H
h\'\'\'
"I" 'i'.split()
"J" """j
J""" 'j'.split()
'K'
'L'
r"M" u'm'
"N" ''"" "n" """"""\'\'\'\'\'\'\'N\'\'\'
"""
O
"""
"""
P
"""
"Q" "q"""
"R" "r""" + "S""""s""S""""s""""
S"""
'''
if PY2:
# ur"" is not valid syntax in Python 3
code += """\
Ur'''
t'''
"""
# Implicit string concatenation of non-bytes and bytes literals is not
# valid syntax in Python 3
code += '''\
r"U" u'u' b"""U"""
'''
block = PythonBlock(dedent(code).lstrip(), startpos=(101,1))
expected_statements = (
PythonStatement('''"A" "a"\n''' , startpos=(101,1)),
PythonStatement('''"B" 'b'\n''' , startpos=(102,1)),
PythonStatement(''''C' 'c'\n''' , startpos=(103,1)),
PythonStatement(''''D' "d"\n''' , startpos=(104,1)),
PythonStatement('''"""E\ne""" 'E'\n''' , startpos=(105,1)),
PythonStatement('''x = """F""" 'f'\n''' , startpos=(107,1)),
PythonStatement("""'G''''g\n\n''' "G"\n""" , startpos=(108,1)),
PythonStatement('''x = """H\nh\nH""" 'h' \'\'\'H\n\n\nh\'\'\'\n''', startpos=(111,1)),
PythonStatement('''"I" 'i'.split()\n''' , startpos=(117,1)),
PythonStatement('''"J" """j\nJ""" 'j'.split()\n''' , startpos=(118,1)),
PythonStatement("""'K'\n""" , startpos=(120,1)),
PythonStatement("""'L'\n""" , startpos=(121,1)),
PythonStatement('''r"M" u\'m\'\n''' , startpos=(122,1)),
PythonStatement('''"N" ''"" "n" """"""\'\'\'\'\'\'\'N\'\'\'\n''' , startpos=(123,1)),
PythonStatement('''"""\nO\n"""\n''' , startpos=(124,1)),
PythonStatement('''"""\nP\n"""\n''' , startpos=(127,1)),
PythonStatement('''"Q" "q"""\n''' , startpos=(130,1)),
PythonStatement('''"R" "r""" + "S""""s""S""""s""""\nS"""\n''' , startpos=(131,1)),
)
if PY2:
expected_statements += (
PythonStatement("""Ur'''\nt'''\n""" , startpos=(133,1)),
PythonStatement('''r"U" u'u' b"""U"""\n''' , startpos=(135,1)),
)
assert block.statements == expected_statements
literals = [(f.s, f.startpos) for f in block.string_literals()]
expected_literals = [
("Aa", FilePos(101,1)),
("Bb", FilePos(102,1)),
("Cc", FilePos(103,1)),
("Dd", FilePos(104,1)),
("E\neE", FilePos(105,1)),
("Ff", FilePos(107,5)),
("Gg\n\nG", FilePos(108,1)),
("H\nh\nHhH\n\n\nh", FilePos(111,5)),
("Ii", FilePos(117,1)),
("Jj\nJj", FilePos(118,1)),
("K", FilePos(120,1)),
("L", FilePos(121,1)),
("Mm", FilePos(122,1)),
("NnN", FilePos(123,1)),
("\nO\n", FilePos(124,1)),
("\nP\n", FilePos(127,1)),
("Qq", FilePos(130,1)),
("Rr", FilePos(131,1)),
('Ss""Ss\nS', FilePos(131,13)),
]
if PY2:
expected_literals += [
('\nt', FilePos(133, 1)),
("UuU", FilePos(135,1)),
]
assert literals == expected_literals
def find_unused_and_missing_imports(codeblock):
"""
Find unused imports and missing imports, taking docstrings into account.
Pyflakes is used to statically analyze for unused and missing imports.
Doctests in docstrings are analyzed as code and epydoc references in
docstrings also prevent removal.
In the following example, 'bar' is not considered unused because there is
a string that references it in braces:
>>> find_unused_and_missing_imports("import foo as bar, baz\\n'{bar}'\\n")
([('baz', 1)], [])
@type codeblock:
L{PythonBlock} or convertible
@return:
C{(unused_imports, missing_imports)} where C{unused_imports} and
C{missing_imports} each are sequences of C{(import_as, lineno)} tuples.
"""
# TODO: rewrite this using our own AST parser.
# Once we do that we can also process doctests and literal brace
# identifiers at the proper scope level. We should treat both doctests
# and literal brace identifiers as "soft" uses that don't trigger missing
# imports but do trigger as used imports.
codeblock = PythonBlock(codeblock)
# Run pyflakes on the main code.
unused_imports, missing_imports = (
_pyflakes_find_unused_and_missing_imports(codeblock))
# Find doctests.
doctest_blocks = codeblock.get_doctests()
if doctest_blocks:
# There are doctests. Re-run pyflakes on main code + doctests. Don't
# report missing imports in doctests, but do treat existing imports as
# 'used' if they are used in doctests.
# Create one data structure to pass to pyflakes. This is going to
# screw up linenos but we won't use them, so it doesn't matter.
bigblock = PythonBlock.concatenate([codeblock] + doctest_blocks,
assume_contiguous=True)
wdt_unused_imports, _ = ( # wdt = with doc tests
_pyflakes_find_unused_and_missing_imports(bigblock))
wdt_unused_asimports = set(import_as
for import_as, lineno in wdt_unused_imports)
# Keep only the intersection of unused imports.
unused_imports = [(import_as, lineno)
for import_as, lineno in unused_imports
if import_as in wdt_unused_asimports]
# Find literal brace identifiers like "... L{Foo} ...".
# TODO: merge this into our own AST-based missing/unused-import-finder
# (replacing pyflakes).
literal_brace_identifiers = set(iden for f in codeblock.string_literals()
for iden in brace_identifiers(f.s))
if literal_brace_identifiers:
# Pyflakes doesn't look at docstrings containing references like
# "L{foo}" which require an import, nor at C{str.format} strings like
# '''"{foo}".format(...)'''. Don't remove supposedly-unused imports
# which match a string literal brace identifier.
unused_imports = [(import_as, lineno)
for import_as, lineno in unused_imports
if import_as not in literal_brace_identifiers]
return unused_imports, missing_imports