def test_ImportSet_without_imports_from_1():
importset = ImportSet(
["from m1 import a, b", "from m2 import b, c", "from m3 import b, x"])
result = importset.without_imports("from m1 import b; from m2 import b")
expected = ImportSet(
["from m1 import a", "from m2 import c", "from m3 import b, x"])
assert result == expected
def test_ImportSet_eqne_1():
s1a = ImportSet("from m1 import a, b, a, c")
s1b = ImportSet("from m1 import c, b; from m1 import a as a")
s2 = ImportSet("import m1.a; import m1.b; import m1.c")
assert (s1a == s1b)
assert not (s1a != s1b)
assert (s1a != s2)
assert not (s1a == s2)
def _from_data(cls, known_imports, mandatory_imports, canonical_imports,
forget_imports):
self = object.__new__(cls)
self.forget_imports = ImportSet(forget_imports)
self.known_imports = ImportSet(known_imports).without_imports(
forget_imports)
self.mandatory_imports = ImportSet(mandatory_imports).without_imports(
forget_imports)
# TODO: provide more fine-grained control about canonical_imports.
self.canonical_imports = ImportMap(canonical_imports).without_imports(
forget_imports)
return self
def test_ImportSet_without_imports_star_dot_1():
importset = ImportSet("""
import m94165726
from m68073152 import f59136817
from .m69396491 import f87639367
from . import m81881832
from m97513719.a import f42218372
""")
result = importset.without_imports("from . import *")
expected = ImportSet("""
import m94165726
from m68073152 import f59136817
from m97513719.a import f42218372
""")
assert result == expected
def test_ImportSet_without_imports_star_1():
importset = ImportSet("""
from m11321086.a import f27811501, f04141733
from m28630179.a import f75932565, f54328537
from m28630179.a.b import f46586889, f53411856
from m28630179.x import f10642186, f95537624
from .m28630179.a import f38714787, f42847225
""")
result = importset.without_imports("from m28630179.a import *")
expected = ImportSet("""
from m11321086.a import f27811501, f04141733
from m28630179.x import f10642186, f95537624
from .m28630179.a import f38714787, f42847225
""")
assert result == expected
def test_ImportDB_pyflyby_forget_1():
with EnvVarCtx(PYFLYBY_PATH=".../f70301376"):
d = mkdtemp("_pyflyby")
os.mkdir("%s/d1" % d)
os.mkdir("%s/d1/d2" % d)
os.mkdir("%s/d1/d2/d3" % d)
with open("%s/f70301376" % d, 'w') as f:
f.write(
dedent("""
from m49790901 import f27626336, f96186952
from m78687343 import f35156295, f43613649
from m78687343.a.b.c import f54583581
from m49790901.a import f27626336, f96186952
"""))
with open("%s/d1/d2/f70301376" % d, 'w') as f:
f.write(
dedent("""
__forget_imports__ = [
'from m49790901 import f27626336',
'from m78687343 import *',
]
"""))
db = ImportDB.get_default("%s/d1/d2/d3/f" % d)
result = db.known_imports
expected = ImportSet("""
from m49790901 import f96186952
from m49790901.a import f27626336, f96186952
""")
assert result == expected
rmtree(d)
def test_ImportDB_from_code_complex_1():
result = ImportDB('''
import foo, bar as barf
from xx import yy, yyy, yyyy
__mandatory_imports__ = ['__future__.division',
'import aa . bb . cc as dd']
__forget_imports__ = ['xx.yy', 'from xx import zz']
__canonical_imports__ = {'bad.baad': 'good.goood'}
''')
assert result.known_imports == ImportSet(
["import foo", "import bar as barf", "from xx import yyy, yyyy"])
assert result.mandatory_imports == ImportSet(
["from __future__ import division", "from aa.bb import cc as dd"])
assert result.forget_imports == ImportSet(
["from xx import yy", "from xx import zz"])
assert result.canonical_imports == ImportMap({"bad.baad": "good.goood"})
def test_ImportSet_constructor_string_1():
importset = ImportSet("from m1 import c, b; from m1 import a as a")
expected = [
Import("from m1 import a"),
Import("from m1 import b"),
Import("from m1 import c")
]
assert list(importset) == expected
def test_ImportSet_constructor_list_2():
importset = ImportSet(["from m1 import c, b", "from m1 import a as a"])
expected = [
Import("from m1 import a"),
Import("from m1 import b"),
Import("from m1 import c")
]
assert list(importset) == expected
def test_ImportSet_constructor_importstmt_1():
importset = ImportSet(ImportStatement("from m1 import a, b, c"))
expected = [
Import("from m1 import a"),
Import("from m1 import b"),
Import("from m1 import c")
]
assert list(importset) == expected
def test_ImportSet_constructor_list_imports_1():
expected = [
Import("from m1 import a"),
Import("from m1 import b"),
Import("from m1 import c")
]
importset = ImportSet(expected)
assert list(importset) == expected
def transform_imports(codeblock, transformations, params=None):
"""
Transform imports as specified by C{transformations}.
transform_imports() perfectly replaces all imports in top-level import
blocks.
For the rest of the code body, transform_imports() does a crude textual
string replacement. This is imperfect but handles most cases. There may
be some false positives, but this is difficult to avoid. Generally we do
want to do replacements even within in strings and comments.
>>> result = transform_imports("from m import x", {"m.x": "m.y.z"})
>>> print result.text.joined.strip()
from m.y import z as x
@type codeblock:
L{PythonBlock} or convertible (C{str})
@type transformations:
C{dict} from C{str} to C{str}
@param transformations:
A map of import prefixes to replace, e.g. {"aa.bb": "xx.yy"}
@rtype:
L{PythonBlock}
"""
codeblock = PythonBlock(codeblock)
params = ImportFormatParams(params)
transformer = SourceToSourceFileImportsTransformation(codeblock)
@memoize
def transform_import(imp):
# Transform a block of imports.
# TODO: optimize
# TODO: handle transformations containing both a.b=>x and a.b.c=>y
for k, v in transformations.iteritems():
imp = imp.replace(k, v)
return imp
def transform_block(block):
# Do a crude string replacement in the PythonBlock.
block = PythonBlock(block)
s = block.text.joined
for k, v in transformations.iteritems():
s = re.sub("\\b%s\\b" % (re.escape(k)), v, s)
return PythonBlock(s, flags=block.flags)
# Loop over transformer blocks.
for block in transformer.blocks:
if isinstance(block, SourceToSourceImportBlockTransformation):
input_imports = block.importset.imports
output_imports = [transform_import(imp) for imp in input_imports]
block.importset = ImportSet(output_imports, ignore_shadowed=True)
else:
block.output = transform_block(block.input)
return transformer.output(params=params)
def test_ImportSet_contains_1():
importset = ImportSet('''
from m1 import f1
from m2 import f1
from m1 import f2
from m1 import f1, f3
import m3.m4 as m34
''')
assert Import("from m1 import f1") in importset
assert Import("from .m1 import f1") not in importset
assert Import("from m2 import f2") not in importset
def _parse_import_set(cls, arg):
if isinstance(arg, six.string_types):
arg = [arg]
if not isinstance(arg, (tuple, list)):
raise ValueError("Expected a list, not a %s" %
(type(arg).__name__, ))
for item in arg:
if not isinstance(item, six.string_types):
raise ValueError("Expected a list of str, not %s" %
(type(item).__name__, ))
return ImportSet(arg)
def test_ImportSet_member_names_1():
importset = ImportSet('''
import numpy.linalg.info
from sys import exit as EXIT
''')
expected = {
'': ('EXIT', 'numpy', 'sys'),
'numpy': ('linalg', ),
'numpy.linalg': ('info', ),
'sys': ('exit', )
}
assert importset.member_names == expected
def test_ImportSet_by_import_as_1():
importset = ImportSet('''
from a1.b1 import c1 as x
from a2.b2 import c2 as x
from a2.b2 import c2 as y
''')
expected = {
'x': (Import('from a1.b1 import c1 as x'),
Import('from a2.b2 import c2 as x')),
'y': (Import('from a2.b2 import c2 as y'), )
}
assert importset.by_import_as == expected
def test_ImportSet_1():
importset = ImportSet('''
from m1 import f1
from m2 import f1
from m1 import f2
from m1 import f1, f3
import m3.m4 as m34
''')
expected = (Import("from m1 import f1"), Import("from m1 import f2"),
Import("from m1 import f3"), Import("from m2 import f1"),
Import("from m3 import m4 as m34"))
print importset.imports
assert importset.imports == expected
assert len(importset) == 5
def test_ImportSet_ignore_shadowed_1():
importset = ImportSet('''
from m1 import f1, f2, f3
from m2 import f2, f4, f6
from m3 import f3, f6, f9
from m1 import f9
''',
ignore_shadowed=True)
expected = (
Import("from m1 import f1"),
Import("from m1 import f9"),
Import("from m2 import f2"),
Import("from m2 import f4"),
Import("from m3 import f3"),
Import("from m3 import f6"),
)
print importset.imports
assert importset.imports == expected
assert len(importset) == 6
def exports(self):
"""
Get symbols exported by this module.
Note that this requires involves actually importing this module, which
may have side effects. (TODO: rewrite to avoid this?)
@rtype:
L{ImportSet} or C{None}
@return:
Exports, or C{None} if nothing exported.
"""
from pyflyby._importclns import ImportStatement, ImportSet
module = self.module
try:
members = module.__all__
except AttributeError:
members = dir(module)
# Filter by non-private.
members = [n for n in members if not n.startswith("_")]
# Filter by definition in the module.
def from_this_module(name):
# TODO: could do this more robustly by parsing the AST and
# looking for STOREs (definitions/assignments/etc).
x = getattr(module, name)
m = getattr(x, "__module__", None)
if not m:
return False
return DottedIdentifier(m).startswith(self.name)
members = [n for n in members if from_this_module(n)]
else:
if not all(type(s) == str for s in members):
raise Exception(
"Module %r contains non-string entries in __all__" %
(str(self.name), ))
# Filter out artificially added "deep" members.
members = [n for n in members if "." not in n]
if not members:
return None
return ImportSet([ImportStatement.from_parts(str(self.name), members)])
def test_ImportDB_from_code_1():
db = ImportDB('from aa.bb import cc as dd, ee')
expected_known = ImportSet(['from aa.bb import cc as dd, ee'])
assert db.known_imports == expected_known
def test_ImportSet_without_imports_no_action_1():
importset = ImportSet("import a, b, c, d")
result = importset.without_imports("import x, y")
assert result is importset
def replace_star_imports(codeblock, params=None):
r"""
Replace lines such as::
from foo.bar import *
with
from foo.bar import f1, f2, f3
Note that this requires involves actually importing C{foo.bar}, which may
have side effects. (TODO: rewrite to avoid this?)
The result includes all imports from the C{email} module. The result
excludes shadowed imports. In this example:
1. The original C{MIMEAudio} import is shadowed, so it is removed.
2. The C{MIMEImage} import in the C{email} module is shadowed by a
subsequent import, so it is omitted.
>>> codeblock = PythonBlock('from keyword import *', filename="/tmp/x.py")
>>> print replace_star_imports(codeblock)
[PYFLYBY] /tmp/x.py: replaced 'from keyword import *' with 2 imports
from keyword import iskeyword, kwlist
<BLANKLINE>
Usually you'll want to remove unused imports after replacing star imports.
@type codeblock:
L{PythonBlock} or convertible (C{str})
@rtype:
L{PythonBlock}
"""
from pyflyby._modules import ModuleHandle
params = ImportFormatParams(params)
codeblock = PythonBlock(codeblock)
filename = codeblock.filename
transformer = SourceToSourceFileImportsTransformation(codeblock)
for block in transformer.import_blocks:
# Iterate over the import statements in C{block.input}. We do this
# instead of using C{block.importset} because the latter doesn't
# preserve the order of inputs. The order is important for
# determining what's shadowed.
imports = [
imp for s in block.input.statements
for imp in ImportStatement(s).imports
]
# Process "from ... import *" statements.
new_imports = []
for imp in imports:
if imp.split.member_name != "*":
new_imports.append(imp)
elif imp.split.module_name.startswith("."):
# The source contains e.g. "from .foo import *". Right now we
# don't have a good way to figure out the absolute module
# name, so we can't get at foo. That said, there's a decent
# chance that this is inside an __init__ anyway, which is one
# of the few justifiable use cases for star imports in library
# code.
logger.warning(
"%s: can't replace star imports in relative import: %s",
filename,
imp.pretty_print().strip())
new_imports.append(imp)
else:
module = ModuleHandle(imp.split.module_name)
try:
with ImportPathForRelativeImportsCtx(codeblock):
exports = module.exports
except Exception as e:
logger.warning(
"%s: couldn't import '%s' to enumerate exports, "
"leaving unchanged: '%s'. %s: %s", filename,
module.name, imp,
type(e).__name__, e)
new_imports.append(imp)
continue
if not exports:
# We found nothing in the target module. This probably
# means that module itself is just importing things from
# other modules. Currently we intentionally exclude those
# imports since usually we don't want them. TODO: do
# something better here.
logger.warning("%s: found nothing to import from %s, ",
"leaving unchanged: '%s'", filename, module,
imp)
new_imports.append(imp)
else:
new_imports.extend(exports)
logger.info("%s: replaced %r with %d imports", filename,
imp.pretty_print().strip(), len(exports))
block.importset = ImportSet(new_imports, ignore_shadowed=True)
return transformer.output(params=params)
def _from_code(
cls,
blocks,
_mandatory_imports_blocks_deprecated=(),
_forget_imports_blocks_deprecated=(),
):
"""
Load an import database from code.
>>> ImportDB._from_code('''
... import foo, bar as barf
... from xx import yy
... __mandatory_imports__ = ['__future__.division',
... 'import aa . bb . cc as dd']
... __forget_imports__ = ['xx.yy', 'from xx import zz']
... __canonical_imports__ = {'bad.baad': 'good.goood'}
... ''')
ImportDB('''
import bar as barf
import foo
<BLANKLINE>
__mandatory_imports__ = [
'from __future__ import division',
'from aa.bb import cc as dd',
]
<BLANKLINE>
__canonical_imports__ = {
'bad.baad': 'good.goood',
}
<BLANKLINE>
__forget_imports__ = [
'from xx import yy',
'from xx import zz',
]
''')
@rtype:
L{ImportDB}
"""
if not isinstance(blocks, (tuple, list)):
blocks = [blocks]
if not isinstance(_mandatory_imports_blocks_deprecated, (tuple, list)):
_mandatory_imports_blocks_deprecated = [
_mandatory_imports_blocks_deprecated
]
if not isinstance(_forget_imports_blocks_deprecated, (tuple, list)):
_forget_imports_blocks_deprecated = [
_forget_imports_blocks_deprecated
]
known_imports = []
mandatory_imports = []
canonical_imports = []
forget_imports = []
blocks = [PythonBlock(b) for b in blocks]
for block in blocks:
for statement in block.statements:
if statement.is_comment_or_blank:
continue
if statement.is_import:
known_imports.extend(ImportStatement(statement).imports)
continue
try:
name, value = statement.get_assignment_literal_value()
if name == "__mandatory_imports__":
mandatory_imports.append(cls._parse_import_set(value))
elif name == "__canonical_imports__":
canonical_imports.append(cls._parse_import_map(value))
elif name == "__forget_imports__":
forget_imports.append(cls._parse_import_set(value))
else:
raise ValueError(
"Unknown assignment to %r (expected one of "
"__mandatory_imports__, __canonical_imports__, "
"__forget_imports__)" % (name, ))
except ValueError as e:
raise ValueError("While parsing %s: error in %r: %s" %
(block.filename, statement, e))
for block in _mandatory_imports_blocks_deprecated:
mandatory_imports.append(ImportSet(block))
for block in _forget_imports_blocks_deprecated:
forget_imports.append(ImportSet(block))
return cls._from_data(known_imports, mandatory_imports,
canonical_imports, forget_imports)
def test_ImportSet_constructor_idempotent_1():
importset = ImportSet("from m1 import c, b, a")
result = ImportSet(importset)
assert result is importset
def preprocess(self):
self.importset = ImportSet(self.input, ignore_shadowed=True)
def test_ImportSet_conflicting_imports_2():
importset = ImportSet('import b\nfrom f import a\n')
assert importset.conflicting_imports == ()
def test_ImportSet_without_imports_1():
importset = ImportSet("import a, b, c, d")
result = importset.without_imports("import x, d, b, y")
expected = ImportSet("import a, c")
assert result == expected
def test_ImportSet_without_imports_exact_2():
importset = ImportSet(
["from m1 import a", "import m1.a", "from m1.a import *"])
result = importset.without_imports("import m1.a")
expected = ImportSet("from m1 import a; from m1.a import *")
assert result == expected