def test_ImportSet_without_imports_from_1():
importset = ImportSet(
["from m1 import a, b", "from m2 import b, c", "from m3 import b, x"])
result = importset.without_imports("from m1 import b; from m2 import b")
expected = ImportSet(
["from m1 import a", "from m2 import c", "from m3 import b, x"])
assert result == expected
def test_ImportSet_eqne_1():
s1a = ImportSet("from m1 import a, b, a, c")
s1b = ImportSet("from m1 import c, b; from m1 import a as a")
s2 = ImportSet("import m1.a; import m1.b; import m1.c")
assert (s1a == s1b)
assert not (s1a != s1b)
assert (s1a != s2)
assert not (s1a == s2)
def _from_data(cls, known_imports, mandatory_imports, canonical_imports,
forget_imports):
self = object.__new__(cls)
self.forget_imports = ImportSet(forget_imports)
self.known_imports = ImportSet(known_imports).without_imports(
forget_imports)
self.mandatory_imports = ImportSet(mandatory_imports).without_imports(
forget_imports)
# TODO: provide more fine-grained control about canonical_imports.
self.canonical_imports = ImportMap(canonical_imports).without_imports(
forget_imports)
return self
def test_ImportSet_without_imports_star_1():
importset = ImportSet("""
from m11321086.a import f27811501, f04141733
from m28630179.a import f75932565, f54328537
from m28630179.a.b import f46586889, f53411856
from m28630179.x import f10642186, f95537624
from .m28630179.a import f38714787, f42847225
""")
result = importset.without_imports("from m28630179.a import *")
expected = ImportSet("""
from m11321086.a import f27811501, f04141733
from m28630179.x import f10642186, f95537624
from .m28630179.a import f38714787, f42847225
""")
assert result == expected
def test_ImportSet_without_imports_star_dot_1():
importset = ImportSet("""
import m94165726
from m68073152 import f59136817
from .m69396491 import f87639367
from . import m81881832
from m97513719.a import f42218372
""")
result = importset.without_imports("from . import *")
expected = ImportSet("""
import m94165726
from m68073152 import f59136817
from m97513719.a import f42218372
""")
assert result == expected
def test_ImportDB_from_code_complex_1():
result = ImportDB('''
import foo, bar as barf
from xx import yy, yyy, yyyy
__mandatory_imports__ = ['__future__.division',
'import aa . bb . cc as dd']
__forget_imports__ = ['xx.yy', 'from xx import zz']
__canonical_imports__ = {'bad.baad': 'good.goood'}
''')
assert result.known_imports == ImportSet(
["import foo", "import bar as barf", "from xx import yyy, yyyy"])
assert result.mandatory_imports == ImportSet(
["from __future__ import division", "from aa.bb import cc as dd"])
assert result.forget_imports == ImportSet(
["from xx import yy", "from xx import zz"])
assert result.canonical_imports == ImportMap({"bad.baad": "good.goood"})
class SourceToSourceImportBlockTransformation(SourceToSourceTransformationBase):
def preprocess(self):
self.importset = ImportSet(self.input, ignore_shadowed=True)
def pretty_print(self, params=None):
params = ImportFormatParams(params)
return self.importset.pretty_print(params)
def test_ImportDB_pyflyby_forget_1():
with EnvVarCtx(PYFLYBY_PATH=".../f70301376"):
d = mkdtemp("_pyflyby")
os.mkdir("%s/d1" % d)
os.mkdir("%s/d1/d2" % d)
os.mkdir("%s/d1/d2/d3" % d)
with open("%s/f70301376" % d, 'w') as f:
f.write(
dedent("""
from m49790901 import f27626336, f96186952
from m78687343 import f35156295, f43613649
from m78687343.a.b.c import f54583581
from m49790901.a import f27626336, f96186952
"""))
with open("%s/d1/d2/f70301376" % d, 'w') as f:
f.write(
dedent("""
__forget_imports__ = [
'from m49790901 import f27626336',
'from m78687343 import *',
]
"""))
db = ImportDB.get_default("%s/d1/d2/d3/f" % d)
result = db.known_imports
expected = ImportSet("""
from m49790901 import f96186952
from m49790901.a import f27626336, f96186952
""")
assert result == expected
rmtree(d)
def test_ImportSet_constructor_importstmt_1():
importset = ImportSet(ImportStatement("from m1 import a, b, c"))
expected = [
Import("from m1 import a"),
Import("from m1 import b"),
Import("from m1 import c")
]
assert list(importset) == expected
def test_ImportSet_constructor_list_2():
importset = ImportSet(["from m1 import c, b", "from m1 import a as a"])
expected = [
Import("from m1 import a"),
Import("from m1 import b"),
Import("from m1 import c")
]
assert list(importset) == expected
def test_ImportSet_constructor_list_imports_1():
expected = [
Import("from m1 import a"),
Import("from m1 import b"),
Import("from m1 import c")
]
importset = ImportSet(expected)
assert list(importset) == expected
def test_ImportSet_constructor_string_1():
importset = ImportSet("from m1 import c, b; from m1 import a as a")
expected = [
Import("from m1 import a"),
Import("from m1 import b"),
Import("from m1 import c")
]
assert list(importset) == expected
def transform_imports(codeblock, transformations, params=None):
"""
Transform imports as specified by C{transformations}.
transform_imports() perfectly replaces all imports in top-level import
blocks.
For the rest of the code body, transform_imports() does a crude textual
string replacement. This is imperfect but handles most cases. There may
be some false positives, but this is difficult to avoid. Generally we do
want to do replacements even within in strings and comments.
>>> result = transform_imports("from m import x", {"m.x": "m.y.z"})
>>> print result.text.joined.strip()
from m.y import z as x
@type codeblock:
L{PythonBlock} or convertible (C{str})
@type transformations:
C{dict} from C{str} to C{str}
@param transformations:
A map of import prefixes to replace, e.g. {"aa.bb": "xx.yy"}
@rtype:
L{PythonBlock}
"""
codeblock = PythonBlock(codeblock)
params = ImportFormatParams(params)
transformer = SourceToSourceFileImportsTransformation(codeblock)
@memoize
def transform_import(imp):
# Transform a block of imports.
# TODO: optimize
# TODO: handle transformations containing both a.b=>x and a.b.c=>y
for k, v in transformations.iteritems():
imp = imp.replace(k, v)
return imp
def transform_block(block):
# Do a crude string replacement in the PythonBlock.
block = PythonBlock(block)
s = block.text.joined
for k, v in transformations.iteritems():
s = re.sub("\\b%s\\b" % (re.escape(k)), v, s)
return PythonBlock(s, flags=block.flags)
# Loop over transformer blocks.
for block in transformer.blocks:
if isinstance(block, SourceToSourceImportBlockTransformation):
input_imports = block.importset.imports
output_imports = [transform_import(imp) for imp in input_imports]
block.importset = ImportSet(output_imports, ignore_shadowed=True)
else:
block.output = transform_block(block.input)
return transformer.output(params=params)
def test_ImportSet_contains_1():
importset = ImportSet('''
from m1 import f1
from m2 import f1
from m1 import f2
from m1 import f1, f3
import m3.m4 as m34
''')
assert Import("from m1 import f1") in importset
assert Import("from .m1 import f1") not in importset
assert Import("from m2 import f2") not in importset
def _parse_import_set(cls, arg):
if isinstance(arg, six.string_types):
arg = [arg]
if not isinstance(arg, (tuple, list)):
raise ValueError("Expected a list, not a %s" %
(type(arg).__name__, ))
for item in arg:
if not isinstance(item, six.string_types):
raise ValueError("Expected a list of str, not %s" %
(type(item).__name__, ))
return ImportSet(arg)
def test_ImportSet_by_import_as_1():
importset = ImportSet('''
from a1.b1 import c1 as x
from a2.b2 import c2 as x
from a2.b2 import c2 as y
''')
expected = {
'x': (Import('from a1.b1 import c1 as x'),
Import('from a2.b2 import c2 as x')),
'y': (Import('from a2.b2 import c2 as y'), )
}
assert importset.by_import_as == expected
def test_ImportSet_member_names_1():
importset = ImportSet('''
import numpy.linalg.info
from sys import exit as EXIT
''')
expected = {
'': ('EXIT', 'numpy', 'sys'),
'numpy': ('linalg', ),
'numpy.linalg': ('info', ),
'sys': ('exit', )
}
assert importset.member_names == expected
def test_ImportSet_1():
importset = ImportSet('''
from m1 import f1
from m2 import f1
from m1 import f2
from m1 import f1, f3
import m3.m4 as m34
''')
expected = (Import("from m1 import f1"), Import("from m1 import f2"),
Import("from m1 import f3"), Import("from m2 import f1"),
Import("from m3 import m4 as m34"))
print importset.imports
assert importset.imports == expected
assert len(importset) == 5
def test_ImportSet_ignore_shadowed_1():
importset = ImportSet('''
from m1 import f1, f2, f3
from m2 import f2, f4, f6
from m3 import f3, f6, f9
from m1 import f9
''',
ignore_shadowed=True)
expected = (
Import("from m1 import f1"),
Import("from m1 import f9"),
Import("from m2 import f2"),
Import("from m2 import f4"),
Import("from m3 import f3"),
Import("from m3 import f6"),
)
print importset.imports
assert importset.imports == expected
assert len(importset) == 6
def exports(self):
"""
Get symbols exported by this module.
Note that this requires involves actually importing this module, which
may have side effects. (TODO: rewrite to avoid this?)
@rtype:
L{ImportSet} or C{None}
@return:
Exports, or C{None} if nothing exported.
"""
from pyflyby._importclns import ImportStatement, ImportSet
module = self.module
try:
members = module.__all__
except AttributeError:
members = dir(module)
# Filter by non-private.
members = [n for n in members if not n.startswith("_")]
# Filter by definition in the module.
def from_this_module(name):
# TODO: could do this more robustly by parsing the AST and
# looking for STOREs (definitions/assignments/etc).
x = getattr(module, name)
m = getattr(x, "__module__", None)
if not m:
return False
return DottedIdentifier(m).startswith(self.name)
members = [n for n in members if from_this_module(n)]
else:
if not all(type(s) == str for s in members):
raise Exception(
"Module %r contains non-string entries in __all__" %
(str(self.name), ))
# Filter out artificially added "deep" members.
members = [n for n in members if "." not in n]
if not members:
return None
return ImportSet([ImportStatement.from_parts(str(self.name), members)])
def replace_star_imports(codeblock, params=None):
r"""
Replace lines such as::
from foo.bar import *
with
from foo.bar import f1, f2, f3
Note that this requires involves actually importing C{foo.bar}, which may
have side effects. (TODO: rewrite to avoid this?)
The result includes all imports from the C{email} module. The result
excludes shadowed imports. In this example:
1. The original C{MIMEAudio} import is shadowed, so it is removed.
2. The C{MIMEImage} import in the C{email} module is shadowed by a
subsequent import, so it is omitted.
>>> codeblock = PythonBlock('from keyword import *', filename="/tmp/x.py")
>>> print replace_star_imports(codeblock)
[PYFLYBY] /tmp/x.py: replaced 'from keyword import *' with 2 imports
from keyword import iskeyword, kwlist
<BLANKLINE>
Usually you'll want to remove unused imports after replacing star imports.
@type codeblock:
L{PythonBlock} or convertible (C{str})
@rtype:
L{PythonBlock}
"""
from pyflyby._modules import ModuleHandle
params = ImportFormatParams(params)
codeblock = PythonBlock(codeblock)
filename = codeblock.filename
transformer = SourceToSourceFileImportsTransformation(codeblock)
for block in transformer.import_blocks:
# Iterate over the import statements in C{block.input}. We do this
# instead of using C{block.importset} because the latter doesn't
# preserve the order of inputs. The order is important for
# determining what's shadowed.
imports = [
imp for s in block.input.statements
for imp in ImportStatement(s).imports
]
# Process "from ... import *" statements.
new_imports = []
for imp in imports:
if imp.split.member_name != "*":
new_imports.append(imp)
elif imp.split.module_name.startswith("."):
# The source contains e.g. "from .foo import *". Right now we
# don't have a good way to figure out the absolute module
# name, so we can't get at foo. That said, there's a decent
# chance that this is inside an __init__ anyway, which is one
# of the few justifiable use cases for star imports in library
# code.
logger.warning(
"%s: can't replace star imports in relative import: %s",
filename,
imp.pretty_print().strip())
new_imports.append(imp)
else:
module = ModuleHandle(imp.split.module_name)
try:
with ImportPathForRelativeImportsCtx(codeblock):
exports = module.exports
except Exception as e:
logger.warning(
"%s: couldn't import '%s' to enumerate exports, "
"leaving unchanged: '%s'. %s: %s", filename,
module.name, imp,
type(e).__name__, e)
new_imports.append(imp)
continue
if not exports:
# We found nothing in the target module. This probably
# means that module itself is just importing things from
# other modules. Currently we intentionally exclude those
# imports since usually we don't want them. TODO: do
# something better here.
logger.warning("%s: found nothing to import from %s, ",
"leaving unchanged: '%s'", filename, module,
imp)
new_imports.append(imp)
else:
new_imports.extend(exports)
logger.info("%s: replaced %r with %d imports", filename,
imp.pretty_print().strip(), len(exports))
block.importset = ImportSet(new_imports, ignore_shadowed=True)
return transformer.output(params=params)
class ImportDB(object):
"""
A database of known, mandatory, canonical imports.
@iattr known_imports:
Set of known imports. For use by tidy-imports and autoimporter.
@iattr mandatory_imports:
Set of imports that must be added by tidy-imports.
@iattr canonical_imports:
Map of imports that tidy-imports transforms on every run.
@iattr forget_imports:
Set of imports to remove from known_imports, mandatory_imports,
canonical_imports.
"""
def __new__(cls, *args):
if len(args) != 1:
raise TypeError
arg, = args
if isinstance(arg, cls):
return arg
if isinstance(arg, ImportSet):
return cls._from_data(arg, [], [], [])
return cls._from_args(arg) # PythonBlock, Filename, etc
_default_cache = {}
@classmethod
def clear_default_cache(cls):
"""
Clear the class cache of default ImportDBs.
Subsequent calls to ImportDB.get_default() will not reuse previously
cached results. Existing ImportDB instances are not affected by this
call.
"""
if cls._default_cache:
if logger.debug_enabled:
allpyfiles = set()
for tup in cls._default_cache:
if tup[0] != 2:
continue
for tup2 in tup[1:]:
for f in tup2:
assert isinstance(f, Filename)
if f.ext == '.py':
allpyfiles.add(f)
nfiles = len(allpyfiles)
logger.debug("ImportDB: Clearing default cache of %d files",
nfiles)
cls._default_cache.clear()
@classmethod
def get_default(cls, target_filename):
"""
Return the default import library for the given target filename.
This will read various .../.pyflyby files as specified by
$PYFLYBY_PATH as well as older deprecated environment variables.
Memoized.
@param target_filename:
The target filename for which to get the import database. Note that
the target filename itself is not read. Instead, the target
filename is relevant because we look for .../.pyflyby based on the
target filename.
@rtype:
L{ImportDB}
"""
# We're going to canonicalize target_filenames in a number of steps.
# At each step, see if we've seen the input so far. We do the cache
# checking incrementally since the steps involve syscalls. Since this
# is going to potentially be executed inside the IPython interactive
# loop, we cache as much as possible.
# TODO: Consider refreshing periodically. Check if files have
# been touched, and if so, return new data. Check file timestamps at
# most once every 60 seconds.
cache_keys = []
target_filename = Filename(target_filename or ".")
if target_filename.startswith("/dev"):
target_filename = Filename(".")
target_dirname = target_filename
# TODO: with StatCache
while True:
cache_keys.append((1, target_dirname, os.getenv("PYFLYBY_PATH"),
os.getenv("PYFLYBY_KNOWN_IMPORTS_PATH"),
os.getenv("PYFLYBY_MANDATORY_IMPORTS_PATH")))
try:
return cls._default_cache[cache_keys[-1]]
except KeyError:
pass
if target_dirname.isdir:
break
target_dirname = target_dirname.dir
target_dirname = target_dirname.real
if target_dirname != cache_keys[-1][0]:
cache_keys.append((1, target_dirname, os.getenv("PYFLYBY_PATH"),
os.getenv("PYFLYBY_KNOWN_IMPORTS_PATH"),
os.getenv("PYFLYBY_MANDATORY_IMPORTS_PATH")))
try:
return cls._default_cache[cache_keys[-1]]
except KeyError:
pass
DEFAULT_PYFLYBY_PATH = []
etc_dir = _find_etc_dir()
if etc_dir:
DEFAULT_PYFLYBY_PATH.append(str(etc_dir))
DEFAULT_PYFLYBY_PATH += [
".../.pyflyby",
"~/.pyflyby",
]
logger.debug("DEFAULT_PYFLYBY_PATH=%s", DEFAULT_PYFLYBY_PATH)
filenames = _get_python_path("PYFLYBY_PATH", DEFAULT_PYFLYBY_PATH,
target_dirname)
mandatory_imports_filenames = ()
if "SUPPORT DEPRECATED BEHAVIOR":
PYFLYBY_PATH = _get_env_var("PYFLYBY_PATH", DEFAULT_PYFLYBY_PATH)
# If the old deprecated environment variables are set, then heed
# them.
if os.getenv("PYFLYBY_KNOWN_IMPORTS_PATH"):
# Use PYFLYBY_PATH as the default for
# PYFLYBY_KNOWN_IMPORTS_PATH. Note that the default is
# relevant even though we only enter this code path when the
# variable is set to anything, because the env var can
# reference "-" to include the default.
# Before pyflyby version 0.8, the default value would have
# been
# [d/"known_imports" for d in PYFLYBY_PATH]
# Instead of using that, we just use PYFLYBY_PATH directly as
# the default. This simplifies things and avoids need for a
# "known_imports=>." symlink for backwards compatibility. It
# means that ~/.pyflyby/**/*.py (as opposed to only
# ~/.pyflyby/known_imports/**/*.py) would be included.
# Although this differs slightly from the old behavior, it
# matches the behavior of the newer PYFLYBY_PATH; matching the
# new behavior seems higher utility than exactly matching the
# old behavior. Files under ~/.pyflyby/mandatory_imports will
# be included in known_imports as well, but that should not
# cause any problems.
default_path = PYFLYBY_PATH
# Expand $PYFLYBY_KNOWN_IMPORTS_PATH.
filenames = _get_python_path("PYFLYBY_KNOWN_IMPORTS_PATH",
default_path, target_dirname)
logger.debug(
"The environment variable PYFLYBY_KNOWN_IMPORTS_PATH is deprecated. "
"Use PYFLYBY_PATH.")
if os.getenv("PYFLYBY_MANDATORY_IMPORTS_PATH"):
# Compute the "default" path.
# Note that we still calculate the erstwhile default value,
# even though it's no longer the defaults, in order to still
# allow the "-" in the variable.
default_path = [
os.path.join(d, "mandatory_imports") for d in PYFLYBY_PATH
]
# Expand $PYFLYBY_MANDATORY_IMPORTS_PATH.
mandatory_imports_filenames = _get_python_path(
"PYFLYBY_MANDATORY_IMPORTS_PATH", default_path,
target_dirname)
logger.debug(
"The environment variable PYFLYBY_MANDATORY_IMPORTS_PATH is deprecated. "
"Use PYFLYBY_PATH and write __mandatory_imports__=['...'] in your files."
)
cache_keys.append((2, filenames, mandatory_imports_filenames))
try:
return cls._default_cache[cache_keys[-1]]
except KeyError:
pass
result = cls._from_filenames(filenames, mandatory_imports_filenames)
for k in cache_keys:
cls._default_cache[k] = result
return result
@classmethod
def interpret_arg(cls, arg, target_filename):
if arg is None:
return cls.get_default(target_filename)
else:
return cls(arg)
@classmethod
def _from_data(cls, known_imports, mandatory_imports, canonical_imports,
forget_imports):
self = object.__new__(cls)
self.forget_imports = ImportSet(forget_imports)
self.known_imports = ImportSet(known_imports).without_imports(
forget_imports)
self.mandatory_imports = ImportSet(mandatory_imports).without_imports(
forget_imports)
# TODO: provide more fine-grained control about canonical_imports.
self.canonical_imports = ImportMap(canonical_imports).without_imports(
forget_imports)
return self
@classmethod
def _from_args(cls, args):
# TODO: support merging input ImportDBs. For now we support
# L{PythonBlock}s and convertibles such as L{Filename}.
return cls._from_code(args)
@classmethod
def _from_code(
cls,
blocks,
_mandatory_imports_blocks_deprecated=(),
_forget_imports_blocks_deprecated=(),
):
"""
Load an import database from code.
>>> ImportDB._from_code('''
... import foo, bar as barf
... from xx import yy
... __mandatory_imports__ = ['__future__.division',
... 'import aa . bb . cc as dd']
... __forget_imports__ = ['xx.yy', 'from xx import zz']
... __canonical_imports__ = {'bad.baad': 'good.goood'}
... ''')
ImportDB('''
import bar as barf
import foo
<BLANKLINE>
__mandatory_imports__ = [
'from __future__ import division',
'from aa.bb import cc as dd',
]
<BLANKLINE>
__canonical_imports__ = {
'bad.baad': 'good.goood',
}
<BLANKLINE>
__forget_imports__ = [
'from xx import yy',
'from xx import zz',
]
''')
@rtype:
L{ImportDB}
"""
if not isinstance(blocks, (tuple, list)):
blocks = [blocks]
if not isinstance(_mandatory_imports_blocks_deprecated, (tuple, list)):
_mandatory_imports_blocks_deprecated = [
_mandatory_imports_blocks_deprecated
]
if not isinstance(_forget_imports_blocks_deprecated, (tuple, list)):
_forget_imports_blocks_deprecated = [
_forget_imports_blocks_deprecated
]
known_imports = []
mandatory_imports = []
canonical_imports = []
forget_imports = []
blocks = [PythonBlock(b) for b in blocks]
for block in blocks:
for statement in block.statements:
if statement.is_comment_or_blank:
continue
if statement.is_import:
known_imports.extend(ImportStatement(statement).imports)
continue
try:
name, value = statement.get_assignment_literal_value()
if name == "__mandatory_imports__":
mandatory_imports.append(cls._parse_import_set(value))
elif name == "__canonical_imports__":
canonical_imports.append(cls._parse_import_map(value))
elif name == "__forget_imports__":
forget_imports.append(cls._parse_import_set(value))
else:
raise ValueError(
"Unknown assignment to %r (expected one of "
"__mandatory_imports__, __canonical_imports__, "
"__forget_imports__)" % (name, ))
except ValueError as e:
raise ValueError("While parsing %s: error in %r: %s" %
(block.filename, statement, e))
for block in _mandatory_imports_blocks_deprecated:
mandatory_imports.append(ImportSet(block))
for block in _forget_imports_blocks_deprecated:
forget_imports.append(ImportSet(block))
return cls._from_data(known_imports, mandatory_imports,
canonical_imports, forget_imports)
@classmethod
def _from_filenames(cls, filenames, _mandatory_filenames_deprecated=[]):
"""
Load an import database from filenames.
This function exists to support deprecated behavior.
When we stop supporting the old behavior, we will delete this function.
@type filenames:
Sequence of L{Filename}s
@param filenames:
Filenames of files to read.
@rtype:
L{ImportDB}
"""
if not isinstance(filenames, (tuple, list)):
filenames = [filenames]
filenames = [Filename(f) for f in filenames]
logger.debug("ImportDB: loading [%s], mandatory=[%s]",
', '.join(map(str, filenames)),
', '.join(map(str, _mandatory_filenames_deprecated)))
if "SUPPORT DEPRECATED BEHAVIOR":
# Before 2014-10, pyflyby read the following:
# * known_imports from $PYFLYBY_PATH/known_imports/**/*.py or
# $PYFLYBY_KNOWN_IMPORTS_PATH/**/*.py,
# * mandatory_imports from $PYFLYBY_PATH/mandatory_imports/**/*.py or
# $PYFLYBY_MANDATORY_IMPORTS_PATH/**/*.py, and
# * forget_imports from $PYFLYBY_PATH/known_imports/**/__remove__.py
# After 2014-10, pyflyby reads the following:
# * $PYFLYBY_PATH/**/*.py
# (with directives inside the file)
# For backwards compatibility, for now we continue supporting the
# old, deprecated behavior.
blocks = []
mandatory_imports_blocks = [
Filename(f) for f in _mandatory_filenames_deprecated
]
forget_imports_blocks = []
for filename in filenames:
if filename.base == "__remove__.py":
forget_imports_blocks.append(filename)
elif "mandatory_imports" in str(filename).split("/"):
mandatory_imports_blocks.append(filename)
else:
blocks.append(filename)
return cls._from_code(blocks, mandatory_imports_blocks,
forget_imports_blocks)
else:
return cls._from_code(filenames)
@classmethod
def _parse_import_set(cls, arg):
if isinstance(arg, six.string_types):
arg = [arg]
if not isinstance(arg, (tuple, list)):
raise ValueError("Expected a list, not a %s" %
(type(arg).__name__, ))
for item in arg:
if not isinstance(item, six.string_types):
raise ValueError("Expected a list of str, not %s" %
(type(item).__name__, ))
return ImportSet(arg)
@classmethod
def _parse_import_map(cls, arg):
if isinstance(arg, six.string_types):
arg = [arg]
if not isinstance(arg, dict):
raise ValueError("Expected a dict, not a %s" %
(type(arg).__name__, ))
for k, v in arg.items():
if not isinstance(k, six.string_types):
raise ValueError("Expected a dict of str, not %s" %
(type(k).__name__, ))
if not isinstance(v, six.string_types):
raise ValueError("Expected a dict of str, not %s" %
(type(v).__name__, ))
return ImportMap(arg)
@cached_attribute
def by_fullname_or_import_as(self):
"""
Map from C{fullname} and C{import_as} to L{Import}s.
>>> import pprint
>>> db = ImportDB('from aa.bb import cc as dd')
>>> pprint.pprint(db.by_fullname_or_import_as)
{'aa': (Import('import aa'),),
'aa.bb': (Import('import aa.bb'),),
'dd': (Import('from aa.bb import cc as dd'),)}
@rtype:
C{dict} mapping from C{str} to tuple of L{Import}s
"""
# TODO: make known_imports take into account the below forget_imports,
# then move this function into ImportSet
d = defaultdict(set)
for imp in self.known_imports.imports:
# Given an import like "from foo.bar import quux as QUUX", add the
# following entries:
# - "QUUX" => "from foo.bar import quux as QUUX"
# - "foo.bar" => "import foo.bar"
# - "foo" => "import foo"
# We don't include an entry labeled "quux" because the user has
# implied he doesn't want to pollute the global namespace with
# "quux", only "QUUX".
d[imp.import_as].add(imp)
for prefix in dotted_prefixes(imp.fullname)[:-1]:
d[prefix].add(Import.from_parts(prefix, prefix))
return dict((k, tuple(sorted(v - set(self.forget_imports.imports))))
for k, v in six.iteritems(d))
def __repr__(self):
printed = self.pretty_print()
lines = "".join(" " + line for line in printed.splitlines(True))
return "%s('''\n%s''')" % (type(self).__name__, lines)
def pretty_print(self):
s = self.known_imports.pretty_print()
if self.mandatory_imports:
s += "\n__mandatory_imports__ = [\n"
for imp in self.mandatory_imports.imports:
s += " '%s',\n" % imp
s += "]\n"
if self.canonical_imports:
s += "\n__canonical_imports__ = {\n"
for k, v in sorted(self.canonical_imports.items()):
s += " '%s': '%s',\n" % (k, v)
s += "}\n"
if self.forget_imports:
s += "\n__forget_imports__ = [\n"
for imp in self.forget_imports.imports:
s += " '%s',\n" % imp
s += "]\n"
return s
def test_ImportSet_constructor_idempotent_1():
importset = ImportSet("from m1 import c, b, a")
result = ImportSet(importset)
assert result is importset
def _from_code(
cls,
blocks,
_mandatory_imports_blocks_deprecated=(),
_forget_imports_blocks_deprecated=(),
):
"""
Load an import database from code.
>>> ImportDB._from_code('''
... import foo, bar as barf
... from xx import yy
... __mandatory_imports__ = ['__future__.division',
... 'import aa . bb . cc as dd']
... __forget_imports__ = ['xx.yy', 'from xx import zz']
... __canonical_imports__ = {'bad.baad': 'good.goood'}
... ''')
ImportDB('''
import bar as barf
import foo
<BLANKLINE>
__mandatory_imports__ = [
'from __future__ import division',
'from aa.bb import cc as dd',
]
<BLANKLINE>
__canonical_imports__ = {
'bad.baad': 'good.goood',
}
<BLANKLINE>
__forget_imports__ = [
'from xx import yy',
'from xx import zz',
]
''')
@rtype:
L{ImportDB}
"""
if not isinstance(blocks, (tuple, list)):
blocks = [blocks]
if not isinstance(_mandatory_imports_blocks_deprecated, (tuple, list)):
_mandatory_imports_blocks_deprecated = [
_mandatory_imports_blocks_deprecated
]
if not isinstance(_forget_imports_blocks_deprecated, (tuple, list)):
_forget_imports_blocks_deprecated = [
_forget_imports_blocks_deprecated
]
known_imports = []
mandatory_imports = []
canonical_imports = []
forget_imports = []
blocks = [PythonBlock(b) for b in blocks]
for block in blocks:
for statement in block.statements:
if statement.is_comment_or_blank:
continue
if statement.is_import:
known_imports.extend(ImportStatement(statement).imports)
continue
try:
name, value = statement.get_assignment_literal_value()
if name == "__mandatory_imports__":
mandatory_imports.append(cls._parse_import_set(value))
elif name == "__canonical_imports__":
canonical_imports.append(cls._parse_import_map(value))
elif name == "__forget_imports__":
forget_imports.append(cls._parse_import_set(value))
else:
raise ValueError(
"Unknown assignment to %r (expected one of "
"__mandatory_imports__, __canonical_imports__, "
"__forget_imports__)" % (name, ))
except ValueError as e:
raise ValueError("While parsing %s: error in %r: %s" %
(block.filename, statement, e))
for block in _mandatory_imports_blocks_deprecated:
mandatory_imports.append(ImportSet(block))
for block in _forget_imports_blocks_deprecated:
forget_imports.append(ImportSet(block))
return cls._from_data(known_imports, mandatory_imports,
canonical_imports, forget_imports)
def preprocess(self):
self.importset = ImportSet(self.input, ignore_shadowed=True)
def test_ImportDB_from_code_1():
db = ImportDB('from aa.bb import cc as dd, ee')
expected_known = ImportSet(['from aa.bb import cc as dd, ee'])
assert db.known_imports == expected_known
def test_ImportSet_without_imports_no_action_1():
importset = ImportSet("import a, b, c, d")
result = importset.without_imports("import x, y")
assert result is importset
def test_ImportSet_conflicting_imports_2():
importset = ImportSet('import b\nfrom f import a\n')
assert importset.conflicting_imports == ()
def test_ImportSet_without_imports_1():
importset = ImportSet("import a, b, c, d")
result = importset.without_imports("import x, d, b, y")
expected = ImportSet("import a, c")
assert result == expected
def test_ImportSet_without_imports_exact_2():
importset = ImportSet(
["from m1 import a", "import m1.a", "from m1.a import *"])
result = importset.without_imports("import m1.a")
expected = ImportSet("from m1 import a; from m1.a import *")
assert result == expected
