def test_ImportSet_without_imports_from_1(): importset = ImportSet( ["from m1 import a, b", "from m2 import b, c", "from m3 import b, x"]) result = importset.without_imports("from m1 import b; from m2 import b") expected = ImportSet( ["from m1 import a", "from m2 import c", "from m3 import b, x"]) assert result == expected
def test_ImportSet_eqne_1(): s1a = ImportSet("from m1 import a, b, a, c") s1b = ImportSet("from m1 import c, b; from m1 import a as a") s2 = ImportSet("import m1.a; import m1.b; import m1.c") assert (s1a == s1b) assert not (s1a != s1b) assert (s1a != s2) assert not (s1a == s2)
def _from_data(cls, known_imports, mandatory_imports, canonical_imports, forget_imports): self = object.__new__(cls) self.forget_imports = ImportSet(forget_imports) self.known_imports = ImportSet(known_imports).without_imports( forget_imports) self.mandatory_imports = ImportSet(mandatory_imports).without_imports( forget_imports) # TODO: provide more fine-grained control about canonical_imports. self.canonical_imports = ImportMap(canonical_imports).without_imports( forget_imports) return self
def test_ImportSet_without_imports_star_dot_1(): importset = ImportSet(""" import m94165726 from m68073152 import f59136817 from .m69396491 import f87639367 from . import m81881832 from m97513719.a import f42218372 """) result = importset.without_imports("from . import *") expected = ImportSet(""" import m94165726 from m68073152 import f59136817 from m97513719.a import f42218372 """) assert result == expected
def test_ImportSet_without_imports_star_1(): importset = ImportSet(""" from m11321086.a import f27811501, f04141733 from m28630179.a import f75932565, f54328537 from m28630179.a.b import f46586889, f53411856 from m28630179.x import f10642186, f95537624 from .m28630179.a import f38714787, f42847225 """) result = importset.without_imports("from m28630179.a import *") expected = ImportSet(""" from m11321086.a import f27811501, f04141733 from m28630179.x import f10642186, f95537624 from .m28630179.a import f38714787, f42847225 """) assert result == expected
def test_ImportDB_pyflyby_forget_1(): with EnvVarCtx(PYFLYBY_PATH=".../f70301376"): d = mkdtemp("_pyflyby") os.mkdir("%s/d1" % d) os.mkdir("%s/d1/d2" % d) os.mkdir("%s/d1/d2/d3" % d) with open("%s/f70301376" % d, 'w') as f: f.write( dedent(""" from m49790901 import f27626336, f96186952 from m78687343 import f35156295, f43613649 from m78687343.a.b.c import f54583581 from m49790901.a import f27626336, f96186952 """)) with open("%s/d1/d2/f70301376" % d, 'w') as f: f.write( dedent(""" __forget_imports__ = [ 'from m49790901 import f27626336', 'from m78687343 import *', ] """)) db = ImportDB.get_default("%s/d1/d2/d3/f" % d) result = db.known_imports expected = ImportSet(""" from m49790901 import f96186952 from m49790901.a import f27626336, f96186952 """) assert result == expected rmtree(d)
def test_ImportDB_from_code_complex_1(): result = ImportDB(''' import foo, bar as barf from xx import yy, yyy, yyyy __mandatory_imports__ = ['__future__.division', 'import aa . bb . cc as dd'] __forget_imports__ = ['xx.yy', 'from xx import zz'] __canonical_imports__ = {'bad.baad': 'good.goood'} ''') assert result.known_imports == ImportSet( ["import foo", "import bar as barf", "from xx import yyy, yyyy"]) assert result.mandatory_imports == ImportSet( ["from __future__ import division", "from aa.bb import cc as dd"]) assert result.forget_imports == ImportSet( ["from xx import yy", "from xx import zz"]) assert result.canonical_imports == ImportMap({"bad.baad": "good.goood"})
def test_ImportSet_constructor_string_1(): importset = ImportSet("from m1 import c, b; from m1 import a as a") expected = [ Import("from m1 import a"), Import("from m1 import b"), Import("from m1 import c") ] assert list(importset) == expected
def test_ImportSet_constructor_list_2(): importset = ImportSet(["from m1 import c, b", "from m1 import a as a"]) expected = [ Import("from m1 import a"), Import("from m1 import b"), Import("from m1 import c") ] assert list(importset) == expected
def test_ImportSet_constructor_importstmt_1(): importset = ImportSet(ImportStatement("from m1 import a, b, c")) expected = [ Import("from m1 import a"), Import("from m1 import b"), Import("from m1 import c") ] assert list(importset) == expected
def test_ImportSet_constructor_list_imports_1(): expected = [ Import("from m1 import a"), Import("from m1 import b"), Import("from m1 import c") ] importset = ImportSet(expected) assert list(importset) == expected
def transform_imports(codeblock, transformations, params=None): """ Transform imports as specified by C{transformations}. transform_imports() perfectly replaces all imports in top-level import blocks. For the rest of the code body, transform_imports() does a crude textual string replacement. This is imperfect but handles most cases. There may be some false positives, but this is difficult to avoid. Generally we do want to do replacements even within in strings and comments. >>> result = transform_imports("from m import x", {"m.x": "m.y.z"}) >>> print result.text.joined.strip() from m.y import z as x @type codeblock: L{PythonBlock} or convertible (C{str}) @type transformations: C{dict} from C{str} to C{str} @param transformations: A map of import prefixes to replace, e.g. {"aa.bb": "xx.yy"} @rtype: L{PythonBlock} """ codeblock = PythonBlock(codeblock) params = ImportFormatParams(params) transformer = SourceToSourceFileImportsTransformation(codeblock) @memoize def transform_import(imp): # Transform a block of imports. # TODO: optimize # TODO: handle transformations containing both a.b=>x and a.b.c=>y for k, v in transformations.iteritems(): imp = imp.replace(k, v) return imp def transform_block(block): # Do a crude string replacement in the PythonBlock. block = PythonBlock(block) s = block.text.joined for k, v in transformations.iteritems(): s = re.sub("\\b%s\\b" % (re.escape(k)), v, s) return PythonBlock(s, flags=block.flags) # Loop over transformer blocks. for block in transformer.blocks: if isinstance(block, SourceToSourceImportBlockTransformation): input_imports = block.importset.imports output_imports = [transform_import(imp) for imp in input_imports] block.importset = ImportSet(output_imports, ignore_shadowed=True) else: block.output = transform_block(block.input) return transformer.output(params=params)
def test_ImportSet_contains_1(): importset = ImportSet(''' from m1 import f1 from m2 import f1 from m1 import f2 from m1 import f1, f3 import m3.m4 as m34 ''') assert Import("from m1 import f1") in importset assert Import("from .m1 import f1") not in importset assert Import("from m2 import f2") not in importset
def _parse_import_set(cls, arg): if isinstance(arg, six.string_types): arg = [arg] if not isinstance(arg, (tuple, list)): raise ValueError("Expected a list, not a %s" % (type(arg).__name__, )) for item in arg: if not isinstance(item, six.string_types): raise ValueError("Expected a list of str, not %s" % (type(item).__name__, )) return ImportSet(arg)
def test_ImportSet_member_names_1(): importset = ImportSet(''' import numpy.linalg.info from sys import exit as EXIT ''') expected = { '': ('EXIT', 'numpy', 'sys'), 'numpy': ('linalg', ), 'numpy.linalg': ('info', ), 'sys': ('exit', ) } assert importset.member_names == expected
def test_ImportSet_by_import_as_1(): importset = ImportSet(''' from a1.b1 import c1 as x from a2.b2 import c2 as x from a2.b2 import c2 as y ''') expected = { 'x': (Import('from a1.b1 import c1 as x'), Import('from a2.b2 import c2 as x')), 'y': (Import('from a2.b2 import c2 as y'), ) } assert importset.by_import_as == expected
def test_ImportSet_1(): importset = ImportSet(''' from m1 import f1 from m2 import f1 from m1 import f2 from m1 import f1, f3 import m3.m4 as m34 ''') expected = (Import("from m1 import f1"), Import("from m1 import f2"), Import("from m1 import f3"), Import("from m2 import f1"), Import("from m3 import m4 as m34")) print importset.imports assert importset.imports == expected assert len(importset) == 5
def test_ImportSet_ignore_shadowed_1(): importset = ImportSet(''' from m1 import f1, f2, f3 from m2 import f2, f4, f6 from m3 import f3, f6, f9 from m1 import f9 ''', ignore_shadowed=True) expected = ( Import("from m1 import f1"), Import("from m1 import f9"), Import("from m2 import f2"), Import("from m2 import f4"), Import("from m3 import f3"), Import("from m3 import f6"), ) print importset.imports assert importset.imports == expected assert len(importset) == 6
def exports(self): """ Get symbols exported by this module. Note that this requires involves actually importing this module, which may have side effects. (TODO: rewrite to avoid this?) @rtype: L{ImportSet} or C{None} @return: Exports, or C{None} if nothing exported. """ from pyflyby._importclns import ImportStatement, ImportSet module = self.module try: members = module.__all__ except AttributeError: members = dir(module) # Filter by non-private. members = [n for n in members if not n.startswith("_")] # Filter by definition in the module. def from_this_module(name): # TODO: could do this more robustly by parsing the AST and # looking for STOREs (definitions/assignments/etc). x = getattr(module, name) m = getattr(x, "__module__", None) if not m: return False return DottedIdentifier(m).startswith(self.name) members = [n for n in members if from_this_module(n)] else: if not all(type(s) == str for s in members): raise Exception( "Module %r contains non-string entries in __all__" % (str(self.name), )) # Filter out artificially added "deep" members. members = [n for n in members if "." not in n] if not members: return None return ImportSet([ImportStatement.from_parts(str(self.name), members)])
def test_ImportDB_from_code_1(): db = ImportDB('from aa.bb import cc as dd, ee') expected_known = ImportSet(['from aa.bb import cc as dd, ee']) assert db.known_imports == expected_known
def test_ImportSet_without_imports_no_action_1(): importset = ImportSet("import a, b, c, d") result = importset.without_imports("import x, y") assert result is importset
def replace_star_imports(codeblock, params=None): r""" Replace lines such as:: from foo.bar import * with from foo.bar import f1, f2, f3 Note that this requires involves actually importing C{foo.bar}, which may have side effects. (TODO: rewrite to avoid this?) The result includes all imports from the C{email} module. The result excludes shadowed imports. In this example: 1. The original C{MIMEAudio} import is shadowed, so it is removed. 2. The C{MIMEImage} import in the C{email} module is shadowed by a subsequent import, so it is omitted. >>> codeblock = PythonBlock('from keyword import *', filename="/tmp/x.py") >>> print replace_star_imports(codeblock) [PYFLYBY] /tmp/x.py: replaced 'from keyword import *' with 2 imports from keyword import iskeyword, kwlist <BLANKLINE> Usually you'll want to remove unused imports after replacing star imports. @type codeblock: L{PythonBlock} or convertible (C{str}) @rtype: L{PythonBlock} """ from pyflyby._modules import ModuleHandle params = ImportFormatParams(params) codeblock = PythonBlock(codeblock) filename = codeblock.filename transformer = SourceToSourceFileImportsTransformation(codeblock) for block in transformer.import_blocks: # Iterate over the import statements in C{block.input}. We do this # instead of using C{block.importset} because the latter doesn't # preserve the order of inputs. The order is important for # determining what's shadowed. imports = [ imp for s in block.input.statements for imp in ImportStatement(s).imports ] # Process "from ... import *" statements. new_imports = [] for imp in imports: if imp.split.member_name != "*": new_imports.append(imp) elif imp.split.module_name.startswith("."): # The source contains e.g. "from .foo import *". Right now we # don't have a good way to figure out the absolute module # name, so we can't get at foo. That said, there's a decent # chance that this is inside an __init__ anyway, which is one # of the few justifiable use cases for star imports in library # code. logger.warning( "%s: can't replace star imports in relative import: %s", filename, imp.pretty_print().strip()) new_imports.append(imp) else: module = ModuleHandle(imp.split.module_name) try: with ImportPathForRelativeImportsCtx(codeblock): exports = module.exports except Exception as e: logger.warning( "%s: couldn't import '%s' to enumerate exports, " "leaving unchanged: '%s'. %s: %s", filename, module.name, imp, type(e).__name__, e) new_imports.append(imp) continue if not exports: # We found nothing in the target module. This probably # means that module itself is just importing things from # other modules. Currently we intentionally exclude those # imports since usually we don't want them. TODO: do # something better here. logger.warning("%s: found nothing to import from %s, ", "leaving unchanged: '%s'", filename, module, imp) new_imports.append(imp) else: new_imports.extend(exports) logger.info("%s: replaced %r with %d imports", filename, imp.pretty_print().strip(), len(exports)) block.importset = ImportSet(new_imports, ignore_shadowed=True) return transformer.output(params=params)
def _from_code( cls, blocks, _mandatory_imports_blocks_deprecated=(), _forget_imports_blocks_deprecated=(), ): """ Load an import database from code. >>> ImportDB._from_code(''' ... import foo, bar as barf ... from xx import yy ... __mandatory_imports__ = ['__future__.division', ... 'import aa . bb . cc as dd'] ... __forget_imports__ = ['xx.yy', 'from xx import zz'] ... __canonical_imports__ = {'bad.baad': 'good.goood'} ... ''') ImportDB(''' import bar as barf import foo <BLANKLINE> __mandatory_imports__ = [ 'from __future__ import division', 'from aa.bb import cc as dd', ] <BLANKLINE> __canonical_imports__ = { 'bad.baad': 'good.goood', } <BLANKLINE> __forget_imports__ = [ 'from xx import yy', 'from xx import zz', ] ''') @rtype: L{ImportDB} """ if not isinstance(blocks, (tuple, list)): blocks = [blocks] if not isinstance(_mandatory_imports_blocks_deprecated, (tuple, list)): _mandatory_imports_blocks_deprecated = [ _mandatory_imports_blocks_deprecated ] if not isinstance(_forget_imports_blocks_deprecated, (tuple, list)): _forget_imports_blocks_deprecated = [ _forget_imports_blocks_deprecated ] known_imports = [] mandatory_imports = [] canonical_imports = [] forget_imports = [] blocks = [PythonBlock(b) for b in blocks] for block in blocks: for statement in block.statements: if statement.is_comment_or_blank: continue if statement.is_import: known_imports.extend(ImportStatement(statement).imports) continue try: name, value = statement.get_assignment_literal_value() if name == "__mandatory_imports__": mandatory_imports.append(cls._parse_import_set(value)) elif name == "__canonical_imports__": canonical_imports.append(cls._parse_import_map(value)) elif name == "__forget_imports__": forget_imports.append(cls._parse_import_set(value)) else: raise ValueError( "Unknown assignment to %r (expected one of " "__mandatory_imports__, __canonical_imports__, " "__forget_imports__)" % (name, )) except ValueError as e: raise ValueError("While parsing %s: error in %r: %s" % (block.filename, statement, e)) for block in _mandatory_imports_blocks_deprecated: mandatory_imports.append(ImportSet(block)) for block in _forget_imports_blocks_deprecated: forget_imports.append(ImportSet(block)) return cls._from_data(known_imports, mandatory_imports, canonical_imports, forget_imports)
def test_ImportSet_constructor_idempotent_1(): importset = ImportSet("from m1 import c, b, a") result = ImportSet(importset) assert result is importset
def preprocess(self): self.importset = ImportSet(self.input, ignore_shadowed=True)
def test_ImportSet_conflicting_imports_2(): importset = ImportSet('import b\nfrom f import a\n') assert importset.conflicting_imports == ()
def test_ImportSet_without_imports_1(): importset = ImportSet("import a, b, c, d") result = importset.without_imports("import x, d, b, y") expected = ImportSet("import a, c") assert result == expected
def test_ImportSet_without_imports_exact_2(): importset = ImportSet( ["from m1 import a", "import m1.a", "from m1.a import *"]) result = importset.without_imports("import m1.a") expected = ImportSet("from m1 import a; from m1.a import *") assert result == expected