def remove_broken_imports(codeblock, params=None): """ Try to execute each import, and remove the ones that don't work. Also formats imports. @type codeblock: L{PythonBlock} or convertible (C{str}) @rtype: L{PythonBlock} """ codeblock = PythonBlock(codeblock) params = ImportFormatParams(params) filename = codeblock.filename transformer = SourceToSourceFileImportsTransformation(codeblock) for block in transformer.import_blocks: broken = [] for imp in list(block.importset.imports): ns = {} try: exec_(imp.pretty_print(), ns) except Exception as e: logger.info("%s: Could not import %r; removing it: %s: %s", filename, imp.fullname, type(e).__name__, e) broken.append(imp) block.importset = block.importset.without_imports(broken) return transformer.output(params=params)
def replace_star_imports(codeblock, params=None): r""" Replace lines such as:: from foo.bar import * with from foo.bar import f1, f2, f3 Note that this requires involves actually importing C{foo.bar}, which may have side effects. (TODO: rewrite to avoid this?) The result includes all imports from the C{email} module. The result excludes shadowed imports. In this example: 1. The original C{MIMEAudio} import is shadowed, so it is removed. 2. The C{MIMEImage} import in the C{email} module is shadowed by a subsequent import, so it is omitted. >>> codeblock = PythonBlock('from keyword import *', filename="/tmp/x.py") >>> print replace_star_imports(codeblock) [PYFLYBY] /tmp/x.py: replaced 'from keyword import *' with 2 imports from keyword import iskeyword, kwlist <BLANKLINE> Usually you'll want to remove unused imports after replacing star imports. @type codeblock: L{PythonBlock} or convertible (C{str}) @rtype: L{PythonBlock} """ from pyflyby._modules import ModuleHandle params = ImportFormatParams(params) codeblock = PythonBlock(codeblock) filename = codeblock.filename transformer = SourceToSourceFileImportsTransformation(codeblock) for block in transformer.import_blocks: # Iterate over the import statements in C{block.input}. We do this # instead of using C{block.importset} because the latter doesn't # preserve the order of inputs. The order is important for # determining what's shadowed. imports = [ imp for s in block.input.statements for imp in ImportStatement(s).imports ] # Process "from ... import *" statements. new_imports = [] for imp in imports: if imp.split.member_name != "*": new_imports.append(imp) elif imp.split.module_name.startswith("."): # The source contains e.g. "from .foo import *". Right now we # don't have a good way to figure out the absolute module # name, so we can't get at foo. That said, there's a decent # chance that this is inside an __init__ anyway, which is one # of the few justifiable use cases for star imports in library # code. logger.warning( "%s: can't replace star imports in relative import: %s", filename, imp.pretty_print().strip()) new_imports.append(imp) else: module = ModuleHandle(imp.split.module_name) try: with ImportPathForRelativeImportsCtx(codeblock): exports = module.exports except Exception as e: logger.warning( "%s: couldn't import '%s' to enumerate exports, " "leaving unchanged: '%s'. %s: %s", filename, module.name, imp, type(e).__name__, e) new_imports.append(imp) continue if not exports: # We found nothing in the target module. This probably # means that module itself is just importing things from # other modules. Currently we intentionally exclude those # imports since usually we don't want them. TODO: do # something better here. logger.warning("%s: found nothing to import from %s, ", "leaving unchanged: '%s'", filename, module, imp) new_imports.append(imp) else: new_imports.extend(exports) logger.info("%s: replaced %r with %d imports", filename, imp.pretty_print().strip(), len(exports)) block.importset = ImportSet(new_imports, ignore_shadowed=True) return transformer.output(params=params)
def fix_unused_and_missing_imports(codeblock, add_missing=True, remove_unused="AUTOMATIC", add_mandatory=True, db=None, params=None): r""" Check for unused and missing imports, and fix them automatically. Also formats imports. In the example below, C{m1} and C{m3} are unused, so are automatically removed. C{np} was undefined, so an C{import numpy as np} was automatically added. >>> codeblock = PythonBlock( ... 'from foo import m1, m2, m3, m4\n' ... 'm2, m4, np.foo', filename="/tmp/foo.py") >>> print fix_unused_and_missing_imports(codeblock, add_mandatory=False) [PYFLYBY] /tmp/foo.py: removed unused 'from foo import m1' [PYFLYBY] /tmp/foo.py: removed unused 'from foo import m3' [PYFLYBY] /tmp/foo.py: added 'import numpy as np' import numpy as np from foo import m2, m4 m2, m4, np.foo @type codeblock: L{PythonBlock} or convertible (C{str}) @rtype: L{PythonBlock} """ codeblock = PythonBlock(codeblock) if remove_unused == "AUTOMATIC": fn = codeblock.filename remove_unused = not (fn and (fn.base == "__init__.py" or ".pyflyby" in str(fn).split("/"))) elif remove_unused is True or remove_unused is False: pass else: raise ValueError("Invalid remove_unused=%r" % (remove_unused, )) params = ImportFormatParams(params) db = ImportDB.interpret_arg(db, target_filename=codeblock.filename) # Do a first pass reformatting the imports to get rid of repeated or # shadowed imports, e.g. L1 here: # import foo # L1 # import foo # L2 # foo # L3 codeblock = reformat_import_statements(codeblock, params=params) filename = codeblock.filename transformer = SourceToSourceFileImportsTransformation(codeblock) missing_imports, unused_imports = scan_for_import_issues( codeblock, find_unused_imports=remove_unused, parse_docstrings=True) logger.debug("missing_imports = %r", missing_imports) logger.debug("unused_imports = %r", unused_imports) if remove_unused and unused_imports: # Go through imports to remove. [This used to be organized by going # through import blocks and removing all relevant blocks from there, # but if one removal caused problems the whole thing would fail. The # CPU cost of calling without_imports() multiple times isn't worth # that.] # TODO: don't remove unused mandatory imports. [This isn't # implemented yet because this isn't necessary for __future__ imports # since they aren't reported as unused, and those are the only ones we # have by default right now.] for lineno, imp in unused_imports: try: imp = transformer.remove_import(imp, lineno) except NoSuchImportError: logger.error( "%s: couldn't remove import %r", filename, imp, ) except LineNumberNotFoundError as e: logger.error("%s: unused import %r on line %d not global", filename, str(imp), e.args[0]) else: logger.info("%s: removed unused '%s'", filename, imp) if add_missing and missing_imports: missing_imports.sort(key=lambda k: (k[1], k[0])) known = db.known_imports.by_import_as # Decide on where to put each import to be added. Find the import # block with the longest common prefix. Tie-break by preferring later # blocks. added_imports = set() for lineno, ident in missing_imports: import_as = ident.parts[0] try: imports = known[import_as] except KeyError: logger.warning( "%s:%s: undefined name %r and no known import for it", filename, lineno, import_as) continue if len(imports) != 1: logger.error("%s: don't know which of %r to use", filename, imports) continue imp_to_add = imports[0] if imp_to_add in added_imports: continue transformer.add_import(imp_to_add, lineno) added_imports.add(imp_to_add) logger.info("%s: added %r", filename, imp_to_add.pretty_print().strip()) if add_mandatory: # Todo: allow not adding to empty __init__ files? mandatory = db.mandatory_imports.imports for imp in mandatory: try: transformer.add_import(imp) except ImportAlreadyExistsError: pass else: logger.info("%s: added mandatory %r", filename, imp.pretty_print().strip()) return transformer.output(params=params)
def _xreload_module(module, filename, force=False): """ Reload a module in place, using livepatch. @type module: C{ModuleType} @param module: Module to reload. @param force: Whether to reload even if the module has not been modified since the previous load. If C{False}, then do nothing. If C{True}, then reload. """ import linecache if not filename or not filename.endswith(".py"): # If there's no *.py source file for this module, then fallback to # built-in reload(). return reload(module) # Compare mtime of the file with the load time of the module. If the file # wasn't touched, we don't need to do anything. try: mtime = os.stat(filename).st_mtime except OSError: logger.info("Can't find %s", filename) return None if not force: try: old_loadtime = module.__loadtime__ except AttributeError: # We only have a __loadtime__ attribute if we were the ones that # loaded it. Otherwise, fall back to the process start time as a # conservative bound. old_loadtime = _PROCESS_START_TIME if old_loadtime > mtime: logger.debug( "NOT reloading %s (file %s modified %s ago but loaded %s ago)", module.__name__, filename, _format_age(mtime), _format_age(old_loadtime)) return None # Keep track of previously imported source. If the file's timestamp # was touched, but the content unchanged, we can avoid reloading. cached_lines = linecache.cache.get(filename, (None, None, None, None))[2] else: cached_lines = None # Re-read source for module from disk, and update the linecache. source = ''.join(linecache.updatecache(filename)) # Skip reload if the content didn't change. if cached_lines is not None and source == ''.join(cached_lines): logger.debug( "NOT reloading %s (file %s touched %s ago but content unchanged)", module.__name__, filename, _format_age(mtime)) return module logger.info("Reloading %s (modified %s ago) from %s", module.__name__, _format_age(mtime), filename) # Compile into AST. We do this as a separate step from compiling to byte # code so that we can get the module docstring. astnode = compile(source, filename, "exec", ast.PyCF_ONLY_AST, 1) # Get the new docstring. try: doc = astnode.body[0].value.s except (AttributeError, IndexError): doc = None # Compile into code. code = compile(astnode, filename, "exec", 0, 1) # Execute the code. We do so in a temporary namespace so that if this # fails, nothing changes. It's important to set __name__ so that relative # imports work correctly. new_mod = types.ModuleType(module.__name__) new_mod.__file__ = filename new_mod.__doc__ = doc if hasattr(module, "__path__"): new_mod.__path__ = module.__path__ MISSING = object() saved_mod = sys.modules.get(module.__name__, MISSING) try: # Temporarily put the temporary module in sys.modules, in case the # code references sys.modules[__name__] for some reason. Normally on # success, we will revert this what that was there before (which # normally should be C{module}). If an error occurs, we'll also # revert. If the user has defined a __livepatch__ hook at the module # level, it's possible for result to not be the old module. sys.modules[module.__name__] = new_mod # *** Execute new code *** exec(code, new_mod.__dict__) # Normally C{module} is of type C{ModuleType}. However, in some # cases, the module might have done a "proxy module" trick where the # module is replaced by a proxy object of some other type. Regardless # of the actual type, we do the update as C{module} were of type # C{ModuleType}. assume_type = types.ModuleType # Livepatch the module. result = livepatch(module, new_mod, module.__name__, assume_type=assume_type) sys.modules[module.__name__] = result except: # Either the module failed executing or the livepatch failed. # Revert to previous state. # Note that this isn't perfect because it's possible that the module # modified some global state in other modules. if saved_mod is MISSING: del sys.modules[module.__name__] else: sys.modules[module.__name__] = saved_mod raise # Update the time we last loaded the module. We intentionally use mtime # here instead of time.time(). If we are on NFS, it's possible for the # filer's mtime and time.time() to not be synchronized. We will be # comparing to mtime next time, so if we use only mtime, we'll be fine. module.__loadtime__ = mtime return module
def action_replace(m): if m.filename == Filename.STDIN: raise Exception("Can't replace stdio in-place") logger.info("%s: *** modified ***", m.filename) atomic_write_file(m.filename, m.output_content)
def symlink_replace(m): if m.filename == Filename.STDIN: return symlink_follow(m) if m.filename.islink: logger.info("Replacing symlink %s" % m.filename)
def symlink_skip(m): if m.filename == Filename.STDIN: return symlink_follow(m) if m.filename.islink: logger.info("Skipping symlink %s" % m.filename) raise AbortActions
def symlink_follow(m): if m.filename == Filename.STDIN: return if m.filename.islink: logger.info("Following symlink %s" % m.filename) m.filename = m.filename.realpath