def remove_broken_imports(codeblock, params=None):
"""
Try to execute each import, and remove the ones that don't work.
Also formats imports.
@type codeblock:
L{PythonBlock} or convertible (C{str})
@rtype:
L{PythonBlock}
"""
codeblock = PythonBlock(codeblock)
params = ImportFormatParams(params)
filename = codeblock.filename
transformer = SourceToSourceFileImportsTransformation(codeblock)
for block in transformer.import_blocks:
broken = []
for imp in list(block.importset.imports):
ns = {}
try:
exec_(imp.pretty_print(), ns)
except Exception as e:
logger.info("%s: Could not import %r; removing it: %s: %s",
filename, imp.fullname,
type(e).__name__, e)
broken.append(imp)
block.importset = block.importset.without_imports(broken)
return transformer.output(params=params)
def replace_star_imports(codeblock, params=None):
r"""
Replace lines such as::
from foo.bar import *
with
from foo.bar import f1, f2, f3
Note that this requires involves actually importing C{foo.bar}, which may
have side effects. (TODO: rewrite to avoid this?)
The result includes all imports from the C{email} module. The result
excludes shadowed imports. In this example:
1. The original C{MIMEAudio} import is shadowed, so it is removed.
2. The C{MIMEImage} import in the C{email} module is shadowed by a
subsequent import, so it is omitted.
>>> codeblock = PythonBlock('from keyword import *', filename="/tmp/x.py")
>>> print replace_star_imports(codeblock)
[PYFLYBY] /tmp/x.py: replaced 'from keyword import *' with 2 imports
from keyword import iskeyword, kwlist
<BLANKLINE>
Usually you'll want to remove unused imports after replacing star imports.
@type codeblock:
L{PythonBlock} or convertible (C{str})
@rtype:
L{PythonBlock}
"""
from pyflyby._modules import ModuleHandle
params = ImportFormatParams(params)
codeblock = PythonBlock(codeblock)
filename = codeblock.filename
transformer = SourceToSourceFileImportsTransformation(codeblock)
for block in transformer.import_blocks:
# Iterate over the import statements in C{block.input}. We do this
# instead of using C{block.importset} because the latter doesn't
# preserve the order of inputs. The order is important for
# determining what's shadowed.
imports = [
imp for s in block.input.statements
for imp in ImportStatement(s).imports
]
# Process "from ... import *" statements.
new_imports = []
for imp in imports:
if imp.split.member_name != "*":
new_imports.append(imp)
elif imp.split.module_name.startswith("."):
# The source contains e.g. "from .foo import *". Right now we
# don't have a good way to figure out the absolute module
# name, so we can't get at foo. That said, there's a decent
# chance that this is inside an __init__ anyway, which is one
# of the few justifiable use cases for star imports in library
# code.
logger.warning(
"%s: can't replace star imports in relative import: %s",
filename,
imp.pretty_print().strip())
new_imports.append(imp)
else:
module = ModuleHandle(imp.split.module_name)
try:
with ImportPathForRelativeImportsCtx(codeblock):
exports = module.exports
except Exception as e:
logger.warning(
"%s: couldn't import '%s' to enumerate exports, "
"leaving unchanged: '%s'. %s: %s", filename,
module.name, imp,
type(e).__name__, e)
new_imports.append(imp)
continue
if not exports:
# We found nothing in the target module. This probably
# means that module itself is just importing things from
# other modules. Currently we intentionally exclude those
# imports since usually we don't want them. TODO: do
# something better here.
logger.warning("%s: found nothing to import from %s, ",
"leaving unchanged: '%s'", filename, module,
imp)
new_imports.append(imp)
else:
new_imports.extend(exports)
logger.info("%s: replaced %r with %d imports", filename,
imp.pretty_print().strip(), len(exports))
block.importset = ImportSet(new_imports, ignore_shadowed=True)
return transformer.output(params=params)
def fix_unused_and_missing_imports(codeblock,
add_missing=True,
remove_unused="AUTOMATIC",
add_mandatory=True,
db=None,
params=None):
r"""
Check for unused and missing imports, and fix them automatically.
Also formats imports.
In the example below, C{m1} and C{m3} are unused, so are automatically
removed. C{np} was undefined, so an C{import numpy as np} was
automatically added.
>>> codeblock = PythonBlock(
... 'from foo import m1, m2, m3, m4\n'
... 'm2, m4, np.foo', filename="/tmp/foo.py")
>>> print fix_unused_and_missing_imports(codeblock, add_mandatory=False)
[PYFLYBY] /tmp/foo.py: removed unused 'from foo import m1'
[PYFLYBY] /tmp/foo.py: removed unused 'from foo import m3'
[PYFLYBY] /tmp/foo.py: added 'import numpy as np'
import numpy as np
from foo import m2, m4
m2, m4, np.foo
@type codeblock:
L{PythonBlock} or convertible (C{str})
@rtype:
L{PythonBlock}
"""
codeblock = PythonBlock(codeblock)
if remove_unused == "AUTOMATIC":
fn = codeblock.filename
remove_unused = not (fn and (fn.base == "__init__.py"
or ".pyflyby" in str(fn).split("/")))
elif remove_unused is True or remove_unused is False:
pass
else:
raise ValueError("Invalid remove_unused=%r" % (remove_unused, ))
params = ImportFormatParams(params)
db = ImportDB.interpret_arg(db, target_filename=codeblock.filename)
# Do a first pass reformatting the imports to get rid of repeated or
# shadowed imports, e.g. L1 here:
# import foo # L1
# import foo # L2
# foo # L3
codeblock = reformat_import_statements(codeblock, params=params)
filename = codeblock.filename
transformer = SourceToSourceFileImportsTransformation(codeblock)
missing_imports, unused_imports = scan_for_import_issues(
codeblock, find_unused_imports=remove_unused, parse_docstrings=True)
logger.debug("missing_imports = %r", missing_imports)
logger.debug("unused_imports = %r", unused_imports)
if remove_unused and unused_imports:
# Go through imports to remove. [This used to be organized by going
# through import blocks and removing all relevant blocks from there,
# but if one removal caused problems the whole thing would fail. The
# CPU cost of calling without_imports() multiple times isn't worth
# that.]
# TODO: don't remove unused mandatory imports. [This isn't
# implemented yet because this isn't necessary for __future__ imports
# since they aren't reported as unused, and those are the only ones we
# have by default right now.]
for lineno, imp in unused_imports:
try:
imp = transformer.remove_import(imp, lineno)
except NoSuchImportError:
logger.error(
"%s: couldn't remove import %r",
filename,
imp,
)
except LineNumberNotFoundError as e:
logger.error("%s: unused import %r on line %d not global",
filename, str(imp), e.args[0])
else:
logger.info("%s: removed unused '%s'", filename, imp)
if add_missing and missing_imports:
missing_imports.sort(key=lambda k: (k[1], k[0]))
known = db.known_imports.by_import_as
# Decide on where to put each import to be added. Find the import
# block with the longest common prefix. Tie-break by preferring later
# blocks.
added_imports = set()
for lineno, ident in missing_imports:
import_as = ident.parts[0]
try:
imports = known[import_as]
except KeyError:
logger.warning(
"%s:%s: undefined name %r and no known import for it",
filename, lineno, import_as)
continue
if len(imports) != 1:
logger.error("%s: don't know which of %r to use", filename,
imports)
continue
imp_to_add = imports[0]
if imp_to_add in added_imports:
continue
transformer.add_import(imp_to_add, lineno)
added_imports.add(imp_to_add)
logger.info("%s: added %r", filename,
imp_to_add.pretty_print().strip())
if add_mandatory:
# Todo: allow not adding to empty __init__ files?
mandatory = db.mandatory_imports.imports
for imp in mandatory:
try:
transformer.add_import(imp)
except ImportAlreadyExistsError:
pass
else:
logger.info("%s: added mandatory %r", filename,
imp.pretty_print().strip())
return transformer.output(params=params)
def _xreload_module(module, filename, force=False):
"""
Reload a module in place, using livepatch.
@type module:
C{ModuleType}
@param module:
Module to reload.
@param force:
Whether to reload even if the module has not been modified since the
previous load. If C{False}, then do nothing. If C{True}, then reload.
"""
import linecache
if not filename or not filename.endswith(".py"):
# If there's no *.py source file for this module, then fallback to
# built-in reload().
return reload(module)
# Compare mtime of the file with the load time of the module. If the file
# wasn't touched, we don't need to do anything.
try:
mtime = os.stat(filename).st_mtime
except OSError:
logger.info("Can't find %s", filename)
return None
if not force:
try:
old_loadtime = module.__loadtime__
except AttributeError:
# We only have a __loadtime__ attribute if we were the ones that
# loaded it. Otherwise, fall back to the process start time as a
# conservative bound.
old_loadtime = _PROCESS_START_TIME
if old_loadtime > mtime:
logger.debug(
"NOT reloading %s (file %s modified %s ago but loaded %s ago)",
module.__name__, filename, _format_age(mtime),
_format_age(old_loadtime))
return None
# Keep track of previously imported source. If the file's timestamp
# was touched, but the content unchanged, we can avoid reloading.
cached_lines = linecache.cache.get(filename,
(None, None, None, None))[2]
else:
cached_lines = None
# Re-read source for module from disk, and update the linecache.
source = ''.join(linecache.updatecache(filename))
# Skip reload if the content didn't change.
if cached_lines is not None and source == ''.join(cached_lines):
logger.debug(
"NOT reloading %s (file %s touched %s ago but content unchanged)",
module.__name__, filename, _format_age(mtime))
return module
logger.info("Reloading %s (modified %s ago) from %s", module.__name__,
_format_age(mtime), filename)
# Compile into AST. We do this as a separate step from compiling to byte
# code so that we can get the module docstring.
astnode = compile(source, filename, "exec", ast.PyCF_ONLY_AST, 1)
# Get the new docstring.
try:
doc = astnode.body[0].value.s
except (AttributeError, IndexError):
doc = None
# Compile into code.
code = compile(astnode, filename, "exec", 0, 1)
# Execute the code. We do so in a temporary namespace so that if this
# fails, nothing changes. It's important to set __name__ so that relative
# imports work correctly.
new_mod = types.ModuleType(module.__name__)
new_mod.__file__ = filename
new_mod.__doc__ = doc
if hasattr(module, "__path__"):
new_mod.__path__ = module.__path__
MISSING = object()
saved_mod = sys.modules.get(module.__name__, MISSING)
try:
# Temporarily put the temporary module in sys.modules, in case the
# code references sys.modules[__name__] for some reason. Normally on
# success, we will revert this what that was there before (which
# normally should be C{module}). If an error occurs, we'll also
# revert. If the user has defined a __livepatch__ hook at the module
# level, it's possible for result to not be the old module.
sys.modules[module.__name__] = new_mod
# *** Execute new code ***
exec(code, new_mod.__dict__)
# Normally C{module} is of type C{ModuleType}. However, in some
# cases, the module might have done a "proxy module" trick where the
# module is replaced by a proxy object of some other type. Regardless
# of the actual type, we do the update as C{module} were of type
# C{ModuleType}.
assume_type = types.ModuleType
# Livepatch the module.
result = livepatch(module,
new_mod,
module.__name__,
assume_type=assume_type)
sys.modules[module.__name__] = result
except:
# Either the module failed executing or the livepatch failed.
# Revert to previous state.
# Note that this isn't perfect because it's possible that the module
# modified some global state in other modules.
if saved_mod is MISSING:
del sys.modules[module.__name__]
else:
sys.modules[module.__name__] = saved_mod
raise
# Update the time we last loaded the module. We intentionally use mtime
# here instead of time.time(). If we are on NFS, it's possible for the
# filer's mtime and time.time() to not be synchronized. We will be
# comparing to mtime next time, so if we use only mtime, we'll be fine.
module.__loadtime__ = mtime
return module
def action_replace(m):
if m.filename == Filename.STDIN:
raise Exception("Can't replace stdio in-place")
logger.info("%s: *** modified ***", m.filename)
atomic_write_file(m.filename, m.output_content)
def symlink_replace(m):
if m.filename == Filename.STDIN:
return symlink_follow(m)
if m.filename.islink:
logger.info("Replacing symlink %s" % m.filename)
def symlink_skip(m):
if m.filename == Filename.STDIN:
return symlink_follow(m)
if m.filename.islink:
logger.info("Skipping symlink %s" % m.filename)
raise AbortActions
def symlink_follow(m):
if m.filename == Filename.STDIN:
return
if m.filename.islink:
logger.info("Following symlink %s" % m.filename)
m.filename = m.filename.realpath
