def containing(cls, identifier):
"""
Try to find the module that defines a name such as C{a.b.c} by trying
to import C{a}, C{a.b}, and C{a.b.c}.
@return:
The name of the 'deepest' module (most commonly it would be C{a.b}
in this example).
@rtype:
L{Module}
"""
# In the code below we catch "Exception" rather than just ImportError
# or AttributeError since importing and __getattr__ing can raise other
# exceptions.
identifier = DottedIdentifier(identifier)
try:
module = ModuleHandle(identifier[:1])
result = module.module
except Exception as e:
raise ImportError(e)
for part, prefix in zip(identifier, prefixes(identifier))[1:]:
try:
result = getattr(result, str(part))
except Exception:
try:
module = cls(prefix)
result = module.module
except Exception as e:
raise ImportError(e)
else:
if isinstance(result, types.ModuleType):
module = cls(result)
logger.debug("Imported %r to get %r", module, identifier)
return module
def initialize_comms():
if in_jupyter():
for target in pyflyby_comm_targets:
_register_target(target)
from ipykernel.comm import Comm
comm = Comm(target_name=INIT_COMMS)
msg = {"type": INIT_COMMS}
logger.debug("Requesting frontend to (re-)initialize comms")
comm.send(msg)
def action(m):
bindir = os.path.dirname(os.path.realpath(sys.argv[0]))
env = os.environ
env['PATH'] = env['PATH'] + ":" + bindir
fullcmd = "%s %s %s" % (
command, m.input_content_filename, m.output_content_filename)
logger.debug("Executing external command: %s", fullcmd)
ret = subprocess.call(fullcmd, shell=True, env=env)
logger.debug("External command returned %d", ret)
def send_comm_message(target_name, msg):
if in_jupyter():
try:
comm = comms[target_name]
except KeyError:
logger.debug("Comm with target_name " + target_name +
" hasn't been opened")
else:
# Help the frontend distinguish between multiple types
# of custom comm messages
msg["type"] = target_name
comm.send(msg)
logger.debug("Sending comm message for target " + target_name)
def _from_filenames(cls, filenames, _mandatory_filenames_deprecated=[]):
"""
Load an import database from filenames.
This function exists to support deprecated behavior.
When we stop supporting the old behavior, we will delete this function.
@type filenames:
Sequence of L{Filename}s
@param filenames:
Filenames of files to read.
@rtype:
L{ImportDB}
"""
if not isinstance(filenames, (tuple, list)):
filenames = [filenames]
filenames = [Filename(f) for f in filenames]
logger.debug("ImportDB: loading [%s], mandatory=[%s]",
', '.join(map(str, filenames)),
', '.join(map(str, _mandatory_filenames_deprecated)))
if "SUPPORT DEPRECATED BEHAVIOR":
# Before 2014-10, pyflyby read the following:
# * known_imports from $PYFLYBY_PATH/known_imports/**/*.py or
# $PYFLYBY_KNOWN_IMPORTS_PATH/**/*.py,
# * mandatory_imports from $PYFLYBY_PATH/mandatory_imports/**/*.py or
# $PYFLYBY_MANDATORY_IMPORTS_PATH/**/*.py, and
# * forget_imports from $PYFLYBY_PATH/known_imports/**/__remove__.py
# After 2014-10, pyflyby reads the following:
# * $PYFLYBY_PATH/**/*.py
# (with directives inside the file)
# For backwards compatibility, for now we continue supporting the
# old, deprecated behavior.
blocks = []
mandatory_imports_blocks = [
Filename(f) for f in _mandatory_filenames_deprecated
]
forget_imports_blocks = []
for filename in filenames:
if filename.base == "__remove__.py":
forget_imports_blocks.append(filename)
elif "mandatory_imports" in str(filename).split("/"):
mandatory_imports_blocks.append(filename)
else:
blocks.append(filename)
return cls._from_code(blocks, mandatory_imports_blocks,
forget_imports_blocks)
else:
return cls._from_code(filenames)
def in_jupyter():
from IPython.core.getipython import get_ipython
ip = get_ipython()
if ip is None:
logger.debug("get_ipython() doesn't exist. Comm targets can only"
"be added in an Jupyter notebook/lab/console environment")
return False
else:
try:
ip.kernel.comm_manager
except AttributeError:
logger.debug("Comm targets can only be added in Jupyter "
"notebook/lab/console environment")
return False
else:
return True
def unadvise(self):
if self._wrapped is None:
return
cur = self._container.get(self._name, _UNSET)
if cur is self._wrapped:
from pyflyby._log import logger
logger.debug("unadvising %s", self._qname)
if self._previous is _UNSET:
del self._container[self._name]
else:
self._container[self._name] = self._previous
elif cur == self._previous:
pass
else:
from pyflyby._log import logger
logger.debug("%s seems modified; not unadvising it", self._name)
self._wrapped = None
def clear_default_cache(cls):
"""
Clear the class cache of default ImportDBs.
Subsequent calls to ImportDB.get_default() will not reuse previously
cached results. Existing ImportDB instances are not affected by this
call.
"""
if cls._default_cache:
if logger.debug_enabled:
allpyfiles = set()
for tup in cls._default_cache:
if tup[0] != 2:
continue
for tup2 in tup[1:]:
for f in tup2:
assert isinstance(f, Filename)
if f.ext == '.py':
allpyfiles.add(f)
nfiles = len(allpyfiles)
logger.debug("ImportDB: Clearing default cache of %d files",
nfiles)
cls._default_cache.clear()
def add_module(self, module):
"""
Adds ``module`` and recreates the DocIndex with the updated set of
modules.
:return:
Whether anything was added.
"""
module = ModuleHandle(module)
for prefix in module.ancestors:
if prefix in self.modules:
# The module, or a prefix of it, was already added.
return False
for existing_module in sorted(self.modules):
if existing_module.startswith(module):
# This supersedes an existing module.
assert existing_module != module
self.modules.remove(existing_module)
logger.debug("Expanding docindex to include %r", module)
self.modules.add(module)
del self.docindex
return True
def advise(self, hook, once=False):
from pyflyby._log import logger
self._previous = self._container.get(self._name, _UNSET)
if once and getattr(self._previous, "__aspect__", None):
# TODO: check that it's the same hook - at least check the name.
logger.debug("already advised %s", self._qname)
return None
logger.debug("advising %s", self._qname)
assert self._previous is _UNSET or self._previous == self._original
assert self._wrapped is None
# Create the wrapped function.
wrapped = FunctionWithGlobals(hook, __original__=self._original)
wrapped.__joinpoint__ = self._joinpoint
wrapped.__original__ = self._original
wrapped.__name__ = "%s__advised__%s" % (self._name, hook.__name__)
wrapped.__doc__ = "%s.\n\nAdvice %s:\n%s" % (
self._original.__doc__, hook.__name__, hook.__doc__)
wrapped.__aspect__ = self
if self._wrapper is not None:
wrapped = self._wrapper(wrapped)
self._wrapped = wrapped
# Install the wrapped function!
self._container[self._name] = wrapped
return self
def import_module(module_name):
module_name = str(module_name)
logger.debug("Importing %r", module_name)
try:
result = __import__(module_name, fromlist=['dummy'])
if result.__name__ != module_name:
logger.debug("Note: import_module(%r).__name__ == %r", module_name,
result.__name__)
return result
except ImportError as e:
# We got an ImportError. Figure out whether this is due to the module
# not existing, or whether the module exists but caused an ImportError
# (perhaps due to trying to import another problematic module).
# Do this by looking at the exception traceback. If the previous
# frame in the traceback is this function (because locals match), then
# it should be the internal import machinery reporting that the module
# doesn't exist. Re-raise the exception as-is.
# If some sys.meta_path or other import hook isn't compatible with
# such a check, here are some things we could do:
# - Use pkgutil.find_loader() after the fact to check if the module
# is supposed to exist. Note that we shouldn't rely solely on
# this before attempting to import, because find_loader() doesn't
# work with meta_path.
# - Write a memoized global function that compares in the current
# environment the difference between attempting to import a
# non-existent module vs a problematic module, and returns a
# function that uses the working discriminators.
real_importerror1 = type(e) is ImportError
real_importerror2 = (sys.exc_info()[2].tb_frame.f_locals is locals())
m = re.match("^No module named (.*)$", str(e))
real_importerror3 = (m and m.group(1) == module_name
or module_name.endswith("." + m.group(1)))
logger.debug("import_module(%r): real ImportError: %s %s %s",
module_name, real_importerror1, real_importerror2,
real_importerror3)
if real_importerror1 and real_importerror2 and real_importerror3:
raise
reraise(
ErrorDuringImportError(
"Error while attempting to import %s: %s: %s" %
(module_name, type(e).__name__, e)), None,
sys.exc_info()[2])
except Exception as e:
reraise(
ErrorDuringImportError(
"Error while attempting to import %s: %s: %s" %
(module_name, type(e).__name__, e)), None,
sys.exc_info()[2])
def remove_comms():
for target_name, comm in six.iteritems(comms):
comm.close()
logger.debug("Closing comm for " + target_name)
def fix_unused_and_missing_imports(codeblock,
add_missing=True,
remove_unused="AUTOMATIC",
add_mandatory=True,
db=None,
params=None):
r"""
Check for unused and missing imports, and fix them automatically.
Also formats imports.
In the example below, C{m1} and C{m3} are unused, so are automatically
removed. C{np} was undefined, so an C{import numpy as np} was
automatically added.
>>> codeblock = PythonBlock(
... 'from foo import m1, m2, m3, m4\n'
... 'm2, m4, np.foo', filename="/tmp/foo.py")
>>> print fix_unused_and_missing_imports(codeblock, add_mandatory=False)
[PYFLYBY] /tmp/foo.py: removed unused 'from foo import m1'
[PYFLYBY] /tmp/foo.py: removed unused 'from foo import m3'
[PYFLYBY] /tmp/foo.py: added 'import numpy as np'
import numpy as np
from foo import m2, m4
m2, m4, np.foo
@type codeblock:
L{PythonBlock} or convertible (C{str})
@rtype:
L{PythonBlock}
"""
codeblock = PythonBlock(codeblock)
if remove_unused == "AUTOMATIC":
fn = codeblock.filename
remove_unused = not (fn and (fn.base == "__init__.py"
or ".pyflyby" in str(fn).split("/")))
elif remove_unused is True or remove_unused is False:
pass
else:
raise ValueError("Invalid remove_unused=%r" % (remove_unused, ))
params = ImportFormatParams(params)
db = ImportDB.interpret_arg(db, target_filename=codeblock.filename)
# Do a first pass reformatting the imports to get rid of repeated or
# shadowed imports, e.g. L1 here:
# import foo # L1
# import foo # L2
# foo # L3
codeblock = reformat_import_statements(codeblock, params=params)
filename = codeblock.filename
transformer = SourceToSourceFileImportsTransformation(codeblock)
missing_imports, unused_imports = scan_for_import_issues(
codeblock, find_unused_imports=remove_unused, parse_docstrings=True)
logger.debug("missing_imports = %r", missing_imports)
logger.debug("unused_imports = %r", unused_imports)
if remove_unused and unused_imports:
# Go through imports to remove. [This used to be organized by going
# through import blocks and removing all relevant blocks from there,
# but if one removal caused problems the whole thing would fail. The
# CPU cost of calling without_imports() multiple times isn't worth
# that.]
# TODO: don't remove unused mandatory imports. [This isn't
# implemented yet because this isn't necessary for __future__ imports
# since they aren't reported as unused, and those are the only ones we
# have by default right now.]
for lineno, imp in unused_imports:
try:
imp = transformer.remove_import(imp, lineno)
except NoSuchImportError:
logger.error(
"%s: couldn't remove import %r",
filename,
imp,
)
except LineNumberNotFoundError as e:
logger.error("%s: unused import %r on line %d not global",
filename, str(imp), e.args[0])
else:
logger.info("%s: removed unused '%s'", filename, imp)
if add_missing and missing_imports:
missing_imports.sort(key=lambda k: (k[1], k[0]))
known = db.known_imports.by_import_as
# Decide on where to put each import to be added. Find the import
# block with the longest common prefix. Tie-break by preferring later
# blocks.
added_imports = set()
for lineno, ident in missing_imports:
import_as = ident.parts[0]
try:
imports = known[import_as]
except KeyError:
logger.warning(
"%s:%s: undefined name %r and no known import for it",
filename, lineno, import_as)
continue
if len(imports) != 1:
logger.error("%s: don't know which of %r to use", filename,
imports)
continue
imp_to_add = imports[0]
if imp_to_add in added_imports:
continue
transformer.add_import(imp_to_add, lineno)
added_imports.add(imp_to_add)
logger.info("%s: added %r", filename,
imp_to_add.pretty_print().strip())
if add_mandatory:
# Todo: allow not adding to empty __init__ files?
mandatory = db.mandatory_imports.imports
for imp in mandatory:
try:
transformer.add_import(imp)
except ImportAlreadyExistsError:
pass
else:
logger.info("%s: added mandatory %r", filename,
imp.pretty_print().strip())
return transformer.output(params=params)
def _xreload_module(module, filename, force=False):
"""
Reload a module in place, using livepatch.
@type module:
C{ModuleType}
@param module:
Module to reload.
@param force:
Whether to reload even if the module has not been modified since the
previous load. If C{False}, then do nothing. If C{True}, then reload.
"""
import linecache
if not filename or not filename.endswith(".py"):
# If there's no *.py source file for this module, then fallback to
# built-in reload().
return reload(module)
# Compare mtime of the file with the load time of the module. If the file
# wasn't touched, we don't need to do anything.
try:
mtime = os.stat(filename).st_mtime
except OSError:
logger.info("Can't find %s", filename)
return None
if not force:
try:
old_loadtime = module.__loadtime__
except AttributeError:
# We only have a __loadtime__ attribute if we were the ones that
# loaded it. Otherwise, fall back to the process start time as a
# conservative bound.
old_loadtime = _PROCESS_START_TIME
if old_loadtime > mtime:
logger.debug(
"NOT reloading %s (file %s modified %s ago but loaded %s ago)",
module.__name__, filename, _format_age(mtime),
_format_age(old_loadtime))
return None
# Keep track of previously imported source. If the file's timestamp
# was touched, but the content unchanged, we can avoid reloading.
cached_lines = linecache.cache.get(filename,
(None, None, None, None))[2]
else:
cached_lines = None
# Re-read source for module from disk, and update the linecache.
source = ''.join(linecache.updatecache(filename))
# Skip reload if the content didn't change.
if cached_lines is not None and source == ''.join(cached_lines):
logger.debug(
"NOT reloading %s (file %s touched %s ago but content unchanged)",
module.__name__, filename, _format_age(mtime))
return module
logger.info("Reloading %s (modified %s ago) from %s", module.__name__,
_format_age(mtime), filename)
# Compile into AST. We do this as a separate step from compiling to byte
# code so that we can get the module docstring.
astnode = compile(source, filename, "exec", ast.PyCF_ONLY_AST, 1)
# Get the new docstring.
try:
doc = astnode.body[0].value.s
except (AttributeError, IndexError):
doc = None
# Compile into code.
code = compile(astnode, filename, "exec", 0, 1)
# Execute the code. We do so in a temporary namespace so that if this
# fails, nothing changes. It's important to set __name__ so that relative
# imports work correctly.
new_mod = types.ModuleType(module.__name__)
new_mod.__file__ = filename
new_mod.__doc__ = doc
if hasattr(module, "__path__"):
new_mod.__path__ = module.__path__
MISSING = object()
saved_mod = sys.modules.get(module.__name__, MISSING)
try:
# Temporarily put the temporary module in sys.modules, in case the
# code references sys.modules[__name__] for some reason. Normally on
# success, we will revert this what that was there before (which
# normally should be C{module}). If an error occurs, we'll also
# revert. If the user has defined a __livepatch__ hook at the module
# level, it's possible for result to not be the old module.
sys.modules[module.__name__] = new_mod
# *** Execute new code ***
exec(code, new_mod.__dict__)
# Normally C{module} is of type C{ModuleType}. However, in some
# cases, the module might have done a "proxy module" trick where the
# module is replaced by a proxy object of some other type. Regardless
# of the actual type, we do the update as C{module} were of type
# C{ModuleType}.
assume_type = types.ModuleType
# Livepatch the module.
result = livepatch(module,
new_mod,
module.__name__,
assume_type=assume_type)
sys.modules[module.__name__] = result
except:
# Either the module failed executing or the livepatch failed.
# Revert to previous state.
# Note that this isn't perfect because it's possible that the module
# modified some global state in other modules.
if saved_mod is MISSING:
del sys.modules[module.__name__]
else:
sys.modules[module.__name__] = saved_mod
raise
# Update the time we last loaded the module. We intentionally use mtime
# here instead of time.time(). If we are on NFS, it's possible for the
# filer's mtime and time.time() to not be synchronized. We will be
# comparing to mtime next time, so if we use only mtime, we'll be fine.
module.__loadtime__ = mtime
return module
def action_ifchanged(m):
if m.output_content.joined == m.input_content.joined:
logger.debug("unmodified: %s", m.filename)
raise AbortActions
def get_default(cls, target_filename):
"""
Return the default import library for the given target filename.
This will read various .../.pyflyby files as specified by
$PYFLYBY_PATH as well as older deprecated environment variables.
Memoized.
@param target_filename:
The target filename for which to get the import database. Note that
the target filename itself is not read. Instead, the target
filename is relevant because we look for .../.pyflyby based on the
target filename.
@rtype:
L{ImportDB}
"""
# We're going to canonicalize target_filenames in a number of steps.
# At each step, see if we've seen the input so far. We do the cache
# checking incrementally since the steps involve syscalls. Since this
# is going to potentially be executed inside the IPython interactive
# loop, we cache as much as possible.
# TODO: Consider refreshing periodically. Check if files have
# been touched, and if so, return new data. Check file timestamps at
# most once every 60 seconds.
cache_keys = []
target_filename = Filename(target_filename or ".")
if target_filename.startswith("/dev"):
target_filename = Filename(".")
target_dirname = target_filename
# TODO: with StatCache
while True:
cache_keys.append((1, target_dirname, os.getenv("PYFLYBY_PATH"),
os.getenv("PYFLYBY_KNOWN_IMPORTS_PATH"),
os.getenv("PYFLYBY_MANDATORY_IMPORTS_PATH")))
try:
return cls._default_cache[cache_keys[-1]]
except KeyError:
pass
if target_dirname.isdir:
break
target_dirname = target_dirname.dir
target_dirname = target_dirname.real
if target_dirname != cache_keys[-1][0]:
cache_keys.append((1, target_dirname, os.getenv("PYFLYBY_PATH"),
os.getenv("PYFLYBY_KNOWN_IMPORTS_PATH"),
os.getenv("PYFLYBY_MANDATORY_IMPORTS_PATH")))
try:
return cls._default_cache[cache_keys[-1]]
except KeyError:
pass
DEFAULT_PYFLYBY_PATH = []
etc_dir = _find_etc_dir()
if etc_dir:
DEFAULT_PYFLYBY_PATH.append(str(etc_dir))
DEFAULT_PYFLYBY_PATH += [
".../.pyflyby",
"~/.pyflyby",
]
logger.debug("DEFAULT_PYFLYBY_PATH=%s", DEFAULT_PYFLYBY_PATH)
filenames = _get_python_path("PYFLYBY_PATH", DEFAULT_PYFLYBY_PATH,
target_dirname)
mandatory_imports_filenames = ()
if "SUPPORT DEPRECATED BEHAVIOR":
PYFLYBY_PATH = _get_env_var("PYFLYBY_PATH", DEFAULT_PYFLYBY_PATH)
# If the old deprecated environment variables are set, then heed
# them.
if os.getenv("PYFLYBY_KNOWN_IMPORTS_PATH"):
# Use PYFLYBY_PATH as the default for
# PYFLYBY_KNOWN_IMPORTS_PATH. Note that the default is
# relevant even though we only enter this code path when the
# variable is set to anything, because the env var can
# reference "-" to include the default.
# Before pyflyby version 0.8, the default value would have
# been
# [d/"known_imports" for d in PYFLYBY_PATH]
# Instead of using that, we just use PYFLYBY_PATH directly as
# the default. This simplifies things and avoids need for a
# "known_imports=>." symlink for backwards compatibility. It
# means that ~/.pyflyby/**/*.py (as opposed to only
# ~/.pyflyby/known_imports/**/*.py) would be included.
# Although this differs slightly from the old behavior, it
# matches the behavior of the newer PYFLYBY_PATH; matching the
# new behavior seems higher utility than exactly matching the
# old behavior. Files under ~/.pyflyby/mandatory_imports will
# be included in known_imports as well, but that should not
# cause any problems.
default_path = PYFLYBY_PATH
# Expand $PYFLYBY_KNOWN_IMPORTS_PATH.
filenames = _get_python_path("PYFLYBY_KNOWN_IMPORTS_PATH",
default_path, target_dirname)
logger.debug(
"The environment variable PYFLYBY_KNOWN_IMPORTS_PATH is deprecated. "
"Use PYFLYBY_PATH.")
if os.getenv("PYFLYBY_MANDATORY_IMPORTS_PATH"):
# Compute the "default" path.
# Note that we still calculate the erstwhile default value,
# even though it's no longer the defaults, in order to still
# allow the "-" in the variable.
default_path = [
os.path.join(d, "mandatory_imports") for d in PYFLYBY_PATH
]
# Expand $PYFLYBY_MANDATORY_IMPORTS_PATH.
mandatory_imports_filenames = _get_python_path(
"PYFLYBY_MANDATORY_IMPORTS_PATH", default_path,
target_dirname)
logger.debug(
"The environment variable PYFLYBY_MANDATORY_IMPORTS_PATH is deprecated. "
"Use PYFLYBY_PATH and write __mandatory_imports__=['...'] in your files."
)
cache_keys.append((2, filenames, mandatory_imports_filenames))
try:
return cls._default_cache[cache_keys[-1]]
except KeyError:
pass
result = cls._from_filenames(filenames, mandatory_imports_filenames)
for k in cache_keys:
cls._default_cache[k] = result
return result