def get_doctests(self):
r"""
Return doctests in this code.
>>> PythonBlock("x\n'''\n >>> foo(bar\n ... + baz)\n'''\n").get_doctests()
[PythonBlock('foo(bar\n + baz)\n', startpos=(3,2))]
:rtype:
``list`` of `PythonStatement` s
"""
import doctest
parser = doctest.DocTestParser()
doctest_blocks = []
filename = self.filename
flags = self.flags
for ast_node in self._get_docstring_nodes():
try:
examples = parser.get_examples(ast_node.s)
except Exception:
blob = ast_node.s
if len(blob) > 60:
blob = blob[:60] + '...'
# TODO: let caller decide how to handle
logger.warning("Can't parse docstring; ignoring: %r", blob)
continue
for example in examples:
lineno = ast_node.startpos.lineno + example.lineno
colno = ast_node.startpos.colno + example.indent # dubious
text = FileText(example.source,
filename=filename,
startpos=(lineno, colno))
try:
block = PythonBlock(text, flags=flags)
block.ast_node # make sure we can parse
except Exception:
blob = text.joined
if len(blob) > 60:
blob = blob[:60] + '...'
logger.warning("Can't parse doctest; ignoring: %r", blob)
continue
doctest_blocks.append(block)
return doctest_blocks
def _get_python_path(env_var_name, default_path, target_dirname):
'''
Expand an environment variable specifying pyflyby input config files.
- Default to C{default_path} if the environment variable is undefined.
- Process colon delimiters.
- Replace "-" with C{default_path}.
- Expand triple dots.
- Recursively traverse directories.
@rtype:
C{tuple} of C{Filename}s
'''
pathnames = _get_env_var(env_var_name, default_path)
if pathnames == ["EMPTY"]:
# The special code PYFLYBY_PATH=EMPTY means we intentionally want to
# use an empty PYFLYBY_PATH (and don't fall back to the default path,
# nor warn about an empty path).
return ()
for p in pathnames:
if re.match("/|[.]/|[.][.][.]/|~/", p):
continue
raise ValueError(
"{env_var_name} components should start with / or ./ or ~/ or .../. "
"Use {env_var_name}=./{p} instead of {env_var_name}={p} if you really "
"want to use the current directory.".format(
env_var_name=env_var_name, p=p))
pathnames = [os.path.expanduser(p) for p in pathnames]
pathnames = _expand_tripledots(pathnames, target_dirname)
pathnames = [Filename(fn) for fn in pathnames]
pathnames = stable_unique(pathnames)
pathnames = expand_py_files_from_args(pathnames)
if not pathnames:
logger.warning(
"No import libraries found (%s=%r, default=%r)" %
(env_var_name, os.environ.get(env_var_name), default_path))
return tuple(pathnames)
def replace_star_imports(codeblock, params=None):
r"""
Replace lines such as::
from foo.bar import *
with
from foo.bar import f1, f2, f3
Note that this requires involves actually importing C{foo.bar}, which may
have side effects. (TODO: rewrite to avoid this?)
The result includes all imports from the C{email} module. The result
excludes shadowed imports. In this example:
1. The original C{MIMEAudio} import is shadowed, so it is removed.
2. The C{MIMEImage} import in the C{email} module is shadowed by a
subsequent import, so it is omitted.
>>> codeblock = PythonBlock('from keyword import *', filename="/tmp/x.py")
>>> print replace_star_imports(codeblock)
[PYFLYBY] /tmp/x.py: replaced 'from keyword import *' with 2 imports
from keyword import iskeyword, kwlist
<BLANKLINE>
Usually you'll want to remove unused imports after replacing star imports.
@type codeblock:
L{PythonBlock} or convertible (C{str})
@rtype:
L{PythonBlock}
"""
from pyflyby._modules import ModuleHandle
params = ImportFormatParams(params)
codeblock = PythonBlock(codeblock)
filename = codeblock.filename
transformer = SourceToSourceFileImportsTransformation(codeblock)
for block in transformer.import_blocks:
# Iterate over the import statements in C{block.input}. We do this
# instead of using C{block.importset} because the latter doesn't
# preserve the order of inputs. The order is important for
# determining what's shadowed.
imports = [
imp for s in block.input.statements
for imp in ImportStatement(s).imports
]
# Process "from ... import *" statements.
new_imports = []
for imp in imports:
if imp.split.member_name != "*":
new_imports.append(imp)
elif imp.split.module_name.startswith("."):
# The source contains e.g. "from .foo import *". Right now we
# don't have a good way to figure out the absolute module
# name, so we can't get at foo. That said, there's a decent
# chance that this is inside an __init__ anyway, which is one
# of the few justifiable use cases for star imports in library
# code.
logger.warning(
"%s: can't replace star imports in relative import: %s",
filename,
imp.pretty_print().strip())
new_imports.append(imp)
else:
module = ModuleHandle(imp.split.module_name)
try:
with ImportPathForRelativeImportsCtx(codeblock):
exports = module.exports
except Exception as e:
logger.warning(
"%s: couldn't import '%s' to enumerate exports, "
"leaving unchanged: '%s'. %s: %s", filename,
module.name, imp,
type(e).__name__, e)
new_imports.append(imp)
continue
if not exports:
# We found nothing in the target module. This probably
# means that module itself is just importing things from
# other modules. Currently we intentionally exclude those
# imports since usually we don't want them. TODO: do
# something better here.
logger.warning("%s: found nothing to import from %s, ",
"leaving unchanged: '%s'", filename, module,
imp)
new_imports.append(imp)
else:
new_imports.extend(exports)
logger.info("%s: replaced %r with %d imports", filename,
imp.pretty_print().strip(), len(exports))
block.importset = ImportSet(new_imports, ignore_shadowed=True)
return transformer.output(params=params)
def fix_unused_and_missing_imports(codeblock,
add_missing=True,
remove_unused="AUTOMATIC",
add_mandatory=True,
db=None,
params=None):
r"""
Check for unused and missing imports, and fix them automatically.
Also formats imports.
In the example below, C{m1} and C{m3} are unused, so are automatically
removed. C{np} was undefined, so an C{import numpy as np} was
automatically added.
>>> codeblock = PythonBlock(
... 'from foo import m1, m2, m3, m4\n'
... 'm2, m4, np.foo', filename="/tmp/foo.py")
>>> print fix_unused_and_missing_imports(codeblock, add_mandatory=False)
[PYFLYBY] /tmp/foo.py: removed unused 'from foo import m1'
[PYFLYBY] /tmp/foo.py: removed unused 'from foo import m3'
[PYFLYBY] /tmp/foo.py: added 'import numpy as np'
import numpy as np
from foo import m2, m4
m2, m4, np.foo
@type codeblock:
L{PythonBlock} or convertible (C{str})
@rtype:
L{PythonBlock}
"""
codeblock = PythonBlock(codeblock)
if remove_unused == "AUTOMATIC":
fn = codeblock.filename
remove_unused = not (fn and (fn.base == "__init__.py"
or ".pyflyby" in str(fn).split("/")))
elif remove_unused is True or remove_unused is False:
pass
else:
raise ValueError("Invalid remove_unused=%r" % (remove_unused, ))
params = ImportFormatParams(params)
db = ImportDB.interpret_arg(db, target_filename=codeblock.filename)
# Do a first pass reformatting the imports to get rid of repeated or
# shadowed imports, e.g. L1 here:
# import foo # L1
# import foo # L2
# foo # L3
codeblock = reformat_import_statements(codeblock, params=params)
filename = codeblock.filename
transformer = SourceToSourceFileImportsTransformation(codeblock)
missing_imports, unused_imports = scan_for_import_issues(
codeblock, find_unused_imports=remove_unused, parse_docstrings=True)
logger.debug("missing_imports = %r", missing_imports)
logger.debug("unused_imports = %r", unused_imports)
if remove_unused and unused_imports:
# Go through imports to remove. [This used to be organized by going
# through import blocks and removing all relevant blocks from there,
# but if one removal caused problems the whole thing would fail. The
# CPU cost of calling without_imports() multiple times isn't worth
# that.]
# TODO: don't remove unused mandatory imports. [This isn't
# implemented yet because this isn't necessary for __future__ imports
# since they aren't reported as unused, and those are the only ones we
# have by default right now.]
for lineno, imp in unused_imports:
try:
imp = transformer.remove_import(imp, lineno)
except NoSuchImportError:
logger.error(
"%s: couldn't remove import %r",
filename,
imp,
)
except LineNumberNotFoundError as e:
logger.error("%s: unused import %r on line %d not global",
filename, str(imp), e.args[0])
else:
logger.info("%s: removed unused '%s'", filename, imp)
if add_missing and missing_imports:
missing_imports.sort(key=lambda k: (k[1], k[0]))
known = db.known_imports.by_import_as
# Decide on where to put each import to be added. Find the import
# block with the longest common prefix. Tie-break by preferring later
# blocks.
added_imports = set()
for lineno, ident in missing_imports:
import_as = ident.parts[0]
try:
imports = known[import_as]
except KeyError:
logger.warning(
"%s:%s: undefined name %r and no known import for it",
filename, lineno, import_as)
continue
if len(imports) != 1:
logger.error("%s: don't know which of %r to use", filename,
imports)
continue
imp_to_add = imports[0]
if imp_to_add in added_imports:
continue
transformer.add_import(imp_to_add, lineno)
added_imports.add(imp_to_add)
logger.info("%s: added %r", filename,
imp_to_add.pretty_print().strip())
if add_mandatory:
# Todo: allow not adding to empty __init__ files?
mandatory = db.mandatory_imports.imports
for imp in mandatory:
try:
transformer.add_import(imp)
except ImportAlreadyExistsError:
pass
else:
logger.info("%s: added mandatory %r", filename,
imp.pretty_print().strip())
return transformer.output(params=params)