def remove_broken_imports(codeblock, params=None):
"""
Try to execute each import, and remove the ones that don't work.
Also formats imports.
@type codeblock:
L{PythonBlock} or convertible (C{str})
@rtype:
L{PythonBlock}
"""
codeblock = PythonBlock(codeblock)
params = ImportFormatParams(params)
filename = codeblock.filename
transformer = SourceToSourceFileImportsTransformation(codeblock)
for block in transformer.import_blocks:
broken = []
for imp in list(block.importset.imports):
ns = {}
try:
exec_(imp.pretty_print(), ns)
except Exception as e:
logger.info("%s: Could not import %r; removing it: %s: %s",
filename, imp.fullname,
type(e).__name__, e)
broken.append(imp)
block.importset = block.importset.without_imports(broken)
return transformer.output(params=params)
def reformat_import_statements(codeblock, params=None):
r"""
Reformat each top-level block of import statements within a block of code.
Blank lines, comments, etc. are left alone and separate blocks of imports.
Parse the entire code block into an ast, group into consecutive import
statements and other lines. Each import block consists entirely of
'import' (or 'from ... import') statements. Other lines, including blanks
and comment lines, are not touched.
>>> print reformat_import_statements(
... 'from foo import bar2 as bar2x, bar1\n'
... 'import foo.bar3 as bar3x\n'
... 'import foo.bar4\n'
... '\n'
... 'import foo.bar0 as bar0\n').text.joined
import foo.bar4
from foo import bar1, bar2 as bar2x, bar3 as bar3x
<BLANKLINE>
from foo import bar0
<BLANKLINE>
@type codeblock:
L{PythonBlock} or convertible (C{str})
@type params:
L{ImportFormatParams}
@rtype:
L{PythonBlock}
"""
params = ImportFormatParams(params)
transformer = SourceToSourceFileImportsTransformation(codeblock)
return transformer.output(params=params)
def transform_imports(codeblock, transformations, params=None):
"""
Transform imports as specified by C{transformations}.
transform_imports() perfectly replaces all imports in top-level import
blocks.
For the rest of the code body, transform_imports() does a crude textual
string replacement. This is imperfect but handles most cases. There may
be some false positives, but this is difficult to avoid. Generally we do
want to do replacements even within in strings and comments.
>>> result = transform_imports("from m import x", {"m.x": "m.y.z"})
>>> print result.text.joined.strip()
from m.y import z as x
@type codeblock:
L{PythonBlock} or convertible (C{str})
@type transformations:
C{dict} from C{str} to C{str}
@param transformations:
A map of import prefixes to replace, e.g. {"aa.bb": "xx.yy"}
@rtype:
L{PythonBlock}
"""
codeblock = PythonBlock(codeblock)
params = ImportFormatParams(params)
transformer = SourceToSourceFileImportsTransformation(codeblock)
@memoize
def transform_import(imp):
# Transform a block of imports.
# TODO: optimize
# TODO: handle transformations containing both a.b=>x and a.b.c=>y
for k, v in transformations.iteritems():
imp = imp.replace(k, v)
return imp
def transform_block(block):
# Do a crude string replacement in the PythonBlock.
block = PythonBlock(block)
s = block.text.joined
for k, v in transformations.iteritems():
s = re.sub("\\b%s\\b" % (re.escape(k)), v, s)
return PythonBlock(s, flags=block.flags)
# Loop over transformer blocks.
for block in transformer.blocks:
if isinstance(block, SourceToSourceImportBlockTransformation):
input_imports = block.importset.imports
output_imports = [transform_import(imp) for imp in input_imports]
block.importset = ImportSet(output_imports, ignore_shadowed=True)
else:
block.output = transform_block(block.input)
return transformer.output(params=params)
def canonicalize_imports(codeblock, params=None, db=None):
"""
Transform C{codeblock} as specified by C{__canonical_imports__} in the
global import library.
@type codeblock:
L{PythonBlock} or convertible (C{str})
@rtype:
L{PythonBlock}
"""
codeblock = PythonBlock(codeblock)
params = ImportFormatParams(params)
db = ImportDB.interpret_arg(db, target_filename=codeblock.filename)
transformations = db.canonical_imports
return transform_imports(codeblock, transformations, params=params)
def canonicalize_imports(codeblock, params=None, db=None):
"""
Transform ``codeblock`` as specified by ``__canonical_imports__`` in the
global import library.
:type codeblock:
`PythonBlock` or convertible (``str``)
:rtype:
`PythonBlock`
"""
codeblock = PythonBlock(codeblock)
params = ImportFormatParams(params)
db = ImportDB.interpret_arg(db, target_filename=codeblock.filename)
transformations = db.canonical_imports
return transform_imports(codeblock, transformations, params=params)
def pretty_print(self, params=None, allow_conflicts=False):
"""
Pretty-print a block of import statements into a single string.
@type params:
L{ImportFormatParams}
@rtype:
C{str}
"""
params = ImportFormatParams(params)
# TODO: instead of complaining about conflicts, just filter out the
# shadowed imports at construction time.
if not allow_conflicts and self.conflicting_imports:
raise ConflictingImportsError(
"Refusing to pretty-print because of conflicting imports: " +
'; '.join("%r imported as %r" %
([imp.fullname for imp in self.by_import_as[i]], i)
for i in self.conflicting_imports))
from_spaces = max(1, params.from_spaces)
def do_align(statement):
return statement.fromname != '__future__' or params.align_future
def pp(statement, import_column):
if do_align(statement):
return statement.pretty_print(params=params,
import_column=import_column,
from_spaces=from_spaces)
else:
return statement.pretty_print(params=params,
import_column=None,
from_spaces=1)
statements = self.get_statements(
separate_from_imports=params.separate_from_imports)
def isint(x):
return isinstance(x, int) and not isinstance(x, bool)
if not statements:
import_column = None
elif isinstance(params.align_imports, bool):
if params.align_imports:
fromimp_stmts = [
s for s in statements if s.fromname and do_align(s)
]
if fromimp_stmts:
import_column = (
max(len(s.fromname)
for s in fromimp_stmts) + from_spaces + 5)
else:
import_column = None
else:
import_column = None
elif isinstance(params.align_imports, int):
import_column = params.align_imports
elif isinstance(params.align_imports, (tuple, list, set)):
# If given a set of candidate alignment columns, then try each
# alignment column and pick the one that yields the fewest number
# of output lines.
if not all(isinstance(x, int) for x in params.align_imports):
raise TypeError("expected set of integers; got %r" %
(params.align_imports, ))
candidates = sorted(set(params.align_imports))
if len(candidates) == 0:
raise ValueError(
"list of zero candidate alignment columns specified")
elif len(candidates) == 1:
# Optimization.
import_column = next(iter(candidates))
else:
def argmin(map):
items = iter(sorted(map.items()))
min_k, min_v = next(items)
for k, v in items:
if v < min_v:
min_k = k
min_v = v
return min_k
def count_lines(import_column):
return sum(
s.pretty_print(params=params,
import_column=import_column,
from_spaces=from_spaces).count("\n")
for s in statements)
# Construct a map from alignment column to total number of
# lines.
col2length = dict((c, count_lines(c)) for c in candidates)
# Pick the column that yields the fewest lines. Break ties by
# picking the smaller column.
import_column = argmin(col2length)
else:
raise TypeError(
"ImportSet.pretty_print(): unexpected params.align_imports type %s"
% (type(params.align_imports).__name__, ))
return ''.join(
pp(statement, import_column) for statement in statements)
def pretty_print(self, params=None):
params = ImportFormatParams(params)
return self.importset.pretty_print(params)
def replace_star_imports(codeblock, params=None):
r"""
Replace lines such as::
from foo.bar import *
with
from foo.bar import f1, f2, f3
Note that this requires involves actually importing C{foo.bar}, which may
have side effects. (TODO: rewrite to avoid this?)
The result includes all imports from the C{email} module. The result
excludes shadowed imports. In this example:
1. The original C{MIMEAudio} import is shadowed, so it is removed.
2. The C{MIMEImage} import in the C{email} module is shadowed by a
subsequent import, so it is omitted.
>>> codeblock = PythonBlock('from keyword import *', filename="/tmp/x.py")
>>> print replace_star_imports(codeblock)
[PYFLYBY] /tmp/x.py: replaced 'from keyword import *' with 2 imports
from keyword import iskeyword, kwlist
<BLANKLINE>
Usually you'll want to remove unused imports after replacing star imports.
@type codeblock:
L{PythonBlock} or convertible (C{str})
@rtype:
L{PythonBlock}
"""
from pyflyby._modules import ModuleHandle
params = ImportFormatParams(params)
codeblock = PythonBlock(codeblock)
filename = codeblock.filename
transformer = SourceToSourceFileImportsTransformation(codeblock)
for block in transformer.import_blocks:
# Iterate over the import statements in C{block.input}. We do this
# instead of using C{block.importset} because the latter doesn't
# preserve the order of inputs. The order is important for
# determining what's shadowed.
imports = [
imp for s in block.input.statements
for imp in ImportStatement(s).imports
]
# Process "from ... import *" statements.
new_imports = []
for imp in imports:
if imp.split.member_name != "*":
new_imports.append(imp)
elif imp.split.module_name.startswith("."):
# The source contains e.g. "from .foo import *". Right now we
# don't have a good way to figure out the absolute module
# name, so we can't get at foo. That said, there's a decent
# chance that this is inside an __init__ anyway, which is one
# of the few justifiable use cases for star imports in library
# code.
logger.warning(
"%s: can't replace star imports in relative import: %s",
filename,
imp.pretty_print().strip())
new_imports.append(imp)
else:
module = ModuleHandle(imp.split.module_name)
try:
with ImportPathForRelativeImportsCtx(codeblock):
exports = module.exports
except Exception as e:
logger.warning(
"%s: couldn't import '%s' to enumerate exports, "
"leaving unchanged: '%s'. %s: %s", filename,
module.name, imp,
type(e).__name__, e)
new_imports.append(imp)
continue
if not exports:
# We found nothing in the target module. This probably
# means that module itself is just importing things from
# other modules. Currently we intentionally exclude those
# imports since usually we don't want them. TODO: do
# something better here.
logger.warning("%s: found nothing to import from %s, ",
"leaving unchanged: '%s'", filename, module,
imp)
new_imports.append(imp)
else:
new_imports.extend(exports)
logger.info("%s: replaced %r with %d imports", filename,
imp.pretty_print().strip(), len(exports))
block.importset = ImportSet(new_imports, ignore_shadowed=True)
return transformer.output(params=params)
def fix_unused_and_missing_imports(codeblock,
add_missing=True,
remove_unused="AUTOMATIC",
add_mandatory=True,
db=None,
params=None):
r"""
Check for unused and missing imports, and fix them automatically.
Also formats imports.
In the example below, C{m1} and C{m3} are unused, so are automatically
removed. C{np} was undefined, so an C{import numpy as np} was
automatically added.
>>> codeblock = PythonBlock(
... 'from foo import m1, m2, m3, m4\n'
... 'm2, m4, np.foo', filename="/tmp/foo.py")
>>> print fix_unused_and_missing_imports(codeblock, add_mandatory=False)
[PYFLYBY] /tmp/foo.py: removed unused 'from foo import m1'
[PYFLYBY] /tmp/foo.py: removed unused 'from foo import m3'
[PYFLYBY] /tmp/foo.py: added 'import numpy as np'
import numpy as np
from foo import m2, m4
m2, m4, np.foo
@type codeblock:
L{PythonBlock} or convertible (C{str})
@rtype:
L{PythonBlock}
"""
codeblock = PythonBlock(codeblock)
if remove_unused == "AUTOMATIC":
fn = codeblock.filename
remove_unused = not (fn and (fn.base == "__init__.py"
or ".pyflyby" in str(fn).split("/")))
elif remove_unused is True or remove_unused is False:
pass
else:
raise ValueError("Invalid remove_unused=%r" % (remove_unused, ))
params = ImportFormatParams(params)
db = ImportDB.interpret_arg(db, target_filename=codeblock.filename)
# Do a first pass reformatting the imports to get rid of repeated or
# shadowed imports, e.g. L1 here:
# import foo # L1
# import foo # L2
# foo # L3
codeblock = reformat_import_statements(codeblock, params=params)
filename = codeblock.filename
transformer = SourceToSourceFileImportsTransformation(codeblock)
missing_imports, unused_imports = scan_for_import_issues(
codeblock, find_unused_imports=remove_unused, parse_docstrings=True)
logger.debug("missing_imports = %r", missing_imports)
logger.debug("unused_imports = %r", unused_imports)
if remove_unused and unused_imports:
# Go through imports to remove. [This used to be organized by going
# through import blocks and removing all relevant blocks from there,
# but if one removal caused problems the whole thing would fail. The
# CPU cost of calling without_imports() multiple times isn't worth
# that.]
# TODO: don't remove unused mandatory imports. [This isn't
# implemented yet because this isn't necessary for __future__ imports
# since they aren't reported as unused, and those are the only ones we
# have by default right now.]
for lineno, imp in unused_imports:
try:
imp = transformer.remove_import(imp, lineno)
except NoSuchImportError:
logger.error(
"%s: couldn't remove import %r",
filename,
imp,
)
except LineNumberNotFoundError as e:
logger.error("%s: unused import %r on line %d not global",
filename, str(imp), e.args[0])
else:
logger.info("%s: removed unused '%s'", filename, imp)
if add_missing and missing_imports:
missing_imports.sort(key=lambda k: (k[1], k[0]))
known = db.known_imports.by_import_as
# Decide on where to put each import to be added. Find the import
# block with the longest common prefix. Tie-break by preferring later
# blocks.
added_imports = set()
for lineno, ident in missing_imports:
import_as = ident.parts[0]
try:
imports = known[import_as]
except KeyError:
logger.warning(
"%s:%s: undefined name %r and no known import for it",
filename, lineno, import_as)
continue
if len(imports) != 1:
logger.error("%s: don't know which of %r to use", filename,
imports)
continue
imp_to_add = imports[0]
if imp_to_add in added_imports:
continue
transformer.add_import(imp_to_add, lineno)
added_imports.add(imp_to_add)
logger.info("%s: added %r", filename,
imp_to_add.pretty_print().strip())
if add_mandatory:
# Todo: allow not adding to empty __init__ files?
mandatory = db.mandatory_imports.imports
for imp in mandatory:
try:
transformer.add_import(imp)
except ImportAlreadyExistsError:
pass
else:
logger.info("%s: added mandatory %r", filename,
imp.pretty_print().strip())
return transformer.output(params=params)
def pretty_print(self, params=None):
params = ImportFormatParams(params)
result = [block.pretty_print(params=params) for block in self.blocks]
return FileText.concatenate(result)
def parse_args(addopts=None, import_format_params=False, modify_action_params=False):
"""
Do setup for a top-level script and parse arguments.
"""
### Setup.
# Register a SIGPIPE handler.
signal.signal(signal.SIGPIPE, _sigpipe_handler)
### Parse args.
parser = optparse.OptionParser(usage='\n'+maindoc())
def log_level_callbacker(level):
def callback(option, opt_str, value, parser):
logger.set_level(level)
return callback
def debug_callback(option, opt_str, value, parser):
logger.set_level("DEBUG")
parser.add_option("--debug", action="callback",
callback=debug_callback,
help="Debug mode (noisy and fail fast).")
parser.add_option("--verbose", action="callback",
callback=log_level_callbacker("DEBUG"),
help="Be noisy.")
parser.add_option("--quiet", action="callback",
callback=log_level_callbacker("ERROR"),
help="Be quiet.")
parser.add_option("--version", action="callback",
callback=lambda *args: print_version_and_exit(),
help="Print pyflyby version and exit.")
if modify_action_params:
group = optparse.OptionGroup(parser, "Action options")
action_diff = action_external_command('pyflyby-diff')
def parse_action(v):
V = v.strip().upper()
if V == 'PRINT':
return action_print
elif V == 'REPLACE':
return action_replace
elif V == 'QUERY':
return action_query()
elif V == "DIFF":
return action_diff
elif V.startswith("QUERY:"):
return action_query(v[6:])
elif V.startswith("EXECUTE:"):
return action_external_command(v[8:])
elif V == "IFCHANGED":
return action_ifchanged
else:
raise Exception(
"Bad argument %r to --action; "
"expected PRINT or REPLACE or QUERY or IFCHANGED "
"or EXECUTE:..." % (v,))
def set_actions(actions):
actions = tuple(actions)
parser.values.actions = actions
def action_callback(option, opt_str, value, parser):
action_args = value.split(',')
set_actions([parse_action(v) for v in action_args])
def action_callbacker(actions):
def callback(option, opt_str, value, parser):
set_actions(actions)
return callback
group.add_option(
"--actions", type='string', action='callback',
callback=action_callback,
metavar='PRINT|REPLACE|IFCHANGED|QUERY|DIFF|EXECUTE:mycommand',
help=hfmt('''
Comma-separated list of action(s) to take. If PRINT, print
the changed file to stdout. If REPLACE, then modify the
file in-place. If EXECUTE:mycommand, then execute
'mycommand oldfile tmpfile'. If DIFF, then execute
'pyflyby-diff'. If QUERY, then query user to continue.
If IFCHANGED, then continue actions only if file was
changed.'''))
group.add_option(
"--print", "-p", action='callback',
callback=action_callbacker([action_print]),
help=hfmt('''
Equivalent to --action=PRINT (default when stdin or stdout is
not a tty) '''))
group.add_option(
"--diff", "-d", action='callback',
callback=action_callbacker([action_diff]),
help=hfmt('''Equivalent to --action=DIFF'''))
group.add_option(
"--replace", "-r", action='callback',
callback=action_callbacker([action_ifchanged, action_replace]),
help=hfmt('''Equivalent to --action=IFCHANGED,REPLACE'''))
group.add_option(
"--diff-replace", "-R", action='callback',
callback=action_callbacker([action_ifchanged, action_diff, action_replace]),
help=hfmt('''Equivalent to --action=IFCHANGED,DIFF,REPLACE'''))
actions_interactive = [
action_ifchanged, action_diff,
action_query("Replace {filename}?"), action_replace]
group.add_option(
"--interactive", "-i", action='callback',
callback=action_callbacker(actions_interactive),
help=hfmt('''
Equivalent to --action=IFCHANGED,DIFF,QUERY,REPLACE (default
when stdin & stdout are ttys) '''))
if os.isatty(0) and os.isatty(1):
default_actions = actions_interactive
else:
default_actions = [action_print]
parser.set_default('actions', tuple(default_actions))
parser.add_option_group(group)
if import_format_params:
group = optparse.OptionGroup(parser, "Pretty-printing options")
group.add_option('--align-imports', '--align', type='str', default="32",
metavar='N',
help=hfmt('''
Whether and how to align the 'import' keyword in
'from modulename import aliases...'. If 0, then
don't align. If 1, then align within each block
of imports. If an integer > 1, then align at
that column, wrapping with a backslash if
necessary. If a comma-separated list of integers
(tab stops), then pick the column that results in
the fewest number of lines total per block.'''))
group.add_option('--from-spaces', type='int', default=3, metavar='N',
help=hfmt('''
The number of spaces after the 'from' keyword.
(Must be at least 1; default is 3.)'''))
group.add_option('--separate-from-imports', action='store_true',
default=False,
help=hfmt('''
Separate 'from ... import ...'
statements from 'import ...' statements.'''))
group.add_option('--no-separate-from-imports', action='store_false',
dest='separate_from_imports',
help=hfmt('''
(Default) Don't separate 'from ... import ...'
statements from 'import ...' statements.'''))
group.add_option('--align-future', action='store_true',
default=False,
help=hfmt('''
Align the 'from __future__ import ...' statement
like others.'''))
group.add_option('--no-align-future', action='store_false',
dest='align_future',
help=hfmt('''
(Default) Don't align the 'from __future__ import
...' statement.'''))
group.add_option('--width', type='int', default=79, metavar='N',
help=hfmt('''
Maximum line length (default: 79).'''))
group.add_option('--hanging-indent', type='choice', default='never',
choices=['never','auto','always'],
metavar='never|auto|always',
dest='hanging_indent',
help=hfmt('''
How to wrap import statements that don't fit on
one line.
If --hanging-indent=always, then always indent
imported tokens at column 4 on the next line.
If --hanging-indent=never (default), then align
import tokens after "import (" (by default column
40); do so even if some symbols are so long that
this would exceed the width (by default 79)).
If --hanging-indent=auto, then use hanging indent
only if it is necessary to prevent exceeding the
width (by default 79).
'''))
def uniform_callback(option, opt_str, value, parser):
parser.values.separate_from_imports = False
parser.values.from_spaces = 3
parser.values.align_imports = '32'
group.add_option('--uniform', '-u', action="callback",
callback=uniform_callback,
help=hfmt('''
(Default) Shortcut for --no-separate-from-imports
--from-spaces=3 --align-imports=32.'''))
def unaligned_callback(option, opt_str, value, parser):
parser.values.separate_from_imports = True
parser.values.from_spaces = 1
parser.values.align_imports = '0'
group.add_option('--unaligned', '-n', action="callback",
callback=unaligned_callback,
help=hfmt('''
Shortcut for --separate-from-imports
--from-spaces=1 --align-imports=0.'''))
parser.add_option_group(group)
if addopts is not None:
addopts(parser)
options, args = parser.parse_args()
if import_format_params:
align_imports_args = [int(x.strip())
for x in options.align_imports.split(",")]
if len(align_imports_args) == 1 and align_imports_args[0] == 1:
align_imports = True
elif len(align_imports_args) == 1 and align_imports_args[0] == 0:
align_imports = False
else:
align_imports = tuple(sorted(set(align_imports_args)))
options.params = ImportFormatParams(
align_imports =align_imports,
from_spaces =options.from_spaces,
separate_from_imports =options.separate_from_imports,
max_line_length =options.width,
align_future =options.align_future,
hanging_indent =options.hanging_indent,
)
return options, args