def list_imports():
import optparse
parser = optparse.OptionParser(__doc__.strip())
parser.add_option(
'-I',
'--ignore',
dest='ignores',
action='append',
default=def_ignores,
help="Add the given directory name to the list to be ignored.")
parser.add_option(
'-u',
'--unified',
action='store_true',
help="Just output the unique set of dependencies found, "
"in no particular order, without the filenames. The default "
"is to output all imports, in order of appearance, along with "
"the filename and line number.")
parser.add_option('-v',
'--verbose',
action='count',
default=0,
help="Output input lines as well.")
opts, args = parser.parse_args()
setup_logging(opts.verbose)
if not args:
logging.warning("Searching for files from root directory.")
args = ['.']
info = logging.info
if opts.unified:
all_symnames = set()
for fn in iter_pyfiles(args, opts.ignores):
all_symnames.update(
x[0] for x in find_imports(fn, opts.verbose, opts.ignores))
for symname in sorted(all_symnames):
print(symname)
else:
for fn in iter_pyfiles(args, opts.ignores):
if opts.verbose:
lines = list(open(fn, 'rU'))
for symname, lineno, islocal in find_imports(
fn, opts.verbose, opts.ignores):
print('%s:%d: %s' % (fn, lineno, symname))
if opts.verbose:
for no in range(lineno - 1, len(lines)):
l = lines[no].rstrip()
print(' %s' % l)
if l[-1] != '\\':
break
print()
def list_imports():
import optparse
parser = optparse.OptionParser(__doc__.strip())
parser.add_option('-I', '--ignore', dest='ignores', action='append',
default=def_ignores,
help="Add the given directory name to the list to be ignored.")
parser.add_option('-u', '--unified', action='store_true',
help="Just output the unique set of dependencies found, "
"in no particular order, without the filenames. The default "
"is to output all imports, in order of appearance, along with "
"the filename and line number.")
parser.add_option('-v', '--verbose', action='count', default=0,
help="Output input lines as well.")
opts, args = parser.parse_args()
setup_logging(opts.verbose)
if not args:
logging.warning("Searching for files from root directory.")
args = ['.']
info = logging.info
if opts.unified:
all_symnames = set()
for fn in iter_pyfiles(args, opts.ignores):
all_symnames.update(x[0] for x in
find_imports(fn, opts.verbose, opts.ignores))
for symname in sorted(all_symnames):
print_(symname)
else:
for fn in iter_pyfiles(args, opts.ignores):
if opts.verbose:
lines = list(open(fn, 'rU'))
for symname, lineno, islocal in find_imports(fn,
opts.verbose,
opts.ignores):
print_('%s:%d: %s' % (fn, lineno, symname))
if opts.verbose:
for no in xrange(lineno-1, len(lines)):
l = lines[no].rstrip()
print_(' %s' % l)
if l[-1] != '\\':
break
print_()
def preload_packages(source_paths, ignores=None):
all_package_roots = find_package_paths(source_paths, ignores)
cache = {}
for package_path in all_package_roots:
pyfiles = iter_pyfiles([package_path], [], False)
for fn in pyfiles:
cache_package(fn, package_path)
cache[fn] = package_path
return cache
def files(self):
"""
:return: files under input package
"""
return [
found_file for found_file in iter_pyfiles(
[self.package_path], None, abspaths=False)
if not self._is_ignored(found_file)
]
def load_dependencies(self):
"""
Find all the dependencies.
Taken from snakefood/gendeps.py
"""
fiter = iter_pyfiles(self.files, self.ignores, False)
in_roots = find_roots(self.files, self.ignores)
for fn in fiter:
if fn in self.processed_files:
continue # Make sure we process each file only once.
self.processed_files.add(fn)
if is_python(fn):
files, errors = finder.find_dependencies(
fn, verbose=False, process_pragmas=True, ignore_unused=True)
self.all_errors.extend(errors)
else:
files = []
if basename(fn) == '__init__.py':
fn = dirname(fn)
self._add_dependencies(fn, in_roots, files)
def main():
import optparse
parser = optparse.OptionParser(__doc__.strip())
parser.add_option('--debug', action='store_true', help="Debugging output.")
parser.add_option(
'-I',
'--ignore',
dest='ignores',
action='append',
default=def_ignores,
help="Add the given directory name to the list to be ignored.")
parser.add_option(
'-d',
'--disable-pragmas',
action='store_false',
dest='do_pragmas',
default=True,
help="Disable processing of pragma directives as strings after imports."
)
parser.add_option(
'-D',
'--duplicates',
'--enable-duplicates',
dest='do_dups',
action='store_true',
help="Enable experimental heuristic for finding duplicate imports.")
parser.add_option(
'-M',
'--missing',
'--enable-missing',
dest='do_missing',
action='store_true',
help="Enable experimental heuristic for finding missing imports.")
opts, args = parser.parse_args()
write = sys.stderr.write
for fn in iter_pyfiles(args or ['.'], opts.ignores, False):
# Parse the file.
ast, lines = parse_python_source(fn)
if ast is None:
continue
found_imports = get_ast_imports(ast)
# Check for duplicate remote names imported.
if opts.do_dups:
found_imports, dups = check_duplicate_imports(found_imports)
for modname, rname, lname, lineno, level, pragma in dups:
write("%s:%d: Duplicate import '%s'\n" % (fn, lineno, lname))
# Filter out the unused imports.
used_imports, unused_imports = filter_unused_imports(
ast, found_imports)
# Output warnings for the unused imports.
for x in unused_imports:
_, _, lname, lineno, _, pragma = x
if opts.do_pragmas and pragma:
continue
# Search for the column in the relevant line.
mo = re.search(r'\b%s\b' % lname, lines[lineno - 1])
colno = 0
if mo:
colno = mo.start() + 1
write("%s:%d:%d: Unused import '%s'\n" %
(fn, lineno, colno, lname))
# (Optionally) Compute the list of names that are being assigned to.
if opts.do_missing or opts.debug:
vis = AssignVisitor()
ast.walk(ast, vis)
assign_names = vis.finalize()
# (Optionally) Check for potentially missing imports (this cannot be
# precise, we are only providing a heuristic here).
if opts.do_missing:
defined = set(modname for modname, _, _, _, _, _ in used_imports)
defined.update(x[0] for x in assign_names)
_, simple_names = get_names_from_ast(ast)
for name, lineno in simple_names:
if name not in defined and name not in builtins.__dict__:
write("%s:%d: Missing import for '%s'\n" %
(fn, lineno, name))
# Print out all the schmoo for debugging.
if opts.debug:
print_()
print_()
print_('------ Imported names:')
for modname, rname, lname, lineno, level, pragma in found_imports:
print_('%s:%d: %s' % (fn, lineno, lname))
## print_()
## print_()
## print_('------ Exported names:')
## for name, lineno in exported:
## print_('%s:%d: %s' % (fn, lineno, name))
## print_()
## print_()
## print_('------ Used names:')
## for name, lineno in dotted_names:
## print_('%s:%d: %s' % (fn, lineno, name))
## print_()
print_()
print_()
print_('------ Assigned names:')
for name, lineno in assign_names:
print_('%s:%d: %s' % (fn, lineno, name))
print_()
print_()
print_('------ AST:')
printAst(ast, indent=' ', stream=sys.stdout, initlevel=1)
print_()
def fetch_dependencies(self):
"""
Fetch all dependencies and follow the target file.
This was inspired by the snakefood library
snakefood-1.4-py2.7.egg/snakefood/gendeps.py
"""
# No need to run this twice
if self.dependency_list: return self.dependency_list
log.info("Fetching internal dependecies: %s" % self.filename)
depends = find_imports(self.filename, 1, 0)
# Get the list of package roots for our input files and prepend them to the
# module search path to insure localized imports.
inroots = find_roots([self.filename], [])
self.file_roots = inroots
if not inroots:
raise NoRoot
for file in inroots:
log.debug("Root found: %s" % file)
sys.path = inroots + sys.path
#log.debug("Using the following import path to search for modules:")
#for dn in sys.path:
# log.debug(" -- %s" % dn)
inroots = frozenset(inroots)
# Find all the dependencies.
log.debug("Processing file:")
allfiles = defaultdict(set)
allerrors = []
processed_files = set()
ignorefiles = []
alldependencies = []
fiter = iter_pyfiles([self.filename], ignorefiles, False)
while 1:
newfiles = set()
for fn in fiter:
log.debug(" post-filter: %s" % fn)
processed_files.add(fn)
if is_python(fn):
files, errors = find_dependencies(fn, 0, 0)
log.debug("dependency file count: %d" % len(files))
allerrors.extend(errors)
else:
# If the file is not a source file, we don't know how to get the
# dependencies of that (without importing, which we want to
# avoid).
files = []
# When packages are the source of dependencies, remove the __init__
# file. This is important because the targets also do not include the
# __init__ (i.e. when "from <package> import <subpackage>" is seen).
if basename(fn) == '__init__.py':
fn = dirname(fn)
# no dependency.
from_ = relfile(fn, ignorefiles)
if from_ is None:
log.debug("from_ empty. Move on")
continue
infrom = from_[0] in inroots
log.debug(" from: %s" % from_[0])
log.debug(" file: %s" % from_[1])
allfiles[from_].add((None, None))
# Add the dependencies.
for dfn in files:
xfn = dfn
if basename(xfn) == '__init__.py':
xfn = dirname(xfn)
to_ = relfile(xfn, ignorefiles)
into = to_[0] in inroots
log.debug(" from: %s" % from_[1])
log.debug(" to: %s" % to_[1])
if dfn in alldependencies:
log.debug("Already added %s to dependency list" % dfn)
else:
log.debug("Add %s to dependency list" % dfn)
allfiles[from_].add(to_)
newfiles.add(dfn)
alldependencies.append(dfn)
if not newfiles:
log.debug("No more new files. all done")
break
else:
fiter = iter(sorted(newfiles))
# Output the list of roots found.
log.debug("Found roots:")
found_roots = set()
for key, files in allfiles.iteritems():
found_roots.add(key[0])
found_roots.update(map(itemgetter(0), files))
if None in found_roots:
found_roots.remove(None)
for root in sorted(found_roots):
log.debug(" %s" % root)
self.dependency_list = allfiles
return self.dependency_list
def scan(self):
"""
Returns an ImportGraph
"""
self.optsVerbose -= self.optsQuiet
setup_logging(self.optsVerbose)
info = logging.info
warning = logging.warning
debug = logging.debug
if self.optsInternal and self.optsExternal:
message = "Using --internal and --external at the same time " \
"does not make sense."
raise SnakefoodScannerException(message)
if self.optsPrintRoots:
inroots = find_roots(self.args, self.optsIgnores)
for dn in sorted(inroots):
print(dn)
return
info("")
info("Input paths:")
for arg in self.args:
fn = os.path.realpath(arg)
info(' {}'.format(fn))
if not os.path.exists(fn):
message = "Filename '{}' does not exist.".format(fn)
raise SnakefoodScannerException(message)
# Get the list of package roots for our input files and prepend
# them to the module search path to insure localized imports.
inroots = find_roots(self.args, self.optsIgnores)
if (self.optsInternal or self.optsExternal) and not inroots:
message = "No package roots found from the given files or " \
"directories. Using --internal with these roots will " \
"generate no dependencies."
raise SnakefoodScannerException(message)
info("")
info("Roots of the input files:")
for root in inroots:
info(' {}'.format(root))
info("")
info("Using the following import path to search for modules:")
sys.path = inroots + sys.path
for dn in sys.path:
info(" {}".format(dn))
inroots = frozenset(inroots)
# Find all the dependencies.
info("")
info("Processing files:")
info("")
allfiles = defaultdict(set)
allerrors = []
processed_files = set()
fiter = iter_pyfiles(self.args, self.optsIgnores, False)
while 1:
newfiles = set()
for fn in fiter:
if fn in processed_files:
continue # Make sure we process each file only once.
info(" {}".format(fn))
processed_files.add(fn)
if is_python(fn):
files, errors = find_dependencies(
fn, self.optsVerbose,
self.optsDoPragmas, self.optsVerbose)
allerrors.extend(errors)
else:
# If the file is not a source file, we don't know how
# to get the dependencies of that (without importing,
# which we want to avoid).
files = []
# When packages are the source of dependencies, remove the
# __init__ file. This is important because the targets
# also do not include the __init__ (i.e. when "from
# <package> import <subpackage>" is seen).
if os.path.basename(fn) == '__init__.py':
fn = os.path.dirname(fn)
# Make sure all the files at least appear in the output,
# even if it has no dependency.
from_ = relfile(fn, self.optsIgnores)
if from_ is None:
continue
infrom = from_[0] in inroots
if self.optsInternal and not infrom:
continue
if not self.optsExternal:
allfiles[from_].add((None, None))
# Add the dependencies.
for dfn in files:
xfn = dfn
if os.path.basename(xfn) == '__init__.py':
xfn = os.path.dirname(xfn)
to_ = relfile(xfn, self.optsIgnores)
into = to_[0] in inroots
if (self.optsInternal and not into) or \
(self.optsExternal and into):
continue
allfiles[from_].add(to_)
newfiles.add(dfn)
if not (self.optsFollow and newfiles):
break
else:
fiter = iter(sorted(newfiles))
# If internal is used twice, we filter down further the
# dependencies to the set of files that were processed only,
# not just to the files that live in the same roots.
if self.optsInternal >= 2:
filtfiles = type(allfiles)()
for from_, tolist in allfiles.iteritems():
filtfiles[from_] = set(
x for x in tolist if x in allfiles or x == (None, None))
allfiles = filtfiles
info("")
info("SUMMARY")
info("=======")
# Output a list of the symbols that could not
# be imported as modules.
reports = [
("Modules that were ignored because not used:",
ERROR_UNUSED, info),
("Modules that could not be imported:",
ERROR_IMPORT, warning),
]
if self.optsVerbose >= 2:
reports.append(
("Symbols that could not be imported as modules:",
ERROR_SYMBOL, debug))
for msg, errtype, efun in reports:
names = set(name for (err, name) in allerrors if err is errtype)
if names:
efun("")
efun(msg)
for name in sorted(names):
efun(" {}".format(name))
# Output the list of roots found.
info("")
info("Found roots:")
foundRoots = set()
for key, files in allfiles.iteritems():
foundRoots.add(key[0])
foundRoots.update(map(operator.itemgetter(0), files))
if None in foundRoots:
foundRoots.remove(None)
for root in sorted(foundRoots):
info(" {}".format(root))
# Output the dependencies.
entries = SnakefoodEntries()
info("")
for (from_root, from_), targets in sorted(
allfiles.iteritems(), key=operator.itemgetter(0)):
for to_root, to_ in sorted(targets):
entry = SnakefoodEntry(from_root, from_, to_root, to_)
entries.append(entry)
graph = ImportGraph()
for entry in entries.iterEntries():
graph.addEntry(entry)
return graph
def main():
import optparse
parser = optparse.OptionParser(__doc__.strip())
parser.add_option('--debug', action='store_true',
help="Debugging output.")
parser.add_option('-I', '--ignore', dest='ignores', action='append',
default=def_ignores,
help="Add the given directory name to the list to be ignored.")
parser.add_option('-d', '--disable-pragmas', action='store_false',
dest='do_pragmas', default=True,
help="Disable processing of pragma directives as strings after imports.")
parser.add_option('-D', '--duplicates', '--enable-duplicates',
dest='do_dups', action='store_true',
help="Enable experimental heuristic for finding duplicate imports.")
parser.add_option('-M', '--missing', '--enable-missing',
dest='do_missing', action='store_true',
help="Enable experimental heuristic for finding missing imports.")
opts, args = parser.parse_args()
write = sys.stderr.write
for fn in iter_pyfiles(args or ['.'], opts.ignores, False):
# Parse the file.
ast, lines = parse_python_source(fn)
if ast is None:
continue
found_imports = get_ast_imports(ast)
# Check for duplicate remote names imported.
if opts.do_dups:
found_imports, dups = check_duplicate_imports(found_imports)
for modname, rname, lname, lineno, level, pragma in dups:
write("%s:%d: Duplicate import '%s'\n" % (fn, lineno, lname))
# Filter out the unused imports.
used_imports, unused_imports = filter_unused_imports(ast, found_imports)
# Output warnings for the unused imports.
for x in unused_imports:
_, _, lname, lineno, _, pragma = x
if opts.do_pragmas and pragma:
continue
# Search for the column in the relevant line.
mo = re.search(r'\b%s\b' % lname, lines[lineno-1])
colno = 0
if mo:
colno = mo.start()+1
write("%s:%d:%d: Unused import '%s'\n" % (fn, lineno, colno, lname))
# (Optionally) Compute the list of names that are being assigned to.
if opts.do_missing or opts.debug:
vis = AssignVisitor()
compiler.walk(ast, vis)
assign_names = vis.finalize()
# (Optionally) Check for potentially missing imports (this cannot be
# precise, we are only providing a heuristic here).
if opts.do_missing:
defined = set(modname for modname, _, _, _, _, _ in used_imports)
defined.update(x[0] for x in assign_names)
for name, lineno in simple_names:
if name not in defined and name not in __builtin__.__dict__:
write("%s:%d: Missing import for '%s'\n" % (fn, lineno, name))
# Print out all the schmoo for debugging.
if opts.debug:
print
print
print '------ Imported names:'
for modname, rname, lname, lineno, level, pragma in found_imports:
print '%s:%d: %s' % (fn, lineno, lname)
## print
## print
## print '------ Exported names:'
## for name, lineno in exported:
## print '%s:%d: %s' % (fn, lineno, name)
## print
## print
## print '------ Used names:'
## for name, lineno in dotted_names:
## print '%s:%d: %s' % (fn, lineno, name)
## print
print
print
print '------ Assigned names:'
for name, lineno in assign_names:
print '%s:%d: %s' % (fn, lineno, name)
print
print
print '------ AST:'
printAst(ast, indent=' ', stream=sys.stdout, initlevel=1)
print
def gendeps():
import optparse
parser = optparse.OptionParser(__doc__.strip())
parser.add_option('-i', '--internal', '--internal-only',
default=0, action='count',
help="Filter out dependencies that are outside of the "
"roots of the input files. If internal is used twice, we "
"filter down further the dependencies to the set of "
"files that were processed only, not just to the files "
"that live in the same roots.")
parser.add_option('-e', '--external', '--external-only',
action='store_true',
help="Filter out dependencies to modules within the "
"roots of the input files. This can be used to find out "
"what external modules a package depends on, for example. "
"Note that it does not make sense to use --internal and "
"--external at the same time, as --internal will reject "
"all the dependencies --external allows would output.")
parser.add_option('-I', '--ignore', dest='ignores', action='append',
default=def_ignores,
help="Add the given directory name to the list to be ignored.")
parser.add_option('-v', '--verbose', action='count', default=0,
help="Output more debugging information")
parser.add_option('-q', '--quiet', action='count', default=0,
help="Output less debugging information")
parser.add_option('-f', '--follow', '-r', '--recursive', action='store_true',
help="Follow the modules depended upon and trace their dependencies. "
"WARNING: This can be slow. Use --internal to limit the scope.")
parser.add_option('--print-roots', action='store_true',
help="Only print the package roots corresponding to the input files."
"This is mostly used for testing and troubleshooting.")
parser.add_option('-d', '--disable-pragmas', action='store_false',
dest='do_pragmas', default=True,
help="Disable processing of pragma directives as strings after imports.")
parser.add_option('-u', '--ignore-unused', action='store_true',
help="Automatically ignore unused imports. (See sfood-checker.)")
opts, args = parser.parse_args()
opts.verbose -= opts.quiet
setup_logging(opts.verbose)
if not args:
logging.warning("Searching for files from current directory.")
args = ['.']
info = logging.info
if opts.internal and opts.external:
parser.error("Using --internal and --external at the same time does not make sense.")
if opts.print_roots:
inroots = find_roots(args, opts.ignores)
for dn in sorted(inroots):
print_(dn)
return
info("")
info("Input paths:")
for arg in args:
fn = realpath(arg)
info(' %s' % fn)
if not exists(fn):
parser.error("Filename '%s' does not exist." % fn)
# Get the list of package roots for our input files and prepend them to the
# module search path to insure localized imports.
inroots = find_roots(args, opts.ignores)
if (opts.internal or opts.external) and not inroots:
parser.error("No package roots found from the given files or directories. "
"Using --internal with these roots will generate no dependencies.")
info("")
info("Roots of the input files:")
for root in inroots:
info(' %s' % root)
info("")
info("Using the following import path to search for modules:")
sys.path = inroots + sys.path
for dn in sys.path:
info(" %s" % dn)
inroots = frozenset(inroots)
# Find all the dependencies.
info("")
info("Processing files:")
info("")
allfiles = defaultdict(set)
allerrors = []
processed_files = set()
fiter = iter_pyfiles(args, opts.ignores, False)
while 1:
newfiles = set()
for fn in fiter:
if fn in processed_files:
continue # Make sure we process each file only once.
info(" %s" % fn)
processed_files.add(fn)
if is_python(fn):
files, errors = find_dependencies(
fn, opts.verbose, opts.do_pragmas, opts.ignore_unused)
allerrors.extend(errors)
else:
# If the file is not a source file, we don't know how to get the
# dependencies of that (without importing, which we want to
# avoid).
files = []
# When packages are the source of dependencies, remove the __init__
# file. This is important because the targets also do not include the
# __init__ (i.e. when "from <package> import <subpackage>" is seen).
if basename(fn) == '__init__.py':
fn = dirname(fn)
# Make sure all the files at least appear in the output, even if it has
# no dependency.
from_ = relfile(fn, opts.ignores)
if from_ is None:
continue
infrom = from_[0] in inroots
if opts.internal and not infrom:
continue
if not opts.external:
allfiles[from_].add((None, None))
# Add the dependencies.
for dfn in files:
xfn = dfn
if basename(xfn) == '__init__.py':
xfn = dirname(xfn)
to_ = relfile(xfn, opts.ignores)
into = to_[0] in inroots
if (opts.internal and not into) or (opts.external and into):
continue
allfiles[from_].add(to_)
newfiles.add(dfn)
if not (opts.follow and newfiles):
break
else:
fiter = iter(sorted(newfiles))
# If internal is used twice, we filter down further the dependencies to the
# set of files that were processed only, not just to the files that live in
# the same roots.
if opts.internal >= 2:
filtfiles = type(allfiles)()
for from_, tolist in allfiles.iteritems():
filtfiles[from_] = set(x for x in tolist if x in allfiles or x == (None, None))
allfiles = filtfiles
info("")
info("SUMMARY")
info("=======")
# Output a list of the symbols that could not be imported as modules.
reports = [
("Modules that were ignored because not used:", ERROR_UNUSED, logging.info),
("Modules that could not be imported:", ERROR_IMPORT, logging.warning),
]
if opts.verbose >= 2:
reports.append(
("Symbols that could not be imported as modules:", ERROR_SYMBOL, logging.debug))
for msg, errtype, efun in reports:
names = set(name for (err, name) in allerrors if err is errtype)
if names:
efun("")
efun(msg)
for name in sorted(names):
efun(" %s" % name)
# Output the list of roots found.
info("")
info("Found roots:")
found_roots = set()
for key, files in allfiles.iteritems():
found_roots.add(key[0])
found_roots.update(map(itemgetter(0),files))
if None in found_roots:
found_roots.remove(None)
for root in sorted(found_roots):
info(" %s" % root)
# Output the dependencies.
info("")
output_depends(allfiles)
def scan(self):
"""
Returns an ImportGraph
"""
self.optsVerbose -= self.optsQuiet
setup_logging(self.optsVerbose)
info = logging.info
warning = logging.warning
debug = logging.debug
if self.optsInternal and self.optsExternal:
message = "Using --internal and --external at the same time " \
"does not make sense."
raise SnakefoodScannerException(message)
if self.optsPrintRoots:
inroots = find_roots(self.args, self.optsIgnores)
for dn in sorted(inroots):
print(dn)
return
info("")
info("Input paths:")
for arg in self.args:
fn = os.path.realpath(arg)
info(' {}'.format(fn))
if not os.path.exists(fn):
message = "Filename '{}' does not exist.".format(fn)
raise SnakefoodScannerException(message)
# Get the list of package roots for our input files and prepend
# them to the module search path to insure localized imports.
inroots = find_roots(self.args, self.optsIgnores)
if (self.optsInternal or self.optsExternal) and not inroots:
message = "No package roots found from the given files or " \
"directories. Using --internal with these roots will " \
"generate no dependencies."
raise SnakefoodScannerException(message)
info("")
info("Roots of the input files:")
for root in inroots:
info(' {}'.format(root))
info("")
info("Using the following import path to search for modules:")
sys.path = inroots + sys.path
for dn in sys.path:
info(" {}".format(dn))
inroots = frozenset(inroots)
# Find all the dependencies.
info("")
info("Processing files:")
info("")
allfiles = defaultdict(set)
allerrors = []
processed_files = set()
fiter = iter_pyfiles(self.args, self.optsIgnores, False)
while 1:
newfiles = set()
for fn in fiter:
if fn in processed_files:
continue # Make sure we process each file only once.
info(" {}".format(fn))
processed_files.add(fn)
if is_python(fn):
files, errors = find_dependencies(
fn, self.optsVerbose,
self.optsDoPragmas, self.optsVerbose)
allerrors.extend(errors)
else:
# If the file is not a source file, we don't know how
# to get the dependencies of that (without importing,
# which we want to avoid).
files = []
# When packages are the source of dependencies, remove the
# __init__ file. This is important because the targets
# also do not include the __init__ (i.e. when "from
# <package> import <subpackage>" is seen).
if os.path.basename(fn) == '__init__.py':
fn = os.path.dirname(fn)
# Make sure all the files at least appear in the output,
# even if it has no dependency.
from_ = relfile(fn, self.optsIgnores)
if from_ is None:
continue
infrom = from_[0] in inroots
if self.optsInternal and not infrom:
continue
if not self.optsExternal:
allfiles[from_].add((None, None))
# Add the dependencies.
for dfn in files:
xfn = dfn
if os.path.basename(xfn) == '__init__.py':
xfn = os.path.dirname(xfn)
to_ = relfile(xfn, self.optsIgnores)
into = to_[0] in inroots
if (self.optsInternal and not into) or \
(self.optsExternal and into):
continue
allfiles[from_].add(to_)
newfiles.add(dfn)
if not (self.optsFollow and newfiles):
break
else:
fiter = iter(sorted(newfiles))
# If internal is used twice, we filter down further the
# dependencies to the set of files that were processed only,
# not just to the files that live in the same roots.
if self.optsInternal >= 2:
filtfiles = type(allfiles)()
for from_, tolist in allfiles.iteritems():
filtfiles[from_] = set(
x for x in tolist if x in allfiles or x == (None, None))
allfiles = filtfiles
info("")
info("SUMMARY")
info("=======")
# Output a list of the symbols that could not
# be imported as modules.
reports = [
("Modules that were ignored because not used:",
ERROR_UNUSED, info),
("Modules that could not be imported:",
ERROR_IMPORT, warning),
]
if self.optsVerbose >= 2:
reports.append(
("Symbols that could not be imported as modules:",
ERROR_SYMBOL, debug))
for msg, errtype, efun in reports:
names = set(name for (err, name) in allerrors if err is errtype)
if names:
efun("")
efun(msg)
for name in sorted(names):
efun(" {}".format(name))
# Output the list of roots found.
info("")
info("Found roots:")
foundRoots = set()
for key, files in allfiles.iteritems():
foundRoots.add(key[0])
foundRoots.update(map(operator.itemgetter(0), files))
if None in foundRoots:
foundRoots.remove(None)
for root in sorted(foundRoots):
info(" {}".format(root))
# Output the dependencies.
entries = SnakefoodEntries()
info("")
for (from_root, from_), targets in sorted(
allfiles.iteritems(), key=operator.itemgetter(0)):
for to_root, to_ in sorted(targets):
entry = SnakefoodEntry(from_root, from_, to_root, to_)
entries.append(entry)
graph = ImportGraph()
for entry in entries.iterEntries():
graph.addEntry(entry)
return graph
def gendeps():
import optparse
parser = optparse.OptionParser(__doc__.strip())
parser.add_option(
'-i',
'--internal',
'--internal-only',
default=0,
action='count',
help="Filter out dependencies that are outside of the "
"roots of the input files. If internal is used twice, we "
"filter down further the dependencies to the set of "
"files that were processed only, not just to the files "
"that live in the same roots.")
parser.add_option(
'-e',
'--external',
'--external-only',
action='store_true',
help="Filter out dependencies to modules within the "
"roots of the input files. This can be used to find out "
"what external modules a package depends on, for example. "
"Note that it does not make sense to use --internal and "
"--external at the same time, as --internal will reject "
"all the dependencies --external allows would output.")
parser.add_option(
'-I',
'--ignore',
dest='ignores',
action='append',
default=def_ignores,
help="Add the given directory name to the list to be ignored.")
parser.add_option('-v',
'--verbose',
action='count',
default=0,
help="Output more debugging information")
parser.add_option('-q',
'--quiet',
action='count',
default=0,
help="Output less debugging information")
parser.add_option(
'-f',
'--follow',
'-r',
'--recursive',
action='store_true',
help="Follow the modules depended upon and trace their dependencies. "
"WARNING: This can be slow. Use --internal to limit the scope.")
parser.add_option(
'--print-roots',
action='store_true',
help="Only print the package roots corresponding to the input files."
"This is mostly used for testing and troubleshooting.")
parser.add_option(
'-d',
'--disable-pragmas',
action='store_false',
dest='do_pragmas',
default=True,
help="Disable processing of pragma directives as strings after imports."
)
parser.add_option(
'-u',
'--ignore-unused',
action='store_true',
help="Automatically ignore unused imports. (See sfood-checker.)")
opts, args = parser.parse_args()
opts.verbose -= opts.quiet
setup_logging(opts.verbose)
if not args:
logging.warning("Searching for files from current directory.")
args = ['.']
info = logging.info
if opts.internal and opts.external:
parser.error(
"Using --internal and --external at the same time does not make sense."
)
if opts.print_roots:
inroots = find_roots(args, opts.ignores)
for dn in sorted(inroots):
print_(dn)
return
info("")
info("Input paths:")
for arg in args:
fn = realpath(arg)
info(' %s' % fn)
if not exists(fn):
parser.error("Filename '%s' does not exist." % fn)
# Get the list of package roots for our input files and prepend them to the
# module search path to insure localized imports.
inroots = find_roots(args, opts.ignores)
if (opts.internal or opts.external) and not inroots:
parser.error(
"No package roots found from the given files or directories. "
"Using --internal with these roots will generate no dependencies.")
info("")
info("Roots of the input files:")
for root in inroots:
info(' %s' % root)
info("")
info("Using the following import path to search for modules:")
sys.path = inroots + sys.path
for dn in sys.path:
info(" %s" % dn)
inroots = frozenset(inroots)
# Find all the dependencies.
info("")
info("Processing files:")
info("")
allfiles = defaultdict(set)
allerrors = []
processed_files = set()
fiter = iter_pyfiles(args, opts.ignores, False)
while 1:
newfiles = set()
for fn in fiter:
if fn in processed_files:
continue # Make sure we process each file only once.
info(" %s" % fn)
processed_files.add(fn)
if is_python(fn):
files, errors = find_dependencies(fn, opts.verbose,
opts.do_pragmas,
opts.ignore_unused)
allerrors.extend(errors)
else:
# If the file is not a source file, we don't know how to get the
# dependencies of that (without importing, which we want to
# avoid).
files = []
# When packages are the source of dependencies, remove the __init__
# file. This is important because the targets also do not include the
# __init__ (i.e. when "from <package> import <subpackage>" is seen).
if basename(fn) == '__init__.py':
fn = dirname(fn)
# Make sure all the files at least appear in the output, even if it has
# no dependency.
from_ = relfile(fn, opts.ignores)
if from_ is None:
continue
infrom = from_[0] in inroots
if opts.internal and not infrom:
continue
if not opts.external:
allfiles[from_].add((None, None))
# Add the dependencies.
for dfn in files:
xfn = dfn
if basename(xfn) == '__init__.py':
xfn = dirname(xfn)
to_ = relfile(xfn, opts.ignores)
into = to_[0] in inroots
if (opts.internal and not into) or (opts.external and into):
continue
allfiles[from_].add(to_)
newfiles.add(dfn)
if not (opts.follow and newfiles):
break
else:
fiter = iter(sorted(newfiles))
# If internal is used twice, we filter down further the dependencies to the
# set of files that were processed only, not just to the files that live in
# the same roots.
if opts.internal >= 2:
filtfiles = type(allfiles)()
for from_, tolist in allfiles.items():
filtfiles[from_] = set(x for x in tolist
if x in allfiles or x == (None, None))
allfiles = filtfiles
info("")
info("SUMMARY")
info("=======")
# Output a list of the symbols that could not be imported as modules.
reports = [
("Modules that were ignored because not used:", ERROR_UNUSED,
logging.info),
("Modules that could not be imported:", ERROR_IMPORT, logging.warning),
]
if opts.verbose >= 2:
reports.append(("Symbols that could not be imported as modules:",
ERROR_SYMBOL, logging.debug))
for msg, errtype, efun in reports:
names = set(name for (err, name) in allerrors if err is errtype)
if names:
efun("")
efun(msg)
for name in sorted(names):
efun(" %s" % name)
# Output the list of roots found.
info("")
info("Found roots:")
found_roots = set()
for key, files in allfiles.items():
found_roots.add(key[0])
found_roots.update(list(map(itemgetter(0), files)))
if None in found_roots:
found_roots.remove(None)
for root in sorted(found_roots):
info(" %s" % root)
# Output the dependencies.
info("")
output_depends_inverted(allfiles, is_json=True)
def fetch_dependencies(self):
"""
Fetch all dependencies and follow the target file.
This was inspired by the snakefood library
snakefood-1.4-py2.7.egg/snakefood/gendeps.py
"""
# No need to run this twice
if self.dependency_list: return self.dependency_list
log.info("Fetching internal dependecies: %s" % self.filename)
depends = find_imports(self.filename, 1, 0)
# Get the list of package roots for our input files and prepend them to the
# module search path to insure localized imports.
inroots = find_roots([self.filename], [])
self.file_roots = inroots
if not inroots:
raise NoRoot
for file in inroots:
log.debug("Root found: %s" % file)
sys.path = inroots + sys.path
#log.debug("Using the following import path to search for modules:")
#for dn in sys.path:
# log.debug(" -- %s" % dn)
inroots = frozenset(inroots)
# Find all the dependencies.
log.debug("Processing file:")
allfiles = defaultdict(set)
allerrors = []
processed_files = set()
ignorefiles = []
alldependencies = []
fiter = iter_pyfiles([self.filename], ignorefiles, False)
while 1:
newfiles = set()
for fn in fiter:
log.debug(" post-filter: %s" % fn)
processed_files.add(fn)
if is_python(fn):
files, errors = find_dependencies(fn, 0, 0)
log.debug("dependency file count: %d" % len(files))
allerrors.extend(errors)
else:
# If the file is not a source file, we don't know how to get the
# dependencies of that (without importing, which we want to
# avoid).
files = []
# When packages are the source of dependencies, remove the __init__
# file. This is important because the targets also do not include the
# __init__ (i.e. when "from <package> import <subpackage>" is seen).
if basename(fn) == '__init__.py':
fn = dirname(fn)
# no dependency.
from_ = relfile(fn, ignorefiles)
if from_ is None:
log.debug("from_ empty. Move on")
continue
infrom = from_[0] in inroots
log.debug( " from: %s" % from_[0])
log.debug( " file: %s" % from_[1])
allfiles[from_].add((None, None))
# Add the dependencies.
for dfn in files:
xfn = dfn
if basename(xfn) == '__init__.py':
xfn = dirname(xfn)
to_ = relfile(xfn, ignorefiles)
into = to_[0] in inroots
log.debug( " from: %s" % from_[1])
log.debug( " to: %s" % to_[1])
if dfn in alldependencies:
log.debug("Already added %s to dependency list" % dfn)
else:
log.debug("Add %s to dependency list" % dfn)
allfiles[from_].add(to_)
newfiles.add(dfn)
alldependencies.append(dfn)
if not newfiles:
log.debug("No more new files. all done")
break
else:
fiter = iter(sorted(newfiles))
# Output the list of roots found.
log.debug("Found roots:")
found_roots = set()
for key, files in allfiles.iteritems():
found_roots.add(key[0])
found_roots.update(map(itemgetter(0),files))
if None in found_roots:
found_roots.remove(None)
for root in sorted(found_roots):
log.debug(" %s" % root)
self.dependency_list = allfiles
return self.dependency_list;
def find_package_paths(source_roots, ignores=None):
return {
os.path.dirname(path)
for path in iter_pyfiles(source_roots, ignores)
if os.path.basename(path) == "setup.py"
}
def files(self):
"""
:return: files under input package
"""
return [found_file for found_file in iter_pyfiles([self.package_path], None, abspaths=False)
if not self._is_ignored(found_file)]
