def GetImports(cls, directory, out_filters=[]):
'''
Lists imports made by python files found in the given directory.
Directory will be scanned recursively
:param str directory:
Path to a directory
:param list(str) out_filters:
List of filename filters
.. see:: FindFiles
:rtype: list(str)
:returns:
List of module imported by python
'''
from ben10.filesystem import FindFiles
from modulefinder import ModuleFinder as Finder
finder = Finder(directory)
for py_filename in FindFiles(directory, in_filters='*.py', out_filters=out_filters):
finder.run_script(py_filename)
return sorted(finder.modules.keys() + finder.badmodules.keys())
def basic_module_find(files):
'''Do a basic module find on all files'''
mf = ModuleFinder()
for fname in files:
mf.run_script(fname)
return mf
def get_imports(self, file):
finder = ModuleFinder()
finder.run_script(file)
im = []
for name, mod in finder.modules.items():
im.append(name)
return im
def GetImports(cls, directory, out_filters=[]):
"""
Lists imports made by python files found in the given directory.
Directory will be scanned recursively
:param unicode directory:
Path to a directory
:param list(unicode) out_filters:
List of filename filters
.. see:: FindFiles
:rtype: list(unicode)
:returns:
List of module imported by python
"""
from zerotk.easyfs import FindFiles
from modulefinder import ModuleFinder as Finder
finder = Finder(bytes(directory))
for py_filename in FindFiles(directory,
in_filters='*.py',
out_filters=out_filters):
finder.run_script(py_filename)
return map(unicode,
sorted(finder.modules.keys() + finder.badmodules.keys()))
def show_import_report(pathname=None, show_loaded=False, show_not_imported=False, fullreport=False):
filename = basename(pathname)
finder = ModuleFinder()
finder.run_script(pathname)
if show_loaded or show_not_imported or fullreport:
print('-'*50)
else:
print("Missing options what to do. Use 'modulereport --help'")
if show_loaded:
print("Loaded modules for: " + filename)
print('-'*50)
for name, mod in finder.modules.items():
line = "{pyname}: {modlist}".format(pyname=name, modlist=','.join(list(mod.globalnames.keys())[:3]))
print(line)
print('-'*50)
if show_not_imported:
print("Modules missing for: " + filename)
print('-'*50)
print('\n'.join(finder.badmodules.keys()))
print('-'*50)
if fullreport:
print("Full report:")
print('-'*50)
finder.report()
def spider(self):
print '-' * 40
while len(self.full_path) != 0:
full_path = self.full_path.pop()
self.full_path_discard.append(full_path)
finder = ModuleFinder()
finder.run_script(full_path)
for name, mod in finder.modules.iteritems():
full_path = str(mod.__file__).replace('.', '/')
if full_path not in self.full_path_discard:
if full_path.startswith('/home/alfred/pipline/code/'):
self.result.append(name.replace('.', '/'))
if not full_path.startswith('/home/alfred/pipline/code/U'):
self.full_path.append(full_path)
else:
self.full_path_discard.append(full_path)
names = finder.any_missing()
for name in names:
for folder in self.all_path:
try:
full_path = folder + name + '.py'
if os.path.isfile(full_path):
if full_path not in self.full_path_discard:
self.full_path.append(full_path)
self.result.append(full_path)
except:
pass
for item in sorted(set(self.result)):
print item
print '='*100
def run(self):
p = self.cmdLineParams
finder = ModuleFinder()
finder.run_script(p.infile.name)
modDict = {}
for name, mod in finder.modules.items():
if mod.__file__ is None:
key = str(mod.__file__)
else:
key = os.path.abspath(mod.__file__)
if not key in modDict:
modDict[key] = []
modDict[key].append(name)
# sort
for key in sorted(modDict):
if p.restrictToProject and not key.startswith(__PROJECT_ROOT__):
continue
modules = modDict[key]
sys.stdout.write(key)
for moduleName in modules:
sys.stdout.write('\t')
sys.stdout.write(moduleName)
sys.stdout.write('\n')
def test_dependencies(self):
"""
EXTREMELY SLOW
"""
path = os.path.join('..', 'lingcorpora', 'corpora',
'%s_corpus.py' % self.lang)
core_path = os.path.join(
'..',
'lingcorpora',
'corpus.py',
)
# legit_deps = pickle.load(open('dependencies.pickle', 'rb'))
mf = ModuleFinder()
mf.run_script(init_path)
legit_deps = set(mf.modules.keys())
mf.run_script(path)
deps = set(mf.modules.keys())
self.assertEqual(len(deps.difference(legit_deps)), 0)
del deps, legit_deps
def find_and_register_qabbage_tasks(outer_scope, exclude_tests=True):
"""
Walks the directory and finds all of the scripts that import
qabbage tasks.
:return: file paths with qabbage tasks
"""
all_files = []
qabbage_files = []
for root, dirs, files in os.walk("."):
path = root.split('/')
for file in files:
_, file_extension = os.path.splitext(file)
if file_extension == '.py':
all_files.append(os.path.join(*path, file))
finder = ModuleFinder()
for file in filter(lambda x: 'test_' not in x, all_files):
finder.run_script(file)
for name, mod in finder.modules.items():
if name == 'qabbage':
qabbage_files.append(file)
promises = load_qabbage_modules(qabbage_files)
for promise in promises:
outer_scope[promise[0]] = promise[1]
return promises
def _reload(obj, name=None):
obj_type = type(obj)
module = inspect.getmodule(obj)
sys_modules_frozen = sys.modules.copy()
sys_modules_sorted = sorted(sys_modules_frozen.items(),
key=lambda t: len(t[0]),
reverse=True)
if not module:
for modname, module in sys_modules_sorted:
if modname.split('.')[0] in STDLIB_NAMES:
continue
candidate = getattr(module, name, Sentinel)
if (candidate is not Sentinel and isinstance(candidate, obj_type)):
break
else:
module = None
if not module:
raise TypeError()
finder = ModuleFinder()
finder.run_script(module.__file__)
for m in finder.modules:
if m.split('.')[0] in STDLIB_NAMES:
continue
try:
reload(sys_modules_frozen[m])
except KeyError:
pass
return reload(module)
def find_failed_imports(module_path):
"""Returns all modules that failed to import when loading a particular
module."""
finder = ModuleFinder()
finder.run_script(module_path)
bad_modules = dict(finder.badmodules)
return return_missing_base_modules(bad_modules)
def get_files_with_modules(directory_path, search_recursively=False):
files_with_modules = []
if search_recursively:
original_file_paths = gfp.get_paths(directory_path)
unique_file_paths = []
for file_path_index, file_path in enumerate(original_file_paths):
files_with_modules.append([
file_path,
os.path.dirname(file_path).rsplit(os.path.sep, 1)[1]
])
finder = ModuleFinder()
for file_path in original_file_paths:
try:
finder.run_script(file_path)
except ImportError:
print("Could not load", file_path)
for key in finder.modules.items():
item_path = key[1].__file__ if key[
1].__file__ else key[1].__name__ + " (built-in)"
if '.' not in item_path or item_path.endswith('.py'):
item_data = [item_path, key[1].__name__]
if item_data[0] not in unique_file_paths and item_data[
0] not in original_file_paths:
unique_file_paths.append(item_data[0])
files_with_modules.append(item_data)
files_with_modules.sort(key=lambda item: item[0])
else:
files_with_modules = gfp.get_paths(directory_path)
for file_path_index, file_path in enumerate(files_with_modules):
files_with_modules[file_path_index] = [
file_path,
os.path.dirname(file_path).rsplit(os.path.sep, 1)[1]
]
files_with_modules.sort(key=lambda item: item[0])
return files_with_modules
def make_bom():
"""Generate a bill of materials and return the tt object."""
finder = ModuleFinder()
finder.run_script(sys.argv[0])
tt = tytable.ttable()
tt.add_option(tt.OPTION_LONGTABLE)
tt.add_head(['name', 'ver', 'path', 'bytes', 'lines', 'sha-1'])
tt.set_col_alignmnets("lllrrl")
for inc in [False, True]:
for name, mod in sorted(finder.modules.items()):
if system_module(mod) != inc:
continue # don't use this one
stats = file_stats(mod.__file__)
ver = mod.globalnames.get('__version__', '---')
if ver == 1 and name in sys.modules:
# It has a version number; get it.
try:
ver = sys.modules[name].__version__
except AttributeError:
ver = '---'
fname = mod.__file__
if type(name) != str:
name = ""
if type(fname) != str:
fname = ""
tt.add_data([
latex_tools.latex_escape(name), ver,
latex_tools.latex_escape(fname), stats[0], stats[1], stats[2]
])
tt.add_data(tt.HR)
return tt
def _get_dependencies(entry_point):
# Dynamically find all imported modules
from modulefinder import ModuleFinder
finder = ModuleFinder()
finder.run_script(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
entry_point))
imports = list(
set([
v.__file__.split('site-packages/', 1)[-1].split('/')[0]
for (k, v) in finder.modules.items()
if v.__file__ is not None and "site-packages" in v.__file__
]))
# Get just the modules, ignore the so and py now (maybe useful for calls to add_file)
modules = [i.split('.py')[0] for i in imports if ".so" not in i]
so_files = list(
set([
v.__file__ for (k, v) in finder.modules.items()
if v.__file__ is not None and "site-packages" in v.__file__
and ".so" in v.__file__
]))
return set(modules), so_files
def add_mocks(filename):
gp = mock.patch('ansible.module_utils.basic.get_platform').start()
gp.return_value = 'linux'
module_mock = mock.MagicMock()
mocks = []
for module_class in MODULE_CLASSES:
mocks.append(
mock.patch('ansible.module_utils.basic.AnsibleModule',
new=module_mock))
for m in mocks:
p = m.start()
p.side_effect = AnsibleModuleCallError()
finder = ModuleFinder()
try:
finder.run_script(filename)
except:
pass
sys_mock = mock.MagicMock()
sys_mock.__version__ = '0.0.0'
sys_mocks = []
for module, sources in finder.badmodules.items():
if module in sys.modules:
continue
if [s for s in sources if s[:7] in ['ansible', '__main_']]:
parts = module.split('.')
for i in range(len(parts)):
dotted = '.'.join(parts[:i + 1])
sys.modules[dotted] = sys_mock
sys_mocks.append(dotted)
return module_mock, mocks, sys_mocks
def add_mocks(filename):
gp = mock.patch('ansible.module_utils.basic.get_platform').start()
gp.return_value = 'linux'
module_mock = mock.MagicMock()
mocks = []
for module_class in MODULE_CLASSES:
mocks.append(
mock.patch('ansible.module_utils.basic.AnsibleModule',
new=module_mock)
)
for m in mocks:
p = m.start()
p.side_effect = AnsibleModuleCallError()
finder = ModuleFinder()
try:
finder.run_script(filename)
except:
pass
sys_mock = mock.MagicMock()
sys_mock.__version__ = '0.0.0'
sys_mocks = []
for module, sources in finder.badmodules.items():
if module in sys.modules:
continue
if [s for s in sources if s[:7] in ['ansible', '__main_']]:
parts = module.split('.')
for i in range(len(parts)):
dotted = '.'.join(parts[:i+1])
sys.modules[dotted] = sys_mock
sys_mocks.append(dotted)
return module_mock, mocks, sys_mocks
def GetImports(cls, directory, out_filters=[]):
'''
Lists imports made by python files found in the given directory.
Directory will be scanned recursively
:param str directory:
Path to a directory
:param list(str) out_filters:
List of filename filters
.. see:: FindFiles
:rtype: list(str)
:returns:
List of module imported by python
'''
from ben10.filesystem import FindFiles
from modulefinder import ModuleFinder as Finder
finder = Finder(directory)
for py_filename in FindFiles(directory,
in_filters='*.py',
out_filters=out_filters):
finder.run_script(py_filename)
return sorted(finder.modules.keys() + finder.badmodules.keys())
def print_type_dir():
finder = ModuleFinder()
finder.run_script('from_import.py')
print('dir ModuleFinder: {}'.format(dir(finder)))
"""
dir ModuleFinder: ['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__',
'__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__',
'__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__',
'__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__',
'__weakref__', '_add_badmodule', '_safe_import_hook', 'add_module', 'any_missing',
'any_missing_maybe', 'badmodules', 'debug', 'determine_parent', 'ensure_fromlist',
'excludes', 'find_all_submodules', 'find_head_package', 'find_module', 'import_hook',
'import_module', 'indent', 'load_file', 'load_module', 'load_package', 'load_tail',
'modules', 'msg', 'msgin', 'msgout', 'path', 'processed_paths', 'replace_paths',
'replace_paths_in_code', 'report', 'run_script', 'scan_code', 'scan_opcodes']
"""
for name, mod in finder.modules.items():
print('type:{}'.format(type(mod)))
#=> type:<class 'modulefinder.Module'>
print('dir Module object:{}'.format(dir(mod)))
"""
dir:['__class__', '__code__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__',
'__file__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__',
'__init_subclass__', '__le__', '__lt__', '__module__', '__name__', '__ne__', '__new__',
'__path__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__',
'__str__', '__subclasshook__', '__weakref__', 'globalnames', 'starimports']
"""
return
def _get_source_of_module(module):
# print("module", module)
try:
module_source = inspect.getsource(module)
except OSError:
return {}
module_finder = ModuleFinder()
module_finder.run_script(module.__file__)
# ret = ast.parse(base_source)
# print(dir(ret))
accum = {
module.__name__: module_source
}
for child_module_name, child_module_tuple in module_finder.modules.items():
if child_module_name == "__main__":
continue
if child_module_name in accum:
continue
# found_module = imp.find_module(child_module_tuple.__name__)
# child_module = imp.load_module(child_module_tuple.__name__, *found_module)
child_module = importlib.import_module(child_module_tuple.__name__)
print("processing module", child_module_tuple.__name__, child_module)
accum.update(_get_source_of_module(child_module))
return accum
def get_files_data(general_path, file_paths):
module_finder = ModuleFinder()
files = []
for file_path in file_paths:
spec = importlib.util.spec_from_file_location(file_path, file_path)
module_from_spec = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module_from_spec)
functions_list = [
function for function in getmembers(module_from_spec)
if isfunction(function[1])
]
module = file.Module(
file_path.rsplit(os.path.sep, 1)[1].rsplit('.')[0], file_path)
module.size = os.path.getsize(file_path)
module_finder.run_script(module.path)
module.fill_imports_and_functions(general_path, module_finder)
module.fill_dependecies()
files.append(module)
for element in files:
element.fill_dependecies(files)
return files
def get_python_module_partials(
pathToModuleFile,
log):
"""
*Get the names of the _partials imported into dryx python modules.*
**Key Arguments:**
- ``pathToModuleFile`` -- the path to the python module we wish to extract the _partial names for
- ``log`` -- logger
**Return:**
- ``partialsDictionary`` -- a dictionary of the _partial names imported into the dryx python module, and a list of their functions
.. todo::
- [ ] when complete, clean get_python_module_partials function & add logging
"""
################ > IMPORTS ################
## STANDARD LIB ##
from modulefinder import ModuleFinder
import re
import os
import sys
## THIRD PARTY ##
## LOCAL APPLICATION ##
log.debug('starting the ``get_python_module_partials`` function')
## VARIABLES ##
partialsDictionary = {}
finder = ModuleFinder()
finder.run_script(pathToModuleFile)
baseName = os.path.basename(pathToModuleFile).replace(".py", "")
if baseName == "__init__":
pathToModuleFile = pathToModuleFile.replace("__init__.py", "")
baseName = os.path.basename(pathToModuleFile)
reBaseName = re.compile(r"%s" % (baseName,))
for name, mod in finder.modules.iteritems():
if reBaseName.search(name):
importList = []
for key in mod.globalnames.keys():
# if ("/Library/Frameworks/Python.framework/" in mod.__file__ or
# "macports" in mod.__file__) and "site-packages" not in
# mod.__file__:
if "/Users/" not in mod.__file__:
# print "skipping %s" % (mod.__file__,)
continue
importList.append(key)
if len(importList):
# print mod.__file__, importList
partialsDictionary[name] = importList
log.debug('completed the ``get_python_module_partials`` function')
return partialsDictionary
def get_module_import_dict(object, scope='project', remove_packages=True):
"""
Given some code object (or the full name of a module), find all the modules that must be imported for this object to
run.
:param object:
:param scope: One of:
'package': Only collect modules from same root package (eg root package of foo.bar.baz is foo.
'project': Only collect modules in same project (ie, modules whose root package is in the same directoy as object's root package)
'all': Collect all dependent modueles (not recommended, as this will, for example include a bunch of numpy code).
:return: A dict<module_name: module_path>
"""
assert scope in ('package', 'project', 'all')
if isinstance(object, (list, tuple)):
dicts, names = zip(
*[get_module_import_dict(ob, scope=scope) for ob in object])
return {k: v for d in dicts for k, v in d.iteritems()}, names
elif isinstance(object, basestring):
module = import_module(object)
else:
module = inspect.getmodule(object)
module_file = get_src_file(module)
finder = ModuleFinder()
this_package = module.__name__.split('.')[0]
LOGGER.info(
'Scanning Dependent Modules in {}. This may take some time...'.format(
this_package))
finder.run_script(module_file)
module_files = {
name: get_src_file(module)
for name, module in finder.modules.iteritems()
if module.__file__ is not None
}
module_files[module.__name__] = get_src_file(
module) # Don't forget yourself!
# module_files =
if scope == 'package':
module_files = {
name: mod
for name, mod in module_files.iteritems()
if name.split('.')[0] == this_package
}
elif scope == 'project':
base_dir = os.path.dirname(os.path.dirname(module.__file__))
module_files = {
name: mod
for name, mod in module_files.iteritems()
if mod.startswith(base_dir)
}
LOGGER.info('Scan Complete. {} dependent modules found.'.format(
len(module_files)))
# module_name_to_module_path = {name: get_src_file(m) for name, m in modules.iteritems()}
if remove_packages:
module_files = {
name: path
for name, path in module_files.iteritems() if not (
path.endswith('__init__.py') or path.endswith('__init__.pyc'))
}
return module_files, module.__name__
def zip_std_lib(src_module, dst_file):
"""Compiles the Python standard library modules used by the source module
and outputs to zip file."""
finder = ModuleFinder()
finder.run_script(src_module)
modules = set()
print('Writing dependencies to "%s"...' % DEP_OUT)
with open(DEP_OUT, 'w') as f:
for name, mod in finder.modules.items():
print('%s: ' % name, end='', file=f)
print(mod.__file__, file=f)
if mod.__file__ is None:
continue
path = os.path.realpath(mod.__file__)
if not path.startswith(os.path.normpath(STD_LIB)):
continue
while (os.path.dirname(path) != os.path.normpath(STD_LIB)):
path = os.path.dirname(path)
if os.path.isfile(path):
modules.add(path)
elif os.path.isdir(path):
for root, dirs, files in os.walk(path):
for i in files:
modules.add(os.path.join(root, i))
print('-' * 50, file=f)
print('### Modules NOT imported ###', file=f)
print('\n'.join(finder.badmodules.keys()), file=f)
modules = sorted([
i for i in modules if i.endswith((
'.py', '.pyc')) and not os.path.dirname(i).endswith('__pycache__')
])
print('Writing standard library to "%s"...' % dst_file)
with zipfile.ZipFile(dst_file, 'w', compression=zipfile.ZIP_DEFLATED) as z:
for i in modules:
root, ext = os.path.splitext(i)
if ext == '.py':
arcname = os.path.relpath(root, STD_LIB) + '.pyc'
pyc = create_pyc(i)
else:
arcname = os.path.relpath(i, STD_LIB)
with open(i, 'rb') as f:
pyc = f.read()
z.writestr(arcname, pyc)
def zip_std_lib(src_module, dst_file):
"""Compiles the Python standard library modules used by the source module
and outputs to zip file."""
finder = ModuleFinder()
finder.run_script(src_module)
modules = set()
print('Writing dependencies to "%s"...' % DEP_OUT)
with open(DEP_OUT, "w") as f:
for name, mod in finder.modules.items():
print("%s: " % name, end="", file=f)
print(mod.__file__, file=f)
if mod.__file__ is None:
continue
path = os.path.realpath(mod.__file__)
if not path.startswith(os.path.normpath(STD_LIB)):
continue
while os.path.dirname(path) != os.path.normpath(STD_LIB):
path = os.path.dirname(path)
if os.path.isfile(path):
modules.add(path)
elif os.path.isdir(path):
for root, dirs, files in os.walk(path):
for i in files:
modules.add(os.path.join(root, i))
print("-" * 50, file=f)
print("### Modules NOT imported ###", file=f)
print("\n".join(finder.badmodules.keys()), file=f)
modules = sorted(
[i for i in modules if i.endswith((".py", ".pyc")) and not os.path.dirname(i).endswith("__pycache__")]
)
print('Writing standard library to "%s"...' % dst_file)
with zipfile.ZipFile(dst_file, "w", compression=zipfile.ZIP_DEFLATED) as z:
for i in modules:
root, ext = os.path.splitext(i)
if ext == ".py":
arcname = os.path.relpath(root, STD_LIB) + ".pyc"
pyc = create_pyc(i)
else:
arcname = os.path.relpath(i, STD_LIB)
with open(i, "rb") as f:
pyc = f.read()
z.writestr(arcname, pyc)
def find_failed_imports_by_directory(directory):
"""Returns all modules that failed to import when loading all modules below
a directory."""
finder = ModuleFinder()
py_files = _find_all_python_modules(directory)
for f in py_files:
finder.run_script(f)
bad_modules = dict(finder.badmodules)
return return_missing_base_modules(bad_modules)
def get_modules(filename):
finder = ModuleFinder()
finder.run_script(filename)
list_modules = {}
for name,mod in finder.modules.iteritems():
print name,mod
return list_modules
def get_imported_module_paths(module_path):
finder = ModuleFinder([os.path.dirname(module_path)])
finder.run_script(module_path)
imported_modules = []
for module_name, module in finder.modules.items():
if module_name != '__main__':
path = getattr(module, '__file__', None)
if path is not None and os.path.isfile(path):
imported_modules.append(path)
return imported_modules
def check_module(self, module_name ):
from modulefinder import ModuleFinder
finder = ModuleFinder(debug=2)
finder.run_script(module_name)
for name, mod in finder.modules.items():
try:
__import__(name, fromlist=mod.globalnames.keys(),level=1)
sys.stdout.write('.')
except ImportError as e:
print("ERROR IMPORTING %s: " % name + " -- "+e.message)
def patched(self, project_dir, spec, sack):
mod = ModuleFinder()
for item in list(project_dir.glob('**/*.py')):
try:
mod.run_script(str(item))
for _, mod in mod.modules.items():
if mod.__file__ and mod.__file__.startswith("/usr/lib"):
spec.required_files.add(mod.__file__)
except ImportError as ie:
logging.warn("Exception was raised by ModuleFinder:\n" +
str(ie) + "\nOn file: " + str(item))
def imports_metadata(path: Union[str, Path]) -> bool:
mf = ModuleFinder()
if isinstance(path, Path):
path = str(path)
mf.run_script(path)
modules = list(mod for name, mod in mf.modules.items()
if "metadata" in name)
if modules:
return True
else:
return False
def patched(self, project_dir, spec, sack):
files = list(project_dir.glob('*.py'))
files.extend(list(project_dir.glob('*/*.py')))
mod = ModuleFinder()
for item in files:
mod.run_script(str(item))
for _, mod in mod.modules.items():
if mod.__file__ and mod.__file__.startswith("/usr/lib"):
spec.required_files.add(mod.__file__)
def patched(self, project_dir, spec, sack):
mod = ModuleFinder()
for item in list(project_dir.glob('**/*.py')):
try:
mod.run_script(str(item))
for _, mod in mod.modules.items():
if mod.__file__ and mod.__file__.startswith("/usr/lib"):
spec.required_files.add(mod.__file__)
except ImportError as ie:
logging.warn("Exception was raised by ModuleFinder:\n" +
str(ie) + "\nOn file: " + str(item))
def main(argv):
path = sys.path[:]
path[0] = os.path.dirname(argv[0])
mf = ModuleFinder(path)
for f in argv:
mf.run_script(f)
paths = sorted(list(set([os.path.abspath(x.__file__) for x in mf.modules.values() if x.__file__])))
cwd = os.getcwd()
paths = [x for x in paths if x.startswith(cwd)]
m = len(cwd) + 1
paths = argv + [x[m:] for x in paths]
print ' '.join(paths)
def usage_mistakes():
files = ['from_import_spam_only.py', 'from_import_ham_only.py']
# ModuleFinderオブジェクトを使いまわす
finder = ModuleFinder()
for f in files:
finder.run_script(f)
for name, mod in finder.modules.items():
print('-' * 10)
print('file:{}'.format(f))
print('name:{}'.format(name))
print('modules:{}'.format(','.join(list(mod.globalnames.keys()))))
"""ModuleFinderオブジェクトを使いまわすと、誤った結果になる
def diff_import_fromimport():
files = ['from_import.py', 'import.py']
for f in files:
print('=' * 10)
print('filename:{}'.format(f))
# ModuleFinderオブジェクトは、ファイルごとに生成する
finder = ModuleFinder()
finder.run_script(f)
for name, mod in finder.modules.items():
print('-' * 10)
print('name:{}'.format(name))
print('modules:{}'.format(','.join(list(mod.globalnames.keys()))))
def get_dependencies(module):
from modulefinder import ModuleFinder
import os
finder = ModuleFinder()
finder.run_script(module)
all_module_paths = [os.path.abspath(m.__file__) for
m in finder.modules.values() if m.__file__ is not None]
def is_in_same_path(p):
return p and os.path.dirname(p).startswith(os.path.dirname(module))
return [x for x in all_module_paths if is_in_same_path(x) and x != module]
def make_module_list(input_file, output_file):
module_list = []
finder = ModuleFinder()
finder.run_script(input_file)
keys = sorted(finder.modules.keys())
for key in keys:
if finder.modules[key].__path__ and key.find(".") == -1:
module_list.append(key)
with open(output_file, "w") as f_o:
f_o.write('pip install ' + ' '.join(module_list))
print('pip install ' + ' '.join(module_list))
def usage_files():
files = ['from_import_spam_only.py', 'from_import_ham_only.py']
for f in files:
# ModuleFinderオブジェクトは、ファイルごとに生成する
finder = ModuleFinder()
finder.run_script(f)
for name, mod in finder.modules.items():
print('-' * 10)
print('file:{}'.format(f))
print('name:{}'.format(name))
print('modules:{}'.format(','.join(list(mod.globalnames.keys()))))
"""
def print_attr():
finder = ModuleFinder()
finder.run_script('from_import_ham_only.py')
for name, mod in finder.modules.items():
print('-' * 10)
print('name:{}'.format(name))
print('globalnames:{}'.format(mod.globalnames))
print('modules:{}'.format(','.join(list(mod.globalnames.keys()))))
print('starimports:{}'.format(mod.starimports))
print('bad modules:{}'.format(','.join(finder.badmodules.keys())))
"""
def myMethod(self):
finder = ModuleFinder()
myDrive = "g:\\"
mySrcPath = "git\\6700Spring16\\CA05\\submissions\\danieljames_1246453_74826857_jhd0008\\softwareProcess\\SoftwareProcess\\Assignment\\prod"
myPath = "git\\6700Spring16\\CA05\\submissions\\danieljames_1246453_74826857_jhd0008\\softwareProcess\\SoftwareProcess\\Assignment\\test"
#myPath = "git\\6700Spring16\\CA05\\submissions"
scriptname = myDrive+myPath+'\\FixTest.py'
with io.open(scriptname) as scriptfile:
code = compile(scriptfile.readall(),scriptname,'exec')
mod = importlib.import_module('FixTest',package="Assignment")
print 'Inspect output:'
for i in inspect.getmembers((mod, inspect.ismodule)):
print i[0]
for root, myDir, files in os.walk(myDrive + myPath):
for myFile in files:
print myFile
sys.path.insert(0,os.path.join(myDrive+myPath))
print sys.path
finder.run_script(os.path.join(myDrive+mySrcPath+'\\Fix.py'))
print 'Loaded modules:'
for name, mod in finder.modules.iteritems():
print '%s: ' % name,
print ','.join(mod.globalnames.keys()[:3])
runpy.run_path(os.path.join(myDrive,myPath,'FixTest.py'))
__all__ = []
module_name = None
for loader, module_name, is_pkg in pkgutil.walk_packages("g:\\"):
print 'in for loop'
#__all__.append(module_name)
#module = loader.find_module(module_name).load_module(module_name)
print 'Found submodule %s ' % module_name
print module_name
'''
src_data = pkgutil.get_data('Assignment.prod',"Fix.py")
if src_data == None:
print "No source data"
else:
print src_data
#mod = loader.load_module(myDrive+myPath+"Fix.py")
myString = "diff --git a/rtb0006/CA01/prod/__init__.pyc b/rtb0006/CA01/prod/__init__.pyc"
testSuite = TestLoader().discover("g:\\git\\6700Spring16\\CA05\\submissions\\danieljames_1246453_74826857_jhd0008\\softwareProcess\\SoftwareProcess\\Assignment\\test", pattern="*.py")
ImpImporter("g:\\git\\6700Spring16\\CA05\\submissions\\danieljames_1246453_74826857_jhd00085\\softwareProcess\\SoftwareProcess\\Assignment\\prod")
print testSuite.countTestCases()
for p in sys.path:
print p
result = TextTestRunner().run(testSuite)
print result.testsRun
'''
'''
def module_finder_test():
from modulefinder import ModuleFinder
finder = ModuleFinder()
finder.run_script('takscollector.py')
print 'Loaded modules:'
for name, mod in finder.modules.iteritems():
print '%s: ' % name,
print ','.join(mod.globalnames.keys()[:3])
print '-' * 50
print 'Modules not imported:'
print '\n'.join(finder.badmodules.iterkeys())
def read_config(self):
"""
We have to split modules by apps.
We use ModuleFinder to get all imports from module.
Than we select those not starting with "/usr".
"""
self.apps = {}
self.apps_rev = {}
if (self.config is None or
not isinstance(self.config, basestring)):
return
finder = ModuleFinder()
# map: app -> modules
apps = None
with open(self.config, 'r') as f:
apps = json.load(f)
if apps is None:
log.debug("Apps are empty. Try setting modules in config: %s" % (
self.config))
f.close()
i = 0
if not isinstance(apps, list):
return
for t in apps:
if not isinstance(t, list):
continue
self.apps[i] = set()
for module in t:
if not isinstance(module, basestring):
continue
self.apps[i].add(module)
try:
finder.run_script(module)
modules = [m.__file__.replace('.pyo', '.py')
for m in finder.modules.values()
if (hasattr(m, "__file__") and
isinstance(m.__file__, basestring) and
not m.__file__.startswith('/usr'))]
self.apps[i].update(modules)
except Exception as e:
log.debug(str(e))
i += 1
# Reverse map: module -> apps
for i, modules in self.apps.items():
for module in modules:
if module not in self.apps_rev:
self.apps_rev[module] = set()
self.apps_rev[module].add(i)
def _analyze_file_deps(build_zip_descriptor, file_to_analyze):
"""
Returns a path normalized list of files a file given by file_to_analyze
depends upon.
"""
mf = ModuleFinder(excludes=build_zip_descriptor.exclude_modules)
mf.run_script(file_to_analyze)
file_list = [ ]
for module_name in mf.modules.keys():
module_file = mf.modules[module_name].__file__
if not module_file:
continue
file_list.append(os.path.abspath(module_file))
return file_list
def main():
description="Convenience script to generate HTML documentation using pydoc"
parser = argparse.ArgumentParser(description=description)
parser.add_argument('--out', required=True, metavar="PATH",
help="Directory in which to write HTML output files.")
parser.add_argument('--recursive', action='store_true', default=False,
help="Recursively import documentation for dependencies. If not recursive, zcall documents will contain broken links to standard modules. Recursive mode generates approximately 180 HTML files comprising 6 MB of data.")
parser.add_argument('--verbose', action='store_true', default=False,
help="Write pydoc information to stdout.")
args = vars(parser.parse_args())
recursive = args['recursive']
verbose = args['verbose']
if not verbose: # suppress stdout chatter from pydoc.writedoc
sys.stdout = open('/dev/null', 'w')
localDir = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.abspath(localDir+"/..")) # import from zcall dir
zcallDir = os.path.abspath(localDir+"/../zcall")
outDir = os.path.abspath(args['out'])
if not (os.access(outDir, os.W_OK) and os.path.isdir(outDir)):
msg = "ERROR: Output path "+outDir+" is not a writable directory.\n"
sys.stderr.write(msg)
sys.exit(1)
os.chdir(outDir)
import zcall
pydoc.writedoc(zcall)
modules = set()
zcall = set()
scripts = []
mf = ModuleFinder()
for script in os.listdir(zcallDir):
if re.search("\.py$", script) and script!="__init__.py":
words = re.split("\.", script)
words.pop()
scriptName = (".".join(words)) # name without .py suffix
modules.add("zcall."+scriptName)
zcall.add(scriptName)
scripts.append(script)
if recursive:
for script in scripts:
mf.run_script(os.path.join(zcallDir, script))
for name, mod in mf.modules.iteritems():
if name not in zcall: modules.add(name)
for module in modules:
pydoc.writedoc(import_module(module))
def _get_dependencies(entry_point):
# Dynamically find all imported modules
from modulefinder import ModuleFinder
finder = ModuleFinder()
finder.run_script(entry_point)
imports = list(set([v.__file__.split('site-packages/', 1)[-1].split('/')[0]
for (k, v) in finder.modules.items()
if v.__file__ is not None and "site-packages" in v.__file__]))
# Get just the modules, ignore the so and py now (maybe useful for calls to add_file)
modules = [i.split('.py')[0] for i in imports if ".so" not in i]
so_files = list(set([v.__file__
for (k, v) in finder.modules.items()
if v.__file__ is not None and "site-packages" in
v.__file__ and ".so" in v.__file__]))
return set(modules), so_files
def main():
finder = ModuleFinder()
finder.run_script('sample.py')
print('dir ModuleFinder: {}'.format(dir(finder)))
for name, mod in finder.modules.items():
print("type:{}".format(type(mod)))
print("dir module object:{}".format(dir(mod)))
for name, mod in finder.modules.items():
print('-'*20)
print("name:%s"%name)
print("globalnames:%s"%mod.globalnames)
print("modules:%s"%','.join(list(mod.globalnames.keys())))
print("starimports:{}".format(mod.starimports))
print("bad modules:{}".format(','.join(finder.badmodules.keys())))
def find_modules(self, scriptname, excludes=[], verbose=False):
""" Find the modules we'd want to include in the
distribution.
"""
path = sys.path[:]
path.insert(0, os.path.dirname(scriptname))
mf = ModuleFinder(path=path, excludes=excludes)
mf.run_script(scriptname)
modulenames = mf.modules.keys()
modulenames.sort()
self.log("Looking for modules used by '%s'...\n" % scriptname)
log_format = "%-2s %-30s %s"
self.log(log_format % ('', 'Module name', 'Module location'))
self.log(log_format % ('--', '-' * 30, '-' * 30))
modules = []
for name in modulenames:
m = mf.modules[name]
# builtin
#
if not m.__file__: continue
mpath = os.path.normcase(m.__file__)
# Modules in Python distribution.
# Pass on only those that live in site-packages
#
if mpath.startswith(self.site_packages):
pass
elif mpath.startswith(self.sys_prefix):
continue
type = "P" if m.__path__ else "m"
modules.append((name, type, m.__file__))
self.log(log_format % (type, name, m.__file__))
self.log("")
return modules
def _getImported_all(cls, moduleObj, isRecursive, allSubs=[]):
if cls.isBuiltin(moduleObj) or cls.isStock(moduleObj):
return
if moduleObj.__name__ == "__main__":
return
for i in allSubs:
if i.__name__ == moduleObj.__name__ and i.__file__ == moduleObj.__file__:
return
# print("adding " + str(moduleObj))
allSubs.append(moduleObj)
if isRecursive or allSubs == []:
# add path relative to current module
newPaths = [os.path.split(cls.convert_objectToFullPath(moduleObj))[0]] + sys.path
finder = ModuleFinder(path=newPaths)
finder.run_script(cls.convert_objectToFullPath(moduleObj))
for subModuleName, subModuleObj in finder.modules.iteritems():
# print("discovered " + str(subModuleObj))
cls._getImported_all(subModuleObj, True, allSubs)
return allSubs
def describe_python_module(type_, location, entities, pkgdb=None):
from modulefinder import ModuleFinder
spec = dict(location=location)
if location.endswith('.so'):
# treat binary extension as a library
return describe_binary(type_, location, entities, pkgdb=pkgdb)
fpath = os.path.realpath(location)
spec['realpath'] = fpath
fhash = sha1sum(fpath)
spec['sha1sum'] = fhash
if fhash in entities:
# do not process twice
return fhash
else:
entities[fhash] = spec
lgr.debug("describe %s at '%s' (%s)" % (type_, fpath, fhash))
spec['type'] = type_
# find all related modules
modfind = ModuleFinder()
try:
modfind.run_script(location)
except ImportError as e:
lgr.warning("cannot determine Python module dependencies of %s (%s)"
% (fpath, e))
if len(modfind.modules):
spec['depmods'] = []
for modname, mod in iteritems(modfind.modules):
if not mod.__file__:
# probably builtin
continue
# XXX STOP HERE FOR NOW UNTIL THERE IS A WAY TO CONTROL THE RECURSION DEPTH
continue
#spec['depmods'].append(
# describe_python_module('python_module', mod.__file__, entities,
# pkgdb=pkgdb))
return fhash
def mymodsImported(scriptfile):
'''导入的模块哪些是通过环境变量找到的？'''
try:
dirs = os.getenv('PYTHONPATH').split(':')
except AttributeError:
return []
if not dirs:
return []
from modulefinder import ModuleFinder
finder = ModuleFinder()
finder.run_script(scriptfile)
def filterdir(mod):
file = mod.__file__
if not file:
return False
for i in dirs:
if file.startswith(i):
return True
return False
return [m for m in finder.modules.values() if filterdir(m)]
def describe_python_module(type_, location, entities, pkgdb=None):
from modulefinder import ModuleFinder
spec = dict(location=location)
if location.endswith(".so"):
# treat binary extension as a library
return describe_binary(type_, location, entities, pkgdb=pkgdb)
fpath = os.path.realpath(location)
spec["realpath"] = fpath
fhash = sha1sum(fpath)
spec["sha1sum"] = fhash
if fhash in entities:
# do not process twice
return fhash
else:
entities[fhash] = spec
lgr.debug("describe %s at '%s' (%s)" % (type_, fpath, fhash))
spec["type"] = type_
# find all related modules
modfind = ModuleFinder()
try:
modfind.run_script(location)
except ImportError, e:
lgr.warning("cannot determine Python module dependencies of %s (%s)" % (fpath, e))
def new_mission(decorated_function):
'''Decorator that starts a new mission'''
global FINDER
global FINDER_PATH
if hasattr(decorated_function, 'decorated_function'):
path = inspect.getfile(decorated_function.decorated_function)
else:
path = inspect.getfile(decorated_function)
if path != FINDER_PATH:
FINDER = ModuleFinder()
FINDER.run_script(path)
FINDER_PATH = path
def wrapper(*args, **kw):
'''Wrapper for the new_mission decorator'''
try:
return decorated_function(*args, **kw)
finally:
finish_mission()
wrapper.__name__ = decorated_function.__name__
wrapper.__doc__ = decorated_function.__doc__
return wrapper
def ModuleFind(self):
from modulefinder import ModuleFinder
from VGenesDialogues import openFile
import os
Pathname = openFile(self, 'CSV')
workingdir, filename = os.path.split(Pathname)
os.chdir(workingdir)
# import filename
finder = ModuleFinder()
finder.run_script(filename)
Doc = 'Loaded modules:\n'
for name, mod in finder.modules.items():
# Doc('%s: ' % name, end=''\n)
Doc += (','.join(list(mod.globalnames.keys())[:3]))
Doc += '\n'
# Doc
Doc +=('-'*50)
from modulefinder import ModuleFinder
finder = ModuleFinder()
finder.run_script('kristian.py')
print 'Loaded modules:'
for name, mod in finder.modules.iteritems():
print '%s: ' % name,
print ','.join(mod.globalnames.keys()[:3])
print '-'*50
print 'Modules not imported:'
print '\n'.join(finder.badmodules.iterkeys())
def python_import_scanner(node, env, path):
# this is the scanner we use to decide if the doctests for a PYC should be
# re-run; it's handed a PYC and other filenames, usually in the build
# directory; but, because SCons doesn't guarantee that a target has been
# built prior to calling the scanner for that target (see logging from
# node_exists_check() and the SCons man-page regarding scanners) we have
# to be clever; in this case, clever means figuring out the PY file in the
# source that corresponds to the target PYC, and then using modulefinder
# to get "dependencies"; these of course are in the source tree, so we
# have to put the variant-build prefix back on them.... so, this scanner
# knows way too much about what's going on with SCons
debug_me = False
add_managed_source(node, env, path)
if debug_me: print 'python_import_scanner:', 'path:', tuple(str(x) for x in path)
if debug_me: print 'python_import_scanner:', 'node:', repr(str(node)), repr(node.path), repr(node.abspath)
if not (node.path.endswith('.pyc') or node.path.endswith('.py')):
# we allow non python files, but we don't scan them, but we do list
# them as dependencies; this causes them to get copied to the build
# tree early enough for tests; see DocTest builder which allows
# multiple sources
assert not node.path.endswith('.pyo')
if debug_me: print '+', repr(node.path)
# These files may have been prefixed with '#' in the SConscript file,
# in which case they won't have the build path prefix on them at this
# point. Or the SConscript may have used a relative name without a
# '#', in which case they will have the prefix, so we have to check here.
prefix = '#' if node.path.startswith(onyx_build) else os.path.join('#', onyx_build)
dependency = os.path.join(prefix, node.path)
if debug_me: print ' ', repr(dependency)
return [dependency]
# a nasty hack: DocTest uses this scanner, so it's called with PYC files
# in the build tree *that may not exist yet* (as of SCons 1.2.0), so we
# strip of the variant build portion of the name
if node.path.startswith(onyx_build):
script_filename = onyx_home + os.sep + node.path[len(onyx_build + os.sep):]
else:
script_filename = node.abspath
# a bit of a hack: DocTest uses this scanner, so it's called with PYC
# files, so we fiddle filenames so that run_script, which needs PY files
# (the way we're using it anyway) works
if script_filename.endswith('.pyc'):
script_filename = script_filename[:-1]
assert script_filename.endswith('.py')
if debug_me: print 'script_filename:', script_filename
finder = ModuleFinder(list(path))
finder.run_script(script_filename)
# argggg, ModuleFinder doesn't seem to implement a documented
# programatic access to the dependencies, so, after looking at
# the implementation of ModuleFinder.report() from Python 2.5.1,
# we do the following to iterate through what it's learned
dependencies = set()
for value in finder.modules.values():
filename = value.__file__
if filename is not None:
if debug_me: print '-', repr(filename)
if filename.lower().startswith(onyx_home_l):
filename = filename[len(onyx_home_l):]
if debug_me: print ' ', repr(filename)
assert filename.lower().endswith('.py')
# returning filenames in the build tree causes the dependencies
# to get built; as of SCons 1.2.0, alternatives, like returning
# the filenames in the source tree, do not work; this feels like
# a very grey area of SCons, that is, running the scanner on
# targets of other builders rather than on sources, and returning
# '#' prefixed filenames
dependency = os.path.join('#', onyx_build, filename)
#dependency = os.path.join('#', filename)
if debug_me: print ' ', repr(dependency)
dependencies.add(dependency)
# also report about PYCs so that dependencies get built and installed
dependency += 'c'
if debug_me: print ' ', repr(dependency)
dependencies.add(dependency)
return sorted(dependencies)
from modulefinder import ModuleFinder
finder = ModuleFinder()
finder.run_script('__main__.py')
print 'Loaded modules: '
for name, mod in finder.modules.iteritems():
print '%s: ' % name,
print ','.join(mod.globalnames.keys()[:3])
print '-'*50
print 'Modules not imported: '
print '\n'.join(finder.badmodules.iterkeys())
# print name, ' \t', path
return path
# Bodge the path setup for finding the imports in the file in question
# Pop the apparent path to this file off the front and append the cwd
first = sys.path.pop(0)
sys.path.insert(0, os.getcwd())
finder = ModuleFinder()
mainName = sys.argv[1]
absPath = os.path.abspath(mainName)
basePath = os.path.dirname(absPath)
missingIncludes = {}
try:
finder.run_script(mainName)
except Exception as ex:
"Probably an error in the user code"
if not ex.filename.startswith(basePath):
raise
fileName = ex.filename[len(basePath) + 1:]
missingIncludes['.error'] = { 'file': fileName,
'line': ex.lineno,
'msg': ex.msg
}
includeOriginal = True
allMissingImports = invertBadmodules(finder)
#print 'All missing imports nice:\n', json.dumps(allMissingImports).replace(']', ']\n')
