def find_failed_imports(module_path):
"""Returns all modules that failed to import when loading a particular
module."""
finder = ModuleFinder()
finder.run_script(module_path)
bad_modules = dict(finder.badmodules)
return return_missing_base_modules(bad_modules)
def show_import_report(pathname=None, show_loaded=False, show_not_imported=False, fullreport=False):
filename = basename(pathname)
finder = ModuleFinder()
finder.run_script(pathname)
if show_loaded or show_not_imported or fullreport:
print('-'*50)
else:
print("Missing options what to do. Use 'modulereport --help'")
if show_loaded:
print("Loaded modules for: " + filename)
print('-'*50)
for name, mod in finder.modules.items():
line = "{pyname}: {modlist}".format(pyname=name, modlist=','.join(list(mod.globalnames.keys())[:3]))
print(line)
print('-'*50)
if show_not_imported:
print("Modules missing for: " + filename)
print('-'*50)
print('\n'.join(finder.badmodules.keys()))
print('-'*50)
if fullreport:
print("Full report:")
print('-'*50)
finder.report()
def _reload(obj, name=None):
obj_type = type(obj)
module = inspect.getmodule(obj)
sys_modules_frozen = sys.modules.copy()
sys_modules_sorted = sorted(sys_modules_frozen.items(),
key=lambda t: len(t[0]),
reverse=True)
if not module:
for modname, module in sys_modules_sorted:
if modname.split('.')[0] in STDLIB_NAMES:
continue
candidate = getattr(module, name, Sentinel)
if (candidate is not Sentinel and isinstance(candidate, obj_type)):
break
else:
module = None
if not module:
raise TypeError()
finder = ModuleFinder()
finder.run_script(module.__file__)
for m in finder.modules:
if m.split('.')[0] in STDLIB_NAMES:
continue
try:
reload(sys_modules_frozen[m])
except KeyError:
pass
return reload(module)
def find_and_register_qabbage_tasks(outer_scope, exclude_tests=True):
"""
Walks the directory and finds all of the scripts that import
qabbage tasks.
:return: file paths with qabbage tasks
"""
all_files = []
qabbage_files = []
for root, dirs, files in os.walk("."):
path = root.split('/')
for file in files:
_, file_extension = os.path.splitext(file)
if file_extension == '.py':
all_files.append(os.path.join(*path, file))
finder = ModuleFinder()
for file in filter(lambda x: 'test_' not in x, all_files):
finder.run_script(file)
for name, mod in finder.modules.items():
if name == 'qabbage':
qabbage_files.append(file)
promises = load_qabbage_modules(qabbage_files)
for promise in promises:
outer_scope[promise[0]] = promise[1]
return promises
def _modules_from_buffer(self, buffer):
finder = ModuleFinder(excludes=self.excludes)
bufdir = os.path.dirname(buffer.name)
sys.path.insert(0, bufdir)
try:
finder.load_module('__main__', buffer, '__main__.py', ('', 'r', 1))
finally:
sys.path.remove(bufdir)
self.badmodules.update(finder.badmodules.keys())
stdlib = sysconfig.get_python_lib(standard_lib=True)
stdlib_local = sysconfig.get_python_lib(standard_lib=True,
prefix='/usr/local')
dist_packages = sysconfig.get_python_lib()
stdlib_user_local = sysconfig.get_python_lib(
standard_lib=True, prefix=f'{os.path.expanduser("~")}/.local')
for module in finder.modules.values():
if module.__file__ is None:
continue
elif module.__file__.startswith(stdlib):
continue
elif module.__file__.startswith(stdlib_local):
continue
elif module.__file__.startswith(dist_packages):
continue
elif module.__file__.startswith(stdlib_user_local):
continue
elif module.__file__ == buffer.name:
continue
elif module.__file__ == '__main__.py':
continue
elif module.__file__ in self.modules:
continue
self.modules.add(module.__file__)
yield module
def get_node_data(general_path, file_paths, verbal=False):
nodes = []
for file_data in file_paths:
file_path = file_data[0]
node = FileNode()
node.has_file = os.path.isfile(file_path)
if verbal:
if node.has_file:
print("Getting data from file:", file_path)
else:
print("Getting data about module without file:", file_path)
try:
node.name = file_path.rsplit(
os.path.sep,
1)[1] if node.has_file else file_path.rsplit(' ')[0]
node.module_name = file_data[1]
node.path = file_path
if node.name == "__init__.py":
node.name = file_path.rsplit(os.path.sep, 2)[1]
node.size = os.path.getsize(file_path) if node.has_file else None
module_finder = ModuleFinder()
node.fill_imports_and_members(module_finder)
nodes.append(node)
except IOError:
print("File with path", file_path,
"could not be opened, ignoring...")
if verbal:
print("Filling node dependencies...")
for node in nodes:
node.fill_dependencies(general_path, nodes)
return nodes
def get_files_with_modules(directory_path, search_recursively=False):
files_with_modules = []
if search_recursively:
original_file_paths = gfp.get_paths(directory_path)
unique_file_paths = []
for file_path_index, file_path in enumerate(original_file_paths):
files_with_modules.append([
file_path,
os.path.dirname(file_path).rsplit(os.path.sep, 1)[1]
])
finder = ModuleFinder()
for file_path in original_file_paths:
try:
finder.run_script(file_path)
except ImportError:
print("Could not load", file_path)
for key in finder.modules.items():
item_path = key[1].__file__ if key[
1].__file__ else key[1].__name__ + " (built-in)"
if '.' not in item_path or item_path.endswith('.py'):
item_data = [item_path, key[1].__name__]
if item_data[0] not in unique_file_paths and item_data[
0] not in original_file_paths:
unique_file_paths.append(item_data[0])
files_with_modules.append(item_data)
files_with_modules.sort(key=lambda item: item[0])
else:
files_with_modules = gfp.get_paths(directory_path)
for file_path_index, file_path in enumerate(files_with_modules):
files_with_modules[file_path_index] = [
file_path,
os.path.dirname(file_path).rsplit(os.path.sep, 1)[1]
]
files_with_modules.sort(key=lambda item: item[0])
return files_with_modules
def run(self):
p = self.cmdLineParams
finder = ModuleFinder()
finder.run_script(p.infile.name)
modDict = {}
for name, mod in finder.modules.items():
if mod.__file__ is None:
key = str(mod.__file__)
else:
key = os.path.abspath(mod.__file__)
if not key in modDict:
modDict[key] = []
modDict[key].append(name)
# sort
for key in sorted(modDict):
if p.restrictToProject and not key.startswith(__PROJECT_ROOT__):
continue
modules = modDict[key]
sys.stdout.write(key)
for moduleName in modules:
sys.stdout.write('\t')
sys.stdout.write(moduleName)
sys.stdout.write('\n')
def _get_dependencies(entry_point):
# Dynamically find all imported modules
from modulefinder import ModuleFinder
finder = ModuleFinder()
finder.run_script(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
entry_point))
imports = list(
set([
v.__file__.split('site-packages/', 1)[-1].split('/')[0]
for (k, v) in finder.modules.items()
if v.__file__ is not None and "site-packages" in v.__file__
]))
# Get just the modules, ignore the so and py now (maybe useful for calls to add_file)
modules = [i.split('.py')[0] for i in imports if ".so" not in i]
so_files = list(
set([
v.__file__ for (k, v) in finder.modules.items()
if v.__file__ is not None and "site-packages" in v.__file__
and ".so" in v.__file__
]))
return set(modules), so_files
def add_mocks(filename):
gp = mock.patch('ansible.module_utils.basic.get_platform').start()
gp.return_value = 'linux'
module_mock = mock.MagicMock()
mocks = []
for module_class in MODULE_CLASSES:
mocks.append(
mock.patch('ansible.module_utils.basic.AnsibleModule',
new=module_mock))
for m in mocks:
p = m.start()
p.side_effect = AnsibleModuleCallError()
finder = ModuleFinder()
try:
finder.run_script(filename)
except:
pass
sys_mock = mock.MagicMock()
sys_mock.__version__ = '0.0.0'
sys_mocks = []
for module, sources in finder.badmodules.items():
if module in sys.modules:
continue
if [s for s in sources if s[:7] in ['ansible', '__main_']]:
parts = module.split('.')
for i in range(len(parts)):
dotted = '.'.join(parts[:i + 1])
sys.modules[dotted] = sys_mock
sys_mocks.append(dotted)
return module_mock, mocks, sys_mocks
def spider(self):
print '-' * 40
while len(self.full_path) != 0:
full_path = self.full_path.pop()
self.full_path_discard.append(full_path)
finder = ModuleFinder()
finder.run_script(full_path)
for name, mod in finder.modules.iteritems():
full_path = str(mod.__file__).replace('.', '/')
if full_path not in self.full_path_discard:
if full_path.startswith('/home/alfred/pipline/code/'):
self.result.append(name.replace('.', '/'))
if not full_path.startswith('/home/alfred/pipline/code/U'):
self.full_path.append(full_path)
else:
self.full_path_discard.append(full_path)
names = finder.any_missing()
for name in names:
for folder in self.all_path:
try:
full_path = folder + name + '.py'
if os.path.isfile(full_path):
if full_path not in self.full_path_discard:
self.full_path.append(full_path)
self.result.append(full_path)
except:
pass
for item in sorted(set(self.result)):
print item
print '='*100
def get_files_data(general_path, file_paths):
module_finder = ModuleFinder()
files = []
for file_path in file_paths:
spec = importlib.util.spec_from_file_location(file_path, file_path)
module_from_spec = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module_from_spec)
functions_list = [
function for function in getmembers(module_from_spec)
if isfunction(function[1])
]
module = file.Module(
file_path.rsplit(os.path.sep, 1)[1].rsplit('.')[0], file_path)
module.size = os.path.getsize(file_path)
module_finder.run_script(module.path)
module.fill_imports_and_functions(general_path, module_finder)
module.fill_dependecies()
files.append(module)
for element in files:
element.fill_dependecies(files)
return files
def get_imports(self, file):
finder = ModuleFinder()
finder.run_script(file)
im = []
for name, mod in finder.modules.items():
im.append(name)
return im
def make_bom():
"""Generate a bill of materials and return the tt object."""
finder = ModuleFinder()
finder.run_script(sys.argv[0])
tt = tytable.ttable()
tt.add_option(tt.OPTION_LONGTABLE)
tt.add_head(['name', 'ver', 'path', 'bytes', 'lines', 'sha-1'])
tt.set_col_alignmnets("lllrrl")
for inc in [False, True]:
for name, mod in sorted(finder.modules.items()):
if system_module(mod) != inc:
continue # don't use this one
stats = file_stats(mod.__file__)
ver = mod.globalnames.get('__version__', '---')
if ver == 1 and name in sys.modules:
# It has a version number; get it.
try:
ver = sys.modules[name].__version__
except AttributeError:
ver = '---'
fname = mod.__file__
if type(name) != str:
name = ""
if type(fname) != str:
fname = ""
tt.add_data([
latex_tools.latex_escape(name), ver,
latex_tools.latex_escape(fname), stats[0], stats[1], stats[2]
])
tt.add_data(tt.HR)
return tt
def print_type_dir():
finder = ModuleFinder()
finder.run_script('from_import.py')
print('dir ModuleFinder: {}'.format(dir(finder)))
"""
dir ModuleFinder: ['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__',
'__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__',
'__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__',
'__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__',
'__weakref__', '_add_badmodule', '_safe_import_hook', 'add_module', 'any_missing',
'any_missing_maybe', 'badmodules', 'debug', 'determine_parent', 'ensure_fromlist',
'excludes', 'find_all_submodules', 'find_head_package', 'find_module', 'import_hook',
'import_module', 'indent', 'load_file', 'load_module', 'load_package', 'load_tail',
'modules', 'msg', 'msgin', 'msgout', 'path', 'processed_paths', 'replace_paths',
'replace_paths_in_code', 'report', 'run_script', 'scan_code', 'scan_opcodes']
"""
for name, mod in finder.modules.items():
print('type:{}'.format(type(mod)))
#=> type:<class 'modulefinder.Module'>
print('dir Module object:{}'.format(dir(mod)))
"""
dir:['__class__', '__code__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__',
'__file__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__',
'__init_subclass__', '__le__', '__lt__', '__module__', '__name__', '__ne__', '__new__',
'__path__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__',
'__str__', '__subclasshook__', '__weakref__', 'globalnames', 'starimports']
"""
return
def _get_source_of_module(module):
# print("module", module)
try:
module_source = inspect.getsource(module)
except OSError:
return {}
module_finder = ModuleFinder()
module_finder.run_script(module.__file__)
# ret = ast.parse(base_source)
# print(dir(ret))
accum = {
module.__name__: module_source
}
for child_module_name, child_module_tuple in module_finder.modules.items():
if child_module_name == "__main__":
continue
if child_module_name in accum:
continue
# found_module = imp.find_module(child_module_tuple.__name__)
# child_module = imp.load_module(child_module_tuple.__name__, *found_module)
child_module = importlib.import_module(child_module_tuple.__name__)
print("processing module", child_module_tuple.__name__, child_module)
accum.update(_get_source_of_module(child_module))
return accum
def test_dependencies(self):
"""
EXTREMELY SLOW
"""
path = os.path.join('..', 'lingcorpora', 'corpora',
'%s_corpus.py' % self.lang)
core_path = os.path.join(
'..',
'lingcorpora',
'corpus.py',
)
# legit_deps = pickle.load(open('dependencies.pickle', 'rb'))
mf = ModuleFinder()
mf.run_script(init_path)
legit_deps = set(mf.modules.keys())
mf.run_script(path)
deps = set(mf.modules.keys())
self.assertEqual(len(deps.difference(legit_deps)), 0)
del deps, legit_deps
def get_module_import_dict(object, scope='project', remove_packages=True):
"""
Given some code object (or the full name of a module), find all the modules that must be imported for this object to
run.
:param object:
:param scope: One of:
'package': Only collect modules from same root package (eg root package of foo.bar.baz is foo.
'project': Only collect modules in same project (ie, modules whose root package is in the same directoy as object's root package)
'all': Collect all dependent modueles (not recommended, as this will, for example include a bunch of numpy code).
:return: A dict<module_name: module_path>
"""
assert scope in ('package', 'project', 'all')
if isinstance(object, (list, tuple)):
dicts, names = zip(
*[get_module_import_dict(ob, scope=scope) for ob in object])
return {k: v for d in dicts for k, v in d.iteritems()}, names
elif isinstance(object, basestring):
module = import_module(object)
else:
module = inspect.getmodule(object)
module_file = get_src_file(module)
finder = ModuleFinder()
this_package = module.__name__.split('.')[0]
LOGGER.info(
'Scanning Dependent Modules in {}. This may take some time...'.format(
this_package))
finder.run_script(module_file)
module_files = {
name: get_src_file(module)
for name, module in finder.modules.iteritems()
if module.__file__ is not None
}
module_files[module.__name__] = get_src_file(
module) # Don't forget yourself!
# module_files =
if scope == 'package':
module_files = {
name: mod
for name, mod in module_files.iteritems()
if name.split('.')[0] == this_package
}
elif scope == 'project':
base_dir = os.path.dirname(os.path.dirname(module.__file__))
module_files = {
name: mod
for name, mod in module_files.iteritems()
if mod.startswith(base_dir)
}
LOGGER.info('Scan Complete. {} dependent modules found.'.format(
len(module_files)))
# module_name_to_module_path = {name: get_src_file(m) for name, m in modules.iteritems()}
if remove_packages:
module_files = {
name: path
for name, path in module_files.iteritems() if not (
path.endswith('__init__.py') or path.endswith('__init__.pyc'))
}
return module_files, module.__name__
def _find_modules(module_name):
""" Ideally we want to find all the modules imported by the provided top level module
The modules that are contained in the top level are returned in submodules
Other modules that are dependancies should be returned in others.
"""
_others = set()
_submodules = set()
mod = importlib.import_module(module_name)
if False:
from modulefinder import ModuleFinder
finder = ModuleFinder()
# finder.load_file(mod.__file__)
pathname = Path(mod.__file__)
if pathname.name == '__init__.py':
pathname = pathname.parent
finder.load_package(module_name, str(pathname))
modules = set(finder.modules) | set(finder.badmodules)
modules ^= {'__main__'}
_others = {name if module_name not in name else name.split('.')[0]
for name in finder.modules.keys()}
_submodules = {name for name in finder.modules.keys() if module_name in name}
if True:
def get_imports(_module_name):
import dis
mod = importlib.import_module(_module_name)
instructions = dis.get_instructions(inspect.getsource(mod))
imports = [i.argval for i in instructions if 'IMPORT' in i.opname]
subs = [getattr(mod, i, None) or importlib.import_module(i, _module_name) or i for i in imports]
return subs
# Just get all submodules from sys.modules
for key, val in sys.modules.items():
if module_name == key or key.startswith(module_name + "."):
_submodules |= {val.__name__}
for attr, val in mod.__dict__.items():
# This misses recursive submodules, and ones that are imported like from <module> import <function>
if inspect.ismodule(val):
if module_name in val.__name__:
_submodules |= {val.__name__}
else:
_others |= {val.__name__}
_submodules ^= {module_name}
for m in _submodules:
s, o = _find_modules(m)
_submodules |= s
_others |= o
return _submodules, _others
def zip_std_lib(src_module, dst_file):
"""Compiles the Python standard library modules used by the source module
and outputs to zip file."""
finder = ModuleFinder()
finder.run_script(src_module)
modules = set()
print('Writing dependencies to "%s"...' % DEP_OUT)
with open(DEP_OUT, 'w') as f:
for name, mod in finder.modules.items():
print('%s: ' % name, end='', file=f)
print(mod.__file__, file=f)
if mod.__file__ is None:
continue
path = os.path.realpath(mod.__file__)
if not path.startswith(os.path.normpath(STD_LIB)):
continue
while (os.path.dirname(path) != os.path.normpath(STD_LIB)):
path = os.path.dirname(path)
if os.path.isfile(path):
modules.add(path)
elif os.path.isdir(path):
for root, dirs, files in os.walk(path):
for i in files:
modules.add(os.path.join(root, i))
print('-' * 50, file=f)
print('### Modules NOT imported ###', file=f)
print('\n'.join(finder.badmodules.keys()), file=f)
modules = sorted([
i for i in modules if i.endswith((
'.py', '.pyc')) and not os.path.dirname(i).endswith('__pycache__')
])
print('Writing standard library to "%s"...' % dst_file)
with zipfile.ZipFile(dst_file, 'w', compression=zipfile.ZIP_DEFLATED) as z:
for i in modules:
root, ext = os.path.splitext(i)
if ext == '.py':
arcname = os.path.relpath(root, STD_LIB) + '.pyc'
pyc = create_pyc(i)
else:
arcname = os.path.relpath(i, STD_LIB)
with open(i, 'rb') as f:
pyc = f.read()
z.writestr(arcname, pyc)
def find_failed_imports_by_directory(directory):
"""Returns all modules that failed to import when loading all modules below
a directory."""
finder = ModuleFinder()
py_files = _find_all_python_modules(directory)
for f in py_files:
finder.run_script(f)
bad_modules = dict(finder.badmodules)
return return_missing_base_modules(bad_modules)
def on_modified(self, event):
if not event.src_path.endswith('.py'):
return
print(f'modules using {event.src_path}')
finder = ModuleFinder()
finder.load_file(event.src_path)
for name, mod in finder.modules.items():
print(name)
print(mod)
def get_imported_module_paths(module_path):
finder = ModuleFinder([os.path.dirname(module_path)])
finder.run_script(module_path)
imported_modules = []
for module_name, module in finder.modules.items():
if module_name != '__main__':
path = getattr(module, '__file__', None)
if path is not None and os.path.isfile(path):
imported_modules.append(path)
return imported_modules
def check_module(self, module_name ):
from modulefinder import ModuleFinder
finder = ModuleFinder(debug=2)
finder.run_script(module_name)
for name, mod in finder.modules.items():
try:
__import__(name, fromlist=mod.globalnames.keys(),level=1)
sys.stdout.write('.')
except ImportError as e:
print("ERROR IMPORTING %s: " % name + " -- "+e.message)
def __init__(self, build_dir: Union[str, Path]):
if isinstance(build_dir, str):
self.build_dir = Path(build_dir)
else:
self.build_dir = build_dir
self.zip_build_dir = self.build_dir / "libs" / "lib"
self.dylib_dir = self.build_dir / "dist" / "lib" / "lib-dynload"
self.finder = ModuleFinder()
def __init__(self, gui_api, TabManger, plugin_manager, parent=None):
super(CreatePluginMenu, self).__init__(parent)
self.setupUi(self)
self.dgui = gui_api.gui_data
self.TabManager = TabManger
self.gui_api = gui_api
self.subscriberID = None
self.targetID = None
self.blockName = None
self.plugin_manager = plugin_manager
self.pluginTree.setDragEnabled(True)
self.pluginTree.setDropIndicatorShown(True)
self.setWindowTitle('Available Plugins')
model = PaPITreeModel()
model.setHorizontalHeaderLabels(['Name'])
self.pluginProxyModel = PaPITreeProxyModel(self)
self.pluginProxyModel.setSourceModel(model)
regex = QRegExp("*", Qt.CaseInsensitive, QRegExp.Wildcard)
self.pluginProxyModel.setFilterRegExp(regex)
self.pluginTree.setModel(self.pluginProxyModel)
self.pluginTree.setUniformRowHeights(True)
self.pluginTree.setSortingEnabled(True)
self.pluginTree.setStyleSheet(pc.TREE_CSS)
self.plugin_roots = {}
self.configuration_inputs = {}
self.pluginTree.clicked.connect(self.pluginItemChanged)
self.plugin_create_dialog = CreatePluginDialog(self.gui_api,
self.TabManager)
self.createButton.clicked.connect(self.show_create_plugin_dialog)
self.helpButton.clicked.connect(self.help_button_triggered)
self.finder = ModuleFinder()
self.pluginSearchText.textChanged.connect(
self.changed_search_plugin_text_field)
self.pluginSearchText.setFocus(Qt.OtherFocusReason)
self.helpButton.setText('')
self.helpButton.setIcon(get16Icon('help.png'))
self.helpButton.setToolTip(
'Opens the documentation for the currently selected plugin.')
def imports_metadata(path: Union[str, Path]) -> bool:
mf = ModuleFinder()
if isinstance(path, Path):
path = str(path)
mf.run_script(path)
modules = list(mod for name, mod in mf.modules.items()
if "metadata" in name)
if modules:
return True
else:
return False
def patched(self, project_dir, spec, sack):
mod = ModuleFinder()
for item in list(project_dir.glob('**/*.py')):
try:
mod.run_script(str(item))
for _, mod in mod.modules.items():
if mod.__file__ and mod.__file__.startswith("/usr/lib"):
spec.required_files.add(mod.__file__)
except ImportError as ie:
logging.warn("Exception was raised by ModuleFinder:\n" +
str(ie) + "\nOn file: " + str(item))
def test_inspect(self):
finder = ModuleFinder()
# finder.run_script(...)
print('Loaded modules:')
for name, mod in finder.modules.items():
print('%s: ' % name, end='')
print(','.join(list(mod.globalnames.keys())[:3]))
for name, obj in inspect.getmembers(datacentric,
lambda x: inspect.isclass(x)):
print(name)
print(inspect.getmembers(obj))
def make_module_list(input_file, output_file):
module_list = []
finder = ModuleFinder()
finder.run_script(input_file)
keys = sorted(finder.modules.keys())
for key in keys:
if finder.modules[key].__path__ and key.find(".") == -1:
module_list.append(key)
with open(output_file, "w") as f_o:
f_o.write('pip install ' + ' '.join(module_list))
print('pip install ' + ' '.join(module_list))
