def python(lines): # filter from lines src = '\n'.join(lines) code = compile(src, 'internal_imports', 'exec') exec(src) g = locals() modules = sorted(set(m for v in g.values() for m in (inspect.getmodule(v), ) if m), key=lambda m: m.__name__) # TODO modules -> pkgs = [] for m in modules: try: pkgname = m.__name__.split('.')[0] pkg = pkg_resources.get_distribution(pkgname) mf = modulefinder.ModuleFinder(m.__file__) print(list(mf.modules.items())) #pkgs.append(pkg) #embed() #print(pkg.project_name, pkg.version) pkgs.append((pkg.project_name, pkg.version)) except pkg_resources.DistributionNotFound: # TODO corelibs? print('WARNING: missing', m.__name__) embed() return pkgs
def sort_by_dependency(tp_modules): paths = [] for mod in tp_modules: paths.append(os.path.dirname(mod)) paths = sorted(list(set(paths))) tp_modules_with_main = [] lt = level_tree.Tree() for mod in tp_modules: finder = modulefinder.ModuleFinder(path=paths) finder.run_script(mod) for name, m in finder.modules.iteritems(): if name is "__main__": if "main" in m.globalnames.keys(): tp_modules_with_main.append(mod) continue if (m.__file__): lt.add_edge((mod, m.__file__)) else: lt.add_edge((mod, name)) lt.levelize() lt_levels_result = lt.get_level() #Sort the module by their dependency levels. lt_levels_result_sorted = sorted(lt_levels_result.iteritems(), key=operator.itemgetter(1)) lt_levels_result_sorted = [(f[0], f[0] in tp_modules_with_main) for f in lt_levels_result_sorted] return lt_levels_result_sorted
def find_modules(self, tartdir): from tart.bbutilities import blackberry_tart tartpydir = os.path.join(tartdir, 'bbutilities') searchpath = [tartpydir, self.project.root] for path in sys.path: if 'site-packages' in path: searchpath.append(path) finder = modulefinder.ModuleFinder(path=searchpath) finder.run_script(blackberry_tart.__file__) modules = set() for name, mod in finder.modules.items(): if mod.__file__ is None: # print('builtin:', name) continue # print(' {}'.format(name), mod.__file__) for base in searchpath: if mod.__file__.startswith(base): relpath = os.path.relpath(mod.__file__, base) modules.add((base, relpath)) break else: print('error?', mod.__file__) # modules.append(mod.__file__) # FIXME: should strip stdlib modules from here before showing print('Modules not found:', ', '.join(finder.badmodules)) # FIXME: true ugliness, plus it doesn't handle x86 if 'tart.dynload' in finder.badmodules: modules.add((tartpydir, 'lib-dynload/_opengl-arm.so')) return modules
def find_dep_modules(src): # Get the dependency list for each py files passed global dep_map global mod_type unresolved_dep_mods = [] py_files = [] dep_mods = [] if os.path.isdir(src): os.path.walk(src, get_py_pyd_files, py_files) else: py_files.append(src) for f in py_files: mf = modulefinder.ModuleFinder(path=[std_module_search_path]) mf.run_script(f) mod_list = [] for mod in mf.modules.iteritems(): mod_list.append(mod[0]) for mod in mf.badmodules.iteritems(): mod_list.append(mod[0]) mod_list.sort() # The dependencies for "directory entries" are taken to be the sum of # all the dependencies of the file in that directory if src not in dirname_list: mod_map = {} f = os.path.splitext(f)[0] f = f.replace(std_module_search_path + '\\', '') f = f.replace('\\', '.') mod_map['deps'] = mod_list dep_map[f].update(mod_map) else: dep_mods = list(set(dep_mods + mod_list)) return dep_mods
def _do_test(self, info, report=False): import_this, modules, missing, maybe_missing, source = info create_package(source) try: mf = modulefinder.ModuleFinder(path=TEST_PATH) mf.import_hook(import_this) if report: mf.report() ## # This wouldn't work in general when executed several times: ## opath = sys.path[:] ## sys.path = TEST_PATH ## try: ## __import__(import_this) ## except: ## import traceback; traceback.print_exc() ## sys.path = opath ## return modules = set(modules) found = set(mf.modules.keys()) more = list(found - modules) less = list(modules - found) # check if we found what we expected, not more, not less self.assertEqual((more, less), ([], [])) # check for missing and maybe missing modules bad, maybe = mf.any_missing_maybe() self.assertEqual(bad, missing) self.assertEqual(maybe, maybe_missing) finally: distutils.dir_util.remove_tree(TEST_DIR)
def python_getImportedUserlandModules(source, sourceFilename): """ Returns a list of direct (userland) dependencies (source is an ATS/Module source code), as a list of module names (not filenames !) @type source: utf-8 string @param source: Python source code @type sourceFilename: utf-8 string @param sourceFilename: the filename of this source, if known @rtype: list of strings @returns: a list of module names ('mylibs.mymodule', 'amodule', etc) """ mf = modulefinder.ModuleFinder() fp = StringIO.StringIO(source) mf.load_module('__main__', fp, '<string>', ("", "r", imp.PY_SOURCE)) rawdeps = mf.any_missing() directdeps = [] # Let's filter and retrieve only missing imports from __main__ # i.e. direct missing dependencies for name in rawdeps: mods = mf.badmodules[name].keys() if '__main__' in mods: directdeps.append(name) getLogger().info('Userland modules imported by file %s:\n%s' % (sourceFilename, "\n".join(directdeps))) return directdeps
def _find_modules(self): """Use modulefinder to find modules Return finder.modules dict """ metascript = self.metascript excludes = set() last_name = "A" * 255 # invalid name max_atemps = 1000 for i in range(max_atemps): try: finder = modulefinder.ModuleFinder(excludes=excludes) finder.run_script(metascript.path) print(metascript.path) return finder.modules except SyntaxError as exc: name = exc.filename.split("site-packages/")[-1] # pylint: disable=no-member name = name.replace(os.sep, ".") name = name[:name.rfind(".")] if last_name in name: last_name = last_name[last_name.find(".") + 1:] else: last_name = name excludes.add(last_name) print_msg(" skip module due syntax error: {} ({}/{})" .format(last_name, i + 1, max_atemps)) return {}
def _do_test(self, info, report=False, debug=0, replace_paths=[]): import_this, modules, missing, maybe_missing, source = info create_package(source) try: mf = modulefinder.ModuleFinder(path=TEST_PATH, debug=debug, replace_paths=replace_paths) mf.import_hook(import_this) if report: mf.report() ## # This wouldn't work in general when executed several times: ## opath = sys.path[:] ## sys.path = TEST_PATH ## try: ## __import__(import_this) ## except: ## import traceback; traceback.print_exc() ## sys.path = opath ## return modules = sorted(set(modules)) found = sorted(mf.modules) # check if we found what we expected, not more, not less self.assertEqual(found, modules) # check for missing and maybe missing modules bad, maybe = mf.any_missing_maybe() self.assertEqual(bad, missing) self.assertEqual(maybe, maybe_missing) finally: shutil.rmtree(TEST_DIR)
def sketch_accusation(targetpath, modulepaths, project_dirs): """Creates a graph of all the modules in `modulepaths` that import the module at `targetpath`. The directories used to resolve an import are `project_dirs` Args: targetpath (str): filepath of the module whose import status is being checked. modulepaths (List[str]): list of modules filepaths to check for imports. project_dirs (List[str]): list of directories used to resolve the imports Returns: networkx.Graph: graph of the module as nodes with their imports as edges. """ graph = networkx.Graph() # Calculate complexity and maintainability with open(targetpath) as source: size, color = scientist.get_size_color(source.read(), initsize=80) # Insert target module info target_info = {'shape':'cross' ,'name':targetpath, 'size':size, 'color':color} graph.add_node(targetpath, target_info) for modulepath in modulepaths: print('processing:\t', modulepath) finder = modulefinder.ModuleFinder(path=project_dirs) finder.run_script(modulepath) for module in finder.modules.values(): if (scientist.include_module(module) and module.__file__ == targetpath): with open(modulepath) as source: size, color = scientist.get_size_color(source.read(), initsize=80) module_info = {'shape':'square' ,'name':modulepath, 'size':size, 'color':color} graph.add_node(modulepath, module_info) graph.add_edge(modulepath, targetpath) return graph
def mod_data(opts, full): ''' Grab the module's data ''' ret = {} finder = modulefinder.ModuleFinder() try: finder.load_file(full) except ImportError as exc: print('ImportError - {0} (Reason: {1})'.format(full, exc), file=sys.stderr) return ret for name, mod in finder.modules.items(): basemod = name.split('.')[0] if basemod in ret: continue if basemod.startswith('_'): continue if not mod.__file__: continue if opts['bif'] not in mod.__file__: # Bif - skip continue if name == os.path.basename(mod.__file__)[:-3]: continue ret[basemod] = mod.__file__ for name, err in finder.badmodules.items(): basemod = name.split('.')[0] if basemod in ret: continue if basemod.startswith('_'): continue ret[basemod] = err return ret
def sketch_blocks(modulepaths, pkg_dirs): """Creates a graph of all the modules in `modulepaths` that are related to each other by their imports. The directories used to resolve an import is `pkg_dirs` Args: modulepaths (List[str]): list of modules filepaths to analyze. pkg_dirs (List[str]): list of directories used to resolve the imports Returns: networkx.Graph: graph of the modules as nodes with their imports as edges. """ attributes = init(pkg_dirs) graph = attributes['graph'] Python = 'python' graph.add_node(Python, attributes[Python]) for filepath in modulepaths: # bug - if the finder is not reinitialized, the previous modules.values() # are kept, thus been useless finder = modulefinder.ModuleFinder(path=pkg_dirs) print('processing:\t', filepath) # Calculate complexity and maintainability indexes with open(filepath) as source: size, color = scientist.get_size_color(source.read(), initsize=80) # Insert current module info module_info = {'shape':'square', 'name':filepath, 'size':size, 'color':color} graph.add_node(filepath, module_info) # Find module imports finder.run_script(filepath) for edge in scientist.compute_edges(filepath, Python, finder.modules.values(), finder.badmodules.keys()): graph.add_edge(*edge) return graph
def recurse_module(module, path): finder = modulefinder.ModuleFinder(path=list(user_paths())) finder.run_script(path) from disco.fileutils import get_valid_path return dict((name, get_valid_path(module.__file__)) for name, module in finder.modules.items() if name != '__main__' and module.__file__)
def print_imported_modules(filename): ''' Print modules imported by filename. Warning: Runs filename. >>> # how do we test this? how we generally use it: >>> # print_imported_modules(__file__) ''' import modulefinder if filename.endswith('.pyc') or filename.endswith('.pyo'): filename = filename[:-1] print(filename) finder = modulefinder.ModuleFinder() finder.run_script(filename) # See /usr/share/doc/python2.7/html/library/modulefinder.html print('Loaded modules:') for name in sorted(finder.modules.keys()): mod = finder.modules[name] print(f' {name}: {mod}') globalnames = list(mod.globalnames.keys()) modules = sorted(globalnames[:3]) print(','.join(modules)) print('Modules not imported:') keys = finder.badmodules.keys() for name in keys: print(f' {name}')
def get_dep_modules(test): # Cache results in case of repitition if test in _DEP_MODULES_CACHE: return _DEP_MODULES_CACHE[test] repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) test_location = os.path.join(repo_root, 'test', test + '.py') finder = modulefinder.ModuleFinder( # Ideally exclude all third party modules, to speed up calculation. excludes=[ 'scipy', 'numpy', 'numba', 'multiprocessing', 'sklearn', 'setuptools', 'hypothesis', 'llvmlite', 'joblib', 'email', 'importlib', 'unittest', 'urllib', 'json', 'collections', ], ) # HACK: some platforms default to ascii, so we can't just run_script :( with open(test_location, 'r', encoding='utf-8') as fp: finder.load_module('__main__', fp, test_location, ('', 'r', 1)) dep_modules = set(finder.modules.keys()) _DEP_MODULES_CACHE[test] = dep_modules return dep_modules
def find_modes(): mf = modulefinder.ModuleFinder() ret = list(mf.find_all_submodules(wayround_i2p.pyeditor.modes)) if 'text_plain' in ret: text_plain_pos = ret.index('text_plain') ret = ret[:text_plain_pos] + ret[text_plain_pos + 1:] + ['text_plain'] # print("{}".format(ret)) return ret
def collect_stdlib_dependencies(script, dest_folder, extra_deps=None): sysprefix = sys.prefix # could be a virtualenv real_lib_prefix = sysconfig.get_config_var('LIBDEST') def is_stdlib_path(path): # A module path is only a stdlib path if it's in either sys.prefix or # sysconfig.get_config_var('prefix') (the 2 are different if we are in a virtualenv) and if # there's no "site-package in the path. if not path: return False if 'site-package' in path: return False if not (path.startswith(sysprefix) or path.startswith(real_lib_prefix)): return False return True ensure_folder(dest_folder) mf = modulefinder.ModuleFinder() mf.run_script(script) modpaths = [mod.__file__ for mod in mf.modules.values()] modpaths = filter(is_stdlib_path, modpaths) for p in modpaths: if p.startswith(real_lib_prefix): relpath = op.relpath(p, real_lib_prefix) elif p.startswith(sysprefix): relpath = op.relpath(p, sysprefix) assert relpath.startswith( 'lib/python3.' ) # we want to get rid of that lib/python3.x part relpath = relpath[len('lib/python3.X/'):] else: raise AssertionError() if relpath.startswith( 'lib-dynload' ): # We copy .so files in lib-dynload directly in our dest relpath = relpath[len('lib-dynload/'):] if relpath.startswith('encodings') or relpath.startswith('distutils'): # We force their inclusion later. continue dest_path = op.join(dest_folder, relpath) ensure_folder(op.dirname(dest_path)) copy(p, dest_path) # stringprep is used by encodings. # We use real_lib_prefix with distutils because virtualenv messes with it and we need to refer # to the original distutils folder. FORCED_INCLUSION = [ 'encodings', 'stringprep', op.join(real_lib_prefix, 'distutils') ] if extra_deps: FORCED_INCLUSION += extra_deps copy_packages(FORCED_INCLUSION, dest_folder) # There's a couple of rather big exe files in the distutils folder that we absolutely don't # need. Remove them. delete_files_with_pattern(op.join(dest_folder, 'distutils'), '*.exe') # And, finally, create an empty "site.py" that Python needs around on startup. open(op.join(dest_folder, 'site.py'), 'w').close()
def module_signatures(py_source, verbose=True): r"""Scan *py_source* for dependencies, and return list of 2-tuples [(hexdigest, modulename), ...], sorted by modulename. To see what signatures signet will use when building your loader:: from signet.command.build_signet import module_signatures for hash, mod in module_signatures('hello.py'): print hash, mod """ signatures = [] finder = modulefinder.ModuleFinder() finder.run_script(py_source) # Iterate over installed modules, and try to # determine what filename they came from my_mod = os.path.basename(py_source) my_mod = os.path.splitext(my_mod)[0] modules = {my_mod: py_source} for modname, mod in finder.modules.items(): if modname == '__main__': continue # If module has a custom loader (ala: egg), # use the name of the archive file. fname = (getattr(mod, '__loader__', None) or getattr(mod, '__file__', None)) if not fname: if verbose: log.warn("can't find module '%s'", modname) else: modules[modname] = fname # Now iterate over the list of filenames we # collected, and calculate each one's hash sha1 = hashlib.sha1 for modname in sorted(modules.keys()): modpath = modules[modname] if modpath.endswith('.pyc'): modpath = modpath[:-1] with open(modpath, 'rb') as fin: digest = sha1(fin.read()).hexdigest() signatures.append([digest, modname]) return sorted(signatures, key=lambda s: s[1])
def main(): script = sys.argv[1] path = sys.argv[2:] or None # if not given use default finder = modulefinder.ModuleFinder(path=path or sys.path) finder.run_script(script) for module in finder.modules.values(): if module.__file__ and module.__file__[-3:].lower() == '.py': print(module.__file__)
def find_modules(script, add_script = False, include_stdlib = False, use_source = False, path = None, debug = 0, excludes = [], replace_paths = []): """Find the modules needed by `script`. Return a list of file-/archive-name tuples for the modules needed by `script`. This format is suitable for deploying the files in an archive or making use of Python's zip-import machinery. By default, only non-standard modules are included; i.e., modules from the Python-install hierarchy aren't included in the list. To include the standard modules, `include_stdlib` can be set to ``True``. Also by default, the script itself and the source aren't included. To include the script, set `add_script` to ``True``. Similarly, to have the source included in the list instead of the compiled byte code, set `use_source` to ``True``. Since this function uses :class:`modulefinder.ModuleFinder` to do the heavy lifting, its arguments are passed through. """ def normabs(p): return os.path.normcase(os.path.normpath(os.path.abspath(p))) prefix = normabs(sys.prefix) modules = [] ma = modules.append finder = modulefinder.ModuleFinder(path = path, debug = debug, excludes = excludes, replace_paths = replace_paths) finder.run_script(script) names = finder.modules.keys() if not add_script: names.remove('__main__') names.sort() for name in names: mod = finder.modules[name] fn = mod.__file__ if not fn: continue if not include_stdlib and normabs(fn).startswith(prefix): continue if not use_source and '.py' == os.path.splitext(fn)[1]: fn += 'c' ans = name.split('.') if mod.__path__: ans.append(os.path.basename(fn)) else: ans[-1] = os.path.basename(fn) ma((fn, '/'.join(ans))) return modules
def findDependencies(self): self.message('Finding module dependencies', 1) import modulefinder mf = modulefinder.ModuleFinder(excludes=self.excludeModules) if self.use_zipimport: mf.import_hook('zlib') site = mf.add_module('site') site.__code__ = self._getSiteCode() mf.scan_code(site.__code__, site) mf.import_hook('warnings') includeModules = self.includeModules[:] for name in self.includePackages: includeModules.extend(findPackageContents(name).keys()) for name in includeModules: try: mf.import_hook(name) except ImportError: self.missingModules.append(name) mf.run_script(self.mainprogram) modules = mf.modules.items() modules.sort() for name, mod in modules: path = mod.__file__ if path and self.semi_standalone: if path.startswith(LIB) and not path.startswith(SITE_PACKAGES): continue if path and mod.__code__ is None: filename = os.path.basename(path) pathitems = name.split('.')[:-1] + [filename] dstpath = pathjoin(*pathitems) if self.use_zipimport: if name != 'zlib': dstpath = pathjoin('ExtensionModules', dstpath) source = EXT_LOADER % {'name': name, 'filename': dstpath} code = compile(source, '<dynloader for %s>' % name, 'exec') mod.__code__ = code self.files.append((path, pathjoin('Contents', 'Resources', dstpath))) if mod.__code__ is not None: ispkg = mod.__path__ is not None if not self.use_zipimport or name != 'site': self.pymodules.append((name, mod.__code__, ispkg)) if hasattr(mf, 'any_missing_maybe'): missing, maybe = mf.any_missing_maybe() else: missing = mf.any_missing() maybe = [] self.missingModules.extend(missing) self.maybeMissingModules.extend(maybe) return
def _get_local_modules(script_name): # return a list of modules used defined in the same directory as `script_name` (including `script_name`) file_list = [] finder = modulefinder.ModuleFinder(path=[ '.', ]) finder.run_script(script_name) for name, mod in finder.modules.items(): if mod.__file__: # Skips standard library imports file_list.append(mod.__file__) return file_list
def main(output, *mnames): mf = modulefinder.ModuleFinder() for mname in mnames: mf.run_script(mname) cwd = os.getcwd() zf = zipfile.ZipFile(output, 'w') for mod in mf.modules.itervalues(): if not mod.__file__: continue modfile = os.path.abspath(mod.__file__) if os.path.commonprefix([cwd, modfile]) == cwd: zf.write(modfile, os.path.relpath(modfile)) zf.close()
def get_imports(file_path: str, to_match: str) -> list: import modulefinder import re from src.constants import Origin sys = list() native = Origin.Buildin_Functions.INBUILD modfind = modulefinder.ModuleFinder() modfind.run_script(file_path) pattern = re.compile(to_match) for k, v in modfind.modules.items(): if pattern.search(str(v.__file__)): sys.append(k) return native, sys
def addModFile(mfile): mfile = mfile.replace(".pyc", ".py") if (mfile and (os.path.splitext(mfile)[1] == ".py") and (not mfile in result) and (not max([False] + [mfile.startswith(x) for x in exclude_paths]))): result.append(mfile) f = modulefinder.ModuleFinder() f.run_script(mfile) for othername, othermod in f.modules.iteritems(): if othermod.__file__ != mfile and othermod.__file__ and not max( [False] + [othermod.__file__.startswith(x) for x in exclude_paths]): addModFile(othermod.__file__)
def notBuiltInModules(script_path): """Returns a list of (module, path) pairs that are not built-in modules.""" # Figure out the paths for "built-in" modules: # Remove any paths in PYTHONPATH, as those are clearly not "built-in" # sys.path[0] is the "script path": ignore it pythonpaths = set({}) if "PYTHONPATH" in os.environ: for path in os.environ["PYTHONPATH"].split(os.pathsep): pythonpaths.add(path) system_paths = [] for path in sys.path[1:]: if path not in pythonpaths: system_paths.append(path) #~ print "system paths:", "; ".join(system_paths) finder = modulefinder.ModuleFinder() finder.run_script(script_path) finder.report() not_builtin = [] for name, module in finder.modules.iteritems(): # The _bisect module has __file__ = None if not hasattr(module, "__file__") or module.__file__ is None: # Skip built-in modules continue system_module = False for system_path in system_paths: if module.__file__.startswith(system_path): system_module = True break if system_module: continue # Skip the script if name == "__main__": assert module.__file__ == script_path continue relative_path = name.replace('.', '/') if module.__path__: #~ # This is a package; originally I skipped it, but that is not a good idea assert module.__file__.endswith("/__init__.py") relative_path += "/__init__.py" else: relative_path += os.path.splitext(module.__file__)[1] assert name != "__main__" not_builtin.append((module.__file__, relative_path)) return not_builtin
def relevantPyFiles(focalScript='test.py'): import modulefinder finder = modulefinder.ModuleFinder(['.']) finder.run_script(focalScript) import os relevantPyFiles = [] for name, mod in finder.modules.items(): if os.path.isfile(name + '.py'): relevantPyFiles.append(name + '.py') #print(f' {name}.py', end='\t: ') #print(', '.join(list(mod.globalnames.keys()))) relevantPyFiles.reverse() return (relevantPyFiles)
def python_files(entrypoint='main.py'): entrypoint = Path(entrypoint) finder = modulefinder.ModuleFinder() finder.run_script(str(entrypoint)) files = {entrypoint} for k, m in finder.modules.items(): if not m.__file__: continue p = Path(m.__file__) try: p = p.relative_to('.') except ValueError: continue files.add(p) return files
def test_duplicate_imports(self): with temppathlib.TemporaryDirectory() as tmp: script_pth = tmp.path / "some_script.py" script_pth.write_text("#!/usr/bin/env python\n" "import os.path\n" "import os.path\n") mfder = modulefinder.ModuleFinder(path=[str(tmp.path)]) mfder.load_file(pathname=str(script_pth)) bad_modules = sorted(mfder.badmodules.keys()) modules = sorted(mfder.modules.keys()) self.assertListEqual(['os.path'], bad_modules) self.assertListEqual(['some_script'], modules)
def _pack(self, zip, script, deps): moduleFinder = modulefinder.ModuleFinder() moduleFinder.run_script(script) deps.add(script) zip.write(script, self._pathRelativeToPythonPath(script)) for module in moduleFinder.modules.itervalues(): if not self._packModule(module): continue relpath = self._pathRelativeToPythonPath(module.__file__) if relpath not in zip.namelist(): if tipoffmodulefinder.fileIsUpsetoPythonNamespaceJoinInit( module.__file__): zip.writestr(relpath, "") else: zip.write(module.__file__, relpath) deps.add(module.__file__)
def show_imported_modules(self, file): finder = modulefinder.ModuleFinder() a = finder.run_script(file) dir(a) print(' ' * 4, 'Loaded modules') print('-' * 50) for name, mod in finder.modules.items(): print(' ' * 4, "|--", ' %s ' % name) print(' ' * 8, ' ', ','.join(mod.globalnames.keys()[:3])) print(' ' * 50) print(' ' * 4, 'Modules - NOT IMPORTED') print('-' * 50) for name in finder.badmodules.iterkeys(): print(' ' * 4, "|--", ' %s' % name)