def parsePlugins(packageName): # Searching generators and views print 'searching for generators and views...' packageName = 'Plugins' package = __import__(packageName) print dir(package) pluginsDict = {'generators':{},'views':{}} for importer, modname, ispkg in pkgutil.walk_packages(package.__path__): if modname[0] <> '_': if ispkg == True and modname in pluginsToLoad: print "Found submodule %s (is a package: %s)" % (modname, ispkg) package2 = eval('package.'+modname) print 'package content: {0}'.format(dir(package2)) for importer2, modname2, ispkg2 in pkgutil.walk_packages(package2.__path__): # Searching for generators if ispkg2 == False and string.find(modname2, 'Generators') >= 0: genModule = eval('package2.'+modname2) print "{0} contains generators".format(genModule.__name__) genList = [o for o in getmembers(genModule) if isfunction(o[1])] pluginsDict['generators'].update(genList) # Searching for for view if ispkg2 == False and string.find(modname2, 'View') >= 0: viewModule = eval('package2.'+modname2) print "{0} contains view".format(viewModule.__name__) viewList = [o for o in getmembers(viewModule) if isclass(o[1]) and o[0] == modname+'Tab'] pluginsDict['views'].update(viewList) print 'search finished' print 'found generators: {0}'.format(pluginsDict['generators']) print 'found views: {0}'.format(pluginsDict['views']) return pluginsDict
def test_walk_packages_raises_on_string_or_bytes_input(self): str_input = 'test_dir' with self.assertRaises((TypeError, ValueError)): list(pkgutil.walk_packages(str_input)) bytes_input = b'test_dir' with self.assertRaises((TypeError, ValueError)): list(pkgutil.walk_packages(bytes_input))
def run(self): if self._show_help: # extra printout, --help-plugins disables _log() print('Installed jobs:') self._log('--- begin') self._log('--- loading plugins:') plugin_path = [os.path.join(os.path.dirname(os.path.abspath(__file__)), 'plugins'), self._conf.plugin_dir] for p in plugin_path: if not os.path.exists(p): self._log('!!! skipping "{0}" - no such file or directory'.format(p)) continue sys.path.append(p) for (importer, name, ispkg) in pkgutil.walk_packages([p]): if name.startswith('ImpJob'): try: __import__(name) except Exception, e: self._log('!!! unable to import "{0}": {1}: {2}'.format( name, e.__class__.__name__, e))
def loadPlugins(self, folder, namespace): for importer, modname, ispkg in pkgutil.walk_packages(path=[folder], prefix=namespace): module = importlib.import_module(modname) for name, t in module.__dict__.items(): if type(t) is type and issubclass(t, pdmt.api.NodeType): self.nodetypes.append(t) self.plugins.append(module)
def run_tests(path=PLUGINS_PATH, test_module_name_prefix="test_", ignored_modules=None, output_stream=sys.stderr): """ Execute all modules containing unit tests located in the `path` directory. The names of the unit test modules start with the specified prefix (`test_` by default). `ignored_modules` is a list of unit test modules or packages to ignore. If a package is specified, all of its underlying modules are ignored. `output_stream` prints the unit test output using the specified output stream (`sys.stderr` by default). """ _fix_streams_for_unittest() if ignored_modules is None: ignored_modules = [] for unused_, module_name, unused_ in pkgutil.walk_packages(path=[path]): if any(module_name.startswith(ignored_module_name) for ignored_module_name in ignored_modules): continue module = load_module(module_name) parts = module_name.split('.') if parts[-1].startswith(test_module_name_prefix): run_test(module, stream=output_stream)
def test_root_import_all_completeness(): EXCEPTIONS = ('utils', 'tests', 'base', 'setup') for _, modname, _ in pkgutil.walk_packages(path=sklearn.__path__, onerror=lambda _: None): if '.' in modname or modname.startswith('_') or modname in EXCEPTIONS: continue assert_in(modname, sklearn.__all__)
def getURISource(self, url): try: sourceDict = [] for package, name, is_pkg in pkgutil.walk_packages(__path__): sourceDict.append((name, is_pkg)) sourceDict = [i[0] for i in sourceDict if i[1] == False] sourceDict = [(i, __import__(i, globals(), locals(), [], -1).source()) for i in sourceDict] domain = (urlparse.urlparse(url).netloc).lower() domains = [(i[0], i[1].domains) for i in sourceDict] domains = [i[0] for i in domains if any(x in domain for x in i[1])] if len(domains) == 0: return False call = [i[1] for i in sourceDict if i[0] == domains[0]][0] self.sources = call.sources(url, self.hostDict, self.hostprDict) for i in range(len(self.sources)): try: self.sources[i]['autoplay'] = True except: pass self.sources = self.sourcesFilter() return self.sources except: pass
def extensions_find(arg_parser): """ Look for command extensions. """ subparser = arg_parser.add_subparsers( title="subcommands", description="Valid subcommands", help="Each subcommands supports --help for additional information.") for package in testbed.settings.PLUGINS: LOGGER.debug("loading commands %s", package) package = importlib.import_module(package) for _, module, ispkg in pkgutil.walk_packages(package.__path__, package.__name__ + ".", onerror=onerror): ## # only include commands from commands.py files. if ispkg or not module.endswith("commands"): continue LOGGER.debug(" loading commands from %s", module) module = importlib.import_module(module) try: module.add_subparser(subparser) except AttributeError, arg: ## # This means that the module is missing the add method. # All modules identified in settings to extend CLI # must have an add method LOGGER.error("adding subparser for %s.%s", package, module) LOGGER.exception(arg)
def load_processors(self): """ load Cfg file processors. this must be done at run-time, not at compile-time, or we get a circular import and things don't work. but finding the right way to do this at runtime was ... problematic. so here it is, writing to a global variable. Sorry 'bout that. """ global PROCESSORS if PROCESSORS is None: PROCESSORS = [] if hasattr(pkgutil, 'walk_packages'): submodules = pkgutil.walk_packages(path=__path__) else: #python 2.4 import glob submodules = [] for path in __path__: for submodule in glob.glob("%s/*.py" % path): mod = '.'.join(submodule.split("/")[-1].split('.')[:-1]) if mod != '__init__': submodules.append((None, mod, True)) for submodule in submodules: module = getattr(__import__("%s.%s" % (__name__, submodule[1])).Server.Plugins.Cfg, submodule[1]) proc = getattr(module, submodule[1]) if set(proc.__mro__).intersection([CfgInfo, CfgFilter, CfgGenerator, CfgVerifier]): PROCESSORS.append(proc)
def get_modules(cls): """ Dynamically loads all the modules in the modules folder and sorts them by the PRIORITY key. If no PRIORITY is defined for a given module, a priority of 0 is assumed. """ logger = logging.getLogger(__name__) locations = [jasperpath.PLUGIN_PATH] logger.debug("Looking for modules in: %s", ', '.join(["'%s'" % location for location in locations])) modules = [] for finder, name, ispkg in pkgutil.walk_packages(locations): try: loader = finder.find_module(name) mod = loader.load_module(name) except: logger.warning("Skipped module '%s' due to an error.", name, exc_info=True) else: if hasattr(mod, 'WORDS'): logger.debug("Found module '%s' with words: %r", name, mod.WORDS) modules.append(mod) else: logger.warning("Skipped module '%s' because it misses " + "the WORDS constant.", name) modules.sort(key=lambda mod: mod.PRIORITY if hasattr(mod, 'PRIORITY') else 0, reverse=True) return modules
def packages_in_stage(stage_name): stage_pymod = self.load_pymodule(stage_name) # where to recursively search for pymodules search_path = stage_pymod.__path__ # Prefix of all pymodules and packages found in this dir. This is a little strange - suppose we are # searching in stage foo.bar which has the following structure: # |-foo # |-bar # |-buzz.py # |-baz # |-file.py # When walk_packages finds the directory `baz`, it won't know that the package's name is `foo.bar.baz` - # it's not aware that bar is contained within another package. Unless we provide a prefix, it will think # that `foo.bar.baz's` name is just `baz`. This sort of makes sense, because if you added foo/bar to the # path then you could `import baz`. However, walk_packages will actually fail because it cannot # `import baz`, which it needs to do in order to get package details that inform the recursive search. # If there are no packages (only pymodules) in foo/bar, then `walk_packages` will succeed, but the output # names will be wrong (e.g. `buzz` instead of `foo.bar.buzz`). # `walk_packages` can generate AttributeError if the system has # Gtk modules, which are not designed to use with reflection or # introspection. Best action to take in this situation is probably # to simply suppress the error. def onerror(name): smv.logger.error("Skipping due to error during walk_packages: " + name) return pkgutil.walk_packages( path=search_path, prefix=stage_name + '.', onerror=onerror)
def load_modules(): # Import modules package. import modules plugins = dict() # Walk recursively through all modules and packages. for loader, module_name, ispkg in pkgutil.walk_packages(modules.__path__, modules.__name__ + '.'): # If current item is a package, skip. if ispkg: continue # Try to import the module, otherwise skip. try: module = __import__(module_name, globals(), locals(), ['dummy'], -1) except ImportError as e: continue # Walk through all members of currently imported modules. for member_name, member_object in inspect.getmembers(module): # Check if current member is a class. if inspect.isclass(member_object): # Yield the class if it's a subclass of Module. if issubclass(member_object, Module) and member_object is not Module: plugins[member_object.cmd] = dict(obj=member_object, description=member_object.description) return plugins
def load_plugins(self): import pkgutil from rez.backport.importlib import import_module type_module_name = 'rezplugins.' + self.type_name package = import_module(type_module_name) # on import, the `__path__` variable of the imported package is extended # to include existing directories on the plugin search path (via # extend_path, above). this means that `walk_packages` will walk over all # modules on the search path at the same level (.e.g in a # 'rezplugins/type_name' sub-directory). paths = [package.__path__] if isinstance(package.__path__, basestring) \ else package.__path__ for path in paths: for loader, modname, ispkg in pkgutil.walk_packages( [path], package.__name__ + '.'): if loader is not None: plugin_name = modname.split('.')[-1] if plugin_name.startswith('_'): continue if config.debug("plugins"): print_debug("loading %s plugin at %s: %s..." % (self.type_name, path, modname)) try: # load_module will force reload the module if it's # already loaded, so check for that module = sys.modules.get(modname) if module is None: module = loader.find_module(modname).load_module(modname) if hasattr(module, 'register_plugin') and \ hasattr(module.register_plugin, '__call__'): plugin_class = module.register_plugin() if plugin_class != None: self.register_plugin(plugin_name, plugin_class, module) else: if config.debug("plugins"): print_warning( "'register_plugin' function at %s: %s did not return a class." % (path, modname)) else: if config.debug("plugins"): print_warning( "no 'register_plugin' function at %s: %s" % (path, modname)) # delete from sys.modules? except Exception as e: nameish = modname.split('.')[-1] self.failed_plugins[nameish] = str(e) if config.debug("plugins"): import traceback from StringIO import StringIO out = StringIO() traceback.print_exc(file=out) print_debug(out.getvalue()) # load config data, _ = _load_config_from_filepaths([os.path.join(path, "rezconfig")]) deep_update(self.config_data, data)
def load_modules(): # Import modules package. import viper.modules as modules plugins = dict() # Walk recursively through all modules and packages. for loader, module_name, ispkg in pkgutil.walk_packages(modules.__path__, modules.__name__ + '.'): # If current item is a package, skip. if ispkg: continue # Try to import the module, otherwise skip. try: module = importlib.import_module(module_name) except ImportError as e: print_warning("Something wrong happened while importing the module {0}: {1}".format(module_name, e)) continue # Walk through all members of currently imported modules. for member_name, member_object in inspect.getmembers(module): # Check if current member is a class. if inspect.isclass(member_object): # Yield the class if it's a subclass of Module. if issubclass(member_object, Module) and member_object is not Module: plugins[member_object.cmd] = dict(obj=member_object, description=member_object.description, parser_args=get_argparse_parser_actions(member_object().parser), subparser_args=get_argparse_subparser_actions(member_object().parser)) return plugins
def all_estimators(): def is_abstract(c): if not(hasattr(c, '__abstractmethods__')): return False if not len(c.__abstractmethods__): return False return True all_classes = [] # get parent folder path = sklearn.__path__ for importer, modname, ispkg in pkgutil.walk_packages(path=path, prefix='sklearn.', onerror=lambda x: None): module = __import__(modname, fromlist="dummy") if ".tests." in modname: continue classes = inspect.getmembers(module, inspect.isclass) all_classes.extend(classes) all_classes = set(all_classes) estimators = [c for c in all_classes if issubclass(c[1], BaseEstimator)] # get rid of abstract base classes estimators = [c for c in estimators if not is_abstract(c[1])] # We sort in order to have reproducible test failures return sorted(estimators)
def walk_packages(packages: List[str]) -> Iterator[str]: """Iterates through all packages and sub-packages in the given list. Python packages have a __path__ attribute defined, which pkgutil uses to determine the package hierarchy. However, packages in C extensions do not have this attribute, so we have to roll out our own. """ for package_name in packages: package = importlib.import_module(package_name) yield package.__name__ # get the path of the object (needed by pkgutil) path = getattr(package, '__path__', None) if path is None: # object has no path; this means it's either a module inside a package # (and thus no sub-packages), or it could be a C extension package. if is_c_module(package): # This is a C extension module, now get the list of all sub-packages # using the inspect module subpackages = [package.__name__ + "." + name for name, val in inspect.getmembers(package) if inspect.ismodule(val) and val.__name__ == package.__name__ + "." + name] # recursively iterate through the subpackages for submodule in walk_packages(subpackages): yield submodule # It's a module inside a package. There's nothing else to walk/yield. else: all_packages = pkgutil.walk_packages(path, prefix=package.__name__ + ".", onerror=lambda r: None) for importer, qualified_name, ispkg in all_packages: yield qualified_name
def _load_handlers(self): # this must be called at run-time, not at compile-time, or we # get wierd circular import issues. self._handlers = dict() if hasattr(pkgutil, 'walk_packages'): submodules = pkgutil.walk_packages(path=__path__) else: # python 2.4 import glob submodules = [] for path in __path__: for submodule in glob.glob(os.path.join(path, "*.py")): mod = os.path.splitext(os.path.basename(submodule))[0] if mod not in ['__init__']: submodules.append((None, mod, True)) for submodule in submodules: if submodule[1] == 'base': continue module = getattr(__import__("%s.%s" % (__name__, submodule[1])).Client.Tools.POSIX, submodule[1]) hdlr = getattr(module, "POSIX" + submodule[1]) if POSIXTool in hdlr.__mro__: # figure out what entry type this handler handles etype = hdlr.__name__[5:].lower() self._handlers[etype] = hdlr(self.logger, self.setup, self.config)
def get_all_methods(dirname, pkg_name): ret = dict() for _, module_name, _ in walk_packages([dirname], pkg_name + '.'): mod = import_module(module_name) for key, val in mod.__dict__.iteritems(): ret[key] = val return ret
def sourcesResolve(self, url, provider): try: provider = provider.lower() if not provider.endswith(('_mv', '_tv', '_mv_tv')): sourceDict = [] for package, name, is_pkg in pkgutil.walk_packages(__path__): sourceDict.append((name, is_pkg)) provider = [i[0] for i in sourceDict if i[1] == False and i[0].startswith(provider + '_')][0] source = __import__(provider, globals(), locals(), [], -1).source() url = source.resolve(url) #control.log("[suurces] my url 1 ************ %s " % url) #url = resolvers.request(url) #control.log("[sources] my url 2 ************ %s " % url) try: headers = dict(urlparse.parse_qsl(url.rsplit('|', 1)[1])) except: headers = dict('') result = client.request(url.split('|')[0], headers=headers, output='chunk', timeout='20') if result == None: raise Exception() #control.log("!!!!!!!!!!!!!!!!!!! %s prov: %s" % (url,provider)) self.url = url return url except: return
def test_pkgutil_walk_packages(self): # This is a dodgy hack to use the test_runpy infrastructure to test # issue #15343. Issue #15348 declares this is indeed a dodgy hack ;) import pkgutil max_depth = 4 base_name = "__runpy_pkg__" package_suffixes = ["uncle", "uncle.cousin"] module_suffixes = ["uncle.cousin.nephew", base_name + ".sibling"] expected_packages = set() expected_modules = set() for depth in range(1, max_depth): pkg_name = ".".join([base_name] * depth) expected_packages.add(pkg_name) for name in package_suffixes: expected_packages.add(pkg_name + "." + name) for name in module_suffixes: expected_modules.add(pkg_name + "." + name) pkg_name = ".".join([base_name] * max_depth) expected_packages.add(pkg_name) expected_modules.add(pkg_name + ".runpy_test") pkg_dir, mod_fname, mod_name, mod_spec = ( self._make_pkg("", max_depth)) self.addCleanup(self._del_pkg, pkg_dir, max_depth, mod_name) for depth in range(2, max_depth+1): self._add_relative_modules(pkg_dir, "", depth) for finder, mod_name, ispkg in pkgutil.walk_packages([pkg_dir]): self.assertIsInstance(finder, importlib.machinery.FileFinder) if ispkg: expected_packages.remove(mod_name) else: expected_modules.remove(mod_name) self.assertEqual(len(expected_packages), 0, expected_packages) self.assertEqual(len(expected_modules), 0, expected_modules)
def _get_modules(self): """Gets modules given the repo paths passed in to init""" modules = [] error = False for repo in self.testrepos: # We're assuming only packages have __file__ attribute strings # that end with __init__.py (or .pyc). If this doesn't match # that pattern, assume it's a module. if "__init__.py" not in getattr(repo, "__file__", ""): modules.append(repo) continue # We're still in a package, so walk it and find the rest of the # modules. prefix = "{0}.".format(repo.__name__) for _, modname, is_pkg in pkgutil.walk_packages( path=repo.__path__, prefix=prefix, onerror=lambda x: None): if not is_pkg: try: modules.append(importlib.import_module(modname)) except Exception as exception: print_exception( "Suite Builder", "import_module", modname, exception) error = True if self.exit_on_error and error: exit(get_error(exception)) return modules
def test_walkpackages_zipfile(self): """Tests the same as test_walkpackages_filesys, only with a zip file.""" zip = 'test_walkpackages_zipfile.zip' pkg1 = 'test_walkpackages_zipfile' pkg2 = 'sub' zip_file = os.path.join(self.dirname, zip) z = zipfile.ZipFile(zip_file, 'w') z.writestr(pkg2 + '/__init__.py', "") z.writestr(pkg2 + '/' + pkg1 + '/__init__.py', "") z.writestr(pkg2 + '/' + pkg1 + '/mod.py', "") z.writestr(pkg1 + '/__init__.py', "") z.writestr(pkg1 + '/' + pkg2 + '/__init__.py', "") z.writestr(pkg1 + '/' + pkg2 + '/mod.py', "") z.close() sys.path.insert(0, zip_file) expected = [ 'sub', 'sub.test_walkpackages_zipfile', 'sub.test_walkpackages_zipfile.mod', 'test_walkpackages_zipfile', 'test_walkpackages_zipfile.sub', 'test_walkpackages_zipfile.sub.mod', ] actual= [e[1] for e in pkgutil.walk_packages([zip_file])] self.assertEqual(actual, expected) del sys.path[0] for pkg in expected: if pkg.endswith('mod'): continue del sys.modules[pkg]
def find_analysis_classes(): analyses = {} package_itr = pkgutil.walk_packages( path=oopa.modules.__path__, prefix=oopa.modules.__name__ + "." ); for importer, mod_name, is_package in package_itr: if is_package: continue mod = importlib.import_module(mod_name) analysis = None for name, member in inspect.getmembers(mod, inspect.isclass): if Analysis.__name__ in [c.__name__ for c in member.__bases__]: analysis = member if analysis is None: print "Error: Analysis subclass not found in %s" % (mod.__path__) analyses[mod_name.split(".")[-1]] = analysis return analyses
def morepath_packages(): namespace_packages = set() paths = [] m = DependencyMap() m.load() for dist in m.relevant_dists('morepath'): if dist.has_metadata('namespace_packages.txt'): data = dist.get_metadata('namespace_packages.txt') for ns in data.split('\n'): ns = ns.strip() if ns: namespace_packages.add(ns) paths.append(dist.location) seen = set() for importer, dotted_name, is_pkg in walk_packages(paths): if not is_pkg: continue if dotted_name in namespace_packages: continue if known_prefix(dotted_name, seen): continue for prefix in prefixes(dotted_name): if prefix not in namespace_packages: seen.add(prefix) m = importer.find_module(dotted_name).load_module(dotted_name) # XXX hack to work around bug in walk_packages that will load # more than one namespace package: http://bugs.python.org/issue14787 # XXX performance if in_path(m, paths): yield m
def get_modules(self): """ walks all modules in the test repo, filters by product and module filter. Filter passed in with -m returns a list of module dotpath strings """ test_repo = import_module(self.test_repo_name) prefix = "{0}.".format(test_repo.__name__) product_path = "{0}{1}".format(prefix, self.product) modnames = [] for importer, modname, is_pkg in pkgutil.walk_packages( path=test_repo.__path__, prefix=prefix, onerror=lambda x: None): if not is_pkg and modname.startswith(product_path): if (not self.module_regex or self.module_regex in modname.rsplit(".", 1)[1]): modnames.append(modname) filter_mods = [] for modname in modnames: add_package = not bool(self.packages) for package in self.packages: if package in modname.rsplit(".", 1)[0]: add_package = True break if add_package: filter_mods.append(modname) filter_mods.sort() return filter_mods
def discover(self, path=os.path.dirname(__file__), prefix=None): dirname = os.path.basename(path) if prefix is None: prefix = dirname for importer, name, ispkg in pkgutil.walk_packages([path]): try: pkg_name = '%s.%s' % (prefix, name) if pkg_name not in sys.modules: module_meta = importer.find_module(name) module = module_meta.load_module(pkg_name) else: module = sys.modules[pkg_name] if ispkg: self.discover(os.path.join(path, name), pkg_name) except PluginFormatError as error: logging.warn(' *** [{name}] Plugin error: {error}' ''.format(name=name, error=error)) except PluginLoadError as error: logging.warn(' *** [{name}] Plugin failed to load: {error}' ''.format(name=name, error=error)) except PluginCrashed as error: logging.warn(' *** [{name}] Plugin crashed: {error}' ''.format(name=name, error=error)) except ImportError as error: logging.exception(error)
def walk_packages(path=None, prefix='', onerror=None): """ Implementation of walk_packages for python 2.5 """ def seen(path, seenpaths={}): # pylint: disable=W0102 """ detect if a path has been 'seen' (i.e., considered for inclusion in the generator). tracks what has been seen through the magic of python default arguments """ if path in seenpaths: return True seenpaths[path] = True for importer, name, ispkg in iter_modules(path, prefix): yield importer, name, ispkg if ispkg: try: __import__(name) except ImportError: if onerror is not None: onerror(name) except Exception: if onerror is not None: onerror(name) else: raise else: path = getattr(sys.modules[name], '__path__', []) # don't traverse path items we've seen before path = [p for p in path if not seen(p)] for item in walk_packages(path, name + '.', onerror): yield item
def load_scrapers(cls): """ Dynamically loads all the scrapers in the 'scrapers' folder that contain 'Scraper' class inheriting from 'ScraperBase'. """ logger = logging.getLogger('filer') locations = ['/Filed/src/scrapers'] logger.info("Loading scrapers from: %s", ', '.join(["'%s'" % location for location in locations])) scrapers = [] for finder, name, ispkg in pkgutil.walk_packages(locations): try: loader = finder.find_module(name) scraper = loader.load_module(name) except: logger.warning("Skipped scraper '%s' due to an error.", name, exc_info=True) else: if hasattr(scraper, 'Scraper') and issubclass(scraper.Scraper, ScraperBase): logger.debug("Found scraper '%s'", name) scrapers.append(scraper) else: logger.warning("Skipped scraper '%s' because it misses " + "the 'Scraper' class or it does not inherit " + "the 'ScraperBase' parent class", name) logger.info("Scrapers loaded") return scrapers
def _load_all_views(cls): for pkg in cls.view_packages: walk = pkgutil.walk_packages(pkg.__path__, pkg.__name__ + '.') for loader, name, is_pkg in walk: if name not in sys.modules: module = loader.find_module(name).load_module(name) yield module
def test_tabs(): """Test that there are no tabs in our source files.""" # avoid importing modules that require mayavi if mayavi is not installed ignore = _tab_ignores[:] try: import mayavi # noqa: F401 analysis:ignore except ImportError: ignore.extend('mne.gui.' + name for name in ('_coreg_gui', '_fiducials_gui', '_file_traits', '_help', '_kit2fiff_gui', '_marker_gui', '_viewer')) for importer, modname, ispkg in walk_packages(mne.__path__, prefix='mne.'): # because we don't import e.g. mne.tests w/mne if not ispkg and modname not in ignore: # mod = importlib.import_module(modname) # not py26 compatible! try: with pytest.warns(None): __import__(modname) except Exception: # can't import properly continue mod = sys.modules[modname] try: source = getsource(mod) except IOError: # user probably should have run "make clean" continue assert '\t' not in source, ('"%s" has tabs, please remove them ' 'or add it to the ignore list' % modname)
__author__ = ['Miguel Ramos Pernas'] __email__ = ['*****@*****.**'] # Python import importlib import inspect import os import pkgutil PACKAGE_PATH = os.path.dirname(os.path.abspath(__file__)) __all__ = ['PACKAGE_PATH'] for loader, module_name, ispkg in pkgutil.walk_packages(__path__): if module_name.endswith('setup') or module_name.endswith('__'): continue # Import all classes and functions mod = importlib.import_module('.' + module_name, package='hep_spt.stats') __all__ += mod.__all__ for n, c in inspect.getmembers(mod): if n in mod.__all__: globals()[n] = c __all__ = list(sorted(__all__))
import importlib import pkgutil from .hooks import get_builder, get_tool # noqa # Import all the packages in this directory so their hooks get run. for _, name, _ in pkgutil.walk_packages(__path__, '.'): importlib.import_module(name, __package__)
def sourcesResolve(self, item, info=False): try: self.url = None u = url = item['url'] d = item['debrid'] direct = item['direct'] provider = item['provider'].lower() host = item['source'].lower() if not provider.endswith(('_movies', '_series', '_movies_series')): sourceDict = [] for package, name, is_pkg in pkgutil.walk_packages(__path__): sourceDict.append((name, is_pkg)) provider = [ i[0] for i in sourceDict if i[1] == False and i[0].startswith(provider + '_') ][0] source = __import__(provider, globals(), locals(), [], -1).source() u = url = source.resolve(url) if url == None or not '://' in str(url): raise Exception() if not d == '': url = debrid.resolver(url, d) elif not direct == True: url = None if host in self.resDict: try: from res import selfresolver url = selfresolver.resolve(u, host) except: pass if url == None: hmf = urlresolver.HostedMediaFile(url=u, include_disabled=True, include_universal=False) if hmf.valid_url() == True: url = hmf.resolve() if url == False or url == None: raise Exception() ext = url.split('?')[0].split('&')[0].split('|')[0].rsplit( '.')[-1].replace('/', '').lower() if ext == 'rar': raise Exception() try: headers = url.rsplit('|', 1)[1] except: headers = '' headers = urllib.quote_plus(headers).replace( '%3D', '=') if ' ' in headers else headers headers = dict(urlparse.parse_qsl(headers)) if url.startswith('http') and '.m3u8' in url: result = client.request(url.split('|')[0], headers=headers, output='geturl', timeout='20') if result == None: raise Exception() #elif url.startswith('http'): # result = client.request(url.split('|')[0], headers=headers, output='chunk', timeout='20') # if result == None: raise Exception() self.url = url return url except: if info == True: self.errorForSources() return
def getSources(self, title, year, imdb, tvdb, season, episode, tvshowtitle, premiered, timeout=30): sourceDict = [] for package, name, is_pkg in pkgutil.walk_packages(__path__): sourceDict.append((name, is_pkg)) sourceDict = [i[0] for i in sourceDict if i[1] == False] content = 'movie' if tvshowtitle == None else 'episode' if content == 'movie': sourceDict = [ i for i in sourceDict if i.endswith(('_movies', '_movies_series')) ] else: sourceDict = [ i for i in sourceDict if i.endswith(('_series', '_movies_series')) ] try: sourceDict = [ (i, control.setting( 'provider.' + re.sub('_movies_series$|_movies$|_series$', '', i))) for i in sourceDict ] except: sourceDict = [(i, 'true') for i in sourceDict] sourceDict = [i[0] for i in sourceDict if not i[1] == 'false'] threads = [] control.makeFile(control.dataPath) self.sourceFile = control.providercacheFile if content == 'movie': title = cleantitle.normalize(title) for source in sourceDict: threads.append( workers.Thread( self.getMovieSource, title, year, imdb, re.sub('_movies_series$|_movies$|_series$', '', source), __import__(source, globals(), locals(), [], -1).source())) else: tvshowtitle = cleantitle.normalize(tvshowtitle) for source in sourceDict: threads.append( workers.Thread( self.getEpisodeSource, title, year, imdb, tvdb, season, episode, tvshowtitle, premiered, re.sub('_movies_series$|_movies$|_series$', '', source), __import__(source, globals(), locals(), [], -1).source())) try: timeout = int(control.setting('scrapers.timeout.1')) except: pass [i.start() for i in threads] sourceLabel = [ re.sub('_movies_series$|_movies$|_series$', '', i) for i in sourceDict ] sourceLabel = [re.sub('v\d+$', '', i).upper() for i in sourceLabel] progressDialog = control.progressDialog if control.setting( 'progress.dialog') == '0' else control.progressDialogBG progressDialog.create(control.addonInfo('name'), '') progressDialog.update(0) string1 = control.lang(32404).encode('utf-8') string2 = control.lang(32405).encode('utf-8') string3 = control.lang(32406).encode('utf-8') for i in range(0, timeout * 2): try: if xbmc.abortRequested == True: return sys.exit() try: info = [ sourceLabel[int(re.sub('[^0-9]', '', str(x.getName()))) - 1] for x in threads if x.is_alive() == True ] except: info = [] try: if progressDialog.iscanceled(): break string4 = string1 % str(int(i * 0.5)) if len(info) > 5: string5 = string3 % str(len(info)) else: string5 = string3 % str(info).translate(None, "[]'") progressDialog.update( int((100 / float(len(threads))) * len([x for x in threads if x.is_alive() == False])), str(string4), str(string5)) except: string4 = string2 % str(int(i * 0.5)) if len(info) > 5: string5 = string3 % str(len(info)) else: string5 = str(info).translate(None, "[]'") progressDialog.update( int((100 / float(len(threads))) * len([x for x in threads if x.is_alive() == False])), str(string4), str(string5)) is_alive = [x.is_alive() for x in threads] if all(x == False for x in is_alive): break time.sleep(0.5) except: pass for i in range(0, 30 * 2): try: if xbmc.abortRequested == True: return sys.exit() try: info = [ sourceLabel[int(re.sub('[^0-9]', '', str(x.getName()))) - 1] for x in threads if x.is_alive() == True ] except: info = [] try: if progressDialog.iscanceled(): break string4 = string1 % str(int(i * 0.5) + timeout) if len(info) > 5: string5 = string3 % str(len(info)) else: string5 = string3 % str(info).translate(None, "[]'") progressDialog.update( int((100 / float(len(threads))) * len([x for x in threads if x.is_alive() == False])), str(string4), str(string5)) except: string4 = string2 % str(int(i * 0.5) + timeout) if len(info) > 5: string5 = string3 % str(len(info)) else: string5 = str(info).translate(None, "[]'") progressDialog.update( int((100 / float(len(threads))) * len([x for x in threads if x.is_alive() == False])), str(string4), str(string5)) is_alive = [x.is_alive() for x in threads] if all(x == False for x in is_alive): break if self.sources: break time.sleep(0.5) except: pass try: progressDialog.close() except: pass self.sourcesFilter() return self.sources
import pkgutil import inspect import datetime import os __all__ = [] for loader, name, is_pkg in pkgutil.walk_packages(__path__): module = loader.find_module(name).load_module(name) for name, value in inspect.getmembers(module): if name.startswith('__'): continue globals()[name] = value __all__.append(name) class Formatter(object): @staticmethod def WinOrLinux(winStr, linStr): if os.name == 'nt': return winStr else: return linStr @staticmethod def day(d): return Formatter.utc_to_local(d).strftime( Formatter.WinOrLinux('%b %d, %Y', '%b %e, %Y'))
""" import pkgutil import os import json from .utils import get_info __author__ = "Andrea Tramacere" pkg_dir = os.path.abspath(os.path.dirname(__file__)) pkg_name = os.path.basename(pkg_dir) __all__ = [] _info = get_info() __version__ = _info['version'] if 'label' in _info.keys(): __label__ = _info['label'] else: __label__ = None for importer, modname, ispkg in pkgutil.walk_packages(path=[pkg_dir], prefix=pkg_name + '.', onerror=lambda x: None): if ispkg == True: __all__.append(modname) else: pass data_dir = os.path.dirname(__file__) + '/data'
def all_estimators(include_meta_estimators=False, include_other=False, type_filter=None, include_dont_test=False): """Get a list of all estimators from sklearn. This function crawls the module and gets all classes that inherit from BaseEstimator. Classes that are defined in test-modules are not included. By default meta_estimators such as GridSearchCV are also not included. Parameters ---------- include_meta_estimators : boolean, default=False Whether to include meta-estimators that can be constructed using an estimator as their first argument. These are currently BaseEnsemble, OneVsOneClassifier, OutputCodeClassifier, OneVsRestClassifier, RFE, RFECV. include_other : boolean, default=False Wether to include meta-estimators that are somehow special and can not be default-constructed sensibly. These are currently Pipeline, FeatureUnion and GridSearchCV include_dont_test : boolean, default=False Whether to include "special" label estimator or test processors. type_filter : string, list of string, or None, default=None Which kind of estimators should be returned. If None, no filter is applied and all estimators are returned. Possible values are 'classifier', 'regressor', 'cluster' and 'transformer' to get estimators only of these specific types, or a list of these to get the estimators that fit at least one of the types. Returns ------- estimators : list of tuples List of (name, class), where ``name`` is the class name as string and ``class`` is the actuall type of the class. """ def is_abstract(c): if not (hasattr(c, '__abstractmethods__')): return False if not len(c.__abstractmethods__): return False return True all_classes = [] # get parent folder path = sklearn.__path__ for importer, modname, ispkg in pkgutil.walk_packages( path=path, prefix='sklearn.', onerror=lambda x: None): if ".tests." in modname: continue module = __import__(modname, fromlist="dummy") classes = inspect.getmembers(module, inspect.isclass) all_classes.extend(classes) all_classes = set(all_classes) estimators = [ c for c in all_classes if (issubclass(c[1], BaseEstimator) and c[0] != 'BaseEstimator') ] # get rid of abstract base classes estimators = [c for c in estimators if not is_abstract(c[1])] if not include_dont_test: estimators = [c for c in estimators if not c[0] in DONT_TEST] if not include_other: estimators = [c for c in estimators if not c[0] in OTHER] # possibly get rid of meta estimators if not include_meta_estimators: estimators = [c for c in estimators if not c[0] in META_ESTIMATORS] if type_filter is not None: if not isinstance(type_filter, list): type_filter = [type_filter] else: type_filter = list(type_filter) # copy filtered_estimators = [] filters = { 'classifier': ClassifierMixin, 'regressor': RegressorMixin, 'transformer': TransformerMixin, 'cluster': ClusterMixin } for name, mixin in filters.items(): if name in type_filter: type_filter.remove(name) filtered_estimators.extend( [est for est in estimators if issubclass(est[1], mixin)]) estimators = filtered_estimators if type_filter: raise ValueError( "Parameter type_filter must be 'classifier', " "'regressor', 'transformer', 'cluster' or None, got" " %s." % repr(type_filter)) # drop duplicates, sort for reproducibility return sorted(set(estimators))
def getSources(self, name, title, year, imdb, tmdb, tvdb, tvrage, season, episode, tvshowtitle, alter, date): sourceDict = [] for package, name, is_pkg in pkgutil.walk_packages(__path__): sourceDict.append((name, is_pkg)) sourceDict = [i[0] for i in sourceDict if i[1] == False] content = 'movie' if tvshowtitle == None else 'episode' if content == 'movie': sourceDict = [i for i in sourceDict if i.endswith(('_mv', '_mv_tv'))] try: sourceDict = [(i, control.setting(re.sub('_mv_tv$|_mv$|_tv$', '', i))) for i in sourceDict] except: sourceDict = [(i, 'true') for i in sourceDict] else: sourceDict = [i for i in sourceDict if i.endswith(('_tv', '_mv_tv'))] try: sourceDict = [(i, control.setting(re.sub('_mv_tv$|_mv$|_tv$', '', i) + '_tv')) for i in sourceDict] except: sourceDict = [(i, 'true') for i in sourceDict] threads = [] control.makeFile(control.dataPath) self.sourceFile = control.sourcescacheFile sourceDict = [i[0] for i in sourceDict if i[1] == 'true'] if content == 'movie': title = cleantitle.normalize(title) for source in sourceDict: threads.append(workers.Thread(self.getMovieSource, title, year, imdb, re.sub('_mv_tv$|_mv$|_tv$', '', source), __import__(source, globals(), locals(), [], -1).source())) else: tvshowtitle = cleantitle.normalize(tvshowtitle) season, episode = alterepisode.alterepisode().get(imdb, tmdb, tvdb, tvrage, season, episode, alter, title, date) for source in sourceDict: threads.append(workers.Thread(self.getEpisodeSource, title, year, imdb, tvdb, season, episode, tvshowtitle, date, re.sub('_mv_tv$|_mv$|_tv$', '', source), __import__(source, globals(), locals(), [], -1).source())) try: timeout = int(control.setting('sources_timeout_40')) except: timeout = 40 [i.start() for i in threads] control.idle() sourceLabel = [re.sub('_mv_tv$|_mv$|_tv$', '', i) for i in sourceDict] sourceLabel = [re.sub('v\d+$', '', i).upper() for i in sourceLabel] self.progressDialog = control.progressDialog self.progressDialog.create(control.addonInfo('name'), '') self.progressDialog.update(0) string1 = control.lang(30512).encode('utf-8') string2 = control.lang(30513).encode('utf-8') string3 = control.lang(30514).encode('utf-8') for i in range(0, timeout * 2): try: if xbmc.abortRequested == True: return sys.exit() try: info = [sourceLabel[int(re.sub('[^0-9]', '', str(x.getName()))) - 1] for x in threads if x.is_alive() == True] except: info = [] if len(info) > 5: info = len(info) self.progressDialog.update(int((100 / float(len(threads))) * len([x for x in threads if x.is_alive() == False])), str('%s: %s %s' % (string1, int(i * 0.5), string2)), str('%s: %s' % (string3, str(info).translate(None, "[]'")))) if self.progressDialog.iscanceled(): break is_alive = [x.is_alive() for x in threads] if all(x == False for x in is_alive): break time.sleep(0.5) except: pass self.progressDialog.close() return self.sources
# -*- coding: UTF-8 -*- # File: __init__.py # Author: Yuxin Wu <*****@*****.**> from pkgutil import walk_packages import os import os.path __all__ = [] def global_import(name): p = __import__(name, globals(), locals(), level=1) lst = p.__all__ if '__all__' in dir(p) else dir(p) del globals()[name] for k in lst: globals()[k] = p.__dict__[k] __all__.append(k) _CURR_DIR = os.path.dirname(__file__) for _, module_name, _ in walk_packages( [_CURR_DIR]): srcpath = os.path.join(_CURR_DIR, module_name + '.py') if not os.path.isfile(srcpath): continue if not module_name.startswith('_'): global_import(module_name)
import warnings import importlib from pkgutil import walk_packages from inspect import getsource, isabstract import sklearn from sklearn.base import signature from sklearn.utils.testing import SkipTest from sklearn.utils.testing import check_docstring_parameters from sklearn.utils.testing import _get_func_name from sklearn.utils.testing import ignore_warnings from sklearn.utils.deprecation import _is_deprecated PUBLIC_MODULES = set([ pckg[1] for pckg in walk_packages(prefix='sklearn.', path=sklearn.__path__) if not ("._" in pckg[1] or ".tests." in pckg[1]) ]) # TODO Uncomment all modules and fix doc inconsistencies everywhere # The list of modules that are not tested for now IGNORED_MODULES = ( 'cluster', 'datasets', 'decomposition', 'feature_extraction', 'gaussian_process', 'linear_model', 'ensemble', 'feature_selection', 'kernel_approximation',
def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False): """ Search for modules under a certain path modulePath is the import string needed to access the parent module. Root modules will be included automatically (like DIRAC). For instance "ConfigurationSystem.Service" reFilter is a regular expression to filter what to load. For instance ".*Handler" parentClass is a class object from which the loaded modules have to import from. For instance RequestHandler """ if 'OrderedDict' in dir(collections): modules = collections.OrderedDict() else: modules = {} if type(reFilter) in types.StringTypes: reFilter = re.compile(reFilter) for rootModule in self.__rootModules: if rootModule: impPath = "%s.%s" % (rootModule, modulePath) else: impPath = modulePath gLogger.debug("Trying to load %s" % impPath) result = self.__recurseImport(impPath) if not result['OK']: return result if not result['Value']: continue parentModule = result['Value'] fsPath = parentModule.__path__[0] gLogger.verbose("Loaded module %s at %s" % (impPath, fsPath)) for _modLoader, modName, isPkg in pkgutil.walk_packages( parentModule.__path__): if reFilter and not reFilter.match(modName): continue if isPkg: if recurse: result = self.getObjects("%s.%s" % (modulePath, modName), reFilter=reFilter, parentClass=parentClass, recurse=recurse) if not result['OK']: return result modules.update(result['Value']) continue modKeyName = "%s.%s" % (modulePath, modName) if modKeyName in modules: continue fullName = "%s.%s" % (impPath, modName) result = self.__recurseImport(modName, parentModule=parentModule, fullName=fullName) if not result['OK']: return result if not result['Value']: continue modObj = result['Value'] try: modClass = getattr(modObj, modName) except AttributeError: gLogger.warn("%s does not contain a %s object" % (fullName, modName)) continue if parentClass and not issubclass(modClass, parentClass): continue #Huge success! modules[modKeyName] = modClass return S_OK(modules)
def load_object(obj_dict): """ Creates an instantiation of a class based on a dictionary representation. We implicitly determine the Class through introspection along with information in the dictionary. We search for a class with the _fw_name property equal to obj_dict['_fw_name'] If the @module key is set, that module is checked first for a matching class to improve speed of lookup. Afterwards, the modules in the USER_PACKAGES global parameter are checked. Refactoring class names, module names, etc. will not break object loading as long as: i) the _fw_name property is maintained the same AND ii) the refactored module is kept within USER_PACKAGES You can get around these limitations if you really want: i) If you want to change the fw_name of an object you can set the FW_NAME_UPDATES key ii) If you want to put a refactored module in a new place add an entry to USER_PACKAGES :param obj_dict: the dict representation of the class """ # override the name in the obj_dict if there's an entry in FW_NAME_UPDATES fw_name = FW_NAME_UPDATES.get(obj_dict['_fw_name'], obj_dict['_fw_name']) obj_dict['_fw_name'] = fw_name # check for explicit serialization, e.g. {{fireworks.tasks.MyTask}} - based on pymatgen method if fw_name.startswith('{{') and fw_name.endswith('}}'): modname, classname = fw_name.strip('{} ').rsplit(".", 1) mod = __import__(modname, globals(), locals(), [classname], 0) if hasattr(mod, classname): cls_ = getattr(mod, classname) return cls_.from_dict(obj_dict) # first try to load from known location if fw_name in SAVED_FW_MODULES: m_module = importlib.import_module(SAVED_FW_MODULES[fw_name]) m_object = _search_module_for_obj(m_module, obj_dict) if m_object is not None: return m_object # failing that, look for the object within all of USER_PACKAGES # this will be slow, but only needed the first time found_objects = [] # used to make sure we don't find multiple hits for package in USER_PACKAGES: root_module = importlib.import_module(package) for loader, mod_name, is_pkg in pkgutil.walk_packages( root_module.__path__, package + '.'): try: m_module = loader.find_module(mod_name).load_module(mod_name) m_object = _search_module_for_obj(m_module, obj_dict) if m_object is not None: found_objects.append((m_object, mod_name)) except ImportError as ex: import warnings warnings.warn( "%s in %s cannot be loaded because of %s. Skipping.." % (m_object, mod_name, str(ex))) traceback.print_exc(ex) if len(found_objects) == 1: SAVED_FW_MODULES[fw_name] = found_objects[0][1] return found_objects[0][0] elif len(found_objects) > 0: raise ValueError( 'load_object() found multiple objects with cls._fw_name {} -- {}'. format(fw_name, found_objects)) raise ValueError( 'load_object() could not find a class with cls._fw_name {}'.format( fw_name))
def init(): # Import all the packages in this directory so their hooks get run. for _, name, _ in pkgutil.walk_packages(__path__, '.'): importlib.import_module(name, __package__)
return self.encoders[t] except KeyError: logger.warn('Requested unknown encoder %s' % t, exc_info=True) return None def get_decoder(self, t): try: return self.decoders[t] except KeyError: logger.warn('Requested unknown decoder %s' % t, exc_info=True) return None _ctx = MarshallingCtx(ENCODERS, DECODERS) packages = pkgutil.walk_packages(encoders.__path__, encoders.__name__ + '.') for module_loader, name, ispkg in packages: m = importlib.import_module(name) try: t, encoder = m.encoder ENCODERS[t] = encoder(_ctx) except AttributeError: pass packages = pkgutil.walk_packages(decoders.__path__, decoders.__name__ + '.') for module_loader, name, ispkg in packages: m = importlib.import_module(name) try: t, decoder = m.decoder DECODERS[t] = decoder(_ctx) except AttributeError:
def find_packages(path=["."], prefix=""): yield prefix prefix = prefix + "." for _, name, ispkg in walk_packages(path, prefix): if ispkg: yield name
if options.copy: copy_qtgui_to_qtwidgets() elif options.generate: generate_common_members() elif options.sort: sort_common_members() else: # Import <binding> binding = __import__(options.binding) for importer, modname, ispkg in pkgutil.walk_packages( path=binding.__path__, prefix=binding.__name__ + '.', onerror=lambda x: None): if modname not in SKIP_MODULES: MODULES.append(modname) basemodule = modname[:modname.rfind('.')] submodule = modname[modname.rfind('.')+1:] try: import_statement = ( 'from ' + basemodule + ' import ' + submodule) exec(import_statement) # print(import_statement) except (ImportError, AttributeError, SyntaxError) as error: # SyntaxError catched here because e.g. _port3 # running on Python 2... print('WARNING: Skipped import', modname, error)
#!/usr/bin/env python # BSD 3-Clause License; see https://github.com/scikit-hep/uproot3-methods/blob/master/LICENSE def hasmethods(name): if name not in globals(): if name in hasmethods.loaders: globals()[name] = hasmethods.loaders[name].load_module(name) elif '_3c_' in name and '_3e_' in name: bare_name = name.split('_3c_')[0] if bare_name in hasmethods.loaders: globals()[name] = hasmethods.loaders[bare_name].load_module(bare_name) return name in globals() and isinstance(getattr(globals()[name], "Methods", None), type) import pkgutil hasmethods.loaders = dict([(module_name, loader.find_module(module_name)) for loader, module_name, is_pkg in pkgutil.walk_packages(__path__)]) del pkgutil
def all_estimators(type_filter=None,): """Get a list of all estimators from imblearn. This function crawls the module and gets all classes that inherit from BaseEstimator. Classes that are defined in test-modules are not included. By default meta_estimators are also not included. This function is adapted from sklearn. Parameters ---------- type_filter : string, list of string, or None, default=None Which kind of estimators should be returned. If None, no filter is applied and all estimators are returned. Possible values are 'sampler' to get estimators only of these specific types, or a list of these to get the estimators that fit at least one of the types. Returns ------- estimators : list of tuples List of (name, class), where ``name`` is the class name as string and ``class`` is the actual type of the class. """ def is_abstract(c): if not (hasattr(c, "__abstractmethods__")): return False if not len(c.__abstractmethods__): return False return True all_classes = [] modules_to_ignore = {"tests"} root = str(Path(__file__).parent.parent) # Ignore deprecation warnings triggered at import time and from walking # packages with ignore_warnings(category=FutureWarning): for importer, modname, ispkg in pkgutil.walk_packages( path=[root], prefix='skclean.'): mod_parts = modname.split(".") if (any(part in modules_to_ignore for part in mod_parts) or '._' in modname): continue module = import_module(modname) classes = inspect.getmembers(module, inspect.isclass) classes = [(name, est_cls) for name, est_cls in classes if not name.startswith("_")] all_classes.extend(classes) all_classes = set(all_classes) estimators = [ c for c in all_classes if (issubclass(c[1], BaseEstimator) and c[0] != "BaseEstimator") ] # get rid of abstract base classes estimators = [c for c in estimators if not is_abstract(c[1])] # get rid of sklearn estimators which have been imported in some classes estimators = [c for c in estimators if "sklearn" not in c[1].__module__] # drop duplicates, sort for reproducibility # itemgetter is used to ensure the sort does not extend to the 2nd item of # the tuple return sorted(set(estimators), key=itemgetter(0))
def command_names(): from seed import commands names = set((pkg[1] for pkg in walk_packages(path=commands.__path__))) return list(names)
from sklearn.utils.estimator_checks import _enforce_estimator_tags_y from sklearn.utils.estimator_checks import _enforce_estimator_tags_x from sklearn.utils.deprecation import _is_deprecated from sklearn.externals._pep562 import Pep562 from sklearn.datasets import make_classification import pytest # walk_packages() ignores DeprecationWarnings, now we need to ignore # FutureWarnings with warnings.catch_warnings(): warnings.simplefilter('ignore', FutureWarning) PUBLIC_MODULES = set([ pckg[1] for pckg in walk_packages( prefix='sklearn.', # mypy error: Module has no attribute "__path__" path=sklearn.__path__) # type: ignore # mypy issue #1422 if not ("._" in pckg[1] or ".tests." in pckg[1]) ]) # functions to ignore args / docstring of _DOCSTRING_IGNORES = [ 'sklearn.utils.deprecation.load_mlcomp', 'sklearn.pipeline.make_pipeline', 'sklearn.pipeline.make_union', 'sklearn.utils.extmath.safe_sparse_dot', 'sklearn.utils._joblib' ] # Methods where y param should be ignored if y=None by default _METHODS_IGNORE_NONE_Y = [
def discover_modules(module_or_name): """Given an import name, provide an iterable of module filepath,name pairs. This function imports package __init__.py but does not import modules. More specifically, given a package name, this function walks the package (which requires importing the package __init__) and yields the filepath and name of each module (as a tuple of strings) in the package. Python treats both packages and modules as module objects, and this function makes no distinction. Give this function an import name and it returns an iterable regardless of whether the import name maps to a package or a module; in the case of a module name, the iterable results in only one element. Example: >>> discover_modules('testsite.stash') # doctest:+ELLIPSIS <generator object discover_modules at 0x...> >>> for item in discover_modules('testsite.stash'): ... print item # doctest: +ELLIPSIS ... testsite.stash testsite.stash.blank testsite.stash.blankexport testsite.stash.dummy testsite.stash.index testsite.stash.multiple testsite.stash.noexports testsite.stash.package testsite.stash.package.module testsite.stash.view_arg >>> list(discover_modules('testsite.stash.index')) # doctest:+ELLIPSIS ['testsite.stash.index'] >>> For flexibility & support, a module may be used in addition to a string. >>> import testsite.stash >>> discover_modules(testsite.stash) # doctest:+ELLIPSIS <generator object discover_modules at 0x...> >>> for item in discover_modules(testsite.stash): ... print item # doctest: +ELLIPSIS ... testsite.stash testsite.stash.blank testsite.stash.blankexport testsite.stash.dummy testsite.stash.index testsite.stash.multiple testsite.stash.noexports testsite.stash.package testsite.stash.package.module testsite.stash.view_arg >>> import testsite.stash.index >>> list(discover_modules(testsite.stash.index)) ['testsite.stash.index'] >>> :param module_or_name: Tango site stash import name, or imported module :type module_or_name: str """ if isinstance(module_or_name, types.ModuleType): module = module_or_name name = module.__name__ yield name else: module = None name = module_or_name yield name if module_is_package(name): if module is None: module = get_module(name) prefix = name + '.' for _, name, _ in pkgutil.walk_packages(module.__path__, prefix): yield name
def import_subpackages(module_prefix: str, module_path: Path): """Import all subpackages of a module.""" for _, module, _ in walk_packages([str(module_path)]): sub_module_name = '%s.%s' % (module_prefix, module) import_module(sub_module_name)
def checkSources(self, name, title, year, imdb, tmdb, tvdb, tvrage, season, episode, tvshowtitle, alter, date): sourceDict = [] for package, name, is_pkg in pkgutil.walk_packages(__path__): sourceDict.append((name, is_pkg)) sourceDict = [i[0] for i in sourceDict if i[1] == False] content = 'movie' if tvshowtitle == None else 'episode' if content == 'movie': sourceDict = [ i for i in sourceDict if i.endswith(('_mv', '_mv_tv')) ] try: sourceDict = [ (i, control.setting(re.sub('_mv_tv$|_mv$|_tv$', '', i))) for i in sourceDict ] except: sourceDict = [(i, 'true') for i in sourceDict] else: sourceDict = [ i for i in sourceDict if i.endswith(('_tv', '_mv_tv')) ] try: sourceDict = [(i, control.setting( re.sub('_mv_tv$|_mv$|_tv$', '', i) + '_tv')) for i in sourceDict] except: sourceDict = [(i, 'true') for i in sourceDict] threads = [] control.makeFile(control.dataPath) self.sourceFile = control.sourcescacheFile sourceDict = [i[0] for i in sourceDict if i[1] == 'true'] if content == 'movie': title = cleantitle.normalize(title) for source in sourceDict: threads.append( workers.Thread( self.getMovieSource, title, year, imdb, re.sub('_mv_tv$|_mv$|_tv$', '', source), __import__(source, globals(), locals(), [], -1).source())) else: tvshowtitle = cleantitle.normalize(tvshowtitle) season, episode = alterepisode.alterepisode().get( imdb, tmdb, tvdb, tvrage, season, episode, alter, title, date) for source in sourceDict: threads.append( workers.Thread( self.getEpisodeSource, title, year, imdb, tvdb, season, episode, tvshowtitle, date, re.sub('_mv_tv$|_mv$|_tv$', '', source), __import__(source, globals(), locals(), [], -1).source())) try: timeout = int(control.setting('sources_timeout_40')) except: timeout = 40 [i.start() for i in threads] for i in range(0, timeout * 2): try: if xbmc.abortRequested == True: return sys.exit() if len(self.sources) >= 10: break is_alive = [x.is_alive() for x in threads] if all(x == False for x in is_alive): break time.sleep(0.5) except: pass if len(self.sources) >= 10: return True else: return False
if upfront or toctree2 is None: if functionname not in common: toctree.write(" " + functionname + "\n") toctree2.write(" " + functionname + " <" + functionname + ">\n") else: toctree2.write(" " + functionname + "\n") for modulename in order: module = importlib.import_module(modulename) if modulename != "uproot": toctree2 = open(modulename + ".toctree", "w") toctree2.write(""".. toctree:: :caption: {0} :hidden: """.format(modulename.replace("uproot.", ""))) handle_module(modulename, module) if module.__file__.endswith("__init__.py") and modulename != "uproot": for submodulename in sorted([ modulename + "." + name for loader, name, is_pkg in pkgutil.walk_packages(module.__path__) ]): submodule = importlib.import_module(submodulename) handle_module(submodulename, submodule) toctree.close() toctree2.close()
def _get_module_names(package_name): return [ "{}.{}".format(package_name, name) for _, name, _ in pkgutil.walk_packages([package_name]) ]
def all_estimators(include_meta_estimators=False, include_other=False, type_filter=None): """Get a list of all estimators from sklearn. This function crawls the module and gets all classes that inherit from BaseEstimator. Classes that are defined in test-modules are not included. By default meta_estimators such as GridSearchCV are also not included. Parameters ---------- include_meta_estimators : boolean, default=False Whether to include meta-estimators that can be constructed using an estimator as their first argument. These are currently BaseEnsemble, OneVsOneClassifier, OutputCodeClassifier, OneVsRestClassifier, RFE, RFECV. include_others : boolean, default=False Wether to include meta-estimators that are somehow special and can not be default-constructed sensibly. These are currently Pipeline, FeatureUnion and GridSearchCV type_filter : string or None, default=None Which kind of estimators should be returned. If None, no filter is applied and all estimators are returned. Possible values are 'classifier', 'regressor', 'cluster' and 'transformer' to get estimators only of these specific types. Returns ------- estimators : list of tuples List of (name, class), where ``name`` is the class name as string and ``class`` is the actuall type of the class. """ def is_abstract(c): if not (hasattr(c, '__abstractmethods__')): return False if not len(c.__abstractmethods__): return False return True all_classes = [] # get parent folder path = sklearn.__path__ for importer, modname, ispkg in pkgutil.walk_packages( path=path, prefix='sklearn.', onerror=lambda x: None): if ".tests." in modname: continue module = __import__(modname, fromlist="dummy") classes = inspect.getmembers(module, inspect.isclass) all_classes.extend(classes) all_classes = set(all_classes) estimators = [ c for c in all_classes if (issubclass(c[1], BaseEstimator) and c[0] != 'BaseEstimator') ] # get rid of abstract base classes estimators = [c for c in estimators if not is_abstract(c[1])] if not include_other: estimators = [c for c in estimators if not c[0] in other] # possibly get rid of meta estimators if not include_meta_estimators: estimators = [c for c in estimators if not c[0] in meta_estimators] if type_filter == 'classifier': estimators = [ est for est in estimators if issubclass(est[1], ClassifierMixin) ] elif type_filter == 'regressor': estimators = [ est for est in estimators if issubclass(est[1], RegressorMixin) ] elif type_filter == 'transformer': estimators = [ est for est in estimators if issubclass(est[1], TransformerMixin) ] elif type_filter == 'cluster': estimators = [ est for est in estimators if issubclass(est[1], ClusterMixin) ] elif type_filter is not None: raise ValueError("Parameter type_filter must be 'classifier', " "'regressor', 'transformer', 'cluster' or None, got" " %s." % repr(type_filter)) # We sort in order to have reproducible test failures return sorted(estimators)
def _backported_all_estimators(type_filter=None): """ Backported from scikit-learn 0.23.2: https://github.com/scikit-learn/scikit-learn/blob/0.23.2/sklearn/utils/__init__.py#L1146 Use this backported `all_estimators` in old versions of sklearn because: 1. An inferior version of `all_estimators` that old versions of sklearn use for testing, might function differently from a newer version. 2. This backported `all_estimators` works on old versions of sklearn that don’t even define the testing utility variant of `all_estimators`. ========== original docstring ========== Get a list of all estimators from sklearn. This function crawls the module and gets all classes that inherit from BaseEstimator. Classes that are defined in test-modules are not included. By default meta_estimators such as GridSearchCV are also not included. Parameters ---------- type_filter : string, list of string, or None, default=None Which kind of estimators should be returned. If None, no filter is applied and all estimators are returned. Possible values are 'classifier', 'regressor', 'cluster' and 'transformer' to get estimators only of these specific types, or a list of these to get the estimators that fit at least one of the types. Returns ------- estimators : list of tuples List of (name, class), where ``name`` is the class name as string and ``class`` is the actuall type of the class. """ # lazy import to avoid circular imports from sklearn.base import pkgutil import platform import sklearn from importlib import import_module from operator import itemgetter # pylint: disable=no-name-in-module, import-error from sklearn.utils.testing import ignore_warnings from sklearn.base import ( BaseEstimator, ClassifierMixin, RegressorMixin, TransformerMixin, ClusterMixin, ) IS_PYPY = platform.python_implementation() == "PyPy" def is_abstract(c): if not (hasattr(c, "__abstractmethods__")): return False if not len(c.__abstractmethods__): return False return True all_classes = [] modules_to_ignore = {"tests", "externals", "setup", "conftest"} root = sklearn.__path__[0] # sklearn package # Ignore deprecation warnings triggered at import time and from walking # packages with ignore_warnings(category=FutureWarning): for _, modname, _ in pkgutil.walk_packages(path=[root], prefix="sklearn."): mod_parts = modname.split(".") if any(part in modules_to_ignore for part in mod_parts) or "._" in modname: continue module = import_module(modname) classes = inspect.getmembers(module, inspect.isclass) classes = [(name, est_cls) for name, est_cls in classes if not name.startswith("_")] # TODO: Remove when FeatureHasher is implemented in PYPY # Skips FeatureHasher for PYPY if IS_PYPY and "feature_extraction" in modname: classes = [(name, est_cls) for name, est_cls in classes if name == "FeatureHasher"] all_classes.extend(classes) all_classes = set(all_classes) estimators = [ c for c in all_classes if (issubclass(c[1], BaseEstimator) and c[0] != "BaseEstimator") ] # get rid of abstract base classes estimators = [c for c in estimators if not is_abstract(c[1])] if type_filter is not None: if not isinstance(type_filter, list): type_filter = [type_filter] else: type_filter = list(type_filter) # copy filtered_estimators = [] filters = { "classifier": ClassifierMixin, "regressor": RegressorMixin, "transformer": TransformerMixin, "cluster": ClusterMixin, } for name, mixin in filters.items(): if name in type_filter: type_filter.remove(name) filtered_estimators.extend( [est for est in estimators if issubclass(est[1], mixin)]) estimators = filtered_estimators if type_filter: raise ValueError("Parameter type_filter must be 'classifier', " "'regressor', 'transformer', 'cluster' or " "None, got" " %s." % repr(type_filter)) # drop duplicates, sort for reproducibility # itemgetter is used to ensure the sort does not extend to the 2nd item of # the tuple return sorted(set(estimators), key=itemgetter(0))
import importlib import logging import pkgutil # Load all repository handlers. modules = dict() _required_functions = ('verify', 'resolve_latest', 'get_artifact', 'repro_arg') for importer, modname, ispkg in pkgutil.walk_packages(path=__path__, prefix=__name__ + '.'): module = importlib.import_module(modname) bad_module = False for function in _required_functions: if not hasattr(module, function): logging.error('Ignoring %s: No %s function' % function) bad_module = True if bad_module: continue assert modname.startswith('%s.' % __name__) shortname = modname[len('%s.' % __name__):] modules[shortname] = module
): for task_measure, task_measure_name in task_related_measures: if custom_measure == task_measure: return if 'CUSTOM' not in AntMeasuresFactory.factory_measures: AntMeasuresFactory.factory_measures['CUSTOM'] = [] AntMeasuresFactory.factory_measures['CUSTOM'].append( (custom_measure, None)) def _global_import(name): p = __import__(name, globals(), locals(), level=1) lst = p.default if 'default' in dir(p) else {} if len(lst) > 0: globals().pop(name) for measure_method, keys in lst.items(): globals()[measure_method] = p.__dict__[measure_method] measure_name, task_type = keys if task_type not in AntMeasuresFactory.factory_measures: AntMeasuresFactory.factory_measures[task_type] = [] AntMeasuresFactory.factory_measures[task_type].append( (p.__dict__[measure_method], measure_name)) for _, module_name, _ in walk_packages([os.path.dirname(__file__)]): if not module_name.startswith('_'): _global_import(module_name)
__author__ = 'dxl' import pkgutil __path__ = pkgutil.extend_path(__path__, __name__) for imp, module, ispackage in pkgutil.walk_packages(path=__path__, prefix=__name__ + '.'): __import__(module)