def ensure_coherence(self): # make sure parent paths show up as updated appropriately self.modified |= {p for a in self.added for p in iter_prefixes(a)} self.modified |= {p for u in self.modified for p in iter_prefixes(u)} self.modified |= {p for t in self.typechanged for p in iter_prefixes(t)} # make sure there is no overlap self.added -= set(self.typechanged.keys()) self.modified -= set(self.typechanged.keys()) self.modified -= self.added
def gather_sources_and_dependencies(globs): dependencies = set() main = Source.create(globs.get('__file__')) sources = {main} experiment_path = os.path.dirname(main.filename) for glob in globs.values(): if isinstance(glob, module): mod_path = glob.__name__ elif hasattr(glob, '__module__'): mod_path = glob.__module__ else: continue # pragma: no cover if not mod_path: continue for modname in iter_prefixes(mod_path): mod = sys.modules.get(modname) create_source_or_dep(modname, mod, dependencies, sources, experiment_path) if opt.has_numpy: # Add numpy as a dependency because it might be used for randomness dependencies.add(PackageDependency.create(opt.np)) return sources, dependencies
def iterate_imported_modules(globs): checked_modules = set(MODULE_BLACKLIST) for glob in globs.values(): if isinstance(glob, module): mod_path = glob.__name__ elif hasattr(glob, '__module__'): mod_path = glob.__module__ else: continue # pragma: no cover if not mod_path: continue for modname in iter_prefixes(mod_path): if modname in checked_modules: continue checked_modules.add(modname) mod = sys.modules.get(modname) if mod is not None: yield modname, mod
def _warn_about_suspicious_changes(self): for add in sorted(self.config_mods.added): if not set(iter_prefixes(add)).intersection(self.captured_args): raise KeyError('Added a new config entry "{}" that is not used' ' anywhere'.format(add)) else: self.logger.warning('Added new config entry: "%s"' % add) for key, (type_old, type_new) in self.config_mods.typechanged.items(): if type_old in (int, float) and type_new in (int, float): continue self.logger.warning( 'Changed type of config entry "%s" from %s to %s' % (key, type_old.__name__, type_new.__name__)) for cfg_summary in self.summaries: for key in cfg_summary.ignored_fallbacks: self.logger.warning( 'Ignored attempt to set value of "%s", because it is an ' 'ingredient.' % key)
def _warn_about_suspicious_changes(self): for add in sorted(self.config_mods.added): if not set(iter_prefixes(add)).intersection(self.captured_args): raise KeyError('Added a new config entry "{}" that is not used' ' anywhere'.format(add)) else: self.logger.warning('Added new config entry: "%s"' % add) for key, (type_old, type_new) in self.config_mods.typechanged.items(): if type_old in (int, float) and type_new in (int, float): continue self.logger.warning( 'Changed type of config entry "%s" from %s to %s' % (key, type_old.__name__, type_new.__name__)) for cfg_summary in self.summaries: for key in cfg_summary.ignored_fallbacks: self.logger.warning( 'Ignored attempt to set value of "%s", because it is an ' 'ingredient.' % key )
def _warn_about_suspicious_changes(self): for add in sorted(self.config_mods.added): if not set(iter_prefixes(add)).intersection(self.captured_args): if self.path: add = join_paths(self.path, add) raise ConfigAddedError(add, config=self.config) else: self.logger.warning('Added new config entry: "%s"' % add) for key, (type_old, type_new) in self.config_mods.typechanged.items(): if type_old in (int, float) and type_new in (int, float): continue self.logger.warning( 'Changed type of config entry "%s" from %s to %s' % (key, type_old.__name__, type_new.__name__)) for cfg_summary in self.summaries: for key in cfg_summary.ignored_fallbacks: self.logger.warning( 'Ignored attempt to set value of "%s", because it is an ' "ingredient." % key)
def gather_sources_and_dependencies(globs, interactive=False): """Scan the given globals for modules and return them as dependencies.""" dependencies = set() filename = globs.get('__file__') if filename is None: if not interactive: raise RuntimeError("Defining an experiment in interactive mode! " "The sourcecode cannot be stored and the " "experiment won't be reproducible. If you still" " want to run it pass interactive=True") sources = set() experiment_path = os.path.abspath(os.path.curdir) main = None else: main = Source.create(globs.get('__file__')) sources = {main} experiment_path = os.path.dirname(main.filename) for glob in globs.values(): if isinstance(glob, module): mod_path = glob.__name__ elif hasattr(glob, '__module__'): mod_path = glob.__module__ else: continue # pragma: no cover if not mod_path: continue for modname in iter_prefixes(mod_path): mod = sys.modules.get(modname) create_source_or_dep(modname, mod, dependencies, sources, experiment_path) if opt.has_numpy: # Add numpy as a dependency because it might be used for randomness dependencies.add(PackageDependency.create(opt.np)) return main, sources, dependencies
def gather_sources_and_dependencies(globs, interactive=False): dependencies = set() filename = globs.get('__file__') if filename is None: if not interactive: raise RuntimeError("Defining an experiment in interactive mode! " "The sourcecode cannot be stored and the " "experiment won't be reproducible. If you still" " want to run it pass interactive=True") sources = set() experiment_path = os.path.abspath(os.path.curdir) else: main = Source.create(globs.get('__file__')) sources = {main} experiment_path = os.path.dirname(main.filename) for glob in globs.values(): if isinstance(glob, module): mod_path = glob.__name__ elif hasattr(glob, '__module__'): mod_path = glob.__module__ else: continue # pragma: no cover if not mod_path: continue for modname in iter_prefixes(mod_path): mod = sys.modules.get(modname) create_source_or_dep(modname, mod, dependencies, sources, experiment_path) if opt.has_numpy: # Add numpy as a dependency because it might be used for randomness dependencies.add(PackageDependency.create(opt.np)) return sources, dependencies
def gather_sources_and_dependencies(globs): dependencies = set() filename = globs.get('__file__') if filename is None: import warnings warnings.warn("Defining an experiment in interactive mode! " "The sourcecode cannot be stored and the experiment " "won't be reproducible") sources = set() experiment_path = os.path.abspath(os.path.curdir) else: main = Source.create(globs.get('__file__')) sources = {main} experiment_path = os.path.dirname(main.filename) for glob in globs.values(): if isinstance(glob, module): mod_path = glob.__name__ elif hasattr(glob, '__module__'): mod_path = glob.__module__ else: continue # pragma: no cover if not mod_path: continue for modname in iter_prefixes(mod_path): mod = sys.modules.get(modname) create_source_or_dep(modname, mod, dependencies, sources, experiment_path) if opt.has_numpy: # Add numpy as a dependency because it might be used for randomness dependencies.add(PackageDependency.create(opt.np)) return sources, dependencies
def test_iter_prefixes(): assert list(iter_prefixes('foo.bar.baz')) == \ ['foo', 'foo.bar', 'foo.bar.baz']
def test_iter_prefixes(): assert list(iter_prefixes("foo.bar.baz")) == ["foo", "foo.bar", "foo.bar.baz"]