def ensure_coherence(self):
# make sure parent paths show up as updated appropriately
self.modified |= {p for a in self.added for p in iter_prefixes(a)}
self.modified |= {p for u in self.modified for p in iter_prefixes(u)}
self.modified |= {p for t in self.typechanged
for p in iter_prefixes(t)}
# make sure there is no overlap
self.added -= set(self.typechanged.keys())
self.modified -= set(self.typechanged.keys())
self.modified -= self.added
def ensure_coherence(self):
# make sure parent paths show up as updated appropriately
self.modified |= {p for a in self.added for p in iter_prefixes(a)}
self.modified |= {p for u in self.modified for p in iter_prefixes(u)}
self.modified |= {p for t in self.typechanged
for p in iter_prefixes(t)}
# make sure there is no overlap
self.added -= set(self.typechanged.keys())
self.modified -= set(self.typechanged.keys())
self.modified -= self.added
def gather_sources_and_dependencies(globs):
dependencies = set()
main = Source.create(globs.get('__file__'))
sources = {main}
experiment_path = os.path.dirname(main.filename)
for glob in globs.values():
if isinstance(glob, module):
mod_path = glob.__name__
elif hasattr(glob, '__module__'):
mod_path = glob.__module__
else:
continue # pragma: no cover
if not mod_path:
continue
for modname in iter_prefixes(mod_path):
mod = sys.modules.get(modname)
create_source_or_dep(modname, mod, dependencies, sources,
experiment_path)
if opt.has_numpy:
# Add numpy as a dependency because it might be used for randomness
dependencies.add(PackageDependency.create(opt.np))
return sources, dependencies
def gather_sources_and_dependencies(globs):
dependencies = set()
main = Source.create(globs.get('__file__'))
sources = {main}
experiment_path = os.path.dirname(main.filename)
for glob in globs.values():
if isinstance(glob, module):
mod_path = glob.__name__
elif hasattr(glob, '__module__'):
mod_path = glob.__module__
else:
continue # pragma: no cover
if not mod_path:
continue
for modname in iter_prefixes(mod_path):
mod = sys.modules.get(modname)
create_source_or_dep(modname, mod, dependencies, sources,
experiment_path)
if opt.has_numpy:
# Add numpy as a dependency because it might be used for randomness
dependencies.add(PackageDependency.create(opt.np))
return sources, dependencies
def iterate_imported_modules(globs):
checked_modules = set(MODULE_BLACKLIST)
for glob in globs.values():
if isinstance(glob, module):
mod_path = glob.__name__
elif hasattr(glob, '__module__'):
mod_path = glob.__module__
else:
continue # pragma: no cover
if not mod_path:
continue
for modname in iter_prefixes(mod_path):
if modname in checked_modules:
continue
checked_modules.add(modname)
mod = sys.modules.get(modname)
if mod is not None:
yield modname, mod
def _warn_about_suspicious_changes(self):
for add in sorted(self.config_mods.added):
if not set(iter_prefixes(add)).intersection(self.captured_args):
raise KeyError('Added a new config entry "{}" that is not used'
' anywhere'.format(add))
else:
self.logger.warning('Added new config entry: "%s"' % add)
for key, (type_old, type_new) in self.config_mods.typechanged.items():
if type_old in (int, float) and type_new in (int, float):
continue
self.logger.warning(
'Changed type of config entry "%s" from %s to %s' %
(key, type_old.__name__, type_new.__name__))
for cfg_summary in self.summaries:
for key in cfg_summary.ignored_fallbacks:
self.logger.warning(
'Ignored attempt to set value of "%s", because it is an '
'ingredient.' % key)
def iterate_imported_modules(globs):
checked_modules = set(MODULE_BLACKLIST)
for glob in globs.values():
if isinstance(glob, module):
mod_path = glob.__name__
elif hasattr(glob, '__module__'):
mod_path = glob.__module__
else:
continue # pragma: no cover
if not mod_path:
continue
for modname in iter_prefixes(mod_path):
if modname in checked_modules:
continue
checked_modules.add(modname)
mod = sys.modules.get(modname)
if mod is not None:
yield modname, mod
def _warn_about_suspicious_changes(self):
for add in sorted(self.config_mods.added):
if not set(iter_prefixes(add)).intersection(self.captured_args):
raise KeyError('Added a new config entry "{}" that is not used'
' anywhere'.format(add))
else:
self.logger.warning('Added new config entry: "%s"' % add)
for key, (type_old, type_new) in self.config_mods.typechanged.items():
if type_old in (int, float) and type_new in (int, float):
continue
self.logger.warning(
'Changed type of config entry "%s" from %s to %s' %
(key, type_old.__name__, type_new.__name__))
for cfg_summary in self.summaries:
for key in cfg_summary.ignored_fallbacks:
self.logger.warning(
'Ignored attempt to set value of "%s", because it is an '
'ingredient.' % key
)
def _warn_about_suspicious_changes(self):
for add in sorted(self.config_mods.added):
if not set(iter_prefixes(add)).intersection(self.captured_args):
if self.path:
add = join_paths(self.path, add)
raise ConfigAddedError(add, config=self.config)
else:
self.logger.warning('Added new config entry: "%s"' % add)
for key, (type_old, type_new) in self.config_mods.typechanged.items():
if type_old in (int, float) and type_new in (int, float):
continue
self.logger.warning(
'Changed type of config entry "%s" from %s to %s' %
(key, type_old.__name__, type_new.__name__))
for cfg_summary in self.summaries:
for key in cfg_summary.ignored_fallbacks:
self.logger.warning(
'Ignored attempt to set value of "%s", because it is an '
"ingredient." % key)
def gather_sources_and_dependencies(globs, interactive=False):
"""Scan the given globals for modules and return them as dependencies."""
dependencies = set()
filename = globs.get('__file__')
if filename is None:
if not interactive:
raise RuntimeError("Defining an experiment in interactive mode! "
"The sourcecode cannot be stored and the "
"experiment won't be reproducible. If you still"
" want to run it pass interactive=True")
sources = set()
experiment_path = os.path.abspath(os.path.curdir)
main = None
else:
main = Source.create(globs.get('__file__'))
sources = {main}
experiment_path = os.path.dirname(main.filename)
for glob in globs.values():
if isinstance(glob, module):
mod_path = glob.__name__
elif hasattr(glob, '__module__'):
mod_path = glob.__module__
else:
continue # pragma: no cover
if not mod_path:
continue
for modname in iter_prefixes(mod_path):
mod = sys.modules.get(modname)
create_source_or_dep(modname, mod, dependencies, sources,
experiment_path)
if opt.has_numpy:
# Add numpy as a dependency because it might be used for randomness
dependencies.add(PackageDependency.create(opt.np))
return main, sources, dependencies
def gather_sources_and_dependencies(globs, interactive=False):
dependencies = set()
filename = globs.get('__file__')
if filename is None:
if not interactive:
raise RuntimeError("Defining an experiment in interactive mode! "
"The sourcecode cannot be stored and the "
"experiment won't be reproducible. If you still"
" want to run it pass interactive=True")
sources = set()
experiment_path = os.path.abspath(os.path.curdir)
else:
main = Source.create(globs.get('__file__'))
sources = {main}
experiment_path = os.path.dirname(main.filename)
for glob in globs.values():
if isinstance(glob, module):
mod_path = glob.__name__
elif hasattr(glob, '__module__'):
mod_path = glob.__module__
else:
continue # pragma: no cover
if not mod_path:
continue
for modname in iter_prefixes(mod_path):
mod = sys.modules.get(modname)
create_source_or_dep(modname, mod, dependencies, sources,
experiment_path)
if opt.has_numpy:
# Add numpy as a dependency because it might be used for randomness
dependencies.add(PackageDependency.create(opt.np))
return sources, dependencies
def gather_sources_and_dependencies(globs):
dependencies = set()
filename = globs.get('__file__')
if filename is None:
import warnings
warnings.warn("Defining an experiment in interactive mode! "
"The sourcecode cannot be stored and the experiment "
"won't be reproducible")
sources = set()
experiment_path = os.path.abspath(os.path.curdir)
else:
main = Source.create(globs.get('__file__'))
sources = {main}
experiment_path = os.path.dirname(main.filename)
for glob in globs.values():
if isinstance(glob, module):
mod_path = glob.__name__
elif hasattr(glob, '__module__'):
mod_path = glob.__module__
else:
continue # pragma: no cover
if not mod_path:
continue
for modname in iter_prefixes(mod_path):
mod = sys.modules.get(modname)
create_source_or_dep(modname, mod, dependencies, sources,
experiment_path)
if opt.has_numpy:
# Add numpy as a dependency because it might be used for randomness
dependencies.add(PackageDependency.create(opt.np))
return sources, dependencies
def test_iter_prefixes():
assert list(iter_prefixes('foo.bar.baz')) == \
['foo', 'foo.bar', 'foo.bar.baz']
def test_iter_prefixes():
assert list(iter_prefixes("foo.bar.baz")) == ["foo", "foo.bar", "foo.bar.baz"]
def test_iter_prefixes():
assert list(iter_prefixes('foo.bar.baz')) == \
['foo', 'foo.bar', 'foo.bar.baz']
