def get_config_modifications(self):
self.config_mods = ConfigSummary(added={
key
for key, value in iterate_flattened(self.config_updates)
})
for cfg_summary in self.summaries:
self.config_mods.update_from(cfg_summary)
def __call__(self, fixed=None, preset=None, fallback=None):
result = dogmatize(fixed or {})
recursive_fill_in(result, self._conf)
recursive_fill_in(result, preset or {})
added = result.revelation()
config_summary = ConfigSummary(added, result.modified, result.typechanges)
config_summary.update(undogmatize(result))
return config_summary
def __call__(self, fixed=None, preset=None, fallback=None):
result = dogmatize(fixed or {})
result.update(preset)
result.update(self._conf)
added = result.revelation()
config_summary = ConfigSummary(added, result.modified, result.typechanges)
config_summary.update(undogmatize(result))
return config_summary
def run():
config = {'a': 17, 'foo': {'bar': True, 'baz': False}, 'seed': 1234}
config_mod = ConfigSummary()
main_func = mock.Mock(return_value=123)
logger = mock.Mock()
observer = [mock.Mock()]
return Run(config, config_mod, main_func, observer, logger, logger, {}, {},
[], [])
def __call__(self, fixed=None, preset=None, fallback=None):
"""
Evaluate this ConfigScope.
This will evaluate the function body and fill the relevant local
variables into entries into keys in this dictionary.
:param fixed: Dictionary of entries that should stay fixed during the
evaluation. All of them will be part of the final config.
:type fixed: dict
:param preset: Dictionary of preset values that will be available
during the evaluation (if they are declared in the
function argument list). All of them will be part of the
final config.
:type preset: dict
:param fallback: Dictionary of fallback values that will be available
during the evaluation (if they are declared in the
function argument list). They will NOT be part of the
final config.
:type fallback: dict
:return: self
:rtype: ConfigScope
"""
cfg_locals = dogmatize(fixed or {})
fallback = fallback or {}
preset = preset or {}
fallback_view = {}
available_entries = set(preset.keys()) | set(fallback.keys())
for arg in self.arg_spec.args:
if arg not in available_entries:
raise KeyError("'{}' not in preset for ConfigScope. "
"Available options are: {}".format(
arg, available_entries))
if arg in preset:
cfg_locals[arg] = preset[arg]
else: # arg in fallback
fallback_view[arg] = fallback[arg]
cfg_locals.fallback = fallback_view
eval(self._body_code, copy(self._func.__globals__), cfg_locals)
added = cfg_locals.revelation()
config_summary = ConfigSummary(added,
cfg_locals.modified,
cfg_locals.typechanges,
cfg_locals.fallback_writes,
docs=self._var_docs)
# fill in the unused presets
recursive_fill_in(cfg_locals, preset)
for key, value in cfg_locals.items():
try:
config_summary[key] = normalize_or_die(value)
except ValueError:
pass
return config_summary
def run():
config = {'a': 17, 'foo': {'bar': True, 'baz': False}, 'seed': 1234}
config_mod = ConfigSummary()
signature = mock.Mock()
signature.name = 'main_func'
main_func = mock.Mock(return_value=123, prefix='', signature=signature)
logger = mock.Mock()
observer = [mock.Mock(priority=10)]
return Run(config, config_mod, main_func, observer, logger, logger, {},
{}, [], [])
def run():
config = {"a": 17, "foo": {"bar": True, "baz": False}, "seed": 1234}
config_mod = ConfigSummary()
signature = mock.Mock()
signature.name = "main_func"
main_func = mock.Mock(return_value=123, prefix="", signature=signature)
logger = mock.Mock()
observer = [mock.Mock(priority=10)]
return Run(config, config_mod, main_func, observer, logger, logger, {}, {},
[], [])
def test_iterate_marked(cfg):
assert list(_iterate_marked(cfg, ConfigSummary())) == [
("a", ConfigEntry("a", 0, False, False, None, None)),
("b", ConfigEntry("b", {}, False, False, None, None)),
("c", PathEntry("c", False, False, None, None)),
("c.cA", ConfigEntry("cA", 3, False, False, None, None)),
("c.cB", ConfigEntry("cB", 4, False, False, None, None)),
("c.cC", PathEntry("cC", False, False, None, None)),
("c.cC.cC1", ConfigEntry("cC1", 6, False, False, None, None)),
("d", PathEntry("d", False, False, None, None)),
("d.dA", ConfigEntry("dA", 8, False, False, None, None)),
]
def test_iterate_marked(cfg):
assert list(_iterate_marked(cfg, ConfigSummary())) == \
[('a', ConfigEntry('a', 0, False, False, None, None)),
('b', ConfigEntry('b', {}, False, False, None, None)),
('c', PathEntry('c', False, False, None, None)),
('c.cA', ConfigEntry('cA', 3, False, False, None, None)),
('c.cB', ConfigEntry('cB', 4, False, False, None, None)),
('c.cC', PathEntry('cC', False, False, None, None)),
('c.cC.cC1', ConfigEntry('cC1', 6, False, False, None, None)),
('d', PathEntry('d', False, False, None, None)),
('d.dA', ConfigEntry('dA', 8, False, False, None, None))
]
def test_format_config(cfg):
cfg_text = _format_config(cfg, ConfigSummary())
lines = cfg_text.split('\n')
assert lines[0].startswith('Configuration')
assert lines[1].find(' a = 0') > -1
assert lines[2].find(' b = {}') > -1
assert lines[3].find(' c:') > -1
assert lines[4].find(' cA = 3') > -1
assert lines[5].find(' cB = 4') > -1
assert lines[6].find(' cC:') > -1
assert lines[7].find(' cC1 = 6') > -1
assert lines[8].find(' d:') > -1
assert lines[9].find(' dA = 8') > -1
def test_iterate_marked_added(cfg):
added = {'a', 'c.cB', 'c.cC.cC1'}
assert list(iterate_marked(cfg, ConfigSummary(added=added))) == \
[('a', ConfigEntry('a', 0, True, False, None)),
('b', ConfigEntry('b', {}, False, False, None)),
('c', PathEntry('c', False, True, None)),
('c.cA', ConfigEntry('cA', 3, False, False, None)),
('c.cB', ConfigEntry('cB', 4, True, False, None)),
('c.cC', PathEntry('cC', False, True, None)),
('c.cC.cC1', ConfigEntry('cC1', 6, True, False, None)),
('d', PathEntry('d', False, False, None)),
('d.dA', ConfigEntry('dA', 8, False, False, None))
]
def test_iterate_marked_updated(cfg):
modified = {"b", "c", "c.cC.cC1"}
assert list(_iterate_marked(cfg, ConfigSummary(modified=modified))) == [
("a", ConfigEntry("a", 0, False, False, None, None)),
("b", ConfigEntry("b", {}, False, True, None, None)),
("c", PathEntry("c", False, True, None, None)),
("c.cA", ConfigEntry("cA", 3, False, False, None, None)),
("c.cB", ConfigEntry("cB", 4, False, False, None, None)),
("c.cC", PathEntry("cC", False, True, None, None)),
("c.cC.cC1", ConfigEntry("cC1", 6, False, True, None, None)),
("d", PathEntry("d", False, False, None, None)),
("d.dA", ConfigEntry("dA", 8, False, False, None, None)),
]
def test_format_config(cfg):
cfg_text = _format_config(cfg, ConfigSummary())
lines = cfg_text.split("\n")
assert lines[0].startswith("Configuration")
assert " a = 0" in lines[1]
assert " b = {}" in lines[2]
assert " c:" in lines[3]
assert " cA = 3" in lines[4]
assert " cB = 4" in lines[5]
assert " cC:" in lines[6]
assert " cC1 = 6" in lines[7]
assert " d:" in lines[8]
assert " dA = 8" in lines[9]
def test_iterate_marked_updated(cfg):
modified = {'b', 'c', 'c.cC.cC1'}
assert list(_iterate_marked(cfg, ConfigSummary(modified=modified))) == \
[('a', ConfigEntry('a', 0, False, False, None, None)),
('b', ConfigEntry('b', {}, False, True, None, None)),
('c', PathEntry('c', False, True, None, None)),
('c.cA', ConfigEntry('cA', 3, False, False, None, None)),
('c.cB', ConfigEntry('cB', 4, False, False, None, None)),
('c.cC', PathEntry('cC', False, True, None, None)),
('c.cC.cC1', ConfigEntry('cC1', 6, False, True, None, None)),
('d', PathEntry('d', False, False, None, None)),
('d.dA', ConfigEntry('dA', 8, False, False, None, None))
]
def test_format_config(cfg):
cfg_text = _format_config(cfg, ConfigSummary())
lines = cfg_text.split('\n')
assert lines[0].startswith('Configuration')
assert ' a = 0' in lines[1]
assert ' b = {}' in lines[2]
assert ' c:' in lines[3]
assert ' cA = 3' in lines[4]
assert ' cB = 4' in lines[5]
assert ' cC:' in lines[6]
assert ' cC1 = 6' in lines[7]
assert ' d:' in lines[8]
assert ' dA = 8' in lines[9]
def test_iterate_marked_typechanged(cfg):
typechanged = {'a': (bool, int), 'd.dA': (float, int)}
result = list(_iterate_marked(cfg, ConfigSummary(typechanged=typechanged)))
assert result == \
[('a', ConfigEntry('a', 0, False, False, (bool, int), None)),
('b', ConfigEntry('b', {}, False, False, None, None)),
('c', PathEntry('c', False, False, None, None)),
('c.cA', ConfigEntry('cA', 3, False, False, None, None)),
('c.cB', ConfigEntry('cB', 4, False, False, None, None)),
('c.cC', PathEntry('cC', False, False, None, None)),
('c.cC.cC1', ConfigEntry('cC1', 6, False, False, None, None)),
('d', PathEntry('d', False, True, None, None)),
('d.dA', ConfigEntry('dA', 8, False, False, (float, int), None))
]
def test_iterate_marked_typechanged(cfg):
typechanged = {"a": (bool, int), "d.dA": (float, int)}
result = list(_iterate_marked(cfg, ConfigSummary(typechanged=typechanged)))
assert result == [
("a", ConfigEntry("a", 0, False, False, (bool, int), None)),
("b", ConfigEntry("b", {}, False, False, None, None)),
("c", PathEntry("c", False, False, None, None)),
("c.cA", ConfigEntry("cA", 3, False, False, None, None)),
("c.cB", ConfigEntry("cB", 4, False, False, None, None)),
("c.cC", PathEntry("cC", False, False, None, None)),
("c.cC.cC1", ConfigEntry("cC1", 6, False, False, None, None)),
("d", PathEntry("d", False, True, None, None)),
("d.dA", ConfigEntry("dA", 8, False, False, (float, int), None)),
]
def run():
config = {"a": 17, "foo": {"bar": True, "baz": False}, "seed": 1234}
config_mod = ConfigSummary()
signature = mock.Mock()
signature.name = "main_func"
def side_effect(*args):
# TODO : Type checking ? Does mock have a function for this ?
for arg in args:
arg = arg + 10
return args
main_func = mock.Mock(return_value=123,
prefix="",
signature=signature,
side_effect=side_effect)
logger = mock.Mock()
observer = [mock.Mock(priority=10)]
return Run(config, config_mod, main_func, observer, logger, logger, {}, {},
[], [])
def get_config_modifications(scaffolding):
config_modifications = ConfigSummary()
for sc_path, scaffold in scaffolding.items():
config_modifications.update_from(scaffold.config_mods, path=sc_path)
return config_modifications
class Scaffold(object):
def __init__(self, config_scopes, subrunners, path, captured_functions,
commands, named_configs, generate_seed):
self.config_scopes = config_scopes
self.named_configs = named_configs
self.subrunners = subrunners
self.path = path
self.generate_seed = generate_seed
self.config_updates = {}
self.named_configs_to_use = []
self.config = None
self.fixture = None # TODO: rename
self.logger = None
self.seed = None
self.rnd = None
self._captured_functions = captured_functions
self.commands = commands
self.config_mods = None
self.summaries = []
def set_up_seed(self, rnd=None):
if self.seed is not None:
return
self.seed = self.config.get('seed') or get_seed(rnd)
self.rnd = create_rnd(self.seed)
if self.generate_seed:
self.config['seed'] = self.seed
if 'seed' in self.config and 'seed' in self.config_mods.added:
self.config_mods.modified.add('seed')
self.config_mods.added -= {'seed'}
# Hierarchically set the seed of proper subrunners
for subrunner_path, subrunner in reversed(list(
self.subrunners.items())):
if is_prefix(self.path, subrunner_path):
subrunner.set_up_seed(self.rnd)
def set_up_config(self):
if self.config is not None:
return self.config
# gather presets
fallback = {}
for sr_path, subrunner in self.subrunners.items():
if self.path and is_prefix(self.path, sr_path):
path = sr_path[len(self.path):].strip('.')
set_by_dotted_path(fallback, path, subrunner.config)
else:
set_by_dotted_path(fallback, sr_path, subrunner.config)
# dogmatize to make the subrunner configurations read-only
const_fallback = dogmatize(fallback)
const_fallback.revelation()
self.config = {}
# named configs first
cfg_list = []
for ncfg in self.named_configs_to_use:
if os.path.exists(ncfg):
cfg_list.append(ConfigDict(load_config_file(ncfg)))
else:
cfg_list.append(self.named_configs[ncfg])
self.config_updates, _ = chain_evaluate_config_scopes(
cfg_list,
fixed=self.config_updates,
preset=self.config,
fallback=const_fallback)
# unnamed (default) configs second
self.config, self.summaries = chain_evaluate_config_scopes(
self.config_scopes,
fixed=self.config_updates,
preset=self.config,
fallback=const_fallback)
self.get_config_modifications()
def get_config_modifications(self):
self.config_mods = ConfigSummary()
for cfg_summary in self.summaries:
self.config_mods.update_from(cfg_summary)
def get_fixture(self):
if self.fixture is not None:
return self.fixture
self.fixture = copy(self.config)
for sr_path, subrunner in self.subrunners.items():
sub_fix = subrunner.get_fixture()
sub_path = sr_path
if is_prefix(self.path, sub_path):
sub_path = sr_path[len(self.path):].strip('.')
# Note: This might fail if we allow non-dict fixtures
set_by_dotted_path(self.fixture, sub_path, sub_fix)
return self.fixture
def finalize_initialization(self, run):
# look at seed again, because it might have changed during the
# configuration process
if 'seed' in self.config:
self.seed = self.config['seed']
self.rnd = create_rnd(self.seed)
for cfunc in self._captured_functions:
cfunc.logger = self.logger.getChild(cfunc.__name__)
cfunc.config = get_by_dotted_path(self.get_fixture(), cfunc.prefix)
seed = get_seed(self.rnd)
cfunc.rnd = create_rnd(seed)
cfunc.run = run
self._warn_about_suspicious_changes()
def _warn_about_suspicious_changes(self):
for add in sorted(self.config_mods.added):
self.logger.warning('Added new config entry: "%s"' % add)
for key, (type_old, type_new) in self.config_mods.typechanged.items():
if (isinstance(type_old, type(None)) or
(type_old in (int, float) and type_new in (int, float))):
continue
self.logger.warning(
'Changed type of config entry "%s" from %s to %s' %
(key, type_old.__name__, type_new.__name__))
for cfg_summary in self.summaries:
for key in cfg_summary.ignored_fallbacks:
self.logger.warning(
'Ignored attempt to set value of "%s", because it is an '
'ingredient.' % key
)
def get_config_modifications(self):
self.config_mods = ConfigSummary()
for cfg_summary in self.summaries:
self.config_mods.update_from(cfg_summary)
class Scaffold(object):
def __init__(self, config_scopes, subrunners, path, captured_functions,
commands, named_configs, config_hooks, generate_seed):
self.config_scopes = config_scopes
self.named_configs = named_configs
self.subrunners = subrunners
self.path = path
self.generate_seed = generate_seed
self.config_hooks = config_hooks
self.config_updates = {}
self.named_configs_to_use = []
self.config = {}
self.fallback = None
self.presets = {}
self.fixture = None # TODO: rename
self.logger = None
self.seed = None
self.rnd = None
self._captured_functions = captured_functions
self.commands = commands
self.config_mods = None
self.summaries = []
self.captured_args = {
join_paths(cf.prefix, n)
for cf in self._captured_functions for n in cf.signature.arguments
}
self.captured_args.add('__doc__') # allow setting the config docstring
def set_up_seed(self, rnd=None):
if self.seed is not None:
return
self.seed = self.config.get('seed')
if self.seed is None:
self.seed = get_seed(rnd)
self.rnd = create_rnd(self.seed)
if self.generate_seed:
self.config['seed'] = self.seed
if 'seed' in self.config and 'seed' in self.config_mods.added:
self.config_mods.modified.add('seed')
self.config_mods.added -= {'seed'}
# Hierarchically set the seed of proper subrunners
for subrunner_path, subrunner in reversed(list(
self.subrunners.items())):
if is_prefix(self.path, subrunner_path):
subrunner.set_up_seed(self.rnd)
def gather_fallbacks(self):
fallback = {'_log': self.logger}
for sr_path, subrunner in self.subrunners.items():
if self.path and is_prefix(self.path, sr_path):
path = sr_path[len(self.path):].strip('.')
set_by_dotted_path(fallback, path, subrunner.config)
else:
set_by_dotted_path(fallback, sr_path, subrunner.config)
# dogmatize to make the subrunner configurations read-only
self.fallback = dogmatize(fallback)
self.fallback.revelation()
def run_named_config(self, config_name):
if os.path.isfile(config_name):
nc = ConfigDict(load_config_file(config_name))
else:
if config_name not in self.named_configs:
raise NamedConfigNotFoundError(named_config=config_name,
available_named_configs=tuple(
self.named_configs.keys()))
nc = self.named_configs[config_name]
cfg = nc(fixed=self.get_config_updates_recursive(),
preset=self.presets,
fallback=self.fallback)
return undogmatize(cfg)
def set_up_config(self):
self.config, self.summaries = chain_evaluate_config_scopes(
self.config_scopes,
fixed=self.config_updates,
preset=self.config,
fallback=self.fallback)
self.get_config_modifications()
def run_config_hooks(self, config, command_name, logger):
final_cfg_updates = {}
for ch in self.config_hooks:
cfg_upup = ch(deepcopy(config), command_name, logger)
if cfg_upup:
recursive_update(final_cfg_updates, cfg_upup)
recursive_update(final_cfg_updates, self.config_updates)
return final_cfg_updates
def get_config_modifications(self):
self.config_mods = ConfigSummary(added={
key
for key, value in iterate_flattened(self.config_updates)
})
for cfg_summary in self.summaries:
self.config_mods.update_from(cfg_summary)
def get_config_updates_recursive(self):
config_updates = self.config_updates.copy()
for sr_path, subrunner in self.subrunners.items():
if not is_prefix(self.path, sr_path):
continue
update = subrunner.get_config_updates_recursive()
if update:
config_updates[rel_path(self.path, sr_path)] = update
return config_updates
def get_fixture(self):
if self.fixture is not None:
return self.fixture
def get_fixture_recursive(runner):
for sr_path, subrunner in runner.subrunners.items():
# I am not sure if it is necessary to trigger all
subrunner.get_fixture()
get_fixture_recursive(subrunner)
sub_fix = copy(subrunner.config)
sub_path = sr_path
if is_prefix(self.path, sub_path):
sub_path = sr_path[len(self.path):].strip('.')
# Note: This might fail if we allow non-dict fixtures
set_by_dotted_path(self.fixture, sub_path, sub_fix)
self.fixture = copy(self.config)
get_fixture_recursive(self)
return self.fixture
def finalize_initialization(self, run):
# look at seed again, because it might have changed during the
# configuration process
if 'seed' in self.config:
self.seed = self.config['seed']
self.rnd = create_rnd(self.seed)
for cfunc in self._captured_functions:
# Setup the captured function
cfunc.logger = self.logger.getChild(cfunc.__name__)
seed = get_seed(self.rnd)
cfunc.rnd = create_rnd(seed)
cfunc.run = run
cfunc.config = get_by_dotted_path(self.get_fixture(),
cfunc.prefix,
default={})
# Make configuration read only if enabled in settings
if SETTINGS.CONFIG.READ_ONLY_CONFIG:
cfunc.config = make_read_only(
cfunc.config,
error_message='The configuration is read-only in a '
'captured function!')
if not run.force:
self._warn_about_suspicious_changes()
def _warn_about_suspicious_changes(self):
for add in sorted(self.config_mods.added):
if not set(iter_prefixes(add)).intersection(self.captured_args):
if self.path:
add = join_paths(self.path, add)
raise ConfigAddedError(add, config=self.config)
else:
self.logger.warning('Added new config entry: "%s"' % add)
for key, (type_old, type_new) in self.config_mods.typechanged.items():
if type_old in (int, float) and type_new in (int, float):
continue
self.logger.warning(
'Changed type of config entry "%s" from %s to %s' %
(key, type_old.__name__, type_new.__name__))
for cfg_summary in self.summaries:
for key in cfg_summary.ignored_fallbacks:
self.logger.warning(
'Ignored attempt to set value of "%s", because it is an '
'ingredient.' % key)
def __repr__(self):
return "<Scaffold: '{}'>".format(self.path)
class Scaffold(object):
def __init__(self, config_scopes, subrunners, path, captured_functions,
commands, named_configs, config_hooks, generate_seed):
self.config_scopes = config_scopes
self.named_configs = named_configs
self.subrunners = subrunners
self.path = path
self.generate_seed = generate_seed
self.config_hooks = config_hooks
self.config_updates = {}
self.named_configs_to_use = []
self.config = None
self.fallback = None
self.fixture = None # TODO: rename
self.logger = None
self.seed = None
self.rnd = None
self._captured_functions = captured_functions
self.commands = commands
self.config_mods = None
self.summaries = []
self.captured_args = {join_paths(cf.prefix, n)
for cf in self._captured_functions
for n in cf.signature.arguments}
def set_up_seed(self, rnd=None):
if self.seed is not None:
return
self.seed = self.config.get('seed') or get_seed(rnd)
self.rnd = create_rnd(self.seed)
if self.generate_seed:
self.config['seed'] = self.seed
if 'seed' in self.config and 'seed' in self.config_mods.added:
self.config_mods.modified.add('seed')
self.config_mods.added -= {'seed'}
# Hierarchically set the seed of proper subrunners
for subrunner_path, subrunner in reversed(list(
self.subrunners.items())):
if is_prefix(self.path, subrunner_path):
subrunner.set_up_seed(self.rnd)
def pick_relevant_config_updates(self, config_updates, past_paths):
if config_updates is None:
return
for path, value in iterate_flattened(config_updates):
for prefix, suffix in reversed(list(iter_path_splits(path))):
if prefix in past_paths:
# don't use config_updates for prior ingredients
break
elif prefix == self.path:
set_by_dotted_path(self.config_updates, suffix, value)
break
def gather_fallbacks(self):
fallback = {}
for sr_path, subrunner in self.subrunners.items():
if self.path and is_prefix(self.path, sr_path):
path = sr_path[len(self.path):].strip('.')
set_by_dotted_path(fallback, path, subrunner.config)
else:
set_by_dotted_path(fallback, sr_path, subrunner.config)
# dogmatize to make the subrunner configurations read-only
self.fallback = dogmatize(fallback)
self.fallback.revelation()
def use_named_config(self, config_name):
if os.path.exists(config_name):
self.named_configs_to_use.append(
ConfigDict(load_config_file(config_name)))
else:
self.named_configs_to_use.append(self.named_configs[config_name])
def set_up_config(self):
# named configs go first
self.config_updates, _ = chain_evaluate_config_scopes(
self.named_configs_to_use,
fixed=self.config_updates,
preset={},
fallback=self.fallback)
# unnamed (default) configs second
self.config, self.summaries = chain_evaluate_config_scopes(
self.config_scopes,
fixed=self.config_updates,
preset=self.config,
fallback=self.fallback)
self.get_config_modifications()
def run_config_hooks(self, config, config_updates, command_name, logger):
final_cfg_updates = {}
for ch in self.config_hooks:
cfg_upup = ch(deepcopy(config), command_name, logger)
if cfg_upup:
recursive_update(final_cfg_updates, cfg_upup)
recursive_update(final_cfg_updates, config_updates)
return final_cfg_updates
def get_config_modifications(self):
self.config_mods = ConfigSummary(
added={key
for key, value in iterate_flattened(self.config_updates)})
for cfg_summary in self.summaries:
self.config_mods.update_from(cfg_summary)
def get_fixture(self):
if self.fixture is not None:
return self.fixture
self.fixture = copy(self.config)
for sr_path, subrunner in self.subrunners.items():
sub_fix = subrunner.get_fixture()
sub_path = sr_path
if is_prefix(self.path, sub_path):
sub_path = sr_path[len(self.path):].strip('.')
# Note: This might fail if we allow non-dict fixtures
set_by_dotted_path(self.fixture, sub_path, sub_fix)
return self.fixture
def finalize_initialization(self, run):
# look at seed again, because it might have changed during the
# configuration process
if 'seed' in self.config:
self.seed = self.config['seed']
self.rnd = create_rnd(self.seed)
for cfunc in self._captured_functions:
cfunc.logger = self.logger.getChild(cfunc.__name__)
cfunc.config = get_by_dotted_path(self.get_fixture(), cfunc.prefix)
seed = get_seed(self.rnd)
cfunc.rnd = create_rnd(seed)
cfunc.run = run
if not run.force:
self._warn_about_suspicious_changes()
def _warn_about_suspicious_changes(self):
for add in sorted(self.config_mods.added):
if not set(iter_prefixes(add)).intersection(self.captured_args):
raise KeyError('Added a new config entry "{}" that is not used'
' anywhere'.format(add))
else:
self.logger.warning('Added new config entry: "%s"' % add)
for key, (type_old, type_new) in self.config_mods.typechanged.items():
if (isinstance(type_old, type(None)) or
(type_old in (int, float) and type_new in (int, float))):
continue
self.logger.warning(
'Changed type of config entry "%s" from %s to %s' %
(key, type_old.__name__, type_new.__name__))
for cfg_summary in self.summaries:
for key in cfg_summary.ignored_fallbacks:
self.logger.warning(
'Ignored attempt to set value of "%s", because it is an '
'ingredient.' % key
)
def get_config_modifications(self):
self.config_mods = ConfigSummary(
added={key
for key, value in iterate_flattened(self.config_updates)})
for cfg_summary in self.summaries:
self.config_mods.update_from(cfg_summary)
class Scaffold(object):
def __init__(self, config_scopes, subrunners, path, captured_functions,
commands, named_configs, config_hooks, generate_seed):
self.config_scopes = config_scopes
self.named_configs = named_configs
self.subrunners = subrunners
self.path = path
self.generate_seed = generate_seed
self.config_hooks = config_hooks
self.config_updates = {}
self.named_configs_to_use = []
self.config = None
self.fallback = None
self.fixture = None # TODO: rename
self.logger = None
self.seed = None
self.rnd = None
self._captured_functions = captured_functions
self.commands = commands
self.config_mods = None
self.summaries = []
def set_up_seed(self, rnd=None):
if self.seed is not None:
return
self.seed = self.config.get('seed') or get_seed(rnd)
self.rnd = create_rnd(self.seed)
if self.generate_seed:
self.config['seed'] = self.seed
if 'seed' in self.config and 'seed' in self.config_mods.added:
self.config_mods.modified.add('seed')
self.config_mods.added -= {'seed'}
# Hierarchically set the seed of proper subrunners
for subrunner_path, subrunner in reversed(list(
self.subrunners.items())):
if is_prefix(self.path, subrunner_path):
subrunner.set_up_seed(self.rnd)
def pick_relevant_config_updates(self, config_updates, past_paths):
if config_updates is None:
return
for path, value in iterate_flattened(config_updates):
for prefix, suffix in reversed(list(iter_path_splits(path))):
if prefix in past_paths:
# don't use config_updates for prior ingredients
break
elif prefix == self.path:
set_by_dotted_path(self.config_updates, suffix, value)
break
def gather_fallbacks(self):
fallback = {}
for sr_path, subrunner in self.subrunners.items():
if self.path and is_prefix(self.path, sr_path):
path = sr_path[len(self.path):].strip('.')
set_by_dotted_path(fallback, path, subrunner.config)
else:
set_by_dotted_path(fallback, sr_path, subrunner.config)
# dogmatize to make the subrunner configurations read-only
self.fallback = dogmatize(fallback)
self.fallback.revelation()
def use_named_config(self, config_name):
if os.path.exists(config_name):
self.named_configs_to_use.append(
ConfigDict(load_config_file(config_name)))
else:
self.named_configs_to_use.append(self.named_configs[config_name])
def set_up_config(self):
# named configs go first
self.config_updates, _ = chain_evaluate_config_scopes(
self.named_configs_to_use,
fixed=self.config_updates,
preset={},
fallback=self.fallback)
# unnamed (default) configs second
self.config, self.summaries = chain_evaluate_config_scopes(
self.config_scopes,
fixed=self.config_updates,
preset=self.config,
fallback=self.fallback)
self.get_config_modifications()
def run_config_hooks(self, config, config_updates, command_name, logger):
final_cfg_updates = {}
for ch in self.config_hooks:
cfg_upup = ch(deepcopy(config), command_name, logger)
if cfg_upup:
recursive_update(final_cfg_updates, cfg_upup)
recursive_update(final_cfg_updates, config_updates)
return final_cfg_updates
def get_config_modifications(self):
self.config_mods = ConfigSummary(added={
key
for key, value in iterate_flattened(self.config_updates)
})
for cfg_summary in self.summaries:
self.config_mods.update_from(cfg_summary)
def get_fixture(self):
if self.fixture is not None:
return self.fixture
self.fixture = copy(self.config)
for sr_path, subrunner in self.subrunners.items():
sub_fix = subrunner.get_fixture()
sub_path = sr_path
if is_prefix(self.path, sub_path):
sub_path = sr_path[len(self.path):].strip('.')
# Note: This might fail if we allow non-dict fixtures
set_by_dotted_path(self.fixture, sub_path, sub_fix)
return self.fixture
def finalize_initialization(self, run):
# look at seed again, because it might have changed during the
# configuration process
if 'seed' in self.config:
self.seed = self.config['seed']
self.rnd = create_rnd(self.seed)
for cfunc in self._captured_functions:
cfunc.logger = self.logger.getChild(cfunc.__name__)
cfunc.config = get_by_dotted_path(self.get_fixture(), cfunc.prefix)
seed = get_seed(self.rnd)
cfunc.rnd = create_rnd(seed)
cfunc.run = run
self._warn_about_suspicious_changes()
def _warn_about_suspicious_changes(self):
for add in sorted(self.config_mods.added):
self.logger.warning('Added new config entry: "%s"' % add)
for key, (type_old, type_new) in self.config_mods.typechanged.items():
if (isinstance(type_old, type(None)) or
(type_old in (int, float) and type_new in (int, float))):
continue
self.logger.warning(
'Changed type of config entry "%s" from %s to %s' %
(key, type_old.__name__, type_new.__name__))
for cfg_summary in self.summaries:
for key in cfg_summary.ignored_fallbacks:
self.logger.warning(
'Ignored attempt to set value of "%s", because it is an '
'ingredient.' % key)
def get_config_modifications(scaffolding, config_updates):
config_modifications = ConfigSummary()
for sc_path, scaffold in scaffolding.items():
config_modifications.update_add(scaffold.config_mods, path=sc_path)
return config_modifications
class Scaffold(object):
def __init__(self, config_scopes, subrunners, path, captured_functions,
commands, named_configs, config_hooks, generate_seed):
self.config_scopes = config_scopes
self.named_configs = named_configs
self.subrunners = subrunners
self.path = path
self.generate_seed = generate_seed
self.config_hooks = config_hooks
self.config_updates = {}
self.named_configs_to_use = []
self.config = {}
self.fallback = None
self.presets = {}
self.fixture = None # TODO: rename
self.logger = None
self.seed = None
self.rnd = None
self._captured_functions = captured_functions
self.commands = commands
self.config_mods = None
self.summaries = []
self.captured_args = {join_paths(cf.prefix, n)
for cf in self._captured_functions
for n in cf.signature.arguments}
self.captured_args.add('__doc__') # allow setting the config docstring
def set_up_seed(self, rnd=None):
if self.seed is not None:
return
self.seed = self.config.get('seed')
if self.seed is None:
self.seed = get_seed(rnd)
self.rnd = create_rnd(self.seed)
if self.generate_seed:
self.config['seed'] = self.seed
if 'seed' in self.config and 'seed' in self.config_mods.added:
self.config_mods.modified.add('seed')
self.config_mods.added -= {'seed'}
# Hierarchically set the seed of proper subrunners
for subrunner_path, subrunner in reversed(list(
self.subrunners.items())):
if is_prefix(self.path, subrunner_path):
subrunner.set_up_seed(self.rnd)
def gather_fallbacks(self):
fallback = {'_log': self.logger}
for sr_path, subrunner in self.subrunners.items():
if self.path and is_prefix(self.path, sr_path):
path = sr_path[len(self.path):].strip('.')
set_by_dotted_path(fallback, path, subrunner.config)
else:
set_by_dotted_path(fallback, sr_path, subrunner.config)
# dogmatize to make the subrunner configurations read-only
self.fallback = dogmatize(fallback)
self.fallback.revelation()
def run_named_config(self, config_name):
if os.path.isfile(config_name):
nc = ConfigDict(load_config_file(config_name))
else:
if config_name not in self.named_configs:
raise NamedConfigNotFoundError(
named_config=config_name,
available_named_configs=tuple(self.named_configs.keys()))
nc = self.named_configs[config_name]
cfg = nc(fixed=self.get_config_updates_recursive(),
preset=self.presets,
fallback=self.fallback)
return undogmatize(cfg)
def set_up_config(self):
self.config, self.summaries = chain_evaluate_config_scopes(
self.config_scopes,
fixed=self.config_updates,
preset=self.config,
fallback=self.fallback)
self.get_config_modifications()
def run_config_hooks(self, config, command_name, logger):
final_cfg_updates = {}
for ch in self.config_hooks:
cfg_upup = ch(deepcopy(config), command_name, logger)
if cfg_upup:
recursive_update(final_cfg_updates, cfg_upup)
recursive_update(final_cfg_updates, self.config_updates)
return final_cfg_updates
def get_config_modifications(self):
self.config_mods = ConfigSummary(
added={key
for key, value in iterate_flattened(self.config_updates)})
for cfg_summary in self.summaries:
self.config_mods.update_from(cfg_summary)
def get_config_updates_recursive(self):
config_updates = self.config_updates.copy()
for sr_path, subrunner in self.subrunners.items():
if not is_prefix(self.path, sr_path):
continue
update = subrunner.get_config_updates_recursive()
if update:
config_updates[rel_path(self.path, sr_path)] = update
return config_updates
def get_fixture(self):
if self.fixture is not None:
return self.fixture
def get_fixture_recursive(runner):
for sr_path, subrunner in runner.subrunners.items():
# I am not sure if it is necessary to trigger all
subrunner.get_fixture()
get_fixture_recursive(subrunner)
sub_fix = copy(subrunner.config)
sub_path = sr_path
if is_prefix(self.path, sub_path):
sub_path = sr_path[len(self.path):].strip('.')
# Note: This might fail if we allow non-dict fixtures
set_by_dotted_path(self.fixture, sub_path, sub_fix)
self.fixture = copy(self.config)
get_fixture_recursive(self)
return self.fixture
def finalize_initialization(self, run):
# look at seed again, because it might have changed during the
# configuration process
if 'seed' in self.config:
self.seed = self.config['seed']
self.rnd = create_rnd(self.seed)
for cfunc in self._captured_functions:
cfunc.logger = self.logger.getChild(cfunc.__name__)
cfunc.config = get_by_dotted_path(self.get_fixture(), cfunc.prefix,
default={})
seed = get_seed(self.rnd)
cfunc.rnd = create_rnd(seed)
cfunc.run = run
if not run.force:
self._warn_about_suspicious_changes()
def _warn_about_suspicious_changes(self):
for add in sorted(self.config_mods.added):
if not set(iter_prefixes(add)).intersection(self.captured_args):
if self.path:
add = join_paths(self.path, add)
raise ConfigAddedError(add, config=self.config)
else:
self.logger.warning('Added new config entry: "%s"' % add)
for key, (type_old, type_new) in self.config_mods.typechanged.items():
if type_old in (int, float) and type_new in (int, float):
continue
self.logger.warning(
'Changed type of config entry "%s" from %s to %s' %
(key, type_old.__name__, type_new.__name__))
for cfg_summary in self.summaries:
for key in cfg_summary.ignored_fallbacks:
self.logger.warning(
'Ignored attempt to set value of "%s", because it is an '
'ingredient.' % key
)
def __repr__(self):
return "<Scaffold: '{}'>".format(self.path)