class Scaffold(object): def __init__(self, config_scopes, subrunners, path, captured_functions, commands, named_configs, config_hooks, generate_seed): self.config_scopes = config_scopes self.named_configs = named_configs self.subrunners = subrunners self.path = path self.generate_seed = generate_seed self.config_hooks = config_hooks self.config_updates = {} self.named_configs_to_use = [] self.config = None self.fallback = None self.fixture = None # TODO: rename self.logger = None self.seed = None self.rnd = None self._captured_functions = captured_functions self.commands = commands self.config_mods = None self.summaries = [] def set_up_seed(self, rnd=None): if self.seed is not None: return self.seed = self.config.get('seed') or get_seed(rnd) self.rnd = create_rnd(self.seed) if self.generate_seed: self.config['seed'] = self.seed if 'seed' in self.config and 'seed' in self.config_mods.added: self.config_mods.modified.add('seed') self.config_mods.added -= {'seed'} # Hierarchically set the seed of proper subrunners for subrunner_path, subrunner in reversed(list( self.subrunners.items())): if is_prefix(self.path, subrunner_path): subrunner.set_up_seed(self.rnd) def pick_relevant_config_updates(self, config_updates, past_paths): if config_updates is None: return for path, value in iterate_flattened(config_updates): for prefix, suffix in reversed(list(iter_path_splits(path))): if prefix in past_paths: # don't use config_updates for prior ingredients break elif prefix == self.path: set_by_dotted_path(self.config_updates, suffix, value) break def gather_fallbacks(self): fallback = {} for sr_path, subrunner in self.subrunners.items(): if self.path and is_prefix(self.path, sr_path): path = sr_path[len(self.path):].strip('.') set_by_dotted_path(fallback, path, subrunner.config) else: set_by_dotted_path(fallback, sr_path, subrunner.config) # dogmatize to make the subrunner configurations read-only self.fallback = dogmatize(fallback) self.fallback.revelation() def use_named_config(self, config_name): if os.path.exists(config_name): self.named_configs_to_use.append( ConfigDict(load_config_file(config_name))) else: self.named_configs_to_use.append(self.named_configs[config_name]) def set_up_config(self): # named configs go first self.config_updates, _ = chain_evaluate_config_scopes( self.named_configs_to_use, fixed=self.config_updates, preset={}, fallback=self.fallback) # unnamed (default) configs second self.config, self.summaries = chain_evaluate_config_scopes( self.config_scopes, fixed=self.config_updates, preset=self.config, fallback=self.fallback) self.get_config_modifications() def run_config_hooks(self, config, config_updates, command_name, logger): final_cfg_updates = {} for ch in self.config_hooks: cfg_upup = ch(deepcopy(config), command_name, logger) if cfg_upup: recursive_update(final_cfg_updates, cfg_upup) recursive_update(final_cfg_updates, config_updates) return final_cfg_updates def get_config_modifications(self): self.config_mods = ConfigSummary(added={ key for key, value in iterate_flattened(self.config_updates) }) for cfg_summary in self.summaries: self.config_mods.update_from(cfg_summary) def get_fixture(self): if self.fixture is not None: return self.fixture self.fixture = copy(self.config) for sr_path, subrunner in self.subrunners.items(): sub_fix = subrunner.get_fixture() sub_path = sr_path if is_prefix(self.path, sub_path): sub_path = sr_path[len(self.path):].strip('.') # Note: This might fail if we allow non-dict fixtures set_by_dotted_path(self.fixture, sub_path, sub_fix) return self.fixture def finalize_initialization(self, run): # look at seed again, because it might have changed during the # configuration process if 'seed' in self.config: self.seed = self.config['seed'] self.rnd = create_rnd(self.seed) for cfunc in self._captured_functions: cfunc.logger = self.logger.getChild(cfunc.__name__) cfunc.config = get_by_dotted_path(self.get_fixture(), cfunc.prefix) seed = get_seed(self.rnd) cfunc.rnd = create_rnd(seed) cfunc.run = run self._warn_about_suspicious_changes() def _warn_about_suspicious_changes(self): for add in sorted(self.config_mods.added): self.logger.warning('Added new config entry: "%s"' % add) for key, (type_old, type_new) in self.config_mods.typechanged.items(): if (isinstance(type_old, type(None)) or (type_old in (int, float) and type_new in (int, float))): continue self.logger.warning( 'Changed type of config entry "%s" from %s to %s' % (key, type_old.__name__, type_new.__name__)) for cfg_summary in self.summaries: for key in cfg_summary.ignored_fallbacks: self.logger.warning( 'Ignored attempt to set value of "%s", because it is an ' 'ingredient.' % key)
def get_config_modifications(scaffolding): config_modifications = ConfigSummary() for sc_path, scaffold in scaffolding.items(): config_modifications.update_from(scaffold.config_mods, path=sc_path) return config_modifications
class Scaffold(object): def __init__(self, config_scopes, subrunners, path, captured_functions, commands, named_configs, generate_seed): self.config_scopes = config_scopes self.named_configs = named_configs self.subrunners = subrunners self.path = path self.generate_seed = generate_seed self.config_updates = {} self.named_configs_to_use = [] self.config = None self.fixture = None # TODO: rename self.logger = None self.seed = None self.rnd = None self._captured_functions = captured_functions self.commands = commands self.config_mods = None self.summaries = [] def set_up_seed(self, rnd=None): if self.seed is not None: return self.seed = self.config.get('seed') or get_seed(rnd) self.rnd = create_rnd(self.seed) if self.generate_seed: self.config['seed'] = self.seed if 'seed' in self.config and 'seed' in self.config_mods.added: self.config_mods.modified.add('seed') self.config_mods.added -= {'seed'} # Hierarchically set the seed of proper subrunners for subrunner_path, subrunner in reversed(list( self.subrunners.items())): if is_prefix(self.path, subrunner_path): subrunner.set_up_seed(self.rnd) def set_up_config(self): if self.config is not None: return self.config # gather presets fallback = {} for sr_path, subrunner in self.subrunners.items(): if self.path and is_prefix(self.path, sr_path): path = sr_path[len(self.path):].strip('.') set_by_dotted_path(fallback, path, subrunner.config) else: set_by_dotted_path(fallback, sr_path, subrunner.config) # dogmatize to make the subrunner configurations read-only const_fallback = dogmatize(fallback) const_fallback.revelation() self.config = {} # named configs first cfg_list = [] for ncfg in self.named_configs_to_use: if os.path.exists(ncfg): cfg_list.append(ConfigDict(load_config_file(ncfg))) else: cfg_list.append(self.named_configs[ncfg]) self.config_updates, _ = chain_evaluate_config_scopes( cfg_list, fixed=self.config_updates, preset=self.config, fallback=const_fallback) # unnamed (default) configs second self.config, self.summaries = chain_evaluate_config_scopes( self.config_scopes, fixed=self.config_updates, preset=self.config, fallback=const_fallback) self.get_config_modifications() def get_config_modifications(self): self.config_mods = ConfigSummary() for cfg_summary in self.summaries: self.config_mods.update_from(cfg_summary) def get_fixture(self): if self.fixture is not None: return self.fixture self.fixture = copy(self.config) for sr_path, subrunner in self.subrunners.items(): sub_fix = subrunner.get_fixture() sub_path = sr_path if is_prefix(self.path, sub_path): sub_path = sr_path[len(self.path):].strip('.') # Note: This might fail if we allow non-dict fixtures set_by_dotted_path(self.fixture, sub_path, sub_fix) return self.fixture def finalize_initialization(self, run): # look at seed again, because it might have changed during the # configuration process if 'seed' in self.config: self.seed = self.config['seed'] self.rnd = create_rnd(self.seed) for cfunc in self._captured_functions: cfunc.logger = self.logger.getChild(cfunc.__name__) cfunc.config = get_by_dotted_path(self.get_fixture(), cfunc.prefix) seed = get_seed(self.rnd) cfunc.rnd = create_rnd(seed) cfunc.run = run self._warn_about_suspicious_changes() def _warn_about_suspicious_changes(self): for add in sorted(self.config_mods.added): self.logger.warning('Added new config entry: "%s"' % add) for key, (type_old, type_new) in self.config_mods.typechanged.items(): if (isinstance(type_old, type(None)) or (type_old in (int, float) and type_new in (int, float))): continue self.logger.warning( 'Changed type of config entry "%s" from %s to %s' % (key, type_old.__name__, type_new.__name__)) for cfg_summary in self.summaries: for key in cfg_summary.ignored_fallbacks: self.logger.warning( 'Ignored attempt to set value of "%s", because it is an ' 'ingredient.' % key )
class Scaffold(object): def __init__(self, config_scopes, subrunners, path, captured_functions, commands, named_configs, config_hooks, generate_seed): self.config_scopes = config_scopes self.named_configs = named_configs self.subrunners = subrunners self.path = path self.generate_seed = generate_seed self.config_hooks = config_hooks self.config_updates = {} self.named_configs_to_use = [] self.config = {} self.fallback = None self.presets = {} self.fixture = None # TODO: rename self.logger = None self.seed = None self.rnd = None self._captured_functions = captured_functions self.commands = commands self.config_mods = None self.summaries = [] self.captured_args = { join_paths(cf.prefix, n) for cf in self._captured_functions for n in cf.signature.arguments } self.captured_args.add('__doc__') # allow setting the config docstring def set_up_seed(self, rnd=None): if self.seed is not None: return self.seed = self.config.get('seed') if self.seed is None: self.seed = get_seed(rnd) self.rnd = create_rnd(self.seed) if self.generate_seed: self.config['seed'] = self.seed if 'seed' in self.config and 'seed' in self.config_mods.added: self.config_mods.modified.add('seed') self.config_mods.added -= {'seed'} # Hierarchically set the seed of proper subrunners for subrunner_path, subrunner in reversed(list( self.subrunners.items())): if is_prefix(self.path, subrunner_path): subrunner.set_up_seed(self.rnd) def gather_fallbacks(self): fallback = {'_log': self.logger} for sr_path, subrunner in self.subrunners.items(): if self.path and is_prefix(self.path, sr_path): path = sr_path[len(self.path):].strip('.') set_by_dotted_path(fallback, path, subrunner.config) else: set_by_dotted_path(fallback, sr_path, subrunner.config) # dogmatize to make the subrunner configurations read-only self.fallback = dogmatize(fallback) self.fallback.revelation() def run_named_config(self, config_name): if os.path.isfile(config_name): nc = ConfigDict(load_config_file(config_name)) else: if config_name not in self.named_configs: raise NamedConfigNotFoundError(named_config=config_name, available_named_configs=tuple( self.named_configs.keys())) nc = self.named_configs[config_name] cfg = nc(fixed=self.get_config_updates_recursive(), preset=self.presets, fallback=self.fallback) return undogmatize(cfg) def set_up_config(self): self.config, self.summaries = chain_evaluate_config_scopes( self.config_scopes, fixed=self.config_updates, preset=self.config, fallback=self.fallback) self.get_config_modifications() def run_config_hooks(self, config, command_name, logger): final_cfg_updates = {} for ch in self.config_hooks: cfg_upup = ch(deepcopy(config), command_name, logger) if cfg_upup: recursive_update(final_cfg_updates, cfg_upup) recursive_update(final_cfg_updates, self.config_updates) return final_cfg_updates def get_config_modifications(self): self.config_mods = ConfigSummary(added={ key for key, value in iterate_flattened(self.config_updates) }) for cfg_summary in self.summaries: self.config_mods.update_from(cfg_summary) def get_config_updates_recursive(self): config_updates = self.config_updates.copy() for sr_path, subrunner in self.subrunners.items(): if not is_prefix(self.path, sr_path): continue update = subrunner.get_config_updates_recursive() if update: config_updates[rel_path(self.path, sr_path)] = update return config_updates def get_fixture(self): if self.fixture is not None: return self.fixture def get_fixture_recursive(runner): for sr_path, subrunner in runner.subrunners.items(): # I am not sure if it is necessary to trigger all subrunner.get_fixture() get_fixture_recursive(subrunner) sub_fix = copy(subrunner.config) sub_path = sr_path if is_prefix(self.path, sub_path): sub_path = sr_path[len(self.path):].strip('.') # Note: This might fail if we allow non-dict fixtures set_by_dotted_path(self.fixture, sub_path, sub_fix) self.fixture = copy(self.config) get_fixture_recursive(self) return self.fixture def finalize_initialization(self, run): # look at seed again, because it might have changed during the # configuration process if 'seed' in self.config: self.seed = self.config['seed'] self.rnd = create_rnd(self.seed) for cfunc in self._captured_functions: # Setup the captured function cfunc.logger = self.logger.getChild(cfunc.__name__) seed = get_seed(self.rnd) cfunc.rnd = create_rnd(seed) cfunc.run = run cfunc.config = get_by_dotted_path(self.get_fixture(), cfunc.prefix, default={}) # Make configuration read only if enabled in settings if SETTINGS.CONFIG.READ_ONLY_CONFIG: cfunc.config = make_read_only( cfunc.config, error_message='The configuration is read-only in a ' 'captured function!') if not run.force: self._warn_about_suspicious_changes() def _warn_about_suspicious_changes(self): for add in sorted(self.config_mods.added): if not set(iter_prefixes(add)).intersection(self.captured_args): if self.path: add = join_paths(self.path, add) raise ConfigAddedError(add, config=self.config) else: self.logger.warning('Added new config entry: "%s"' % add) for key, (type_old, type_new) in self.config_mods.typechanged.items(): if type_old in (int, float) and type_new in (int, float): continue self.logger.warning( 'Changed type of config entry "%s" from %s to %s' % (key, type_old.__name__, type_new.__name__)) for cfg_summary in self.summaries: for key in cfg_summary.ignored_fallbacks: self.logger.warning( 'Ignored attempt to set value of "%s", because it is an ' 'ingredient.' % key) def __repr__(self): return "<Scaffold: '{}'>".format(self.path)
class Scaffold(object): def __init__(self, config_scopes, subrunners, path, captured_functions, commands, named_configs, config_hooks, generate_seed): self.config_scopes = config_scopes self.named_configs = named_configs self.subrunners = subrunners self.path = path self.generate_seed = generate_seed self.config_hooks = config_hooks self.config_updates = {} self.named_configs_to_use = [] self.config = None self.fallback = None self.fixture = None # TODO: rename self.logger = None self.seed = None self.rnd = None self._captured_functions = captured_functions self.commands = commands self.config_mods = None self.summaries = [] self.captured_args = {join_paths(cf.prefix, n) for cf in self._captured_functions for n in cf.signature.arguments} def set_up_seed(self, rnd=None): if self.seed is not None: return self.seed = self.config.get('seed') or get_seed(rnd) self.rnd = create_rnd(self.seed) if self.generate_seed: self.config['seed'] = self.seed if 'seed' in self.config and 'seed' in self.config_mods.added: self.config_mods.modified.add('seed') self.config_mods.added -= {'seed'} # Hierarchically set the seed of proper subrunners for subrunner_path, subrunner in reversed(list( self.subrunners.items())): if is_prefix(self.path, subrunner_path): subrunner.set_up_seed(self.rnd) def pick_relevant_config_updates(self, config_updates, past_paths): if config_updates is None: return for path, value in iterate_flattened(config_updates): for prefix, suffix in reversed(list(iter_path_splits(path))): if prefix in past_paths: # don't use config_updates for prior ingredients break elif prefix == self.path: set_by_dotted_path(self.config_updates, suffix, value) break def gather_fallbacks(self): fallback = {} for sr_path, subrunner in self.subrunners.items(): if self.path and is_prefix(self.path, sr_path): path = sr_path[len(self.path):].strip('.') set_by_dotted_path(fallback, path, subrunner.config) else: set_by_dotted_path(fallback, sr_path, subrunner.config) # dogmatize to make the subrunner configurations read-only self.fallback = dogmatize(fallback) self.fallback.revelation() def use_named_config(self, config_name): if os.path.exists(config_name): self.named_configs_to_use.append( ConfigDict(load_config_file(config_name))) else: self.named_configs_to_use.append(self.named_configs[config_name]) def set_up_config(self): # named configs go first self.config_updates, _ = chain_evaluate_config_scopes( self.named_configs_to_use, fixed=self.config_updates, preset={}, fallback=self.fallback) # unnamed (default) configs second self.config, self.summaries = chain_evaluate_config_scopes( self.config_scopes, fixed=self.config_updates, preset=self.config, fallback=self.fallback) self.get_config_modifications() def run_config_hooks(self, config, config_updates, command_name, logger): final_cfg_updates = {} for ch in self.config_hooks: cfg_upup = ch(deepcopy(config), command_name, logger) if cfg_upup: recursive_update(final_cfg_updates, cfg_upup) recursive_update(final_cfg_updates, config_updates) return final_cfg_updates def get_config_modifications(self): self.config_mods = ConfigSummary( added={key for key, value in iterate_flattened(self.config_updates)}) for cfg_summary in self.summaries: self.config_mods.update_from(cfg_summary) def get_fixture(self): if self.fixture is not None: return self.fixture self.fixture = copy(self.config) for sr_path, subrunner in self.subrunners.items(): sub_fix = subrunner.get_fixture() sub_path = sr_path if is_prefix(self.path, sub_path): sub_path = sr_path[len(self.path):].strip('.') # Note: This might fail if we allow non-dict fixtures set_by_dotted_path(self.fixture, sub_path, sub_fix) return self.fixture def finalize_initialization(self, run): # look at seed again, because it might have changed during the # configuration process if 'seed' in self.config: self.seed = self.config['seed'] self.rnd = create_rnd(self.seed) for cfunc in self._captured_functions: cfunc.logger = self.logger.getChild(cfunc.__name__) cfunc.config = get_by_dotted_path(self.get_fixture(), cfunc.prefix) seed = get_seed(self.rnd) cfunc.rnd = create_rnd(seed) cfunc.run = run if not run.force: self._warn_about_suspicious_changes() def _warn_about_suspicious_changes(self): for add in sorted(self.config_mods.added): if not set(iter_prefixes(add)).intersection(self.captured_args): raise KeyError('Added a new config entry "{}" that is not used' ' anywhere'.format(add)) else: self.logger.warning('Added new config entry: "%s"' % add) for key, (type_old, type_new) in self.config_mods.typechanged.items(): if (isinstance(type_old, type(None)) or (type_old in (int, float) and type_new in (int, float))): continue self.logger.warning( 'Changed type of config entry "%s" from %s to %s' % (key, type_old.__name__, type_new.__name__)) for cfg_summary in self.summaries: for key in cfg_summary.ignored_fallbacks: self.logger.warning( 'Ignored attempt to set value of "%s", because it is an ' 'ingredient.' % key )
class Scaffold(object): def __init__(self, config_scopes, subrunners, path, captured_functions, commands, named_configs, config_hooks, generate_seed): self.config_scopes = config_scopes self.named_configs = named_configs self.subrunners = subrunners self.path = path self.generate_seed = generate_seed self.config_hooks = config_hooks self.config_updates = {} self.named_configs_to_use = [] self.config = {} self.fallback = None self.presets = {} self.fixture = None # TODO: rename self.logger = None self.seed = None self.rnd = None self._captured_functions = captured_functions self.commands = commands self.config_mods = None self.summaries = [] self.captured_args = {join_paths(cf.prefix, n) for cf in self._captured_functions for n in cf.signature.arguments} self.captured_args.add('__doc__') # allow setting the config docstring def set_up_seed(self, rnd=None): if self.seed is not None: return self.seed = self.config.get('seed') if self.seed is None: self.seed = get_seed(rnd) self.rnd = create_rnd(self.seed) if self.generate_seed: self.config['seed'] = self.seed if 'seed' in self.config and 'seed' in self.config_mods.added: self.config_mods.modified.add('seed') self.config_mods.added -= {'seed'} # Hierarchically set the seed of proper subrunners for subrunner_path, subrunner in reversed(list( self.subrunners.items())): if is_prefix(self.path, subrunner_path): subrunner.set_up_seed(self.rnd) def gather_fallbacks(self): fallback = {'_log': self.logger} for sr_path, subrunner in self.subrunners.items(): if self.path and is_prefix(self.path, sr_path): path = sr_path[len(self.path):].strip('.') set_by_dotted_path(fallback, path, subrunner.config) else: set_by_dotted_path(fallback, sr_path, subrunner.config) # dogmatize to make the subrunner configurations read-only self.fallback = dogmatize(fallback) self.fallback.revelation() def run_named_config(self, config_name): if os.path.isfile(config_name): nc = ConfigDict(load_config_file(config_name)) else: if config_name not in self.named_configs: raise NamedConfigNotFoundError( named_config=config_name, available_named_configs=tuple(self.named_configs.keys())) nc = self.named_configs[config_name] cfg = nc(fixed=self.get_config_updates_recursive(), preset=self.presets, fallback=self.fallback) return undogmatize(cfg) def set_up_config(self): self.config, self.summaries = chain_evaluate_config_scopes( self.config_scopes, fixed=self.config_updates, preset=self.config, fallback=self.fallback) self.get_config_modifications() def run_config_hooks(self, config, command_name, logger): final_cfg_updates = {} for ch in self.config_hooks: cfg_upup = ch(deepcopy(config), command_name, logger) if cfg_upup: recursive_update(final_cfg_updates, cfg_upup) recursive_update(final_cfg_updates, self.config_updates) return final_cfg_updates def get_config_modifications(self): self.config_mods = ConfigSummary( added={key for key, value in iterate_flattened(self.config_updates)}) for cfg_summary in self.summaries: self.config_mods.update_from(cfg_summary) def get_config_updates_recursive(self): config_updates = self.config_updates.copy() for sr_path, subrunner in self.subrunners.items(): if not is_prefix(self.path, sr_path): continue update = subrunner.get_config_updates_recursive() if update: config_updates[rel_path(self.path, sr_path)] = update return config_updates def get_fixture(self): if self.fixture is not None: return self.fixture def get_fixture_recursive(runner): for sr_path, subrunner in runner.subrunners.items(): # I am not sure if it is necessary to trigger all subrunner.get_fixture() get_fixture_recursive(subrunner) sub_fix = copy(subrunner.config) sub_path = sr_path if is_prefix(self.path, sub_path): sub_path = sr_path[len(self.path):].strip('.') # Note: This might fail if we allow non-dict fixtures set_by_dotted_path(self.fixture, sub_path, sub_fix) self.fixture = copy(self.config) get_fixture_recursive(self) return self.fixture def finalize_initialization(self, run): # look at seed again, because it might have changed during the # configuration process if 'seed' in self.config: self.seed = self.config['seed'] self.rnd = create_rnd(self.seed) for cfunc in self._captured_functions: cfunc.logger = self.logger.getChild(cfunc.__name__) cfunc.config = get_by_dotted_path(self.get_fixture(), cfunc.prefix, default={}) seed = get_seed(self.rnd) cfunc.rnd = create_rnd(seed) cfunc.run = run if not run.force: self._warn_about_suspicious_changes() def _warn_about_suspicious_changes(self): for add in sorted(self.config_mods.added): if not set(iter_prefixes(add)).intersection(self.captured_args): if self.path: add = join_paths(self.path, add) raise ConfigAddedError(add, config=self.config) else: self.logger.warning('Added new config entry: "%s"' % add) for key, (type_old, type_new) in self.config_mods.typechanged.items(): if type_old in (int, float) and type_new in (int, float): continue self.logger.warning( 'Changed type of config entry "%s" from %s to %s' % (key, type_old.__name__, type_new.__name__)) for cfg_summary in self.summaries: for key in cfg_summary.ignored_fallbacks: self.logger.warning( 'Ignored attempt to set value of "%s", because it is an ' 'ingredient.' % key ) def __repr__(self): return "<Scaffold: '{}'>".format(self.path)