def load_yaml(filename, **options): loaded_yaml = AttrDict() # Find the requested yaml in the yaml dir if os.path.exists(filename): with open(filename) as config_fh: try: loaded_yaml.update(yaml.load(config_fh, Loader=OrderedDictYAMLLoader)) except: warn('Unable to parse configuration file at {}'.format(filename), ConfigInvalid) return loaded_yaml
def load_yaml(filename, **options): loaded_yaml = AttrDict() # Find the requested yaml in the yaml dir if os.path.exists(filename): with open(filename) as config_fh: try: loaded_yaml.update( yaml.load(config_fh, Loader=OrderedDictYAMLLoader)) except: warn( 'Unable to parse configuration file at {}'.format( filename), ConfigInvalid) return loaded_yaml
class Config(dict): """Configuration YAML loader and cache""" def __init__(self, config_dir, **kwargs): # private yaycl conf, plugins can stash info here for configuration self._yaycl = AttrDict({'config_dir': config_dir}) self._runtime = ConfigTree(self) self._extension = kwargs.pop('extension', '.yaml') self._yaycl.update(kwargs) def _runtime_overrides(self): return self._runtime def _set_runtime_overrides(self, overrides_dict): self._runtime.update(overrides_dict) def _del_runtime_overrides(self): self._runtime.clear() runtime = property(_runtime_overrides, _set_runtime_overrides, _del_runtime_overrides) def save(self, key): """Write out an in-memory config to the conf dir Warning: This will destroy any formatting or ordering that existed in the original yaml """ with open(os.path.join(self._yaycl.config_dir, '%s.yaml' % key), 'w') as conf_file: self[key].dump(conf_file) # Support for descriptor access, e.g. instance.attrname # Note that this is only on the get side, for support of nefarious things # like setting and deleting, use the normal dict interface. def __getattribute__(self, attr): # Attempt normal object attr lookup; delegate to the dict interface if that fails try: return super(Config, self).__getattribute__(attr) except AttributeError: # try to load from cache first if this conf is already known if attr in self: return self[attr] if attr.startswith('_'): # don't try to load private names raise # If we're here, trigger the loader via getitem return self[attr] def __getitem__(self, key): # Attempt a normal dict lookup to pull a cached conf if key not in self: super(Config, self).__setitem__(key, AttrDict()) # Cache miss, populate this conf key # Call out to dict's setitem here since this is the only place where we're allowed to # create a new key and we want the default behavior instead of the override below self._populate(key) return super(Config, self).__getitem__(key) def __setitem__(self, key, value): self[key].clear() self[key].update(value) def __delitem__(self, key): self[key].clear() self._populate(key) def _inherit(self, conf_key): """Recurses through an object looking for 'inherit' clauses and replaces them with their real counterparts. In the case of a dict, the inherit clause remains, in the case of anything else, a replacement occurs such that: sim5: key: value newkey: newvalue sim6: tags: - tag1 - tag2 sim7: inherit: management_systems/sim5 test: tags: inherit: management_systems/sim6/tags Will produce the following output if requesting management_systems/sim7 inherit: management_systems/sim5 key: value newkey: newvalue test: tags: - tag1 - tag2 """ for keys in self._needs_inherit(conf_key): # get the dict containing inherit key keys, root_key = keys[:-1], keys[-1] root = self[conf_key] for k in keys: root = root[k] # find the dict we're inheriting based on value base = self[conf_key] try: for path_element in root[root_key]['inherit'].split('/'): base = base[path_element] except KeyError: warn('{} path cannot be traversed, {} does not exist'.format( root[root_key]['inherit'], path_element), InvalidInheritPath) # rebase if the base was an attrdict, # otherwise overwrite the key in-place if isinstance(base, AttrDict): del(root[root_key]['inherit']) root[root_key].rebase(base) else: root[root_key] = base def _needs_inherit(self, conf_key): conf = self[conf_key] # loop over keys until all the inherits are gone while True: seen_inherit = False for k, v in conf.flatten_dict(conf): if k[-1] == 'inherit': # give back the keys needed to get to a dict containing an inherit key seen_inherit = True yield k[:-1] if not seen_inherit: break def _populate(self, key): yaml_dict = self._load_yaml(key) # Graft in local yaml updates if they're available with catch_warnings(): local_yaml = '%s.local' % key local_yaml_dict = self._load_yaml(local_yaml, warn_on_fail=False) if local_yaml_dict: yaml_dict.update_dict(local_yaml_dict) # Graft on the runtime overrides if key in self._runtime: AttrDict(self._runtime)[key]._.apply_flat( partial(self._apply_runtime_overrides, yaml_dict)) self[key].update(yaml_dict) self._inherit(key) def _apply_runtime_overrides(self, yaml_dict, keys, value): base = yaml_dict for key in keys[:-1]: if key not in base: base[key] = AttrDict() base = base[key] base[keys[-1]] = value def clear(self): # because of the 'from conf import foo' mechanism, we need to clear each key in-place, # and reload the runtime overrides. Once a key is created in this dict, its value MUST NOT # change to a different dict object. for key in self: # clear the conf dict in-place self[key].clear() self._populate(key) def _load_yaml(self, conf_key, warn_on_fail=True): return config_file(self.file_path(conf_key), warn_on_fail=warn_on_fail, **self._yaycl) def file_path(self, conf_key): file_name = '{}{}'.format(conf_key, self._extension) return os.path.join(self._yaycl.config_dir, file_name)
# Default blocking time before giving up on an ssh command execution, # in seconds (float) RUNCMD_TIMEOUT = 1200.0 SSHResult = namedtuple("SSHResult", ["rc", "output"]) _ssh_key_file = project_path.join('.generated_ssh_key') _ssh_pubkey_file = project_path.join('.generated_ssh_key.pub') # enum _ssh_keystate = AttrDict({ 'not_installed': 0, 'installing': 1, 'installed': 2 }) # enum reverse lookup _ssh_keystate.update({v: k for k, v in _ssh_keystate.items()}) _client_session = [] class SSHClient(paramiko.SSHClient): """paramiko.SSHClient wrapper Allows copying/overriding and use as a context manager Constructor kwargs are handed directly to paramiko.SSHClient.connect() """ def __init__(self, stream_output=False, keystate=_ssh_keystate.not_installed, **connect_kwargs): super(SSHClient, self).__init__() self._streaming = stream_output self._keystate = keystate
class Config(dict): """Configuration YAML loader and cache""" def __init__(self, config_dir, **kwargs): # private yaycl conf, plugins can stash info here for configuration self._yaycl = AttrDict({'config_dir': config_dir}) self._runtime = ConfigTree(self) self._extension = kwargs.pop('extension', '.yaml') self._yaycl.update(kwargs) def _runtime_overrides(self): return self._runtime def _set_runtime_overrides(self, overrides_dict): self._runtime.update(overrides_dict) def _del_runtime_overrides(self): self._runtime.clear() runtime = property(_runtime_overrides, _set_runtime_overrides, _del_runtime_overrides) def save(self, key): """Write out an in-memory config to the conf dir Warning: This will destroy any formatting or ordering that existed in the original yaml """ with open(os.path.join(self._yaycl.config_dir, '%s.yaml' % key), 'w') as conf_file: self[key].dump(conf_file) # Support for descriptor access, e.g. instance.attrname # Note that this is only on the get side, for support of nefarious things # like setting and deleting, use the normal dict interface. def __getattribute__(self, attr): # Attempt normal object attr lookup; delegate to the dict interface if that fails try: return super(Config, self).__getattribute__(attr) except AttributeError: # try to load from cache first if this conf is already known if attr in self: return self[attr] if attr.startswith('_'): # don't try to load private names raise # If we're here, trigger the loader via getitem return self[attr] def __getitem__(self, key): # Attempt a normal dict lookup to pull a cached conf if key not in self: super(Config, self).__setitem__(key, AttrDict()) # Cache miss, populate this conf key # Call out to dict's setitem here since this is the only place where we're allowed to # create a new key and we want the default behavior instead of the override below self._populate(key) return super(Config, self).__getitem__(key) def __setitem__(self, key, value): self[key].clear() self[key].update(value) def __delitem__(self, key): self[key].clear() self._populate(key) def _inherit(self, conf_key): """Recurses through an object looking for 'inherit' clauses and replaces them with their real counterparts. In the case of a dict, the inherit clause remains, in the case of anything else, a replacement occurs such that: sim5: key: value newkey: newvalue sim6: tags: - tag1 - tag2 sim7: inherit: management_systems/sim5 test: tags: inherit: management_systems/sim6/tags Will produce the following output if requesting management_systems/sim7 inherit: management_systems/sim5 key: value newkey: newvalue test: tags: - tag1 - tag2 """ for keys in self._needs_inherit(conf_key): # get the dict containing inherit key keys, root_key = keys[:-1], keys[-1] root = self[conf_key] for k in keys: root = root[k] # find the dict we're inheriting based on value base = self[conf_key] try: for path_element in root[root_key]['inherit'].split('/'): base = base[path_element] except KeyError: warn( '{} path cannot be traversed, {} does not exist'.format( root[root_key]['inherit'], path_element), InvalidInheritPath) # rebase if the base was an attrdict, # otherwise overwrite the key in-place if isinstance(base, AttrDict): del (root[root_key]['inherit']) root[root_key].rebase(base) else: root[root_key] = base def _needs_inherit(self, conf_key): conf = self[conf_key] # loop over keys until all the inherits are gone while True: seen_inherit = False for k, v in conf.flatten_dict(conf): if k[-1] == 'inherit': # give back the keys needed to get to a dict containing an inherit key seen_inherit = True yield k[:-1] if not seen_inherit: break def _populate(self, key): yaml_dict = self._load_yaml(key) # Graft in local yaml updates if they're available with catch_warnings(): local_yaml = '%s.local' % key local_yaml_dict = self._load_yaml(local_yaml, warn_on_fail=False) if local_yaml_dict: yaml_dict.update_dict(local_yaml_dict) # Graft on the runtime overrides if key in self._runtime: AttrDict(self._runtime)[key]._.apply_flat( partial(self._apply_runtime_overrides, yaml_dict)) self[key].update(yaml_dict) self._inherit(key) def _apply_runtime_overrides(self, yaml_dict, keys, value): base = yaml_dict for key in keys[:-1]: if key not in base: base[key] = AttrDict() base = base[key] base[keys[-1]] = value def clear(self): # because of the 'from conf import foo' mechanism, we need to clear each key in-place, # and reload the runtime overrides. Once a key is created in this dict, its value MUST NOT # change to a different dict object. for key in self: # clear the conf dict in-place self[key].clear() self._populate(key) def _load_yaml(self, conf_key, warn_on_fail=True): return config_file(self.file_path(conf_key), warn_on_fail=warn_on_fail, **self._yaycl) def file_path(self, conf_key): file_name = '{}{}'.format(conf_key, self._extension) return os.path.join(self._yaycl.config_dir, file_name)