def load_yaml(filename, **options):
loaded_yaml = AttrDict()
# Find the requested yaml in the yaml dir
if os.path.exists(filename):
with open(filename) as config_fh:
try:
loaded_yaml.update(yaml.load(config_fh, Loader=OrderedDictYAMLLoader))
except:
warn('Unable to parse configuration file at {}'.format(filename), ConfigInvalid)
return loaded_yaml
def load_yaml(filename, **options):
loaded_yaml = AttrDict()
# Find the requested yaml in the yaml dir
if os.path.exists(filename):
with open(filename) as config_fh:
try:
loaded_yaml.update(
yaml.load(config_fh, Loader=OrderedDictYAMLLoader))
except:
warn(
'Unable to parse configuration file at {}'.format(
filename), ConfigInvalid)
return loaded_yaml
class Config(dict):
"""Configuration YAML loader and cache"""
def __init__(self, config_dir, **kwargs):
# private yaycl conf, plugins can stash info here for configuration
self._yaycl = AttrDict({'config_dir': config_dir})
self._runtime = ConfigTree(self)
self._extension = kwargs.pop('extension', '.yaml')
self._yaycl.update(kwargs)
def _runtime_overrides(self):
return self._runtime
def _set_runtime_overrides(self, overrides_dict):
self._runtime.update(overrides_dict)
def _del_runtime_overrides(self):
self._runtime.clear()
runtime = property(_runtime_overrides, _set_runtime_overrides, _del_runtime_overrides)
def save(self, key):
"""Write out an in-memory config to the conf dir
Warning: This will destroy any formatting or ordering that existed in the original yaml
"""
with open(os.path.join(self._yaycl.config_dir, '%s.yaml' % key), 'w') as conf_file:
self[key].dump(conf_file)
# Support for descriptor access, e.g. instance.attrname
# Note that this is only on the get side, for support of nefarious things
# like setting and deleting, use the normal dict interface.
def __getattribute__(self, attr):
# Attempt normal object attr lookup; delegate to the dict interface if that fails
try:
return super(Config, self).__getattribute__(attr)
except AttributeError:
# try to load from cache first if this conf is already known
if attr in self:
return self[attr]
if attr.startswith('_'):
# don't try to load private names
raise
# If we're here, trigger the loader via getitem
return self[attr]
def __getitem__(self, key):
# Attempt a normal dict lookup to pull a cached conf
if key not in self:
super(Config, self).__setitem__(key, AttrDict())
# Cache miss, populate this conf key
# Call out to dict's setitem here since this is the only place where we're allowed to
# create a new key and we want the default behavior instead of the override below
self._populate(key)
return super(Config, self).__getitem__(key)
def __setitem__(self, key, value):
self[key].clear()
self[key].update(value)
def __delitem__(self, key):
self[key].clear()
self._populate(key)
def _inherit(self, conf_key):
"""Recurses through an object looking for 'inherit' clauses and replaces them with their
real counterparts. In the case of a dict, the inherit clause remains, in the case of
anything else, a replacement occurs such that:
sim5:
key: value
newkey: newvalue
sim6:
tags:
- tag1
- tag2
sim7:
inherit: management_systems/sim5
test:
tags:
inherit: management_systems/sim6/tags
Will produce the following output if requesting management_systems/sim7
inherit: management_systems/sim5
key: value
newkey: newvalue
test:
tags:
- tag1
- tag2
"""
for keys in self._needs_inherit(conf_key):
# get the dict containing inherit key
keys, root_key = keys[:-1], keys[-1]
root = self[conf_key]
for k in keys:
root = root[k]
# find the dict we're inheriting based on value
base = self[conf_key]
try:
for path_element in root[root_key]['inherit'].split('/'):
base = base[path_element]
except KeyError:
warn('{} path cannot be traversed, {} does not exist'.format(
root[root_key]['inherit'], path_element), InvalidInheritPath)
# rebase if the base was an attrdict,
# otherwise overwrite the key in-place
if isinstance(base, AttrDict):
del(root[root_key]['inherit'])
root[root_key].rebase(base)
else:
root[root_key] = base
def _needs_inherit(self, conf_key):
conf = self[conf_key]
# loop over keys until all the inherits are gone
while True:
seen_inherit = False
for k, v in conf.flatten_dict(conf):
if k[-1] == 'inherit':
# give back the keys needed to get to a dict containing an inherit key
seen_inherit = True
yield k[:-1]
if not seen_inherit:
break
def _populate(self, key):
yaml_dict = self._load_yaml(key)
# Graft in local yaml updates if they're available
with catch_warnings():
local_yaml = '%s.local' % key
local_yaml_dict = self._load_yaml(local_yaml, warn_on_fail=False)
if local_yaml_dict:
yaml_dict.update_dict(local_yaml_dict)
# Graft on the runtime overrides
if key in self._runtime:
AttrDict(self._runtime)[key]._.apply_flat(
partial(self._apply_runtime_overrides, yaml_dict))
self[key].update(yaml_dict)
self._inherit(key)
def _apply_runtime_overrides(self, yaml_dict, keys, value):
base = yaml_dict
for key in keys[:-1]:
if key not in base:
base[key] = AttrDict()
base = base[key]
base[keys[-1]] = value
def clear(self):
# because of the 'from conf import foo' mechanism, we need to clear each key in-place,
# and reload the runtime overrides. Once a key is created in this dict, its value MUST NOT
# change to a different dict object.
for key in self:
# clear the conf dict in-place
self[key].clear()
self._populate(key)
def _load_yaml(self, conf_key, warn_on_fail=True):
return config_file(self.file_path(conf_key), warn_on_fail=warn_on_fail, **self._yaycl)
def file_path(self, conf_key):
file_name = '{}{}'.format(conf_key, self._extension)
return os.path.join(self._yaycl.config_dir, file_name)
# Default blocking time before giving up on an ssh command execution,
# in seconds (float)
RUNCMD_TIMEOUT = 1200.0
SSHResult = namedtuple("SSHResult", ["rc", "output"])
_ssh_key_file = project_path.join('.generated_ssh_key')
_ssh_pubkey_file = project_path.join('.generated_ssh_key.pub')
# enum
_ssh_keystate = AttrDict({
'not_installed': 0,
'installing': 1,
'installed': 2
})
# enum reverse lookup
_ssh_keystate.update({v: k for k, v in _ssh_keystate.items()})
_client_session = []
class SSHClient(paramiko.SSHClient):
"""paramiko.SSHClient wrapper
Allows copying/overriding and use as a context manager
Constructor kwargs are handed directly to paramiko.SSHClient.connect()
"""
def __init__(self, stream_output=False, keystate=_ssh_keystate.not_installed,
**connect_kwargs):
super(SSHClient, self).__init__()
self._streaming = stream_output
self._keystate = keystate
class Config(dict):
"""Configuration YAML loader and cache"""
def __init__(self, config_dir, **kwargs):
# private yaycl conf, plugins can stash info here for configuration
self._yaycl = AttrDict({'config_dir': config_dir})
self._runtime = ConfigTree(self)
self._extension = kwargs.pop('extension', '.yaml')
self._yaycl.update(kwargs)
def _runtime_overrides(self):
return self._runtime
def _set_runtime_overrides(self, overrides_dict):
self._runtime.update(overrides_dict)
def _del_runtime_overrides(self):
self._runtime.clear()
runtime = property(_runtime_overrides, _set_runtime_overrides,
_del_runtime_overrides)
def save(self, key):
"""Write out an in-memory config to the conf dir
Warning: This will destroy any formatting or ordering that existed in the original yaml
"""
with open(os.path.join(self._yaycl.config_dir, '%s.yaml' % key),
'w') as conf_file:
self[key].dump(conf_file)
# Support for descriptor access, e.g. instance.attrname
# Note that this is only on the get side, for support of nefarious things
# like setting and deleting, use the normal dict interface.
def __getattribute__(self, attr):
# Attempt normal object attr lookup; delegate to the dict interface if that fails
try:
return super(Config, self).__getattribute__(attr)
except AttributeError:
# try to load from cache first if this conf is already known
if attr in self:
return self[attr]
if attr.startswith('_'):
# don't try to load private names
raise
# If we're here, trigger the loader via getitem
return self[attr]
def __getitem__(self, key):
# Attempt a normal dict lookup to pull a cached conf
if key not in self:
super(Config, self).__setitem__(key, AttrDict())
# Cache miss, populate this conf key
# Call out to dict's setitem here since this is the only place where we're allowed to
# create a new key and we want the default behavior instead of the override below
self._populate(key)
return super(Config, self).__getitem__(key)
def __setitem__(self, key, value):
self[key].clear()
self[key].update(value)
def __delitem__(self, key):
self[key].clear()
self._populate(key)
def _inherit(self, conf_key):
"""Recurses through an object looking for 'inherit' clauses and replaces them with their
real counterparts. In the case of a dict, the inherit clause remains, in the case of
anything else, a replacement occurs such that:
sim5:
key: value
newkey: newvalue
sim6:
tags:
- tag1
- tag2
sim7:
inherit: management_systems/sim5
test:
tags:
inherit: management_systems/sim6/tags
Will produce the following output if requesting management_systems/sim7
inherit: management_systems/sim5
key: value
newkey: newvalue
test:
tags:
- tag1
- tag2
"""
for keys in self._needs_inherit(conf_key):
# get the dict containing inherit key
keys, root_key = keys[:-1], keys[-1]
root = self[conf_key]
for k in keys:
root = root[k]
# find the dict we're inheriting based on value
base = self[conf_key]
try:
for path_element in root[root_key]['inherit'].split('/'):
base = base[path_element]
except KeyError:
warn(
'{} path cannot be traversed, {} does not exist'.format(
root[root_key]['inherit'], path_element),
InvalidInheritPath)
# rebase if the base was an attrdict,
# otherwise overwrite the key in-place
if isinstance(base, AttrDict):
del (root[root_key]['inherit'])
root[root_key].rebase(base)
else:
root[root_key] = base
def _needs_inherit(self, conf_key):
conf = self[conf_key]
# loop over keys until all the inherits are gone
while True:
seen_inherit = False
for k, v in conf.flatten_dict(conf):
if k[-1] == 'inherit':
# give back the keys needed to get to a dict containing an inherit key
seen_inherit = True
yield k[:-1]
if not seen_inherit:
break
def _populate(self, key):
yaml_dict = self._load_yaml(key)
# Graft in local yaml updates if they're available
with catch_warnings():
local_yaml = '%s.local' % key
local_yaml_dict = self._load_yaml(local_yaml, warn_on_fail=False)
if local_yaml_dict:
yaml_dict.update_dict(local_yaml_dict)
# Graft on the runtime overrides
if key in self._runtime:
AttrDict(self._runtime)[key]._.apply_flat(
partial(self._apply_runtime_overrides, yaml_dict))
self[key].update(yaml_dict)
self._inherit(key)
def _apply_runtime_overrides(self, yaml_dict, keys, value):
base = yaml_dict
for key in keys[:-1]:
if key not in base:
base[key] = AttrDict()
base = base[key]
base[keys[-1]] = value
def clear(self):
# because of the 'from conf import foo' mechanism, we need to clear each key in-place,
# and reload the runtime overrides. Once a key is created in this dict, its value MUST NOT
# change to a different dict object.
for key in self:
# clear the conf dict in-place
self[key].clear()
self._populate(key)
def _load_yaml(self, conf_key, warn_on_fail=True):
return config_file(self.file_path(conf_key),
warn_on_fail=warn_on_fail,
**self._yaycl)
def file_path(self, conf_key):
file_name = '{}{}'.format(conf_key, self._extension)
return os.path.join(self._yaycl.config_dir, file_name)
