def __init__(self, path=None):
'''Create a new RunConfig object
Arguments
path -- If present, a yaml file to load last, overwriting earlier values.
If it is an array, load only the files in the array.
If path is not an array, the file is added to an array of files, and
each of the files is loaded. These files are:
/etc/databundles.yaml
~/.databundles.yaml
databundles.yaml
'''
self.config = AttrDict()
self.config['loaded'] = []
if isinstance(path, list):
self.files = path
else:
self.files = [ RunConfig.ROOT_CONFIG, RunConfig.USER_CONFIG,
RunConfig.DIR_CONFIG, path]
loaded = False
for f in self.files:
if f is not None and os.path.exists(f):
loaded = True
self.config.loaded.append(f)
self.config.update_yaml(f)
if not loaded:
raise Exception("Failed to load any config from: {}".format(self.files))
def load_yaml(self, *args):
"""Load a yaml file from the bundle file system. Arguments are passed to self.path()
And if the first path element is not absolute, pre-pends the bundle path.
Returns an AttrDict of the results.
This will load yaml files the same way as RunConfig files.
"""
from databundles.util import AttrDict
f = self.path(*args)
ad = AttrDict()
ad.update_yaml(f)
return ad
def __init__(self, path=None, is_server = False):
'''Create a new RunConfig object
Arguments
path -- If present, a yaml file to load last, overwriting earlier values.
If it is an array, load only the files in the array.
'''
self.config = AttrDict()
self.config['loaded'] = []
if isinstance(path, (list, tuple, set)):
self.files = path
else:
self.files = [
RunConfig.SERVER_CONFIG if is_server else RunConfig.CLIENT_CONFIG ,
RunConfig.ROOT_CONFIG,
RunConfig.USER_CONFIG,
RunConfig.DIR_CONFIG,
path]
loaded = False
for f in self.files:
if f is not None and os.path.exists(f):
try:
loaded = True
self.config.loaded.append(f)
self.config.update_yaml(f)
except TypeError as e:
pass # Empty files will produce a type error
if not loaded:
raise Exception("Failed to load any config from: {}".format(self.files))
class RunConfig(object):
'''Runtime configuration object
The RunConfig object will search for a databundles.yaml file in multiple locations,
including::
/etc/databundles.yaml
~user/.databundles.yaml
./databundles.yaml
A named path ( --config option )
It will start from the first directory, and for each one, try to load the file
and copy the values into an accumulator, with later values overwritting
earlier ones.
'''
ROOT_CONFIG = '/etc/databundles/config.yaml'
SERVER_CONFIG = '/etc/databundles/server.yaml'
CLIENT_CONFIG = '/etc/databundles/client.yaml'
USER_CONFIG = os.path.expanduser('~/.databundles.yaml')
DIR_CONFIG = os.path.join(os.getcwd(),'databundles.yaml')
def __init__(self, path=None, is_server = False):
'''Create a new RunConfig object
Arguments
path -- If present, a yaml file to load last, overwriting earlier values.
If it is an array, load only the files in the array.
'''
self.config = AttrDict()
self.config['loaded'] = []
if isinstance(path, (list, tuple, set)):
self.files = path
else:
self.files = [
RunConfig.SERVER_CONFIG if is_server else RunConfig.CLIENT_CONFIG ,
RunConfig.ROOT_CONFIG,
RunConfig.USER_CONFIG,
RunConfig.DIR_CONFIG,
path]
loaded = False
for f in self.files:
if f is not None and os.path.exists(f):
try:
loaded = True
self.config.loaded.append(f)
self.config.update_yaml(f)
except TypeError as e:
pass # Empty files will produce a type error
if not loaded:
raise Exception("Failed to load any config from: {}".format(self.files))
def __getattr__(self, group):
'''Fetch a confiration group and return the contents as an
attribute-accessible dict'''
return self.config.get(group,{})
def group(self, name):
'''return a dict for a group of configuration items.'''
return self.config.get(name,{})
def dump(self, stream=None):
to_string = False
if stream is None:
import StringIO
stream = StringIO.StringIO()
to_string = True
self.config.dump(stream)
if to_string:
stream.seek(0)
return stream.read()
else:
return stream
class RunConfig(object):
'''Runtime configuration object
The RunConfig object will search for a databundles.yaml file in multiple locations,
including::
/etc/databundles.yaml
~user/.databundles.yaml
./databundles.yaml
A named path ( --config option )
It will start from the first directory, and for each one, try to load the file
and copy the values into an accumulator, with later values overwritting
earlier ones.
'''
ROOT_CONFIG = '/etc/databundles/config.yaml'
SERVER_CONFIG = '/etc/databundles/server.yaml'
CLIENT_CONFIG = '/etc/databundles/client.yaml'
USER_CONFIG = os.path.expanduser('~/.databundles.yaml')
DIR_CONFIG = os.path.join(os.getcwd(),'databundles.yaml')
def __init__(self, path=None, is_server = False):
'''Create a new RunConfig object
Arguments
path -- If present, a yaml file to load last, overwriting earlier values.
If it is an array, load only the files in the array.
'''
self.config = AttrDict()
self.config['loaded'] = []
if isinstance(path, (list, tuple, set)):
self.files = path
else:
self.files = [
RunConfig.SERVER_CONFIG if is_server else RunConfig.CLIENT_CONFIG ,
RunConfig.ROOT_CONFIG,
RunConfig.USER_CONFIG,
RunConfig.DIR_CONFIG,
path]
loaded = False
for f in self.files:
if f is not None and os.path.exists(f):
try:
loaded = True
self.config.loaded.append(f)
self.config.update_yaml(f)
except TypeError:
pass # Empty files will produce a type error
if not loaded:
raise Exception("Failed to load any config from: {}".format(self.files))
self.sourcerepo = self._sourcerepo(self)
def __getattr__(self, group):
'''Fetch a confiration group and return the contents as an
attribute-accessible dict'''
return self.config.get(group,{})
def group(self, name):
'''return a dict for a group of configuration items.'''
return self.config.get(name,{})
def group_item(self, group, name):
import copy
g = self.group(group)
if not name in g:
raise KeyError("Could not find name '{}' in group '{}'. Config has: {}".format(name, group, g.keys()))
return copy.deepcopy(g[name])
def _yield_string(self, e):
'''Recursively descend a data structure to find string values.
This will locate values that should be expanded by reference. '''
from util import walk_dict
for path, subdicts, values in walk_dict(e):
for k,v in values:
if v is None:
import pprint
pprint.pprint(e.to_dict())
raise Exception('Got None value: {} {} {} {} '.format(path, subdicts, k, v))
path_parts = path.split('/')
path_parts.pop()
path_parts.pop(0)
path_parts.append(k)
def setter(nv):
sd = e
for pp in path_parts:
if not isinstance(sd[pp], dict ):
break
sd = sd[pp]
# Save the oroginal value as a name
sd[pp] = nv
if isinstance(sd[pp], dict):
sd[pp]['_name'] = v
yield k,v,setter
def _sub_strings(self, e, subs):
'''Substitute keys in the dict e with functions defined in subs'''
iters = 0
while (iters < 100):
sub_count = 0
for k,v,setter in self._yield_string(e):
if k in subs:
setter(subs[k](k,v))
sub_count += 1
if sub_count == 0:
break
iters += 1
return e
def dump(self, stream=None):
to_string = False
if stream is None:
import StringIO
stream = StringIO.StringIO()
to_string = True
self.config.dump(stream)
if to_string:
stream.seek(0)
return stream.read()
else:
return stream
def filesystem(self,name):
e = self.group_item('filesystem', name)
fs = self.group('filesystem')
root_dir = fs['root_dir'] if 'root_dir' in fs else '/tmp/norootdir'
return self._sub_strings(e, {
'upstream': lambda k,v: self.filesystem(v),
'account': lambda k,v: self.account(v),
'dir' : lambda k,v: v.format(root=root_dir)
} )
def account(self,name):
e = self.group_item('accounts', name)
e = self._sub_strings(e, {'store': lambda k,v: self.filesystem(v)} )
e['_name'] = name
return e
def datarepo(self,name):
e = self.group_item('datarepo', name)
return self._sub_strings(e, {
'filesystem': lambda k,v: self.filesystem(v)
} )
def library(self,name):
e = self.group_item('library', name)
e = self._sub_strings(e, {
'filesystem': lambda k,v: self.filesystem(v),
'remote': lambda k,v: self.filesystem(v),
'database': lambda k,v: self.database(v),
'account': lambda k,v: self.account(v),
'upstream': lambda k,v: self.filesystem(v),
'cdn': lambda k,v: self.account(v),
} )
e['_name'] = name
return e
#
# This object is attached to the sourcerepo property, to provide a variety of interfaces
# other than just being callable.
class _sourcerepo(object):
def __init__(self,this):
self.this = this
def __call__(self, name):
e = self.this.group_item('sourcerepo', name)
e = self.this._sub_strings(e, {
'account': lambda k,v: self.this.account(v)
} )
e['_name'] = name
e['dir'] = self.dir
return e
@property
def list(self):
from source.repository import new_repository
return [ new_repository(self.this.sourcerepo(r_name))
for r_name in self.this.group('sourcerepo').keys()
if r_name != 'dir'
]
@property
def dir(self):
fs = self.this.group('filesystem')
root_dir = fs['root_dir'] if 'root_dir' in fs else '/tmp/norootdir'
return self.this.group('sourcerepo')['dir'].format(root=root_dir)
def warehouse(self,name):
e = self.group_item('warehouse', name)
return self._sub_strings(e, {
'database': lambda k,v: self.database(v),
'account': lambda k,v: self.account(v),
'library': lambda k,v: self.database(v),
} )
def database(self,name):
fs = self.group('filesystem')
root_dir = fs['root_dir'] if 'root_dir' in fs else '/tmp/norootdir'
e = self.group_item('database', name)
e = self._sub_strings(e, {'dbname' : lambda k,v: v.format(root=root_dir),
'account': lambda k,v: self.account(v),} )
# Copy account credentials into the database record, so there is consistent access
# pattern
if 'account' in e:
account = e['account']
if 'password' in account:
e['user'] = account['user']
e['password'] = account['password']
e = e.to_dict()
return e
def python_dir(self):
fs = self.group('filesystem')
if not 'python' in fs:
return None
root_dir = fs['root_dir'] if 'root_dir' in fs else '/tmp/norootdir'
python_dir = fs['python'].format(root=root_dir)
return python_dir
