def __init__(self, path=None): '''Create a new RunConfig object Arguments path -- If present, a yaml file to load last, overwriting earlier values. If it is an array, load only the files in the array. If path is not an array, the file is added to an array of files, and each of the files is loaded. These files are: /etc/databundles.yaml ~/.databundles.yaml databundles.yaml ''' self.config = AttrDict() self.config['loaded'] = [] if isinstance(path, list): self.files = path else: self.files = [ RunConfig.ROOT_CONFIG, RunConfig.USER_CONFIG, RunConfig.DIR_CONFIG, path] loaded = False for f in self.files: if f is not None and os.path.exists(f): loaded = True self.config.loaded.append(f) self.config.update_yaml(f) if not loaded: raise Exception("Failed to load any config from: {}".format(self.files))
def load_yaml(self, *args): """Load a yaml file from the bundle file system. Arguments are passed to self.path() And if the first path element is not absolute, pre-pends the bundle path. Returns an AttrDict of the results. This will load yaml files the same way as RunConfig files. """ from databundles.util import AttrDict f = self.path(*args) ad = AttrDict() ad.update_yaml(f) return ad
def __init__(self, path=None, is_server = False): '''Create a new RunConfig object Arguments path -- If present, a yaml file to load last, overwriting earlier values. If it is an array, load only the files in the array. ''' self.config = AttrDict() self.config['loaded'] = [] if isinstance(path, (list, tuple, set)): self.files = path else: self.files = [ RunConfig.SERVER_CONFIG if is_server else RunConfig.CLIENT_CONFIG , RunConfig.ROOT_CONFIG, RunConfig.USER_CONFIG, RunConfig.DIR_CONFIG, path] loaded = False for f in self.files: if f is not None and os.path.exists(f): try: loaded = True self.config.loaded.append(f) self.config.update_yaml(f) except TypeError as e: pass # Empty files will produce a type error if not loaded: raise Exception("Failed to load any config from: {}".format(self.files))
class RunConfig(object): '''Runtime configuration object The RunConfig object will search for a databundles.yaml file in multiple locations, including:: /etc/databundles.yaml ~user/.databundles.yaml ./databundles.yaml A named path ( --config option ) It will start from the first directory, and for each one, try to load the file and copy the values into an accumulator, with later values overwritting earlier ones. ''' ROOT_CONFIG = '/etc/databundles/config.yaml' SERVER_CONFIG = '/etc/databundles/server.yaml' CLIENT_CONFIG = '/etc/databundles/client.yaml' USER_CONFIG = os.path.expanduser('~/.databundles.yaml') DIR_CONFIG = os.path.join(os.getcwd(),'databundles.yaml') def __init__(self, path=None, is_server = False): '''Create a new RunConfig object Arguments path -- If present, a yaml file to load last, overwriting earlier values. If it is an array, load only the files in the array. ''' self.config = AttrDict() self.config['loaded'] = [] if isinstance(path, (list, tuple, set)): self.files = path else: self.files = [ RunConfig.SERVER_CONFIG if is_server else RunConfig.CLIENT_CONFIG , RunConfig.ROOT_CONFIG, RunConfig.USER_CONFIG, RunConfig.DIR_CONFIG, path] loaded = False for f in self.files: if f is not None and os.path.exists(f): try: loaded = True self.config.loaded.append(f) self.config.update_yaml(f) except TypeError as e: pass # Empty files will produce a type error if not loaded: raise Exception("Failed to load any config from: {}".format(self.files)) def __getattr__(self, group): '''Fetch a confiration group and return the contents as an attribute-accessible dict''' return self.config.get(group,{}) def group(self, name): '''return a dict for a group of configuration items.''' return self.config.get(name,{}) def dump(self, stream=None): to_string = False if stream is None: import StringIO stream = StringIO.StringIO() to_string = True self.config.dump(stream) if to_string: stream.seek(0) return stream.read() else: return stream
class RunConfig(object): '''Runtime configuration object The RunConfig object will search for a databundles.yaml file in multiple locations, including:: /etc/databundles.yaml ~user/.databundles.yaml ./databundles.yaml A named path ( --config option ) It will start from the first directory, and for each one, try to load the file and copy the values into an accumulator, with later values overwritting earlier ones. ''' ROOT_CONFIG = '/etc/databundles/config.yaml' SERVER_CONFIG = '/etc/databundles/server.yaml' CLIENT_CONFIG = '/etc/databundles/client.yaml' USER_CONFIG = os.path.expanduser('~/.databundles.yaml') DIR_CONFIG = os.path.join(os.getcwd(),'databundles.yaml') def __init__(self, path=None, is_server = False): '''Create a new RunConfig object Arguments path -- If present, a yaml file to load last, overwriting earlier values. If it is an array, load only the files in the array. ''' self.config = AttrDict() self.config['loaded'] = [] if isinstance(path, (list, tuple, set)): self.files = path else: self.files = [ RunConfig.SERVER_CONFIG if is_server else RunConfig.CLIENT_CONFIG , RunConfig.ROOT_CONFIG, RunConfig.USER_CONFIG, RunConfig.DIR_CONFIG, path] loaded = False for f in self.files: if f is not None and os.path.exists(f): try: loaded = True self.config.loaded.append(f) self.config.update_yaml(f) except TypeError: pass # Empty files will produce a type error if not loaded: raise Exception("Failed to load any config from: {}".format(self.files)) self.sourcerepo = self._sourcerepo(self) def __getattr__(self, group): '''Fetch a confiration group and return the contents as an attribute-accessible dict''' return self.config.get(group,{}) def group(self, name): '''return a dict for a group of configuration items.''' return self.config.get(name,{}) def group_item(self, group, name): import copy g = self.group(group) if not name in g: raise KeyError("Could not find name '{}' in group '{}'. Config has: {}".format(name, group, g.keys())) return copy.deepcopy(g[name]) def _yield_string(self, e): '''Recursively descend a data structure to find string values. This will locate values that should be expanded by reference. ''' from util import walk_dict for path, subdicts, values in walk_dict(e): for k,v in values: if v is None: import pprint pprint.pprint(e.to_dict()) raise Exception('Got None value: {} {} {} {} '.format(path, subdicts, k, v)) path_parts = path.split('/') path_parts.pop() path_parts.pop(0) path_parts.append(k) def setter(nv): sd = e for pp in path_parts: if not isinstance(sd[pp], dict ): break sd = sd[pp] # Save the oroginal value as a name sd[pp] = nv if isinstance(sd[pp], dict): sd[pp]['_name'] = v yield k,v,setter def _sub_strings(self, e, subs): '''Substitute keys in the dict e with functions defined in subs''' iters = 0 while (iters < 100): sub_count = 0 for k,v,setter in self._yield_string(e): if k in subs: setter(subs[k](k,v)) sub_count += 1 if sub_count == 0: break iters += 1 return e def dump(self, stream=None): to_string = False if stream is None: import StringIO stream = StringIO.StringIO() to_string = True self.config.dump(stream) if to_string: stream.seek(0) return stream.read() else: return stream def filesystem(self,name): e = self.group_item('filesystem', name) fs = self.group('filesystem') root_dir = fs['root_dir'] if 'root_dir' in fs else '/tmp/norootdir' return self._sub_strings(e, { 'upstream': lambda k,v: self.filesystem(v), 'account': lambda k,v: self.account(v), 'dir' : lambda k,v: v.format(root=root_dir) } ) def account(self,name): e = self.group_item('accounts', name) e = self._sub_strings(e, {'store': lambda k,v: self.filesystem(v)} ) e['_name'] = name return e def datarepo(self,name): e = self.group_item('datarepo', name) return self._sub_strings(e, { 'filesystem': lambda k,v: self.filesystem(v) } ) def library(self,name): e = self.group_item('library', name) e = self._sub_strings(e, { 'filesystem': lambda k,v: self.filesystem(v), 'remote': lambda k,v: self.filesystem(v), 'database': lambda k,v: self.database(v), 'account': lambda k,v: self.account(v), 'upstream': lambda k,v: self.filesystem(v), 'cdn': lambda k,v: self.account(v), } ) e['_name'] = name return e # # This object is attached to the sourcerepo property, to provide a variety of interfaces # other than just being callable. class _sourcerepo(object): def __init__(self,this): self.this = this def __call__(self, name): e = self.this.group_item('sourcerepo', name) e = self.this._sub_strings(e, { 'account': lambda k,v: self.this.account(v) } ) e['_name'] = name e['dir'] = self.dir return e @property def list(self): from source.repository import new_repository return [ new_repository(self.this.sourcerepo(r_name)) for r_name in self.this.group('sourcerepo').keys() if r_name != 'dir' ] @property def dir(self): fs = self.this.group('filesystem') root_dir = fs['root_dir'] if 'root_dir' in fs else '/tmp/norootdir' return self.this.group('sourcerepo')['dir'].format(root=root_dir) def warehouse(self,name): e = self.group_item('warehouse', name) return self._sub_strings(e, { 'database': lambda k,v: self.database(v), 'account': lambda k,v: self.account(v), 'library': lambda k,v: self.database(v), } ) def database(self,name): fs = self.group('filesystem') root_dir = fs['root_dir'] if 'root_dir' in fs else '/tmp/norootdir' e = self.group_item('database', name) e = self._sub_strings(e, {'dbname' : lambda k,v: v.format(root=root_dir), 'account': lambda k,v: self.account(v),} ) # Copy account credentials into the database record, so there is consistent access # pattern if 'account' in e: account = e['account'] if 'password' in account: e['user'] = account['user'] e['password'] = account['password'] e = e.to_dict() return e def python_dir(self): fs = self.group('filesystem') if not 'python' in fs: return None root_dir = fs['root_dir'] if 'root_dir' in fs else '/tmp/norootdir' python_dir = fs['python'].format(root=root_dir) return python_dir