def load(path=None, root=None, db=None, load_user=True): "Load all of the config files. " config = load_config(path, load_user=load_user) remotes = load_remotes(path, load_user=load_user) # The external file overwrites the main config if remotes: if not 'remotes' in config: config.remotes = AttrDict() for k, v in remotes.remotes.items(): config.remotes[k] = v accounts = load_accounts(path, load_user=load_user) # The external file overwrites the main config if accounts: if not 'accounts' in config: config.accounts = AttrDict() for k, v in accounts.accounts.items(): config.accounts[k] = v update_config(config) if root: config.library.filesystem_root = root if db: config.library.database = db return config
def load_accounts(extra_path=None, load_user=True): """Load the yaml account files :param load_user: :return: An `AttrDict` """ from os.path import getmtime try: accts_file = find_config_file(ACCOUNTS_FILE, extra_path=extra_path, load_user=load_user) except ConfigurationError: accts_file = None if accts_file is not None and os.path.exists(accts_file): config = AttrDict() config.update_yaml(accts_file) if not 'accounts' in config: config.remotes = AttrDict() config.accounts.loaded = [accts_file, getmtime(accts_file)] return config else: return None
def load_remotes(extra_path=None, load_user=True): """Load the YAML remotes file, which sort of combines the Accounts file with part of the remotes sections from the main config :return: An `AttrDict` """ from os.path import getmtime try: remotes_file = find_config_file(REMOTES_FILE, extra_path=extra_path, load_user=load_user) except ConfigurationError: remotes_file = None if remotes_file is not None and os.path.exists(remotes_file): config = AttrDict() config.update_yaml(remotes_file) if not 'remotes' in config: config.remotes = AttrDict() config.remotes.loaded = [remotes_file, getmtime(remotes_file)] return config else: return None
def dump_key(key, subs): values = [] for path, value in rc.config.flatten(): dot_path = '.'.join(path) if key: if key == dot_path: # Exact matches return sub_value(value, subs) elif dot_path.startswith(key): values.append((dot_path.split('.'), sub_value(value, subs) )) else: return ''.join(dot_path, '=', sub_value(value, subs)) if not values: return d = AttrDict() d.update_flat(values) return d
def test_dump_metadata(self): from ambry.util import AttrDict l = self.library() b = self.import_single_bundle('build.example.com/casters') v = 'Packaged for [Ambry](http://ambry.io) by {{contact_bundle.creator.org}}' self.assertEqual(v, AttrDict(b.metadata.about.items()).processed) self.assertEqual(v, b.metadata.about.processed) self.assertEqual( v, b.build_source_files.bundle_meta.record.unpacked_contents['about'] ['processed']) self.assertEqual( v, b.build_source_files.bundle_meta.get_object().about.processed) b.metadata.about.processed = 'foobar' b.commit() self.assertEqual('foobar', b.metadata.about.processed) self.assertNotEqual( b.metadata.about.processed, b.build_source_files.bundle_meta.get_object().about.processed) b.build_source_files.bundle_meta.objects_to_record() self.assertEqual( b.metadata.about.processed, b.build_source_files.bundle_meta.get_object().about.processed)
def config_edit(args, l, rc): from ambry.dbexceptions import ConfigurationError from ambry.util import AttrDict edit_args = ' '.join(args.args) if args.yaml or args.json: if args.yaml: import yaml v = yaml.load(edit_args) elif args.json: import json v = json.loads(edit_args) d = AttrDict() d.update(v) print d rc.config.update_flat(d.flatten()) else: key, value = edit_args.split('=') value = value.strip() key = key.strip() key_parts = key.split('.') e = rc.config for k in key_parts: k = k.strip() #print(k, str(key_parts[-1])) if str(k) == str(key_parts[-1]): e[k] = value else: e = e[k] configs = rc.config['loaded']['configs'] if len(configs) != 1: raise ConfigurationError("Configuration was loaded from multiple files; don't know which to edit; " "'{}'".format(configs)) try: del rc.config['accounts'] except KeyError: pass try: del rc.config['loaded'] except KeyError: pass with open(configs[0], 'w') as f: rc.config.dump(f)
def default_bundle_config(): """Return the default bundle config file as an AttrDict.""" import os from ambry.util import AttrDict config = AttrDict() f = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'bundle.yaml') config.update_yaml(f) return config
def load_config(path=None, load_user=True): """ Load configuration information from a config directory. Tries directories in this order: - A path provided as an argument - A path specified by the AMBRY_CONFIG environmenal variable - ambry in a path specified by the VIRTUAL_ENV environmental variable - /etc/ambry - ~/ambry :param path: An iterable of additional paths to load. :return: An `AttrDict` of configuration information """ from os.path import getmtime config = AttrDict() if not path: path = ROOT_DIR config_file = find_config_file(CONFIG_FILE, extra_path=path, load_user=load_user) if os.path.exists(config_file): config.update_yaml(config_file) config.loaded = [config_file, getmtime(config_file)] else: # Probably never get here, since the find_config_dir would have thrown a ConfigurationError config = AttrDict() config.loaded = [None, 0] return config
def mp_run(mp_run_args): ''' Run a bundle in a multi-processor child process. ''' import traceback import sys bundle_dir, run_args, method_name, args = mp_run_args try: bundle_file = sys.argv[1] if not os.path.exists(os.path.join(os.getcwd(), 'bundle.yaml')): print >> sys.stderr, "ERROR: Current directory '{}' does not have a bundle.yaml file, so it isn't a bundle file. Did you mean to run 'cli'?".format( os.getcwd()) sys.exit(1) # Import the bundle file from the rp = os.path.realpath(os.path.join(bundle_dir, 'bundle.py')) mod = import_file(rp) dir_ = os.path.dirname(rp) b = mod.Bundle(dir_) b.run_args = AttrDict(run_args) method = getattr(b, method_name) b.log("MP Run: pid={} {}{} ".format(os.getpid(), method.__name__, args)) try: # This close is really important; the child process can't be allowed to use the database # connection created by the parent; you get horrible breakages in # random places. b.close() method(*args) except: b.close() raise except: tb = traceback.format_exc() print '==========vvv MP Run Exception: {} pid = {} ==========='.format( args, os.getpid()) print tb print '==========^^^ MP Run Exception: {} pid = {} ==========='.format( args, os.getpid()) raise
def load_yaml(self, *args): """Load a yaml file from the bundle file system. Arguments are passed to self.path() And if the first path element is not absolute, pre-pends the bundle path. Returns an AttrDict of the results. This will load yaml files the same way as RunConfig files. """ from ambry.util import AttrDict f = self.path(*args) ad = AttrDict() ad.update_yaml(f) return ad
def __init__(self, path=None): """Create a new RunConfig object. Arguments path -- If present, a yaml file to load last, overwriting earlier values. If it is an array, load only the files in the array. """ config = AttrDict() config['loaded'] = [] if not path: pass if isinstance(path, (list, tuple, set)): files = path else: files = [ RunConfig.ROOT_CONFIG, path if path else RunConfig.USER_CONFIG, RunConfig.USER_ACCOUNTS, RunConfig.DIR_CONFIG ] loaded = False for f in files: if f is not None and os.path.exists(f): try: loaded = True config.loaded.append(f) config.update_yaml(f) except TypeError: pass # Empty files will produce a type error if not loaded: raise ConfigurationError( "Failed to load any config from: {}".format(files)) object.__setattr__(self, 'config', config) object.__setattr__(self, 'files', files)
def test_returns_false_if_partition_is_not_indexed(self): # to test we need just vid from the given object. So do not create partition, create partition like # instead. It makes that test more quick. partition = AttrDict(vid='vid1') ret = self.backend.partition_index.is_indexed(partition) self.assertFalse(ret)
def config_install(args, l, rc): import yaml import pkgutil import os from os.path import join, dirname import getpass import ambry.support from ambry.run import ROOT_DIR, USER_DIR, CONFIG_FILE, ACCOUNTS_FILE from ambry.util import AttrDict user = getpass.getuser() default_config_file = join(dirname(ambry.support.__file__),'ambry-{}.yaml'.format(args.template)) d = AttrDict().update_yaml(default_config_file) user_config_dir = os.path.join(os.path.expanduser('~'), USER_DIR) if user == 'root': # Root user config_dir = ROOT_DIR default_root = d.library.filesystem_root elif os.getenv('VIRTUAL_ENV'): # Special case for python virtual environments config_dir = os.path.join(os.getenv('VIRTUAL_ENV'), USER_DIR) default_root = os.path.join(os.getenv('VIRTUAL_ENV'), 'data') else: # Non-root user, outside of virtualenv config_dir = user_config_dir warn(("Installing as non-root, to '{}'\n" + "Run as root to install for all users.").format(config_dir)) default_root = os.path.join(os.path.expanduser('~'), 'ambry') if args.root: default_root = args.root if not os.path.exists(user_config_dir): os.makedirs(user_config_dir) if not os.path.exists(config_dir): os.makedirs(config_dir) if os.path.exists(os.path.join(config_dir, CONFIG_FILE)): if args.force: prt("File output file exists, overwriting: {}".format(config_dir)) else: fatal("Output file {} exists. Use -f to overwrite".format(config_dir)) d['library']['filesystem_root'] = default_root s = d.dump() if args.prt: prt(s.replace("{", "{{").replace("}", "}}")) return #Create an empty accounts file, if it does not exist user_accounts_file =os.path.join(user_config_dir, ACCOUNTS_FILE) if not os.path.exists(user_accounts_file): with open(user_accounts_file, 'w') as f: from ambry.util import random_string d = dict(accounts=dict( password=random_string(16), ambry=dict( name=None, email=None ) ) ) prt('Writing accounts file: {}'.format(user_accounts_file)) f.write(yaml.dump(d, indent=4, default_flow_style=False)) config_file = os.path.join(config_dir, CONFIG_FILE) with open(config_file, 'w') as f: prt('Writing config file: {}'.format(config_file)) f.write(s) # Make the directories. from ..run import get_runconfig rc = get_runconfig(config_file) for name, v in iteritems(rc.filesystem): dr = v.format(root=rc.library.filesystem_root) try: if not os.path.exists(dr): prt("Making directory: {}".format(dr)) os.makedirs(dr) except KeyError: pass
def update_config(config, use_environ=True): """Update the configuration from environmental variables. Updates: - config.library.database from the AMBRY_DB environmental variable. - config.library.filesystem_root from the AMBRY_ROOT environmental variable. - config.accounts.password from the AMBRY_PASSWORD environmental variable. :param config: An `attrDict` of configuration information. """ from ambry.util import select_from_url try: _ = config.library except KeyError: config.library = AttrDict() try: _ = config.filesystem except KeyError: config.filesystem = AttrDict() try: _ = config.accounts except KeyError: config.accounts = AttrDict() if not config.accounts.get('loaded'): config.accounts.loaded = [None, 0] try: _ = config.accounts.password except KeyError: config.accounts.password = None try: _ = config.remotes except KeyError: config.remotes = AttrDict() # Default empty if not config.remotes.get('loaded'): config.remotes.loaded = [None, 0] if use_environ: if os.getenv(ENVAR.DB): config.library.database = os.getenv(ENVAR.DB) if os.getenv(ENVAR.ROOT): config.library.filesystem_root = os.getenv(ENVAR.ROOT) if os.getenv(ENVAR.PASSWORD): config.accounts.password = os.getenv(ENVAR.PASSWORD) # Move any remotes that were configured under the library to the remotes section try: for k, v in config.library.remotes.items(): config.remotes[k] = {'url': v} del config.library['remotes'] except KeyError as e: pass # Then move any of the account entries that are linked to remotes into the remotes. try: for k, v in config.remotes.items(): if 'url' in v: host = select_from_url(v['url'], 'netloc') if host in config.accounts: config.remotes[k].update(config.accounts[host]) del config.accounts[host] except KeyError: pass # Set a default for the library database try: _ = config.library.database except KeyError: config.library.database = 'sqlite:///{root}/library.db' # Raise exceptions on missing items checks = [ 'config.library.filesystem_root', ] for check in checks: try: _ = eval(check) except KeyError: raise ConfigurationError( "Configuration is missing '{}'; loaded from {} ".format( check, config.loaded[0])) _, config.library.database = normalize_dsn_or_dict(config.library.database) for k, v in filesystem_defaults.items(): if k not in config.filesystem: config.filesystem[k] = v config.modtime = max(config.loaded[1], config.remotes.loaded[1], config.accounts.loaded[1])
def test_metadata(self): from ambry.bundle.meta import Metadata, ScalarTerm, TypedDictGroup, VarDictGroup, DictGroup, DictTerm, ListGroup import yaml from ambry.util import AttrDict class TestDictTerm(DictTerm): dterm1 = ScalarTerm() dterm2 = ScalarTerm() unset_term = ScalarTerm() class TestListGroup(ListGroup): _proto = TestDictTerm() class TestGroup(DictGroup): term = ScalarTerm() term2 = ScalarTerm() dterm = TestDictTerm() class TestTDGroup(TypedDictGroup): _proto = TestDictTerm() class TestTop(Metadata): group = TestGroup() tdgroup = TestTDGroup() lgroup = TestListGroup() vdgroup = VarDictGroup() tt = TestTop() ## ## Dict Group tt.group.term = 'Term' tt.group.term2 = 'Term2' with self.assertRaises(AttributeError): tt.group.term3 = 'Term3' self.assertEquals('Term', tt.group.term) self.assertEquals('Term2', tt.group.term2) self.assertEquals('Term', tt.group['term']) self.assertEquals(['term', 'term2', 'dterm'], tt.group.keys()) self.assertEquals([ 'Term', 'Term2', AttrDict([('dterm1', None), ('unset_term', None), ('dterm2', None)]) ], tt.group.values()) ## ## Dict Term tt.group.dterm.dterm1 = 'dterm1' tt.group.dterm.dterm2 = 'dterm2' with self.assertRaises(AttributeError): tt.group.dterm.dterm3 = 'dterm3' self.assertEquals('dterm1', tt.group.dterm.dterm1) self.assertEquals(['dterm1', 'unset_term', 'dterm2'], tt.group.dterm.keys()) self.assertEquals(['dterm1', None, 'dterm2'], tt.group.dterm.values()) ## List Group tt.lgroup.append({'k1': 'v1'}) tt.lgroup.append({'k2': 'v2'}) self.assertEquals('v1', tt.lgroup[0]['k1']) self.assertEquals('v2', tt.lgroup[1]['k2']) ## TypedDictGroup tt.tdgroup.foo.dterm1 = 'foo.dterm1' self.assertEqual('foo.dterm1', tt.tdgroup.foo.dterm1) tt.tdgroup.foo.dterm2 = 'foo.dterm2' tt.tdgroup.baz.dterm1 = 'foo.dterm1' ## VarDict Group tt.vdgroup.k1['v1'] = 'v1' tt.vdgroup.k1.v2 = 'v2' d = dict( about=dict(title='title', subject='subject', rights='rights', summary='Summary', tags='Foobotom'), contact=dict( creator=dict(name='Name', email='Email', bingo='bingo')), # These are note part of the defined set, so aren't converted to terms build=dict(foo='foo', bar='bar'), partitions=[ dict(name='foo', grain='bar'), dict(time='foo', space='bar'), dict( name='name', time='time', space='space', grain='grain', segment=0, ), ]) top = Top(d) t2 = Top() self.assertIn(('contact', 'creator', 'bingo'), top.errors) self.assertIn('publisher', top.contact.keys()) self.assertIn('url', dict(top.contact.creator)) self.assertEqual('Email', top.contact.creator.email) self.assertIn('name', top.partitions[0]) self.assertNotIn('space', top.partitions[0]) self.assertIn('space', top.partitions[2]) self.assertEquals('foo', top.partitions[0]['name']) top.sources.foo.url = 'url' top.sources.bar.description = 'description' top = Top(yaml.load(config_str)) self.assertEquals(['foo', 'baz'], top.build.keys()) self.assertEquals('bar', top.build.foo) self.assertEquals('bar', top.build['foo']) self.assertEqual(6, len(top.partitions)) self.assertIn('google', top.sources.keys()) self.assertIn('yahoo', top.sources.keys()) self.assertEquals('http://yahoo.com', top.sources.yahoo.url) #print top.write_to_dir(None) #for (group, term, subterm),v in top.rows: # print group, term, subterm,v t3 = Top() t3.load_rows(top.rows)
def test_basic(self): from ambry.bundle.meta import Metadata, ScalarTerm, TypedDictGroup,\ VarDictGroup, DictGroup, DictTerm, ListGroup from ambry.util import AttrDict class TestDictTerm(DictTerm): dterm1 = ScalarTerm() dterm2 = ScalarTerm() unset_term = ScalarTerm() class TestListGroup(ListGroup): _proto = TestDictTerm() class TestGroup(DictGroup): term = ScalarTerm() term2 = ScalarTerm() dterm = TestDictTerm() class TestTDGroup(TypedDictGroup): _proto = TestDictTerm() class TestTop(Metadata): group = TestGroup() tdgroup = TestTDGroup() lgroup = TestListGroup() vdgroup = VarDictGroup() tt = TestTop() # # Dict Group tt.group.term = 'Term' tt.group.term2 = 'Term2' with self.assertRaises(AttributeError): tt.group.term3 = 'Term3' self.assertEquals('Term', tt.group.term) self.assertEquals('Term2', tt.group.term2) self.assertEquals('Term', tt.group['term']) self.assertEquals(['term', 'term2', 'dterm'], tt.group.keys()) self.assertEquals([ 'Term', 'Term2', AttrDict([('dterm1', None), ('unset_term', None), ('dterm2', None)]) ], tt.group.values()) # # Dict Term tt.group.dterm.dterm1 = 'dterm1' tt.group.dterm.dterm2 = 'dterm2' self.assertEquals('dterm1', tt.group.dterm.dterm1) self.assertEquals(['dterm1', 'unset_term', 'dterm2'], tt.group.dterm.keys()) self.assertEquals(['dterm1', None, 'dterm2'], tt.group.dterm.values()) # # List Group tt.lgroup.append({'dterm1': 'dterm1'}) tt.lgroup.append({'dterm2': 'dterm2'}) self.assertEquals('dterm2', tt.lgroup[1]['dterm2']) self.assertEquals('dterm1', tt.lgroup[0]['dterm1']) # # TypedDictGroup tt.tdgroup.foo.dterm1 = 'foo.dterm1' self.assertEqual('foo.dterm1', tt.tdgroup.foo.dterm1) tt.tdgroup.foo.dterm2 = 'foo.dterm2' tt.tdgroup.baz.dterm1 = 'foo.dterm1' # # VarDict Group tt.vdgroup.k1['v1'] = 'v1' tt.vdgroup.k1.v2 = 'v2'