def load(path=None, root=None, db=None, load_user=True):
"Load all of the config files. "
config = load_config(path, load_user=load_user)
remotes = load_remotes(path, load_user=load_user)
# The external file overwrites the main config
if remotes:
if not 'remotes' in config:
config.remotes = AttrDict()
for k, v in remotes.remotes.items():
config.remotes[k] = v
accounts = load_accounts(path, load_user=load_user)
# The external file overwrites the main config
if accounts:
if not 'accounts' in config:
config.accounts = AttrDict()
for k, v in accounts.accounts.items():
config.accounts[k] = v
update_config(config)
if root:
config.library.filesystem_root = root
if db:
config.library.database = db
return config
def dump_key(key, subs):
values = []
for path, value in rc.config.flatten():
dot_path = '.'.join(path)
if key:
if key == dot_path: # Exact matches
return sub_value(value, subs)
elif dot_path.startswith(key):
values.append((dot_path.split('.'), sub_value(value, subs) ))
else:
return ''.join(dot_path, '=', sub_value(value, subs))
if not values:
return
d = AttrDict()
d.update_flat(values)
return d
def config_edit(args, l, rc):
from ambry.dbexceptions import ConfigurationError
from ambry.util import AttrDict
edit_args = ' '.join(args.args)
if args.yaml or args.json:
if args.yaml:
import yaml
v = yaml.load(edit_args)
elif args.json:
import json
v = json.loads(edit_args)
d = AttrDict()
d.update(v)
print d
rc.config.update_flat(d.flatten())
else:
key, value = edit_args.split('=')
value = value.strip()
key = key.strip()
key_parts = key.split('.')
e = rc.config
for k in key_parts:
k = k.strip()
#print(k, str(key_parts[-1]))
if str(k) == str(key_parts[-1]):
e[k] = value
else:
e = e[k]
configs = rc.config['loaded']['configs']
if len(configs) != 1:
raise ConfigurationError("Configuration was loaded from multiple files; don't know which to edit; "
"'{}'".format(configs))
try:
del rc.config['accounts']
except KeyError:
pass
try:
del rc.config['loaded']
except KeyError:
pass
with open(configs[0], 'w') as f:
rc.config.dump(f)
def default_bundle_config():
'''Return the default bundle config file as an AttrDict'''
import os
from ambry.util import AttrDict
config = AttrDict()
f = os.path.join(os.path.dirname(os.path.realpath(__file__)),'bundle.yaml')
config.update_yaml(f)
return config
def default_bundle_config():
"""Return the default bundle config file as an AttrDict."""
import os
from ambry.util import AttrDict
config = AttrDict()
f = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'bundle.yaml')
config.update_yaml(f)
return config
def test_dump_metadata(self):
from ambry.util import AttrDict
l = self.library()
b = self.import_single_bundle('build.example.com/casters')
v = 'Packaged for [Ambry](http://ambry.io) by {{contact_bundle.creator.org}}'
self.assertEqual(v, AttrDict(b.metadata.about.items()).processed)
self.assertEqual(v, b.metadata.about.processed)
self.assertEqual(
v,
b.build_source_files.bundle_meta.record.unpacked_contents['about']
['processed'])
self.assertEqual(
v,
b.build_source_files.bundle_meta.get_object().about.processed)
b.metadata.about.processed = 'foobar'
b.commit()
self.assertEqual('foobar', b.metadata.about.processed)
self.assertNotEqual(
b.metadata.about.processed,
b.build_source_files.bundle_meta.get_object().about.processed)
b.build_source_files.bundle_meta.objects_to_record()
self.assertEqual(
b.metadata.about.processed,
b.build_source_files.bundle_meta.get_object().about.processed)
def load_yaml(self,*args):
"""Load a yaml file from the bundle file system. Arguments are passed to self.path()
And if the first path element is not absolute, pre-pends the bundle path.
Returns an AttrDict of the results.
This will load yaml files the same way as RunConfig files.
"""
from ambry.util import AttrDict
f = self.path(*args)
ad = AttrDict()
ad.update_yaml(f)
return ad
def load_yaml(self, *args):
"""Load a yaml file from the bundle file system. Arguments are passed to self.path()
And if the first path element is not absolute, pre-pends the bundle path.
Returns an AttrDict of the results.
This will load yaml files the same way as RunConfig files.
"""
from ambry.util import AttrDict
f = self.path(*args)
ad = AttrDict()
ad.update_yaml(f)
return ad
def __init__(self, path=None):
"""Create a new RunConfig object.
Arguments
path -- If present, a yaml file to load last, overwriting earlier values.
If it is an array, load only the files in the array.
"""
config = AttrDict()
config['loaded'] = []
if not path:
pass
if isinstance(path, (list, tuple, set)):
files = path
else:
files = [
RunConfig.ROOT_CONFIG,
path if path else RunConfig.USER_CONFIG,
RunConfig.USER_ACCOUNTS,
RunConfig.DIR_CONFIG]
loaded = False
for f in files:
if f is not None and os.path.exists(f):
try:
loaded = True
config.loaded.append(f)
config.update_yaml(f)
except TypeError:
pass # Empty files will produce a type error
if not loaded:
raise ConfigurationError(
"Failed to load any config from: {}".format(files))
object.__setattr__(self, 'config', config)
object.__setattr__(self, 'files', files)
def __init__(self, path=None):
"""Create a new RunConfig object.
Arguments
path -- If present, a yaml file to load last, overwriting earlier values.
If it is an array, load only the files in the array.
"""
config = AttrDict()
config['loaded'] = []
if not path:
pass
if isinstance(path, (list, tuple, set)):
files = path
else:
files = [
RunConfig.ROOT_CONFIG, path if path else RunConfig.USER_CONFIG,
RunConfig.USER_ACCOUNTS, RunConfig.DIR_CONFIG
]
loaded = False
for f in files:
if f is not None and os.path.exists(f):
try:
loaded = True
config.loaded.append(f)
config.update_yaml(f)
except TypeError:
pass # Empty files will produce a type error
if not loaded:
raise ConfigurationError(
"Failed to load any config from: {}".format(files))
object.__setattr__(self, 'config', config)
object.__setattr__(self, 'files', files)
def load_accounts(extra_path=None, load_user=True):
"""Load the yaml account files
:param load_user:
:return: An `AttrDict`
"""
from os.path import getmtime
try:
accts_file = find_config_file(ACCOUNTS_FILE,
extra_path=extra_path,
load_user=load_user)
except ConfigurationError:
accts_file = None
if accts_file is not None and os.path.exists(accts_file):
config = AttrDict()
config.update_yaml(accts_file)
if not 'accounts' in config:
config.remotes = AttrDict()
config.accounts.loaded = [accts_file, getmtime(accts_file)]
return config
else:
return None
def load_remotes(extra_path=None, load_user=True):
"""Load the YAML remotes file, which sort of combines the Accounts file with part of the
remotes sections from the main config
:return: An `AttrDict`
"""
from os.path import getmtime
try:
remotes_file = find_config_file(REMOTES_FILE,
extra_path=extra_path,
load_user=load_user)
except ConfigurationError:
remotes_file = None
if remotes_file is not None and os.path.exists(remotes_file):
config = AttrDict()
config.update_yaml(remotes_file)
if not 'remotes' in config:
config.remotes = AttrDict()
config.remotes.loaded = [remotes_file, getmtime(remotes_file)]
return config
else:
return None
def mp_run(mp_run_args):
''' Run a bundle in a multi-processor child process. '''
import traceback
import sys
bundle_dir, run_args, method_name, args = mp_run_args
try:
bundle_file = sys.argv[1]
if not os.path.exists(os.path.join(os.getcwd(), 'bundle.yaml')):
print >> sys.stderr, "ERROR: Current directory '{}' does not have a bundle.yaml file, so it isn't a bundle file. Did you mean to run 'cli'?".format(
os.getcwd())
sys.exit(1)
# Import the bundle file from the
rp = os.path.realpath(os.path.join(bundle_dir, 'bundle.py'))
mod = import_file(rp)
dir_ = os.path.dirname(rp)
b = mod.Bundle(dir_)
b.run_args = AttrDict(run_args)
method = getattr(b, method_name)
b.log("MP Run: pid={} {}{} ".format(os.getpid(), method.__name__,
args))
try:
# This close is really important; the child process can't be allowed to use the database
# connection created by the parent; you get horrible breakages in
# random places.
b.close()
method(*args)
except:
b.close()
raise
except:
tb = traceback.format_exc()
print '==========vvv MP Run Exception: {} pid = {} ==========='.format(
args, os.getpid())
print tb
print '==========^^^ MP Run Exception: {} pid = {} ==========='.format(
args, os.getpid())
raise
def load_config(path=None, load_user=True):
"""
Load configuration information from a config directory. Tries directories in this order:
- A path provided as an argument
- A path specified by the AMBRY_CONFIG environmenal variable
- ambry in a path specified by the VIRTUAL_ENV environmental variable
- /etc/ambry
- ~/ambry
:param path: An iterable of additional paths to load.
:return: An `AttrDict` of configuration information
"""
from os.path import getmtime
config = AttrDict()
if not path:
path = ROOT_DIR
config_file = find_config_file(CONFIG_FILE,
extra_path=path,
load_user=load_user)
if os.path.exists(config_file):
config.update_yaml(config_file)
config.loaded = [config_file, getmtime(config_file)]
else:
# Probably never get here, since the find_config_dir would have thrown a ConfigurationError
config = AttrDict()
config.loaded = [None, 0]
return config
def test_basic(self):
from ambry.bundle.meta import Metadata, ScalarTerm, TypedDictGroup,\
VarDictGroup, DictGroup, DictTerm, ListGroup
from ambry.util import AttrDict
class TestDictTerm(DictTerm):
dterm1 = ScalarTerm()
dterm2 = ScalarTerm()
unset_term = ScalarTerm()
class TestListGroup(ListGroup):
_proto = TestDictTerm()
class TestGroup(DictGroup):
term = ScalarTerm()
term2 = ScalarTerm()
dterm = TestDictTerm()
class TestTDGroup(TypedDictGroup):
_proto = TestDictTerm()
class TestTop(Metadata):
group = TestGroup()
tdgroup = TestTDGroup()
lgroup = TestListGroup()
vdgroup = VarDictGroup()
tt = TestTop()
#
# Dict Group
tt.group.term = 'Term'
tt.group.term2 = 'Term2'
with self.assertRaises(AttributeError):
tt.group.term3 = 'Term3'
self.assertEquals('Term', tt.group.term)
self.assertEquals('Term2', tt.group.term2)
self.assertEquals('Term', tt.group['term'])
self.assertEquals(['term', 'term2', 'dterm'], tt.group.keys())
self.assertEquals([
'Term', 'Term2',
AttrDict([('dterm1', None), ('unset_term', None),
('dterm2', None)])
], tt.group.values())
#
# Dict Term
tt.group.dterm.dterm1 = 'dterm1'
tt.group.dterm.dterm2 = 'dterm2'
self.assertEquals('dterm1', tt.group.dterm.dterm1)
self.assertEquals(['dterm1', 'unset_term', 'dterm2'],
tt.group.dterm.keys())
self.assertEquals(['dterm1', None, 'dterm2'], tt.group.dterm.values())
#
# List Group
tt.lgroup.append({'dterm1': 'dterm1'})
tt.lgroup.append({'dterm2': 'dterm2'})
self.assertEquals('dterm2', tt.lgroup[1]['dterm2'])
self.assertEquals('dterm1', tt.lgroup[0]['dterm1'])
#
# TypedDictGroup
tt.tdgroup.foo.dterm1 = 'foo.dterm1'
self.assertEqual('foo.dterm1', tt.tdgroup.foo.dterm1)
tt.tdgroup.foo.dterm2 = 'foo.dterm2'
tt.tdgroup.baz.dterm1 = 'foo.dterm1'
#
# VarDict Group
tt.vdgroup.k1['v1'] = 'v1'
tt.vdgroup.k1.v2 = 'v2'
def test_metadata(self):
from ambry.bundle.meta import Metadata, ScalarTerm, TypedDictGroup, VarDictGroup, DictGroup, DictTerm, ListGroup
import yaml
from ambry.util import AttrDict
class TestDictTerm(DictTerm):
dterm1 = ScalarTerm()
dterm2 = ScalarTerm()
unset_term = ScalarTerm()
class TestListGroup(ListGroup):
_proto = TestDictTerm()
class TestGroup(DictGroup):
term = ScalarTerm()
term2 = ScalarTerm()
dterm = TestDictTerm()
class TestTDGroup(TypedDictGroup):
_proto = TestDictTerm()
class TestTop(Metadata):
group = TestGroup()
tdgroup = TestTDGroup()
lgroup = TestListGroup()
vdgroup = VarDictGroup()
tt = TestTop()
##
## Dict Group
tt.group.term = 'Term'
tt.group.term2 = 'Term2'
with self.assertRaises(AttributeError):
tt.group.term3 = 'Term3'
self.assertEquals('Term', tt.group.term)
self.assertEquals('Term2', tt.group.term2)
self.assertEquals('Term', tt.group['term'])
self.assertEquals(['term', 'term2', 'dterm'], tt.group.keys())
self.assertEquals([
'Term', 'Term2',
AttrDict([('dterm1', None), ('unset_term', None),
('dterm2', None)])
], tt.group.values())
##
## Dict Term
tt.group.dterm.dterm1 = 'dterm1'
tt.group.dterm.dterm2 = 'dterm2'
with self.assertRaises(AttributeError):
tt.group.dterm.dterm3 = 'dterm3'
self.assertEquals('dterm1', tt.group.dterm.dterm1)
self.assertEquals(['dterm1', 'unset_term', 'dterm2'],
tt.group.dterm.keys())
self.assertEquals(['dterm1', None, 'dterm2'], tt.group.dterm.values())
## List Group
tt.lgroup.append({'k1': 'v1'})
tt.lgroup.append({'k2': 'v2'})
self.assertEquals('v1', tt.lgroup[0]['k1'])
self.assertEquals('v2', tt.lgroup[1]['k2'])
## TypedDictGroup
tt.tdgroup.foo.dterm1 = 'foo.dterm1'
self.assertEqual('foo.dterm1', tt.tdgroup.foo.dterm1)
tt.tdgroup.foo.dterm2 = 'foo.dterm2'
tt.tdgroup.baz.dterm1 = 'foo.dterm1'
## VarDict Group
tt.vdgroup.k1['v1'] = 'v1'
tt.vdgroup.k1.v2 = 'v2'
d = dict(
about=dict(title='title',
subject='subject',
rights='rights',
summary='Summary',
tags='Foobotom'),
contact=dict(
creator=dict(name='Name', email='Email', bingo='bingo')),
# These are note part of the defined set, so aren't converted to terms
build=dict(foo='foo', bar='bar'),
partitions=[
dict(name='foo', grain='bar'),
dict(time='foo', space='bar'),
dict(
name='name',
time='time',
space='space',
grain='grain',
segment=0,
),
])
top = Top(d)
t2 = Top()
self.assertIn(('contact', 'creator', 'bingo'), top.errors)
self.assertIn('publisher', top.contact.keys())
self.assertIn('url', dict(top.contact.creator))
self.assertEqual('Email', top.contact.creator.email)
self.assertIn('name', top.partitions[0])
self.assertNotIn('space', top.partitions[0])
self.assertIn('space', top.partitions[2])
self.assertEquals('foo', top.partitions[0]['name'])
top.sources.foo.url = 'url'
top.sources.bar.description = 'description'
top = Top(yaml.load(config_str))
self.assertEquals(['foo', 'baz'], top.build.keys())
self.assertEquals('bar', top.build.foo)
self.assertEquals('bar', top.build['foo'])
self.assertEqual(6, len(top.partitions))
self.assertIn('google', top.sources.keys())
self.assertIn('yahoo', top.sources.keys())
self.assertEquals('http://yahoo.com', top.sources.yahoo.url)
#print top.write_to_dir(None)
#for (group, term, subterm),v in top.rows:
# print group, term, subterm,v
t3 = Top()
t3.load_rows(top.rows)
def test_returns_false_if_partition_is_not_indexed(self):
# to test we need just vid from the given object. So do not create partition, create partition like
# instead. It makes that test more quick.
partition = AttrDict(vid='vid1')
ret = self.backend.partition_index.is_indexed(partition)
self.assertFalse(ret)
def config_install(args, l, rc):
import yaml
import pkgutil
import os
from os.path import join, dirname
import getpass
import ambry.support
from ambry.run import ROOT_DIR, USER_DIR, CONFIG_FILE, ACCOUNTS_FILE
from ambry.util import AttrDict
user = getpass.getuser()
default_config_file = join(dirname(ambry.support.__file__),'ambry-{}.yaml'.format(args.template))
d = AttrDict().update_yaml(default_config_file)
user_config_dir = os.path.join(os.path.expanduser('~'), USER_DIR)
if user == 'root': # Root user
config_dir = ROOT_DIR
default_root = d.library.filesystem_root
elif os.getenv('VIRTUAL_ENV'): # Special case for python virtual environments
config_dir = os.path.join(os.getenv('VIRTUAL_ENV'), USER_DIR)
default_root = os.path.join(os.getenv('VIRTUAL_ENV'), 'data')
else: # Non-root user, outside of virtualenv
config_dir = user_config_dir
warn(("Installing as non-root, to '{}'\n" +
"Run as root to install for all users.").format(config_dir))
default_root = os.path.join(os.path.expanduser('~'), 'ambry')
if args.root:
default_root = args.root
if not os.path.exists(user_config_dir):
os.makedirs(user_config_dir)
if not os.path.exists(config_dir):
os.makedirs(config_dir)
if os.path.exists(os.path.join(config_dir, CONFIG_FILE)):
if args.force:
prt("File output file exists, overwriting: {}".format(config_dir))
else:
fatal("Output file {} exists. Use -f to overwrite".format(config_dir))
d['library']['filesystem_root'] = default_root
s = d.dump()
if args.prt:
prt(s.replace("{", "{{").replace("}", "}}"))
return
#Create an empty accounts file, if it does not exist
user_accounts_file =os.path.join(user_config_dir, ACCOUNTS_FILE)
if not os.path.exists(user_accounts_file):
with open(user_accounts_file, 'w') as f:
from ambry.util import random_string
d = dict(accounts=dict(
password=random_string(16),
ambry=dict(
name=None, email=None
)
)
)
prt('Writing accounts file: {}'.format(user_accounts_file))
f.write(yaml.dump(d, indent=4, default_flow_style=False))
config_file = os.path.join(config_dir, CONFIG_FILE)
with open(config_file, 'w') as f:
prt('Writing config file: {}'.format(config_file))
f.write(s)
# Make the directories.
from ..run import get_runconfig
rc = get_runconfig(config_file)
for name, v in iteritems(rc.filesystem):
dr = v.format(root=rc.library.filesystem_root)
try:
if not os.path.exists(dr):
prt("Making directory: {}".format(dr))
os.makedirs(dr)
except KeyError:
pass
def update_config(config, use_environ=True):
"""Update the configuration from environmental variables. Updates:
- config.library.database from the AMBRY_DB environmental variable.
- config.library.filesystem_root from the AMBRY_ROOT environmental variable.
- config.accounts.password from the AMBRY_PASSWORD environmental variable.
:param config: An `attrDict` of configuration information.
"""
from ambry.util import select_from_url
try:
_ = config.library
except KeyError:
config.library = AttrDict()
try:
_ = config.filesystem
except KeyError:
config.filesystem = AttrDict()
try:
_ = config.accounts
except KeyError:
config.accounts = AttrDict()
if not config.accounts.get('loaded'):
config.accounts.loaded = [None, 0]
try:
_ = config.accounts.password
except KeyError:
config.accounts.password = None
try:
_ = config.remotes
except KeyError:
config.remotes = AttrDict() # Default empty
if not config.remotes.get('loaded'):
config.remotes.loaded = [None, 0]
if use_environ:
if os.getenv(ENVAR.DB):
config.library.database = os.getenv(ENVAR.DB)
if os.getenv(ENVAR.ROOT):
config.library.filesystem_root = os.getenv(ENVAR.ROOT)
if os.getenv(ENVAR.PASSWORD):
config.accounts.password = os.getenv(ENVAR.PASSWORD)
# Move any remotes that were configured under the library to the remotes section
try:
for k, v in config.library.remotes.items():
config.remotes[k] = {'url': v}
del config.library['remotes']
except KeyError as e:
pass
# Then move any of the account entries that are linked to remotes into the remotes.
try:
for k, v in config.remotes.items():
if 'url' in v:
host = select_from_url(v['url'], 'netloc')
if host in config.accounts:
config.remotes[k].update(config.accounts[host])
del config.accounts[host]
except KeyError:
pass
# Set a default for the library database
try:
_ = config.library.database
except KeyError:
config.library.database = 'sqlite:///{root}/library.db'
# Raise exceptions on missing items
checks = [
'config.library.filesystem_root',
]
for check in checks:
try:
_ = eval(check)
except KeyError:
raise ConfigurationError(
"Configuration is missing '{}'; loaded from {} ".format(
check, config.loaded[0]))
_, config.library.database = normalize_dsn_or_dict(config.library.database)
for k, v in filesystem_defaults.items():
if k not in config.filesystem:
config.filesystem[k] = v
config.modtime = max(config.loaded[1], config.remotes.loaded[1],
config.accounts.loaded[1])
