def load(context: Context, backend: Mongo, config: Config):
# Load Mongo client using configuration.
backend.dsn = config.get('backends', backend.name, 'dsn', required=True)
backend.db_name = config.get('backends', backend.name, 'db', required=True)
backend.client = pymongo.MongoClient(backend.dsn)
backend.db = backend.client[backend.db_name]
def test_update_config_from_cli():
config = Config()
config.read(env_vars=False, env_files=False, cli_args=[
'backends.default.backend=psycopg2',
'backends.new.backend=psycopg2',
])
assert config.get('backends', 'default', 'backend') == 'psycopg2'
assert config.get('backends', 'new', 'backend') == 'psycopg2'
assert config.keys('backends') == ['default', 'new']
def test_update_config_from_env():
config = Config()
config.read(env_files=False, env_vars={
'SPINTA_BACKENDS': 'default,new',
'SPINTA_BACKENDS_DEFAULT_BACKEND': 'psycopg2',
'SPINTA_BACKENDS_NEW_BACKEND': 'psycopg2',
})
assert config.get('backends', 'default', 'backend') == 'psycopg2'
assert config.get('backends', 'new', 'backend') == 'psycopg2'
assert config.keys('backends') == ['default', 'new']
def load(context: Context, config: components.Config, c: Config) -> Config:
# Load commands.
config.commands = {}
for scope in c.keys('commands'):
if scope == 'modules':
continue
config.commands[scope] = {}
for name in c.keys('commands', scope):
command = c.get('commands', scope, name, cast=importstr)
config.commands[scope][name] = command
# Load components.
config.components = {}
for group in c.keys('components'):
config.components[group] = {}
for name in c.keys('components', group):
component = c.get('components', group, name, cast=importstr)
config.components[group][name] = component
# Load exporters.
config.exporters = {}
for name in c.keys('exporters'):
exporter = c.get('exporters', name, cast=importstr)
config.exporters[name] = exporter()
# Load everything else.
config.debug = c.get('debug', default=False)
config.config_path = c.get('config_path', cast=pathlib.Path, exists=True)
config.server_url = c.get('server_url')
config.scope_prefix = c.get('scope_prefix')
config.scope_max_length = c.get('scope_max_length', cast=int)
config.default_auth_client = c.get('default_auth_client')
return config
def test_update_config_from_env_file(tmpdir):
envfile = tmpdir.join('.env')
envfile.write(
'# comment line\n'
'\n'
'SPINTA_BACKENDS=default,new\n'
'SPINTA_BACKENDS_DEFAULT_BACKEND=foo\n'
'SPINTA_BACKENDS_NEW_BACKEND=bar\n',
)
config = Config()
config.read(env_vars=False, env_files=[str(envfile)])
assert config.get('backends', 'default', 'backend') == 'foo'
assert config.get('backends', 'new', 'backend') == 'bar'
assert config.keys('backends') == ['default', 'new']
def test_custom_config_fron_environ():
config = Config()
config.read(env_files=False, env_vars={
'SPINTA_CONFIG': f'{__name__}:_TEST_CONFIG',
})
assert config.get('backends', 'custom', 'dsn') == 'config'
assert config.keys('backends') == ['default', 'custom', 'mongo']
def test_custom_config():
config = Config()
config.read(env_vars=False, env_files=False, config={
'config': [f'{__name__}:_TEST_CONFIG'],
})
assert config.get('backends', 'custom', 'dsn') == 'config'
assert config.keys('backends') == ['default', 'custom', 'mongo']
def test_udpate_config():
config = Config()
config.read(env_vars=False, env_files=False, config={
'components': {
'nodes': {
'new': 'component',
},
},
'manifests': {
'new': {
'path': 'here',
},
},
})
assert config.get('components', 'nodes', 'new') == 'component'
assert config.get('manifests', 'new', 'path') == 'here'
assert config.keys('manifests') == ['default', 'new']
def main(ctx, option):
c = Config()
c.read(cli_args=option)
load_commands(c.get('commands', 'modules', cast=list))
Context = c.get('components', 'core', 'context', cast=importstr)
context = Context()
config = context.set('config', components.Config())
store = context.set('store', Store())
commands.load(context, config, c)
commands.check(context, config)
commands.load(context, store, c)
commands.check(context, store)
ctx.ensure_object(dict)
ctx.obj['context'] = context
def test_custom_environment():
config = Config()
config.read(env_vars=False, env_files=False, config={
'env': 'testing',
'environments': {
'testing': {
'backends': {
'default': {
'dsn': 'foo',
},
'new': {
'dsn': 'bar',
},
}
}
}
})
assert config.get('backends', 'default', 'dsn') == 'foo'
assert config.get('backends', 'new', 'dsn') == 'bar'
assert config.keys('backends') == ['default', 'new']
def wait(context: Context, store: Store, config: Config):
# Wait while all backends are up.
seconds = config.get('wait', cast=int, required=True)
for backend in store.backends.values():
for i in range(1, seconds + 1):
if wait(context, backend, config):
break
time.sleep(1)
print(f"Waiting for {backend.name!r} backend %s..." % i)
else:
wait(context, backend, config, fail=True)
def load(context: Context, store: Store, config: Config) -> Store:
"""Load backends and manifests from configuration."""
# Load backends.
store.backends = {}
for name in config.keys('backends'):
Backend = config.get('backends', name, 'backend', cast=importstr)
backend = store.backends[name] = Backend()
backend.name = name
load(context, backend, config)
# Load intrnal manifest.
internal = store.internal = Manifest()
internal.name = 'internal'
internal.path = pathlib.Path(pres.resource_filename('spinta', 'manifest'))
internal.backend = store.backends['default']
load(context, internal, config)
# Load manifests
store.manifests = {}
for name in config.keys('manifests'):
manifest = store.manifests[name] = Manifest()
manifest.name = name
manifest.path = config.get('manifests',
name,
'path',
cast=pathlib.Path,
required=True)
manifest.backend = store.backends[config.get('manifests',
name,
'backend',
required=True)]
load(context, manifest, config)
if 'default' not in store.manifests:
raise Exception("'default' manifest must be set in the configuration.")
return store
def wait(context: Context, backend: PostgreSQL, config: Config, *, fail: bool = False):
dsn = config.get('backends', backend.name, 'dsn', required=True)
engine = sa.create_engine(dsn, connect_args={'connect_timeout': 0})
try:
conn = engine.connect()
except sqlalchemy.exc.OperationalError:
if fail:
raise
else:
return False
else:
conn.close()
engine.dispose()
return True
def load(context: Context, manifest: Manifest, c: Config):
config = context.get('config')
ignore = c.get('ignore', default=[], cast=list)
# Add all supported node types.
for name in config.components['nodes'].keys():
manifest.objects[name] = {}
for file in manifest.path.glob('**/*.yml'):
if is_ignored(ignore, manifest.path, file):
continue
try:
data = yaml.load(file.read_text())
except (ParserError, ScannerError) as e:
context.error(f"{file}: {e}.")
if not isinstance(data, dict):
context.error(f"{file}: expected dict got {data.__class__.__name__}.")
if 'type' not in data:
raise Exception(f"'type' is not defined in {file}.")
if data['type'] not in manifest.objects:
raise Exception(f"Unknown type {data['type']!r} in {file}.")
node = config.components['nodes'][data['type']]()
data = {
'path': file,
'parent': manifest,
'backend': manifest.backend,
**data,
}
load(context, node, data, manifest)
if node.name in manifest.objects[node.type]:
raise Exception(f"Object {node.type} with name {node.name} already exist.")
manifest.objects[node.type][node.name] = node
from spinta.components import Store
from spinta.utils.commands import load_commands
from spinta import components
from spinta.config import Config
from spinta.auth import AuthorizationServer, ResourceProtector, BearerTokenValidator
from spinta.utils.imports import importstr
logging.basicConfig(
level=logging.INFO,
format='%(levelname)s: %(message)s',
)
c = Config()
c.read()
load_commands(c.get('commands', 'modules', cast=list))
Context = c.get('components', 'core', 'context', cast=importstr)
context = Context()
config = context.set('config', components.Config())
store = context.set('store', Store())
load(context, config, c)
check(context, config)
load(context, store, c)
check(context, store)
wait(context, store, c)
prepare(context, store.internal)
prepare(context, store)
from spinta.api import set_context
from spinta.commands import load, wait, prepare, check
from spinta.components import Context, Store
from spinta.utils.commands import load_commands
from spinta import components
from spinta.config import Config
logging.basicConfig(
level=logging.INFO,
format='%(levelname)s: %(message)s',
)
c = Config()
c.read()
load_commands(c.get('commands', 'modules', cast=list))
context = Context()
config = context.set('config', components.Config())
store = context.set('store', Store())
load(context, config, c)
load(context, store, c)
check(context, store)
wait(context, store, c)
prepare(context, store.internal)
prepare(context, store)
set_context(context)
def load(context: Context, backend: PostgreSQL, config: Config):
backend.dsn = config.get('backends', backend.name, 'dsn', required=True)
backend.engine = sa.create_engine(backend.dsn, echo=False)
backend.schema = sa.MetaData(backend.engine)
backend.tables = {}