def __init__(self, config_path=None):
"""
:param config_path:
:return:
"""
from ambry.run import load_config, update_config, load_accounts
from ambry.util import parse_url_to_dict, unparse_url_dict
self._root = DEFAULT_ROOT
# TODO: Update root from config.
ensure_dir_exists(self._root)
if config_path is None:
import test.support
config_path = os.path.join(os.path.dirname(test.support.__file__), 'test-config')
self.config = load_config(config_path)
self.config.update(load_accounts())
update_config(self.config, use_environ=False)
assert self.config.loaded[0] == config_path + '/config.yaml'
# Populate library and proto DSNs
if os.environ.get('AMBRY_TEST_DB'):
library_dsn = os.environ['AMBRY_TEST_DB']
else:
# Derive from library.database setting.
dsn = self.config.library.database
if dsn.startswith('post'):
# postgres case.
p = parse_url_to_dict(dsn)
parsed_library = dict(p, path=p['path'] )
elif dsn.startswith('sqlite'):
# sqlite case
p = parse_url_to_dict(dsn)
parsed_library = dict(p, path=p['path'] )
library_dsn = unparse_url_dict(parsed_library)
if library_dsn.startswith('post'):
self._db_type = 'postgres'
p = parse_url_to_dict(library_dsn)
parsed_proto = dict(p, path=p['path'] + '-proto')
proto_dsn = unparse_url_dict(parsed_proto)
elif library_dsn.startswith('sqlite'):
self._db_type = 'sqlite'
p = parse_url_to_dict(library_dsn)
parsed_proto = dict(p, path=p['path'] )
proto_dsn = unparse_url_dict(parsed_proto)
else:
raise Exception('Do not know how to process {} database.'.format(library_dsn))
self.proto_dsn = proto_dsn
self.config.library.database = library_dsn
def setup_bundle(self, name, source_url=None, build_url=None, library=None):
"""Configure a bundle from existing sources"""
from test import bundles
from os.path import dirname, join
from fs.opener import fsopendir
import yaml
if not library:
library = self.library()
if not source_url:
source_url = 'mem://source'.format(name)
if not build_url:
build_url = 'mem://build'.format(name)
for fs_url in (source_url, build_url):
d = parse_url_to_dict(fs_url)
# For persistent fs types, make sure it is empty before the test.
if d['scheme'] not in ('temp', 'mem'):
assert fsopendir(fs_url).isdirempty('/')
test_source_fs = fsopendir(join(dirname(bundles.__file__), 'example.com', name))
config = yaml.load(test_source_fs.getcontents('bundle.yaml'))
b = library.new_from_bundle_config(config)
b.set_file_system(source_url=source_url, build_url=build_url)
self.copy_bundle_files(test_source_fs, b.source_fs)
return b
def account(self):
"""Return an account record, based on the host in the url"""
from ambry.util import parse_url_to_dict
d = parse_url_to_dict(self.url)
return self._bundle.library.account(d['netloc'])
def account(self):
"""Return an account record, based on the host in the url"""
from ambry.util import parse_url_to_dict
d = parse_url_to_dict(self.url)
return self._bundle.library.account(d['netloc'])
def s3(self, url, account_acessor=None, access=None, secret=None):
"""Setup an S3 pyfs, with account credentials, fixing an ssl matching problem"""
from ambry.util.ambrys3 import AmbryS3FS
from ambry.util import parse_url_to_dict
import ssl
pd = parse_url_to_dict(url)
if account_acessor:
account = account_acessor(pd['hostname'])
assert account['account_id'] == pd['hostname']
aws_access_key = account['access_key'],
aws_secret_key = account['secret']
else:
aws_access_key = access
aws_secret_key = secret
assert access, url
assert secret, url
s3 = AmbryS3FS(
bucket=pd['netloc'],
prefix=pd['path'].strip('/')+'/',
aws_access_key=aws_access_key,
aws_secret_key=aws_secret_key,
)
return s3
def account(self, url):
"""
Return accounts references for the given account id.
:param account_id:
:param accounts_password: The password for decrypting the secret
:return:
"""
from sqlalchemy.orm.exc import NoResultFound
from ambry.orm.exc import NotFoundError
from ambry.util import parse_url_to_dict
from ambry.orm import Account
pd = parse_url_to_dict(url)
# Old method of storing account information.
try:
act = self.database.session.query(Account).filter(Account.account_id == pd['netloc']).one()
act.secret_password = self._account_password
return act
except NoResultFound:
pass
# Try the remotes.
for r in self.remotes:
if url.startswith(r.url):
return r
raise NotFoundError("Did not find account for url: '{}' ".format(url))
def account(self, url):
"""
Return accounts references for the given account id.
:param account_id:
:param accounts_password: The password for decrypting the secret
:return:
"""
from sqlalchemy.orm.exc import NoResultFound
from ambry.orm.exc import NotFoundError
from ambry.util import parse_url_to_dict
from ambry.orm import Account
pd = parse_url_to_dict(url)
# Old method of storing account information.
try:
act = self.database.session.query(Account).filter(
Account.account_id == pd['netloc']).one()
act.secret_password = self._account_password
return act
except NoResultFound:
pass
# Try the remotes.
for r in self.remotes:
if url.startswith(r.url):
return r
raise NotFoundError("Did not find account for url: '{}' ".format(url))
def s3(self, url, account_acessor=None, access=None, secret=None):
"""Setup an S3 pyfs, with account credentials, fixing an ssl matching problem"""
from ambry.util.ambrys3 import AmbryS3FS
from ambry.util import parse_url_to_dict
import ssl
pd = parse_url_to_dict(url)
if account_acessor:
account = account_acessor(pd['hostname'])
assert account['account_id'] == pd['hostname']
aws_access_key = account['access_key'],
aws_secret_key = account['secret']
else:
aws_access_key = access
aws_secret_key = secret
assert access, url
assert secret, url
s3 = AmbryS3FS(
bucket=pd['netloc'],
prefix=pd['path'].strip('/') + '/',
aws_access_key=aws_access_key,
aws_secret_key=aws_secret_key,
)
return s3
def abbrev_url(self):
from ..util import parse_url_to_dict, unparse_url_dict
if self.url and len(self.url) > 100:
d = parse_url_to_dict(self.url)
d['path'] = '/.../' + d['path'].split('/').pop()
return unparse_url_dict(d)
else:
return self.url
def abbrev_url(self):
from ..util import parse_url_to_dict, unparse_url_dict
if self.url and len(self.url) > 100:
d = parse_url_to_dict(self.url)
d['path'] = '/.../' + d['path'].split('/').pop()
return unparse_url_dict(d)
else:
return self.url
def test_search_sqlite_fails(self):
db_path = parse_url_to_dict(self.config.library.database)['path']
if os.path.exists(db_path):
os.remove(db_path)
library = self.library()
# Prior to the fix, this triggers the error
for r in library.search.search('foobar'):
pass
def test_search_sqlite_fails(self):
db_path = parse_url_to_dict(self.config.library.database)['path']
if os.path.exists(db_path):
os.remove(db_path)
library = self.library()
# Prior to the fix, this triggers the error
for r in library.search.search('foobar'):
pass
def _fs_remote(self, url):
from ambry.util import parse_url_to_dict
d = parse_url_to_dict(url)
if d['scheme'] == 's3':
return self.s3(url, access=self.access, secret=self.secret)
else:
from fs.opener import fsopendir
return fsopendir(url)
def _fs_remote(self, url):
from ambry.util import parse_url_to_dict
d = parse_url_to_dict(url)
if d['scheme'] == 's3':
return self.s3(url, access=self.access, secret=self.secret)
else:
from fs.opener import fsopendir
return fsopendir(url)
def warehouse(self, dsn=None):
from ambry.library.warehouse import Warehouse
if self.database.dsn.startswith('sqlite') and dsn is None:
from ambry.util import parse_url_to_dict
d = parse_url_to_dict(self.database.dsn)
dsn = self.database.dsn.replace(os.path.basename(d['path']), 'warehouse.db')
return Warehouse(self, dsn=dsn)
def warehouse(self, dsn=None):
from ambry.library.warehouse import Warehouse
if self.database.dsn.startswith('sqlite') and dsn is None:
from ambry.util import parse_url_to_dict
d = parse_url_to_dict(self.database.dsn)
dsn = self.database.dsn.replace(os.path.basename(d['path']),
'warehouse.db')
return Warehouse(self, dsn=dsn)
def __init__(self,
dsn,
echo=False,
foreign_keys=True,
engine_kwargs=None,
application_prefix='ambry'):
""" Initializes database.
Args:
dsn (str): database connect string, 'sqlite://' for example.
echo (boolean): echo parameter of the create_engine.
engine_kwargs (dict): parameters to pass to the create_engine method of the Sqlalchemy.
"""
self.dsn = dsn
d = parse_url_to_dict(self.dsn)
self.path = d['path'].replace('//', '/')
self.driver = d['scheme']
self.engine_kwargs = engine_kwargs or {}
self.Session = None
self._session = None
self._engine = None
self._connection = None
self._echo = echo
self._foreign_keys = foreign_keys
self._raise_on_commit = False # For debugging
if self.driver in [
'postgres', 'postgresql', 'postgresql+psycopg2', 'postgis'
]:
self.driver = 'postgres'
self._schema = POSTGRES_SCHEMA_NAME
else:
self._schema = None
self.logger = logger
self.library = None # Set externally when checking in in
self._application_prefix = application_prefix
def setUpClass(cls):
cls.dbname = os.environ.get('AMBRY_TEST_DB') or 'sqlite'
config = ambry.run.load() # not cached; get_config is
cls.test_dsn_key = 'test-{}'.format(cls.dbname)
cls.library_test_dsn = config.get('database', {}).get(cls.test_dsn_key)
cls.library_prod_dsn = config.library.database
if not cls.library_test_dsn:
if cls.dbname not in config.library.database:
raise Exception(
'Can not run tests on {dbname} database without credentials.\n'
' HINT: Set library.database or database.test-{dbname} keys '
' in the ~/.ambry/config.yaml'
.format(dbname=cls.dbname))
if cls.dbname == 'sqlite':
d = parse_url_to_dict(config.library.database)
last_part = d['path'].split('/')[-1]
if last_part.endswith('.db'):
new_last_part = last_part.replace('.db', '_test_1k.db')
else:
new_last_part = last_part + '_test_1k'
d['path'] = d['path'].rstrip(last_part) + new_last_part
cls.library_test_dsn = unparse_url_dict(d)
elif cls.dbname == 'postgres':
# create test database dsn and write it to the config.
parsed_url = urlparse(config.library.database)
db_name = parsed_url.path.replace('/', '')
test_db_name = '{}_test_{}'.format(db_name, SAFETY_POSTFIX)
cls.library_test_dsn = parsed_url._replace(path=test_db_name).geturl()
cls._is_postgres = cls.dbname == 'postgres'
cls._is_sqlite = cls.dbname == 'sqlite'
cls._delete_db_in_teardown = True
def test_dstk(self):
import pprint
from ambry.util import parse_url_to_dict, unparse_url_dict
# Test that the services configuration works
urls = [
'http://*****:*****@localhost:5432/mydatabase?foo=bar'
'http://*****:*****@localhost/mydatabase?foo=bar'
'http://scott@localhost:5432/mydatabase?foo=bar'
'http://*****:*****@foo.bar.bingo:5432/mydatabase',
unparse_url_dict(self.rc.service('dstk')))
self.assertEquals('postgres://*****:*****@foo.bar.bingo:5432/mydatabase',
unparse_url_dict(self.rc.service('geocoder')))
def setUpClass(cls):
cls.dbname = os.environ.get('AMBRY_TEST_DB') or 'sqlite'
config = ambry.run.load() # not cached; get_config is
cls.test_dsn_key = 'test-{}'.format(cls.dbname)
cls.library_test_dsn = config.get('database', {}).get(cls.test_dsn_key)
cls.library_prod_dsn = config.library.database
if not cls.library_test_dsn:
if cls.dbname not in config.library.database:
raise Exception(
'Can not run tests on {dbname} database without credentials.\n'
' HINT: Set library.database or database.test-{dbname} keys '
' in the ~/.ambry/config.yaml'.format(dbname=cls.dbname))
if cls.dbname == 'sqlite':
d = parse_url_to_dict(config.library.database)
last_part = d['path'].split('/')[-1]
if last_part.endswith('.db'):
new_last_part = last_part.replace('.db', '_test_1k.db')
else:
new_last_part = last_part + '_test_1k'
d['path'] = d['path'].rstrip(last_part) + new_last_part
cls.library_test_dsn = unparse_url_dict(d)
elif cls.dbname == 'postgres':
# create test database dsn and write it to the config.
parsed_url = urlparse(config.library.database)
db_name = parsed_url.path.replace('/', '')
test_db_name = '{}_test_{}'.format(db_name, SAFETY_POSTFIX)
cls.library_test_dsn = parsed_url._replace(
path=test_db_name).geturl()
cls._is_postgres = cls.dbname == 'postgres'
cls._is_sqlite = cls.dbname == 'sqlite'
cls._delete_db_in_teardown = True
def __init__(self, dsn, echo=False, foreign_keys=True, engine_kwargs=None, application_prefix='ambry'):
""" Initializes database.
Args:
dsn (str): database connect string, 'sqlite://' for example.
echo (boolean): echo parameter of the create_engine.
engine_kwargs (dict): parameters to pass to the create_engine method of the Sqlalchemy.
"""
self.dsn = dsn
d = parse_url_to_dict(self.dsn)
self.path = d['path'].replace('//', '/')
self.driver = d['scheme']
self.engine_kwargs = engine_kwargs or {}
self.Session = None
self._session = None
self._engine = None
self._connection = None
self._echo = echo
self._foreign_keys = foreign_keys
self._raise_on_commit = False # For debugging
if self.driver in ['postgres', 'postgresql', 'postgresql+psycopg2', 'postgis']:
self.driver = 'postgres'
self._schema = POSTGRES_SCHEMA_NAME
else:
self._schema = None
self.logger = logger
self.library = None # Set externally when checking in in
self._application_prefix = application_prefix
def _get_connection(self):
""" Returns connection to the postgres database.
Returns:
connection to postgres database who stores mpr data.
"""
if not getattr(self, '_connection', None):
logger.debug(
'Creating new connection.\n dsn: {}'
.format(self._dsn))
d = parse_url_to_dict(self._dsn)
self._connection = psycopg2.connect(
database=d['path'].strip('/'), user=d['username'], password=d['password'],
port=d['port'], host=d['hostname'])
# It takes some time to find the way how to get raw connection from sqlalchemy. So,
# I leave the commented code.
#
# self._engine = create_engine(self._dsn)
# self._connection = self._engine.raw_connection()
#
return self._connection
def test_dstk(self):
import pprint
from ambry.util import parse_url_to_dict, unparse_url_dict
# Test that the services configuration works
urls = [
'http://*****:*****@localhost:5432/mydatabase?foo=bar'
'http://*****:*****@localhost/mydatabase?foo=bar'
'http://scott@localhost:5432/mydatabase?foo=bar'
'http://*****:*****@foo.bar.bingo:5432/mydatabase',
unparse_url_dict(self.rc.service('dstk')))
self.assertEquals(
'postgres://*****:*****@foo.bar.bingo:5432/mydatabase',
unparse_url_dict(self.rc.service('geocoder')))
def setup_bundle(self,
name,
source_url=None,
build_url=None,
library=None):
"""Configure a bundle from existing sources"""
from test import bundles
from os.path import dirname, join
from fs.opener import fsopendir
import yaml
if not library:
library = self.library()
if not source_url:
source_url = 'mem://source'.format(name)
if not build_url:
build_url = 'mem://build'.format(name)
for fs_url in (source_url, build_url):
d = parse_url_to_dict(fs_url)
# For persistent fs types, make sure it is empty before the test.
if d['scheme'] not in ('temp', 'mem'):
assert fsopendir(fs_url).isdirempty('/')
test_source_fs = fsopendir(
join(dirname(bundles.__file__), 'example.com', name))
config = yaml.load(test_source_fs.getcontents('bundle.yaml'))
b = library.new_from_bundle_config(config)
b.set_file_system(source_url=source_url, build_url=build_url)
self.copy_bundle_files(test_source_fs, b.source_fs)
return b
def _get_connection(self):
""" Returns connection to the postgres database.
Returns:
connection to postgres database who stores mpr data.
"""
if not getattr(self, '_connection', None):
logger.debug('Creating new connection.\n dsn: {}'.format(
self._dsn))
d = parse_url_to_dict(self._dsn)
self._connection = psycopg2.connect(database=d['path'].strip('/'),
user=d['username'],
password=d['password'],
port=d['port'],
host=d['hostname'])
# It takes some time to find the way how to get raw connection from sqlalchemy. So,
# I leave the commented code.
#
# self._engine = create_engine(self._dsn)
# self._connection = self._engine.raw_connection()
#
return self._connection
def db_host(self):
from ambry.util import parse_url_to_dict
d = parse_url_to_dict(self.db_dsn)
return d['hostname']
def db_host(self):
from ambry.util import parse_url_to_dict
d = parse_url_to_dict(self.db_dsn)
return d['hostname']
def db_password(self):
from ambry.util import parse_url_to_dict
d = parse_url_to_dict(self.db_dsn)
return d['password']
def __init__(self, config_path=None):
"""
:param config_path:
:return:
"""
from ambry.run import load_config, update_config, load_accounts
from ambry.util import parse_url_to_dict, unparse_url_dict
self._root = DEFAULT_ROOT
# TODO: Update root from config.
ensure_dir_exists(self._root)
if config_path is None:
import test.support
config_path = os.path.join(os.path.dirname(test.support.__file__),
'test-config')
self.config = load_config(config_path)
self.config.update(load_accounts())
update_config(self.config, use_environ=False)
assert self.config.loaded[0] == config_path + '/config.yaml'
# Populate library and proto DSNs
if os.environ.get('AMBRY_TEST_DB'):
library_dsn = os.environ['AMBRY_TEST_DB']
else:
# Derive from library.database setting.
dsn = self.config.library.database
if dsn.startswith('post'):
# postgres case.
p = parse_url_to_dict(dsn)
parsed_library = dict(p, path=p['path'])
elif dsn.startswith('sqlite'):
# sqlite case
p = parse_url_to_dict(dsn)
parsed_library = dict(p, path=p['path'])
library_dsn = unparse_url_dict(parsed_library)
if library_dsn.startswith('post'):
self._db_type = 'postgres'
p = parse_url_to_dict(library_dsn)
parsed_proto = dict(p, path=p['path'] + '-proto')
proto_dsn = unparse_url_dict(parsed_proto)
elif library_dsn.startswith('sqlite'):
self._db_type = 'sqlite'
p = parse_url_to_dict(library_dsn)
parsed_proto = dict(p, path=p['path'])
proto_dsn = unparse_url_dict(parsed_proto)
else:
raise Exception(
'Do not know how to process {} database.'.format(library_dsn))
self.proto_dsn = proto_dsn
self.config.library.database = library_dsn
def normalize_dsn_or_dict(d):
"""Clean up a database DSN, or dict version of a DSN, returning both the cleaned DSN and dict version"""
if isinstance(d, dict):
try:
# Convert from an AttrDict to a real dict
d = d.to_dict()
except AttributeError:
pass # Already a real dict
config = d
dsn = None
elif isinstance(d, string_types):
config = None
dsn = d
else:
raise ConfigurationError(
"Can't deal with database config '{}' type '{}' ".format(
d, type(d)))
if dsn:
if dsn.startswith('sqlite') or dsn.startswith('spatialite'):
driver, path = dsn.split(':', 1)
slashes, path = path[:2], path[2:]
if slashes != '//':
raise ConfigurationError(
"Sqlite DSNs must start with at least 2 slashes")
if len(path) == 1 and path[0] == '/':
raise ConfigurationError(
"Sqlite DSNs can't have only 3 slashes in path")
if len(path) > 1 and path[0] != '/':
raise ConfigurationError(
"Sqlite DSNs with a path must have 3 or 4 slashes.")
path = path[1:]
config = dict(server=None,
username=None,
password=None,
driver=driver,
dbname=path)
else:
d = parse_url_to_dict(dsn)
config = dict(server=d['hostname'],
dbname=d['path'].strip('/'),
driver=d['scheme'],
password=d.get('password', None),
username=d.get('username', None))
else:
up = d.get('username', '') or ''
if d.get('password'):
up += ':' + d.get('password', '')
if up:
up += '@'
if up and not d.get('server'):
raise ConfigurationError(
"Can't construct a DSN with a username or password without a hostname"
)
host_part = up + d.get('server', '') if d.get('server') else ''
if d.get('dbname', False):
path_part = '/' + d.get('dbname')
# if d['driver'] in ('sqlite3', 'sqlite', 'spatialite'):
# path_part = '/' + path_part
else:
path_part = '' # w/ no dbname, Sqlite should use memory, which required 2 slash. Rel dir is 3, abs dir is 4
dsn = '{}://{}{}'.format(d['driver'], host_part, path_part)
return config, dsn
def db_password(self):
from ambry.util import parse_url_to_dict
d = parse_url_to_dict(self.db_dsn)
return d['password']
