def __init__(self, config_path=None): """ :param config_path: :return: """ from ambry.run import load_config, update_config, load_accounts from ambry.util import parse_url_to_dict, unparse_url_dict self._root = DEFAULT_ROOT # TODO: Update root from config. ensure_dir_exists(self._root) if config_path is None: import test.support config_path = os.path.join(os.path.dirname(test.support.__file__), 'test-config') self.config = load_config(config_path) self.config.update(load_accounts()) update_config(self.config, use_environ=False) assert self.config.loaded[0] == config_path + '/config.yaml' # Populate library and proto DSNs if os.environ.get('AMBRY_TEST_DB'): library_dsn = os.environ['AMBRY_TEST_DB'] else: # Derive from library.database setting. dsn = self.config.library.database if dsn.startswith('post'): # postgres case. p = parse_url_to_dict(dsn) parsed_library = dict(p, path=p['path'] ) elif dsn.startswith('sqlite'): # sqlite case p = parse_url_to_dict(dsn) parsed_library = dict(p, path=p['path'] ) library_dsn = unparse_url_dict(parsed_library) if library_dsn.startswith('post'): self._db_type = 'postgres' p = parse_url_to_dict(library_dsn) parsed_proto = dict(p, path=p['path'] + '-proto') proto_dsn = unparse_url_dict(parsed_proto) elif library_dsn.startswith('sqlite'): self._db_type = 'sqlite' p = parse_url_to_dict(library_dsn) parsed_proto = dict(p, path=p['path'] ) proto_dsn = unparse_url_dict(parsed_proto) else: raise Exception('Do not know how to process {} database.'.format(library_dsn)) self.proto_dsn = proto_dsn self.config.library.database = library_dsn
def setup_bundle(self, name, source_url=None, build_url=None, library=None): """Configure a bundle from existing sources""" from test import bundles from os.path import dirname, join from fs.opener import fsopendir import yaml if not library: library = self.library() if not source_url: source_url = 'mem://source'.format(name) if not build_url: build_url = 'mem://build'.format(name) for fs_url in (source_url, build_url): d = parse_url_to_dict(fs_url) # For persistent fs types, make sure it is empty before the test. if d['scheme'] not in ('temp', 'mem'): assert fsopendir(fs_url).isdirempty('/') test_source_fs = fsopendir(join(dirname(bundles.__file__), 'example.com', name)) config = yaml.load(test_source_fs.getcontents('bundle.yaml')) b = library.new_from_bundle_config(config) b.set_file_system(source_url=source_url, build_url=build_url) self.copy_bundle_files(test_source_fs, b.source_fs) return b
def account(self): """Return an account record, based on the host in the url""" from ambry.util import parse_url_to_dict d = parse_url_to_dict(self.url) return self._bundle.library.account(d['netloc'])
def s3(self, url, account_acessor=None, access=None, secret=None): """Setup an S3 pyfs, with account credentials, fixing an ssl matching problem""" from ambry.util.ambrys3 import AmbryS3FS from ambry.util import parse_url_to_dict import ssl pd = parse_url_to_dict(url) if account_acessor: account = account_acessor(pd['hostname']) assert account['account_id'] == pd['hostname'] aws_access_key = account['access_key'], aws_secret_key = account['secret'] else: aws_access_key = access aws_secret_key = secret assert access, url assert secret, url s3 = AmbryS3FS( bucket=pd['netloc'], prefix=pd['path'].strip('/')+'/', aws_access_key=aws_access_key, aws_secret_key=aws_secret_key, ) return s3
def account(self, url): """ Return accounts references for the given account id. :param account_id: :param accounts_password: The password for decrypting the secret :return: """ from sqlalchemy.orm.exc import NoResultFound from ambry.orm.exc import NotFoundError from ambry.util import parse_url_to_dict from ambry.orm import Account pd = parse_url_to_dict(url) # Old method of storing account information. try: act = self.database.session.query(Account).filter(Account.account_id == pd['netloc']).one() act.secret_password = self._account_password return act except NoResultFound: pass # Try the remotes. for r in self.remotes: if url.startswith(r.url): return r raise NotFoundError("Did not find account for url: '{}' ".format(url))
def account(self, url): """ Return accounts references for the given account id. :param account_id: :param accounts_password: The password for decrypting the secret :return: """ from sqlalchemy.orm.exc import NoResultFound from ambry.orm.exc import NotFoundError from ambry.util import parse_url_to_dict from ambry.orm import Account pd = parse_url_to_dict(url) # Old method of storing account information. try: act = self.database.session.query(Account).filter( Account.account_id == pd['netloc']).one() act.secret_password = self._account_password return act except NoResultFound: pass # Try the remotes. for r in self.remotes: if url.startswith(r.url): return r raise NotFoundError("Did not find account for url: '{}' ".format(url))
def s3(self, url, account_acessor=None, access=None, secret=None): """Setup an S3 pyfs, with account credentials, fixing an ssl matching problem""" from ambry.util.ambrys3 import AmbryS3FS from ambry.util import parse_url_to_dict import ssl pd = parse_url_to_dict(url) if account_acessor: account = account_acessor(pd['hostname']) assert account['account_id'] == pd['hostname'] aws_access_key = account['access_key'], aws_secret_key = account['secret'] else: aws_access_key = access aws_secret_key = secret assert access, url assert secret, url s3 = AmbryS3FS( bucket=pd['netloc'], prefix=pd['path'].strip('/') + '/', aws_access_key=aws_access_key, aws_secret_key=aws_secret_key, ) return s3
def abbrev_url(self): from ..util import parse_url_to_dict, unparse_url_dict if self.url and len(self.url) > 100: d = parse_url_to_dict(self.url) d['path'] = '/.../' + d['path'].split('/').pop() return unparse_url_dict(d) else: return self.url
def test_search_sqlite_fails(self): db_path = parse_url_to_dict(self.config.library.database)['path'] if os.path.exists(db_path): os.remove(db_path) library = self.library() # Prior to the fix, this triggers the error for r in library.search.search('foobar'): pass
def _fs_remote(self, url): from ambry.util import parse_url_to_dict d = parse_url_to_dict(url) if d['scheme'] == 's3': return self.s3(url, access=self.access, secret=self.secret) else: from fs.opener import fsopendir return fsopendir(url)
def warehouse(self, dsn=None): from ambry.library.warehouse import Warehouse if self.database.dsn.startswith('sqlite') and dsn is None: from ambry.util import parse_url_to_dict d = parse_url_to_dict(self.database.dsn) dsn = self.database.dsn.replace(os.path.basename(d['path']), 'warehouse.db') return Warehouse(self, dsn=dsn)
def __init__(self, dsn, echo=False, foreign_keys=True, engine_kwargs=None, application_prefix='ambry'): """ Initializes database. Args: dsn (str): database connect string, 'sqlite://' for example. echo (boolean): echo parameter of the create_engine. engine_kwargs (dict): parameters to pass to the create_engine method of the Sqlalchemy. """ self.dsn = dsn d = parse_url_to_dict(self.dsn) self.path = d['path'].replace('//', '/') self.driver = d['scheme'] self.engine_kwargs = engine_kwargs or {} self.Session = None self._session = None self._engine = None self._connection = None self._echo = echo self._foreign_keys = foreign_keys self._raise_on_commit = False # For debugging if self.driver in [ 'postgres', 'postgresql', 'postgresql+psycopg2', 'postgis' ]: self.driver = 'postgres' self._schema = POSTGRES_SCHEMA_NAME else: self._schema = None self.logger = logger self.library = None # Set externally when checking in in self._application_prefix = application_prefix
def setUpClass(cls): cls.dbname = os.environ.get('AMBRY_TEST_DB') or 'sqlite' config = ambry.run.load() # not cached; get_config is cls.test_dsn_key = 'test-{}'.format(cls.dbname) cls.library_test_dsn = config.get('database', {}).get(cls.test_dsn_key) cls.library_prod_dsn = config.library.database if not cls.library_test_dsn: if cls.dbname not in config.library.database: raise Exception( 'Can not run tests on {dbname} database without credentials.\n' ' HINT: Set library.database or database.test-{dbname} keys ' ' in the ~/.ambry/config.yaml' .format(dbname=cls.dbname)) if cls.dbname == 'sqlite': d = parse_url_to_dict(config.library.database) last_part = d['path'].split('/')[-1] if last_part.endswith('.db'): new_last_part = last_part.replace('.db', '_test_1k.db') else: new_last_part = last_part + '_test_1k' d['path'] = d['path'].rstrip(last_part) + new_last_part cls.library_test_dsn = unparse_url_dict(d) elif cls.dbname == 'postgres': # create test database dsn and write it to the config. parsed_url = urlparse(config.library.database) db_name = parsed_url.path.replace('/', '') test_db_name = '{}_test_{}'.format(db_name, SAFETY_POSTFIX) cls.library_test_dsn = parsed_url._replace(path=test_db_name).geturl() cls._is_postgres = cls.dbname == 'postgres' cls._is_sqlite = cls.dbname == 'sqlite' cls._delete_db_in_teardown = True
def test_dstk(self): import pprint from ambry.util import parse_url_to_dict, unparse_url_dict # Test that the services configuration works urls = [ 'http://*****:*****@localhost:5432/mydatabase?foo=bar' 'http://*****:*****@localhost/mydatabase?foo=bar' 'http://scott@localhost:5432/mydatabase?foo=bar' 'http://*****:*****@foo.bar.bingo:5432/mydatabase', unparse_url_dict(self.rc.service('dstk'))) self.assertEquals('postgres://*****:*****@foo.bar.bingo:5432/mydatabase', unparse_url_dict(self.rc.service('geocoder')))
def setUpClass(cls): cls.dbname = os.environ.get('AMBRY_TEST_DB') or 'sqlite' config = ambry.run.load() # not cached; get_config is cls.test_dsn_key = 'test-{}'.format(cls.dbname) cls.library_test_dsn = config.get('database', {}).get(cls.test_dsn_key) cls.library_prod_dsn = config.library.database if not cls.library_test_dsn: if cls.dbname not in config.library.database: raise Exception( 'Can not run tests on {dbname} database without credentials.\n' ' HINT: Set library.database or database.test-{dbname} keys ' ' in the ~/.ambry/config.yaml'.format(dbname=cls.dbname)) if cls.dbname == 'sqlite': d = parse_url_to_dict(config.library.database) last_part = d['path'].split('/')[-1] if last_part.endswith('.db'): new_last_part = last_part.replace('.db', '_test_1k.db') else: new_last_part = last_part + '_test_1k' d['path'] = d['path'].rstrip(last_part) + new_last_part cls.library_test_dsn = unparse_url_dict(d) elif cls.dbname == 'postgres': # create test database dsn and write it to the config. parsed_url = urlparse(config.library.database) db_name = parsed_url.path.replace('/', '') test_db_name = '{}_test_{}'.format(db_name, SAFETY_POSTFIX) cls.library_test_dsn = parsed_url._replace( path=test_db_name).geturl() cls._is_postgres = cls.dbname == 'postgres' cls._is_sqlite = cls.dbname == 'sqlite' cls._delete_db_in_teardown = True
def __init__(self, dsn, echo=False, foreign_keys=True, engine_kwargs=None, application_prefix='ambry'): """ Initializes database. Args: dsn (str): database connect string, 'sqlite://' for example. echo (boolean): echo parameter of the create_engine. engine_kwargs (dict): parameters to pass to the create_engine method of the Sqlalchemy. """ self.dsn = dsn d = parse_url_to_dict(self.dsn) self.path = d['path'].replace('//', '/') self.driver = d['scheme'] self.engine_kwargs = engine_kwargs or {} self.Session = None self._session = None self._engine = None self._connection = None self._echo = echo self._foreign_keys = foreign_keys self._raise_on_commit = False # For debugging if self.driver in ['postgres', 'postgresql', 'postgresql+psycopg2', 'postgis']: self.driver = 'postgres' self._schema = POSTGRES_SCHEMA_NAME else: self._schema = None self.logger = logger self.library = None # Set externally when checking in in self._application_prefix = application_prefix
def _get_connection(self): """ Returns connection to the postgres database. Returns: connection to postgres database who stores mpr data. """ if not getattr(self, '_connection', None): logger.debug( 'Creating new connection.\n dsn: {}' .format(self._dsn)) d = parse_url_to_dict(self._dsn) self._connection = psycopg2.connect( database=d['path'].strip('/'), user=d['username'], password=d['password'], port=d['port'], host=d['hostname']) # It takes some time to find the way how to get raw connection from sqlalchemy. So, # I leave the commented code. # # self._engine = create_engine(self._dsn) # self._connection = self._engine.raw_connection() # return self._connection
def test_dstk(self): import pprint from ambry.util import parse_url_to_dict, unparse_url_dict # Test that the services configuration works urls = [ 'http://*****:*****@localhost:5432/mydatabase?foo=bar' 'http://*****:*****@localhost/mydatabase?foo=bar' 'http://scott@localhost:5432/mydatabase?foo=bar' 'http://*****:*****@foo.bar.bingo:5432/mydatabase', unparse_url_dict(self.rc.service('dstk'))) self.assertEquals( 'postgres://*****:*****@foo.bar.bingo:5432/mydatabase', unparse_url_dict(self.rc.service('geocoder')))
def setup_bundle(self, name, source_url=None, build_url=None, library=None): """Configure a bundle from existing sources""" from test import bundles from os.path import dirname, join from fs.opener import fsopendir import yaml if not library: library = self.library() if not source_url: source_url = 'mem://source'.format(name) if not build_url: build_url = 'mem://build'.format(name) for fs_url in (source_url, build_url): d = parse_url_to_dict(fs_url) # For persistent fs types, make sure it is empty before the test. if d['scheme'] not in ('temp', 'mem'): assert fsopendir(fs_url).isdirempty('/') test_source_fs = fsopendir( join(dirname(bundles.__file__), 'example.com', name)) config = yaml.load(test_source_fs.getcontents('bundle.yaml')) b = library.new_from_bundle_config(config) b.set_file_system(source_url=source_url, build_url=build_url) self.copy_bundle_files(test_source_fs, b.source_fs) return b
def _get_connection(self): """ Returns connection to the postgres database. Returns: connection to postgres database who stores mpr data. """ if not getattr(self, '_connection', None): logger.debug('Creating new connection.\n dsn: {}'.format( self._dsn)) d = parse_url_to_dict(self._dsn) self._connection = psycopg2.connect(database=d['path'].strip('/'), user=d['username'], password=d['password'], port=d['port'], host=d['hostname']) # It takes some time to find the way how to get raw connection from sqlalchemy. So, # I leave the commented code. # # self._engine = create_engine(self._dsn) # self._connection = self._engine.raw_connection() # return self._connection
def db_host(self): from ambry.util import parse_url_to_dict d = parse_url_to_dict(self.db_dsn) return d['hostname']
def db_password(self): from ambry.util import parse_url_to_dict d = parse_url_to_dict(self.db_dsn) return d['password']
def __init__(self, config_path=None): """ :param config_path: :return: """ from ambry.run import load_config, update_config, load_accounts from ambry.util import parse_url_to_dict, unparse_url_dict self._root = DEFAULT_ROOT # TODO: Update root from config. ensure_dir_exists(self._root) if config_path is None: import test.support config_path = os.path.join(os.path.dirname(test.support.__file__), 'test-config') self.config = load_config(config_path) self.config.update(load_accounts()) update_config(self.config, use_environ=False) assert self.config.loaded[0] == config_path + '/config.yaml' # Populate library and proto DSNs if os.environ.get('AMBRY_TEST_DB'): library_dsn = os.environ['AMBRY_TEST_DB'] else: # Derive from library.database setting. dsn = self.config.library.database if dsn.startswith('post'): # postgres case. p = parse_url_to_dict(dsn) parsed_library = dict(p, path=p['path']) elif dsn.startswith('sqlite'): # sqlite case p = parse_url_to_dict(dsn) parsed_library = dict(p, path=p['path']) library_dsn = unparse_url_dict(parsed_library) if library_dsn.startswith('post'): self._db_type = 'postgres' p = parse_url_to_dict(library_dsn) parsed_proto = dict(p, path=p['path'] + '-proto') proto_dsn = unparse_url_dict(parsed_proto) elif library_dsn.startswith('sqlite'): self._db_type = 'sqlite' p = parse_url_to_dict(library_dsn) parsed_proto = dict(p, path=p['path']) proto_dsn = unparse_url_dict(parsed_proto) else: raise Exception( 'Do not know how to process {} database.'.format(library_dsn)) self.proto_dsn = proto_dsn self.config.library.database = library_dsn
def normalize_dsn_or_dict(d): """Clean up a database DSN, or dict version of a DSN, returning both the cleaned DSN and dict version""" if isinstance(d, dict): try: # Convert from an AttrDict to a real dict d = d.to_dict() except AttributeError: pass # Already a real dict config = d dsn = None elif isinstance(d, string_types): config = None dsn = d else: raise ConfigurationError( "Can't deal with database config '{}' type '{}' ".format( d, type(d))) if dsn: if dsn.startswith('sqlite') or dsn.startswith('spatialite'): driver, path = dsn.split(':', 1) slashes, path = path[:2], path[2:] if slashes != '//': raise ConfigurationError( "Sqlite DSNs must start with at least 2 slashes") if len(path) == 1 and path[0] == '/': raise ConfigurationError( "Sqlite DSNs can't have only 3 slashes in path") if len(path) > 1 and path[0] != '/': raise ConfigurationError( "Sqlite DSNs with a path must have 3 or 4 slashes.") path = path[1:] config = dict(server=None, username=None, password=None, driver=driver, dbname=path) else: d = parse_url_to_dict(dsn) config = dict(server=d['hostname'], dbname=d['path'].strip('/'), driver=d['scheme'], password=d.get('password', None), username=d.get('username', None)) else: up = d.get('username', '') or '' if d.get('password'): up += ':' + d.get('password', '') if up: up += '@' if up and not d.get('server'): raise ConfigurationError( "Can't construct a DSN with a username or password without a hostname" ) host_part = up + d.get('server', '') if d.get('server') else '' if d.get('dbname', False): path_part = '/' + d.get('dbname') # if d['driver'] in ('sqlite3', 'sqlite', 'spatialite'): # path_part = '/' + path_part else: path_part = '' # w/ no dbname, Sqlite should use memory, which required 2 slash. Rel dir is 3, abs dir is 4 dsn = '{}://{}{}'.format(d['driver'], host_part, path_part) return config, dsn