def test_find_config(): files = write_files({ 'base.conf': dedent("""\ [datacube] db_hostname: fakehost.test.lan """), 'override.conf': dedent("""\ [datacube] db_hostname: overridden.test.lan db_database: overridden_db """) }) # One config file config = LocalConfig.find(paths=[str(files.joinpath('base.conf'))]) assert config['db_hostname'] == 'fakehost.test.lan' # Not set: uses default assert config['db_database'] == 'datacube' # Now two config files, with the latter overriding earlier options. config = LocalConfig.find(paths=[ str(files.joinpath('base.conf')), str(files.joinpath('override.conf')) ]) assert config['db_hostname'] == 'overridden.test.lan' assert config['db_database'] == 'overridden_db'
def test_cfg_from_env(monkeypatch): def set_env(**kw): _clear_cfg_env(monkeypatch) for e, v in kw.items(): monkeypatch.setenv(e, v) set_env(DATACUBE_DB_URL='postgresql://*****:*****@host.tld:3344/db') cfg = LocalConfig.find() assert '3344' in str(cfg) assert '3344' in repr(cfg) assert cfg['db_username'] == 'uu' assert cfg['db_password'] == ' pass@' assert cfg['db_hostname'] == 'host.tld' assert cfg['db_database'] == 'db' assert cfg['db_port'] == '3344' set_env(DB_DATABASE='dc2', DB_HOSTNAME='remote.db', DB_PORT='4433', DB_USERNAME='******', DB_PASSWORD='******') cfg = LocalConfig.find() assert cfg['db_username'] == 'dcu' assert cfg['db_password'] == 'gg' assert cfg['db_hostname'] == 'remote.db' assert cfg['db_database'] == 'dc2' assert cfg['db_port'] == '4433'
def index_connect(local_config: LocalConfig = None, application_name: str = None, validate_connection: bool = True) -> Index: """ Create a Data Cube Index that can connect to a PostgreSQL server It contains all the required connection parameters, but doesn't actually check that the server is available. :param application_name: A short, alphanumeric name to identify this application. :param local_config: Config object to use. (optional) :param validate_connection: Validate database connection and schema immediately :raises datacube.index.Exceptions.IndexSetupError: """ if local_config is None: local_config = LocalConfig.find() driver_name = local_config.get('index_driver', 'default') index_driver = index_driver_by_name(driver_name) if not index_driver: raise RuntimeError( "No index driver found for %r. %s available: %s" % (driver_name, len(index_drivers()), ', '.join(index_drivers()))) return index_driver.connect_to_index( local_config, application_name=application_name, validate_connection=validate_connection)
def test_auto_config(monkeypatch, tmpdir): from pathlib import Path cfg_file = Path(str(tmpdir / "dc.cfg")) assert cfg_file.exists() is False cfg_file_name = str(cfg_file) _clear_cfg_env(monkeypatch) monkeypatch.setenv('DATACUBE_CONFIG_PATH', cfg_file_name) assert auto_config() == cfg_file_name assert cfg_file.exists() is True monkeypatch.setenv('DB_HOSTNAME', 'should-not-be-used.local') # second run should skip overwriting assert auto_config() == cfg_file_name config = LocalConfig.find(paths=cfg_file_name) assert config['db_hostname'] == '' assert config['db_database'] == 'datacube' cfg_file.unlink() assert cfg_file.exists() is False _clear_cfg_env(monkeypatch) monkeypatch.setenv('DATACUBE_CONFIG_PATH', cfg_file_name) monkeypatch.setenv('DB_HOSTNAME', 'some.db') monkeypatch.setenv('DB_USERNAME', 'user') assert auto_config() == cfg_file_name config = LocalConfig.find(paths=cfg_file_name) assert config['db_hostname'] == 'some.db' assert config['db_database'] == 'datacube' assert config['db_username'] == 'user'
def index_connect(local_config=None, application_name=None, validate_connection=True): # type: (LocalConfig, str, bool) -> Index """ Create a Data Cube Index that can connect to a PostgreSQL server It contains all the required connection parameters, but doesn't actually check that the server is available. :param application_name: A short, alphanumeric name to identify this application. :param local_config: Config object to use. :type local_config: :py:class:`datacube.config.LocalConfig`, optional :param validate_connection: Validate database connection and schema immediately :raises datacube.index.postgres._api.EnvironmentError: :rtype: datacube.index.index.Index """ if local_config is None: local_config = LocalConfig.find() index_driver = index_driver_by_name( local_config.get('index_driver', 'default')) return index_driver.connect_to_index( local_config, application_name=application_name, validate_connection=validate_connection)
def check(local_config: LocalConfig): """ Verify & view current configuration """ def echo_field(name, value): echo('{:<15}'.format(name + ':') + style(str(value), bold=True)) echo_field('Version', datacube.__version__) echo_field('Config files', ','.join(local_config.files_loaded)) echo_field( 'Host', '{}:{}'.format(local_config['db_hostname'] or 'localhost', local_config.get('db_port', None) or '5432')) echo_field('Database', local_config['db_database']) echo_field('User', local_config['db_username']) echo_field('Environment', local_config['env']) echo_field('Index Driver', local_config['index_driver']) echo() echo('Valid connection:\t', nl=False) try: index = index_connect(local_config=local_config) echo(style('YES', bold=True)) for role, user, description in index.users.list_users(): if user == local_config['db_username']: echo('You have %s privileges.' % style(role.upper(), bold=True)) except OperationalError as e: handle_exception('Error Connecting to Database: %s', e) except IndexSetupError as e: handle_exception('Database not initialised: %s', e)
def db_connect(cfg=None): """ Create database connection from datacube config. cfg: None -- use default datacube config str -- use config with a given name LocalConfig -- use loaded config object """ from datacube.config import LocalConfig import psycopg2 if isinstance(cfg, str) or cfg is None: cfg = LocalConfig.find(env=cfg) cfg_remap = dict(dbname='db_database', user='******', password='******', host='db_hostname', port='db_port') pg_cfg = {k: cfg.get(cfg_name, None) for k, cfg_name in cfg_remap.items()} return psycopg2.connect(**pg_cfg)
def test_multiple_environment_config(tmpdir): config_path = tmpdir.join('second.conf') config_path.write(""" [DEFAULT] db_username: test_user index_driver: default [default] db_hostname: db.opendatacube.test [test_alt] db_hostname: alt-db.opendatacube.test """) config_path = str(config_path) config = LocalConfig.find([config_path]) assert config['db_hostname'] == 'db.opendatacube.test' alt_config = LocalConfig.find([config_path], env='test_alt') assert alt_config['db_hostname'] == 'alt-db.opendatacube.test' # Make sure the correct config is passed through the API # Parsed config: db_url = 'postgresql://{user}@db.opendatacube.test:5432/datacube'.format( user=config['db_username']) alt_db_url = 'postgresql://{user}@alt-db.opendatacube.test:5432/datacube'.format( user=config['db_username']) with Datacube(config=config, validate_connection=False) as dc: assert str(dc.index.url) == db_url # When none specified, default environment is loaded with Datacube(config=str(config_path), validate_connection=False) as dc: assert str(dc.index.url) == db_url # When specific environment is loaded with Datacube(config=config_path, env='test_alt', validate_connection=False) as dc: assert str(dc.index.url) == alt_db_url # An environment that isn't in any config files with pytest.raises(ValueError): with Datacube(config=config_path, env='undefined-env', validate_connection=False) as dc: pass
def null_config(): """Provides a :class:`LocalConfig` configured with suitable config file paths. .. seealso:: The :func:`integration_config_paths` fixture sets up the config files. """ return LocalConfig.find(CONFIG_FILE_PATHS, env="null_driver")
def local_config(datacube_env_name): """Provides a :class:`LocalConfig` configured with suitable config file paths. .. seealso:: The :func:`integration_config_paths` fixture sets up the config files. """ return LocalConfig.find(CONFIG_FILE_PATHS, env=datacube_env_name)
def create_views(dc): try: from datacube.config import LocalConfig odc_cfg = LocalConfig.find() dbname = odc_cfg.get("db_database") except ImportError: dbname = os.environ.get("DB_DATABASE") run_sql(dc, "extent_views/create", database=dbname)
def test_search_returning(index: Index, local_config: LocalConfig, pseudo_ls8_type: DatasetType, pseudo_ls8_dataset: Dataset, ls5_dataset_w_children) -> None: assert index.datasets.count() == 4, "Expected four test datasets" # Expect one product with our one dataset. results = list( index.datasets.search_returning( ('id', 'sat_path', 'sat_row'), platform='LANDSAT_8', instrument='OLI_TIRS', )) assert len(results) == 1 id_, path_range, sat_range = results[0] assert id_ == pseudo_ls8_dataset.id # TODO: output nicer types? assert path_range == NumericRange(Decimal('116'), Decimal('116'), '[]') assert sat_range == NumericRange(Decimal('74'), Decimal('84'), '[]') results = list( index.datasets.search_returning( ( 'id', 'metadata_doc', ), platform='LANDSAT_8', instrument='OLI_TIRS', )) assert len(results) == 1 id_, document = results[0] assert id_ == pseudo_ls8_dataset.id assert document == pseudo_ls8_dataset.metadata_doc my_username = local_config.get('db_username', DEFAULT_DB_USER) # Mixture of document and native fields results = list( index.datasets.search_returning( ('id', 'creation_time', 'format', 'label'), platform='LANDSAT_8', indexed_by=my_username, )) assert len(results) == 1 id_, creation_time, format_, label = results[0] assert id_ == pseudo_ls8_dataset.id assert format_ == 'PSEUDOMD' # It's always UTC in the document expected_time = creation_time.astimezone(tz.tzutc()).replace(tzinfo=None) assert expected_time.isoformat( ) == pseudo_ls8_dataset.metadata_doc['creation_dt'] assert label == pseudo_ls8_dataset.metadata_doc['ga_label']
def test_multiple_environment_config(tmpdir): config_path = tmpdir.join('second.conf') config_path.write(""" [user] default_environment: test_default [test_default] db_hostname: db.opendatacube.test [test_alt] db_hostname: alt-db.opendatacube.test """) config_path = str(config_path) config = LocalConfig.find([config_path]) assert config.db_hostname == 'db.opendatacube.test' alt_config = LocalConfig.find([config_path], env='test_alt') assert alt_config.db_hostname == 'alt-db.opendatacube.test' # Lazily connect: they shouldn't try to connect during this test as we're not using the API args = dict(validate_connection=False) # Make sure the correct config is passed through the API # Parsed config: db_url = 'postgresql://{user}@db.opendatacube.test:5432/datacube'.format(user=config.db_username) alt_db_url = 'postgresql://{user}@alt-db.opendatacube.test:5432/datacube'.format(user=config.db_username) with Datacube(config=config, **args) as dc: assert str(dc.index.url) == db_url # When none specified, default environment is loaded with Datacube(config=str(config_path), **args) as dc: assert str(dc.index.url) == db_url # When specific environment is loaded with Datacube(config=config_path, env='test_alt', **args) as dc: assert str(dc.index.url) == alt_db_url # An environment that isn't in any config files with pytest.raises(ValueError): with Datacube(config=config_path, env='undefined-env', **args) as dc: pass
def get_config(username): config = configparser.ConfigParser() config['datacube'] = { 'db_password': settings.DATABASES['default']['PASSWORD'], 'db_connection_timeout': '60', 'db_username': settings.DATABASES['default']['USER'], 'db_database': username, 'db_hostname': settings.MASTER_NODE } return LocalConfig(config)
def get_config(username): config = configparser.ConfigParser() config['datacube'] = { 'db_password': '******', 'db_connection_timeout': '60', 'db_username': '******', 'db_database': username, 'db_hostname': settings.MASTER_NODE } return LocalConfig(config)
def from_config(cls, config=LocalConfig.find(), application_name=None, validate_db=True): app_name = cls._expand_app_name(application_name) return PostgresDb.connect( config.db_hostname, config.db_database, config.db_username, config.db_password, config.db_port, application_name=app_name, validate=validate_db )
def test_find_config(): files = util.write_files({ 'base.conf': """[datacube] db_hostname: fakehost.test.lan """, 'override.conf': """[datacube] db_hostname: overridden.test.lan db_database: overridden_db """ }) # One config file config = LocalConfig.find(paths=[str(files.joinpath('base.conf'))]) assert config.db_hostname == 'fakehost.test.lan' # Not set: uses default assert config.db_database == 'datacube' # Now two config files, with the latter overriding earlier options. config = LocalConfig.find(paths=[str(files.joinpath('base.conf')), str(files.joinpath('override.conf'))]) assert config.db_hostname == 'overridden.test.lan' assert config.db_database == 'overridden_db'
def connect(local_config=LocalConfig.find(), application_name=None, validate_connection=True): """ Connect to the index. Default Postgres implementation. :param application_name: A short, alphanumeric name to identify this application. :param local_config: Config object to use. :type local_config: :py:class:`datacube.config.LocalConfig`, optional :rtype: Index :raises: datacube.index.postgres._api.EnvironmentError """ return Index( PostgresDb.from_config(local_config, application_name=application_name, validate_db=validate_connection), local_config )
def test_get_locations(): files = util.write_files({ 'base.conf': """[locations] ls7_ortho: file:///tmp/test/ls7_ortho t_archive: file:///tmp/test/t_archive """, 'override.conf': """[locations] t_archive: file:///tmp/override """ }) config = LocalConfig.find(paths=[str(files.joinpath('base.conf'))]) assert config.location_mappings == { 'ls7_ortho': 'file:///tmp/test/ls7_ortho', 't_archive': 'file:///tmp/test/t_archive' } config = LocalConfig.find(paths=[str(files.joinpath('base.conf')), str(files.joinpath('override.conf'))]) assert config.location_mappings == { 'ls7_ortho': 'file:///tmp/test/ls7_ortho', 't_archive': 'file:///tmp/override' }
def config_from_settings(): """Create or load a Datacube configuration from the django settings""" if hasattr(settings, 'DATACUBE_CONF_PATH'): return settings.DATACUBE_CONF_PATH config = configparser.ConfigParser() config['datacube'] = { 'db_password': settings.DATABASES['default']['PASSWORD'], 'db_connection_timeout': '60', 'db_username': settings.DATABASES['default']['USER'], 'db_database': settings.DATABASES['default']['NAME'], 'db_hostname': settings.DATABASES['default']['HOST'] } return LocalConfig(config)
def from_config(cls, config=None, application_name=None, validate_connection=True): config = LocalConfig.find() if config is None else config app_name = cls._expand_app_name(application_name) return PostgresDb.create( config.db_hostname, config.db_database, config.db_username, config.db_password, config.db_port, application_name=app_name, validate=validate_connection, pool_timeout=config.db_connection_timeout )
def db_connect(cfg=None): from datacube.config import LocalConfig import psycopg2 if isinstance(cfg, str) or cfg is None: cfg = LocalConfig.find(env=cfg) cfg_remap = dict(dbname='db_database', user='******', password='******', host='db_hostname', port='db_port') pg_cfg = {k: cfg.get(cfg_name, None) for k, cfg_name in cfg_remap.items()} return psycopg2.connect(**pg_cfg)
def connect(local_config=LocalConfig.find(), application_name=None, validate_connection=True): """ Connect to the index. Default Postgres implementation. :param application_name: A short, alphanumeric name to identify this application. :param local_config: Config object to use. :type local_config: :py:class:`datacube.config.LocalConfig`, optional :rtype: Index :raises: datacube.index.postgres._api.EnvironmentError """ return Index( PostgresDb.from_config(local_config, application_name=application_name, validate_db=validate_connection), local_config)
def connect(local_config=None, application_name=None, validate_connection=True): """ Connect to the index. Default Postgres implementation. :param application_name: A short, alphanumeric name to identify this application. :param local_config: Config object to use. :type local_config: :py:class:`datacube.config.LocalConfig`, optional :param validate_connection: Validate database connection and schema immediately :rtype: Index :raises datacube.index.postgres._api.EnvironmentError: """ if local_config is None: local_config = LocalConfig.find() return Index( PostgresDb.from_config(local_config, application_name=application_name, validate_connection=validate_connection) )
def run_ingestion(ingestion_definition): """Kick off the standard system database ingestion process using a user defined configuration Args: ingestion_definition: dict representing a Data Cube ingestion def produced using the utils func. Returns: The primary key of the new dataset type. """ conf_path = os.environ.get('DATACUBE_CONFIG_PATH') index = index_connect(local_config=LocalConfig.find([conf_path])) source_type, output_type = ingest.make_output_type(index, ingestion_definition) ingestion_work.delay(output_type, source_type, ingestion_definition) index.close() return output_type.id
def run_ingestion(ingestion_definition): """Kick off the standard system database ingestion process using a user defined configuration Args: ingestion_definition: dict representing a Data Cube ingestion def produced using the utils func. Returns: The primary key of the new dataset type. """ conf_path = '/home/' + settings.LOCAL_USER + '/Datacube/NE-GeoCloud/config/.datacube.conf' index = index_connect(local_config=LocalConfig.find([conf_path])) source_type, output_type = ingest.make_output_type(index, ingestion_definition) ingestion_work.delay(output_type, source_type, ingestion_definition) index.close() return output_type.id
def inner(*args, **kwargs): obj = click.get_current_context().obj paths = obj.get("config_files", None) # If the user is overriding the defaults specific_environment = obj.get("config_environment") parsed_config = None try: parsed_config = LocalConfig.find(paths=paths, env=specific_environment) except ValueError as e: if specific_environment: raise click.ClickException( f"No datacube config found for '{specific_environment}'" ) from e elif required: raise click.ClickException( "No datacube config found") from e return fn(parsed_config, *args, **kwargs)
def ingestion_work(output_type, source_type, ingestion_definition): """Run the ingestion process for a user defined configuration Args: output_type, source_type: types produced by ingest.make_output_type ingestion_definition: dict representing a Data Cube ingestion def produced using the utils func. """ conf_path = os.environ.get('DATACUBE_CONFIG_PATH') index = index_connect(local_config=LocalConfig.find([conf_path])) tasks = ingest.create_task_list(index, output_type, None, source_type, ingestion_definition) # this is a dry run # paths = [ingest.get_filename(ingestion_definition, task['tile_index'], task['tile'].sources) for task in tasks] # ingest.check_existing_files(paths) # this actually ingests stuff successful, failed = ingest.process_tasks(index, ingestion_definition, source_type, output_type, tasks, 3200, get_executor(None, None)) index.close() return 0
def __init__(self, driver_manager, index=None, *args, **kargs): """Initialise the generic index. :param index: An index object behaving like :class:`datacube.index._api.Index` and used for testing purposes only. In the current implementation, only the `index._db` variable is used, and is passed to the index initialisation method, that should basically replace the existing DB connection with that variable. :param args: Optional positional arguments to be passed to the index on initialisation. Caution: In the current implementation all parameters get passed to all potential indexes. :param kargs: Optional keyword arguments to be passed to the index on initialisation. Caution: In the current implementation all parameters get passed to all potential indexes. """ self.logger = logging.getLogger(self.__class__.__name__) if index is None: local_config = kargs[ 'local_config'] if 'local_config' in kargs else None application_name = kargs[ 'application_name'] if 'application_name' in kargs else None validate_connection = kargs[ 'validate_connection'] if 'validate_connection' in kargs else True if local_config is None: local_config = LocalConfig.find() db = PostgresDb.from_config( local_config, application_name=application_name, validate_connection=validate_connection) else: db = index._db # pylint: disable=protected-access super(Index, self).__init__(driver_manager, db)
def read_remote_config(self): remote_config = ConfigParser() remote_config.read_string(_DEFAULT_CONF) with self.sftp.open('.datacube.conf') as fin: remote_config.read_file(fin) self.remote_dc_config = LocalConfig(remote_config)
def test_find_defaults(): config = LocalConfig.find(paths=[]) assert config.db_hostname == '' assert config.db_database == 'datacube'
def local_config(integration_config_paths): return LocalConfig.find(integration_config_paths)
def normalise_config(config): if config is None: return LocalConfig.find(env=env) if isinstance(config, string_types): return LocalConfig.find([config], env=env) return config