def check(config_file):
"""
Verify & view current configuration
"""
echo('Version:\t' + style(str(datacube.__version__), bold=True))
echo('Config files:\t' +
style(','.join(config_file.files_loaded), bold=True))
echo('Host:\t\t' + style('{}:{}'.format(
config_file.db_hostname or 'localhost', config_file.db_port or '5432'),
bold=True))
echo('Database:\t' +
style('{}'.format(config_file.db_database), bold=True))
echo('User:\t\t' + style('{}'.format(config_file.db_username), bold=True))
echo()
echo('Valid connection:\t', nl=False)
try:
# Initialise driver manager singleton
index = DriverManager(default_driver_name=None,
index=None,
local_config=config_file).index
echo(style('YES', bold=True))
for role, user, description in index.users.list_users():
if user == config_file.db_username:
echo('You have %s privileges.' %
style(role.upper(), bold=True))
except OperationalError as e:
handle_exception('Error Connecting to Database: %s', e)
except IndexSetupError as e:
handle_exception('Database not initialised: %s', e)
def __init__(self,
index,
grid_spec=None,
product=None,
driver_manager=None):
"""
Create a grid workflow tool.
Either grid_spec or product must be supplied.
:param Index index: The database index to use. This feature
will become deprecated, so `driver_manager` should be used
instead, unless a specific index DB needs to be set in the
driver manager for testing purposes.
:param GridSpec grid_spec: The grid projection and resolution
:param str product: The name of an existing product, if no grid_spec is supplied.
:param DriverManager driver_manager: The driver manager to
use. If not specified, an new manager will be created using
the index if specified, or the default configuration
otherwise.
"""
self.driver_manager = driver_manager if driver_manager else DriverManager(
index=index)
self.index = self.driver_manager.index
if grid_spec is None:
product = self.index.products.get_by_name(product)
grid_spec = product and product.grid_spec
self.grid_spec = grid_spec
def test_index_dataset():
mock_db = MockDb()
mock_index = MockIndex(mock_db)
driver_manager = DriverManager(index=mock_index)
mock_types = MockTypesResource(_EXAMPLE_DATASET_TYPE)
datasets = DatasetResource(driver_manager, mock_db, mock_types)
dataset = datasets.add(_EXAMPLE_NBAR_DATASET)
ids = {d.id for d in mock_db.dataset.values()}
assert ids == {_nbar_uuid, _ortho_uuid, _telemetry_uuid}
# Three datasets (ours and the two embedded source datasets)
assert len(mock_db.dataset) == 3
# Our three datasets should be linked together
# Nbar -> Ortho -> Telemetry
assert len(mock_db.dataset_source) == 2
assert mock_db.dataset_source == {('ortho', _nbar_uuid, _ortho_uuid),
('satellite_telemetry_data', _ortho_uuid,
_telemetry_uuid)}
# Nothing ingested, because we reported the first as already ingested.
dataset = datasets.add(_EXAMPLE_NBAR_DATASET)
assert len(mock_db.dataset) == 3
assert len(mock_db.dataset_source) == 2
ds2 = deepcopy(_EXAMPLE_NBAR_DATASET)
ds2.metadata_doc['product_type'] = 'zzzz'
with pytest.raises(DocumentMismatchError):
dataset = datasets.add(ds2)
def test_get_descriptor_some_data():
from mock import MagicMock, Mock
band_10 = MagicMock(dtype='int16', )
my_dict = {'band10': band_10}
def getitem(name):
return my_dict[name]
def setitem(name, val):
my_dict[name] = val
mock_measurements = MagicMock()
mock_measurements.__getitem__.side_effect = getitem
mock_measurements.__setitem__.side_effect = setitem
su = MagicMock()
su.storage_type.dimensions.return_value = ['t', 'x', 'y']
su.storage_type.measurements = mock_measurements
su.coordinates.items
su.storage_type.name
su.variables.values.return_value = ['t', 'x', 'y']
mock_index = PickableMock()
DriverManager(index=mock_index)
# mock_index.datasets.get_fields.return_value = dict(product=None)
mock_index.storage.search.return_value = [su]
api = API(index=mock_index)
descriptor = api.get_descriptor({})
assert descriptor == {}
def check(local_config # type: LocalConfig
):
"""
Verify & view current configuration
"""
def echo_field(name, value):
echo('{:<15}'.format(name + ':') + style(str(value), bold=True))
echo_field('Version', datacube.__version__)
echo_field('Config files', ','.join(local_config.files_loaded))
echo_field(
'Host', '{}:{}'.format(local_config.db_hostname or 'localhost',
local_config.db_port or '5432'))
echo_field('Database', local_config.db_database)
echo_field('User', local_config.db_username)
echo_field('Environment', local_config.environment)
echo()
echo('Valid connection:\t', nl=False)
try:
with DriverManager(default_driver_name=local_config.default_driver,
local_config=local_config) as driver_manager:
index = driver_manager.index
echo(style('YES', bold=True))
for role, user, description in index.users.list_users():
if user == local_config.db_username:
echo('You have %s privileges.' %
style(role.upper(), bold=True))
except OperationalError as e:
handle_exception('Error Connecting to Database: %s', e)
except IndexSetupError as e:
handle_exception('Database not initialised: %s', e)
def driver_manager(local_config, request):
timezone = request.param
with DriverManager(index=None,
local_config=local_config,
application_name='test-run',
validate_connection=False) as driver_manager:
db = driver_manager.index._db
# Drop and recreate tables so our tests have a clean db.
with db.connect() as connection:
_core.drop_db(connection._connection)
remove_dynamic_indexes()
# Disable informational messages since we're doing this on every test run.
with _increase_logging(_core._LOG) as _:
_core.ensure_db(db._engine, with_s3_tables=True)
c = db._engine.connect()
c.execute('alter database %s set timezone = %r' % (local_config.db_database, str(timezone)))
c.close()
# We don't need informational create/drop messages for every config change.
_dynamic._LOG.setLevel(logging.WARN)
yield driver_manager
def __init__(self,
index=None,
config=None,
app=None,
env=None,
driver_manager=None,
validate_connection=None):
"""
Create the interface for the query and storage access.
If no index or config is given, the default configuration is used for database connection.
:param Index index: The database index to use. This feature
will become deprecated, so `driver_manager` should be used
instead, unless a specific index DB needs to be set in the
driver manager for testing purposes.
:type index: :py:class:`datacube.index._api.Index` or None.
:param Union[LocalConfig|str] config: A config object or a path to a config file that defines the connection.
If an index is supplied, config is ignored.
:param str app: A short, alphanumeric name to identify this application.
The application name is used to track down problems with database queries, so it is strongly
advised that be used. Required if an index is not supplied, otherwise ignored.
:param str env: Name of the datacube environment to use.
ie. the section name in any config files. Defaults to 'datacube' for backwards
compatibility with old config files.
Allows you to have multiple datacube instances in one configuration, specified on load,
eg. 'dev', 'test' or 'landsat', 'modis' etc.
:param DriverManager driver_manager: The driver manager to
use. If not specified, a new manager will be created using
the index if specified, or the default configuration
otherwise.
:return: Datacube object
"""
self._to_close = None
if not driver_manager:
if not config:
config = LocalConfig.find(env=env)
# The 'config' parameter could be a string path
elif isinstance(config, (string_types, PurePath)):
config = LocalConfig.find(paths=[config], env=env)
driver_manager = DriverManager(index=index,
local_config=config,
application_name=app,
validate_connection=validate_connection)
self._to_close = driver_manager
self.driver_manager = driver_manager
self.index = self.driver_manager.index
def __init__(self, index=None, config=None, app=None, driver_manager=None):
"""
Create the interface for the query and storage access.
If no index or config is given, the default configuration is used for database connection.
:param Index index: The database index to use. This feature
will become deprecated, so `driver_manager` should be used
instead, unless a specific index DB needs to be set in the
driver manager for testing purposes.
:type index: :py:class:`datacube.index._api.Index` or None.
:param LocalConfig config: A config object or a path to a config file that defines the connection.
If an index is supplied, config is ignored.
:param str app: A short, alphanumeric name to identify this application.
The application name is used to track down problems with database queries, so it is strongly
advised that be used. Required if an index is not supplied, otherwise ignored.
:param DriverManager driver_manager: The driver manager to
use. If not specified, an new manager will be created using
the index if specified, or the default configuration
otherwise.
:return: Datacube object
"""
self._to_close = None
if index:
self.driver_manager = DriverManager(index=index)
self._to_close = self.driver_manager
elif driver_manager:
self.driver_manager = driver_manager
else:
if config is not None:
if isinstance(config, string_types):
config = LocalConfig.find([config])
self.driver_manager = DriverManager(local_config=config,
application_name=app)
else:
self.driver_manager = DriverManager(application_name=app)
self._to_close = self.driver_manager
self.index = self.driver_manager.index
def test_get_descriptor_no_data():
from mock import MagicMock, Mock
mock_index = PickableMock()
DriverManager(index=mock_index)
api = API(index=mock_index)
descriptor = api.get_descriptor({})
assert descriptor == {}
def __init__(self, config):
self.remote_host = config['remote_host']
self.remote_user = config['remote_user']
self.db_password = config['db_password']
self.remote_dir = config['remote_dir']
self.local_dir = config['local_dir']
self.replication_defns = config['replicated_data']
self.client = None
self.sftp = None
self.tunnel = None
self.remote_dc_config = None
self.remote_dc = None
self.local_index = DriverManager().index
def test_index_already_ingested_source_dataset():
mock_db = MockDb()
mock_index = MockIndex(mock_db)
driver_manager = DriverManager(index=mock_index)
mock_types = MockTypesResource(_EXAMPLE_DATASET_TYPE)
datasets = DatasetResource(driver_manager, mock_db, mock_types)
dataset = datasets.add(_EXAMPLE_NBAR_DATASET.sources['ortho'])
assert len(mock_db.dataset) == 2
assert len(mock_db.dataset_source) == 1
dataset = datasets.add(_EXAMPLE_NBAR_DATASET)
assert len(mock_db.dataset) == 3
assert len(mock_db.dataset_source) == 2
def with_driver_manager(*args, **kwargs):
ctx = click.get_current_context()
try:
with DriverManager(index=None,
local_config=ctx.obj['config_file'],
application_name=app_name or ctx.command_path,
validate_connection=expect_initialised) as driver_manager:
driver_manager.set_current_driver(ctx.obj['driver'])
ctx.obj['index'] = driver_manager.index
_LOG.debug("Driver manager ready. Connected to index: %s",
driver_manager.index)
return f(driver_manager, *args, **kwargs)
except (OperationalError, ProgrammingError) as e:
handle_exception('Error Connecting to database: %s', e)
def test_index_two_levels_already_ingested():
mock_db = MockDb()
mock_index = MockIndex(mock_db)
driver_manager = DriverManager(index=mock_index)
mock_types = MockTypesResource(_EXAMPLE_DATASET_TYPE)
datasets = DatasetResource(driver_manager, mock_db, mock_types)
dataset = datasets.add(_EXAMPLE_NBAR_DATASET.sources['ortho'].
sources['satellite_telemetry_data'])
assert len(mock_db.dataset) == 1
assert len(mock_db.dataset_source) == 0
dataset = datasets.add(_EXAMPLE_NBAR_DATASET)
assert len(mock_db.dataset) == 3
assert len(mock_db.dataset_source) == 2
def load_data(sources,
geobox,
measurements,
fuse_func=None,
dask_chunks=None,
skip_broken_datasets=False,
use_threads=False,
driver_manager=None):
"""
Load data from :meth:`group_datasets` into an :class:`xarray.Dataset`.
:param xarray.DataArray sources:
DataArray holding a list of :class:`datacube.model.Dataset`, grouped along the time dimension
:param GeoBox geobox:
A GeoBox defining the output spatial projection and resolution
:param measurements:
list of measurement dicts with keys: {'name', 'dtype', 'nodata', 'units'}
:param fuse_func:
function to merge successive arrays as an output
:param dict dask_chunks:
If provided, the data will be loaded on demand using using :class:`dask.array.Array`.
Should be a dictionary specifying the chunking size for each output dimension.
See the documentation on using `xarray with dask <http://xarray.pydata.org/en/stable/dask.html>`_
for more information.
:param bool use_threads:
Optional. If this is set to True, IO will be multi-thread.
May not work for all drivers due to locking/GIL.
Default is False.
:param DriverManager driver_manager: The driver manager to
use. If not specified, an new manager will be created using
the index if specified, or the default configuration
otherwise.
:rtype: xarray.Dataset
.. seealso:: :meth:`find_datasets` :meth:`group_datasets`
"""
if driver_manager is None:
driver_manager = DriverManager()
if use_threads and ('SharedArray' not in sys.modules
or 'pathos.threading' not in sys.modules):
use_threads = False
if dask_chunks is None:
def data_func(measurement):
if not use_threads:
data = numpy.full(sources.shape + geobox.shape,
measurement['nodata'],
dtype=measurement['dtype'])
for index, datasets in numpy.ndenumerate(sources.values):
_fuse_measurement(
data[index],
datasets,
geobox,
measurement,
fuse_func=fuse_func,
skip_broken_datasets=skip_broken_datasets,
driver_manager=driver_manager)
else:
def work_load_data(array_name, index, datasets):
data = sa.attach(array_name)
_fuse_measurement(
data[index],
datasets,
geobox,
measurement,
fuse_func=fuse_func,
skip_broken_datasets=skip_broken_datasets,
driver_manager=driver_manager)
array_name = '_'.join(
['DCCORE',
str(uuid.uuid4()),
str(os.getpid())])
sa.create(array_name,
shape=sources.shape + geobox.shape,
dtype=measurement['dtype'])
data = sa.attach(array_name)
data[:] = measurement['nodata']
pool = ThreadPool(32)
pool.map(work_load_data, repeat(array_name),
*zip(*numpy.ndenumerate(sources.values)))
sa.delete(array_name)
return data
else:
def data_func(measurement):
return _make_dask_array(sources,
geobox,
measurement,
fuse_func,
dask_chunks,
driver_manager=driver_manager)
return Datacube.create_storage(
OrderedDict((dim, sources.coords[dim]) for dim in sources.dims),
geobox, measurements, data_func, use_threads)
@author: yan073
'''
from flask import Flask
from flask import request
from flask import jsonify
from datetime import datetime
import datacube
import dcweb.status as httpstatus
import dcweb.download_util as downloadUtil
import json
from datacube.drivers.manager import DriverManager
DriverManager(default_driver_name='s3')
dc = datacube.Datacube(app='dc-example')
app = Flask(__name__)
@app.route("/run/<function>", methods=["GET"]) # list_products, list_variables
def run(function):
#app.logger.error('start running method');
if function == "list_products":
response = list_products()
return jsonify(response), httpstatus.HTTP_200_OK
elif function == "list_measurements":
response = list(dc.list_measurements().index
) #convertDataFrame( dc.list_measurements() )
return jsonify(response), httpstatus.HTTP_200_OK
return "error"