def check_get_data_subset(index):
from datetime import datetime
from datacube.api import API
g = API(index=index)
platform = 'LANDSAT_5'
product = 'nbar'
var1 = 'red'
var2 = 'nir'
data_request_descriptor = {
'platform': platform,
'product': product,
'variables': (var1, var2),
'dimensions': {
'x': {
'range': (149.07, 149.18),
'array_range': (5, 10)
},
'y': {
'range': (-35.32, -35.28),
'array_range': (5, 10)
},
'time': {
'range': (datetime(1992, 1, 1), datetime(1992, 12, 31))
}
}
}
d = g.get_data(data_request_descriptor)
assert d['arrays'][var1].shape == (1, 5, 5)
assert d['arrays'][var2].shape == (1, 5, 5)
def check_get_data_subset(index):
from datetime import datetime
from datacube.api import API
g = API(index=index)
platform = 'LANDSAT_5'
product = 'nbar'
var1 = 'red'
var2 = 'nir'
data_request_descriptor = {
'platform': platform,
'product': product,
'variables': (var1, var2),
'dimensions': {
'x': {
'range': (149.07, 149.18),
'array_range': (5, 10)
},
'y': {
'range': (-35.32, -35.28),
'array_range': (5, 10)
},
'time': {
'range': (datetime(1992, 1, 1), datetime(1992, 12, 31))
}
}
}
d = g.get_data(data_request_descriptor)
assert d['arrays'][var1].shape == (1, 5, 5)
assert d['arrays'][var2].shape == (1, 5, 5)
def test_get_descriptor_some_data():
from mock import MagicMock
band_10 = MagicMock(dtype='int16', )
my_dict = {'band10': band_10}
def getitem(name):
return my_dict[name]
def setitem(name, val):
my_dict[name] = val
mock_measurements = MagicMock()
mock_measurements.__getitem__.side_effect = getitem
mock_measurements.__setitem__.side_effect = setitem
su = MagicMock()
su.storage_type.dimensions.return_value = ['t', 'x', 'y']
su.storage_type.measurements = mock_measurements
su.coordinates.items
su.storage_type.name
su.variables.values.return_value = ['t', 'x', 'y']
mock_index = MagicMock()
# mock_index.datasets.get_fields.return_value = dict(product=None)
mock_index.storage.search.return_value = [su]
api = API(index=mock_index)
descriptor = api.get_descriptor({})
assert descriptor == {}
def test_get_descriptor_some_data():
from mock import MagicMock, Mock
band_10 = MagicMock(dtype='int16', )
my_dict = {'band10': band_10}
def getitem(name):
return my_dict[name]
def setitem(name, val):
my_dict[name] = val
mock_measurements = MagicMock()
mock_measurements.__getitem__.side_effect = getitem
mock_measurements.__setitem__.side_effect = setitem
su = MagicMock()
su.storage_type.dimensions.return_value = ['t', 'x', 'y']
su.storage_type.measurements = mock_measurements
su.coordinates.items
su.storage_type.name
su.variables.values.return_value = ['t', 'x', 'y']
mock_index = PickableMock()
DriverManager(index=mock_index)
# mock_index.datasets.get_fields.return_value = dict(product=None)
mock_index.storage.search.return_value = [su]
api = API(index=mock_index)
descriptor = api.get_descriptor({})
assert descriptor == {}
def check_open_with_api(driver_manager):
from datacube.api import API
api = API(driver_manager=driver_manager)
# fields = api.list_fields()
# assert 'product' in fields
descriptor = api.get_descriptor()
assert 'ls5_nbar_albers' in descriptor
groups = descriptor['ls5_nbar_albers']['groups']
query = {
'variables': ['blue'],
'dimensions': {
'latitude': {
'range': (-35, -36)
},
'longitude': {
'range': (149, 150)
}
}
}
data = api.get_data(query) # , dataset_groups=groups)
assert abs(data['element_sizes'][1] - ALBERS_ELEMENT_SIZE) < .0000001
assert abs(data['element_sizes'][2] - ALBERS_ELEMENT_SIZE) < .0000001
def main():
print('Instantiating API and NDexpr')
g = API()
nd = NDexpr()
print('Retrieving data from API')
# construct data request parameters for band_30 and band_40
data_request_descriptor = {
'platform': 'LANDSAT_5',
'product': 'nbar',
'variables': ('red', 'nir'),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1990, 1, 1), datetime(1990, 12, 31))
}
}
}
# get data
d1 = g.get_data(data_request_descriptor)
# construct data request parameters for PQ
pq_request_descriptor = {
'platform': 'LANDSAT_5',
'product': 'pqa',
'variables': ('pixelquality'),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1990, 1, 1), datetime(1990, 12, 31))
}
}
}
# get data
d2 = g.get_data(pq_request_descriptor)
# The following 3 lines shouldn't be done like this
# Currently done like this for the sake of the example.
b30 = d1['arrays']['red']
b40 = d1['arrays']['nir']
pq = d2['arrays']['pixelquality']
print('NDexpr demo begins here')
# perform ndvi as expressed in this language.
ndvi = nd.evaluate('((b40 - b30) / (b40 + b30))')
# perform mask on ndvi as expressed in this language.
masked_ndvi = nd.evaluate('ndvi{(pq == 32767) | (pq == 16383) | (pq == 2457)}')
print(masked_ndvi.values)
def test_get_descriptor_no_data():
mock_index = PickableMock()
api = API(index=mock_index)
descriptor = api.get_descriptor({})
assert descriptor == {}
def test_get_descriptor_no_data():
from mock import MagicMock
mock_index = MagicMock()
api = API(index=mock_index)
descriptor = api.get_descriptor({})
assert descriptor == {}
def test_get_descriptor_no_data():
from mock import MagicMock
mock_index = MagicMock()
api = API(index=mock_index)
descriptor = api.get_descriptor({})
assert descriptor == {}
def test_get_descriptor_no_data():
from mock import MagicMock, Mock
mock_index = PickableMock()
DriverManager(index=mock_index)
api = API(index=mock_index)
descriptor = api.get_descriptor({})
assert descriptor == {}
def __init__(self, api=None, index=None, driver_manager=None):
LOG.info('Initialise Analytics Module.')
self.plan = []
self.plan_dict = {}
self.api = api or API(index=index, driver_manager=driver_manager)
self.api_descriptors = {}
self.api_products = []
def __init__(self, api=None, index=None):
LOG.info('Initialise Execution Module.')
self.cache = {}
self.nd = NDexpr()
self.nd.set_ae(True)
self.api = api or API(index=index)
self.udfuncs = {}
def check_open_with_api(index):
from datacube.api import API
api = API(index=index)
# fields = api.list_fields()
# assert 'product' in fields
descriptor = api.get_descriptor()
assert 'ls5_nbar_albers' in descriptor
groups = descriptor['ls5_nbar_albers']['groups']
query = {
'variables': ['blue'],
'dimensions': {
'latitude': {'range': (-34, -35)},
'longitude': {'range': (149, 150)}}
}
data = api.get_data(query) # , dataset_groups=groups)
assert abs(data['element_sizes'][1] - ALBERS_ELEMENT_SIZE) < .0000001
assert abs(data['element_sizes'][2] - ALBERS_ELEMENT_SIZE) < .0000001
def check_get_descriptor_data(index):
from datetime import datetime
from datacube.api import API
g = API(index=index)
platform = 'LANDSAT_5'
product = 'nbar'
var1 = 'red'
var2 = 'nir'
data_request_descriptor = {
'platform': platform,
'product': product,
'variables': (var1, var2),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1992, 1, 1), datetime(1992, 12, 31))
}
}
}
d1 = g.get_descriptor(data_request_descriptor)
d2 = g.get_data(data_request_descriptor)
assert list(d1.values())[0]['result_shape'] == \
d2['size'] == \
d2['arrays'][var1].shape == \
d2['arrays'][var2].shape
assert d2['arrays'][var1].shape[0] > 0
assert d2['arrays'][var1].shape[1] > 0
assert d2['arrays'][var1].shape[2] > 0
assert d2['arrays'][var1].shape[0] > 0
assert d2['arrays'][var2].shape[1] > 0
assert d2['arrays'][var2].shape[2] > 0
def check_get_descriptor_data(index):
from datetime import datetime
from datacube.api import API
g = API(index=index)
platform = 'LANDSAT_5'
product = 'nbar'
var1 = 'red'
var2 = 'nir'
data_request_descriptor = {
'platform': platform,
'product': product,
'variables': (var1, var2),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1992, 1, 1), datetime(1992, 12, 31))
}
}
}
d1 = g.get_descriptor(data_request_descriptor)
d2 = g.get_data(data_request_descriptor)
assert list(d1.values())[0]['result_shape'] == \
d2['size'] == \
d2['arrays'][var1].shape == \
d2['arrays'][var2].shape
assert d2['arrays'][var1].shape[0] > 0
assert d2['arrays'][var1].shape[1] > 0
assert d2['arrays'][var1].shape[2] > 0
assert d2['arrays'][var1].shape[0] > 0
assert d2['arrays'][var2].shape[1] > 0
assert d2['arrays'][var2].shape[2] > 0
def check_get_descriptor_data_storage_type(index):
from datetime import datetime
from datacube.api import API
g = API(index=index)
storage_type = 'ls5_nbar_albers'
var1 = 'red'
var2 = 'nir'
data_request_descriptor = {
'storage_type': storage_type,
'variables': (var1, var2),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1992, 1, 1), datetime(1992, 12, 31))
}
}
}
d1 = g.get_descriptor(data_request_descriptor)
d2 = g.get_data(data_request_descriptor)
assert list(d1.values())[0]['result_shape'] == \
d2['size'] == \
d2['arrays'][var1].shape == \
d2['arrays'][var2].shape
assert d2['arrays'][var1].shape[0] > 0
assert d2['arrays'][var1].shape[1] > 0
assert d2['arrays'][var1].shape[2] > 0
assert d2['arrays'][var1].shape[0] > 0
assert d2['arrays'][var2].shape[1] > 0
assert d2['arrays'][var2].shape[2] > 0
def check_get_descriptor_data_storage_type(index):
from datetime import datetime
from datacube.api import API
g = API(index=index)
storage_type = 'ls5_nbar_albers'
var1 = 'red'
var2 = 'nir'
data_request_descriptor = {
'storage_type': storage_type,
'variables': (var1, var2),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1992, 1, 1), datetime(1992, 12, 31))
}
}
}
d1 = g.get_descriptor(data_request_descriptor)
d2 = g.get_data(data_request_descriptor)
assert list(d1.values())[0]['result_shape'] == \
d2['size'] == \
d2['arrays'][var1].shape == \
d2['arrays'][var2].shape
assert d2['arrays'][var1].shape[0] > 0
assert d2['arrays'][var1].shape[1] > 0
assert d2['arrays'][var1].shape[2] > 0
assert d2['arrays'][var1].shape[0] > 0
assert d2['arrays'][var2].shape[1] > 0
assert d2['arrays'][var2].shape[2] > 0
def check_get_data(index):
import numpy as np
import xarray as xr
from datetime import datetime
from datacube.api import API
g = API(index=index)
platform = 'LANDSAT_5'
product = 'nbar'
var1 = 'red'
var2 = 'nir'
data_request_descriptor = {
'platform': platform,
'product': product,
'variables': (var1, var2),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1992, 1, 1), datetime(1992, 12, 31))
}
}
}
d = g.get_data(data_request_descriptor)
assert 'dimensions' in list(d.keys())
assert 'arrays' in list(d.keys())
assert 'element_sizes' in list(d.keys())
assert 'indices' in list(d.keys())
assert 'coordinate_reference_systems' in list(d.keys())
assert 'size' in list(d.keys())
assert isinstance(d['dimensions'], list)
assert isinstance(d['arrays'], dict)
assert isinstance(d['element_sizes'], list)
assert isinstance(d['indices'], dict)
assert isinstance(d['coordinate_reference_systems'], list)
assert isinstance(d['size'], tuple)
assert len(list(d['dimensions'])) == \
len(list(d['coordinate_reference_systems'])) == \
len(list(d['element_sizes'])) == \
len(list(d['indices'])) == \
len(list(d['size']))
for key in list(d['indices'].keys()):
assert key in list(d['dimensions'])
assert var1 in list(d['arrays'].keys())
assert var2 in list(d['arrays'].keys())
for crs in d['coordinate_reference_systems']:
assert 'reference_system_definition' in crs
assert 'reference_system_unit' in crs
assert isinstance(crs['reference_system_definition'], string_types)
assert isinstance(crs['reference_system_unit'], string_types)
for dim in d['indices']:
assert isinstance(d['indices'][dim], np.ndarray)
assert isinstance(d['arrays'][var1], xr.core.dataarray.DataArray)
assert isinstance(d['arrays'][var2], xr.core.dataarray.DataArray)
assert d['arrays'][var1].shape == d['size']
assert d['arrays'][var2].shape == d['size']
assert d['arrays'][var1].name == var1
assert d['arrays'][var2].name == var2
assert len(list(d['arrays'][var1].dims)) == len(list(d['dimensions']))
assert len(list(d['arrays'][var2].dims)) == len(list(d['dimensions']))
for dim in list(d['dimensions']):
assert dim in list(d['arrays'][var1].dims)
assert dim in list(d['arrays'][var2].dims)
def check_get_descriptor(index):
from datetime import datetime
from datacube.api import API
g = API(index=index)
platform = 'LANDSAT_5'
product = 'nbar'
var1 = 'red'
var2 = 'nir'
data_request_descriptor = {
'platform': platform,
'product': product,
'variables': (var1, var2),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1992, 1, 1), datetime(1992, 12, 31))
}
}
}
d = g.get_descriptor(data_request_descriptor)
assert 'storage_units' in list(d.values())[0].keys()
assert 'dimensions' in list(d.values())[0].keys()
assert 'result_max' in list(d.values())[0].keys()
assert 'irregular_indices' in list(d.values())[0].keys()
assert 'variables' in list(d.values())[0].keys()
assert 'result_min' in list(d.values())[0].keys()
assert 'result_shape' in list(d.values())[0].keys()
assert isinstance(list(d.values())[0]['storage_units'], dict)
assert isinstance(list(d.values())[0]['dimensions'], list)
assert isinstance(list(d.values())[0]['result_max'], tuple)
assert isinstance(list(d.values())[0]['irregular_indices'], dict)
assert isinstance(list(d.values())[0]['result_min'], tuple)
assert isinstance(list(d.values())[0]['variables'], dict)
assert isinstance(list(d.values())[0]['result_shape'], tuple)
assert len(list(d.values())[0]['dimensions']) == \
len(list(d.values())[0]['dimensions']) == \
len(list(d.values())[0]['result_shape']) == \
len(list(d.values())[0]['result_max']) == \
len(list(d.values())[0]['result_min'])
for key in list(d.values())[0]['irregular_indices'].keys():
assert key in list(d.values())[0]['dimensions']
assert var1 in list(d.values())[0]['variables']
assert var2 in list(d.values())[0]['variables']
assert 'datatype_name' in list(d.values())[0]['variables'][var1].keys()
assert 'nodata_value' in list(d.values())[0]['variables'][var1].keys()
assert 'datatype_name' in list(d.values())[0]['variables'][var2].keys()
assert 'nodata_value' in list(d.values())[0]['variables'][var2].keys()
for su in list(d.values())[0]['storage_units'].values():
assert 'irregular_indices' in su
assert 'storage_max' in su
assert 'storage_min' in su
assert 'storage_path' in su
assert 'storage_shape' in su
assert isinstance(su['irregular_indices'], dict)
assert isinstance(su['storage_max'], tuple)
assert isinstance(su['storage_min'], tuple)
assert isinstance(su['storage_path'], string_types)
assert isinstance(su['storage_shape'], tuple)
def main():
print('Instantiating API and NDexpr')
g = API()
nd = NDexpr()
print('Retrieving data from API')
# construct data request parameters for band_30 and band_40
data_request_descriptor = {
'platform': 'LANDSAT_5',
'product': 'nbar',
'variables': ('red', 'nir'),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1990, 1, 1), datetime(1990, 12, 31))
}
}
}
# get data
d1 = g.get_data(data_request_descriptor)
# construct data request parameters for PQ
pq_request_descriptor = {
'platform': 'LANDSAT_5',
'product': 'pqa',
'variables': ('pixelquality'),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1990, 1, 1), datetime(1990, 12, 31))
}
}
}
# get data
d2 = g.get_data(pq_request_descriptor)
# The following 3 lines shouldn't be done like this
# Currently done like this for the sake of the example.
b30 = d1['arrays']['red']
b40 = d1['arrays']['nir']
pq = d2['arrays']['pixelquality']
print('NDexpr demo begins here')
# perform ndvi as expressed in this language.
ndvi = nd.evaluate('((b40 - b30) / (b40 + b30))')
# perform mask on ndvi as expressed in this language.
masked_ndvi = nd.evaluate(
'ndvi{(pq == 32767) | (pq == 16383) | (pq == 2457)}')
# currently dimensions are integer indices, later will be labels when
# Data Access API Interface has been finalised.
reduction_on_dim0 = nd.evaluate('median(masked_ndvi, 0)')
reduction_on_dim01 = nd.evaluate('median(masked_ndvi, 0, 1)')
reduction_on_dim012 = nd.evaluate('median(masked_ndvi, 0, 1, 2)')
print(reduction_on_dim0)
print(reduction_on_dim01)
print(reduction_on_dim012)
def dict_api(index):
"""
:type index: datacube.index._api.Index
"""
return API(index=index)
def check_get_data(index):
import numpy as np
import xarray as xr
from datetime import datetime
from datacube.api import API
g = API(index=index)
platform = 'LANDSAT_5'
product = 'nbar'
var1 = 'red'
var2 = 'nir'
data_request_descriptor = {
'platform': platform,
'product': product,
'variables': (var1, var2),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1992, 1, 1), datetime(1992, 12, 31))
}
}
}
d = g.get_data(data_request_descriptor)
assert 'dimensions' in list(d.keys())
assert 'arrays' in list(d.keys())
assert 'element_sizes' in list(d.keys())
assert 'indices' in list(d.keys())
assert 'coordinate_reference_systems' in list(d.keys())
assert 'size' in list(d.keys())
assert isinstance(d['dimensions'], list)
assert isinstance(d['arrays'], dict)
assert isinstance(d['element_sizes'], list)
assert isinstance(d['indices'], dict)
assert isinstance(d['coordinate_reference_systems'], list)
assert isinstance(d['size'], tuple)
assert len(list(d['dimensions'])) == \
len(list(d['coordinate_reference_systems'])) == \
len(list(d['element_sizes'])) == \
len(list(d['indices'])) == \
len(list(d['size']))
for key in list(d['indices'].keys()):
assert key in list(d['dimensions'])
assert var1 in list(d['arrays'].keys())
assert var2 in list(d['arrays'].keys())
for crs in d['coordinate_reference_systems']:
assert 'reference_system_definition' in crs
assert 'reference_system_unit' in crs
assert isinstance(crs['reference_system_definition'], string_types)
assert isinstance(crs['reference_system_unit'], string_types)
for dim in d['indices']:
assert isinstance(d['indices'][dim], np.ndarray)
assert isinstance(d['arrays'][var1], xr.core.dataarray.DataArray)
assert isinstance(d['arrays'][var2], xr.core.dataarray.DataArray)
assert d['arrays'][var1].shape == d['size']
assert d['arrays'][var2].shape == d['size']
assert d['arrays'][var1].name == var1
assert d['arrays'][var2].name == var2
assert len(list(d['arrays'][var1].dims)) == len(list(d['dimensions']))
assert len(list(d['arrays'][var2].dims)) == len(list(d['dimensions']))
for dim in list(d['dimensions']):
assert dim in list(d['arrays'][var1].dims)
assert dim in list(d['arrays'][var2].dims)
def check_get_descriptor(index):
from datetime import datetime
from datacube.api import API
g = API(index=index)
platform = 'LANDSAT_5'
product = 'nbar'
var1 = 'red'
var2 = 'nir'
data_request_descriptor = {
'platform': platform,
'product': product,
'variables': (var1, var2),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1992, 1, 1), datetime(1992, 12, 31))
}
}
}
d = g.get_descriptor(data_request_descriptor)
assert 'storage_units' in list(d.values())[0].keys()
assert 'dimensions' in list(d.values())[0].keys()
assert 'result_max' in list(d.values())[0].keys()
assert 'irregular_indices' in list(d.values())[0].keys()
assert 'variables' in list(d.values())[0].keys()
assert 'result_min' in list(d.values())[0].keys()
assert 'result_shape' in list(d.values())[0].keys()
assert isinstance(list(d.values())[0]['storage_units'], dict)
assert isinstance(list(d.values())[0]['dimensions'], list)
assert isinstance(list(d.values())[0]['result_max'], tuple)
assert isinstance(list(d.values())[0]['irregular_indices'], dict)
assert isinstance(list(d.values())[0]['result_min'], tuple)
assert isinstance(list(d.values())[0]['variables'], dict)
assert isinstance(list(d.values())[0]['result_shape'], tuple)
assert len(list(d.values())[0]['dimensions']) == \
len(list(d.values())[0]['dimensions']) == \
len(list(d.values())[0]['result_shape']) == \
len(list(d.values())[0]['result_max']) == \
len(list(d.values())[0]['result_min'])
for key in list(d.values())[0]['irregular_indices'].keys():
assert key in list(d.values())[0]['dimensions']
assert var1 in list(d.values())[0]['variables']
assert var2 in list(d.values())[0]['variables']
assert 'datatype_name' in list(d.values())[0]['variables'][var1].keys()
assert 'nodata_value' in list(d.values())[0]['variables'][var1].keys()
assert 'datatype_name' in list(d.values())[0]['variables'][var2].keys()
assert 'nodata_value' in list(d.values())[0]['variables'][var2].keys()
for su in list(d.values())[0]['storage_units'].values():
assert 'irregular_indices' in su
assert 'storage_max' in su
assert 'storage_min' in su
assert 'storage_path' in su
assert 'storage_shape' in su
assert isinstance(su['irregular_indices'], dict)
assert isinstance(su['storage_max'], tuple)
assert isinstance(su['storage_min'], tuple)
assert isinstance(su['storage_path'], string_types)
assert isinstance(su['storage_shape'], tuple)
