def test_expressions_transform(dc, query):
bluegreen = construct_from_yaml("""
transform: expressions
output:
bluegreen:
formula: blue + green
nodata: -999
blue: blue
input:
product: ls8_nbar_albers
measurements: [blue, green]
""")
measurements = bluegreen.output_measurements(
{product.name: product
for product in dc.index.products.get_all()})
assert 'bluegreen' in measurements
with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube:
mock_datacube.load_data = load_data
mock_datacube.group_datasets = group_datasets
data = bluegreen.load(dc, **query)
assert 'bluegreen' in data
assert numpy.all((data.bluegreen == -999).values)
assert 'blue' in data
assert numpy.all((data.blue == -999).values)
assert 'green' not in data
bluegreen = construct_from_yaml("""
transform: expressions
output:
bluegreen:
formula: blue + green
dtype: float32
blue: blue
input:
product: ls8_nbar_albers
measurements: [blue, green]
""")
measurements = bluegreen.output_measurements(
{product.name: product
for product in dc.index.products.get_all()})
assert 'bluegreen' in measurements
assert measurements['bluegreen'].dtype == numpy.dtype('float32')
assert numpy.isnan(measurements['bluegreen'].nodata)
with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube:
mock_datacube.load_data = load_data
mock_datacube.group_datasets = group_datasets
data = bluegreen.load(dc, **query)
assert 'bluegreen' in data
assert numpy.all(numpy.isnan(data.bluegreen.values))
assert 'blue' in data
assert numpy.all((data.blue == -999).values)
assert 'green' not in data
def test_register(dc, query):
class BlueGreen(Transformation):
def compute(self, data):
return (data.blue + data.green).to_dataset(name='bluegreen').assign_attrs(data.blue.attrs)
def measurements(self, input_measurements):
bluegreen = deepcopy(input_measurements['blue'])
bluegreen.name = 'bluegreen'
return {'bluegreen': bluegreen}
resolver = deepcopy(DEFAULT_RESOLVER)
resolver.register('transform', 'bluegreen', BlueGreen)
bluegreen = construct_from_yaml("""
transform: bluegreen
input:
product: ls8_nbar_albers
measurements: [blue, green]
""", name_resolver=resolver)
measurements = bluegreen.output_measurements({product.name: product
for product in dc.index.products.get_all()})
assert 'bluegreen' in measurements
with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube:
mock_datacube.load_data = load_data
mock_datacube.group_datasets = group_datasets
data = bluegreen.load(dc, **query)
assert 'bluegreen' in data
def test_explode(dc, query):
collate = construct_from_yaml("""
collate:
- product: ls8_nbar_albers
- product: ls7_nbar_albers
""")
bag = collate.query(dc, **query)
# three datasets in two products
assert len(list(bag.contained_datasets())) == 3
bags = list(bag.explode())
# each element should contain one dataset
assert len(bags) == 3
for bag in bags:
assert len(list(bag.contained_datasets())) == 1
# the smaller bags should have the same structure
assert 'collate' in bag.bag
# there were two products (only one of them should have the single dataset)
assert len(bag.bag['collate']) == 2
def test_aggregate(dc, query):
aggr = construct_from_yaml("""
aggregate: mean
group_by: month
input:
transform: to_float
input:
collate:
- product: ls7_nbar_albers
measurements: [blue]
- product: ls8_nbar_albers
measurements: [blue]
""")
measurements = aggr.output_measurements(
{product.name: product
for product in dc.index.products.get_all()})
assert 'blue' in measurements
with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube:
mock_datacube.load_data = load_data
mock_datacube.group_datasets = group_datasets
data = aggr.load(dc, **query)
assert data.time.shape == (2, )
def test_to_float(dc, query):
to_float = construct_from_yaml("""
transform: to_float
input:
product: ls8_nbar_albers
measurements: [blue]
""")
with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube:
mock_datacube.load_data = load_data
mock_datacube.group_datasets = group_datasets
data = to_float.load(dc, **query)
assert numpy.all(numpy.isnan(data.blue.values))
assert data.blue.dtype == 'float32'
def test_select(dc, query):
select = construct_from_yaml("""
transform: select
measurement_names:
- green
input:
product: ls8_nbar_albers
measurements: [blue, green]
""")
with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube:
mock_datacube.load_data = load_data
mock_datacube.group_datasets = group_datasets
data = select.load(dc, **query)
assert 'green' in data
assert 'blue' not in data
def test_aliases(dc, query):
verde = construct_from_yaml("""
product: ls8_nbar_albers
measurements: [verde]
""")
measurements = verde.output_measurements({product.name: product
for product in dc.index.products.get_all()})
assert 'verde' in measurements
assert 'green' not in measurements
with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube:
mock_datacube.load_data = load_data
mock_datacube.group_datasets = group_datasets
data = verde.load(dc, **query)
assert 'verde' in data
assert 'green' not in data
def test_rename(dc, query):
rename = construct_from_yaml("""
transform: rename
measurement_names:
green: verde
input:
product: ls8_nbar_albers
measurements: [blue, green]
""")
with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube:
mock_datacube.load_data = load_data
mock_datacube.group_datasets = group_datasets
data = rename.load(dc, **query)
assert 'verde' in data
assert 'blue' in data
assert 'green' not in data
def test_formula(dc, query):
bluegreen = construct_from_yaml("""
transform: formula
output:
bluegreen:
formula: blue + green
input:
product: ls8_nbar_albers
measurements: [blue, green]
""")
measurements = bluegreen.output_measurements(
{product.name: product
for product in dc.index.products.get_all()})
assert 'bluegreen' in measurements
with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube:
mock_datacube.load_data = load_data
mock_datacube.group_datasets = group_datasets
data = bluegreen.load(dc, **query)
assert 'bluegreen' in data
def cloud_free_nbar():
return construct_from_yaml("""
collate:
- transform: apply_mask
mask_measurement_name: pixelquality
input:
&mask
transform: make_mask
flags:
blue_saturated: false
cloud_acca: no_cloud
cloud_fmask: no_cloud
cloud_shadow_acca: no_cloud_shadow
cloud_shadow_fmask: no_cloud_shadow
contiguous: true
green_saturated: false
nir_saturated: false
red_saturated: false
swir1_saturated: false
swir2_saturated: false
mask_measurement_name: pixelquality
input:
juxtapose:
- product: ls8_nbar_albers
measurements: ['blue', 'green']
- product: ls8_pq_albers
- transform: datacube.virtual.transformations.ApplyMask
mask_measurement_name: pixelquality
input:
<<: *mask
input:
juxtapose:
- product: ls7_nbar_albers
measurements: ['blue', 'green']
- product: ls7_pq_albers
index_measurement_name: source_index
""")
def test_misspelled_product(dc, query):
ls8_nbar = construct_from_yaml("product: ls8_nbar")
with pytest.raises(VirtualProductException):
datasets = ls8_nbar.query(dc, **query)
def measurements(self, input_measurements):
return {
'vegetat_veg_cat':
Measurement(name='vegetat_veg_cat',
dtype='float32',
nodata=float('nan'),
units='1')
}
vsvg = construct(aggregate=FC_summary,
group_by='time',
input=dict(product='fc'))
fc_with_wofs = construct_from_yaml("""
juxtapose:
- product: ls8_fc_albers
- product: wofs_albers
""")
vegetat_veg_cat = construct(transform=FC_summary, input=fc_with_wofs)
vegetat_veg_cat_data = vegetat_veg_cat.load(dc,
dask_chunks={
'x': 512,
'y': 512,
'time': -1
},
**search_terms)
xmin = bounds[0]
xmax = bounds[2]
ymin = bounds[3]
ymax = bounds[1]
crs = int(grid.crs['init'].split(':')[1])
srs = osr.SpatialReference()
srs.ImportFromEPSG(crs)
combined_ls_sref = construct_from_yaml("""
collate:
- product: ls4_arcsi_sref_global_mangroves
measurements: [green, NIR, red]
- product: ls5_arcsi_sref_global_mangroves
measurements: [green, NIR, red]
- product: ls7_arcsi_sref_global_mangroves
measurements: [green, NIR, red]
- product: ls8_arcsi_sref_global_mangroves
measurements: [green, NIR, red]
""")
def getDataset(crs, xmin, xmax, ymin, ymax):
"Fetch all data for the given area."
print("Fetching data...")
fetch_ds = combined_ls_sref.query(dc,
x=(xmin, xmax),
