def test_expressions_transform(dc, query): bluegreen = construct_from_yaml(""" transform: expressions output: bluegreen: formula: blue + green nodata: -999 blue: blue input: product: ls8_nbar_albers measurements: [blue, green] """) measurements = bluegreen.output_measurements( {product.name: product for product in dc.index.products.get_all()}) assert 'bluegreen' in measurements with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube: mock_datacube.load_data = load_data mock_datacube.group_datasets = group_datasets data = bluegreen.load(dc, **query) assert 'bluegreen' in data assert numpy.all((data.bluegreen == -999).values) assert 'blue' in data assert numpy.all((data.blue == -999).values) assert 'green' not in data bluegreen = construct_from_yaml(""" transform: expressions output: bluegreen: formula: blue + green dtype: float32 blue: blue input: product: ls8_nbar_albers measurements: [blue, green] """) measurements = bluegreen.output_measurements( {product.name: product for product in dc.index.products.get_all()}) assert 'bluegreen' in measurements assert measurements['bluegreen'].dtype == numpy.dtype('float32') assert numpy.isnan(measurements['bluegreen'].nodata) with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube: mock_datacube.load_data = load_data mock_datacube.group_datasets = group_datasets data = bluegreen.load(dc, **query) assert 'bluegreen' in data assert numpy.all(numpy.isnan(data.bluegreen.values)) assert 'blue' in data assert numpy.all((data.blue == -999).values) assert 'green' not in data
def test_register(dc, query): class BlueGreen(Transformation): def compute(self, data): return (data.blue + data.green).to_dataset(name='bluegreen').assign_attrs(data.blue.attrs) def measurements(self, input_measurements): bluegreen = deepcopy(input_measurements['blue']) bluegreen.name = 'bluegreen' return {'bluegreen': bluegreen} resolver = deepcopy(DEFAULT_RESOLVER) resolver.register('transform', 'bluegreen', BlueGreen) bluegreen = construct_from_yaml(""" transform: bluegreen input: product: ls8_nbar_albers measurements: [blue, green] """, name_resolver=resolver) measurements = bluegreen.output_measurements({product.name: product for product in dc.index.products.get_all()}) assert 'bluegreen' in measurements with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube: mock_datacube.load_data = load_data mock_datacube.group_datasets = group_datasets data = bluegreen.load(dc, **query) assert 'bluegreen' in data
def test_explode(dc, query): collate = construct_from_yaml(""" collate: - product: ls8_nbar_albers - product: ls7_nbar_albers """) bag = collate.query(dc, **query) # three datasets in two products assert len(list(bag.contained_datasets())) == 3 bags = list(bag.explode()) # each element should contain one dataset assert len(bags) == 3 for bag in bags: assert len(list(bag.contained_datasets())) == 1 # the smaller bags should have the same structure assert 'collate' in bag.bag # there were two products (only one of them should have the single dataset) assert len(bag.bag['collate']) == 2
def test_aggregate(dc, query): aggr = construct_from_yaml(""" aggregate: mean group_by: month input: transform: to_float input: collate: - product: ls7_nbar_albers measurements: [blue] - product: ls8_nbar_albers measurements: [blue] """) measurements = aggr.output_measurements( {product.name: product for product in dc.index.products.get_all()}) assert 'blue' in measurements with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube: mock_datacube.load_data = load_data mock_datacube.group_datasets = group_datasets data = aggr.load(dc, **query) assert data.time.shape == (2, )
def test_to_float(dc, query): to_float = construct_from_yaml(""" transform: to_float input: product: ls8_nbar_albers measurements: [blue] """) with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube: mock_datacube.load_data = load_data mock_datacube.group_datasets = group_datasets data = to_float.load(dc, **query) assert numpy.all(numpy.isnan(data.blue.values)) assert data.blue.dtype == 'float32'
def test_select(dc, query): select = construct_from_yaml(""" transform: select measurement_names: - green input: product: ls8_nbar_albers measurements: [blue, green] """) with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube: mock_datacube.load_data = load_data mock_datacube.group_datasets = group_datasets data = select.load(dc, **query) assert 'green' in data assert 'blue' not in data
def test_aliases(dc, query): verde = construct_from_yaml(""" product: ls8_nbar_albers measurements: [verde] """) measurements = verde.output_measurements({product.name: product for product in dc.index.products.get_all()}) assert 'verde' in measurements assert 'green' not in measurements with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube: mock_datacube.load_data = load_data mock_datacube.group_datasets = group_datasets data = verde.load(dc, **query) assert 'verde' in data assert 'green' not in data
def test_rename(dc, query): rename = construct_from_yaml(""" transform: rename measurement_names: green: verde input: product: ls8_nbar_albers measurements: [blue, green] """) with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube: mock_datacube.load_data = load_data mock_datacube.group_datasets = group_datasets data = rename.load(dc, **query) assert 'verde' in data assert 'blue' in data assert 'green' not in data
def test_formula(dc, query): bluegreen = construct_from_yaml(""" transform: formula output: bluegreen: formula: blue + green input: product: ls8_nbar_albers measurements: [blue, green] """) measurements = bluegreen.output_measurements( {product.name: product for product in dc.index.products.get_all()}) assert 'bluegreen' in measurements with mock.patch('datacube.virtual.impl.Datacube') as mock_datacube: mock_datacube.load_data = load_data mock_datacube.group_datasets = group_datasets data = bluegreen.load(dc, **query) assert 'bluegreen' in data
def cloud_free_nbar(): return construct_from_yaml(""" collate: - transform: apply_mask mask_measurement_name: pixelquality input: &mask transform: make_mask flags: blue_saturated: false cloud_acca: no_cloud cloud_fmask: no_cloud cloud_shadow_acca: no_cloud_shadow cloud_shadow_fmask: no_cloud_shadow contiguous: true green_saturated: false nir_saturated: false red_saturated: false swir1_saturated: false swir2_saturated: false mask_measurement_name: pixelquality input: juxtapose: - product: ls8_nbar_albers measurements: ['blue', 'green'] - product: ls8_pq_albers - transform: datacube.virtual.transformations.ApplyMask mask_measurement_name: pixelquality input: <<: *mask input: juxtapose: - product: ls7_nbar_albers measurements: ['blue', 'green'] - product: ls7_pq_albers index_measurement_name: source_index """)
def test_misspelled_product(dc, query): ls8_nbar = construct_from_yaml("product: ls8_nbar") with pytest.raises(VirtualProductException): datasets = ls8_nbar.query(dc, **query)
def measurements(self, input_measurements): return { 'vegetat_veg_cat': Measurement(name='vegetat_veg_cat', dtype='float32', nodata=float('nan'), units='1') } vsvg = construct(aggregate=FC_summary, group_by='time', input=dict(product='fc')) fc_with_wofs = construct_from_yaml(""" juxtapose: - product: ls8_fc_albers - product: wofs_albers """) vegetat_veg_cat = construct(transform=FC_summary, input=fc_with_wofs) vegetat_veg_cat_data = vegetat_veg_cat.load(dc, dask_chunks={ 'x': 512, 'y': 512, 'time': -1 }, **search_terms)
xmin = bounds[0] xmax = bounds[2] ymin = bounds[3] ymax = bounds[1] crs = int(grid.crs['init'].split(':')[1]) srs = osr.SpatialReference() srs.ImportFromEPSG(crs) combined_ls_sref = construct_from_yaml(""" collate: - product: ls4_arcsi_sref_global_mangroves measurements: [green, NIR, red] - product: ls5_arcsi_sref_global_mangroves measurements: [green, NIR, red] - product: ls7_arcsi_sref_global_mangroves measurements: [green, NIR, red] - product: ls8_arcsi_sref_global_mangroves measurements: [green, NIR, red] """) def getDataset(crs, xmin, xmax, ymin, ymax): "Fetch all data for the given area." print("Fetching data...") fetch_ds = combined_ls_sref.query(dc, x=(xmin, xmax),