def test_no_epsg(self): crs = rasterio.crs.CRS() img_path = join(self.tmp_dir, 'tmp.tif') height = 100 width = 100 nb_channels = 3 with rasterio.open( img_path, 'w', driver='GTiff', height=height, width=width, count=nb_channels, dtype=np.uint8, crs=crs) as img_dataset: im = np.zeros((height, width, nb_channels)).astype(np.uint8) for channel in range(nb_channels): img_dataset.write(im[:, :, channel], channel + 1) try: config = RasterioSourceConfig(uris=[img_path]) config.build(tmp_dir=self.tmp_dir) except Exception: self.fail( 'Creating RasterioSource with CRS with no EPSG attribute ' 'raised an exception when it should not have.')
def test_mask(self): # make geotiff filled with ones and zeros and mask the whole image img_path = join(self.tmp_dir, 'tmp.tif') height = 100 width = 100 nb_channels = 3 with rasterio.open( img_path, 'w', driver='GTiff', height=height, width=width, count=nb_channels, dtype=np.uint8) as img_dataset: im = np.random.randint(0, 2, (height, width, nb_channels)).astype( np.uint8) for channel in range(nb_channels): img_dataset.write(im[:, :, channel], channel + 1) img_dataset.write_mask(np.zeros(im.shape[0:2]).astype(np.bool)) config = RasterioSourceConfig(uris=[img_path]) source = config.build(tmp_dir=self.tmp_dir) with source.activate(): out_chip = source.get_image_array() expected_out_chip = np.zeros((height, width, nb_channels)) np.testing.assert_equal(out_chip, expected_out_chip)
def test_nonidentical_extents_and_resolutions(self): img_path_1 = data_file_path( 'multi_raster_source/const_100_600x600.tiff') img_path_2 = data_file_path('multi_raster_source/const_175_60x60.tiff') img_path_3 = data_file_path('multi_raster_source/const_250_6x6.tiff') source_1 = RasterioSourceConfig(uris=[img_path_1], channel_order=[0]) source_2 = RasterioSourceConfig(uris=[img_path_2], channel_order=[0]) source_3 = RasterioSourceConfig(uris=[img_path_3], channel_order=[0]) cfg = MultiRasterSourceConfig(raster_sources=[ SubRasterSourceConfig(raster_source=source_1, target_channels=[0]), SubRasterSourceConfig(raster_source=source_2, target_channels=[1]), SubRasterSourceConfig(raster_source=source_3, target_channels=[2]) ]) rs = cfg.build(tmp_dir=self.tmp_dir) with rs.activate(): for get_chip_fn in [rs._get_chip, rs.get_chip]: ch_1_only = get_chip_fn(Box(0, 0, 10, 10)) self.assertEqual(tuple(ch_1_only.reshape(-1, 3).mean(axis=0)), (100, 0, 0)) ch_2_only = get_chip_fn(Box(0, 600, 10, 600 + 10)) self.assertEqual(tuple(ch_2_only.reshape(-1, 3).mean(axis=0)), (0, 175, 0)) ch_3_only = get_chip_fn(Box(600, 0, 600 + 10, 10)) self.assertEqual(tuple(ch_3_only.reshape(-1, 3).mean(axis=0)), (0, 0, 250)) full_img = get_chip_fn(Box(0, 0, 600, 600)) self.assertEqual(set(np.unique(full_img[..., 0])), {100}) self.assertEqual(set(np.unique(full_img[..., 1])), {0, 175}) self.assertEqual(set(np.unique(full_img[..., 2])), {0, 250})
def test_gets_raw_chip(self): img_path = data_file_path('small-rgb-tile.tif') channel_order = [0, 1] config = RasterioSourceConfig( uris=[img_path], channel_order=channel_order) source = config.build(tmp_dir=self.tmp_dir) with source.activate(): out_chip = source.get_raw_image_array() self.assertEqual(out_chip.shape[2], 3)
def test_channel_order_error(self): img_path = join(self.tmp_dir, 'img.tif') chip = np.ones((2, 2, 3)).astype(np.uint8) chip[:, :, :] *= np.array([0, 1, 2]).astype(np.uint8) save_img(chip, img_path) channel_order = [3, 1, 0] with self.assertRaises(ChannelOrderError): config = RasterioSourceConfig( uris=[img_path], channel_order=channel_order) config.build(tmp_dir=self.tmp_dir)
def make_cfg(img_path='small-rgb-tile.tif', **kwargs): img_path = data_file_path(img_path) r_source = RasterioSourceConfig(uris=[img_path], channel_order=[0]) g_source = RasterioSourceConfig(uris=[img_path], channel_order=[1]) b_source = RasterioSourceConfig(uris=[img_path], channel_order=[2]) cfg = MultiRasterSourceConfig(raster_sources=[ SubRasterSourceConfig(raster_source=r_source, target_channels=[0]), SubRasterSourceConfig(raster_source=g_source, target_channels=[1]), SubRasterSourceConfig(raster_source=b_source, target_channels=[2]) ], **kwargs) return cfg
def test_extent(self): img_path = data_file_path('small-rgb-tile.tif') cfg = RasterioSourceConfig(uris=[img_path]) rs = cfg.build(tmp_dir=self.tmp_dir) extent = rs.get_extent() h, w = extent.get_height(), extent.get_width() ymin, xmin, ymax, xmax = extent self.assertEqual(h, 256) self.assertEqual(w, 256) self.assertEqual(ymin, 0) self.assertEqual(xmin, 0) self.assertEqual(ymax, 256) self.assertEqual(xmax, 256)
def test_get_chip(self): # create a 3-channel raster from a 1-channel raster # using a ReclassTransformer to give each channel a different value # (100, 175, and 250 respectively) path = data_file_path('multi_raster_source/const_100_600x600.tiff') source_1 = RasterioSourceConfig(uris=[path], channel_order=[0]) source_2 = RasterioSourceConfig( uris=[path], channel_order=[0], transformers=[ReclassTransformerConfig(mapping={100: 175})]) source_3 = RasterioSourceConfig( uris=[path], channel_order=[0], transformers=[ReclassTransformerConfig(mapping={100: 250})]) cfg = MultiRasterSourceConfig(raster_sources=[ SubRasterSourceConfig(raster_source=source_1, target_channels=[0]), SubRasterSourceConfig(raster_source=source_2, target_channels=[1]), SubRasterSourceConfig(raster_source=source_3, target_channels=[2]) ], channel_order=[2, 1, 0], transformers=[ ReclassTransformerConfig(mapping={ 100: 10, 175: 17, 250: 25 }) ]) rs = cfg.build(tmp_dir=self.tmp_dir) with rs.activate(): window = Box(0, 0, 100, 100) # sub transformers and channel_order applied sub_chips = rs._get_sub_chips(window, raw=False) self.assertEqual(tuple(c.mean() for c in sub_chips), (100, 175, 250)) # sub transformers, channel_order, and transformer applied chip = rs.get_chip(window) self.assertEqual(tuple(chip.reshape(-1, 3).mean(axis=0)), (25, 17, 10)) # none of sub transformers, channel_order, and transformer applied sub_chips = rs._get_sub_chips(window, raw=True) self.assertEqual(tuple(c.mean() for c in sub_chips), (100, 100, 100)) chip = rs._get_chip(window) self.assertEqual(tuple(chip.reshape(-1, 3).mean(axis=0)), (100, 100, 100))
def test_accounts_for_aoi(self): class_config = ClassConfig(names=['car', 'building', 'background']) label_source_uri = data_file_path('evaluator/cc-label-filtered.json') label_source_cfg = ChipClassificationLabelSourceConfig( vector_source=GeoJSONVectorSourceConfig( uri=label_source_uri, default_class_id=None)) label_store_uri = data_file_path('evaluator/cc-label-full.json') label_store_cfg = ChipClassificationGeoJSONStoreConfig( uri=label_store_uri) raster_source_uri = data_file_path('evaluator/cc-label-img-blank.tif') raster_source_cfg = RasterioSourceConfig(uris=[raster_source_uri]) aoi_uri = data_file_path('evaluator/cc-label-aoi.json') s = SceneConfig( id='test', raster_source=raster_source_cfg, label_source=label_source_cfg, label_store=label_store_cfg, aoi_uris=[aoi_uri]) with rv_config.get_tmp_dir() as tmp_dir: scene = s.build(class_config, tmp_dir) output_uri = os.path.join(tmp_dir, 'eval.json') evaluator = ChipClassificationEvaluatorConfig( output_uri=output_uri).build(class_config) evaluator.process([scene], tmp_dir) overall = file_to_json(output_uri)['overall'] for item in overall: self.assertEqual(item['f1'], 1.0)
def hrefs_to_sceneconfig( imagery: str, labels: Optional[str], name: str, class_id_filter_dict: Dict[int, str], extent_crop: Optional[CropOffsets] = None) -> SceneConfig: transformers = [CastTransformerConfig(to_dtype='float16')] image_source = RasterioSourceConfig( channel_order=list(range(224)), uris=[imagery], allow_streaming=True, transformers=transformers, extent_crop=extent_crop, ) label_vector_source = GeoJSONVectorSourceConfig( uri=labels, class_id_to_filter=class_id_filter_dict, default_class_id=0xff) label_raster_source = RasterizedSourceConfig( vector_source=label_vector_source, rasterizer_config=RasterizerConfig(background_class_id=0, all_touched=True)) label_source = SemanticSegmentationLabelSourceConfig( raster_source=label_raster_source) return SceneConfig(id=name, raster_source=image_source, label_source=label_source)
def test_uses_channel_order(self): img_path = join(self.tmp_dir, 'img.tif') chip = np.ones((2, 2, 4)).astype(np.uint8) chip[:, :, :] *= np.array([0, 1, 2, 3]).astype(np.uint8) save_img(chip, img_path) channel_order = [0, 1, 2] config = RasterioSourceConfig( uris=[img_path], channel_order=channel_order) source = config.build(tmp_dir=self.tmp_dir) with source.activate(): out_chip = source.get_image_array() expected_out_chip = np.ones((2, 2, 3)).astype(np.uint8) expected_out_chip[:, :, :] *= np.array([0, 1, 2]).astype(np.uint8) np.testing.assert_equal(out_chip, expected_out_chip)
def test_extent_crop(self): f = 1 / 4 img_path = data_file_path('small-rgb-tile.tif') cfg_crop = RasterioSourceConfig( uris=[img_path], extent_crop=(f, f, f, f)) rs_crop = cfg_crop.build(tmp_dir=self.tmp_dir) # test extent box extent_crop = rs_crop.get_extent() self.assertEqual(extent_crop.ymin, 64) self.assertEqual(extent_crop.xmin, 64) self.assertEqual(extent_crop.ymax, 192) self.assertEqual(extent_crop.xmax, 192) # test windows windows = extent_crop.get_windows(64, 64) self.assertEqual(windows[0].ymin, 64) self.assertEqual(windows[0].xmin, 64) self.assertEqual(windows[-1].ymax, 192) self.assertEqual(windows[-1].xmax, 192) # test CropOffsets class cfg_crop = RasterioSourceConfig( uris=[img_path], extent_crop=CropOffsets(skip_top=.5, skip_right=.5)) rs_crop = cfg_crop.build(tmp_dir=self.tmp_dir) extent_crop = rs_crop.get_extent() self.assertEqual(extent_crop.ymin, 128) self.assertEqual(extent_crop.xmin, 0) self.assertEqual(extent_crop.ymax, 256) self.assertEqual(extent_crop.xmax, 128) # test validation extent_crop = CropOffsets(skip_top=.5, skip_bottom=.5) self.assertRaises( ValidationError, lambda: RasterioSourceConfig(uris=[img_path], extent_crop=extent_crop)) extent_crop = CropOffsets(skip_left=.5, skip_right=.5) self.assertRaises( ValidationError, lambda: RasterioSourceConfig(uris=[img_path], extent_crop=extent_crop)) # test extent_crop=None try: _ = RasterioSourceConfig(uris=[img_path], extent_crop=None) # noqa except Exception: self.fail('extent_crop=None caused an error.')
def test_rgb_class_config_null_class(self): raster_source_cfg = RasterioSourceConfig(uris=['/abc/def.tif']) rgb_class_config = ClassConfig(names=['a'], colors=['#010101']) cfg = SemanticSegmentationLabelSourceConfig( raster_source=raster_source_cfg, rgb_class_config=rgb_class_config) cfg.update() self.assertEqual(len(cfg.rgb_class_config), 2) self.assertIn('null', cfg.rgb_class_config.names)
def make_scene(id, img_path, label_path): raster_source = RasterioSourceConfig(channel_order=[0, 1, 2], uris=[img_path]) label_source = SemanticSegmentationLabelSourceConfig( rgb_class_config=class_config, raster_source=RasterioSourceConfig(uris=[label_path])) label_store = SemanticSegmentationLabelStoreConfig( rgb=True, vector_output=[ PolygonVectorOutputConfig(class_id=0), BuildingVectorOutputConfig(class_id=1) ]) return SceneConfig(id=id, raster_source=raster_source, label_source=label_source, label_store=label_store)
def test_shift_x(self): # Specially-engineered image w/ one meter per pixel resolution # in the x direction. img_path = data_file_path('ones.tif') channel_order = [0] config = RasterioSourceConfig(uris=[img_path], channel_order=channel_order, x_shift=1.0, y_shift=0.0) source = config.build(tmp_dir=self.tmp_dir) with source.activate(): extent = source.get_extent() data = source.get_chip(extent) self.assertEqual(data.sum(), 2**16 - 256) column = data[:, 255, 0] self.assertEqual(column.sum(), 0)
def make_scene(scene_id, img_path, label_path): raster_source = RasterioSourceConfig(channel_order=[0, 1, 2], uris=[img_path]) label_source = ObjectDetectionLabelSourceConfig( vector_source=GeoJSONVectorSourceConfig(uri=label_path, default_class_id=None)) return SceneConfig(id=scene_id, raster_source=raster_source, label_source=label_source)
def test_non_geo(self): # Check if non-georeferenced image files can be read and CRSTransformer # implements the identity function. img_path = join(self.tmp_dir, 'img.png') chip = np.ones((2, 2, 3)).astype(np.uint8) save_img(chip, img_path) config = RasterioSourceConfig(uris=[img_path]) source = config.build(tmp_dir=self.tmp_dir) with source.activate(): out_chip = source.get_image_array() np.testing.assert_equal(out_chip, chip) p = (3, 4) out_p = source.get_crs_transformer().map_to_pixel(p) np.testing.assert_equal(out_p, p) out_p = source.get_crs_transformer().pixel_to_map(p) np.testing.assert_equal(out_p, p)
def test_detects_alpha(self): # Set first channel to alpha. Expectation is that when omitting channel_order, # only the second and third channels will be in output. img_path = join(self.tmp_dir, 'img.tif') chip = np.ones((2, 2, 3)).astype(np.uint8) chip[:, :, :] *= np.array([0, 1, 2]).astype(np.uint8) save_img(chip, img_path) ci = (ColorInterp.alpha, ColorInterp.blue, ColorInterp.green) with rasterio.open(img_path, 'r+') as src: src.colorinterp = ci config = RasterioSourceConfig(uris=[img_path]) source = config.build(tmp_dir=self.tmp_dir) with source.activate(): out_chip = source.get_image_array() expected_out_chip = np.ones((2, 2, 2)).astype(np.uint8) expected_out_chip[:, :, :] *= np.array([1, 2]).astype(np.uint8) np.testing.assert_equal(out_chip, expected_out_chip)
def make_multi_raster_source( rgbir_raster_uri: Union[UriPath, str], elevation_raster_uri: Union[UriPath, str]) -> MultiRasterSourceConfig: """ Create multi raster source by combining rgbir and elevation sources. """ rgbir_raster_uri = str(rgbir_raster_uri) elevation_raster_uri = str(elevation_raster_uri) rgbir_source = RasterioSourceConfig(uris=[rgbir_raster_uri], channel_order=[0, 1, 2, 3]) elevation_source = RasterioSourceConfig(uris=[elevation_raster_uri], channel_order=[0]) raster_source = MultiRasterSourceConfig(raster_sources=[ SubRasterSourceConfig(raster_source=rgbir_source, target_channels=[0, 1, 2, 3]), SubRasterSourceConfig(raster_source=elevation_source, target_channels=[4]) ]) return raster_source
def test_extent_crop_overflow(self): f = 1 / 10 arr = np.ones((100, 100), dtype=np.uint8) mask = np.zeros_like(arr).astype(bool) mask[10:90, 10:90] = 1 with NamedTemporaryFile('wb') as fp: uri = fp.name with rasterio.open(uri, 'w', driver='GTiff', height=100, width=100, count=1, dtype=np.uint8) as ds: ds.write_band(1, arr) cfg = RasterioSourceConfig(uris=[uri], extent_crop=(f, f, f, f)) rs = cfg.build(tmp_dir=self.tmp_dir) with rs.activate(): out = rs.get_chip(Box(0, 0, 100, 100))[..., 0] self.assertTrue(np.all(out[mask] == 1)) self.assertTrue(np.all(out[~mask] == 0))
def make_label_source( class_config: ClassConfig, label_uri: Union[UriPath, str] ) -> Tuple[SemanticSegmentationLabelSourceConfig, SemanticSegmentationLabelStoreConfig]: label_uri = str(label_uri) # Using with_rgb_class_map because label TIFFs have classes encoded as # RGB colors. label_source = SemanticSegmentationLabelSourceConfig( rgb_class_config=class_config, raster_source=RasterioSourceConfig(uris=[label_uri])) # URI will be injected by scene config. # Using rgb=True because we want prediction TIFFs to be in # RGB format. label_store = SemanticSegmentationLabelStoreConfig( rgb=True, vector_output=[PolygonVectorOutputConfig(class_id=0)]) return label_source, label_store
def make_scene(img_path, label_path): id = basename(img_path) label_source = ChipClassificationLabelSourceConfig( vector_source=GeoJSONVectorSourceConfig(uri=label_path, default_class_id=None, ignore_crs_field=True), ioa_thresh=0.5, use_intersection_over_cell=False, pick_min_class_id=True, background_class_id=2, infer_cells=True) raster_source = RasterioSourceConfig( channel_order=[0, 1, 2], uris=[img_path], transformers=[StatsTransformerConfig()]) return SceneConfig(id=id, raster_source=raster_source, label_source=label_source)
def test_gets_raw_chip_from_uint16_transformed_proto(self): img_path = data_file_path('small-uint16-tile.tif') channel_order = [0, 1] config = RasterioSourceConfig(uris=[img_path]) raw_rs = config.build(tmp_dir=self.tmp_dir) stats_uri = join(self.tmp_dir, 'tmp.tif') stats = RasterStats() stats.compute([raw_rs]) stats.save(stats_uri) transformer = StatsTransformerConfig(stats_uri=stats_uri) config = RasterioSourceConfig(uris=[img_path], channel_order=channel_order, transformers=[transformer]) rs = config.build(tmp_dir=self.tmp_dir) with rs.activate(): out_chip = rs.get_raw_image_array() self.assertEqual(out_chip.shape[2], 3)
def test_get_dtype(self): img_path = data_file_path('small-rgb-tile.tif') config = RasterioSourceConfig(uris=[img_path]) source = config.build(tmp_dir=self.tmp_dir) self.assertEqual(source.get_dtype(), np.uint8)