def test_no_epsg(self):
crs = rasterio.crs.CRS()
img_path = join(self.tmp_dir, 'tmp.tif')
height = 100
width = 100
nb_channels = 3
with rasterio.open(
img_path,
'w',
driver='GTiff',
height=height,
width=width,
count=nb_channels,
dtype=np.uint8,
crs=crs) as img_dataset:
im = np.zeros((height, width, nb_channels)).astype(np.uint8)
for channel in range(nb_channels):
img_dataset.write(im[:, :, channel], channel + 1)
try:
config = RasterioSourceConfig(uris=[img_path])
config.build(tmp_dir=self.tmp_dir)
except Exception:
self.fail(
'Creating RasterioSource with CRS with no EPSG attribute '
'raised an exception when it should not have.')
def test_mask(self):
# make geotiff filled with ones and zeros and mask the whole image
img_path = join(self.tmp_dir, 'tmp.tif')
height = 100
width = 100
nb_channels = 3
with rasterio.open(
img_path,
'w',
driver='GTiff',
height=height,
width=width,
count=nb_channels,
dtype=np.uint8) as img_dataset:
im = np.random.randint(0, 2, (height, width, nb_channels)).astype(
np.uint8)
for channel in range(nb_channels):
img_dataset.write(im[:, :, channel], channel + 1)
img_dataset.write_mask(np.zeros(im.shape[0:2]).astype(np.bool))
config = RasterioSourceConfig(uris=[img_path])
source = config.build(tmp_dir=self.tmp_dir)
with source.activate():
out_chip = source.get_image_array()
expected_out_chip = np.zeros((height, width, nb_channels))
np.testing.assert_equal(out_chip, expected_out_chip)
def test_nonidentical_extents_and_resolutions(self):
img_path_1 = data_file_path(
'multi_raster_source/const_100_600x600.tiff')
img_path_2 = data_file_path('multi_raster_source/const_175_60x60.tiff')
img_path_3 = data_file_path('multi_raster_source/const_250_6x6.tiff')
source_1 = RasterioSourceConfig(uris=[img_path_1], channel_order=[0])
source_2 = RasterioSourceConfig(uris=[img_path_2], channel_order=[0])
source_3 = RasterioSourceConfig(uris=[img_path_3], channel_order=[0])
cfg = MultiRasterSourceConfig(raster_sources=[
SubRasterSourceConfig(raster_source=source_1, target_channels=[0]),
SubRasterSourceConfig(raster_source=source_2, target_channels=[1]),
SubRasterSourceConfig(raster_source=source_3, target_channels=[2])
])
rs = cfg.build(tmp_dir=self.tmp_dir)
with rs.activate():
for get_chip_fn in [rs._get_chip, rs.get_chip]:
ch_1_only = get_chip_fn(Box(0, 0, 10, 10))
self.assertEqual(tuple(ch_1_only.reshape(-1, 3).mean(axis=0)),
(100, 0, 0))
ch_2_only = get_chip_fn(Box(0, 600, 10, 600 + 10))
self.assertEqual(tuple(ch_2_only.reshape(-1, 3).mean(axis=0)),
(0, 175, 0))
ch_3_only = get_chip_fn(Box(600, 0, 600 + 10, 10))
self.assertEqual(tuple(ch_3_only.reshape(-1, 3).mean(axis=0)),
(0, 0, 250))
full_img = get_chip_fn(Box(0, 0, 600, 600))
self.assertEqual(set(np.unique(full_img[..., 0])), {100})
self.assertEqual(set(np.unique(full_img[..., 1])), {0, 175})
self.assertEqual(set(np.unique(full_img[..., 2])), {0, 250})
def test_gets_raw_chip(self):
img_path = data_file_path('small-rgb-tile.tif')
channel_order = [0, 1]
config = RasterioSourceConfig(
uris=[img_path], channel_order=channel_order)
source = config.build(tmp_dir=self.tmp_dir)
with source.activate():
out_chip = source.get_raw_image_array()
self.assertEqual(out_chip.shape[2], 3)
def test_channel_order_error(self):
img_path = join(self.tmp_dir, 'img.tif')
chip = np.ones((2, 2, 3)).astype(np.uint8)
chip[:, :, :] *= np.array([0, 1, 2]).astype(np.uint8)
save_img(chip, img_path)
channel_order = [3, 1, 0]
with self.assertRaises(ChannelOrderError):
config = RasterioSourceConfig(
uris=[img_path], channel_order=channel_order)
config.build(tmp_dir=self.tmp_dir)
def make_cfg(img_path='small-rgb-tile.tif', **kwargs):
img_path = data_file_path(img_path)
r_source = RasterioSourceConfig(uris=[img_path], channel_order=[0])
g_source = RasterioSourceConfig(uris=[img_path], channel_order=[1])
b_source = RasterioSourceConfig(uris=[img_path], channel_order=[2])
cfg = MultiRasterSourceConfig(raster_sources=[
SubRasterSourceConfig(raster_source=r_source, target_channels=[0]),
SubRasterSourceConfig(raster_source=g_source, target_channels=[1]),
SubRasterSourceConfig(raster_source=b_source, target_channels=[2])
],
**kwargs)
return cfg
def test_extent(self):
img_path = data_file_path('small-rgb-tile.tif')
cfg = RasterioSourceConfig(uris=[img_path])
rs = cfg.build(tmp_dir=self.tmp_dir)
extent = rs.get_extent()
h, w = extent.get_height(), extent.get_width()
ymin, xmin, ymax, xmax = extent
self.assertEqual(h, 256)
self.assertEqual(w, 256)
self.assertEqual(ymin, 0)
self.assertEqual(xmin, 0)
self.assertEqual(ymax, 256)
self.assertEqual(xmax, 256)
def test_get_chip(self):
# create a 3-channel raster from a 1-channel raster
# using a ReclassTransformer to give each channel a different value
# (100, 175, and 250 respectively)
path = data_file_path('multi_raster_source/const_100_600x600.tiff')
source_1 = RasterioSourceConfig(uris=[path], channel_order=[0])
source_2 = RasterioSourceConfig(
uris=[path],
channel_order=[0],
transformers=[ReclassTransformerConfig(mapping={100: 175})])
source_3 = RasterioSourceConfig(
uris=[path],
channel_order=[0],
transformers=[ReclassTransformerConfig(mapping={100: 250})])
cfg = MultiRasterSourceConfig(raster_sources=[
SubRasterSourceConfig(raster_source=source_1, target_channels=[0]),
SubRasterSourceConfig(raster_source=source_2, target_channels=[1]),
SubRasterSourceConfig(raster_source=source_3, target_channels=[2])
],
channel_order=[2, 1, 0],
transformers=[
ReclassTransformerConfig(mapping={
100: 10,
175: 17,
250: 25
})
])
rs = cfg.build(tmp_dir=self.tmp_dir)
with rs.activate():
window = Box(0, 0, 100, 100)
# sub transformers and channel_order applied
sub_chips = rs._get_sub_chips(window, raw=False)
self.assertEqual(tuple(c.mean() for c in sub_chips),
(100, 175, 250))
# sub transformers, channel_order, and transformer applied
chip = rs.get_chip(window)
self.assertEqual(tuple(chip.reshape(-1, 3).mean(axis=0)),
(25, 17, 10))
# none of sub transformers, channel_order, and transformer applied
sub_chips = rs._get_sub_chips(window, raw=True)
self.assertEqual(tuple(c.mean() for c in sub_chips),
(100, 100, 100))
chip = rs._get_chip(window)
self.assertEqual(tuple(chip.reshape(-1, 3).mean(axis=0)),
(100, 100, 100))
def test_accounts_for_aoi(self):
class_config = ClassConfig(names=['car', 'building', 'background'])
label_source_uri = data_file_path('evaluator/cc-label-filtered.json')
label_source_cfg = ChipClassificationLabelSourceConfig(
vector_source=GeoJSONVectorSourceConfig(
uri=label_source_uri, default_class_id=None))
label_store_uri = data_file_path('evaluator/cc-label-full.json')
label_store_cfg = ChipClassificationGeoJSONStoreConfig(
uri=label_store_uri)
raster_source_uri = data_file_path('evaluator/cc-label-img-blank.tif')
raster_source_cfg = RasterioSourceConfig(uris=[raster_source_uri])
aoi_uri = data_file_path('evaluator/cc-label-aoi.json')
s = SceneConfig(
id='test',
raster_source=raster_source_cfg,
label_source=label_source_cfg,
label_store=label_store_cfg,
aoi_uris=[aoi_uri])
with rv_config.get_tmp_dir() as tmp_dir:
scene = s.build(class_config, tmp_dir)
output_uri = os.path.join(tmp_dir, 'eval.json')
evaluator = ChipClassificationEvaluatorConfig(
output_uri=output_uri).build(class_config)
evaluator.process([scene], tmp_dir)
overall = file_to_json(output_uri)['overall']
for item in overall:
self.assertEqual(item['f1'], 1.0)
def hrefs_to_sceneconfig(
imagery: str,
labels: Optional[str],
name: str,
class_id_filter_dict: Dict[int, str],
extent_crop: Optional[CropOffsets] = None) -> SceneConfig:
transformers = [CastTransformerConfig(to_dtype='float16')]
image_source = RasterioSourceConfig(
channel_order=list(range(224)),
uris=[imagery],
allow_streaming=True,
transformers=transformers,
extent_crop=extent_crop,
)
label_vector_source = GeoJSONVectorSourceConfig(
uri=labels,
class_id_to_filter=class_id_filter_dict,
default_class_id=0xff)
label_raster_source = RasterizedSourceConfig(
vector_source=label_vector_source,
rasterizer_config=RasterizerConfig(background_class_id=0,
all_touched=True))
label_source = SemanticSegmentationLabelSourceConfig(
raster_source=label_raster_source)
return SceneConfig(id=name,
raster_source=image_source,
label_source=label_source)
def test_uses_channel_order(self):
img_path = join(self.tmp_dir, 'img.tif')
chip = np.ones((2, 2, 4)).astype(np.uint8)
chip[:, :, :] *= np.array([0, 1, 2, 3]).astype(np.uint8)
save_img(chip, img_path)
channel_order = [0, 1, 2]
config = RasterioSourceConfig(
uris=[img_path], channel_order=channel_order)
source = config.build(tmp_dir=self.tmp_dir)
with source.activate():
out_chip = source.get_image_array()
expected_out_chip = np.ones((2, 2, 3)).astype(np.uint8)
expected_out_chip[:, :, :] *= np.array([0, 1, 2]).astype(np.uint8)
np.testing.assert_equal(out_chip, expected_out_chip)
def test_extent_crop(self):
f = 1 / 4
img_path = data_file_path('small-rgb-tile.tif')
cfg_crop = RasterioSourceConfig(
uris=[img_path], extent_crop=(f, f, f, f))
rs_crop = cfg_crop.build(tmp_dir=self.tmp_dir)
# test extent box
extent_crop = rs_crop.get_extent()
self.assertEqual(extent_crop.ymin, 64)
self.assertEqual(extent_crop.xmin, 64)
self.assertEqual(extent_crop.ymax, 192)
self.assertEqual(extent_crop.xmax, 192)
# test windows
windows = extent_crop.get_windows(64, 64)
self.assertEqual(windows[0].ymin, 64)
self.assertEqual(windows[0].xmin, 64)
self.assertEqual(windows[-1].ymax, 192)
self.assertEqual(windows[-1].xmax, 192)
# test CropOffsets class
cfg_crop = RasterioSourceConfig(
uris=[img_path],
extent_crop=CropOffsets(skip_top=.5, skip_right=.5))
rs_crop = cfg_crop.build(tmp_dir=self.tmp_dir)
extent_crop = rs_crop.get_extent()
self.assertEqual(extent_crop.ymin, 128)
self.assertEqual(extent_crop.xmin, 0)
self.assertEqual(extent_crop.ymax, 256)
self.assertEqual(extent_crop.xmax, 128)
# test validation
extent_crop = CropOffsets(skip_top=.5, skip_bottom=.5)
self.assertRaises(
ValidationError,
lambda: RasterioSourceConfig(uris=[img_path],
extent_crop=extent_crop))
extent_crop = CropOffsets(skip_left=.5, skip_right=.5)
self.assertRaises(
ValidationError,
lambda: RasterioSourceConfig(uris=[img_path],
extent_crop=extent_crop))
# test extent_crop=None
try:
_ = RasterioSourceConfig(uris=[img_path], extent_crop=None) # noqa
except Exception:
self.fail('extent_crop=None caused an error.')
def test_rgb_class_config_null_class(self):
raster_source_cfg = RasterioSourceConfig(uris=['/abc/def.tif'])
rgb_class_config = ClassConfig(names=['a'], colors=['#010101'])
cfg = SemanticSegmentationLabelSourceConfig(
raster_source=raster_source_cfg, rgb_class_config=rgb_class_config)
cfg.update()
self.assertEqual(len(cfg.rgb_class_config), 2)
self.assertIn('null', cfg.rgb_class_config.names)
def make_scene(id, img_path, label_path):
raster_source = RasterioSourceConfig(channel_order=[0, 1, 2],
uris=[img_path])
label_source = SemanticSegmentationLabelSourceConfig(
rgb_class_config=class_config,
raster_source=RasterioSourceConfig(uris=[label_path]))
label_store = SemanticSegmentationLabelStoreConfig(
rgb=True,
vector_output=[
PolygonVectorOutputConfig(class_id=0),
BuildingVectorOutputConfig(class_id=1)
])
return SceneConfig(id=id,
raster_source=raster_source,
label_source=label_source,
label_store=label_store)
def test_shift_x(self):
# Specially-engineered image w/ one meter per pixel resolution
# in the x direction.
img_path = data_file_path('ones.tif')
channel_order = [0]
config = RasterioSourceConfig(uris=[img_path],
channel_order=channel_order,
x_shift=1.0,
y_shift=0.0)
source = config.build(tmp_dir=self.tmp_dir)
with source.activate():
extent = source.get_extent()
data = source.get_chip(extent)
self.assertEqual(data.sum(), 2**16 - 256)
column = data[:, 255, 0]
self.assertEqual(column.sum(), 0)
def make_scene(scene_id, img_path, label_path):
raster_source = RasterioSourceConfig(channel_order=[0, 1, 2],
uris=[img_path])
label_source = ObjectDetectionLabelSourceConfig(
vector_source=GeoJSONVectorSourceConfig(uri=label_path,
default_class_id=None))
return SceneConfig(id=scene_id,
raster_source=raster_source,
label_source=label_source)
def test_non_geo(self):
# Check if non-georeferenced image files can be read and CRSTransformer
# implements the identity function.
img_path = join(self.tmp_dir, 'img.png')
chip = np.ones((2, 2, 3)).astype(np.uint8)
save_img(chip, img_path)
config = RasterioSourceConfig(uris=[img_path])
source = config.build(tmp_dir=self.tmp_dir)
with source.activate():
out_chip = source.get_image_array()
np.testing.assert_equal(out_chip, chip)
p = (3, 4)
out_p = source.get_crs_transformer().map_to_pixel(p)
np.testing.assert_equal(out_p, p)
out_p = source.get_crs_transformer().pixel_to_map(p)
np.testing.assert_equal(out_p, p)
def test_detects_alpha(self):
# Set first channel to alpha. Expectation is that when omitting channel_order,
# only the second and third channels will be in output.
img_path = join(self.tmp_dir, 'img.tif')
chip = np.ones((2, 2, 3)).astype(np.uint8)
chip[:, :, :] *= np.array([0, 1, 2]).astype(np.uint8)
save_img(chip, img_path)
ci = (ColorInterp.alpha, ColorInterp.blue, ColorInterp.green)
with rasterio.open(img_path, 'r+') as src:
src.colorinterp = ci
config = RasterioSourceConfig(uris=[img_path])
source = config.build(tmp_dir=self.tmp_dir)
with source.activate():
out_chip = source.get_image_array()
expected_out_chip = np.ones((2, 2, 2)).astype(np.uint8)
expected_out_chip[:, :, :] *= np.array([1, 2]).astype(np.uint8)
np.testing.assert_equal(out_chip, expected_out_chip)
def make_multi_raster_source(
rgbir_raster_uri: Union[UriPath, str],
elevation_raster_uri: Union[UriPath, str]) -> MultiRasterSourceConfig:
""" Create multi raster source by combining rgbir and elevation sources. """
rgbir_raster_uri = str(rgbir_raster_uri)
elevation_raster_uri = str(elevation_raster_uri)
rgbir_source = RasterioSourceConfig(uris=[rgbir_raster_uri],
channel_order=[0, 1, 2, 3])
elevation_source = RasterioSourceConfig(uris=[elevation_raster_uri],
channel_order=[0])
raster_source = MultiRasterSourceConfig(raster_sources=[
SubRasterSourceConfig(raster_source=rgbir_source,
target_channels=[0, 1, 2, 3]),
SubRasterSourceConfig(raster_source=elevation_source,
target_channels=[4])
])
return raster_source
def test_extent_crop_overflow(self):
f = 1 / 10
arr = np.ones((100, 100), dtype=np.uint8)
mask = np.zeros_like(arr).astype(bool)
mask[10:90, 10:90] = 1
with NamedTemporaryFile('wb') as fp:
uri = fp.name
with rasterio.open(uri,
'w',
driver='GTiff',
height=100,
width=100,
count=1,
dtype=np.uint8) as ds:
ds.write_band(1, arr)
cfg = RasterioSourceConfig(uris=[uri], extent_crop=(f, f, f, f))
rs = cfg.build(tmp_dir=self.tmp_dir)
with rs.activate():
out = rs.get_chip(Box(0, 0, 100, 100))[..., 0]
self.assertTrue(np.all(out[mask] == 1))
self.assertTrue(np.all(out[~mask] == 0))
def make_label_source(
class_config: ClassConfig, label_uri: Union[UriPath, str]
) -> Tuple[SemanticSegmentationLabelSourceConfig,
SemanticSegmentationLabelStoreConfig]:
label_uri = str(label_uri)
# Using with_rgb_class_map because label TIFFs have classes encoded as
# RGB colors.
label_source = SemanticSegmentationLabelSourceConfig(
rgb_class_config=class_config,
raster_source=RasterioSourceConfig(uris=[label_uri]))
# URI will be injected by scene config.
# Using rgb=True because we want prediction TIFFs to be in
# RGB format.
label_store = SemanticSegmentationLabelStoreConfig(
rgb=True, vector_output=[PolygonVectorOutputConfig(class_id=0)])
return label_source, label_store
def make_scene(img_path, label_path):
id = basename(img_path)
label_source = ChipClassificationLabelSourceConfig(
vector_source=GeoJSONVectorSourceConfig(uri=label_path,
default_class_id=None,
ignore_crs_field=True),
ioa_thresh=0.5,
use_intersection_over_cell=False,
pick_min_class_id=True,
background_class_id=2,
infer_cells=True)
raster_source = RasterioSourceConfig(
channel_order=[0, 1, 2],
uris=[img_path],
transformers=[StatsTransformerConfig()])
return SceneConfig(id=id,
raster_source=raster_source,
label_source=label_source)
def test_gets_raw_chip_from_uint16_transformed_proto(self):
img_path = data_file_path('small-uint16-tile.tif')
channel_order = [0, 1]
config = RasterioSourceConfig(uris=[img_path])
raw_rs = config.build(tmp_dir=self.tmp_dir)
stats_uri = join(self.tmp_dir, 'tmp.tif')
stats = RasterStats()
stats.compute([raw_rs])
stats.save(stats_uri)
transformer = StatsTransformerConfig(stats_uri=stats_uri)
config = RasterioSourceConfig(uris=[img_path],
channel_order=channel_order,
transformers=[transformer])
rs = config.build(tmp_dir=self.tmp_dir)
with rs.activate():
out_chip = rs.get_raw_image_array()
self.assertEqual(out_chip.shape[2], 3)
def test_get_dtype(self):
img_path = data_file_path('small-rgb-tile.tif')
config = RasterioSourceConfig(uris=[img_path])
source = config.build(tmp_dir=self.tmp_dir)
self.assertEqual(source.get_dtype(), np.uint8)
