def read_labels(geojson, extent=None):
"""Convert GeoJSON to ChipClassificationLabels.
If the GeoJSON already contains a grid of cells, then it can be constructed
in a straightforward manner without having to infer the class of cells.
If extent is given, only labels that intersect with the extent are returned.
Args:
geojson: dict in normalized GeoJSON format (see VectorSource)
extent: Box in pixel coords
Returns:
ChipClassificationLabels
"""
labels = ChipClassificationLabels()
for f in geojson['features']:
geom = shape(f['geometry'])
(xmin, ymin, xmax, ymax) = geom.bounds
cell = Box(ymin, xmin, ymax, xmax)
if extent is not None and not cell.to_shapely().intersects(
extent.to_shapely()):
continue
props = f['properties']
class_id = props['class_id']
scores = props.get('scores')
labels.set_cell(cell, class_id, scores)
return labels
def nodata_transform(min_val, in_path, out_path):
"""Convert values less than min_val to 0/NODATA.
This script was developed to deal with imagery over Belize City which contains values
that appear to be NODATA values on the periphery of the image, but are actually very
small values < 7. This script converts those small values to zeros which Raster
Vision knows to ignore when making predictions.
"""
min_val = float(min_val)
chip_size = 1000
with rasterio.open(in_path, 'r') as in_data:
with rasterio.open(out_path, 'w', **in_data.profile) as out_data:
extent = Box(0, 0, in_data.height, in_data.width)
windows = extent.get_windows(chip_size, chip_size)
for w in windows:
# Avoid going off edge of array.
if w.ymax > in_data.height:
w.ymax = in_data.height
if w.xmax > in_data.width:
w.xmax = in_data.width
print('.', end='', flush=True)
w = w.rasterio_format()
im = in_data.read(window=w)
nodata_mask = np.all(im < min_val, axis=0)
for b in range(im.shape[0]):
band = im[b, :, :]
band[nodata_mask] = 0
out_data.write_band(b + 1, band, window=w)
def test_enough_target_pixels_true(self):
data = np.zeros((10, 10, 1), dtype=np.uint8)
data[4:, 4:, :] = 1
null_class_id = 2
raster_source = MockRasterSource([0], 1)
raster_source.set_raster(data)
label_source = SemanticSegmentationLabelSource(raster_source,
null_class_id)
with label_source.activate():
extent = Box(0, 0, 10, 10)
self.assertTrue(label_source.enough_target_pixels(extent, 30, [1]))
def set_return_vals(self, raster=None):
self.mock.get_extent.return_value = Box.make_square(0, 0, 2)
self.mock.get_dtype.return_value = np.uint8
self.mock.get_crs_transformer.return_value = IdentityCRSTransformer()
self.mock._get_chip.return_value = np.random.rand(1, 2, 2, 3)
if raster is not None:
self.mock.get_extent.return_value = Box(0, 0, raster.shape[0],
raster.shape[1])
self.mock.get_dtype.return_value = raster.dtype
def get_chip(window):
return raster[window.ymin:window.ymax,
window.xmin:window.xmax, :]
self.mock._get_chip.side_effect = get_chip
def process(self, training_data, tmp_dir):
augmented = TrainingData()
nodata_aug_prob = self.aug_prob
for chip, window, labels in training_data:
# If negative chip, with some probability, add a random black square
# to chip.
if len(labels) == 0 and random.uniform(0, 1) < nodata_aug_prob:
size = round(random.uniform(0, 1) * chip.shape[0])
square = Box(0, 0, chip.shape[0],
chip.shape[1]).make_random_square(size)
chip = np.copy(chip)
chip[square.ymin:square.ymax, square.xmin:square.xmax, :] = 0
augmented.append(chip, window, labels)
return augmented
def test_command_run_with_mocks(self):
task_config = rv.TaskConfig.builder(mk.MOCK_TASK).build()
backend_config = rv.BackendConfig.builder(mk.MOCK_BACKEND).build()
backend = backend_config.create_backend(task_config)
backend_config.mock.create_backend.return_value = backend
task = task_config.create_task(backend)
task_config.mock.create_task.return_value = task
scene = mk.create_mock_scene()
task.mock.get_predict_windows.return_value = [Box(0, 0, 1, 1)]
cmd = rv.CommandConfig.builder(rv.PREDICT) \
.with_task(task_config) \
.with_backend(backend_config) \
.with_scenes([scene]) \
.with_root_uri('.') \
.build() \
.create_command()
cmd.run()
self.assertTrue(backend.mock.predict.called)
def make_labels(self, class_ids):
"""Make 2x2 grid label store.
Args:
class_ids: 2x2 array of class_ids to use
"""
cell_size = 200
y_cells = 2
x_cells = 2
labels = ChipClassificationLabels()
for yind in range(y_cells):
for xind in range(x_cells):
ymin = yind * cell_size
xmin = xind * cell_size
ymax = ymin + cell_size
xmax = xmin + cell_size
window = Box(ymin, xmin, ymax, xmax)
class_id = class_ids[yind][xind]
new_labels = ChipClassificationLabels()
new_labels.set_cell(window, class_id)
labels.extend(new_labels)
return labels
def save_image_crop(image_uri,
image_crop_uri,
label_uri=None,
label_crop_uri=None,
size=600,
min_features=10,
vector_labels=True,
class_config=None):
"""Save a crop of an image to use for testing.
If label_uri is set, the crop needs to cover >= min_features.
Args:
image_uri: URI of original image
image_crop_uri: URI of cropped image to save
label_uri: optional URI of label file
label_crop_uri: optional URI of cropped labels to save
size: height and width of crop
Raises:
ValueError if cannot find a crop satisfying min_features constraint.
"""
if not file_exists(image_crop_uri):
print('Saving test crop to {}...'.format(image_crop_uri))
old_environ = os.environ.copy()
try:
request_payer = S3FileSystem.get_request_payer()
if request_payer == 'requester':
os.environ['AWS_REQUEST_PAYER'] = request_payer
im_dataset = rasterio.open(image_uri)
h, w = im_dataset.height, im_dataset.width
extent = Box(0, 0, h, w)
windows = extent.get_windows(size, size)
if label_uri and vector_labels:
crs_transformer = RasterioCRSTransformer.from_dataset(
im_dataset)
geojson_vs_config = GeoJSONVectorSourceConfig(
uri=label_uri, default_class_id=0, ignore_crs_field=True)
vs = geojson_vs_config.build(class_config, crs_transformer)
geojson = vs.get_geojson()
geoms = []
for f in geojson['features']:
g = shape(f['geometry'])
geoms.append(g)
tree = STRtree(geoms)
def p2m(x, y, z=None):
return crs_transformer.pixel_to_map((x, y))
for w in windows:
use_window = True
if label_uri and vector_labels:
w_polys = tree.query(w.to_shapely())
use_window = len(w_polys) >= min_features
if use_window and label_crop_uri is not None:
print('Saving test crop labels to {}...'.format(
label_crop_uri))
label_crop_features = [
mapping(transform(p2m, wp)) for wp in w_polys
]
label_crop_json = {
'type':
'FeatureCollection',
'features': [{
'geometry': f
} for f in label_crop_features]
}
json_to_file(label_crop_json, label_crop_uri)
if use_window:
crop_image(image_uri, w, image_crop_uri)
if not vector_labels and label_uri and label_crop_uri:
crop_image(label_uri, w, label_crop_uri)
break
if not use_window:
raise ValueError('Could not find a good crop.')
finally:
os.environ.clear()
os.environ.update(old_environ)
def test_mocks(self):
"""Test to ensure all mocks are working as expected."""
task_config = rv.TaskConfig.builder(mk.MOCK_TASK) \
.build()
backend_config = rv.BackendConfig.builder(mk.MOCK_BACKEND) \
.build()
raster_transformer_config = rv.RasterTransformerConfig.builder(
mk.MOCK_TRANSFORMER).build()
raster_source_config = rv.RasterSourceConfig.builder(mk.MOCK_SOURCE) \
.with_transformer(
raster_transformer_config) \
.build()
label_source_config = rv.LabelSourceConfig.builder(mk.MOCK_SOURCE) \
.build()
label_store_config = rv.LabelStoreConfig.builder(mk.MOCK_STORE) \
.build()
scene_config = rv.SceneConfig.builder() \
.with_id('test') \
.with_raster_source(raster_source_config) \
.with_label_source(label_source_config) \
.with_label_store(label_store_config) \
.build()
augmentor_config = rv.AugmentorConfig.builder(mk.MOCK_AUGMENTOR) \
.build()
dataset = rv.DatasetConfig.builder() \
.with_train_scene(scene_config) \
.with_validation_scene(scene_config) \
.with_augmentor(augmentor_config) \
.build()
analyzer_config = rv.AnalyzerConfig.builder(mk.MOCK_ANALYZER).build()
evaluator_config = rv.EvaluatorConfig.builder(
mk.MOCK_EVALUATOR).build()
# Create entities from configuration
backend = backend_config.create_backend(task_config)
task = task_config.create_task(backend)
scene = scene_config.create_scene(task_config, '.')
_ = augmentor_config.create_augmentor() # noqa
_ = analyzer_config.create_analyzer() # noqa
_ = evaluator_config.create_evaluator() # noqa
# Assert some things
task_config.mock.create_task.assert_called_once_with(backend)
self.assertEqual(task.get_predict_windows(Box(0, 0, 1, 1)), [])
_ = scene.raster_source.get_chip(Box(0, 0, 1, 1)) # noqa
self.assertTrue(
scene.raster_source.raster_transformers[0].mock.transform.called)
# Create and run experiment
with tempfile.TemporaryDirectory() as tmp_dir:
e = rv.ExperimentConfig.builder() \
.with_task(task_config) \
.with_backend(backend_config) \
.with_dataset(dataset) \
.with_analyzer(analyzer_config) \
.with_evaluator(evaluator_config) \
.with_root_uri(tmp_dir) \
.with_id('test') \
.build()
rv.ExperimentRunner.get_runner(rv.LOCAL).run(e, splits=7)
def save_image_crop(image_uri,
crop_uri,
label_uri=None,
size=600,
min_features=10):
"""Save a crop of an image to use for testing.
If label_uri is set, the crop needs to cover >= min_features.
Args:
image_uri: URI of original image
crop_uri: URI of cropped image to save
label_uri: optional URI of GeoJSON file
size: height and width of crop
Raises:
ValueError if cannot find a crop satisfying min_features constraint.
"""
if not file_exists(crop_uri):
print('Saving test crop to {}...'.format(crop_uri))
old_environ = os.environ.copy()
try:
request_payer = S3FileSystem.get_request_payer()
if request_payer == 'requester':
os.environ['AWS_REQUEST_PAYER'] = request_payer
im_dataset = rasterio.open(image_uri)
h, w = im_dataset.height, im_dataset.width
extent = Box(0, 0, h, w)
windows = extent.get_windows(size, size)
if label_uri is not None:
crs_transformer = RasterioCRSTransformer.from_dataset(
im_dataset)
vs = GeoJSONVectorSource(label_uri, crs_transformer)
geojson = vs.get_geojson()
geoms = []
for f in geojson['features']:
g = shape(f['geometry'])
geoms.append(g)
tree = STRtree(geoms)
for w in windows:
use_window = True
if label_uri is not None:
w_polys = tree.query(w.to_shapely())
use_window = len(w_polys) >= min_features
if use_window:
w = w.rasterio_format()
im = im_dataset.read(window=w)
if np.mean(np.sum(im, axis=2).ravel() == 0) < 0.9:
with tempfile.TemporaryDirectory() as tmp_dir:
crop_path = get_local_path(crop_uri, tmp_dir)
make_dir(crop_path, use_dirname=True)
meta = im_dataset.meta
meta['width'], meta['height'] = size, size
meta['transform'] = rasterio.windows.transform(
w, im_dataset.transform)
with rasterio.open(crop_path, 'w', **meta) as dst:
dst.colorinterp = im_dataset.colorinterp
dst.write(im)
upload_or_copy(crop_path, crop_uri)
break
if not use_window:
raise ValueError('Could not find a good crop.')
finally:
os.environ.clear()
os.environ.update(old_environ)