def test_evaluator(self):
output_uri = join(self.tmp_dir.name, 'out.json')
scenes = [self.get_scene(0), self.get_scene(1)]
evaluator = SemanticSegmentationEvaluator(self.class_config,
output_uri, None)
evaluator.process(scenes, self.tmp_dir.name)
eval_json = file_to_json(output_uri)
exp_eval_json = file_to_json(data_file_path('expected-eval.json'))
self.assertDictEqual(eval_json, exp_eval_json)
def collect_eval_dir(root_uri):
eval_json_uris = list_paths(join(root_uri, 'eval'), ext='eval.json')
for eval_json_uri in eval_json_uris:
eval_json = file_to_json(eval_json_uri)
print(basename(dirname(eval_json_uri)))
print(eval_json['overall'][-1]['f1'])
print()
def collect_experiment(key, root_uri, output_dir, get_pred_package=False):
print('\nCollecting experiment {}...\n'.format(key))
if root_uri.startswith('s3://'):
predict_package_uris = list_paths(join(root_uri, key, 'bundle'), ext='predict_package.zip')
eval_json_uris = list_paths(join(root_uri, key, 'eval'), ext='eval.json')
else:
predict_package_uris = glob.glob(join(root_uri, key, 'bundle', '*', 'predict_package.zip'))
eval_json_uris = glob.glob(join(root_uri, key, 'eval', '*', 'eval.json'))
if len(predict_package_uris) > 1 or len(eval_json_uris) > 1:
print('Cannot collect from key with multiple experiments!!!')
return
if len(predict_package_uris) == 0 or len(eval_json_uris) == 0:
print('Missing output!!!')
return
predict_package_uri = predict_package_uris[0]
eval_json_uri = eval_json_uris[0]
make_dir(join(output_dir, key))
if get_pred_package:
download_or_copy(predict_package_uri, join(output_dir, key))
download_or_copy(eval_json_uri, join(output_dir, key))
eval_json = file_to_json(join(output_dir, key, 'eval.json'))
pprint.pprint(eval_json['overall'], indent=4)
def test_accounts_for_aoi(self):
class_config = ClassConfig(names=['car', 'building', 'background'])
label_source_uri = data_file_path('evaluator/cc-label-filtered.json')
label_source_cfg = ChipClassificationLabelSourceConfig(
vector_source=GeoJSONVectorSourceConfig(uri=label_source_uri,
default_class_id=None))
label_store_uri = data_file_path('evaluator/cc-label-full.json')
label_store_cfg = ChipClassificationGeoJSONStoreConfig(
uri=label_store_uri)
raster_source_uri = data_file_path('evaluator/cc-label-img-blank.tif')
raster_source_cfg = RasterioSourceConfig(uris=[raster_source_uri])
aoi_uri = data_file_path('evaluator/cc-label-aoi.json')
s = SceneConfig(id='test',
raster_source=raster_source_cfg,
label_source=label_source_cfg,
label_store=label_store_cfg,
aoi_uris=[aoi_uri])
with rv_config.get_tmp_dir() as tmp_dir:
scene = s.build(class_config, tmp_dir)
output_uri = os.path.join(tmp_dir, 'eval.json')
evaluator = ChipClassificationEvaluatorConfig(
output_uri=output_uri).build(class_config)
evaluator.process([scene], tmp_dir)
overall = file_to_json(output_uri)['overall']
for item in overall:
self.assertEqual(item['f1'], 1.0)
def test_vector_evaluator_with_aoi(self):
output_uri = join(self.tmp_dir.name, 'raster-out.json')
vector_output_uri = join(self.tmp_dir.name, 'vector-out.json')
scenes = [self.get_vector_scene(0, use_aoi=True)]
evaluator = SemanticSegmentationEvaluator(self.class_config,
output_uri,
vector_output_uri)
evaluator.process(scenes, self.tmp_dir.name)
vector_eval_json = file_to_json(vector_output_uri)
exp_vector_eval_json = file_to_json(
data_file_path('expected-vector-eval-with-aoi.json'))
# NOTE: The precision and recall values found in the file
# `expected-vector-eval.json` are equal to fractions of the
# form (n-1)/n for n <= 7 which can be seen to be (and have
# been manually verified to be) correct.
self.assertDictEqual(vector_eval_json, exp_vector_eval_json)
def from_model_bundle(model_bundle_uri, tmp_dir):
model_bundle_path = download_if_needed(model_bundle_uri, tmp_dir)
model_bundle_dir = join(tmp_dir, 'model-bundle')
unzip(model_bundle_path, model_bundle_dir)
config_path = join(model_bundle_dir, 'config.json')
model_path = join(model_bundle_dir, 'model.pth')
cfg = build_config(file_to_json(config_path))
return cfg.build(tmp_dir, model_path=model_path)
def from_model_bundle(model_bundle_uri: str, tmp_dir: str):
"""Create a Learner from a model bundle."""
model_bundle_path = download_if_needed(model_bundle_uri, tmp_dir)
model_bundle_dir = join(tmp_dir, 'model-bundle')
unzip(model_bundle_path, model_bundle_dir)
config_path = join(model_bundle_dir, 'learner-config.json')
model_path = join(model_bundle_dir, 'model.pth')
cfg = build_config(file_to_json(config_path))
return cfg.build(tmp_dir, model_path=model_path)
def check_eval(test, tmp_dir):
errors = []
actual_eval_path = get_actual_eval_path(test, tmp_dir)
expected_eval_path = get_expected_eval_path(test)
if isfile(actual_eval_path):
expected_eval = file_to_json(expected_eval_path)['overall']
actual_eval = file_to_json(actual_eval_path)['overall']
for expected_item in expected_eval:
class_name = expected_item['class_name']
actual_item = \
next(filter(
lambda x: x['class_name'] == class_name, actual_eval))
errors.extend(check_eval_item(test, expected_item, actual_item))
else:
errors.append(
TestError(test, 'actual eval file does not exist',
actual_eval_path))
return errors
def _run_command(cfg_json_uri: str,
command: str,
split_ind: Optional[int] = None,
num_splits: Optional[int] = None,
runner: Optional[str] = None):
"""Run a single command using a serialized PipelineConfig.
Args:
cfg_json_uri: URI of a JSON file with a serialized PipelineConfig
command: name of command to run
split_ind: the index that a split command should assume
num_splits: the total number of splits to use
runner: the name of the runner to use
"""
pipeline_cfg_dict = file_to_json(cfg_json_uri)
rv_config_dict = pipeline_cfg_dict.get('rv_config')
rv_config.set_everett_config(profile=rv_config.profile,
config_overrides=rv_config_dict)
tmp_dir_obj = rv_config.get_tmp_dir()
tmp_dir = tmp_dir_obj.name
cfg = build_config(pipeline_cfg_dict)
pipeline = cfg.build(tmp_dir)
if num_splits is not None and split_ind is None and runner is not None:
runner = registry.get_runner(runner)()
split_ind = runner.get_split_ind()
command_fn = getattr(pipeline, command)
if num_splits is not None and num_splits > 1:
msg = 'Running {} command split {}/{}...'.format(
command, split_ind + 1, num_splits)
click.echo(click.style(msg, fg='green'))
command_fn(split_ind=split_ind, num_splits=num_splits)
else:
msg = 'Running {} command...'.format(command)
click.echo(click.style(msg, fg='green'))
command_fn()
def __init__(self, img_dir, annotation_uri, transform=None):
self.img_dir = img_dir
self.annotation_uri = annotation_uri
self.transform = transform
self.img_ids = []
self.id2ann = {}
ann_json = file_to_json(annotation_uri)
for img in ann_json['images']:
img_id = img['id']
self.img_ids.append(img_id)
self.id2ann[img_id] = {
'image': img['file_name'],
'bboxes': [],
'category_id': []
}
for ann in ann_json['annotations']:
img_id = ann['image_id']
bboxes = self.id2ann[img_id]['bboxes']
category_ids = self.id2ann[img_id]['category_id']
bboxes.append(ann['bbox'])
category_ids.append(ann['category_id'])
def get_vector_scene(self, class_id, use_aoi=False):
gt_uri = data_file_path('{}-gt-polygons.geojson'.format(class_id))
pred_uri = data_file_path('{}-pred-polygons.geojson'.format(class_id))
scene_id = str(class_id)
rs = MockRasterSource(channel_order=[0, 1, 3], num_channels=3)
rs.set_raster(np.zeros((10, 10, 3)))
crs_transformer = IdentityCRSTransformer()
extent = Box.make_square(0, 0, 360)
config = RasterizedSourceConfig(
vector_source=GeoJSONVectorSourceConfig(uri=gt_uri,
default_class_id=0),
rasterizer_config=RasterizerConfig(background_class_id=1))
gt_rs = config.build(self.class_config, crs_transformer, extent)
gt_ls = SemanticSegmentationLabelSource(raster_source=gt_rs)
config = RasterizedSourceConfig(
vector_source=GeoJSONVectorSourceConfig(uri=pred_uri,
default_class_id=0),
rasterizer_config=RasterizerConfig(background_class_id=1))
pred_rs = config.build(self.class_config, crs_transformer, extent)
pred_ls = SemanticSegmentationLabelSource(raster_source=pred_rs)
pred_ls.vector_output = [
PolygonVectorOutputConfig(uri=pred_uri,
denoise=0,
class_id=class_id)
]
if use_aoi:
aoi_uri = data_file_path('{}-aoi.geojson'.format(class_id))
aoi_geojson = file_to_json(aoi_uri)
aoi_polygons = [shape(aoi_geojson['features'][0]['geometry'])]
return Scene(scene_id, rs, gt_ls, pred_ls, aoi_polygons)
return Scene(scene_id, rs, gt_ls, pred_ls)