def main(tests, root_uri, verbose):
"""Runs RV end-to-end and checks that evaluation metrics are correct."""
if len(tests) == 0:
tests = all_tests
if verbose:
rv_config.set_verbosity(verbosity=Verbosity.DEBUG)
with rv_config.get_tmp_dir() as tmp_dir:
if root_uri:
tmp_dir = root_uri
errors = []
for test in tests:
if test not in all_tests:
print('{} is not a valid test.'.format(test))
return
errors.extend(run_test(test, tmp_dir))
for error in errors:
print(error)
for test in tests:
nb_test_errors = len(
list(filter(lambda error: error.test == test, errors)))
if nb_test_errors == 0:
print('{} test passed!'.format(test))
if errors:
exit(1)
def setUp(self):
self.tmp_dir = rv_config.get_tmp_dir()
self.class_config = ClassConfig(names=['one', 'two'])
self.class_config.update()
self.class_config.ensure_null_class()
self.null_class_id = self.class_config.get_null_class_id()
def test_accounts_for_aoi(self):
class_config = ClassConfig(names=['car', 'building', 'background'])
label_source_uri = data_file_path('evaluator/cc-label-filtered.json')
label_source_cfg = ChipClassificationLabelSourceConfig(
vector_source=GeoJSONVectorSourceConfig(
uri=label_source_uri, default_class_id=None))
label_store_uri = data_file_path('evaluator/cc-label-full.json')
label_store_cfg = ChipClassificationGeoJSONStoreConfig(
uri=label_store_uri)
raster_source_uri = data_file_path('evaluator/cc-label-img-blank.tif')
raster_source_cfg = RasterioSourceConfig(uris=[raster_source_uri])
aoi_uri = data_file_path('evaluator/cc-label-aoi.json')
s = SceneConfig(
id='test',
raster_source=raster_source_cfg,
label_source=label_source_cfg,
label_store=label_store_cfg,
aoi_uris=[aoi_uri])
with rv_config.get_tmp_dir() as tmp_dir:
scene = s.build(class_config, tmp_dir)
output_uri = os.path.join(tmp_dir, 'eval.json')
evaluator = ChipClassificationEvaluatorConfig(
output_uri=output_uri).build(class_config)
evaluator.process([scene], tmp_dir)
overall = file_to_json(output_uri)['overall']
for item in overall:
self.assertEqual(item['f1'], 1.0)
def setUp(self):
self.crs_transformer = IdentityCRSTransformer()
self.extent = Box.make_square(0, 0, 10)
self.tmp_dir_obj = rv_config.get_tmp_dir()
self.tmp_dir = self.tmp_dir_obj.name
self.class_id = 0
self.background_class_id = 1
self.line_buffer = 1
self.class_config = ClassConfig(names=['a'])
self.uri = join(self.tmp_dir, 'tmp.json')
def main(tests, root_uri, verbose):
"""Runs RV end-to-end and checks that evaluation metrics are correct."""
if verbose:
rv_config.set_verbosity(verbosity=Verbosity.DEBUG)
if len(tests) == 0:
# no tests specified, so run all
tests = list(ALL_TESTS.keys())
else:
# run all tests that start with the given string e.g "chip" will match
# both "chip_classification.basic" and "chip_classification.nochip"
_tests = []
for t in tests:
t = t.strip().lower()
matching_tests = [k for k in ALL_TESTS.keys() if k.startswith(t)]
_tests.extend(matching_tests)
if len(matching_tests) == 0:
console_error(
f'{t} does not match any valid tests. Valid tests are: ')
console_error(pformat(list(ALL_TESTS.keys())))
continue
tests = _tests
console_info('The following tests will be run:')
console_info(pformat(tests, compact=False))
with rv_config.get_tmp_dir() as tmp_dir:
if root_uri:
tmp_dir = root_uri
num_failed = 0
errors = {}
for test_id in tests:
test_cfg = ALL_TESTS[test_id]
errors[test_id] = run_test(test_id, test_cfg, tmp_dir)
if len(errors[test_id]) > 0:
num_failed += 1
for error in errors[test_id]:
console_error(str(error))
for test_id in tests:
if test_id not in errors:
continue
if len(errors[test_id]) == 0:
console_success(f'{test_id}: test passed!', bold=True)
else:
console_error(f'{test_id}: test failed!', bold=True)
if num_failed > 0:
console_error(
f'Tests passed: {len(tests) - num_failed} of {len(tests)}')
console_error('Error counts:')
console_error(pformat({k: len(es) for k, es in errors.items()}))
exit(1)
def setUp(self):
# Setup mock S3 bucket.
self.mock_s3 = mock_s3()
self.mock_s3.start()
self.s3 = boto3.client('s3')
self.bucket_name = 'mock_bucket'
self.s3.create_bucket(Bucket=self.bucket_name)
self.content_str = 'hello'
self.file_name = 'hello.txt'
self.tmp_dir = rv_config.get_tmp_dir()
self.cache_dir = os.path.join(self.tmp_dir.name, 'cache')
def setUp(self):
self.lorem = LOREM
# Mock S3 bucket
self.mock_s3 = mock_s3()
self.mock_s3.start()
self.s3 = boto3.client('s3')
self.bucket_name = 'mock_bucket'
self.s3.create_bucket(Bucket=self.bucket_name)
# temporary directory
self.tmp_dir = rv_config.get_tmp_dir()
def setUp(self):
self.crs_transformer = DoubleCRSTransformer()
self.geojson = {
'type':
'FeatureCollection',
'features': [{
'type': 'Feature',
'geometry': {
'type':
'MultiPolygon',
'coordinates': [[[[0., 0.], [0., 2.], [2., 2.], [2., 0.],
[0., 0.]]]]
},
'properties': {
'class_name': 'car',
'class_id': 0,
'score': 0.0
}
}, {
'type': 'Feature',
'geometry': {
'type':
'Polygon',
'coordinates': [[[2., 2.], [2., 4.], [4., 4.], [4., 2.],
[2., 2.]]]
},
'properties': {
'score': 0.0,
'class_name': 'house',
'class_id': 1
}
}]
}
self.class_config = ClassConfig(names=['car', 'house'])
self.box1 = Box.make_square(0, 0, 4)
self.box2 = Box.make_square(4, 4, 4)
self.class_id1 = 0
self.class_id2 = 1
self.background_class_id = 2
geoms = []
for f in self.geojson['features']:
g = shape(f['geometry'])
g.class_id = f['properties']['class_id']
geoms.append(g)
self.str_tree = STRtree(geoms)
self.file_name = 'labels.json'
self.tmp_dir = rv_config.get_tmp_dir()
self.uri = os.path.join(self.tmp_dir.name, self.file_name)
json_to_file(self.geojson, self.uri)
def predict(model_bundle, image_uri, output_uri, update_stats, channel_order):
"""Make predictions on the images at IMAGE_URI
using MODEL_BUNDLE and store the prediction output at OUTPUT_URI.
"""
if channel_order is not None:
channel_order = [
int(channel_ind) for channel_ind in channel_order.split(' ')
]
with rv_config.get_tmp_dir() as tmp_dir:
predictor = Predictor(model_bundle, tmp_dir, update_stats,
channel_order)
predictor.predict([image_uri], output_uri)
def setUp(self):
# Setup mock S3 bucket.
self.mock_s3 = mock_s3()
self.mock_s3.start()
self.s3 = boto3.client('s3')
self.bucket_name = 'mock_bucket'
self.s3.create_bucket(Bucket=self.bucket_name)
self.content_str = 'hello'
self.file_name = 'hello.txt'
self.s3_path = 's3://{}/{}'.format(self.bucket_name, self.file_name)
self.tmp_dir = rv_config.get_tmp_dir()
self.local_path = os.path.join(self.tmp_dir.name, self.file_name)
def test_stats_transformer(self):
raster_stats = RasterStats()
raster_stats.means = list(np.ones((4, )))
raster_stats.stds = list(np.ones((4, )) * 2)
with rv_config.get_tmp_dir() as tmp_dir:
stats_uri = os.path.join(tmp_dir, 'stats.json')
raster_stats.save(stats_uri)
# All values have z-score of 1, which translates to
# uint8 value of 170.
transformer = StatsTransformerConfig(stats_uri=stats_uri).build()
chip = np.ones((2, 2, 4)) * 3
out_chip = transformer.transform(chip)
expected_out_chip = np.ones((2, 2, 4)) * 170
np.testing.assert_equal(out_chip, expected_out_chip)
def _run_command(cfg_json_uri: str,
command: str,
split_ind: Optional[int] = None,
num_splits: Optional[int] = None,
runner: Optional[str] = None):
"""Run a single command using a serialized PipelineConfig.
Args:
cfg_json_uri: URI of a JSON file with a serialized PipelineConfig
command: name of command to run
split_ind: the index that a split command should assume
num_splits: the total number of splits to use
runner: the name of the runner to use
"""
pipeline_cfg_dict = file_to_json(cfg_json_uri)
rv_config_dict = pipeline_cfg_dict.get('rv_config')
rv_config.set_everett_config(profile=rv_config.profile,
config_overrides=rv_config_dict)
tmp_dir_obj = rv_config.get_tmp_dir()
tmp_dir = tmp_dir_obj.name
cfg = build_config(pipeline_cfg_dict)
pipeline = cfg.build(tmp_dir)
if num_splits is not None and split_ind is None and runner is not None:
runner = registry.get_runner(runner)()
split_ind = runner.get_split_ind()
command_fn = getattr(pipeline, command)
if num_splits is not None and num_splits > 1:
msg = 'Running {} command split {}/{}...'.format(
command, split_ind + 1, num_splits)
click.secho(msg, fg='green', bold=True)
command_fn(split_ind=split_ind, num_splits=num_splits)
else:
msg = 'Running {} command...'.format(command)
click.secho(msg, fg='green', bold=True)
command_fn()
def setUp(self):
self.file_name = 'labels.json'
self.tmp_dir = rv_config.get_tmp_dir()
self.file_path = os.path.join(self.tmp_dir.name, self.file_name)
self.crs_transformer = DoubleCRSTransformer()
self.geojson = {
'type':
'FeatureCollection',
'features': [{
'type': 'Feature',
'geometry': {
'type':
'Polygon',
'coordinates': [[[0., 0.], [0., 1.], [1., 1.], [1., 0.],
[0., 0.]]]
},
'properties': {
'class_id': 0,
'score': 0.9
}
}, {
'type': 'Feature',
'geometry': {
'type':
'Polygon',
'coordinates': [[[1., 1.], [1., 2.], [2., 2.], [2., 1.],
[1., 1.]]]
},
'properties': {
'score': 0.9,
'class_id': 1
}
}]
}
self.extent = Box.make_square(0, 0, 10)
self.class_config = ClassConfig(names=['car', 'house'])
json_to_file(self.geojson, self.file_path)
def run(runner: str, cfg_module: str, commands: List[str],
arg: List[Tuple[str, str]], splits: int):
"""Run COMMANDS within pipelines in CFG_MODULE using RUNNER.
RUNNER: name of the Runner to use
CFG_MODULE: the module with `get_configs` function that returns PipelineConfigs.
This can either be a Python module path or a local path to a .py file.
COMMANDS: space separated sequence of commands to run within pipeline. The order in
which to run them is based on the Pipeline.commands attribute. If this is omitted,
all commands will be run.
"""
tmp_dir_obj = rv_config.get_tmp_dir()
tmp_dir = tmp_dir_obj.name
args = dict(arg)
args = convert_bool_args(args)
cfgs = get_configs(cfg_module, runner, args)
runner = registry.get_runner(runner)()
for cfg in cfgs:
_run_pipeline(cfg, runner, tmp_dir, splits, commands)
def setUp(self):
self.lorem = LOREM
self.tmp_dir = rv_config.get_tmp_dir()
def setUp(self):
self.tmp_dir = rv_config.get_tmp_dir()
self.uri = os.path.join(self.tmp_dir.name, 'vectors.json')
def setUp(self):
self.tmp_dir_obj = rv_config.get_tmp_dir()
self.tmp_dir = self.tmp_dir_obj.name
def setUp(self):
self.tmp_dir = rv_config.get_tmp_dir()
