def run_command(command_config_uri, tempdir): """Run a command from a serialized command configuration at COMMAND_CONFIG_URI. """ if tempdir is not None: RVConfig.set_tmp_dir(tempdir) rv.runner.CommandRunner.run(command_config_uri)
def test_set_tmp_dir(self): if platform.system() == 'Linux': directory = '/tmp/xxx/' while os.path.exists(directory): directory = directory + 'xxx/' self.assertFalse(os.path.exists(directory)) RVConfig.set_tmp_dir(directory) self.assertTrue(os.path.exists(directory)) self.assertTrue(os.path.isdir(directory)) shutil.rmtree(directory)
def test_command_run_with_mocks(self): with RVConfig.get_tmp_dir() as tmp_dir: predict_package_uri = os.path.join(tmp_dir, 'predict_package.zip') task_config = rv.TaskConfig.builder(mk.MOCK_TASK).build() backend_config = rv.BackendConfig.builder(mk.MOCK_BACKEND).build() scene = mk.create_mock_scene() analyzer_config = rv.AnalyzerConfig.builder( mk.MOCK_ANALYZER).build() cmd_conf = rv.command.rv.CommandConfig.builder(rv.BUNDLE) \ .with_task(task_config) \ .with_backend(backend_config) \ .with_scene(scene) \ .with_analyzers([analyzer_config]) \ .with_root_uri('.') \ .build() cmd_conf = rv.command.CommandConfig.from_proto(cmd_conf.to_proto()) cmd_conf.task.predict_package_uri = predict_package_uri analyzer_config = cmd_conf.analyzers[0] cmd = cmd_conf.create_command() cmd.run() self.assertTrue(os.path.exists(predict_package_uri)) self.assertTrue(analyzer_config.mock.save_bundle_files.called)
def test_respects_utilizes_gpu(self): config = self.mock_config() config['AWS_BATCH_job_queue'] = 'GPU_JOB_QUEUE' config['AWS_BATCH_job_definition'] = 'GPU_JOB_DEF' config['AWS_BATCH_cpu_job_queue'] = 'CPU_JOB_QUEUE' config['AWS_BATCH_cpu_job_definition'] = 'CPU_JOB_DEF' rv._registry.initialize_config(config_overrides=config) with RVConfig.get_tmp_dir() as tmp_dir: e = mk.create_mock_experiment().to_builder() \ .with_root_uri(tmp_dir) \ .clear_command_uris() \ .build() runner = MockAwsBatchExperimentRunner() runner.run( e, commands_to_run=[rv.CHIP, rv.TRAIN, rv.PREDICT, rv.EVAL]) submit_args = runner.mock_client.submit_job.call_args_list self.assertEqual(len(submit_args), 4) for args in submit_args: jobName, jobQueue = args[1]['jobName'], args[1]['jobQueue'] if 'EVAL' in jobName or 'CHIP' in jobName: self.assertTrue('CPU' in jobQueue) else: self.assertTrue('GPU' in jobQueue)
def test_mask(self): with RVConfig.get_tmp_dir() as temp_dir: # make geotiff filled with ones and zeros and mask the whole image image_path = os.path.join(temp_dir, 'temp.tif') height = 100 width = 100 nb_channels = 3 with rasterio.open(image_path, 'w', driver='GTiff', height=height, width=width, count=nb_channels, dtype=np.uint8) as image_dataset: im = np.random.randint( 0, 2, (height, width, nb_channels)).astype(np.uint8) for channel in range(nb_channels): image_dataset.write(im[:, :, channel], channel + 1) image_dataset.write_mask( np.zeros(im.shape[0:2]).astype(np.bool)) source = rv.RasterSourceConfig.builder(rv.RASTERIO_SOURCE) \ .with_uri(image_path) \ .build() \ .create_source(tmp_dir=temp_dir) with source.activate(): out_chip = source.get_image_array() expected_out_chip = np.zeros((height, width, nb_channels)) np.testing.assert_equal(out_chip, expected_out_chip)
def test_command_through_experiment(self): src_path = data_file_path('small-rgb-tile.tif') with RVConfig.get_tmp_dir() as tmp_dir: cog_path = os.path.join(tmp_dir, 'cog.tif') e = mk.create_mock_experiment().to_builder() \ .with_root_uri(tmp_dir) \ .with_custom_config({ 'cogify': { 'key': 'test', 'config': { 'uris': [(src_path, cog_path)], 'block_size': 128 } } }) \ .build() rv.ExperimentRunner.get_runner(rv.LOCAL).run( e, splits=2, commands_to_run=[rv.COGIFY]) # Check that it's cogified with rasterio.open(cog_path) as ds: self.assertEqual(ds.block_shapes, [(128, 128), (128, 128), (128, 128)]) self.assertEqual(ds.overviews(1), [2, 4, 8, 16, 32])
def main(tests, rv_root, verbose, use_tf): """Runs RV end-to-end and checks that evaluation metrics are correct.""" if len(tests) == 0: tests = all_tests if verbose: rv._registry.initialize_config( verbosity=rv.cli.verbosity.Verbosity.DEBUG) tests = list(map(lambda x: x.upper(), tests)) with RVConfig.get_tmp_dir() as temp_dir: if rv_root: temp_dir = rv_root errors = [] for test in tests: if test not in all_tests: print('{} is not a valid test.'.format(test)) return errors.extend(run_test(test, use_tf, temp_dir)) for error in errors: print(error) for test in tests: nb_test_errors = len( list(filter(lambda error: error.test == test, errors))) if nb_test_errors == 0: print('{} test passed!'.format(test)) if errors: exit(1)
def test_make_predict_windows_with_aoi(self): task_config = rv.TaskConfig.builder(rv.CHIP_CLASSIFICATION) \ .with_chip_size(200) \ .with_classes(['car', 'building', 'background']) \ .build() backend_config = rv.BackendConfig.builder(rv.KERAS_CLASSIFICATION) \ .with_task(task_config) \ .with_model_defaults(rv.RESNET50_IMAGENET) \ .with_pretrained_model(None) \ .build() label_source_uri = data_file_path('evaluator/cc-label-full.json') label_source = rv.LabelSourceConfig.builder(rv.CHIP_CLASSIFICATION_GEOJSON) \ .with_uri(label_source_uri) \ .build() label_source_2_uri = data_file_path('evaluator/cc-label-filtered.json') label_source_2 = rv.LabelSourceConfig.builder(rv.CHIP_CLASSIFICATION_GEOJSON) \ .with_uri(label_source_2_uri) \ .build() source_uri = data_file_path('evaluator/cc-label-img-blank.tif') raster_source = rv.RasterSourceConfig.builder(rv.RASTERIO_SOURCE) \ .with_uri(source_uri) \ .build() aoi_uri = data_file_path('evaluator/cc-label-aoi.json') s = rv.SceneConfig.builder() \ .with_id('test') \ .with_raster_source(raster_source) \ .with_label_source(label_source) \ .with_aoi_uri(aoi_uri) \ .build() with RVConfig.get_tmp_dir() as tmp_dir: scene = s.create_scene(task_config, tmp_dir) backend = backend_config.create_backend(task_config) task = task_config.create_task(backend) with scene.activate(): windows = task.get_train_windows(scene) from rastervision.data import (ChipClassificationLabels, ChipClassificationGeoJSONStore) labels = ChipClassificationLabels() for w in windows: labels.set_cell(w, 1) store = ChipClassificationGeoJSONStore( os.path.join(tmp_dir, 'test.json'), scene.raster_source.get_crs_transformer(), task_config.class_map) store.save(labels) ls = label_source_2.create_source( task_config, scene.raster_source.get_extent(), scene.raster_source.get_crs_transformer(), tmp_dir) actual = ls.get_labels().get_cells() self.assertEqual(len(windows), len(actual))
def test_command_create(self): task = rv.TaskConfig.builder(mk.MOCK_TASK).build() with RVConfig.get_tmp_dir() as tmp_dir: img_path = os.path.join(tmp_dir, 'img.tif') chip = np.ones((2, 2, 4)).astype(np.uint8) chip[:, :, :] *= np.array([0, 1, 2, 3]).astype(np.uint8) save_img(chip, img_path) source = rv.data.RasterioSourceConfig(img_path) scenes = [rv.data.SceneConfig('', source)] analyzers = [ rv.analyzer.StatsAnalyzerConfig(stats_uri='dummy_path') ] cmd_conf = rv.command.AnalyzeCommandConfig.builder() \ .with_task(task) \ .with_root_uri(tmp_dir) \ .with_scenes(scenes) \ .with_analyzers(analyzers) \ .build() cmd_conf = rv.command.CommandConfig.from_proto(cmd_conf.to_proto()) cmd = cmd_conf.create_command() self.assertTrue(cmd, rv.command.AnalyzeCommand)
def _run_experiment(self, command_dag): """Runs all commands on this machine.""" def run_commands(tmp_dir): for command_config in command_dag.get_sorted_commands(): msg = command_config.to_proto() builder = rv._registry.get_command_config_builder( msg.command_type)() command_config = builder.from_proto(msg).build() command_root_uri = command_config.root_uri command_basename = 'command-config-{}.json'.format( command_config.split_id) command_uri = os.path.join(command_root_uri, command_basename) log.info('Saving command configuration to {}...'.format( command_uri)) save_json_config(command_config.to_proto(), command_uri) command = command_config.create_command() command.run(tmp_dir) if self.tmp_dir: run_commands(self.tmp_dir) else: with RVConfig.get_tmp_dir() as tmp_dir: run_commands(tmp_dir)
def test_get_dtype(self): img_path = data_file_path('small-rgb-tile.tif') with RVConfig.get_tmp_dir() as tmp_dir: source = rv.data.RasterioSourceConfig(uris=[img_path]) \ .create_source(tmp_dir) self.assertEqual(source.get_dtype(), np.uint8)
def test_command_create_no_compression(self): src_path = data_file_path('small-rgb-tile.tif') with RVConfig.get_tmp_dir() as tmp_dir: cog_path = os.path.join(tmp_dir, 'cog.tif') cmd_conf = rv.CommandConfig.builder(rv.COGIFY) \ .with_root_uri(tmp_dir) \ .with_config(uris=[(src_path, cog_path)], block_size=128, compression='none') \ .build() cmd_conf = rv.command.CommandConfig.from_proto(cmd_conf.to_proto()) cmd = cmd_conf.create_command() self.assertTrue(cmd, rv.command.aux.CogifyCommand) cmd.run(tmp_dir) # Check that it's cogified with rasterio.open(cog_path) as ds: self.assertEqual(ds.block_shapes, [(128, 128), (128, 128), (128, 128)]) self.assertEqual(ds.overviews(1), [2, 4, 8, 16, 32]) self.assertIsNone(ds.compression)
def test_no_epsg(self): crs = rasterio.crs.CRS() with RVConfig.get_tmp_dir() as tmp_dir: image_path = os.path.join(tmp_dir, 'temp.tif') height = 100 width = 100 nb_channels = 3 with rasterio.open(image_path, 'w', driver='GTiff', height=height, width=width, count=nb_channels, dtype=np.uint8, crs=crs) as image_dataset: im = np.zeros((height, width, nb_channels)).astype(np.uint8) for channel in range(nb_channels): image_dataset.write(im[:, :, channel], channel + 1) try: rv.RasterSourceConfig.builder(rv.RASTERIO_SOURCE) \ .with_uri(image_path) \ .build() \ .create_source(tmp_dir=tmp_dir) except Exception: self.fail( 'Creating RasterioSource with CRS with no EPSG attribute ' 'raised an exception when it should not have.')
def test_gets_raw_chip_from_uint16_transformed_proto(self): img_path = data_file_path('small-uint16-tile.tif') channel_order = [0, 1] with RVConfig.get_tmp_dir() as temp_dir: stats_uri = os.path.join(temp_dir, 'temp.tif') stats = RasterStats() stats.compute([ rv.RasterSourceConfig.builder(rv.RASTERIO_SOURCE).with_uri( img_path).build().create_source(temp_dir) ]) stats.save(stats_uri) transformer = rv.RasterTransformerConfig.builder(rv.STATS_TRANSFORMER) \ .with_stats_uri(stats_uri) \ .build() msg = rv.RasterSourceConfig.builder(rv.RASTERIO_SOURCE) \ .with_uri(img_path) \ .with_channel_order(channel_order) \ .with_transformer(transformer) \ .build() \ .to_proto() source = rv.RasterSourceConfig.from_proto(msg) \ .create_source(tmp_dir=None) with source.activate(): out_chip = source.get_raw_image_array() self.assertEqual(out_chip.shape[2], 3)
def setUp(self): self.crs_transformer = DoubleCRSTransformer() self.geojson = { 'type': 'FeatureCollection', 'features': [{ 'type': 'Feature', 'geometry': { 'type': 'MultiPolygon', 'coordinates': [[[[0., 0.], [0., 2.], [2., 2.], [2., 0.], [0., 0.]]]] }, 'properties': { 'class_name': 'car', 'class_id': 1, 'score': 0.0 } }, { 'type': 'Feature', 'geometry': { 'type': 'Polygon', 'coordinates': [[[2., 2.], [2., 4.], [4., 4.], [4., 2.], [2., 2.]]] }, 'properties': { 'score': 0.0, 'class_name': 'house', 'class_id': 2 } }] } self.class_map = ClassMap([ClassItem(1, 'car'), ClassItem(2, 'house')]) class MockTaskConfig(): def __init__(self, class_map): self.class_map = class_map self.task_config = MockTaskConfig(self.class_map) self.box1 = Box.make_square(0, 0, 4) self.box2 = Box.make_square(4, 4, 4) self.class_id1 = 1 self.class_id2 = 2 self.background_class_id = 3 geoms = [] for f in self.geojson['features']: g = shape(f['geometry']) g.class_id = f['properties']['class_id'] geoms.append(g) self.str_tree = STRtree(geoms) self.file_name = 'labels.json' self.temp_dir = RVConfig.get_tmp_dir() self.uri = os.path.join(self.temp_dir.name, self.file_name) json_to_file(self.geojson, self.uri)
def setUp(self): self.crs_transformer = IdentityCRSTransformer() self.extent = Box.make_square(0, 0, 10) self.tmp_dir = RVConfig.get_tmp_dir() self.class_id = 2 self.background_class_id = 3 self.line_buffer = 1 self.uri = os.path.join(self.tmp_dir.name, 'temp.json')
def get_request_payer(): # Import here to avoid circular reference. from rastervision.rv_config import RVConfig rv_config = RVConfig.get_instance() s3_config = rv_config.get_subconfig('AWS_S3') # 'None' needs the quotes because boto3 cannot handle None. return ('requester' if s3_config( 'requester_pays', parser=bool, default='False') else 'None')
def save_debug_predict_image(self, scene, debug_dir_uri): img = draw_debug_predict_image(scene, self.config.class_map) # Saving to a jpg leads to segfault for unknown reasons. debug_image_uri = join(debug_dir_uri, scene.id + '.png') with RVConfig.get_tmp_dir() as temp_dir: debug_image_path = get_local_path(debug_image_uri, temp_dir) make_dir(debug_image_path, use_dirname=True) img.save(debug_image_path) upload_or_copy(debug_image_path, debug_image_uri)
def get_tile_features(tile_uri, z, x, y): """Get GeoJSON features for a specific tile using in-memory caching.""" cache_dir = os.path.join(RVConfig.get_tmp_dir_root(), 'vector-tiles') tile_path = get_cached_file(cache_dir, tile_uri) cmd = ['tippecanoe-decode', '-f', '-c', tile_path, str(z), str(x), str(y)] tile_geojson_str = check_output(cmd).decode('utf-8') tile_features = [json.loads(ts) for ts in tile_geojson_str.split('\n')] return tile_features
def test_required_fields(self): with RVConfig.get_tmp_dir() as tmp_dir: b = rv.CommandConfig.builder(mk.MOCK_AUX_COMMAND) \ .with_config() \ .with_root_uri(tmp_dir) with self.assertRaises(rv.ConfigError) as context: b.build() self.assertTrue('uris' in str(context.exception))
def get_tmp_dir(self): if hasattr(self, '_tmp_dir') and self._tmp_dir: if isinstance(self._tmp_dir, str): return self._tmp_dir else: return self._tmp_dir.name else: tmp_dir = RVConfig.get_tmp_dir() self.set_tmp_dir(tmp_dir) return tmp_dir.name
def setup_plugin_logger(root_name): plugin_logger = logging.getLogger(root_name) sh = logging.StreamHandler() sh.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s:%(name)s: %(levelname)s - %(message)s', '%Y-%m-%d %H:%M:%S') sh.setFormatter(formatter) plugin_logger.addHandler(sh) plugin_logger.setLevel(RVConfig.get_instance().verbosity)
def create_command(self, tmp_dir=None): if not tmp_dir: _tmp_dir = RVConfig.get_tmp_dir() tmp_dir = _tmp_dir.name else: _tmp_dir = tmp_dir retval = BundleCommand(self) retval.set_tmp_dir(_tmp_dir) return retval
def test_bundle_od_command(self): def get_task(tmp_dir): predict_package_uri = os.path.join(tmp_dir, 'predict_package.zip') t = rv.TaskConfig.builder(rv.OBJECT_DETECTION) \ .with_predict_package_uri(predict_package_uri) \ .with_classes(['class1']) \ .build() return t def get_backend(task, tmp_dir): model_uri = os.path.join(tmp_dir, 'model') template_uri = data_file_path( 'tf_object_detection/embedded_ssd_mobilenet_v1_coco.config') with open(model_uri, 'w') as f: f.write('DUMMY') b = rv.BackendConfig.builder(rv.TF_OBJECT_DETECTION) \ .with_task(task) \ .with_template(template_uri) \ .with_model_uri(model_uri) \ .build() return b with RVConfig.get_tmp_dir() as tmp_dir: task = get_task(tmp_dir) backend = get_backend(task, tmp_dir) analyzer = self.get_analyzer(tmp_dir) scene = self.get_scene(tmp_dir) cmd = rv.CommandConfig.builder(rv.BUNDLE) \ .with_task(task) \ .with_root_uri(tmp_dir) \ .with_backend(backend) \ .with_analyzers([analyzer]) \ .with_scene(scene) \ .build() \ .create_command() cmd.run(tmp_dir) package_dir = os.path.join(tmp_dir, 'package') make_dir(package_dir) with zipfile.ZipFile(task.predict_package_uri, 'r') as package_zip: package_zip.extractall(path=package_dir) bundle_config_path = os.path.join(package_dir, 'bundle_config.json') bundle_config = load_json_config(bundle_config_path, CommandConfigMsg()) self.assertEqual(bundle_config.command_type, rv.BUNDLE) actual = set(os.listdir(package_dir)) expected = set(['stats.json', 'model', 'bundle_config.json']) self.assertEqual(actual, expected)
def initialize_config(self, profile=None, rv_home=None, config_overrides=None, verbosity=Verbosity.NORMAL): self._rv_config = RVConfig(profile=profile, rv_home=rv_home, config_overrides=config_overrides, verbosity=verbosity) # Reset the plugins in case this is a re-initialization, self._plugin_registry = None
def create_command(self, tmp_dir=None): if not tmp_dir: _tmp_dir = RVConfig.get_tmp_dir() tmp_dir = _tmp_dir.name else: _tmp_dir = tmp_dir retval = self.command_class(self.config) retval.set_tmp_dir(_tmp_dir) return retval
def _merge_training_results(results, record_path, split): # merge each scene's tfrecord into "split" tf record merge_tf_records(record_path, results) # Save debug chips. if self.config.debug: debug_zip_path = training_package.get_local_path( training_package.get_debug_chips_uri(split)) with RVConfig.get_tmp_dir() as debug_dir: make_debug_images(record_path, self.class_map, debug_dir) shutil.make_archive( os.path.splitext(debug_zip_path)[0], 'zip', debug_dir)
def test_no_config_error(self): task = rv.task.ChipClassificationConfig({}) try: with RVConfig.get_tmp_dir() as tmp_dir: rv.command.AnalyzeCommandConfig.builder() \ .with_task(task) \ .with_root_uri(tmp_dir) \ .with_scenes(['']) \ .with_analyzers(['']) \ .build() except rv.ConfigError: self.fail('rv.ConfigError raised unexpectedly')
def test_no_config_error(self): task = rv.task.ChipClassificationConfig({}) backend = rv.backend.KerasClassificationConfig('') try: with RVConfig.get_tmp_dir() as tmp_dir: rv.command.TrainCommandConfig.builder() \ .with_task(task) \ .with_root_uri(tmp_dir) \ .with_backend(backend) \ .build() except rv.ConfigError: self.fail('rv.ConfigError raised unexpectedly')
def setUp(self): # Setup mock S3 bucket. self.mock_s3 = mock_s3() self.mock_s3.start() self.s3 = boto3.client('s3') self.bucket_name = 'mock_bucket' self.s3.create_bucket(Bucket=self.bucket_name) self.content_str = 'hello' self.file_name = 'hello.txt' self.temp_dir = RVConfig.get_tmp_dir() self.cache_dir = os.path.join(self.temp_dir.name, 'cache')