def _run_pipeline(cfg, runner, tmp_dir, splits=1, commands=None):
cfg.update()
cfg.recursive_validate_config()
# This is to run the validation again to check any fields that may have changed
# after the Config was constructed, possibly by the update method.
build_config(cfg.dict())
cfg_json_uri = cfg.get_config_uri()
save_pipeline_config(cfg, cfg_json_uri)
pipeline = cfg.build(tmp_dir)
if not commands:
commands = pipeline.commands
runner.run(cfg_json_uri, pipeline, commands, num_splits=splits)
def _run_command(cfg_json_uri,
command,
split_ind=None,
num_splits=None,
runner=None):
pipeline_cfg_dict = file_to_json(cfg_json_uri)
rv_config_dict = pipeline_cfg_dict.get('rv_config')
rv_config.reset(
config_overrides=rv_config_dict,
verbosity=rv_config.verbosity,
tmp_dir=rv_config.tmp_dir)
tmp_dir_obj = rv_config.get_tmp_dir()
tmp_dir = tmp_dir_obj.name
cfg = build_config(pipeline_cfg_dict)
pipeline = cfg.build(tmp_dir)
if num_splits is not None and split_ind is None and runner is not None:
runner = registry.get_runner(runner)()
split_ind = runner.get_split_ind()
command_fn = getattr(pipeline, command)
if num_splits is not None and num_splits > 1:
msg = 'Running {} command split {}/{}...'.format(
command, split_ind + 1, num_splits)
click.echo(click.style(msg, fg='green'))
command_fn(split_ind=split_ind, num_splits=num_splits)
else:
msg = 'Running {} command...'.format(command)
click.echo(click.style(msg, fg='green'))
command_fn()
def from_model_bundle(model_bundle_uri, tmp_dir):
model_bundle_path = download_if_needed(model_bundle_uri, tmp_dir)
model_bundle_dir = join(tmp_dir, 'model-bundle')
unzip(model_bundle_path, model_bundle_dir)
config_path = join(model_bundle_dir, 'config.json')
model_path = join(model_bundle_dir, 'model.pth')
cfg = build_config(file_to_json(config_path))
return cfg.build(tmp_dir, model_path=model_path)
def from_model_bundle(model_bundle_uri: str, tmp_dir: str):
"""Create a Learner from a model bundle."""
model_bundle_path = download_if_needed(model_bundle_uri, tmp_dir)
model_bundle_dir = join(tmp_dir, 'model-bundle')
unzip(model_bundle_path, model_bundle_dir)
config_path = join(model_bundle_dir, 'learner-config.json')
model_path = join(model_bundle_dir, 'model.pth')
cfg = build_config(file_to_json(config_path))
return cfg.build(tmp_dir, model_path=model_path)
def run(runner: str, cfg_module: str, commands: List[str],
arg: List[Tuple[str, str]], splits: int):
"""Run COMMANDS within pipelines in CFG_MODULE using RUNNER.
RUNNER: name of the Runner to use
CFG_MODULE: the module with `get_configs` function that returns PipelineConfigs.
This can either be a Python module path or a local path to a .py file.
COMMANDS: space separated sequence of commands to run within pipeline. The order in
which to run them is based on the Pipeline.commands attribute. If this is omitted,
all commands will be run.
"""
tmp_dir_obj = rv_config.get_tmp_dir()
tmp_dir = tmp_dir_obj.name
args = dict(arg)
args = convert_bool_args(args)
cfgs = get_configs(cfg_module, runner, args)
runner = registry.get_runner(runner)()
for cfg in cfgs:
cfg.update()
cfg.rv_config = rv_config.get_config_dict(registry.rv_config_schema)
cfg.recursive_validate_config()
# This is to run the validation again to check any fields that may have changed
# after the Config was constructed, possibly by the update method.
build_config(cfg.dict())
cfg_json = cfg.json()
cfg_json_uri = cfg.get_config_uri()
str_to_file(cfg_json, cfg_json_uri)
pipeline = cfg.build(tmp_dir)
if not commands:
commands = pipeline.commands
runner.run(cfg_json_uri, pipeline, commands, num_splits=splits)
def test_to_from(self):
cfg = CConfig(al=[AConfig(), ASub1Config(),
ASub2Config()],
bl=[BConfig()],
a=ASub1Config(),
b=BConfig(),
plugin_versions=self.plugin_versions,
root_uri=None,
rv_config=None)
exp_dict = {
'plugin_versions':
self.plugin_versions,
'root_uri':
None,
'rv_config':
None,
'type_hint':
'c',
'a': {
'type_hint': 'asub1',
'x': 'x',
'y': 'y'
},
'al': [{
'type_hint': 'a',
'x': 'x'
}, {
'type_hint': 'asub1',
'x': 'x',
'y': 'y'
}, {
'type_hint': 'asub2',
'x': 'x',
'y': 'y'
}],
'b': {
'type_hint': 'b',
'x': 'x'
},
'bl': [{
'type_hint': 'b',
'x': 'x'
}],
'x':
'x'
}
self.assertDictEqual(cfg.dict(), exp_dict)
self.assertEqual(build_config(exp_dict), cfg)
def test_to_from(self):
cfg = CConfig(
al=[AConfig(), ASub1Config(),
ASub2Config()],
bl=[BConfig()],
a=ASub1Config(),
b=BConfig())
exp_dict = {
'type_hint':
'c',
'version':
1,
'a': {
'type_hint': 'asub1',
'x': 'x',
'y': 'y'
},
'al': [{
'x': 'x'
}, {
'type_hint': 'asub1',
'x': 'x',
'y': 'y'
}, {
'type_hint': 'asub2',
'x': 'x',
'y': 'y'
}],
'b': {
'x': 'x'
},
'bl': [{
'x': 'x'
}],
'x':
'x'
}
self.assertDictEqual(cfg.dict(), exp_dict)
self.assertEqual(build_config(exp_dict), cfg)
def _run_command(cfg_json_uri: str,
command: str,
split_ind: Optional[int] = None,
num_splits: Optional[int] = None,
runner: Optional[str] = None):
"""Run a single command using a serialized PipelineConfig.
Args:
cfg_json_uri: URI of a JSON file with a serialized PipelineConfig
command: name of command to run
split_ind: the index that a split command should assume
num_splits: the total number of splits to use
runner: the name of the runner to use
"""
pipeline_cfg_dict = file_to_json(cfg_json_uri)
rv_config_dict = pipeline_cfg_dict.get('rv_config')
rv_config.set_everett_config(profile=rv_config.profile,
config_overrides=rv_config_dict)
tmp_dir_obj = rv_config.get_tmp_dir()
tmp_dir = tmp_dir_obj.name
cfg = build_config(pipeline_cfg_dict)
pipeline = cfg.build(tmp_dir)
if num_splits is not None and split_ind is None and runner is not None:
runner = registry.get_runner(runner)()
split_ind = runner.get_split_ind()
command_fn = getattr(pipeline, command)
if num_splits is not None and num_splits > 1:
msg = 'Running {} command split {}/{}...'.format(
command, split_ind + 1, num_splits)
click.echo(click.style(msg, fg='green'))
command_fn(split_ind=split_ind, num_splits=num_splits)
else:
msg = 'Running {} command...'.format(command)
click.echo(click.style(msg, fg='green'))
command_fn()
def __init__(self,
model_bundle_uri,
tmp_dir,
update_stats=False,
channel_order=None):
"""Creates a new Predictor.
Args:
model_bundle_uri: URI of the model bundle to use. Can be any
type of URI that Raster Vision can read.
tmp_dir: Temporary directory in which to store files that are used
by the Predictor. This directory is not cleaned up by this
class.
channel_order: Option for a new channel order to use for the
imagery being predicted against. If not present, the
channel_order from the original configuration in the predict
package will be used.
"""
self.tmp_dir = tmp_dir
self.update_stats = update_stats
self.model_loaded = False
bundle_path = download_if_needed(model_bundle_uri, tmp_dir)
bundle_dir = join(tmp_dir, 'bundle')
make_dir(bundle_dir)
with zipfile.ZipFile(bundle_path, 'r') as bundle_zip:
bundle_zip.extractall(path=bundle_dir)
config_path = join(bundle_dir, 'pipeline-config.json')
config_dict = file_to_json(config_path)
rv_config.set_everett_config(
config_overrides=config_dict.get('rv_config'))
config_dict = upgrade_config(config_dict)
self.pipeline = build_config(config_dict).build(tmp_dir)
self.scene = None
if not hasattr(self.pipeline, 'predict'):
raise Exception(
'pipeline in model bundle must have predict method')
self.scene = self.pipeline.config.dataset.validation_scenes[0]
if not hasattr(self.scene.raster_source, 'uris'):
raise Exception(
'raster_source in model bundle must have uris as field')
if not hasattr(self.scene.label_store, 'uri'):
raise Exception(
'label_store in model bundle must have uri as field')
for t in self.scene.raster_source.transformers:
t.update_root(bundle_dir)
if self.update_stats:
stats_analyzer = StatsAnalyzerConfig(
output_uri=join(bundle_dir, 'stats.json'))
self.pipeline.config.analyzers = [stats_analyzer]
self.scene.label_source = None
self.scene.aoi_uris = None
self.pipeline.config.dataset.train_scenes = [self.scene]
self.pipeline.config.dataset.validation_scenes = [self.scene]
self.pipeline.config.dataset.test_scenes = None
if channel_order is not None:
self.scene.raster_source.channel_order = channel_order
