def test_create_runner_parser_check_fails_unknown() -> None: """ Test parsing of commandline arguments: Check if unknown arguments fail the parsing """ azure_parser = create_runner_parser(SegmentationModelBase) valid_args = ["--model=Lung"] invalid_args = ["--unknown=1"] # Ensure that the valid arguments that we provide later do actually parse correctly with mock.patch("sys.argv", [""] + valid_args): result = parse_args_and_add_yaml_variables(azure_parser, fail_on_unknown_args=True) assert "model" in result.args assert result.args["model"] == "Lung" # Supply both valid and invalid arguments, and we expect a failure because of the invalid ones: with mock.patch("sys.argv", [""] + valid_args + invalid_args): with pytest.raises(Exception) as e_info: parse_args_and_add_yaml_variables(azure_parser, fail_on_unknown_args=True) assert str(e_info.value) == 'Unknown arguments: [\'--unknown=1\']' # Supply both valid and invalid arguments, and we expect that the invalid ones are silently ignored: with mock.patch("sys.argv", [""] + valid_args + invalid_args): result = parse_args_and_add_yaml_variables(azure_parser, fail_on_unknown_args=False) assert "model" in result.args assert result.args["model"] == "Lung" assert invalid_args[0] in result.unknown
def parse_and_load_model(self) -> ParserResult: """ Parses the command line arguments, and creates configuration objects for the model itself, and for the Azure-related parameters. Sets self.azure_config and self.model_config to their proper values. Returns the parser output from parsing the model commandline arguments. If no "model" argument is provided on the commandline, self.model_config will be set to None, and the return value is None. """ # Create a parser that will understand only the args we need for an AzureConfig parser1 = create_runner_parser() parser_result = parse_args_and_add_yaml_variables(parser1, yaml_config_file=self.yaml_config_file, project_root=self.project_root, fail_on_unknown_args=False) azure_config = AzureConfig(**parser_result.args) azure_config.project_root = self.project_root self.azure_config = azure_config self.model_config = None if not azure_config.model: raise ValueError("Parameter 'model' needs to be set to tell InnerEye which model to run.") model_config_loader: ModelConfigLoader = ModelConfigLoader(**parser_result.args) # Create the model as per the "model" commandline option. This can return either a built-in config # of type DeepLearningConfig, or a LightningContainer. config_or_container = model_config_loader.create_model_config_from_name(model_name=azure_config.model) def parse_overrides_and_apply(c: object, previous_parser_result: ParserResult) -> ParserResult: assert isinstance(c, GenericConfig) parser = type(c).create_argparser() # For each parser, feed in the unknown settings from the previous parser. All commandline args should # be consumed by name, hence fail if there is something that is still unknown. parser_result = parse_arguments(parser, settings_from_yaml=previous_parser_result.unknown_settings_from_yaml, args=previous_parser_result.unknown, fail_on_unknown_args=True) # Apply the overrides and validate. Overrides can come from either YAML settings or the commandline. c.apply_overrides(parser_result.known_settings_from_yaml) c.apply_overrides(parser_result.overrides) c.validate() return parser_result # Now create a parser that understands overrides at model/container level. parser_result = parse_overrides_and_apply(config_or_container, parser_result) if isinstance(config_or_container, LightningContainer): self.lightning_container = config_or_container elif isinstance(config_or_container, ModelConfigBase): # Built-in InnerEye models use a fake container self.model_config = config_or_container self.lightning_container = InnerEyeContainer(config_or_container) else: raise ValueError(f"Don't know how to handle a loaded configuration of type {type(config_or_container)}") if azure_config.extra_code_directory: exist = "exists" if Path(azure_config.extra_code_directory).exists() else "does not exist" logging.info(f"extra_code_directory is {azure_config.extra_code_directory}, which {exist}") else: logging.info("extra_code_directory is unset") return parser_result
def create_parser(yaml_file_path: Path) -> ParserResult: """ Create a parser for all runner arguments, even though we are only using a subset of the arguments. This way, we can get secrets handling in a consistent way. In particular, this will create arguments for --local_dataset --azure_dataset_id """ parser = create_runner_parser(SegmentationModelBase) NormalizeAndVisualizeConfig.add_args(parser) return parse_args_and_add_yaml_variables(parser, yaml_config_file=yaml_file_path, fail_on_unknown_args=True)
def parse_and_load_model(self) -> Optional[ParserResult]: """ Parses the command line arguments, and creates configuration objects for the model itself, and for the Azure-related parameters. Sets self.azure_config and self.model_config to their proper values. Returns the parser output from parsing the model commandline arguments. If no "model" argument is provided on the commandline, self.model_config will be set to None, and the return value is None. """ # Create a parser that will understand only the args we need for an AzureConfig parser1 = create_runner_parser() parser1_result = parse_args_and_add_yaml_variables(parser1, yaml_config_file=self.yaml_config_file, project_root=self.project_root, args=self.command_line_args, fail_on_unknown_args=False) azure_config = AzureConfig(**parser1_result.args) azure_config.project_root = self.project_root self.azure_config = azure_config self.model_config = None # type: ignore if not azure_config.model: return None model_config_loader: ModelConfigLoader = ModelConfigLoader(**parser1_result.args) # Create the model as per the "model" commandline option model_config = model_config_loader.create_model_config_from_name( model_name=azure_config.model ) # This model will be either a classification model or a segmentation model. Those have different # fields that could be overridden on the command line. Create a parser that understands the fields we need # for the actual model type. We feed this parser will the YAML settings and commandline arguments that the # first parser did not recognize. parser2 = type(model_config).create_argparser() parser2_result = parse_arguments(parser2, settings_from_yaml=parser1_result.unknown_settings_from_yaml, args=parser1_result.unknown, fail_on_unknown_args=True) # Apply the overrides and validate. Overrides can come from either YAML settings or the commandline. model_config.apply_overrides(parser1_result.unknown_settings_from_yaml) model_config.apply_overrides(parser2_result.overrides) model_config.validate() # Set the file system related configs, they might be affected by the overrides that were applied. logging.info("Creating the adjusted output folder structure.") model_config.create_filesystem(self.project_root) if azure_config.extra_code_directory: exist = "exists" if Path(azure_config.extra_code_directory).exists() else "does not exist" logging.info(f"extra_code_directory is {azure_config.extra_code_directory}, which {exist}") else: logging.info("extra_code_directory is unset") self.model_config = model_config return parser2_result
def main() -> None: parser = create_runner_parser(SegmentationModelBase) parser_result = parse_args_and_add_yaml_variables( parser, fail_on_unknown_args=True) surface_distance_config = SurfaceDistanceConfig.parse_args() azure_config = AzureConfig(**parser_result.args) config_model = azure_config.model if config_model is None: raise ValueError( "The name of the model to train must be given in the --model argument." ) model_config = ModelConfigLoader().create_model_config_from_name( config_model) model_config.apply_overrides(parser_result.overrides, should_validate=True) execution_mode = surface_distance_config.execution_mode run_mode = surface_distance_config.run_mode if run_mode == SurfaceDistanceRunType.IOV: ct_path = Path( "outputs") / SurfaceDistanceRunType.IOV.value.lower() / "ct.nii.gz" ct = load_nifti_image(ct_path).image else: ct = None annotators = [ annotator.strip() for annotator in surface_distance_config.annotators ] extended_annotators = annotators + [surface_distance_config.model_name] outlier_range = surface_distance_config.outlier_range predictions = load_predictions(run_mode, azure_config, model_config, execution_mode, extended_annotators, outlier_range) segmentations = [ load_nifti_image(Path(pred_seg.segmentation_path)) for pred_seg in predictions ] img_shape = segmentations[0].image.shape # transpose spacing to match image which is transposed in io_util voxel_spacing = segmentations[0].header.spacing[::-1] overall_gold_standard = np.zeros(img_shape) sds_for_annotator = sd_util.initialise_surface_distance_dictionary( extended_annotators, img_shape) plane = surface_distance_config.plane output_img_dir = Path(surface_distance_config.output_img_dir) subject_id: Optional[int] = None for prediction, pred_seg_w_header in zip(predictions, segmentations): subject_id = prediction.subject_id structure_name = prediction.structure_name annotator = prediction.annotator pred_segmentation = pred_seg_w_header.image if run_mode == SurfaceDistanceRunType.OUTLIERS: try: ground_truth = sd_util.load_ground_truth_from_run( model_config, surface_distance_config, subject_id, structure_name) except FileNotFoundError as e: logging.warning(e) continue elif run_mode == SurfaceDistanceRunType.IOV: ground_truth = sd_util.get_annotations_and_majority_vote( model_config, annotators, structure_name) else: raise ValueError( f'Unrecognised run mode: {run_mode}. Expected either IOV or OUTLIERS' ) binary_prediction_mask = multi_label_array_to_binary( pred_segmentation, 2)[1] # For comparison, plot gold standard vs predicted segmentation segmentation_and_groundtruth_plot(binary_prediction_mask, ground_truth, subject_id, structure_name, plane, output_img_dir, annotator=annotator) if run_mode == SurfaceDistanceRunType.IOV: overall_gold_standard += ground_truth # Calculate and plot surface distance sds_full = sd_util.calculate_surface_distances(ground_truth, binary_prediction_mask, list(voxel_spacing)) surface_distance_ground_truth_plot(ct, ground_truth, sds_full, subject_id, structure_name, plane, output_img_dir, annotator=annotator) if annotator is not None: sds_for_annotator[annotator] += sds_full # Plot all structures SDs for each annotator if run_mode == SurfaceDistanceRunType.IOV and subject_id is not None: for annotator, sds in sds_for_annotator.items(): num_classes = int(np.amax(np.unique(overall_gold_standard))) binarised_gold_standard = multi_label_array_to_binary( overall_gold_standard, num_classes)[1:].sum(axis=0) surface_distance_ground_truth_plot(ct, binarised_gold_standard, sds, subject_id, 'All', plane, output_img_dir, annotator=annotator)
def test_create_runner_parser(with_config: bool) -> None: """ Test parsing of commandline arguments: From arguments to the runner, can we reconstruct arguments for AzureConfig and for the model config? Check that default and non-default arguments are set correctly and recognized as default/non-default. """ azure_parser = create_runner_parser( SegmentationModelBase if with_config else None) args_list = [ "--model=Lung", "--train=False", "--l_rate=100.0", "--unknown=1", "--subscription_id", "Test1", "--tenant_id=Test2", "--application_id", "Test3", "--log_level=INFO", # Normally we don't use extra index URLs in InnerEye, hence this won't be set in YAML. "--pip_extra_index_url=foo" ] with mock.patch("sys.argv", [""] + args_list): parser_result = parse_args_and_add_yaml_variables( azure_parser, yaml_config_file=fixed_paths.SETTINGS_YAML_FILE) azure_config = AzureConfig(**parser_result.args) # These values have been set on the commandline, to values that are not the parser defaults. non_default_args = { "train": False, "model": "Lung", "subscription_id": "Test1", "application_id": "Test3", } for prop, value in non_default_args.items(): assert prop in parser_result.args, f"Property {prop} missing in args" assert parser_result.args[ prop] == value, f"Property {prop} does not have the expected value" assert getattr(azure_config, prop) == value, f"Property {prop} not in object" assert parser_result.overrides[prop] == value, \ f"Property {prop} has a non-default value, and should be recognized as such." # log_level is set on the commandline, to a value that is equal to the default. It should be recognized as an # override. log_level = "log_level" assert log_level in parser_result.args assert parser_result.args[log_level] == "INFO" assert log_level in parser_result.overrides assert parser_result.overrides[log_level] == "INFO" # These next variables should have been read from YAML. They should be in the args dictionary and in the object, # but not in the list overrides from_yaml = { "workspace_name": "InnerEye-DeepLearning", "azureml_datastore": "innereyedatasets", } for prop, value in from_yaml.items(): assert prop in parser_result.args, f"Property {prop} missing in args" assert parser_result.args[ prop] == value, f"Property {prop} does not have the expected value" assert getattr(azure_config, prop) == value, f"Property {prop} not in object" assert prop not in parser_result.overrides, f"Property {prop} should not be listed as having a " \ f"non-default value" assert "unknown" not in parser_result.args l_rate = "l_rate" if with_config: assert l_rate in parser_result.args assert parser_result.args[l_rate] == 100.0 assert parser_result.unknown == ["--unknown=1"] else: assert l_rate not in parser_result.args assert parser_result.unknown == ["--l_rate=100.0", "--unknown=1"]