def test_create_runner_parser_check_fails_unknown() -> None:
"""
Test parsing of commandline arguments: Check if unknown arguments fail the parsing
"""
azure_parser = create_runner_parser(SegmentationModelBase)
valid_args = ["--model=Lung"]
invalid_args = ["--unknown=1"]
# Ensure that the valid arguments that we provide later do actually parse correctly
with mock.patch("sys.argv", [""] + valid_args):
result = parse_args_and_add_yaml_variables(azure_parser,
fail_on_unknown_args=True)
assert "model" in result.args
assert result.args["model"] == "Lung"
# Supply both valid and invalid arguments, and we expect a failure because of the invalid ones:
with mock.patch("sys.argv", [""] + valid_args + invalid_args):
with pytest.raises(Exception) as e_info:
parse_args_and_add_yaml_variables(azure_parser,
fail_on_unknown_args=True)
assert str(e_info.value) == 'Unknown arguments: [\'--unknown=1\']'
# Supply both valid and invalid arguments, and we expect that the invalid ones are silently ignored:
with mock.patch("sys.argv", [""] + valid_args + invalid_args):
result = parse_args_and_add_yaml_variables(azure_parser,
fail_on_unknown_args=False)
assert "model" in result.args
assert result.args["model"] == "Lung"
assert invalid_args[0] in result.unknown
def parse_and_load_model(self) -> ParserResult:
"""
Parses the command line arguments, and creates configuration objects for the model itself, and for the
Azure-related parameters. Sets self.azure_config and self.model_config to their proper values. Returns the
parser output from parsing the model commandline arguments.
If no "model" argument is provided on the commandline, self.model_config will be set to None, and the return
value is None.
"""
# Create a parser that will understand only the args we need for an AzureConfig
parser1 = create_runner_parser()
parser_result = parse_args_and_add_yaml_variables(parser1,
yaml_config_file=self.yaml_config_file,
project_root=self.project_root,
fail_on_unknown_args=False)
azure_config = AzureConfig(**parser_result.args)
azure_config.project_root = self.project_root
self.azure_config = azure_config
self.model_config = None
if not azure_config.model:
raise ValueError("Parameter 'model' needs to be set to tell InnerEye which model to run.")
model_config_loader: ModelConfigLoader = ModelConfigLoader(**parser_result.args)
# Create the model as per the "model" commandline option. This can return either a built-in config
# of type DeepLearningConfig, or a LightningContainer.
config_or_container = model_config_loader.create_model_config_from_name(model_name=azure_config.model)
def parse_overrides_and_apply(c: object, previous_parser_result: ParserResult) -> ParserResult:
assert isinstance(c, GenericConfig)
parser = type(c).create_argparser()
# For each parser, feed in the unknown settings from the previous parser. All commandline args should
# be consumed by name, hence fail if there is something that is still unknown.
parser_result = parse_arguments(parser,
settings_from_yaml=previous_parser_result.unknown_settings_from_yaml,
args=previous_parser_result.unknown,
fail_on_unknown_args=True)
# Apply the overrides and validate. Overrides can come from either YAML settings or the commandline.
c.apply_overrides(parser_result.known_settings_from_yaml)
c.apply_overrides(parser_result.overrides)
c.validate()
return parser_result
# Now create a parser that understands overrides at model/container level.
parser_result = parse_overrides_and_apply(config_or_container, parser_result)
if isinstance(config_or_container, LightningContainer):
self.lightning_container = config_or_container
elif isinstance(config_or_container, ModelConfigBase):
# Built-in InnerEye models use a fake container
self.model_config = config_or_container
self.lightning_container = InnerEyeContainer(config_or_container)
else:
raise ValueError(f"Don't know how to handle a loaded configuration of type {type(config_or_container)}")
if azure_config.extra_code_directory:
exist = "exists" if Path(azure_config.extra_code_directory).exists() else "does not exist"
logging.info(f"extra_code_directory is {azure_config.extra_code_directory}, which {exist}")
else:
logging.info("extra_code_directory is unset")
return parser_result
def create_parser(yaml_file_path: Path) -> ParserResult:
"""
Create a parser for all runner arguments, even though we are only using a subset of the arguments.
This way, we can get secrets handling in a consistent way.
In particular, this will create arguments for
--local_dataset
--azure_dataset_id
"""
parser = create_runner_parser(SegmentationModelBase)
NormalizeAndVisualizeConfig.add_args(parser)
return parse_args_and_add_yaml_variables(parser,
yaml_config_file=yaml_file_path,
fail_on_unknown_args=True)
def parse_and_load_model(self) -> Optional[ParserResult]:
"""
Parses the command line arguments, and creates configuration objects for the model itself, and for the
Azure-related parameters. Sets self.azure_config and self.model_config to their proper values. Returns the
parser output from parsing the model commandline arguments.
If no "model" argument is provided on the commandline, self.model_config will be set to None, and the return
value is None.
"""
# Create a parser that will understand only the args we need for an AzureConfig
parser1 = create_runner_parser()
parser1_result = parse_args_and_add_yaml_variables(parser1,
yaml_config_file=self.yaml_config_file,
project_root=self.project_root,
args=self.command_line_args,
fail_on_unknown_args=False)
azure_config = AzureConfig(**parser1_result.args)
azure_config.project_root = self.project_root
self.azure_config = azure_config
self.model_config = None # type: ignore
if not azure_config.model:
return None
model_config_loader: ModelConfigLoader = ModelConfigLoader(**parser1_result.args)
# Create the model as per the "model" commandline option
model_config = model_config_loader.create_model_config_from_name(
model_name=azure_config.model
)
# This model will be either a classification model or a segmentation model. Those have different
# fields that could be overridden on the command line. Create a parser that understands the fields we need
# for the actual model type. We feed this parser will the YAML settings and commandline arguments that the
# first parser did not recognize.
parser2 = type(model_config).create_argparser()
parser2_result = parse_arguments(parser2,
settings_from_yaml=parser1_result.unknown_settings_from_yaml,
args=parser1_result.unknown,
fail_on_unknown_args=True)
# Apply the overrides and validate. Overrides can come from either YAML settings or the commandline.
model_config.apply_overrides(parser1_result.unknown_settings_from_yaml)
model_config.apply_overrides(parser2_result.overrides)
model_config.validate()
# Set the file system related configs, they might be affected by the overrides that were applied.
logging.info("Creating the adjusted output folder structure.")
model_config.create_filesystem(self.project_root)
if azure_config.extra_code_directory:
exist = "exists" if Path(azure_config.extra_code_directory).exists() else "does not exist"
logging.info(f"extra_code_directory is {azure_config.extra_code_directory}, which {exist}")
else:
logging.info("extra_code_directory is unset")
self.model_config = model_config
return parser2_result
def main() -> None:
parser = create_runner_parser(SegmentationModelBase)
parser_result = parse_args_and_add_yaml_variables(
parser, fail_on_unknown_args=True)
surface_distance_config = SurfaceDistanceConfig.parse_args()
azure_config = AzureConfig(**parser_result.args)
config_model = azure_config.model
if config_model is None:
raise ValueError(
"The name of the model to train must be given in the --model argument."
)
model_config = ModelConfigLoader().create_model_config_from_name(
config_model)
model_config.apply_overrides(parser_result.overrides, should_validate=True)
execution_mode = surface_distance_config.execution_mode
run_mode = surface_distance_config.run_mode
if run_mode == SurfaceDistanceRunType.IOV:
ct_path = Path(
"outputs") / SurfaceDistanceRunType.IOV.value.lower() / "ct.nii.gz"
ct = load_nifti_image(ct_path).image
else:
ct = None
annotators = [
annotator.strip() for annotator in surface_distance_config.annotators
]
extended_annotators = annotators + [surface_distance_config.model_name]
outlier_range = surface_distance_config.outlier_range
predictions = load_predictions(run_mode, azure_config, model_config,
execution_mode, extended_annotators,
outlier_range)
segmentations = [
load_nifti_image(Path(pred_seg.segmentation_path))
for pred_seg in predictions
]
img_shape = segmentations[0].image.shape
# transpose spacing to match image which is transposed in io_util
voxel_spacing = segmentations[0].header.spacing[::-1]
overall_gold_standard = np.zeros(img_shape)
sds_for_annotator = sd_util.initialise_surface_distance_dictionary(
extended_annotators, img_shape)
plane = surface_distance_config.plane
output_img_dir = Path(surface_distance_config.output_img_dir)
subject_id: Optional[int] = None
for prediction, pred_seg_w_header in zip(predictions, segmentations):
subject_id = prediction.subject_id
structure_name = prediction.structure_name
annotator = prediction.annotator
pred_segmentation = pred_seg_w_header.image
if run_mode == SurfaceDistanceRunType.OUTLIERS:
try:
ground_truth = sd_util.load_ground_truth_from_run(
model_config, surface_distance_config, subject_id,
structure_name)
except FileNotFoundError as e:
logging.warning(e)
continue
elif run_mode == SurfaceDistanceRunType.IOV:
ground_truth = sd_util.get_annotations_and_majority_vote(
model_config, annotators, structure_name)
else:
raise ValueError(
f'Unrecognised run mode: {run_mode}. Expected either IOV or OUTLIERS'
)
binary_prediction_mask = multi_label_array_to_binary(
pred_segmentation, 2)[1]
# For comparison, plot gold standard vs predicted segmentation
segmentation_and_groundtruth_plot(binary_prediction_mask,
ground_truth,
subject_id,
structure_name,
plane,
output_img_dir,
annotator=annotator)
if run_mode == SurfaceDistanceRunType.IOV:
overall_gold_standard += ground_truth
# Calculate and plot surface distance
sds_full = sd_util.calculate_surface_distances(ground_truth,
binary_prediction_mask,
list(voxel_spacing))
surface_distance_ground_truth_plot(ct,
ground_truth,
sds_full,
subject_id,
structure_name,
plane,
output_img_dir,
annotator=annotator)
if annotator is not None:
sds_for_annotator[annotator] += sds_full
# Plot all structures SDs for each annotator
if run_mode == SurfaceDistanceRunType.IOV and subject_id is not None:
for annotator, sds in sds_for_annotator.items():
num_classes = int(np.amax(np.unique(overall_gold_standard)))
binarised_gold_standard = multi_label_array_to_binary(
overall_gold_standard, num_classes)[1:].sum(axis=0)
surface_distance_ground_truth_plot(ct,
binarised_gold_standard,
sds,
subject_id,
'All',
plane,
output_img_dir,
annotator=annotator)
def test_create_runner_parser(with_config: bool) -> None:
"""
Test parsing of commandline arguments:
From arguments to the runner, can we reconstruct arguments for AzureConfig and for the model config?
Check that default and non-default arguments are set correctly and recognized as default/non-default.
"""
azure_parser = create_runner_parser(
SegmentationModelBase if with_config else None)
args_list = [
"--model=Lung",
"--train=False",
"--l_rate=100.0",
"--unknown=1",
"--subscription_id",
"Test1",
"--tenant_id=Test2",
"--application_id",
"Test3",
"--log_level=INFO",
# Normally we don't use extra index URLs in InnerEye, hence this won't be set in YAML.
"--pip_extra_index_url=foo"
]
with mock.patch("sys.argv", [""] + args_list):
parser_result = parse_args_and_add_yaml_variables(
azure_parser, yaml_config_file=fixed_paths.SETTINGS_YAML_FILE)
azure_config = AzureConfig(**parser_result.args)
# These values have been set on the commandline, to values that are not the parser defaults.
non_default_args = {
"train": False,
"model": "Lung",
"subscription_id": "Test1",
"application_id": "Test3",
}
for prop, value in non_default_args.items():
assert prop in parser_result.args, f"Property {prop} missing in args"
assert parser_result.args[
prop] == value, f"Property {prop} does not have the expected value"
assert getattr(azure_config,
prop) == value, f"Property {prop} not in object"
assert parser_result.overrides[prop] == value, \
f"Property {prop} has a non-default value, and should be recognized as such."
# log_level is set on the commandline, to a value that is equal to the default. It should be recognized as an
# override.
log_level = "log_level"
assert log_level in parser_result.args
assert parser_result.args[log_level] == "INFO"
assert log_level in parser_result.overrides
assert parser_result.overrides[log_level] == "INFO"
# These next variables should have been read from YAML. They should be in the args dictionary and in the object,
# but not in the list overrides
from_yaml = {
"workspace_name": "InnerEye-DeepLearning",
"azureml_datastore": "innereyedatasets",
}
for prop, value in from_yaml.items():
assert prop in parser_result.args, f"Property {prop} missing in args"
assert parser_result.args[
prop] == value, f"Property {prop} does not have the expected value"
assert getattr(azure_config,
prop) == value, f"Property {prop} not in object"
assert prop not in parser_result.overrides, f"Property {prop} should not be listed as having a " \
f"non-default value"
assert "unknown" not in parser_result.args
l_rate = "l_rate"
if with_config:
assert l_rate in parser_result.args
assert parser_result.args[l_rate] == 100.0
assert parser_result.unknown == ["--unknown=1"]
else:
assert l_rate not in parser_result.args
assert parser_result.unknown == ["--l_rate=100.0", "--unknown=1"]
