def test_prostate_paper_with_optional_params() -> None:
"""
Check that optional parameters can be passed in to ProstatePaper class.
"""
ground_truth_ids = DEFAULT_PROSTATE_GROUND_TRUTH_IDS
ground_truth_count = len(ground_truth_ids)
ground_truth_ids_display_names = generate_random_display_ids(
ground_truth_count)
colours = generate_random_colours_list(RANDOM_COLOUR_GENERATOR,
ground_truth_count)
fill_holes = generate_random_fill_holes(ground_truth_count)
class_weights = generate_random_class_weights(ground_truth_count + 1)
largest_connected_component_foreground_classes = DEFAULT_PROSTATE_GROUND_TRUTH_IDS[
1:3]
config = ProstatePaper(
local_dataset=Path("foo"),
ground_truth_ids_display_names=ground_truth_ids_display_names,
colours=colours,
fill_holes=fill_holes,
class_weights=class_weights,
largest_connected_component_foreground_classes=
largest_connected_component_foreground_classes)
assert config.ground_truth_ids == ground_truth_ids
assert config.ground_truth_ids_display_names == ground_truth_ids_display_names
assert config.colours == colours
assert config.fill_holes == fill_holes
assert config.class_weights == class_weights
expected_lccfc = [(c, None)
for c in largest_connected_component_foreground_classes]
assert config.largest_connected_component_foreground_classes == expected_lccfc
def __init__(self, **kwargs: Any) -> None:
fg_classes = ["spinalcord", "lung_r", "lung_l", "heart", "esophagus"]
fg_display_names = [
"SpinalCord", "Lung_R", "Lung_L", "Heart", "Esophagus"
]
super().__init__(
should_validate=False,
# Set as UNet3D only because this does not shrink patches in the forward pass.
architecture=ModelArchitectureConfig.UNet3D,
azure_dataset_id=AZURE_DATASET_ID,
crop_size=(64, 224, 224),
num_dataload_workers=1,
# Disable monitoring so that we can use VS Code remote debugging
monitoring_interval_seconds=0,
image_channels=["ct"],
ground_truth_ids=fg_classes,
ground_truth_ids_display_names=fg_display_names,
colours=generate_random_colours_list(RANDOM_COLOUR_GENERATOR,
len(fg_classes)),
fill_holes=[False] * len(fg_classes),
roi_interpreted_types=["ORGAN"] * len(fg_classes),
inference_batch_size=1,
class_weights=equally_weighted_classes(fg_classes,
background_weight=0.02),
feature_channels=[1],
start_epoch=0,
num_epochs=1,
# Necessary to avoid https://github.com/pytorch/pytorch/issues/45324
max_num_gpus=1,
)
self.add_and_validate(kwargs)
def test_head_and_neck_base_with_optional_params() -> None:
"""
Check that optional parameters can be passed in to HeadAndNeckBase class.
"""
ground_truth_ids = DEFAULT_HEAD_AND_NECK_GROUND_TRUTH_IDS
ground_truth_count = len(ground_truth_ids)
ground_truth_ids_display_names = generate_random_display_ids(
ground_truth_count)
colours = generate_random_colours_list(RANDOM_COLOUR_GENERATOR,
ground_truth_count)
fill_holes = generate_random_fill_holes(ground_truth_count)
class_weights = generate_random_class_weights(ground_truth_count + 1)
num_feature_channels = random.randint(1, ground_truth_count)
slice_exclusion_rules = generate_random_slice_exclusion_rules(
ground_truth_ids)
summed_probability_rules = generate_random_summed_probability_rules(
ground_truth_ids)
config = HeadAndNeckBase(
local_dataset=Path("foo"),
ground_truth_ids=ground_truth_ids,
ground_truth_ids_display_names=ground_truth_ids_display_names,
colours=colours,
fill_holes=fill_holes,
class_weights=class_weights,
slice_exclusion_rules=slice_exclusion_rules,
summed_probability_rules=summed_probability_rules,
num_feature_channels=num_feature_channels)
assert config.ground_truth_ids == ground_truth_ids
assert config.ground_truth_ids_display_names == ground_truth_ids_display_names
assert config.colours == colours
assert config.fill_holes == fill_holes
assert config.class_weights == class_weights
assert config.feature_channels == [num_feature_channels]
assert config.slice_exclusion_rules == slice_exclusion_rules
assert config.summed_probability_rules == summed_probability_rules
def test_head_and_neck_paper_with_mismatched_colours_raises() -> None:
"""
Check that passing too many colours raises ValueError exception.
"""
ground_truth_count = len(DEFAULT_HEAD_AND_NECK_GROUND_TRUTH_IDS) - 2
colours = generate_random_colours_list(RANDOM_COLOUR_GENERATOR,
ground_truth_count - 1)
with pytest.raises(ValueError) as e:
assert HeadAndNeckPaper(num_structures=ground_truth_count,
colours=colours)
assert str(e.value) == "len(ground_truth_ids_display_names)!=len(colours)"
def test_generate_random_colours_list() -> None:
"""
Test list of random colours
"""
rng = random.Random(0)
expected_list_len = 10
list_colours = generate_random_colours_list(rng, 10)
assert len(list_colours) == expected_list_len
for r, g, b in list_colours:
assert 0 <= r < 255
assert 0 <= g < 255
assert 0 <= b < 255
def __init__(self, **kwargs: Any) -> None:
fg_classes = ["region", "region_1"]
super().__init__(
# Data definition - in this section we define where to load the dataset from
local_dataset=full_ml_test_data_path(),
# Model definition - in this section we define what model to use and some related configurations
architecture="UNet3D",
feature_channels=[4],
crop_size=(64, 64, 64),
image_channels=["channel1", "channel2"],
ground_truth_ids=fg_classes,
class_weights=equally_weighted_classes(fg_classes,
background_weight=0.02),
mask_id="mask",
# Model training and testing - in this section we define configurations pertaining to the model
# training loop (ie: batch size, how many epochs to train, number of epochs to save)
# and testing (ie: how many epochs to test)
use_gpu=False,
num_dataload_workers=0,
train_batch_size=2,
start_epoch=0,
num_epochs=2,
save_start_epoch=1,
save_step_epochs=1,
test_start_epoch=2,
test_diff_epochs=1,
test_step_epochs=1,
use_mixed_precision=True,
# Pre-processing - in this section we define how to normalize our inputs, in this case we are doing
# CT Level and Window based normalization.
norm_method=PhotometricNormalizationMethod.CtWindow,
level=50,
window=200,
# Post-processing - in this section we define our post processing configurations, in this case
# we are filling holes in the generated segmentation masks for all of the foreground classes.
fill_holes=[True] * len(fg_classes),
# Output - in this section we define settings that determine how our output looks like in this case
# we define the structure names and colours to use.
ground_truth_ids_display_names=fg_classes,
colours=generate_random_colours_list(Random(5), len(fg_classes)),
)
self.add_and_validate(kwargs)
from InnerEye.ML.deep_learning_config import OptimizerType
from InnerEye.ML.utils.model_metadata_util import generate_random_colours_list
from InnerEye.ML.utils.split_dataset import DatasetSplits
# List of structures to segment. The order is important, because different values of num_structures
# in the constructor will select different prefixes of the list.
STRUCTURE_LIST = [
"external", "parotid_l", "parotid_r", "smg_l", "smg_r", "spinal_cord",
"brainstem", "globe_l", "globe_r", "mandible", "spc_muscle", "mpc_muscle",
"cochlea_l", "cochlea_r", "lens_l", "lens_r", "optic_chiasm",
"optic_nerve_l", "optic_nerve_r", "pituitary_gland", "lacrimal_gland_l",
"lacrimal_gland_r"
]
RANDOM_COLOUR_GENERATOR = random.Random(0)
COLOURS = generate_random_colours_list(RANDOM_COLOUR_GENERATOR,
len(STRUCTURE_LIST))
FILL_HOLES = [True] * len(STRUCTURE_LIST)
# This configuration needs to be supplied with a value for azure_dataset_id that refers to your
# dataset. You may also supply a value for num_structures, feature_channels or any other feature. For example,
# with the appropriate dataset, this would build the model whose results are reported in the InnerEye team's
# paper:
#
# class HeadAndNeckPaper(HeadAndNeckBase):
#
# def __init__(self):
# super().__init__(
# azure_dataset_id="foo_bar_baz",
# num_structures=10)
def __init__(self,
ground_truth_ids: List[str],
ground_truth_ids_display_names: Optional[List[str]] = None,
colours: Optional[List[TupleInt3]] = None,
fill_holes: Optional[List[bool]] = None,
class_weights: Optional[List[float]] = None,
slice_exclusion_rules: Optional[List[SliceExclusionRule]] = None,
summed_probability_rules: Optional[List[SummedProbabilityRule]] = None,
num_feature_channels: Optional[int] = None,
**kwargs: Any) -> None:
"""
Creates a new instance of the class.
:param ground_truth_ids: List of ground truth ids.
:param ground_truth_ids_display_names: Optional list of ground truth id display names. If
present then must be of the same length as ground_truth_ids.
:param colours: Optional list of colours. If
present then must be of the same length as ground_truth_ids.
:param fill_holes: Optional list of fill hole flags. If
present then must be of the same length as ground_truth_ids.
:param class_weights: Optional list of class weights. If
present then must be of the same length as ground_truth_ids + 1.
:param slice_exclusion_rules: Optional list of SliceExclusionRules.
:param summed_probability_rules: Optional list of SummedProbabilityRule.
:param num_feature_channels: Optional number of feature channels.
:param kwargs: Additional arguments that will be passed through to the SegmentationModelBase constructor.
"""
# Number of training epochs
num_epochs = 120
num_structures = len(ground_truth_ids)
colours = colours or generate_random_colours_list(RANDOM_COLOUR_GENERATOR, num_structures)
fill_holes = fill_holes or [True] * num_structures
ground_truth_ids_display_names = ground_truth_ids_display_names or [f"zz_{x}" for x in ground_truth_ids]
# The amount of GPU memory required increases with both the number of structures and the
# number of feature channels. The following is a sensible default to avoid out-of-memory,
# but you can override is by passing in another (singleton list) value for feature_channels
# from a subclass.
num_feature_channels = num_feature_channels or (32 if num_structures <= 20 else 26)
bg_weight = 0.02 if len(ground_truth_ids) > 1 else 0.25
class_weights = class_weights or equally_weighted_classes(ground_truth_ids, background_weight=bg_weight)
# In case of vertical overlap between brainstem and spinal_cord, we separate them
# by converting brainstem voxels to cord, as the latter is clinically more sensitive.
# We do the same to separate SPC and MPC; in this case, the direction of change is unimportant,
# so we choose SPC-to-MPC arbitrarily.
slice_exclusion_rules = slice_exclusion_rules or []
summed_probability_rules = summed_probability_rules or []
super().__init__(
should_validate=False, # we'll validate after kwargs are added
num_epochs=num_epochs,
recovery_checkpoint_save_interval=10,
architecture="UNet3D",
kernel_size=3,
train_batch_size=1,
inference_batch_size=1,
feature_channels=[num_feature_channels],
crop_size=(96, 288, 288),
test_crop_size=(144, 512, 512),
inference_stride_size=(72, 256, 256),
image_channels=["ct"],
norm_method=PhotometricNormalizationMethod.CtWindow,
level=50,
window=600,
start_epoch=0,
l_rate=1e-3,
min_l_rate=1e-5,
l_rate_polynomial_gamma=0.9,
optimizer_type=OptimizerType.Adam,
opt_eps=1e-4,
adam_betas=(0.9, 0.999),
momentum=0.9,
use_mixed_precision=True,
use_model_parallel=True,
monitoring_interval_seconds=0,
num_dataload_workers=2,
loss_type=SegmentationLoss.Mixture,
mixture_loss_components=[MixtureLossComponent(0.5, SegmentationLoss.Focal, 0.2),
MixtureLossComponent(0.5, SegmentationLoss.SoftDice, 0.1)],
ground_truth_ids=ground_truth_ids,
ground_truth_ids_display_names=ground_truth_ids_display_names,
largest_connected_component_foreground_classes=ground_truth_ids,
colours=colours,
fill_holes=fill_holes,
class_weights=class_weights,
slice_exclusion_rules=slice_exclusion_rules,
summed_probability_rules=summed_probability_rules,
)
self.add_and_validate(kwargs)
