def test_predict_non_ensemble(batch_size: int, empty_labels: bool) -> None:
config = ConstantScalarConfig(1.)
model_and_info = ModelAndInfo(config=config,
model_execution_mode=ModelExecutionMode.TEST,
checkpoint_path=None)
model_loaded = model_and_info.try_create_model_load_from_checkpoint_and_adjust(
)
assert model_loaded
model = model_and_info.model
pipeline = ScalarInferencePipeline(model, config, 0, 0)
actual_labels = torch.zeros(
(batch_size, 1)) * np.nan if empty_labels else torch.zeros(
(batch_size, 1))
data = {
"metadata": [GeneralSampleMetadata(id='2')] * batch_size,
"label": actual_labels,
"images": torch.zeros(
((batch_size, 1) + config.expected_image_size_zyx)),
"numerical_non_image_features": torch.tensor([]),
"categorical_non_image_features": torch.tensor([]),
"segmentations": torch.tensor([])
}
results = pipeline.predict(data)
ids, labels, predicted = results.subject_ids, results.labels, results.model_outputs
assert ids == ['2'] * batch_size
assert torch.allclose(labels, actual_labels, equal_nan=True)
# The model always returns 1, so predicted should be sigmoid(1)
assert torch.allclose(predicted, torch.full((batch_size, 1), 0.731058578))
def test_predict_non_ensemble(batch_size: int, empty_labels: bool) -> None:
config = ConstantScalarConfig(1.)
model = create_lightning_model(config, set_optimizer_and_scheduler=False)
assert isinstance(model, ScalarLightning)
pipeline = ScalarInferencePipeline(model, config, 0)
actual_labels = torch.zeros(
(batch_size, 1)) * np.nan if empty_labels else torch.zeros(
(batch_size, 1))
data = {
"metadata": [GeneralSampleMetadata(id='2')] * batch_size,
"label": actual_labels,
"images": torch.zeros(
((batch_size, 1) + config.expected_image_size_zyx)),
"numerical_non_image_features": torch.tensor([]),
"categorical_non_image_features": torch.tensor([]),
"segmentations": torch.tensor([])
}
results = pipeline.predict(data)
ids, labels, predicted = results.subject_ids, results.labels, results.posteriors
assert ids == ['2'] * batch_size
assert torch.allclose(labels, actual_labels, equal_nan=True)
# The model always returns 1, so predicted should be sigmoid(1)
assert torch.allclose(predicted, torch.full((batch_size, 1), 0.731058578))
def test_predict_ensemble(batch_size: int) -> None:
config_returns_0 = ConstantScalarConfig(0.)
model_and_info_returns_0 = ModelAndInfo(config=config_returns_0, model_execution_mode=ModelExecutionMode.TEST,
is_mean_teacher=False, checkpoint_path=None)
model_loaded = model_and_info_returns_0.try_create_model_load_from_checkpoint_and_adjust()
assert model_loaded
model_returns_0 = model_and_info_returns_0.model
config_returns_1 = ConstantScalarConfig(1.)
model_and_info_returns_1 = ModelAndInfo(config=config_returns_1, model_execution_mode=ModelExecutionMode.TEST,
is_mean_teacher=False, checkpoint_path=None)
model_loaded = model_and_info_returns_1.try_create_model_load_from_checkpoint_and_adjust()
assert model_loaded
model_returns_1 = model_and_info_returns_1.model
pipeline_0 = ScalarInferencePipeline(model_returns_0, config_returns_0, 0, 0)
pipeline_1 = ScalarInferencePipeline(model_returns_0, config_returns_0, 0, 1)
pipeline_2 = ScalarInferencePipeline(model_returns_0, config_returns_0, 0, 2)
pipeline_3 = ScalarInferencePipeline(model_returns_1, config_returns_1, 0, 3)
pipeline_4 = ScalarInferencePipeline(model_returns_1, config_returns_1, 0, 4)
ensemble_pipeline = ScalarEnsemblePipeline([pipeline_0, pipeline_1, pipeline_2, pipeline_3, pipeline_4],
config_returns_0, EnsembleAggregationType.Average)
data = {"metadata": [GeneralSampleMetadata(id='2')] * batch_size,
"label": torch.zeros((batch_size, 1)),
"images": torch.zeros(((batch_size, 1) + config_returns_0.expected_image_size_zyx)),
"numerical_non_image_features": torch.tensor([]),
"categorical_non_image_features": torch.tensor([]),
"segmentations": torch.tensor([])}
results = ensemble_pipeline.predict(data)
ids, labels, predicted = results.subject_ids, results.labels, results.model_outputs
assert ids == ['2'] * batch_size
assert torch.equal(labels, torch.zeros((batch_size, 1)))
# 3 models return 0, 2 return 1, so predicted should be ((sigmoid(0)*3)+(sigmoid(1)*2))/5
assert torch.allclose(predicted, torch.full((batch_size, 1), 0.592423431))
def test_predict_non_ensemble(batch_size: int, empty_labels: bool) -> None:
config = ClassificationModelForTesting()
model: Any = ScalarOnesModel(config.expected_image_size_zyx, 1.)
update_model_for_multiple_gpus(ModelAndInfo(model),
args=config,
execution_mode=ModelExecutionMode.TEST)
pipeline = ScalarInferencePipeline(model, config, 0, 0)
actual_labels = torch.zeros(
(batch_size, 1)) * np.nan if empty_labels else torch.zeros(
(batch_size, 1))
data = {
"metadata": [GeneralSampleMetadata(id='2')] * batch_size,
"label": actual_labels,
"images": torch.zeros(
((batch_size, 1) + config.expected_image_size_zyx)),
"numerical_non_image_features": torch.tensor([]),
"categorical_non_image_features": torch.tensor([]),
"segmentations": torch.tensor([])
}
results = pipeline.predict(data)
ids, labels, predicted = results.subject_ids, results.labels, results.model_outputs
assert ids == ['2'] * batch_size
assert torch.allclose(labels, actual_labels, equal_nan=True)
# The model always returns 1, so predicted should be sigmoid(1)
assert torch.allclose(predicted, torch.full((batch_size, 1), 0.731058578))
def test_create_from_checkpoint_non_ensemble(
test_output_dirs: OutputFolderForTests) -> None:
config = ClassificationModelForTesting()
# when checkpoint does not exist, return None
path_to_checkpoint = test_output_dirs.root_dir / "foo.ckpt"
inference_pipeline = ScalarInferencePipeline.create_from_checkpoint(
path_to_checkpoint, config)
assert inference_pipeline is None
create_model_and_store_checkpoint(config, path_to_checkpoint)
inference_pipeline = ScalarInferencePipeline.create_from_checkpoint(
path_to_checkpoint, config)
assert isinstance(inference_pipeline, ScalarInferencePipeline)
def test_create_from_checkpoint_non_ensemble() -> None:
config = ClassificationModelForTesting()
# when checkpoint does not exist, return None
checkpoint_folder = "classification_data_generated_random/checkpoints/non_exist.pth.tar"
path_to_checkpoint = full_ml_test_data_path(checkpoint_folder)
inference_pipeline = ScalarInferencePipeline.create_from_checkpoint(path_to_checkpoint, config)
assert inference_pipeline is None
checkpoint_folder = "classification_data_generated_random/checkpoints/1_checkpoint.pth.tar"
path_to_checkpoint = full_ml_test_data_path(checkpoint_folder)
inference_pipeline = ScalarInferencePipeline.create_from_checkpoint(path_to_checkpoint, config)
assert isinstance(inference_pipeline, ScalarInferencePipeline)
assert inference_pipeline.epoch == 1
def create_pipeline_from_checkpoint_paths(
config: ModelConfigBase,
checkpoint_paths: List[Path]) -> Optional[InferencePipelineBase]:
"""
Attempt to create a pipeline from the provided checkpoint paths. If the files referred to by the paths
do not exist, or if there are no paths, None will be returned.
"""
if len(checkpoint_paths) > 1:
if config.is_segmentation_model:
assert isinstance(config, SegmentationModelBase)
return EnsemblePipeline.create_from_checkpoints(
path_to_checkpoints=checkpoint_paths, model_config=config)
elif config.is_scalar_model:
assert isinstance(config, ScalarModelBase)
return ScalarEnsemblePipeline.create_from_checkpoint(
paths_to_checkpoint=checkpoint_paths, config=config)
else:
raise NotImplementedError(
"Cannot create inference pipeline for unknown model type")
if len(checkpoint_paths) == 1:
if config.is_segmentation_model:
assert isinstance(config, SegmentationModelBase)
return InferencePipeline.create_from_checkpoint(
path_to_checkpoint=checkpoint_paths[0], model_config=config)
elif config.is_scalar_model:
assert isinstance(config, ScalarModelBase)
return ScalarInferencePipeline.create_from_checkpoint(
path_to_checkpoint=checkpoint_paths[0], config=config)
else:
raise NotImplementedError(
"Cannot create ensemble pipeline for unknown model type")
return None
def create_inference_pipeline(config: ModelConfigBase,
checkpoint_paths: List[Path]) -> Optional[InferencePipelineBase]:
"""
If multiple checkpoints are found in run_recovery then create EnsemblePipeline otherwise InferencePipeline.
If no checkpoint files exist in the run recovery or current run checkpoint folder, None will be returned.
:param config: Model related configs.
:param epoch: The epoch for which to create pipeline for.
:param run_recovery: RunRecovery data if applicable
:return: FullImageInferencePipelineBase or ScalarInferencePipelineBase
"""
if not checkpoint_paths:
return None
if len(checkpoint_paths) > 1:
if config.is_segmentation_model:
assert isinstance(config, SegmentationModelBase)
return EnsemblePipeline.create_from_checkpoints(path_to_checkpoints=checkpoint_paths, model_config=config)
elif config.is_scalar_model:
assert isinstance(config, ScalarModelBase)
return ScalarEnsemblePipeline.create_from_checkpoint(paths_to_checkpoint=checkpoint_paths, config=config)
else:
raise NotImplementedError("Cannot create inference pipeline for unknown model type")
if len(checkpoint_paths) == 1:
if config.is_segmentation_model:
assert isinstance(config, SegmentationModelBase)
return InferencePipeline.create_from_checkpoint(path_to_checkpoint=checkpoint_paths[0],
model_config=config)
elif config.is_scalar_model:
assert isinstance(config, ScalarModelBase)
return ScalarInferencePipeline.create_from_checkpoint(path_to_checkpoint=checkpoint_paths[0],
config=config)
else:
raise NotImplementedError("Cannot create ensemble pipeline for unknown model type")
return None
def test_predict_ensemble(batch_size: int) -> None:
config_returns_0 = ConstantScalarConfig(0.)
model_returns_0 = create_lightning_model(config_returns_0,
set_optimizer_and_scheduler=False)
assert isinstance(model_returns_0, ScalarLightning)
config_returns_1 = ConstantScalarConfig(1.)
model_returns_1 = create_lightning_model(config_returns_1,
set_optimizer_and_scheduler=False)
assert isinstance(model_returns_1, ScalarLightning)
pipeline_0 = ScalarInferencePipeline(model_returns_0, config_returns_0, 0)
pipeline_1 = ScalarInferencePipeline(model_returns_0, config_returns_0, 1)
pipeline_2 = ScalarInferencePipeline(model_returns_0, config_returns_0, 2)
pipeline_3 = ScalarInferencePipeline(model_returns_1, config_returns_1, 3)
pipeline_4 = ScalarInferencePipeline(model_returns_1, config_returns_1, 4)
ensemble_pipeline = ScalarEnsemblePipeline(
[pipeline_0, pipeline_1, pipeline_2, pipeline_3, pipeline_4],
config_returns_0, EnsembleAggregationType.Average)
data = {
"metadata": [GeneralSampleMetadata(id='2')] * batch_size,
"label":
torch.zeros((batch_size, 1)),
"images":
torch.zeros(
((batch_size, 1) + config_returns_0.expected_image_size_zyx)),
"numerical_non_image_features":
torch.tensor([]),
"categorical_non_image_features":
torch.tensor([]),
"segmentations":
torch.tensor([])
}
results = ensemble_pipeline.predict(data)
ids, labels, predicted = results.subject_ids, results.labels, results.posteriors
assert ids == ['2'] * batch_size
assert torch.equal(labels, torch.zeros((batch_size, 1)))
# 3 models return 0, 2 return 1, so predicted should be ((sigmoid(0)*3)+(sigmoid(1)*2))/5
assert torch.allclose(predicted, torch.full((batch_size, 1), 0.592423431))
def test_predict_ensemble(batch_size: int) -> None:
config = ClassificationModelForTesting()
model_returns_0: Any = ScalarOnesModel(config.expected_image_size_zyx, 0.)
model_returns_1: Any = ScalarOnesModel(config.expected_image_size_zyx, 1.)
model_and_opt_0 = update_model_for_multiple_gpus(
ModelAndInfo(model_returns_0),
args=config,
execution_mode=ModelExecutionMode.TEST)
model_returns_0 = model_and_opt_0.model
model_and_opt_1 = update_model_for_multiple_gpus(
ModelAndInfo(model_returns_1),
args=config,
execution_mode=ModelExecutionMode.TEST)
model_returns_1 = model_and_opt_1.model
pipeline_0 = ScalarInferencePipeline(model_returns_0, config, 0, 0)
pipeline_1 = ScalarInferencePipeline(model_returns_0, config, 0, 1)
pipeline_2 = ScalarInferencePipeline(model_returns_0, config, 0, 2)
pipeline_3 = ScalarInferencePipeline(model_returns_1, config, 0, 3)
pipeline_4 = ScalarInferencePipeline(model_returns_1, config, 0, 4)
ensemble_pipeline = ScalarEnsemblePipeline(
[pipeline_0, pipeline_1, pipeline_2, pipeline_3, pipeline_4], config,
EnsembleAggregationType.Average)
data = {
"metadata": [GeneralSampleMetadata(id='2')] * batch_size,
"label": torch.zeros((batch_size, 1)),
"images": torch.zeros(
((batch_size, 1) + config.expected_image_size_zyx)),
"numerical_non_image_features": torch.tensor([]),
"categorical_non_image_features": torch.tensor([]),
"segmentations": torch.tensor([])
}
results = ensemble_pipeline.predict(data)
ids, labels, predicted = results.subject_ids, results.labels, results.model_outputs
assert ids == ['2'] * batch_size
assert torch.equal(labels, torch.zeros((batch_size, 1)))
# 3 models return 0, 2 return 1, so predicted should be ((sigmoid(0)*3)+(sigmoid(1)*2))/5
assert torch.allclose(predicted, torch.full((batch_size, 1), 0.592423431))