def test_inputs_without_conflicting_names_do_not_require_adapter(data):
step = Step(
name='test_inputs_without_conflicting_names_do_not_require_adapter_1',
transformer=IdentityOperation(),
input_data=['input_1'])
output = step.fit_transform(data)
assert output == data['input_1']
step = Step(
name='test_inputs_without_conflicting_names_do_not_require_adapter_2',
transformer=IdentityOperation(),
input_data=['input_1', 'input_2'])
output = step.fit_transform(data)
assert output == {**data['input_1'], **data['input_2']}
def postprocessing_pipeline_simplified(cache_dirpath, loader_mode):
if loader_mode == 'resize_and_pad':
size_adjustment_function = partial(crop_image, target_size=ORIGINAL_SIZE)
elif loader_mode == 'resize':
size_adjustment_function = partial(resize_image, target_size=ORIGINAL_SIZE)
else:
raise NotImplementedError
mask_resize = Step(name='mask_resize',
transformer=make_apply_transformer(size_adjustment_function,
output_name='resized_images',
apply_on=['images']),
input_data=['unet_output'],
adapter=Adapter({'images': E('unet_output', 'mask_prediction'),
}),
experiment_directory=cache_dirpath)
binarizer = Step(name='binarizer',
transformer=make_apply_transformer(
partial(binarize, threshold=THRESHOLD),
output_name='binarized_images',
apply_on=['images']),
input_steps=[mask_resize],
adapter=Adapter({'images': E(mask_resize.name, 'resized_images'),
}),
experiment_directory=cache_dirpath)
output = Step(name='output',
transformer=IdentityOperation(),
input_steps=[binarizer],
adapter=Adapter({'y_pred': E(binarizer.name, 'binarized_images'),
}),
experiment_directory=cache_dirpath)
return output
def test_inputs_with_conflicting_names_require_adapter(data):
step = Step(name='test_inputs_with_conflicting_names_require_adapter',
transformer=IdentityOperation(),
input_data=['input_1', 'input_3'],
cache_dirpath=CACHE_DIRPATH)
with pytest.raises(StepsError):
step.fit_transform(data)
def retinanet(config, train_mode, visualize=False):
persist_output = False
load_persisted_output = False
loader = preprocessing_generator(config, is_train=train_mode)
retinanet = Step(name='retinanet',
transformer=Retina(**config.retinanet,
train_mode=train_mode),
input_steps=[loader],
experiment_directory=config.env.cache_dirpath,
persist_output=persist_output,
is_trainable=True,
load_persisted_output=load_persisted_output)
if train_mode:
return retinanet
if visualize:
return visualizer(retinanet, loader.get_step('label_encoder'), config)
postprocessor = postprocessing(retinanet, loader.get_step('label_encoder'),
config)
output = Step(name='output',
transformer=IdentityOperation(),
input_steps=[postprocessor],
adapter=Adapter(
{'y_pred': E(postprocessor.name, 'submission')}),
experiment_directory=config.env.cache_dirpath,
persist_output=persist_output,
load_persisted_output=load_persisted_output)
return output
def test_inputs_with_conflicting_names_require_adapter(data):
step = Step(name='test_inputs_with_conflicting_names_require_adapter',
transformer=IdentityOperation(),
input_data=['input_1', 'input_3'],
experiment_directory=EXP_DIR)
with pytest.raises(StepsError):
step.fit_transform(data)
def unet_tta(config, suffix=''):
preprocessing, tta_generator = preprocessing_inference_tta(
config, model_name='unet')
unet = Step(name='unet{}'.format(suffix),
transformer=PyTorchUNet(**config.model['unet']),
input_data=['callback_input'],
input_steps=[preprocessing],
is_trainable=True,
experiment_directory=config.env.experiment_dir)
tta_aggregator = aggregator('tta_aggregator{}'.format(suffix),
unet,
tta_generator=tta_generator,
experiment_directory=config.env.experiment_dir,
config=config.tta_aggregator)
prediction_renamed = Step(name='prediction_renamed{}'.format(suffix),
transformer=IdentityOperation(),
input_steps=[tta_aggregator],
adapter=Adapter({
'mask_prediction':
E(tta_aggregator.name,
'aggregated_prediction')
}),
experiment_directory=config.env.experiment_dir)
return prediction_renamed
def postprocessing_pipeline_simplified(cache_dirpath):
mask_resize = Step(
name='mask_resize',
transformer=make_apply_transformer(post.resize_image,
output_name='resized_images',
apply_on=['images',
'target_sizes']),
input_data=['unet_output', 'callback_input'],
adapter={
'images': ([('unet_output', 'multichannel_map_prediction')]),
'target_sizes': ([('callback_input', 'target_sizes')]),
},
cache_dirpath=cache_dirpath)
category_mapper = Step(name='category_mapper',
transformer=make_apply_transformer(
post.categorize_image,
output_name='categorized_images'),
input_steps=[mask_resize],
adapter={
'images': ([('mask_resize', 'resized_images')]),
},
cache_dirpath=cache_dirpath)
labeler = Step(
name='labeler',
transformer=make_apply_transformer(post.label_multiclass_image,
output_name='labeled_images'),
input_steps=[category_mapper],
adapter={
'images': ([(category_mapper.name, 'categorized_images')]),
},
cache_dirpath=cache_dirpath)
score_builder = Step(name='score_builder',
transformer=make_apply_transformer(
post.build_score,
output_name='images_with_scores',
apply_on=['images', 'probabilities']),
input_steps=[labeler, mask_resize],
adapter={
'images': ([(labeler.name, 'labeled_images')]),
'probabilities':
([(mask_resize.name, 'resized_images')]),
},
cache_dirpath=cache_dirpath)
output = Step(name='output',
transformer=IdentityOperation(),
input_steps=[score_builder],
adapter={
'y_pred': ([(score_builder.name, 'images_with_scores')]),
},
cache_dirpath=cache_dirpath)
return output
def unet_tta(config, suffix=''):
preprocessing, tta_generator = pipelines.preprocessing_inference_tta(
config, model_name='unet')
unet = Step(name='unet{}'.format(suffix),
transformer=models.PyTorchUNet(**config.model['unet']),
input_data=['callback_input'],
input_steps=[preprocessing],
is_trainable=True,
experiment_directory=config.execution.experiment_dir)
tta_aggregator = pipelines.aggregator(
'tta_aggregator{}'.format(suffix),
unet,
tta_generator=tta_generator,
experiment_directory=config.execution.experiment_dir,
config=config.tta_aggregator)
prediction_renamed = Step(
name='prediction_renamed{}'.format(suffix),
transformer=IdentityOperation(),
input_steps=[tta_aggregator],
adapter=Adapter({
'mask_prediction':
E(tta_aggregator.name, 'aggregated_prediction')
}),
experiment_directory=config.execution.experiment_dir)
if config.general.loader_mode == 'resize_and_pad':
size_adjustment_function = partial(
postprocessing.crop_image,
target_size=config.general.original_size)
elif config.general.loader_mode == 'resize':
size_adjustment_function = partial(
postprocessing.resize_image,
target_size=config.general.original_size)
else:
raise NotImplementedError
mask_resize = Step(name='mask_resize{}'.format(suffix),
transformer=utils.make_apply_transformer(
size_adjustment_function,
output_name='resized_images',
apply_on=['images']),
input_steps=[prediction_renamed],
adapter=Adapter({
'images':
E(prediction_renamed.name, 'mask_prediction'),
}),
experiment_directory=config.execution.experiment_dir)
return mask_resize
def stacking_features(config, train_mode, suffix, **kwargs):
features = Step(name='stacking_features{}'.format(suffix),
transformer=IdentityOperation(),
input_data=['input'],
adapter=Adapter({'numerical_features': E('input', 'X')}),
experiment_directory=config.pipeline.experiment_directory,
**kwargs)
if train_mode:
features_valid = Step(
name='stacking_features_valid{}'.format(suffix),
transformer=IdentityOperation(),
input_data=['input'],
adapter=Adapter({'numerical_features': E('input', 'X_valid')}),
experiment_directory=config.pipeline.experiment_directory,
**kwargs)
feature_combiner, feature_combiner_valid = _join_features(
numerical_features=[features],
numerical_features_valid=[features_valid],
categorical_features=[],
categorical_features_valid=[],
config=config,
train_mode=train_mode,
suffix=suffix,
**kwargs)
return feature_combiner, feature_combiner_valid
else:
feature_combiner = _join_features(numerical_features=[features],
numerical_features_valid=[],
categorical_features=[],
categorical_features_valid=[],
config=config,
train_mode=train_mode,
suffix=suffix,
**kwargs)
return feature_combiner
def network_tta(config, suffix=''):
if SECOND_LEVEL:
raise NotImplementedError('Second level does not work with TTA')
preprocessing, tta_generator = pipelines.preprocessing_inference_tta(config, model_name='network')
if USE_DEPTH:
Network = models.SegmentationModelWithDepth
else:
Network = models.SegmentationModel
network = Step(name='network{}'.format(suffix),
transformer=Network(**config.model['network']),
input_data=['callback_input'],
input_steps=[preprocessing],
is_trainable=True,
experiment_directory=config.execution.experiment_dir)
tta_aggregator = pipelines.aggregator('tta_aggregator{}'.format(suffix), network,
tta_generator=tta_generator,
experiment_directory=config.execution.experiment_dir,
config=config.tta_aggregator)
prediction_renamed = Step(name='prediction_renamed{}'.format(suffix),
transformer=IdentityOperation(),
input_steps=[tta_aggregator],
adapter=Adapter({'mask_prediction': E(tta_aggregator.name, 'aggregated_prediction')
}),
experiment_directory=config.execution.experiment_dir)
if config.general.loader_mode == 'resize_and_pad':
size_adjustment_function = partial(postprocessing.crop_image, target_size=config.general.original_size)
elif config.general.loader_mode == 'resize' or config.general.loader_mode == 'stacking':
size_adjustment_function = partial(postprocessing.resize_image, target_size=config.general.original_size)
else:
raise NotImplementedError
mask_resize = Step(name='mask_resize{}'.format(suffix),
transformer=utils.make_apply_transformer(size_adjustment_function,
output_name='resized_images',
apply_on=['images']),
input_steps=[prediction_renamed],
adapter=Adapter({'images': E(prediction_renamed.name, 'mask_prediction'),
}),
experiment_directory=config.execution.experiment_dir)
return mask_resize
def test_step_with_adapted_inputs(data):
step = Step(name='test_step_wit_adapted_inputs',
transformer=IdentityOperation(),
input_data=['input_1', 'input_3'],
adapter=Adapter({
'img': E('input_3', 'images'),
'fea': E('input_1', 'features'),
'l1': E('input_3', 'labels'),
'l2': E('input_1', 'labels'),
}))
output = step.fit_transform(data)
expected = {
'img': data['input_3']['images'],
'fea': data['input_1']['features'],
'l1': data['input_3']['labels'],
'l2': data['input_1']['labels'],
}
assert output == expected
def postprocessing_pipeline_simplified(cache_dirpath, loader_mode):
if loader_mode == "resize_and_pad":
size_adjustment_function = partial(crop_image, target_size=ORIGINAL_SIZE)
elif loader_mode == "resize":
size_adjustment_function = partial(resize_image, target_size=ORIGINAL_SIZE)
else:
raise NotImplementedError
mask_resize = Step(
name="mask_resize",
transformer=make_apply_transformer(
size_adjustment_function, output_name="resized_images", apply_on=["images"]
),
input_data=["unet_output"],
adapter=Adapter({"images": E("unet_output", "mask_prediction")}),
experiment_directory=cache_dirpath,
)
binarizer = Step(
name="binarizer",
transformer=make_apply_transformer(
partial(binarize, threshold=THRESHOLD),
output_name="binarized_images",
apply_on=["images"],
),
input_steps=[mask_resize],
adapter=Adapter({"images": E(mask_resize.name, "resized_images")}),
experiment_directory=cache_dirpath,
)
output = Step(
name="output",
transformer=IdentityOperation(),
input_steps=[binarizer],
adapter=Adapter({"y_pred": E(binarizer.name, "binarized_images")}),
experiment_directory=cache_dirpath,
)
return output
def unet(config, train_mode, suffix=''):
if train_mode:
preprocessing = preprocessing_train(config,
model_name='unet',
suffix=suffix)
else:
preprocessing = preprocessing_inference(config, suffix=suffix)
unet = Step(name='unet{}'.format(suffix),
transformer=PyTorchUNet(**config.model['unet']),
input_data=['callback_input'],
input_steps=[preprocessing],
adapter=Adapter({
'datagen':
E(preprocessing.name, 'datagen'),
'validation_datagen':
E(preprocessing.name, 'validation_datagen'),
'meta_valid':
E('callback_input', 'meta_valid'),
}),
is_trainable=True,
experiment_directory=config.env.experiment_dir)
if train_mode:
return unet
mask_postprocessed = mask_postprocessing(unet, config, suffix)
output = Step(name='output{}'.format(suffix),
transformer=IdentityOperation(),
input_steps=[mask_postprocessed],
adapter=Adapter({
'y_pred':
E(mask_postprocessed.name, 'binarized_images'),
}),
experiment_directory=config.env.experiment_dir)
return output
def inference_segmentation_pipeline(config):
if config.general.loader_mode == 'resize_and_pad':
size_adjustment_function = partial(
postprocessing.crop_image,
target_size=config.general.original_size)
elif config.general.loader_mode == 'resize' or config.general.loader_mode == 'stacking':
size_adjustment_function = partial(
postprocessing.resize_image,
target_size=config.general.original_size)
else:
raise NotImplementedError
if USE_TTA:
preprocessing, tta_generator = pipelines.preprocessing_inference_tta(
config, model_name='segmentation_network')
segmentation_network = Step(
name='segmentation_network',
transformer=models.SegmentationModel(
**config.model['segmentation_network']),
input_steps=[preprocessing])
tta_aggregator = pipelines.aggregator('tta_aggregator',
segmentation_network,
tta_generator=tta_generator,
config=config.tta_aggregator)
prediction_renamed = Step(name='prediction_renamed',
transformer=IdentityOperation(),
input_steps=[tta_aggregator],
adapter=Adapter({
'mask_prediction':
E(tta_aggregator.name,
'aggregated_prediction')
}))
mask_resize = Step(name='mask_resize',
transformer=misc.make_apply_transformer(
size_adjustment_function,
output_name='resized_images',
apply_on=['images'],
n_threads=config.execution.num_threads,
),
input_steps=[prediction_renamed],
adapter=Adapter({
'images':
E(prediction_renamed.name, 'mask_prediction'),
}))
else:
preprocessing = pipelines.preprocessing_inference(
config, model_name='segmentation_network')
segmentation_network = misc.FineTuneStep(
name='segmentation_network',
transformer=models.SegmentationModel(
**config.model['segmentation_network']),
input_steps=[preprocessing],
)
mask_resize = Step(
name='mask_resize',
transformer=misc.make_apply_transformer(
size_adjustment_function,
output_name='resized_images',
apply_on=['images'],
n_threads=config.execution.num_threads,
),
input_steps=[segmentation_network],
adapter=Adapter({
'images':
E(segmentation_network.name, 'mask_prediction'),
}),
)
binarizer = Step(name='binarizer',
transformer=misc.make_apply_transformer(
partial(postprocessing.binarize,
threshold=config.thresholder.threshold_masks),
output_name='binarized_images',
apply_on=['images'],
n_threads=config.execution.num_threads),
input_steps=[mask_resize],
adapter=Adapter({
'images':
E(mask_resize.name, 'resized_images'),
}))
labeler = Step(name='labeler',
transformer=misc.make_apply_transformer(
postprocessing.label,
output_name='labeled_images',
apply_on=['images'],
n_threads=config.execution.num_threads,
),
input_steps=[binarizer],
adapter=Adapter({
'images':
E(binarizer.name, 'binarized_images'),
}))
mask_postprocessing = Step(name='mask_postprocessing',
transformer=misc.make_apply_transformer(
postprocessing.mask_postprocessing,
output_name='labeled_images',
apply_on=['images'],
n_threads=config.execution.num_threads,
),
input_steps=[labeler],
adapter=Adapter({
'images':
E(labeler.name, 'labeled_images'),
}))
mask_postprocessing.set_mode_inference()
mask_postprocessing.set_parameters_upstream({
'experiment_directory':
config.execution.experiment_dir,
'is_fittable':
False
})
segmentation_network.is_fittable = True
return mask_postprocessing