def test_inputs_without_conflicting_names_do_not_require_adapter(data): step = Step( name='test_inputs_without_conflicting_names_do_not_require_adapter_1', transformer=IdentityOperation(), input_data=['input_1']) output = step.fit_transform(data) assert output == data['input_1'] step = Step( name='test_inputs_without_conflicting_names_do_not_require_adapter_2', transformer=IdentityOperation(), input_data=['input_1', 'input_2']) output = step.fit_transform(data) assert output == {**data['input_1'], **data['input_2']}
def postprocessing_pipeline_simplified(cache_dirpath, loader_mode): if loader_mode == 'resize_and_pad': size_adjustment_function = partial(crop_image, target_size=ORIGINAL_SIZE) elif loader_mode == 'resize': size_adjustment_function = partial(resize_image, target_size=ORIGINAL_SIZE) else: raise NotImplementedError mask_resize = Step(name='mask_resize', transformer=make_apply_transformer(size_adjustment_function, output_name='resized_images', apply_on=['images']), input_data=['unet_output'], adapter=Adapter({'images': E('unet_output', 'mask_prediction'), }), experiment_directory=cache_dirpath) binarizer = Step(name='binarizer', transformer=make_apply_transformer( partial(binarize, threshold=THRESHOLD), output_name='binarized_images', apply_on=['images']), input_steps=[mask_resize], adapter=Adapter({'images': E(mask_resize.name, 'resized_images'), }), experiment_directory=cache_dirpath) output = Step(name='output', transformer=IdentityOperation(), input_steps=[binarizer], adapter=Adapter({'y_pred': E(binarizer.name, 'binarized_images'), }), experiment_directory=cache_dirpath) return output
def test_inputs_with_conflicting_names_require_adapter(data): step = Step(name='test_inputs_with_conflicting_names_require_adapter', transformer=IdentityOperation(), input_data=['input_1', 'input_3'], cache_dirpath=CACHE_DIRPATH) with pytest.raises(StepsError): step.fit_transform(data)
def retinanet(config, train_mode, visualize=False): persist_output = False load_persisted_output = False loader = preprocessing_generator(config, is_train=train_mode) retinanet = Step(name='retinanet', transformer=Retina(**config.retinanet, train_mode=train_mode), input_steps=[loader], experiment_directory=config.env.cache_dirpath, persist_output=persist_output, is_trainable=True, load_persisted_output=load_persisted_output) if train_mode: return retinanet if visualize: return visualizer(retinanet, loader.get_step('label_encoder'), config) postprocessor = postprocessing(retinanet, loader.get_step('label_encoder'), config) output = Step(name='output', transformer=IdentityOperation(), input_steps=[postprocessor], adapter=Adapter( {'y_pred': E(postprocessor.name, 'submission')}), experiment_directory=config.env.cache_dirpath, persist_output=persist_output, load_persisted_output=load_persisted_output) return output
def test_inputs_with_conflicting_names_require_adapter(data): step = Step(name='test_inputs_with_conflicting_names_require_adapter', transformer=IdentityOperation(), input_data=['input_1', 'input_3'], experiment_directory=EXP_DIR) with pytest.raises(StepsError): step.fit_transform(data)
def unet_tta(config, suffix=''): preprocessing, tta_generator = preprocessing_inference_tta( config, model_name='unet') unet = Step(name='unet{}'.format(suffix), transformer=PyTorchUNet(**config.model['unet']), input_data=['callback_input'], input_steps=[preprocessing], is_trainable=True, experiment_directory=config.env.experiment_dir) tta_aggregator = aggregator('tta_aggregator{}'.format(suffix), unet, tta_generator=tta_generator, experiment_directory=config.env.experiment_dir, config=config.tta_aggregator) prediction_renamed = Step(name='prediction_renamed{}'.format(suffix), transformer=IdentityOperation(), input_steps=[tta_aggregator], adapter=Adapter({ 'mask_prediction': E(tta_aggregator.name, 'aggregated_prediction') }), experiment_directory=config.env.experiment_dir) return prediction_renamed
def postprocessing_pipeline_simplified(cache_dirpath): mask_resize = Step( name='mask_resize', transformer=make_apply_transformer(post.resize_image, output_name='resized_images', apply_on=['images', 'target_sizes']), input_data=['unet_output', 'callback_input'], adapter={ 'images': ([('unet_output', 'multichannel_map_prediction')]), 'target_sizes': ([('callback_input', 'target_sizes')]), }, cache_dirpath=cache_dirpath) category_mapper = Step(name='category_mapper', transformer=make_apply_transformer( post.categorize_image, output_name='categorized_images'), input_steps=[mask_resize], adapter={ 'images': ([('mask_resize', 'resized_images')]), }, cache_dirpath=cache_dirpath) labeler = Step( name='labeler', transformer=make_apply_transformer(post.label_multiclass_image, output_name='labeled_images'), input_steps=[category_mapper], adapter={ 'images': ([(category_mapper.name, 'categorized_images')]), }, cache_dirpath=cache_dirpath) score_builder = Step(name='score_builder', transformer=make_apply_transformer( post.build_score, output_name='images_with_scores', apply_on=['images', 'probabilities']), input_steps=[labeler, mask_resize], adapter={ 'images': ([(labeler.name, 'labeled_images')]), 'probabilities': ([(mask_resize.name, 'resized_images')]), }, cache_dirpath=cache_dirpath) output = Step(name='output', transformer=IdentityOperation(), input_steps=[score_builder], adapter={ 'y_pred': ([(score_builder.name, 'images_with_scores')]), }, cache_dirpath=cache_dirpath) return output
def unet_tta(config, suffix=''): preprocessing, tta_generator = pipelines.preprocessing_inference_tta( config, model_name='unet') unet = Step(name='unet{}'.format(suffix), transformer=models.PyTorchUNet(**config.model['unet']), input_data=['callback_input'], input_steps=[preprocessing], is_trainable=True, experiment_directory=config.execution.experiment_dir) tta_aggregator = pipelines.aggregator( 'tta_aggregator{}'.format(suffix), unet, tta_generator=tta_generator, experiment_directory=config.execution.experiment_dir, config=config.tta_aggregator) prediction_renamed = Step( name='prediction_renamed{}'.format(suffix), transformer=IdentityOperation(), input_steps=[tta_aggregator], adapter=Adapter({ 'mask_prediction': E(tta_aggregator.name, 'aggregated_prediction') }), experiment_directory=config.execution.experiment_dir) if config.general.loader_mode == 'resize_and_pad': size_adjustment_function = partial( postprocessing.crop_image, target_size=config.general.original_size) elif config.general.loader_mode == 'resize': size_adjustment_function = partial( postprocessing.resize_image, target_size=config.general.original_size) else: raise NotImplementedError mask_resize = Step(name='mask_resize{}'.format(suffix), transformer=utils.make_apply_transformer( size_adjustment_function, output_name='resized_images', apply_on=['images']), input_steps=[prediction_renamed], adapter=Adapter({ 'images': E(prediction_renamed.name, 'mask_prediction'), }), experiment_directory=config.execution.experiment_dir) return mask_resize
def stacking_features(config, train_mode, suffix, **kwargs): features = Step(name='stacking_features{}'.format(suffix), transformer=IdentityOperation(), input_data=['input'], adapter=Adapter({'numerical_features': E('input', 'X')}), experiment_directory=config.pipeline.experiment_directory, **kwargs) if train_mode: features_valid = Step( name='stacking_features_valid{}'.format(suffix), transformer=IdentityOperation(), input_data=['input'], adapter=Adapter({'numerical_features': E('input', 'X_valid')}), experiment_directory=config.pipeline.experiment_directory, **kwargs) feature_combiner, feature_combiner_valid = _join_features( numerical_features=[features], numerical_features_valid=[features_valid], categorical_features=[], categorical_features_valid=[], config=config, train_mode=train_mode, suffix=suffix, **kwargs) return feature_combiner, feature_combiner_valid else: feature_combiner = _join_features(numerical_features=[features], numerical_features_valid=[], categorical_features=[], categorical_features_valid=[], config=config, train_mode=train_mode, suffix=suffix, **kwargs) return feature_combiner
def network_tta(config, suffix=''): if SECOND_LEVEL: raise NotImplementedError('Second level does not work with TTA') preprocessing, tta_generator = pipelines.preprocessing_inference_tta(config, model_name='network') if USE_DEPTH: Network = models.SegmentationModelWithDepth else: Network = models.SegmentationModel network = Step(name='network{}'.format(suffix), transformer=Network(**config.model['network']), input_data=['callback_input'], input_steps=[preprocessing], is_trainable=True, experiment_directory=config.execution.experiment_dir) tta_aggregator = pipelines.aggregator('tta_aggregator{}'.format(suffix), network, tta_generator=tta_generator, experiment_directory=config.execution.experiment_dir, config=config.tta_aggregator) prediction_renamed = Step(name='prediction_renamed{}'.format(suffix), transformer=IdentityOperation(), input_steps=[tta_aggregator], adapter=Adapter({'mask_prediction': E(tta_aggregator.name, 'aggregated_prediction') }), experiment_directory=config.execution.experiment_dir) if config.general.loader_mode == 'resize_and_pad': size_adjustment_function = partial(postprocessing.crop_image, target_size=config.general.original_size) elif config.general.loader_mode == 'resize' or config.general.loader_mode == 'stacking': size_adjustment_function = partial(postprocessing.resize_image, target_size=config.general.original_size) else: raise NotImplementedError mask_resize = Step(name='mask_resize{}'.format(suffix), transformer=utils.make_apply_transformer(size_adjustment_function, output_name='resized_images', apply_on=['images']), input_steps=[prediction_renamed], adapter=Adapter({'images': E(prediction_renamed.name, 'mask_prediction'), }), experiment_directory=config.execution.experiment_dir) return mask_resize
def test_step_with_adapted_inputs(data): step = Step(name='test_step_wit_adapted_inputs', transformer=IdentityOperation(), input_data=['input_1', 'input_3'], adapter=Adapter({ 'img': E('input_3', 'images'), 'fea': E('input_1', 'features'), 'l1': E('input_3', 'labels'), 'l2': E('input_1', 'labels'), })) output = step.fit_transform(data) expected = { 'img': data['input_3']['images'], 'fea': data['input_1']['features'], 'l1': data['input_3']['labels'], 'l2': data['input_1']['labels'], } assert output == expected
def postprocessing_pipeline_simplified(cache_dirpath, loader_mode): if loader_mode == "resize_and_pad": size_adjustment_function = partial(crop_image, target_size=ORIGINAL_SIZE) elif loader_mode == "resize": size_adjustment_function = partial(resize_image, target_size=ORIGINAL_SIZE) else: raise NotImplementedError mask_resize = Step( name="mask_resize", transformer=make_apply_transformer( size_adjustment_function, output_name="resized_images", apply_on=["images"] ), input_data=["unet_output"], adapter=Adapter({"images": E("unet_output", "mask_prediction")}), experiment_directory=cache_dirpath, ) binarizer = Step( name="binarizer", transformer=make_apply_transformer( partial(binarize, threshold=THRESHOLD), output_name="binarized_images", apply_on=["images"], ), input_steps=[mask_resize], adapter=Adapter({"images": E(mask_resize.name, "resized_images")}), experiment_directory=cache_dirpath, ) output = Step( name="output", transformer=IdentityOperation(), input_steps=[binarizer], adapter=Adapter({"y_pred": E(binarizer.name, "binarized_images")}), experiment_directory=cache_dirpath, ) return output
def unet(config, train_mode, suffix=''): if train_mode: preprocessing = preprocessing_train(config, model_name='unet', suffix=suffix) else: preprocessing = preprocessing_inference(config, suffix=suffix) unet = Step(name='unet{}'.format(suffix), transformer=PyTorchUNet(**config.model['unet']), input_data=['callback_input'], input_steps=[preprocessing], adapter=Adapter({ 'datagen': E(preprocessing.name, 'datagen'), 'validation_datagen': E(preprocessing.name, 'validation_datagen'), 'meta_valid': E('callback_input', 'meta_valid'), }), is_trainable=True, experiment_directory=config.env.experiment_dir) if train_mode: return unet mask_postprocessed = mask_postprocessing(unet, config, suffix) output = Step(name='output{}'.format(suffix), transformer=IdentityOperation(), input_steps=[mask_postprocessed], adapter=Adapter({ 'y_pred': E(mask_postprocessed.name, 'binarized_images'), }), experiment_directory=config.env.experiment_dir) return output
def inference_segmentation_pipeline(config): if config.general.loader_mode == 'resize_and_pad': size_adjustment_function = partial( postprocessing.crop_image, target_size=config.general.original_size) elif config.general.loader_mode == 'resize' or config.general.loader_mode == 'stacking': size_adjustment_function = partial( postprocessing.resize_image, target_size=config.general.original_size) else: raise NotImplementedError if USE_TTA: preprocessing, tta_generator = pipelines.preprocessing_inference_tta( config, model_name='segmentation_network') segmentation_network = Step( name='segmentation_network', transformer=models.SegmentationModel( **config.model['segmentation_network']), input_steps=[preprocessing]) tta_aggregator = pipelines.aggregator('tta_aggregator', segmentation_network, tta_generator=tta_generator, config=config.tta_aggregator) prediction_renamed = Step(name='prediction_renamed', transformer=IdentityOperation(), input_steps=[tta_aggregator], adapter=Adapter({ 'mask_prediction': E(tta_aggregator.name, 'aggregated_prediction') })) mask_resize = Step(name='mask_resize', transformer=misc.make_apply_transformer( size_adjustment_function, output_name='resized_images', apply_on=['images'], n_threads=config.execution.num_threads, ), input_steps=[prediction_renamed], adapter=Adapter({ 'images': E(prediction_renamed.name, 'mask_prediction'), })) else: preprocessing = pipelines.preprocessing_inference( config, model_name='segmentation_network') segmentation_network = misc.FineTuneStep( name='segmentation_network', transformer=models.SegmentationModel( **config.model['segmentation_network']), input_steps=[preprocessing], ) mask_resize = Step( name='mask_resize', transformer=misc.make_apply_transformer( size_adjustment_function, output_name='resized_images', apply_on=['images'], n_threads=config.execution.num_threads, ), input_steps=[segmentation_network], adapter=Adapter({ 'images': E(segmentation_network.name, 'mask_prediction'), }), ) binarizer = Step(name='binarizer', transformer=misc.make_apply_transformer( partial(postprocessing.binarize, threshold=config.thresholder.threshold_masks), output_name='binarized_images', apply_on=['images'], n_threads=config.execution.num_threads), input_steps=[mask_resize], adapter=Adapter({ 'images': E(mask_resize.name, 'resized_images'), })) labeler = Step(name='labeler', transformer=misc.make_apply_transformer( postprocessing.label, output_name='labeled_images', apply_on=['images'], n_threads=config.execution.num_threads, ), input_steps=[binarizer], adapter=Adapter({ 'images': E(binarizer.name, 'binarized_images'), })) mask_postprocessing = Step(name='mask_postprocessing', transformer=misc.make_apply_transformer( postprocessing.mask_postprocessing, output_name='labeled_images', apply_on=['images'], n_threads=config.execution.num_threads, ), input_steps=[labeler], adapter=Adapter({ 'images': E(labeler.name, 'labeled_images'), })) mask_postprocessing.set_mode_inference() mask_postprocessing.set_parameters_upstream({ 'experiment_directory': config.execution.experiment_dir, 'is_fittable': False }) segmentation_network.is_fittable = True return mask_postprocessing