def test(self):
heatmap_test = HeatmapTest(channel_axis=0, invert_transformation=False)
landmark_statistics = LandmarkStatistics()
landmarks = {}
for i in range(self.dataset_val.num_entries()):
dataset_entry = self.dataset_val.get_next()
current_id = dataset_entry['id']['image_id']
datasources = dataset_entry['datasources']
reference_image = datasources['image_datasource']
groundtruth_landmarks = datasources['landmarks_datasource']
image, prediction, transform = self.test_full_image(dataset_entry)
utils.io.image.write_np((prediction * 128).astype(np.int8), self.output_file_for_current_iteration(current_id + '_heatmap.mha'))
predicted_landmarks = heatmap_test.get_landmarks(prediction, reference_image, transformation=transform)
tensorflow_train.utils.tensorflow_util.print_progress_bar(i, self.dataset_val.num_entries())
landmarks[current_id] = predicted_landmarks
landmark_statistics.add_landmarks(current_id, predicted_landmarks, groundtruth_landmarks, normalization_factor=50, normalization_indizes=[1, 5])
tensorflow_train.utils.tensorflow_util.print_progress_bar(self.dataset_val.num_entries(), self.dataset_val.num_entries())
print('ipe', landmark_statistics.get_ipe_statistics())
print('pe', landmark_statistics.get_pe_statistics())
print('outliers', landmark_statistics.get_num_outliers([2.0, 4.0, 10.0]))
# finalize loss values
self.val_loss_aggregator.finalize(self.current_iter)
utils.io.landmark.save_points_csv(landmarks, self.output_file_for_current_iteration('prediction.csv'))
def test(self):
heatmap_test = HeatmapTest(channel_axis=0, invert_transformation=False)
challenge_landmark_statistics = LandmarkStatistics()
senior_landmark_statistics = LandmarkStatistics()
junior_landmark_statistics = LandmarkStatistics()
mean_landmark_statistics = LandmarkStatistics()
landmarks = {}
for i in range(self.dataset_val.num_entries()):
dataset_entry = self.dataset_val.get_next()
current_id = dataset_entry['id']['image_id']
datasources = dataset_entry['datasources']
reference_image = datasources['image_datasource']
groundtruth_challenge_landmarks = datasources['challenge_landmarks_datasource']
groundtruth_senior_landmarks = datasources['senior_landmarks_datasource']
groundtruth_junior_landmarks = datasources['junior_landmarks_datasource']
groundtruth_mean_landmarks = datasources['mean_landmarks_datasource']
image, prediction, target_heatmaps, transform = self.test_full_image(dataset_entry)
utils.io.image.write_np(image, self.output_file_for_current_iteration(current_id + '_image.mha'))
utils.io.image.write_np(prediction, self.output_file_for_current_iteration(current_id + '_prediction.mha'))
utils.io.image.write_np(target_heatmaps, self.output_file_for_current_iteration(current_id + '_target_heatmap.mha'))
predicted_landmarks = heatmap_test.get_landmarks(prediction, reference_image, output_spacing=self.image_spacing, transformation=transform)
tensorflow_train.utils.tensorflow_util.print_progress_bar(i, self.dataset_val.num_entries())
landmarks[current_id] = predicted_landmarks
challenge_landmark_statistics.add_landmarks(current_id, predicted_landmarks, groundtruth_challenge_landmarks)
senior_landmark_statistics.add_landmarks(current_id, predicted_landmarks, groundtruth_senior_landmarks)
junior_landmark_statistics.add_landmarks(current_id, predicted_landmarks, groundtruth_junior_landmarks)
mean_landmark_statistics.add_landmarks(current_id, predicted_landmarks, groundtruth_mean_landmarks)
tensorflow_train.utils.tensorflow_util.print_progress_bar(self.dataset_val.num_entries(), self.dataset_val.num_entries())
challenge_summaries = self.finalize_landmark_statistics(challenge_landmark_statistics, 'challenge')
senior_summaries = self.finalize_landmark_statistics(senior_landmark_statistics, 'senior')
junior_summaries = self.finalize_landmark_statistics(junior_landmark_statistics, 'junior')
mean_summaries = self.finalize_landmark_statistics(mean_landmark_statistics, 'mean')
additional_summaries = OrderedDict(chain(senior_summaries.items(), junior_summaries.items(), challenge_summaries.items(), mean_summaries.items()))
# finalize loss values
self.val_loss_aggregator.finalize(self.current_iter, additional_summaries)
utils.io.landmark.save_points_csv(landmarks, self.output_file_for_current_iteration('prediction.csv'))
def test(self):
"""
The test function. Performs inference on the the validation images and calculates the loss.
"""
print('Testing...')
channel_axis = 0
if self.data_format == 'channels_last':
channel_axis = 3
if self.use_spine_postprocessing:
heatmap_maxima = HeatmapTest(channel_axis,
False,
return_multiple_maxima=True,
min_max_distance=7,
min_max_value=0.25,
multiple_min_max_value_factor=0.1)
spine_postprocessing = SpinePostprocessing(
num_landmarks=self.num_landmarks,
image_spacing=self.image_spacing)
else:
heatmap_maxima = HeatmapTest(channel_axis, False)
landmark_statistics = LandmarkStatistics()
landmarks = {}
num_entries = self.dataset_val.num_entries()
for i in range(num_entries):
dataset_entry = self.dataset_val.get_next()
current_id = dataset_entry['id']['image_id']
datasources = dataset_entry['datasources']
input_image = datasources['image']
target_landmarks = datasources['landmarks']
image, prediction, transformation = self.test_cropped_image(
dataset_entry)
if self.save_output_images:
if self.save_output_images_as_uint:
image_normalization = 'min_max'
heatmap_normalization = (0, 1)
output_image_type = np.uint8
else:
image_normalization = None
heatmap_normalization = None
output_image_type = np.float32
origin = transformation.TransformPoint(np.zeros(3, np.float64))
utils.io.image.write_multichannel_np(
image,
self.output_file_for_current_iteration(current_id +
'_input.mha'),
output_normalization_mode=image_normalization,
data_format=self.data_format,
image_type=output_image_type,
spacing=self.image_spacing,
origin=origin)
utils.io.image.write_multichannel_np(
prediction,
self.output_file_for_current_iteration(current_id +
'_prediction.mha'),
output_normalization_mode=heatmap_normalization,
data_format=self.data_format,
image_type=output_image_type,
spacing=self.image_spacing,
origin=origin)
if self.use_spine_postprocessing:
local_maxima_landmarks = heatmap_maxima.get_landmarks(
prediction, input_image, self.image_spacing,
transformation)
landmark_sequence = spine_postprocessing.postprocess_landmarks(
local_maxima_landmarks, prediction.shape)
landmarks[current_id] = landmark_sequence
else:
maxima_landmarks = heatmap_maxima.get_landmarks(
prediction, input_image, self.image_spacing,
transformation)
landmarks[current_id] = maxima_landmarks
if self.has_validation_groundtruth:
landmark_statistics.add_landmarks(current_id,
landmark_sequence,
target_landmarks)
print_progress_bar(i,
num_entries,
prefix='Testing ',
suffix=' complete')
utils.io.landmark.save_points_csv(
landmarks, self.output_file_for_current_iteration('points.csv'))
# finalize loss values
if self.has_validation_groundtruth:
print(landmark_statistics.get_pe_overview_string())
print(landmark_statistics.get_correct_id_string(20.0))
summary_values = OrderedDict(
zip(
self.point_statistics_names,
list(landmark_statistics.get_pe_statistics()) +
[landmark_statistics.get_correct_id(20)]))
# finalize loss values
self.val_loss_aggregator.finalize(self.current_iter,
summary_values)
overview_string = landmark_statistics.get_overview_string(
[2, 2.5, 3, 4, 10, 20], 10, 20.0)
utils.io.text.save_string_txt(
overview_string,
self.output_file_for_current_iteration('eval.txt'))
def test(self):
print('Testing...')
if self.data_format == 'channels_first':
np_channel_index = 0
else:
np_channel_index = 3
heatmap_maxima = HeatmapTest(np_channel_index, False)
landmark_statistics = LandmarkStatistics()
landmarks = {}
for i in range(self.dataset_val.num_entries()):
dataset_entry = self.dataset_val.get_next()
current_id = dataset_entry['id']['image_id']
datasources = dataset_entry['datasources']
image_datasource = datasources['image_datasource']
landmarks_datasource = datasources['landmarks_datasource']
if not self.cropped_training:
image, heatmaps, heatmap_transform = self.test_full_image(
dataset_entry)
else:
image, heatmaps, heatmap_transform = self.test_cropped_image(
dataset_entry)
utils.io.image.write_np(
ShiftScaleClamp(scale=255, clamp_min=0,
clamp_max=255)(heatmaps).astype(np.uint8),
self.output_file_for_current_iteration(current_id +
'_heatmaps.mha'))
utils.io.image.write_np(
image,
self.output_file_for_current_iteration(current_id +
'_image.mha'))
predicted_landmarks = heatmap_maxima.get_landmarks(
heatmaps, image_datasource, self.image_spacing,
heatmap_transform)
landmarks[current_id] = predicted_landmarks
landmark_statistics.add_landmarks(current_id, predicted_landmarks,
landmarks_datasource)
tensorflow_train.utils.tensorflow_util.print_progress_bar(
i,
self.dataset_val.num_entries(),
prefix='Testing ',
suffix=' complete')
tensorflow_train.utils.tensorflow_util.print_progress_bar(
self.dataset_val.num_entries(),
self.dataset_val.num_entries(),
prefix='Testing ',
suffix=' complete')
print(landmark_statistics.get_pe_overview_string())
print(landmark_statistics.get_correct_id_string(20.0))
summary_values = OrderedDict(
zip(
self.point_statistics_names,
list(landmark_statistics.get_pe_statistics()) +
[landmark_statistics.get_correct_id(20)]))
# finalize loss values
self.val_loss_aggregator.finalize(self.current_iter, summary_values)
utils.io.landmark.save_points_csv(
landmarks, self.output_file_for_current_iteration('points.csv'))
overview_string = landmark_statistics.get_overview_string(
[2, 2.5, 3, 4, 10, 20], 10, 20.0)
utils.io.text.save_string_txt(
overview_string,
self.output_file_for_current_iteration('eval.txt'))
def test(self):
"""
The test function. Performs inference on the the validation images and calculates the loss.
"""
print('Testing...')
channel_axis = 0
if self.data_format == 'channels_last':
channel_axis = 3
landmark_statistics = LandmarkStatistics()
landmarks = {}
num_entries = self.dataset_val.num_entries()
for i in range(num_entries):
dataset_entry = self.dataset_val.get_next()
current_id = dataset_entry['id']['image_id']
datasources = dataset_entry['datasources']
if self.has_validation_groundtruth:
groundtruth_landmarks = datasources['landmarks']
groundtruth_landmark = [
get_mean_landmark(groundtruth_landmarks)
]
input_image = datasources['image']
image, prediction, transformation = self.test_full_image(
dataset_entry)
predictions_sitk = utils.sitk_image.transform_np_output_to_sitk_input(
output_image=prediction,
output_spacing=self.image_spacing,
channel_axis=channel_axis,
input_image_sitk=input_image,
transform=transformation,
interpolator='linear',
output_pixel_type=sitk.sitkFloat32)
if self.save_output_images:
if self.save_output_images_as_uint:
image_normalization = 'min_max'
heatmap_normalization = (0, 1)
output_image_type = np.uint8
else:
image_normalization = None
heatmap_normalization = None
output_image_type = np.float32
origin = transformation.TransformPoint(np.zeros(3, np.float64))
utils.io.image.write_multichannel_np(
image,
self.output_file_for_current_iteration(current_id +
'_input.mha'),
output_normalization_mode=image_normalization,
data_format=self.data_format,
image_type=output_image_type,
spacing=self.image_spacing,
origin=origin)
utils.io.image.write_multichannel_np(
prediction,
self.output_file_for_current_iteration(current_id +
'_prediction.mha'),
output_normalization_mode=heatmap_normalization,
data_format=self.data_format,
image_type=output_image_type,
spacing=self.image_spacing,
origin=origin)
#utils.io.image.write(predictions_sitk[0], self.output_file_for_current_iteration(current_id + '_prediction_original.mha'))
predictions_com = input_image.TransformContinuousIndexToPhysicalPoint(
list(
reversed(
utils.np_image.center_of_mass(
utils.sitk_np.sitk_to_np_no_copy(
predictions_sitk[0])))))
current_landmark = [Landmark(predictions_com)]
landmarks[current_id] = current_landmark
if self.has_validation_groundtruth:
landmark_statistics.add_landmarks(current_id, current_landmark,
groundtruth_landmark)
print_progress_bar(i,
num_entries,
prefix='Testing ',
suffix=' complete')
utils.io.landmark.save_points_csv(
landmarks, self.output_file_for_current_iteration('points.csv'))
# finalize loss values
if self.has_validation_groundtruth:
print(landmark_statistics.get_pe_overview_string())
summary_values = OrderedDict(
zip(self.point_statistics_names,
list(landmark_statistics.get_pe_statistics())))
# finalize loss values
self.val_loss_aggregator.finalize(self.current_iter,
summary_values)
overview_string = landmark_statistics.get_overview_string(
[2, 2.5, 3, 4, 10, 20], 10)
utils.io.text.save_string_txt(
overview_string,
self.output_file_for_current_iteration('eval.txt'))
def test(self):
"""
The test function. Performs inference on the the validation images and calculates the loss.
"""
print('Testing...')
vis = LandmarkVisualizationMatplotlib(dim=3,
annotations=dict([(i, f'C{i + 1}') for i in range(7)] + # 0-6: C1-C7
[(i, f'T{i - 6}') for i in range(7, 19)] + # 7-18: T1-12
[(i, f'L{i - 18}') for i in range(19, 25)] + # 19-24: L1-6
[(25, 'T13')])) # 25: T13
channel_axis = 0
if self.data_format == 'channels_last':
channel_axis = 3
heatmap_maxima = HeatmapTest(channel_axis,
False,
return_multiple_maxima=True,
min_max_value=0.05,
smoothing_sigma=2.0)
with open('possible_successors.pickle', 'rb') as f:
possible_successors = pickle.load(f)
with open('units_distances.pickle', 'rb') as f:
offsets_mean, distances_mean, distances_std = pickle.load(f)
spine_postprocessing = SpinePostprocessingGraph(num_landmarks=self.num_landmarks,
possible_successors=possible_successors,
offsets_mean=offsets_mean,
distances_mean=distances_mean,
distances_std=distances_std,
bias=2.0,
l=0.2)
landmark_statistics = LandmarkStatistics()
landmarks = {}
landmark_statistics_no_postprocessing = LandmarkStatistics()
landmarks_no_postprocessing = {}
all_local_maxima_landmarks = {}
num_entries = self.dataset_val.num_entries()
for _ in tqdm(range(num_entries), desc='Testing'):
dataset_entry = self.dataset_val.get_next()
current_id = dataset_entry['id']['image_id']
datasources = dataset_entry['datasources']
input_image = datasources['image']
if self.has_validation_groundtruth:
target_landmarks = datasources['landmarks']
else:
target_landmarks = None
image, prediction, prediction_local, prediction_spatial, transformation = self.test_cropped_image(dataset_entry)
origin = transformation.TransformPoint(np.zeros(3, np.float64))
if self.save_output_images:
heatmap_normalization_mode = (-1, 1)
image_type = np.uint8
utils.io.image.write_multichannel_np(image,self.output_folder_handler.path_for_iteration(self.current_iter, current_id + '_input.mha'), output_normalization_mode='min_max', sitk_image_output_mode='vector', data_format=self.data_format, image_type=image_type, spacing=self.image_spacing, origin=origin)
utils.io.image.write_multichannel_np(prediction, self.output_folder_handler.path_for_iteration(self.current_iter, current_id + '_prediction.mha'), output_normalization_mode=heatmap_normalization_mode, sitk_image_output_mode='vector', data_format=self.data_format, image_type=image_type, spacing=self.image_spacing, origin=origin)
utils.io.image.write_multichannel_np(prediction_local, self.output_folder_handler.path_for_iteration(self.current_iter, current_id + '_prediction_local.mha'), output_normalization_mode=heatmap_normalization_mode, sitk_image_output_mode='vector', data_format=self.data_format, image_type=image_type, spacing=self.image_spacing, origin=origin)
utils.io.image.write_multichannel_np(prediction_spatial, self.output_folder_handler.path_for_iteration(self.current_iter, current_id + '_prediction_spatial.mha'), output_normalization_mode=heatmap_normalization_mode, sitk_image_output_mode='vector', data_format=self.data_format, image_type=image_type, spacing=self.image_spacing, origin=origin)
local_maxima_landmarks = heatmap_maxima.get_landmarks(prediction, input_image, self.image_spacing, transformation)
# landmarks without postprocessing are the first local maxima (with the largest value)
curr_landmarks_no_postprocessing = [l[0] if len(l) > 0 else Landmark(coords=[np.nan] * 3, is_valid=False) for l in local_maxima_landmarks]
landmarks_no_postprocessing[current_id] = curr_landmarks_no_postprocessing
if self.has_validation_groundtruth:
landmark_statistics_no_postprocessing.add_landmarks(current_id, curr_landmarks_no_postprocessing, target_landmarks)
vis.visualize_landmark_projections(input_image, target_landmarks, filename=self.output_folder_handler.path_for_iteration(self.current_iter, current_id + '_landmarks_gt.png'))
vis.visualize_prediction_groundtruth_projections(input_image, curr_landmarks_no_postprocessing, target_landmarks, filename=self.output_folder_handler.path_for_iteration(self.current_iter, current_id + '_landmarks.png'))
else:
vis.visualize_landmark_projections(input_image, curr_landmarks_no_postprocessing, filename=self.output_folder_handler.path_for_iteration(self.current_iter, current_id + '_landmarks.png'))
if self.evaluate_landmarks_postprocessing:
try:
local_maxima_landmarks = add_landmarks_from_neighbors(local_maxima_landmarks)
curr_landmarks = spine_postprocessing.solve_local_heatmap_maxima(local_maxima_landmarks)
curr_landmarks = reshift_landmarks(curr_landmarks)
curr_landmarks = filter_landmarks_top_bottom(curr_landmarks, input_image)
except Exception:
print('error in postprocessing', current_id)
curr_landmarks = curr_landmarks_no_postprocessing
landmarks[current_id] = curr_landmarks
if self.has_validation_groundtruth:
landmark_statistics.add_landmarks(current_id, curr_landmarks, target_landmarks)
vis.visualize_prediction_groundtruth_projections(input_image, curr_landmarks, target_landmarks, filename=self.output_folder_handler.path_for_iteration(self.current_iter, current_id + '_landmarks_pp.png'))
else:
vis.visualize_landmark_projections(input_image, curr_landmarks, filename=self.output_folder_handler.path_for_iteration(self.current_iter, current_id + '_landmarks_pp.png'))
utils.io.landmark.save_points_csv(landmarks, self.output_folder_handler.path_for_iteration(self.current_iter, 'points.csv'))
utils.io.landmark.save_points_csv(landmarks_no_postprocessing, self.output_folder_handler.path_for_iteration(self.current_iter, 'points_no_postprocessing.csv'))
# finalize loss values
if self.has_validation_groundtruth:
summary_values = OrderedDict()
if self.evaluate_landmarks_postprocessing:
print(landmark_statistics.get_pe_overview_string())
print(landmark_statistics.get_correct_id_string(20.0))
overview_string = landmark_statistics.get_overview_string([2, 2.5, 3, 4, 10, 20], 10, 20.0)
utils.io.text.save_string_txt(overview_string, self.output_folder_handler.path_for_iteration(self.current_iter, 'eval.txt'))
summary_values.update(OrderedDict(zip(['pe_mean', 'pe_stdev', 'pe_median', 'num_correct'], list(landmark_statistics.get_pe_statistics()) + [landmark_statistics.get_num_correct_id(20)])))
print(landmark_statistics_no_postprocessing.get_pe_overview_string())
print(landmark_statistics_no_postprocessing.get_correct_id_string(20.0))
overview_string = landmark_statistics_no_postprocessing.get_overview_string([2, 2.5, 3, 4, 10, 20], 10, 20.0)
utils.io.text.save_string_txt(overview_string, self.output_folder_handler.path_for_iteration(self.current_iter, 'eval_no_postprocessing.txt'))
summary_values.update(OrderedDict(zip(['pe_mean_np', 'pe_stdev_np', 'pe_median_np', 'num_correct_np'], list(landmark_statistics_no_postprocessing.get_pe_statistics()) + [landmark_statistics_no_postprocessing.get_num_correct_id(20)])))
self.loss_metric_logger_val.update_metrics(summary_values)
# finalize loss values
self.loss_metric_logger_val.finalize(self.current_iter)