def _load_filenames(self):
"""Load preprocessed pair data (input and gt) in handler."""
for input_filename, gt_filename, roi_filename, metadata in self.filename_pairs:
segpair = SegmentationPair(input_filename,
gt_filename,
metadata=metadata,
slice_axis=self.slice_axis,
soft_gt=self.soft_gt)
input_data, gt_data = segpair.get_pair_data()
metadata = segpair.get_pair_metadata()
seg_pair = {
'input': input_data,
'gt': gt_data,
'input_metadata': metadata['input_metadata'],
'gt_metadata': metadata['gt_metadata']
}
self.has_bounding_box = imed_obj_detect.verify_metadata(
seg_pair, self.has_bounding_box)
if self.has_bounding_box:
self.prepro_transforms = imed_obj_detect.adjust_transforms(
self.prepro_transforms,
seg_pair,
length=self.length,
stride=self.stride)
seg_pair, roi_pair = imed_transforms.apply_preprocessing_transforms(
self.prepro_transforms, seg_pair=seg_pair)
for metadata in seg_pair['input_metadata']:
metadata['index_shape'] = seg_pair['input'][0].shape
self.handlers.append((seg_pair, roi_pair))
def load_filenames(self):
"""Load preprocessed pair data (input and gt) in handler."""
for input_filenames, gt_filenames, roi_filename, metadata in self.filename_pairs:
roi_pair = SegmentationPair(input_filenames, roi_filename, metadata=metadata, slice_axis=self.slice_axis,
cache=self.cache, prepro_transforms=self.prepro_transforms)
seg_pair = SegmentationPair(input_filenames, gt_filenames, metadata=metadata, slice_axis=self.slice_axis,
cache=self.cache, prepro_transforms=self.prepro_transforms,
soft_gt=self.soft_gt)
input_data_shape, _ = seg_pair.get_pair_shapes()
for idx_pair_slice in range(input_data_shape[-1]):
slice_seg_pair = seg_pair.get_pair_slice(idx_pair_slice, gt_type=self.task)
self.has_bounding_box = imed_obj_detect.verify_metadata(slice_seg_pair, self.has_bounding_box)
if self.has_bounding_box:
self.prepro_transforms = imed_obj_detect.adjust_transforms(self.prepro_transforms, slice_seg_pair)
if self.slice_filter_fn and not self.slice_filter_fn(slice_seg_pair):
continue
# Note: we force here gt_type=segmentation since ROI slice is needed to Crop the image
slice_roi_pair = roi_pair.get_pair_slice(idx_pair_slice, gt_type="segmentation")
if self.slice_filter_roi and imed_loader_utils.filter_roi(slice_roi_pair['gt'], self.roi_thr):
continue
item = imed_transforms.apply_preprocessing_transforms(self.prepro_transforms,
slice_seg_pair,
slice_roi_pair)
self.indexes.append(item)
def _slice_seg_pair(self, idx_pair_slice, seg_pair, roi_pair,
useful_slices, input_volumes, gt_volume, roi_volume):
""" Helper function to slice segmentation pair at load time """
slice_seg_pair = seg_pair.get_pair_slice(idx_pair_slice)
self.has_bounding_box = imed_obj_detect.verify_metadata(
slice_seg_pair, self.has_bounding_box)
if self.has_bounding_box:
imed_obj_detect.adjust_transforms(self.prepro_transforms,
slice_seg_pair)
# keeping idx of slices with gt
if self.slice_filter_fn:
filter_fn_ret_seg = self.slice_filter_fn(slice_seg_pair)
if self.slice_filter_fn and filter_fn_ret_seg:
useful_slices.append(idx_pair_slice)
roi_pair_slice = roi_pair.get_pair_slice(idx_pair_slice)
slice_seg_pair, roi_pair_slice = imed_transforms.apply_preprocessing_transforms(
self.prepro_transforms, slice_seg_pair, roi_pair_slice)
input_volumes.append(slice_seg_pair["input"][0])
# Handle unlabeled data
if not len(slice_seg_pair["gt"]):
gt_volume = []
else:
gt_volume.append(
(slice_seg_pair["gt"][0] * 255).astype(np.uint8) / 255.)
# Handle data with no ROI provided
if not len(roi_pair_slice["gt"]):
roi_volume = []
else:
roi_volume.append(
(roi_pair_slice["gt"][0] * 255).astype(np.uint8) / 255.)
return slice_seg_pair, roi_pair_slice
def load_filenames(self):
"""Load preprocessed pair data (input and gt) in handler."""
for input_filenames, gt_filenames, roi_filename, metadata in self.filename_pairs:
roi_pair = SegmentationPair(
input_filenames,
roi_filename,
metadata=metadata,
slice_axis=self.slice_axis,
cache=self.cache,
prepro_transforms=self.prepro_transforms)
seg_pair = SegmentationPair(
input_filenames,
gt_filenames,
metadata=metadata,
slice_axis=self.slice_axis,
cache=self.cache,
prepro_transforms=self.prepro_transforms,
soft_gt=self.soft_gt)
input_data_shape, _ = seg_pair.get_pair_shapes()
path_temp = Path(create_temp_directory())
for idx_pair_slice in range(input_data_shape[-1]):
slice_seg_pair = seg_pair.get_pair_slice(idx_pair_slice,
gt_type=self.task)
self.has_bounding_box = imed_obj_detect.verify_metadata(
slice_seg_pair, self.has_bounding_box)
if self.has_bounding_box:
self.prepro_transforms = imed_obj_detect.adjust_transforms(
self.prepro_transforms, slice_seg_pair)
if self.slice_filter_fn and not self.slice_filter_fn(
slice_seg_pair):
continue
# Note: we force here gt_type=segmentation since ROI slice is needed to Crop the image
slice_roi_pair = roi_pair.get_pair_slice(
idx_pair_slice, gt_type="segmentation")
if self.slice_filter_roi and imed_loader_utils.filter_roi(
slice_roi_pair['gt'], self.roi_thr):
continue
item: Tuple[
dict,
dict] = imed_transforms.apply_preprocessing_transforms(
self.prepro_transforms, slice_seg_pair, slice_roi_pair)
# Run once code to keep track if disk cache is used
if self.disk_cache is None:
self.determine_cache_need(item, input_data_shape[-1])
# If is_2d_patch, create handlers list for indexing patch
if self.is_2d_patch:
for metadata in item[0][MetadataKW.INPUT_METADATA]:
metadata[
MetadataKW.INDEX_SHAPE] = item[0]['input'][0].shape
if self.disk_cache:
path_item = path_temp / f"item_{get_timestamp()}.pkl"
with path_item.open(mode="wb") as f:
pickle.dump(item, f)
self.handlers.append((path_item))
else:
self.handlers.append((item))
# else, append the whole slice to self.indexes
else:
if self.disk_cache:
path_item = path_temp / f"item_{get_timestamp()}.pkl"
with path_item.open(mode="wb") as f:
pickle.dump(item, f)
self.indexes.append(path_item)
else:
self.indexes.append(item)
# If is_2d_patch, prepare indices of patches
if self.is_2d_patch:
self.prepare_indices()
def _load_filenames(self):
"""Load preprocessed pair data (input and gt) in handler."""
for subject_id, input_filename, gt_filename, roi_filename, metadata in self.filename_pairs:
# Creating/ getting the subject group
if str(subject_id) in self.hdf5_file.keys():
grp = self.hdf5_file[str(subject_id)]
else:
grp = self.hdf5_file.create_group(str(subject_id))
roi_pair = imed_loader.SegmentationPair(input_filename,
roi_filename,
metadata=metadata,
slice_axis=self.slice_axis,
cache=False,
soft_gt=self.soft_gt)
seg_pair = imed_loader.SegmentationPair(input_filename,
gt_filename,
metadata=metadata,
slice_axis=self.slice_axis,
cache=False,
soft_gt=self.soft_gt)
print("gt filename", gt_filename)
input_data_shape, _ = seg_pair.get_pair_shapes()
useful_slices = []
input_volumes = []
gt_volume = []
roi_volume = []
for idx_pair_slice in range(input_data_shape[-1]):
slice_seg_pair = seg_pair.get_pair_slice(idx_pair_slice)
self.has_bounding_box = imed_obj_detect.verify_metadata(
slice_seg_pair, self.has_bounding_box)
if self.has_bounding_box:
imed_obj_detect.adjust_transforms(self.prepro_transforms,
slice_seg_pair)
# keeping idx of slices with gt
if self.slice_filter_fn:
filter_fn_ret_seg = self.slice_filter_fn(slice_seg_pair)
if self.slice_filter_fn and filter_fn_ret_seg:
useful_slices.append(idx_pair_slice)
roi_pair_slice = roi_pair.get_pair_slice(idx_pair_slice)
slice_seg_pair, roi_pair_slice = imed_transforms.apply_preprocessing_transforms(
self.prepro_transforms, slice_seg_pair, roi_pair_slice)
input_volumes.append(slice_seg_pair["input"][0])
# Handle unlabeled data
if not len(slice_seg_pair["gt"]):
gt_volume = []
else:
gt_volume.append(
(slice_seg_pair["gt"][0] * 255).astype(np.uint8) /
255.)
# Handle data with no ROI provided
if not len(roi_pair_slice["gt"]):
roi_volume = []
else:
roi_volume.append(
(roi_pair_slice["gt"][0] * 255).astype(np.uint8) /
255.)
# Getting metadata using the one from the last slice
input_metadata = slice_seg_pair['input_metadata'][0]
gt_metadata = slice_seg_pair['gt_metadata'][0]
roi_metadata = roi_pair_slice['input_metadata'][0]
if grp.attrs.__contains__('slices'):
grp.attrs['slices'] = list(
set(np.concatenate((grp.attrs['slices'], useful_slices))))
else:
grp.attrs['slices'] = useful_slices
# Creating datasets and metadata
contrast = input_metadata['contrast']
# Inputs
print(len(input_volumes))
print("grp= ", str(subject_id))
key = "inputs/{}".format(contrast)
print("key = ", key)
if len(input_volumes) < 1:
print("list empty")
continue
grp.create_dataset(key, data=input_volumes)
# Sub-group metadata
if grp['inputs'].attrs.__contains__('contrast'):
attr = grp['inputs'].attrs['contrast']
new_attr = [c for c in attr]
new_attr.append(contrast)
grp['inputs'].attrs.create('contrast', new_attr, dtype=self.dt)
else:
grp['inputs'].attrs.create('contrast', [contrast],
dtype=self.dt)
# dataset metadata
grp[key].attrs['input_filenames'] = input_metadata[
'input_filenames']
grp[key].attrs['data_type'] = input_metadata['data_type']
if "zooms" in input_metadata.keys():
grp[key].attrs["zooms"] = input_metadata['zooms']
if "data_shape" in input_metadata.keys():
grp[key].attrs["data_shape"] = input_metadata['data_shape']
if "bounding_box" in input_metadata.keys():
grp[key].attrs["bounding_box"] = input_metadata['bounding_box']
# GT
key = "gt/{}".format(contrast)
grp.create_dataset(key, data=gt_volume)
# Sub-group metadata
if grp['gt'].attrs.__contains__('contrast'):
attr = grp['gt'].attrs['contrast']
new_attr = [c for c in attr]
new_attr.append(contrast)
grp['gt'].attrs.create('contrast', new_attr, dtype=self.dt)
else:
grp['gt'].attrs.create('contrast', [contrast], dtype=self.dt)
# dataset metadata
grp[key].attrs['gt_filenames'] = input_metadata['gt_filenames']
grp[key].attrs['data_type'] = gt_metadata['data_type']
if "zooms" in gt_metadata.keys():
grp[key].attrs["zooms"] = gt_metadata['zooms']
if "data_shape" in gt_metadata.keys():
grp[key].attrs["data_shape"] = gt_metadata['data_shape']
if gt_metadata['bounding_box'] is not None:
grp[key].attrs["bounding_box"] = gt_metadata['bounding_box']
# ROI
key = "roi/{}".format(contrast)
grp.create_dataset(key, data=roi_volume)
# Sub-group metadata
if grp['roi'].attrs.__contains__('contrast'):
attr = grp['roi'].attrs['contrast']
new_attr = [c for c in attr]
new_attr.append(contrast)
grp['roi'].attrs.create('contrast', new_attr, dtype=self.dt)
else:
grp['roi'].attrs.create('contrast', [contrast], dtype=self.dt)
# dataset metadata
grp[key].attrs['roi_filename'] = roi_metadata['gt_filenames']
grp[key].attrs['data_type'] = roi_metadata['data_type']
if "zooms" in roi_metadata.keys():
grp[key].attrs["zooms"] = roi_metadata['zooms']
if "data_shape" in roi_metadata.keys():
grp[key].attrs["data_shape"] = roi_metadata['data_shape']
# Adding contrast to group metadata
if grp.attrs.__contains__('contrast'):
attr = grp.attrs['contrast']
new_attr = [c for c in attr]
new_attr.append(contrast)
grp.attrs.create('contrast', new_attr, dtype=self.dt)
else:
grp.attrs.create('contrast', [contrast], dtype=self.dt)