def sliding_window_inference( inputs: torch.Tensor, roi_size: Union[Sequence[int], int], sw_batch_size: int, predictor: Callable[[torch.Tensor], torch.Tensor], overlap: float = 0.25, mode: Union[BlendMode, str] = BlendMode.CONSTANT, padding_mode: Union[PytorchPadMode, str] = PytorchPadMode.CONSTANT, cval: float = 0.0, device: Optional[torch.device] = None, ) -> torch.Tensor: """ Sliding window inference on `inputs` with `predictor`. When roi_size is larger than the inputs' spatial size, the input image are padded during inference. To maintain the same spatial sizes, the output image will be cropped to the original input size. Args: inputs: input image to be processed (assuming NCHW[D]) roi_size: the spatial window size for inferences. When its components have None or non-positives, the corresponding inputs dimension will be used. if the components of the `roi_size` are non-positive values, the transform will use the corresponding components of img size. For example, `roi_size=(32, -1)` will be adapted to `(32, 64)` if the second spatial dimension size of img is `64`. sw_batch_size: the batch size to run window slices. predictor: given input tensor `patch_data` in shape NCHW[D], `predictor(patch_data)` should return a prediction with the same spatial shape and batch_size, i.e. NMHW[D]; where HW[D] represents the patch spatial size, M is the number of output channels, N is `sw_batch_size`. overlap: Amount of overlap between scans. mode: {``"constant"``, ``"gaussian"``} How to blend output of overlapping windows. Defaults to ``"constant"``. - ``"constant``": gives equal weight to all predictions. - ``"gaussian``": gives less weight to predictions on edges of windows. padding_mode: {``"constant"``, ``"reflect"``, ``"replicate"``, ``"circular"``} Padding mode when ``roi_size`` is larger than inputs. Defaults to ``"constant"`` See also: https://pytorch.org/docs/stable/nn.functional.html#pad cval: fill value for 'constant' padding mode. Default: 0 device: device running the concatenation of the windows. By default the device and accordingly the memory of the input device is used. If for example set to device=torch.device('cpu') the gpu memory consumption is less and independent of the input and roi_size parameter. Output is on the device set or if not set the inputs device. Raises: NotImplementedError: When ``inputs`` does not have batch size = 1. Note: - input must be channel-first and have a batch dim, support both spatial 2D and 3D. - currently only supports `inputs` with batch_size=1. """ num_spatial_dims = len(inputs.shape) - 2 assert 0 <= overlap < 1, "overlap must be >= 0 and < 1." # determine image spatial size and batch size # Note: all input images must have the same image size and batch size image_size_ = list(inputs.shape[2:]) batch_size = inputs.shape[0] # TODO: Enable batch sizes > 1 in future if batch_size > 1: raise NotImplementedError("Currently only inputs with batch size = 1 are supported.") if device is None: device = inputs.device roi_size = fall_back_tuple(roi_size, image_size_) # in case that image size is smaller than roi size image_size = tuple(max(image_size_[i], roi_size[i]) for i in range(num_spatial_dims)) pad_size = [] for k in range(len(inputs.shape) - 1, 1, -1): diff = max(roi_size[k - 2] - inputs.shape[k], 0) half = diff // 2 pad_size.extend([half, diff - half]) inputs = F.pad(inputs, pad=pad_size, mode=PytorchPadMode(padding_mode).value, value=cval) scan_interval = _get_scan_interval(image_size, roi_size, num_spatial_dims, overlap) # Store all slices in list slices = dense_patch_slices(image_size, roi_size, scan_interval) slice_batches = [] for slice_index in range(0, len(slices), sw_batch_size): slice_index_range = range(slice_index, min(slice_index + sw_batch_size, len(slices))) input_slices = [] for curr_index in slice_index_range: curr_slice = slices[curr_index] if len(curr_slice) == 3: input_slices.append(inputs[0, :, curr_slice[0], curr_slice[1], curr_slice[2]]) else: input_slices.append(inputs[0, :, curr_slice[0], curr_slice[1]]) slice_batches.append(torch.stack(input_slices)) # Perform predictions output_rois = list() for data in slice_batches: seg_prob = predictor(data) # batched patch segmentation output_rois.append(seg_prob.to(device)) # stitching output image output_classes = output_rois[0].shape[1] output_shape = [batch_size, output_classes] + list(image_size) # Create importance map importance_map = compute_importance_map(get_valid_patch_size(image_size, roi_size), mode=mode, device=device) # allocate memory to store the full output and the count for overlapping parts output_image = torch.zeros(output_shape, dtype=torch.float32, device=device) count_map = torch.zeros(output_shape, dtype=torch.float32, device=device) for window_id, slice_index in enumerate(range(0, len(slices), sw_batch_size)): slice_index_range = range(slice_index, min(slice_index + sw_batch_size, len(slices))) # store the result in the proper location of the full output. Apply weights from importance map. for curr_index in slice_index_range: curr_slice = slices[curr_index] if len(curr_slice) == 3: output_image[0, :, curr_slice[0], curr_slice[1], curr_slice[2]] += ( importance_map * output_rois[window_id][curr_index - slice_index, :] ) count_map[0, :, curr_slice[0], curr_slice[1], curr_slice[2]] += importance_map else: output_image[0, :, curr_slice[0], curr_slice[1]] += ( importance_map * output_rois[window_id][curr_index - slice_index, :] ) count_map[0, :, curr_slice[0], curr_slice[1]] += importance_map # account for any overlapping sections output_image = output_image / count_map if num_spatial_dims == 3: return output_image[ ..., pad_size[4] : image_size_[0] + pad_size[4], pad_size[2] : image_size_[1] + pad_size[2], pad_size[0] : image_size_[2] + pad_size[0], ] return output_image[ ..., pad_size[2] : image_size_[0] + pad_size[2], pad_size[0] : image_size_[1] + pad_size[0] ] # 2D
def sliding_window_inference( inputs: torch.Tensor, roi_size: Union[Sequence[int], int], sw_batch_size: int, predictor: Callable[..., torch.Tensor], overlap: float = 0.25, mode: Union[BlendMode, str] = BlendMode.CONSTANT, sigma_scale: Union[Sequence[float], float] = 0.125, padding_mode: Union[PytorchPadMode, str] = PytorchPadMode.CONSTANT, cval: float = 0.0, sw_device: Union[torch.device, str, None] = None, device: Union[torch.device, str, None] = None, *args: Any, **kwargs: Any, ) -> torch.Tensor: """ Sliding window inference on `inputs` with `predictor`. When roi_size is larger than the inputs' spatial size, the input image are padded during inference. To maintain the same spatial sizes, the output image will be cropped to the original input size. Args: inputs: input image to be processed (assuming NCHW[D]) roi_size: the spatial window size for inferences. When its components have None or non-positives, the corresponding inputs dimension will be used. if the components of the `roi_size` are non-positive values, the transform will use the corresponding components of img size. For example, `roi_size=(32, -1)` will be adapted to `(32, 64)` if the second spatial dimension size of img is `64`. sw_batch_size: the batch size to run window slices. predictor: given input tensor `patch_data` in shape NCHW[D], `predictor(patch_data)` should return a prediction with the same spatial shape and batch_size, i.e. NMHW[D]; where HW[D] represents the patch spatial size, M is the number of output channels, N is `sw_batch_size`. overlap: Amount of overlap between scans. mode: {``"constant"``, ``"gaussian"``} How to blend output of overlapping windows. Defaults to ``"constant"``. - ``"constant``": gives equal weight to all predictions. - ``"gaussian``": gives less weight to predictions on edges of windows. sigma_scale: the standard deviation coefficient of the Gaussian window when `mode` is ``"gaussian"``. Default: 0.125. Actual window sigma is ``sigma_scale`` * ``dim_size``. When sigma_scale is a sequence of floats, the values denote sigma_scale at the corresponding spatial dimensions. padding_mode: {``"constant"``, ``"reflect"``, ``"replicate"``, ``"circular"``} Padding mode for ``inputs``, when ``roi_size`` is larger than inputs. Defaults to ``"constant"`` See also: https://pytorch.org/docs/stable/nn.functional.html#pad cval: fill value for 'constant' padding mode. Default: 0 sw_device: device for the window data. By default the device (and accordingly the memory) of the `inputs` is used. Normally `sw_device` should be consistent with the device where `predictor` is defined. device: device for the stitched output prediction. By default the device (and accordingly the memory) of the `inputs` is used. If for example set to device=torch.device('cpu') the gpu memory consumption is less and independent of the `inputs` and `roi_size`. Output is on the `device`. args: optional args to be passed to ``predictor``. kwargs: optional keyword args to be passed to ``predictor``. Note: - input must be channel-first and have a batch dim, supports N-D sliding window. """ num_spatial_dims = len(inputs.shape) - 2 assert 0 <= overlap < 1, "overlap must be >= 0 and < 1." # determine image spatial size and batch size # Note: all input images must have the same image size and batch size image_size_ = list(inputs.shape[2:]) batch_size = inputs.shape[0] if device is None: device = inputs.device if sw_device is None: sw_device = inputs.device roi_size = fall_back_tuple(roi_size, image_size_) # in case that image size is smaller than roi size image_size = tuple( max(image_size_[i], roi_size[i]) for i in range(num_spatial_dims)) pad_size = [] for k in range(len(inputs.shape) - 1, 1, -1): diff = max(roi_size[k - 2] - inputs.shape[k], 0) half = diff // 2 pad_size.extend([half, diff - half]) inputs = F.pad(inputs, pad=pad_size, mode=PytorchPadMode(padding_mode).value, value=cval) scan_interval = _get_scan_interval(image_size, roi_size, num_spatial_dims, overlap) # Store all slices in list slices = dense_patch_slices(image_size, roi_size, scan_interval) num_win = len(slices) # number of windows per image total_slices = num_win * batch_size # total number of windows # Create window-level importance map importance_map = compute_importance_map(get_valid_patch_size( image_size, roi_size), mode=mode, sigma_scale=sigma_scale, device=device) importance_map = importance_map.to('cpu') # Perform predictions output_image = torch.tensor(0.0) #, device=device) count_map = torch.tensor(0.0) #, device=device) _initialized = False for slice_g in range(0, total_slices, sw_batch_size): slice_range = range(slice_g, min(slice_g + sw_batch_size, total_slices)) unravel_slice = [ [slice(int(idx / num_win), int(idx / num_win) + 1), slice(None)] + list(slices[idx % num_win]) for idx in slice_range ] window_data = torch.cat( [inputs[win_slice] for win_slice in unravel_slice]).to(sw_device) seg_prob = predictor(window_data, *args, **kwargs).to(device) # batched patch segmentation seg_prob = seg_prob.to('cpu') if not _initialized: # init. buffer at the first iteration output_classes = seg_prob.shape[1] output_shape = [batch_size, output_classes] + list(image_size) # allocate memory to store the full output and the count for overlapping parts output_image = torch.zeros(output_shape, dtype=torch.float32) #, device=device) count_map = torch.zeros(output_shape, dtype=torch.float32) #, device=device) _initialized = True # store the result in the proper location of the full output. Apply weights from importance map. for idx, original_idx in zip(slice_range, unravel_slice): output_image[original_idx] += importance_map * seg_prob[idx - slice_g] count_map[original_idx] += importance_map # account for any overlapping sections output_image = output_image / count_map final_slicing: List[slice] = [] for sp in range(num_spatial_dims): slice_dim = slice( pad_size[sp * 2], image_size_[num_spatial_dims - sp - 1] + pad_size[sp * 2]) final_slicing.insert(0, slice_dim) while len(final_slicing) < len(output_image.shape): final_slicing.insert(0, slice(None)) if str(sw_device)[0:3] == 'hpu': return output_image[final_slicing].to(sw_device).contiguous( memory_format=torch.channels_last_3d) return output_image[final_slicing]
def sliding_window_classification( inputs: torch.Tensor, roi_size: Union[Sequence[int], int], sw_batch_size: int, predictor: Callable[[torch.Tensor], torch.Tensor], overlap: float = 0.25, mode: Union[BlendMode, str] = BlendMode.CONSTANT, padding_mode: Union[PytorchPadMode, str] = PytorchPadMode.CONSTANT, cval: float = 0.0, device: Optional[torch.device] = None, ) -> torch.Tensor: num_spatial_dims = len(inputs.shape) - 2 assert 0 <= overlap < 1, "overlap must be >= 0 and < 1." # determine image spatial size and batch size # Note: all input images must have the same image size and batch size image_size_ = list(inputs.shape[2:]) batch_size = inputs.shape[0] # TODO: Enable batch sizes > 1 in future if batch_size > 1: raise NotImplementedError( "Currently only inputs with batch size = 1 are supported.") if device is None: device = inputs.device roi_size = fall_back_tuple(roi_size, image_size_) # in case that image size is smaller than roi size image_size = tuple( max(image_size_[i], roi_size[i]) for i in range(num_spatial_dims)) pad_size = [] for k in range(len(inputs.shape) - 1, 1, -1): diff = max(roi_size[k - 2] - inputs.shape[k], 0) half = diff // 2 pad_size.extend([half, diff - half]) inputs = F.pad(inputs, pad=pad_size, mode=PytorchPadMode(padding_mode).value, value=cval) scan_interval = _get_scan_interval(image_size, roi_size, num_spatial_dims, overlap) # Store all slices in list slices = dense_patch_slices(image_size, roi_size, scan_interval) slice_batches = [] for slice_index in range(0, len(slices), sw_batch_size): slice_index_range = range( slice_index, min(slice_index + sw_batch_size, len(slices))) input_slices = [] for curr_index in slice_index_range: curr_slice = slices[curr_index] if len(curr_slice) == 3: input_slices.append(inputs[0, :, curr_slice[0], curr_slice[1], curr_slice[2]]) else: input_slices.append(inputs[0, :, curr_slice[0], curr_slice[1]]) slice_batches.append(torch.stack(input_slices)) # Perform predictions output_rois = list() for data in slice_batches: cls_prob = predictor(data) # batched patch classification output_rois.append(cls_prob.to(device)) # stitching output image output_classes = output_rois[0].shape[1] output_shape = [batch_size, output_classes] + list(image_size) # Create importance map importance_map = compute_importance_map(get_valid_patch_size( image_size, roi_size), mode=mode, device=device) # allocate memory to store the full output and the count for overlapping parts output_image = torch.zeros(output_shape, dtype=torch.float32, device=device) count_map = torch.zeros(output_shape, dtype=torch.float32, device=device) for window_id, slice_index in enumerate( range(0, len(slices), sw_batch_size)): slice_index_range = range( slice_index, min(slice_index + sw_batch_size, len(slices))) # store the result in the proper location of the full output. Apply weights from importance map. for curr_index in slice_index_range: curr_slice = slices[curr_index] curr_roi = output_rois[window_id][curr_index - slice_index, :] if len(curr_slice) == 3: output_image[0, :, curr_slice[0], curr_slice[1], curr_slice[2]] += (importance_map * curr_roi.view(-1, 1, 1, 1)) count_map[0, :, curr_slice[0], curr_slice[1], curr_slice[2]] += importance_map else: output_image[0, :, curr_slice[0], curr_slice[1]] += (importance_map * curr_roi.view(-1, 1, 1)) count_map[0, :, curr_slice[0], curr_slice[1]] += importance_map # account for any overlapping sections output_image = output_image / count_map if num_spatial_dims == 3: return output_image[..., pad_size[4]:image_size_[0] + pad_size[4], pad_size[2]:image_size_[1] + pad_size[2], pad_size[0]:image_size_[2] + pad_size[0], ] return output_image[..., pad_size[2]:image_size_[0] + pad_size[2], pad_size[0]:image_size_[1] + pad_size[0]] # 2D
def sliding_window_2d_inference_3d( inputs: torch.Tensor, roi_size: Union[Sequence[int], int], sw_batch_size: int, predictor: Callable[..., torch.Tensor], overlap: float = 0.25, mode: Union[BlendMode, str] = BlendMode.CONSTANT, sigma_scale: Union[Sequence[float], float] = 0.125, padding_mode: Union[PytorchPadMode, str] = PytorchPadMode.CONSTANT, cval: float = 0.0, sw_device: Union[torch.device, str, None] = None, device: Union[torch.device, str, None] = None, z_axis: int = 2, *args: Any, **kwargs: Any, ) -> torch.Tensor: num_spatial_dims = len(inputs.shape) - 2 if overlap < 0 or overlap >= 1: raise AssertionError("overlap must be >= 0 and < 1.") # determine image spatial size and batch size # Note: all input images must have the same image size and batch size image_size_ = list(inputs.shape[2:]) batch_size = inputs.shape[0] if device is None: device = inputs.device if sw_device is None: sw_device = inputs.device roi_size = ensure_list(roi_size) if len(roi_size) == 2: roi_size.insert(z_axis, 1) elif len(roi_size) == 3 and 1 not in roi_size: raise ValueError( f'Need one dimension = 1, e.g. (16,16,1), but got {roi_size}') roi_size = fall_back_tuple(roi_size, image_size_) # in case that image size is smaller than roi size image_size = tuple( max(image_size_[i], roi_size[i]) for i in range(num_spatial_dims)) pad_size = [] for k in range(len(inputs.shape) - 1, 1, -1): diff = max(roi_size[k - 2] - inputs.shape[k], 0) half = diff // 2 pad_size.extend([half, diff - half]) inputs = F.pad(inputs, pad=pad_size, mode=PytorchPadMode(padding_mode).value, value=cval) scan_interval = _get_scan_interval(image_size, roi_size, num_spatial_dims, overlap) # Store all slices in list slices = dense_patch_slices(image_size, roi_size, scan_interval) num_win = len(slices) # number of windows per image total_slices = num_win * batch_size # total number of windows # Create window-level importance map importance_map = compute_importance_map(get_valid_patch_size( image_size, roi_size), mode=mode, sigma_scale=sigma_scale, device=device) # Perform predictions output_image, count_map = torch.tensor(0.0, device=device), torch.tensor( 0.0, device=device) _initialized = False for slice_g in range(0, total_slices, sw_batch_size): slice_range = range(slice_g, min(slice_g + sw_batch_size, total_slices)) unravel_slice = [ [slice(int(idx / num_win), int(idx / num_win) + 1), slice(None)] + list(slices[idx % num_win]) for idx in slice_range ] window_data = torch.cat( [inputs[win_slice] for win_slice in unravel_slice]).to(sw_device) window_data = window_data.squeeze(z_axis + 2) seg_prob = predictor(window_data, *args, **kwargs).to(device) # batched patch segmentation seg_prob = seg_prob.unsqueeze(z_axis + 2) if not _initialized: # init. buffer at the first iteration output_classes = seg_prob.shape[1] output_shape = [batch_size, output_classes] + list(image_size) # allocate memory to store the full output and the count for overlapping parts output_image = torch.zeros(output_shape, dtype=torch.float32, device=device) count_map = torch.zeros(output_shape, dtype=torch.float32, device=device) _initialized = True # store the result in the proper location of the full output. Apply weights from importance map. for idx, original_idx in zip(slice_range, unravel_slice): output_image[original_idx] += importance_map * seg_prob[idx - slice_g] count_map[original_idx] += importance_map # account for any overlapping sections output_image = output_image / count_map final_slicing: List[slice] = [] for sp in range(num_spatial_dims): slice_dim = slice( pad_size[sp * 2], image_size_[num_spatial_dims - sp - 1] + pad_size[sp * 2]) final_slicing.insert(0, slice_dim) while len(final_slicing) < len(output_image.shape): final_slicing.insert(0, slice(None)) return output_image[final_slicing]