def LoadHippocampusData(root_dir, y_shape, z_shape): ''' This function loads our dataset form disk into memory, reshaping output to common size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: Array of dictionaries with data stored in seg and image fields as Numpy arrays of shape [AXIAL_WIDTH, Y_SHAPE, Z_SHAPE] ''' image_dir = os.path.join(root_dir, 'images') label_dir = os.path.join(root_dir, 'labels') images = [ f for f in listdir(image_dir) if (isfile(join(image_dir, f)) and f[0] != ".") ] out = [] for f in images: # We would benefit from mmap load method here if dataset doesn't fit into memory # Images are loaded here using MedPy's load method. We will ignore header # since we will not use it image, _ = load(os.path.join(image_dir, f)) label, _ = load(os.path.join(label_dir, f)) # TASK: normalize all images (but not labels) so that values are in [0..1] range # <YOUR CODE GOES HERE> image_max = image.max(axis=(1, 2)).reshape(-1, 1, 1) image_min = image.min(axis=(1, 2)).reshape(-1, 1, 1) image = (image - image_min) / (image_max - image_min) # We need to reshape data since CNN tensors that represent minibatches # in our case will be stacks of slices and stacks need to be of the same size. # In the inference pathway we will need to crop the output to that # of the input image. # Note that since we feed individual slices to the CNN, we only need to # extend 2 dimensions out of 3. We choose to extend coronal and sagittal here # TASK: med_reshape function is not complete. Go and fix it! image = med_reshape(image, new_shape=(image.shape[0], y_shape, z_shape)) label = med_reshape(label, new_shape=(label.shape[0], y_shape, z_shape)).astype(int) # TASK: Why do we need to cast label to int? # ANSWER: Because the target value is used to calculate "cross entropy loss" in which the class number (= k) is used as a index to get the k-th element of the probability vector, see the usage of x[class] in the pytorch document https://pytorch.org/docs/stable/nn.html#torch.nn.CrossEntropyLoss) out.append({"image": image, "seg": label, "filename": f}) # Hippocampus dataset only takes about 300 Mb RAM, so we can afford to keep it all in RAM print( f"Processed {len(out)} files, total {sum([x['image'].shape[0] for x in out])} slices" ) return np.array(out)
def LoadHippocampusData(root_dir, y_shape, z_shape): ''' This function loads our dataset form disk into memory, reshaping output to common size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: Array of dictionaries with data stored in seg and image fields as Numpy arrays of shape [AXIAL_WIDTH, Y_SHAPE, Z_SHAPE] ''' image_dir = os.path.join(root_dir, 'images') label_dir = os.path.join(root_dir, 'labels') images = [ f for f in listdir(image_dir) if (isfile(join(image_dir, f)) and f[0] != ".") ] out = [] for f in images: # We would benefit from mmap load method here if dataset doesn't fit into memory # Images are loaded here using MedPy's load method. We will ignore header # since we will not use it image, _ = load(os.path.join(image_dir, f)) label, _ = load(os.path.join(label_dir, f)) # TASK_COMPLETED: normalize all images (but not labels) so that values are in [0..1] range # <YOUR CODE GOES HERE> image = np.array(image) image = image.astype(np.single) / 0xff # We need to reshape data since CNN tensors that represent minibatches # in our case will be stacks of slices and stacks need to be of the same size. # In the inference pathway we will need to crop the output to that # of the input image. # Note that since we feed individual slices to the CNN, we only need to # extend 2 dimensions out of 3. We choose to extend coronal and sagittal here # TASK_COMPLETED: med_reshape function is not complete. Go and fix it! image = med_reshape(image, new_shape=(image.shape[0], y_shape, z_shape)) label = med_reshape(label, new_shape=(label.shape[0], y_shape, z_shape)).astype(int) # TASK_COMPLETED: Why do we need to cast label to int? # ANSWER: We want a categorical label, so we cast to int out.append({"image": image, "seg": label, "filename": f}) # Hippocampus dataset only takes about 300 Mb RAM, so we can afford to keep it all in RAM print( f"Processed {len(out)} files, total {sum([x['image'].shape[0] for x in out])} slices" ) return np.array(out)
def LoadHippocampusData(root_dir, y_shape, z_shape): ''' This function loads our dataset form disk into memory, reshaping output to common size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: Array of dictionaries with data stored in seg and image fields as Numpy arrays of shape [AXIAL_WIDTH, Y_SHAPE, Z_SHAPE] ''' image_dir = os.path.join(root_dir, 'images') label_dir = os.path.join(root_dir, 'labels') images = [ f for f in listdir(image_dir) if (isfile(join(image_dir, f)) and f[0] != ".") ] out = [] for f in images: # We would benefit from mmap load method here if dataset doesn't fit into memory # Images are loaded here using MedPy's load method. We will ignore header # since we will not use it image, _ = load(os.path.join(image_dir, f)) label, _ = load(os.path.join(label_dir, f)) print(f) image = np.asarray(image).astype('float32') image = image / 255.0 # We need to reshape data since CNN tensors that represent minibatches # in our case will be stacks of slices and stacks need to be of the same size. # In the inference pathway we will need to crop the output to that # of the input image. # Note that since we feed individual slices to the CNN, we only need to # extend 2 dimensions out of 3. We choose to extend coronal and sagittal here image = med_reshape(image, new_shape=(image.shape[0], y_shape, z_shape)) label = med_reshape(label, new_shape=(label.shape[0], y_shape, z_shape)).astype(int) # ANSWER: Loss function need to be in integer to ccalculate loss, it gives error if not calculated on numeric. out.append({"image": image, "seg": label, "filename": f}) # Hippocampus dataset only takes about 300 Mb RAM, so we can afford to keep it all in RAM print( f"Processed {len(out)} files, total {sum([x['image'].shape[0] for x in out])} slices" ) return np.array(out)
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ # Working through this: this scenario is the same as running `single_volume_inference`, but we have to # pad it first (with the `med_reshape` function. # First, reshape. Previous implementations look like this: # med_reshape(image, new_shape=(image.shape[0], y_shape, z_shape)) # I thought we could pull this. For now, hard code patch_size = 64 print("New shape ", volume.shape[0], patch_size, patch_size) reshaped_volume = med_reshape(volume, new_shape=(volume.shape[0], patch_size, patch_size)) # Now we run the `singled_volume_inference` function on this reshaped volume # We have precedent for this too: # inference_agent.single_volume_inference(x["image"]) # I think we can run direclty on this reshaped_volume... return self.single_volume_inference(reshaped_volume)
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ volume = med_reshape(volume, new_shape=(volume.shape[0], self.patch_size, self.patch_size)) out = np.zeros(volume.shape) for slc_ix in range(volume.shape[0]): img = volume[slc_ix,:,:] img = torch.from_numpy(img.astype(np.single)/np.max(img)).unsqueeze(0).unsqueeze(0) pred = self.model(img.to(self.device)) pred = np.squeeze(pred.cpu().detach()) out[slc_ix,:,:] = torch.argmax(pred, dim=0) return out
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ x, y, z = volume.shape new_shape = (x, self.patch_size, self.patch_size) new_volume = med_reshape(volume, new_shape) slices = np.zeros(new_volume.shape) for slice_idx in range(new_volume.shape[0]): slc = new_volume[slice_idx, :, :] slice = torch.from_numpy(slc.astype(np.single) / np.max(slc)).unsqueeze(0).unsqueeze(0).to( self.device) prediction = self.model(slice) prediction = np.squeeze(prediction).cpu().detach() slices[slice_idx, :, :] = torch.argmax(prediction, dim=0) return slices
def single_volume_inference(self, volume): """ Runs inference on a single volume of conformant patch size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ self.model.eval() # Assuming volume is a numpy array of shape [X,Y,Z] and we need to slice X axis slices = [] # TASK: Write code that will create mask for each slice across the X (0th) dimension. After # that, put all slices into a 3D Numpy array. You can verify if your method is # correct by running it on one of the volumes in your training set and comparing # with the label in 3D Slicer. img_ = med_reshape(volume, new_shape=(volume.shape[0], self.patch_size, self.patch_size)) for i in range(img_.shape[0]): image = torch.from_numpy(img_[i] / np.max(img_[i])).unsqueeze(0).unsqueeze(0) output = self.model(image.to(self.device, dtype=torch.float)) output = np.squeeze(output.cpu().detach()) slices.append(torch.argmax(output, dim=0).numpy()) return np.asarray(slices)
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ # normalize the data volume image = (volume.astype(np.single) - np.min(volume)) / (np.max(volume) - np.min(volume)) # reshape the image volume to the same patch size used for training img_reshaped = med_reshape(image, new_shape=(self.patch_size, self.patch_size, image.shape[2])) # create a new 3d mask to store predicted results mask3d = np.zeros(img_reshaped.shape) # iterate over the image array and predict the all the slices for slc_idx in range(img_reshaped.shape[2]): # compute for each slice slc = torch.from_numpy(img_reshaped[:, :, slc_idx].astype( np.single)).unsqueeze(0).unsqueeze(0) # make prediction pred = self.model(slc.to(self.device)) pred = np.squeeze(pred.cpu().detach()) # store predicted data mask3d[:, :, slc_idx] = torch.argmax(pred, dim=0) # return the predicted volume return mask3d
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ volume = med_reshape(volume, new_shape = (volume.shape[0], self.patch_size, self.patch_size)) self.model.eval() # Assuming volume is a numpy array of shape [X,Y,Z] and we need to slice X axis slices = [] slices = np.zeros(volume.shape) for slc_ix in range(volume.shape[0]): img = volume[slc_ix,:,:] tsr_test = torch.from_numpy(img.astype(np.single)/255.0).unsqueeze(0).unsqueeze(0).type(torch.FloatTensor) pred = self.model(tsr_test.to(self.device)) pred = np.squeeze(pred.cpu().detach()) slices[slc_ix,:,:] = torch.argmax(pred, dim=0) return slices
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ patch_size = 64 volume = (volume - volume.min()) / (volume.max() - volume.min()) volume = med_reshape(volume, new_shape=(volume.shape[0], patch_size, patch_size)) masks = np.zeros(volume.shape) for slice_idx in range(masks.shape[0]): # normalize the image slice_0 = volume[slice_idx, :, :] #slice0_norm = (slice0-slice0.min())/(slice0.max()-slice0.min()) data = torch.from_numpy(slice_0).unsqueeze(0).unsqueeze( 0).float().to(self.device) pred = self.model(data) pred = np.squeeze(pred.cpu().detach()) pred = pred.argmax(axis=0) masks[slice_idx, :, :] = pred return masks
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ self.model.eval() # Assuming volume is a numpy array of shape [X,Y,Z] and we need to slice X axis slices = [] # TASK: Write code that will create mask for each slice across the X (0th) dimension. After # that, put all slices into a 3D Numpy array. You can verify if your method is # correct by running it on one of the volumes in your training set and comparing # with the label in 3D Slicer. # <YOUR CODE HERE> patch_size = 64 volume = med_reshape(volume, new_shape=(volume.shape[0], patch_size, patch_size)) slices = np.zeros(volume.shape) for index in range(volume.shape[0]): slc = volume[index, :, :] slice = torch.from_numpy(slc.astype(np.single) / np.max(slc)).unsqueeze(0).unsqueeze(0) prediction = np.squeeze( self.model(slice.to(self.device)).cpu().detach()) slices[index, :, :] = torch.argmax(prediction, dim=0) return slices
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ new_volume = np.moveaxis(volume, -1, 0) print("new volume", new_volume.shape) reshaped_volume = med_reshape(new_volume, new_shape=(new_volume.shape[0], self.patch_size, self.patch_size)) pred = self.single_volume_inference(reshaped_volume.astype(np.float32)) reshaped_pred = np.zeros_like(new_volume) print("hey", reshaped_volume.shape, pred.shape, reshaped_pred.shape) resize_dim = [reshaped_pred.shape[2], reshaped_pred.shape[1]] print("lala", resize_dim) for i in range(reshaped_pred.shape[0]): reshaped_pred[i] = Image.fromarray(pred[i].astype( np.uint8)).resize(resize_dim) return np.moveaxis(reshaped_pred, 0, -1)
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ patch_size = 64 #shape [BATCH_SIZE, 1, PATCH_SIZE, PATCH_SIZE] volume = med_reshape(volume, new_shape=(volume.shape[0], patch_size, patch_size)) def inference(img): tsr_test = torch.from_numpy(img.astype(np.single)/np.max(img)).unsqueeze(0).unsqueeze(0) pred = self.model(tsr_test.to(self.device)) return np.squeeze(pred.cpu().detach()) mask3d = np.zeros(volume.shape) for slc_ix in range(volume.shape[0]): pred = inference(volume[slc_ix,:,:]) mask3d[slc_ix,:,:] = torch.argmax(pred, dim=0) return mask3d
def LoadHippocampusData(root_dir, y_shape, z_shape): ''' This function loads our dataset form disk into memory, reshaping output to common size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: Array of dictionaries with data stored in seg and image fields as Numpy arrays of shape [AXIAL_WIDTH, Y_SHAPE, Z_SHAPE] ''' image_dir = os.path.join(root_dir, 'images') label_dir = os.path.join(root_dir, 'labels') images = [f for f in listdir(image_dir) if ( isfile(join(image_dir, f)) and f[0] != ".")] out = [] for f in images: image, _ = load(os.path.join(image_dir, f)) label, _ = load(os.path.join(label_dir, f)) normalize all images (but not labels) so that values are in [0..1] range image = image.astype(np.single)/np.max(image) # We need to reshape data since CNN tensors that represent minibatches # in our case will be stacks of slices and stacks need to be of the same size. # In the inference pathway we will need to crop the output to that # of the input image. # Note that since we feed individual slices to the CNN, we only need to # extend 2 dimensions out of 3. We choose to extend coronal and sagittal here image = med_reshape(image, new_shape=(image.shape[0], y_shape, z_shape)) label = med_reshape(label, new_shape=(label.shape[0], y_shape, z_shape)).astype(int) out.append({"image": image, "seg": label, "filename": f}) print(f"Processed {len(out)} files, total {sum([x['image'].shape[0] for x in out])} slices") return np.array(out)
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ volume = med_reshape(volume, (volume.shape[0], patch_size, patch_size)) return self.single_volume_inference(volume)
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ PATCH_SIZE = 64 reshaped_vol = med_reshape(volume, (volume.shape[0], PATCH_SIZE, PATCH_SIZE)) pred_vol = self.single_volume_inference(reshaped_vol) return pred_vol[:volume.shape[0], :volume.shape[1], :volume.shape[2]]
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ # TASK: Write code that will create mask for each slice across the X (0th) dimension. After # that, put all slices into a 3D Numpy array. You can verify if your method is # correct by running it on one of the volumes in your training set and comparing # with the label in 3D Slicer. # <YOUR CODE HERE> volume = med_reshape( volume, (volume.shape[0], self.patch_size, self.patch_size)) return self.single_volume_inference(volume)
def single_volume_inference(self, volume): """ Runs inference on a single volume of conformant patch size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ self.model.eval() # Assuming volume is a numpy array of shape [X,Y,Z] and we need to slice X axis slices = [] # Write code that will create mask for each slice across the X (0th) dimension. After # that, put all slices into a 3D Numpy array. You can verify if your method is # correct by running it on one of the volumes in your training set and comparing # with the label in 3D Slicer. # normalize image = (volume.astype(np.single) - np.min(volume)) / (np.max(volume) - np.min(volume)) #image = volume.astype(np.single)/np.max(volume) new_image = med_reshape(image, new_shape=(self.patch_size, self.patch_size, image.shape[2])) mask3d = np.zeros(new_image.shape) for slc_ix in range(new_image.shape[2]): tsr_test = torch.from_numpy(new_image[:, :, slc_ix].astype( np.single)).unsqueeze(0).unsqueeze(0) #image = torch.from_numpy(self.data[slc[0]]["image"][:,:,slc[1]]).unsqueeze(0) #tsr_test = torch.from_numpy(slc.astype(np.single)).unsqueeze(0).unsqueeze(0) pred = self.model(tsr_test.to(self.device)) pred = np.squeeze(pred.cpu().detach()) mask3d[:, :, slc_ix] = torch.argmax(pred, dim=0) return mask3d
def single_volume_inference_unpadded(self, volume, size=64): """ Runs inference on a single volume of conformant patch size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ # self.model.eval() # Assuming volume is a numpy array of shape [X,Y,Z] and we need to slice X axis # TASK: Write code that will create mask for each slice across the X (0th) dimension. After # that, put all slices into a 3D Numpy array. You can verify if your method is # correct by running it on one of the volumes in your training set and comparing # with the label in 3D Slicer. volume = med_reshape(volume, (volume.shape[0], size, size)) return self.single_volume_inference(volume)
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ # Set model to eval no gradients self.model.eval() # Initialise slices with zeros slices = np.zeros(volume.shape) # Reshape volume to conform to required model patch size volume = med_reshape(volume, new_shape=(volume.shape[0], self.patch_size, self.patch_size)) # For each x slice in the volume for x_index in range(volume.shape[0]): # Get the x slice x_slice = volume[x_index, :, :].astype(np.single) # Convert to tensor tensor_x_slice = torch.from_numpy(x_slice).unsqueeze(0).unsqueeze( 0).to(self.device) # Pass slice through model to get predictions predictions = self.model(tensor_x_slice) # Resize predictions pred_resized = np.squeeze(predictions.cpu().detach()) # Append predictions slices[x_index, :, :] = torch.argmax(pred_resized, dim=0) # Return volume of predictions return slices
def single_volume_inference_unpadded(self, volume): """ Runs inference on a single volume of arbitrary patch size, padding it to the conformant size first Arguments: volume {Numpy array} -- 3D array representing the volume Returns: 3D NumPy array with prediction mask """ # Pad volume to match patch size new_shape = (volume.shape[0], self.patch_size, self.patch_size) volume = med_reshape(volume, new_shape) # Run inference on a single volume of arbitrary patch size slices = np.zeros(volume.shape) for slice_ix in range(volume.shape[0]): # Normalize the volume to 0..1 range slc = volume[slice_ix, :, :].astype(np.single) / np.max( volume[slice_ix, :, :]) # Convert slice to PyTorch tensor and move data to our device slc_tensor = torch.from_numpy(slc).unsqueeze(0).unsqueeze(0).to( self.device) # Do the forward pass pred = self.model(slc_tensor) # Calculate prediction mask slices[slice_ix, :, :] = torch.argmax(np.squeeze( pred.cpu().detach()), dim=0) return slices
def LoadHippocampusData(root_dir, y_shape, z_shape): ''' This function loads our dataset form disk into memory, reshaping output to common size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: Array of dictionaries with data stored in seg and image fields as Numpy arrays of shape [AXIAL_WIDTH, Y_SHAPE, Z_SHAPE] ''' image_dir = os.path.join(root_dir, 'images') label_dir = os.path.join(root_dir, 'labels') images = [ f for f in listdir(image_dir) if (isfile(join(image_dir, f)) and f[0] != ".") ] out = [] for f in images: # We would benefit from mmap load method here if dataset doesn't fit into memory # Images are loaded here using MedPy's load method. We will ignore header # since we will not use it image, _ = load(os.path.join(image_dir, f)) label, _ = load(os.path.join(label_dir, f)) # DONE: normalize all images (but not labels) so that values are in [0..1] range image = image / image.max() # We need to reshape data since CNN tensors that represent minibatches # in our case will be stacks of slices and stacks need to be of the same size. # In the inference pathway we will need to crop the output to that # of the input image. # Note that since we feed individual slices to the CNN, we only need to # extend 2 dimensions out of 3. We choose to extend coronal and sagittal here # DONE: med_reshape function is not complete. Go and fix it! image = med_reshape(image, new_shape=(image.shape[0], y_shape, z_shape)) label = med_reshape(label, new_shape=(label.shape[0], y_shape, z_shape)).astype(int) if image is None or label is None: print(f'Skipping {f} due to error in parsing') continue # DONE: Why do we need to cast label to int? # ANSWER: # ------------------------------------------------------------------------------------- # If casting is not done, it will take the float type by default. # Compare to int, float requires more resources such as memory, processing power, etc. # As the label values are 1 and 0, it makes sense to cast to int and save resources # ------------------------------------------------------------------------------------- out.append({"image": image, "seg": label, "filename": f}) # Hippocampus dataset only takes about 300 Mb RAM, so we can afford to keep it all in RAM print( f"Processed {len(out)} files, total {sum([x['image'].shape[0] for x in out])} slices" ) return np.array(out)
def LoadHippocampusData(root_dir, y_shape, z_shape): ''' This function loads our dataset form disk into memory, reshaping output to common size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: Array of dictionaries with data stored in seg and image fields as Numpy arrays of shape [AXIAL_WIDTH, Y_SHAPE, Z_SHAPE] ''' image_dir = os.path.join(root_dir, 'images/') label_dir = os.path.join(root_dir, 'labels/') images = [ f for f in listdir(image_dir) if (isfile(join(image_dir, f)) and f[0] != ".") ] images = sorted(images) for image in images: if not isfile(join(label_dir, image)): images.remove(image) print(f'File removed: {image} does not exist in {label_dir}') out = [] for f in images: # We would benefit from mmap load method here if dataset doesn't fit into memory # Images are loaded here using MedPy's load method. We will ignore header # since we will not use it image, _ = load(os.path.join(image_dir, f)) label, _ = load(os.path.join(label_dir, f)) # normalize all images (but not labels) so that values are in [0..1] range if image.shape[0] < 100: image = image / np.max(image) # We need to reshape data since CNN tensors that represent minibatches # in our case will be stacks of slices and stacks need to be of the same size. # In the inference pathway we will need to crop the output to that # of the input image. # Note that since we feed individual slices to the CNN, we only need to # extend 2 dimensions out of 3. We choose to extend coronal and sagittal here (l, w, h) = image.shape image = med_reshape(image, new_shape=(image.shape[0], y_shape, z_shape)) label = med_reshape(label, new_shape=(label.shape[0], y_shape, z_shape)).astype(int) # Why do we need to cast label to int?: To get distinct class of labels out.append({"image": image, "seg": label, "filename": f}) # Hippocampus dataset only takes about 300 Mb RAM, so we can afford to keep it all in RAM print( f"Processed {len(out)} files, total {sum([x['image'].shape[0] for x in out])} slices" ) return np.array(out)
def LoadHippocampusData(root_dir, y_shape, z_shape): ''' This function loads our dataset form disk into memory, reshaping output to common size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: Array of dictionaries with data stored in seg and image fields as Numpy arrays of shape [AXIAL_WIDTH, Y_SHAPE, Z_SHAPE] ''' image_dir = os.path.join(root_dir, 'images') label_dir = os.path.join(root_dir, 'labels') images = [ f for f in listdir(image_dir) if (isfile(join(image_dir, f)) and f[0] != ".") ] out = [] for f in images: # We would benefit from mmap load method here if dataset doesn't fit into memory # Images are loaded here using MedPy's load method. We will ignore header # since we will not use it image, _ = load(os.path.join(image_dir, f)) label, _ = load(os.path.join(label_dir, f)) # TASK: normalize all images (but not labels) so that values are in [0..1] range # <YOUR CODE GOES HERE> # image = np.clip(image, 0., 1.) image = image.astype(np.single) / np.max(image) # print("check image max and min...") # print(np.max(image), np.min(image)) # We need to reshape data since CNN tensors that represent minibatches # in our case will be stacks of slices and stacks need to be of the same size. # In the inference pathway we will need to crop the output to that # of the input image. # Note that since we feed individual slices to the CNN, we only need to # extend 2 dimensions out of 3. We choose to extend coronal and sagittal here ################################################# My Comments ############################################### # why don't you just say we need to keep each image with the same size of coronal and sagittal dimensions # # because the raw data has different sizes of coronal and sagittal dimensions # ################################################# My Guess ################################################## # My guess is the images and labels should be converted into (num_of_2D_images, 64, 64) # # ############################################################################################################# # TASK: med_reshape function is not complete. Go and fix it! new_shape = ( image.shape[0], y_shape, z_shape ) # x_shape = axial, y_shape = coronal, z_shape = sagittal image = med_reshape(image, new_shape=new_shape) label = med_reshape(label, new_shape=new_shape).astype(int) # TASK: Why do we need to cast label to int? # ANSWER: Label to be used as mask in the project is for classification purposes. # For example, we have 3 classes in this project "background", "anterior", and "posterior". # Also NumPy is better at handling numeric data, it makes more sense to use integers to represent classes. # "labels": { # "0": "background", # "1": "Anterior", # "2": "Posterior" # }, out.append({"image": image, "seg": label, "filename": f}) # Hippocampus dataset only takes about 300 Mb RAM, so we can afford to keep it all in RAM print( f"Processed {len(out)} files, total {sum([x['image'].shape[0] for x in out])} slices" ) return np.array(out)
def create_report(inference, header, orig_vol, pred_vol): """Generates an image with inference report Arguments: inference {Dictionary} -- dict containing anterior, posterior and full volume values header {PyDicom Dataset} -- DICOM header orig_vol {Numpy array} -- original volume pred_vol {Numpy array} -- predicted label Returns: PIL image """ # The code below uses PIL image library to compose an RGB image that will go into the report # A standard way of storing measurement data in DICOM archives is creating such report and # sending them on as Secondary Capture IODs (http://dicom.nema.org/medical/dicom/current/output/chtml/part03/sect_A.8.html) # Essentially, our report is just a standard RGB image, with some metadata, packed into # DICOM format. pimg = Image.new("RGB", (1000, 1000)) draw = ImageDraw.Draw(pimg) header_font = ImageFont.truetype("assets/Roboto-Regular.ttf", 40) main_font = ImageFont.truetype("assets/Roboto-Regular.ttf", 20) #slice_nums = [orig_vol.shape[2]//3, orig_vol.shape[2]//2, orig_vol.shape[2]*3//4] # is there a better choice? # TASK: Create the report here and show information that you think would be relevant to # clinicians. A sample code is provided below, but feel free to use your creative # genius to make if shine. After all, the is the only part of all our machine learning # efforts that will be visible to the world. The usefulness of your computations will largely # depend on how you present them. # SAMPLE CODE BELOW: UNCOMMENT AND CUSTOMIZE # draw.text((10, 0), "HippoVolume.AI", (255, 255, 255), font=header_font) # draw.multiline_text((10, 90), # f"Patient ID: {header.PatientID}\n" # <WHAT OTHER INFORMATION WOULD BE RELEVANT?> # (255, 255, 255), font=main_font) # STAND-OUT SUGGESTION: # In addition to text data in the snippet above, can you show some images? # Think, what would be relevant to show? Can you show an overlay of mask on top of original data? # Hint: here's one way to convert a numpy array into a PIL image and draw it inside our pimg object: # # Create a PIL image from array: # Numpy array needs to flipped, transposed and normalized to a matrix of values in the range of [0..255] # nd_img = np.flip((slice/np.max(slice))*0xff).T.astype(np.uint8) # This is how you create a PIL image from numpy array # pil_i = Image.fromarray(nd_img, mode="L").convert("RGBA").resize(<dimensions>) # Paste the PIL image into our main report image object (pimg) # pimg.paste(pil_i, box=(10, 280)) def get_image(slice_index): slice = orig_vol[slice_index, :, :] slice = ((slice/np.max(slice))*0xff).astype(np.uint8) #slice = np.flip((slice/np.max(slice))*0xff).T.astype(np.uint8) return Image.fromarray(slice, mode='L').convert("RGBA") def get_mask(slice_index, mask_number): slice = pred_vol[slice_index, :, :] slice = v_compute_volume_1(slice) if mask_number == 1 else v_compute_volume_2(slice) #slice = np.flip(slice * 0xff).T.astype(np.uint8) slice = (slice * 0xff).astype(np.uint8) return Image.fromarray(slice, mode='L') def get_overlay(mask_size, mask1, mask2): overlay = Image.new('RGBA', mask_size, (255, 255, 255, 0)) drawing = ImageDraw.Draw(overlay) drawing.bitmap((0, 0), mask1, fill=(255, 0, 0, 128)) drawing.bitmap((0, 0), mask2, fill=(0, 255, 0, 128)) return overlay def get_slice_thumbnail(slice_index, new_size): image = get_image(slice_index) mask1 = get_mask(slice_index, 1) mask2 = get_mask(slice_index, 2) overlay = get_overlay(mask1.size, mask1, mask2) print('image:', image.size, image) print('overlay:', overlay.size, overlay) thumbnail = Image.alpha_composite(image, overlay) thumbnail = thumbnail.resize(new_size) return thumbnail n_slices = orig_vol.shape[0] #orig_vol = np.flip(orig_vol, 0) #orig_vol = np.flip(orig_vol, 1) orig_vol = med_reshape(orig_vol, (n_slices, 64, 64)) side = 200 new_size = (side, side) range1 = range(0, 1000 - 1, side) n_thumbnails = 0 for y in range1: for x in range1: n_thumbnails += 1 black = (0, 0, 0) white = (255, 255, 255) slice_step = 1. * n_slices / n_thumbnails slice_index = 0 for y in range1: for x in range1: slice_index += slice_step int_slice_index = int(np.round(slice_index)) if int_slice_index >= n_slices: continue print(f'Thumbnail of slice_index={int_slice_index} of {n_slices - 1} ({slice_index:.2f})') pimg.paste(get_slice_thumbnail(int_slice_index, new_size), box=(x, y)) slice_string = f'{int_slice_index}' offset = 25 draw.text((x + side - offset + 1, y + side - offset + 1), slice_string, black, font=main_font) draw.text((x + side - offset, y + side - offset), slice_string, white, font=main_font) title = "HippoVolume.AI" draw.text((11, 1), title, black, font=header_font) draw.text((10, 0), title, white, font=header_font) global date_and_time volume_factor = header.SliceThickness * header.PixelSpacing[0] * header.PixelSpacing[1] print(f'volume_factor={volume_factor}') anterior_volume = inference['anterior'] * volume_factor posterior_volume = inference['posterior'] * volume_factor total_volume = inference['total'] * volume_factor long_text = (f'Patient ID: {header.PatientID}\n' f'Time: {date_and_time}\n' f"Anterior volume: {anterior_volume}, posterior volume: {posterior_volume}, total volume: {total_volume}\n") draw.multiline_text((11, 91), long_text, black, font = main_font) draw.multiline_text((10, 90), long_text, white, font = main_font) shape = [(0, 0), (1000 - 1, 1000 - 1)] draw.rectangle(shape, outline ="red") return pimg
def create_report(inference, header, orig_vol, pred_vol): """Generates an image with inference report Arguments: inference {Dictionary} -- dict containing anterior, posterior and full volume values header {PyDicom Dataset} -- DICOM header orig_vol {Numpy array} -- original volume pred_vol {Numpy array} -- predicted label Returns: PIL image """ # The code below uses PIL image library to compose an RGB image that will go into the report # A standard way of storing measurement data in DICOM archives is creating such report and # sending them on as Secondary Capture IODs (http://dicom.nema.org/medical/dicom/current/output/chtml/part03/sect_A.8.html) # Essentially, our report is just a standard RGB image, with some metadata, packed into # DICOM format. pimg = Image.new("RGB", (1000, 1000)) draw = ImageDraw.Draw(pimg) header_font = ImageFont.truetype("assets/RobotoMono-Regular.ttf", size=40) main_font = ImageFont.truetype("assets/RobotoMono-Regular.ttf", size=20) def get_image(slice_index): slice = orig_vol[slice_index, :, :] slice = ((slice / np.max(slice)) * 0xff).astype(np.uint8) return Image.fromarray(slice, mode='L').convert("RGBA") def get_mask(slice_index, mask_number): slice = pred_vol[slice_index, :, :] slice = v_compute_volume_1( slice) if mask_number == 1 else v_compute_volume_2(slice) slice = (slice * 0xff).astype(np.uint8) return Image.fromarray(slice, mode='L') def get_overlay(mask_size, mask1, mask2): overlay = Image.new('RGBA', mask_size, (255, 255, 255, 0)) drawing = ImageDraw.Draw(overlay) drawing.bitmap((0, 0), mask1, fill=(255, 0, 0, 128)) drawing.bitmap((0, 0), mask2, fill=(0, 255, 0, 128)) return overlay def get_slice_thumbnail(slice_index, new_size): image = get_image(slice_index) mask1 = get_mask(slice_index, 1) mask2 = get_mask(slice_index, 2) overlay = get_overlay(mask1.size, mask1, mask2) thumbnail = Image.alpha_composite(image, overlay) thumbnail = thumbnail.resize(new_size) return thumbnail n_slices = orig_vol.shape[0] orig_vol = med_reshape(orig_vol, (n_slices, 64, 64)) side = 200 new_size = (side, side) range1 = range(0, 1000 - 1, side) n_thumbnails = 0 for y in range1: for x in range1: n_thumbnails += 1 black = (0, 0, 0) white = (255, 255, 255) slice_step = 1. * n_slices / n_thumbnails slice_index = 0 for y in range1: for x in range1: slice_index += slice_step int_slice_index = int(np.round(slice_index)) if int_slice_index >= n_slices: continue pimg.paste(get_slice_thumbnail(int_slice_index, new_size), box=(x, y)) slice_string = f'{int_slice_index}' offset = 25 draw.text((x + side - offset - 50 + 1, y + side - offset + 1), slice_string, black, font=main_font) draw.text((x + side - offset - 50, y + side - offset), slice_string, white, font=main_font) title = "HippoVolume.AI" draw.text((11, 1), title, black, font=header_font) draw.text((10, 0), title, white, font=header_font) global date_and_time volume_factor = header.SliceThickness * header.PixelSpacing[ 0] * header.PixelSpacing[1] anterior_volume = inference['anterior'] * volume_factor posterior_volume = inference['posterior'] * volume_factor total_volume = inference['total'] * volume_factor long_text = ( f'Patient ID: {header.PatientID}\n' f'Time: {date_and_time}\n' f"Anterior volume: {anterior_volume}, posterior volume: {posterior_volume}, total volume: {total_volume}\n" ) draw.multiline_text((11, 91), long_text, black, font=main_font) draw.multiline_text((10, 90), long_text, white, font=main_font) shape = [(0, 0), (1000 - 1, 1000 - 1)] draw.rectangle(shape, outline="red") return pimg
def LoadHippocampusData(root_dir, y_shape, z_shape): ''' This function loads our dataset form disk into memory, reshaping output to common size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: Array of dictionaries with data stored in seg and image fields as Numpy arrays of shape [AXIAL_WIDTH, Y_SHAPE, Z_SHAPE] ''' image_dir = os.path.join(root_dir, 'images') label_dir = os.path.join(root_dir, 'labels') images = [ f for f in listdir(image_dir) if (isfile(join(image_dir, f)) and f[0] != ".") ] out = [] for f in images: # We would benefit from mmap load method here if dataset doesn't fit into memory # Images are loaded here using MedPy's load method. We will ignore header # since we will not use it image, _ = load(os.path.join(image_dir, f)) label, _ = load(os.path.join(label_dir, f)) # TASK: normalize all images (but not labels) so that values are in [0..1] range image = image.astype('float') / 255.0 # Handle a noisy dataset containing (512, 241) image pixels but has segmentation label of (36, 50, 31) updated_image, updated_label = [], [] for im, lbl in zip(image, label): if im.shape[0] == 512 and im.shape[1] == 512: print(f'Skipping image having shape: {im.shape}') continue updated_image.append(im) updated_label.append(lbl) updated_image = np.array(updated_image) updated_label = np.array(updated_label) # We need to reshape data since CNN tensors that represent minibatches # in our case will be stacks of slices and stacks need to be of the same size. # In the inference pathway we will need to crop the output to that # of the input image. # Note that since we feed individual slices to the CNN, we only need to # extend 2 dimensions out of 3. We choose to extend coronal and sagittal here # TASK: med_reshape function is not complete. Go and fix it! image = med_reshape(updated_image, new_shape=(updated_image.shape[0], y_shape, z_shape)) label = med_reshape(updated_label, new_shape=(updated_label.shape[0], y_shape, z_shape)).astype(int) # TASK: Why do we need to cast label to int? # ANSWER: For consistency among all the dataset. If few labels are in float or str like '1', casting solves the issue. out.append({"image": image, "seg": label, "filename": f}) # Hippocampus dataset only takes about 300 Mb RAM, so we can afford to keep it all in RAM print( f"Processed {len(out)} files, total {sum([x['image'].shape[0] for x in out])} slices" ) return np.array(out)
def LoadHippocampusData(root_dir, y_shape, z_shape): ''' This function loads our dataset form disk into memory, reshaping output to common size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: Array of dictionaries with data stored in seg and image fields as Numpy arrays of shape [AXIAL_WIDTH, Y_SHAPE, Z_SHAPE] ''' image_dir = os.path.join(root_dir, 'images') label_dir = os.path.join(root_dir, 'labels') images = [ f for f in listdir(image_dir) if (isfile(join(image_dir, f)) and f[0] != ".") ] out = [] for f in images: # We would benefit from mmap load method here if dataset doesn't fit into memory # Images are loaded here using MedPy's load method. We will ignore header # since we will not use it image, _ = load(os.path.join(image_dir, f)) label, _ = load(os.path.join(label_dir, f)) # TASK: normalize all images (but not labels) so that values are in [0..1] range # <YOUR CODE GOES HERE> # Let's apply a min-max scaler. # By the letter, I think that a min-max scaler is "statistically" correct. I'm curious however if we # could have gotten away with something simple like a division by 255... xmax, xmin = image.max(), image.min() image = (image - xmin) / (xmax - xmin) # We need to reshape data since CNN tensors that represent minibatches # in our case will be stacks of slices and stacks need to be of the same size. # In the inference pathway we will need to crop the output to that # of the input image. # Note that since we feed individual slices to the CNN, we only need to # extend 2 dimensions out of 3. We choose to extend coronal and sagittal here # TASK: med_reshape function is not complete. Go and fix it! image = med_reshape(image, new_shape=(image.shape[0], y_shape, z_shape)) label = med_reshape(label, new_shape=(label.shape[0], y_shape, z_shape)).astype(int) # TASK: Why do we need to cast label to int? # ANSWER: If I recall correctly, we're using these values as classes (n=2). Recast to make categorical. out.append({"image": image, "seg": label, "filename": f}) # Hippocampus dataset only takes about 300 Mb RAM, so we can afford to keep it all in RAM print( f"Processed {len(out)} files, total {sum([x['image'].shape[0] for x in out])} slices" ) return np.array(out)
def create_report(inference, header, orig_vol, pred_lbl): """Generates an image with inference report Arguments: inference {Dictionary} -- dict containing anterior, posterior and full volume values header {PyDicom Dataset} -- DICOM header orig_vol {Numpy array} -- original volume pred_lbl {Numpy array} -- predicted label Returns: PIL image """ # The code below uses PIL image library to compose an RGB image that will go into the report # A standard way of storing measurement data in DICOM archives is creating such report and # sending them on as Secondary Capture IODs (http://dicom.nema.org/medical/dicom/current/output/chtml/part03/sect_A.8.html) # Essentially, our report is just a standard RGB image, with some metadata, packed into # DICOM format. pimg = Image.new("RGB", (1000, 1000)) draw = ImageDraw.Draw(pimg) header_font = ImageFont.truetype("assets/Roboto-Regular.ttf", size=40) main_font = ImageFont.truetype("assets/Roboto-Regular.ttf", size=20) slice_nums = [orig_vol.shape[2]//3, orig_vol.shape[2]//2, orig_vol.shape[2]*3//4] # is there a better choice? # Create the report here and show information that you think would be relevant to # clinicians. A sample code is provided below, but feel free to use your creative # genius to make if shine. After all, this is the only part of all our machine learning # efforts that will be visible to the world. The usefulness of your computations will largely # depend on how you present them. draw.text((10, 0), "HippoVolume.AI", (255, 255, 255), font=header_font) draw.multiline_text((10, 90), f"Patient ID: {header.PatientID}, Modality: {header.Modality}, SeriesDescription: {header.SeriesDescription}\n", (255, 255, 255), font=main_font) x = 10 y = 140 draw.multiline_text((x, y), f"Anterior({slice_nums[0]}) Posterior({slice_nums[0]}) Overlay({slice_nums[0]})", (255, 255, 255), font=main_font) draw.multiline_text((x, y+200), f"Anterior({slice_nums[1]}) Posterior({slice_nums[1]}) Overlay({slice_nums[1]})", (255, 255, 255), font=main_font) draw.multiline_text((x, y+2*200), f"Anterior({slice_nums[2]}) Posterior({slice_nums[2]}) Overlay({slice_nums[2]})", (255, 255, 255), font=main_font) # STAND-OUT SUGGESTION: # In addition to text data in the snippet above, can you show some images? # Think, what would be relevant to show? Can you show an overlay of mask on top of original data? # Hint: here's one way to convert a numpy array into a PIL image and draw it inside our pimg object: # # Create a PIL image from array: # Numpy array needs to flipped, transposed and normalized to a matrix of values in the range of [0..255] # reshape image to allow overlay new_vol = med_reshape(orig_vol, new_shape=(pred_lbl.shape[0], pred_lbl.shape[1], orig_vol.shape[2])) slice1 = slice2 = slice3 = np.zeros((new_vol.shape[0], new_vol.shape[1])) for i in slice_nums: slice1 = inference["anterior"][i] pil_i1 = conv_arr2pil(slice1) pimg.paste(pil_i1, box=(x, y+20)) slice2 = inference["posterior"][i] pil_i2 = conv_arr2pil(slice2) pimg.paste(pil_i2, box=(x+200, y+20)) slice3 = new_vol[:,:,i] + slice1 + slice2 pil_i3 = conv_arr2pil(slice3) pimg.paste(pil_i3, box=(x+400, y+20)) y += 200 slice1 *= 0 slice2 *= 0 slice3 *= 0 return pimg
def LoadHippocampusData(root_dir, y_shape, z_shape): ''' This function loads our dataset form disk into memory, reshaping output to common size Arguments: volume {Numpy array} -- 3D array representing the volume Returns: Array of dictionaries with data stored in seg and image fields as Numpy arrays of shape [AXIAL_WIDTH, Y_SHAPE, Z_SHAPE] ''' image_dir = os.path.join(root_dir, 'images') label_dir = os.path.join(root_dir, 'labels') images = [ f for f in listdir(image_dir) if (isfile(join(image_dir, f)) and f[0] != ".") ] out = [] for f in images: # We would benefit from mmap load method here if dataset doesn't fit into memory # Images are loaded here using MedPy's load method. We will ignore header # since we will not use it image, _ = load(os.path.join(image_dir, f)) label, _ = load(os.path.join(label_dir, f)) # TASK: normalize all images (but not labels) so that values are in [0..1] range # <YOUR CODE GOES HERE> min = np.min(image) max = np.max(image) if (min < 0 or max > 1): #print(f"Rescale image to a zero based value: min: {np.min(image)}, max: {np.max(image)}") wc = (abs(np.min(image)) + abs(np.max(image))) / 2 ww = np.max(image) - np.min(image) #print(f"wc: {wc}, ww: {ww}") hu_min = wc - ww / 2 hu_max = wc + ww / 2 image[np.where(image < hu_min)] = hu_min image[np.where(image > hu_max)] = hu_max image = (image - hu_min) / (hu_max - hu_min) #print(f"New Scaled range min: {hu_min}, max: {hu_max}") #print(f"Scale to O:1 range") image = (image - np.min(image)) / (np.max(image) - np.min(image)) #print(f"Final Scaled range min: {np.min(image)}, max: {np.max(image)}") # We need to reshape data since CNN tensors that represent minibatches # in our case will be stacks of slices and stacks need to be of the same size. # In the inference pathway we will need to crop the output to that # of the input image. # Note that since we feed individual slices to the CNN, we only need to # extend 2 dimensions out of 3. We choose to extend coronal and sagittal here # TASK: med_reshape function is not complete. Go and fix it! image = med_reshape(image, new_shape=(image.shape[0], y_shape, z_shape)) label = med_reshape(label, new_shape=(label.shape[0], y_shape, z_shape)).astype(int) # TASK: Why do we need to cast label to int? # ANSWER: We want each pixel in the label to be explicitly classified as posterior, anterior, or not hippocampus. # Using an integer forces an exact segment that the label pixel must belong to. out.append({"image": image, "seg": label, "filename": f}) # Hippocampus dataset only takes about 300 Mb RAM, so we can afford to keep it all in RAM print( f"Processed {len(out)} files, total {sum([x['image'].shape[0] for x in out])} slices" ) return np.array(out)