def process_augment_inputs(model_input, random_number_generator, parameters):
"""
Augment, normalize and convert inputs to np.ndarray
"""
cropped_image, cropped_heatmaps = loading.augment_and_normalize_image(
image=model_input.image,
auxiliary_image=model_input.heatmaps,
view=model_input.metadata["full_view"],
best_center=model_input.metadata["best_center"],
random_number_generator=random_number_generator,
augmentation=parameters["augmentation"],
max_crop_noise=parameters["max_crop_noise"],
max_crop_size_noise=parameters["max_crop_size_noise"],
)
if parameters["use_heatmaps"]:
return np.concatenate([
cropped_image[:, :, np.newaxis],
cropped_heatmaps,
],
axis=2)
else:
return cropped_image[:, :, np.newaxis]
def run_model(model, device, exam_list, parameters):
"""
Returns predictions of image only model or image+heatmaps model.
Prediction for each exam is averaged for a given number of epochs.
"""
random_number_generator = np.random.RandomState(parameters["seed"])
image_extension = ".hdf5" if parameters["use_hdf5"] else ".png"
with torch.no_grad():
predictions_ls = []
for datum in tqdm.tqdm(exam_list):
predictions_for_datum = []
loaded_image_dict = {view: [] for view in VIEWS.LIST}
loaded_heatmaps_dict = {view: [] for view in VIEWS.LIST}
for view in VIEWS.LIST:
for short_file_path in datum[view]:
loaded_image = loading.load_image(
image_path=os.path.join(
parameters["image_path"],
short_file_path + image_extension),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
if parameters["use_heatmaps"]:
loaded_heatmaps = loading.load_heatmaps(
benign_heatmap_path=os.path.join(
parameters["heatmaps_path"], "heatmap_benign",
short_file_path + ".hdf5"),
malignant_heatmap_path=os.path.join(
parameters["heatmaps_path"],
"heatmap_malignant",
short_file_path + ".hdf5"),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
else:
loaded_heatmaps = None
loaded_image_dict[view].append(loaded_image)
loaded_heatmaps_dict[view].append(loaded_heatmaps)
for data_batch in tools.partition_batch(
range(parameters["num_epochs"]), parameters["batch_size"]):
batch_dict = {view: [] for view in VIEWS.LIST}
for _ in data_batch:
for view in VIEWS.LIST:
image_index = 0
if parameters["augmentation"]:
image_index = random_number_generator.randint(
low=0, high=len(datum[view]))
cropped_image, cropped_heatmaps = loading.augment_and_normalize_image(
image=loaded_image_dict[view][image_index],
auxiliary_image=loaded_heatmaps_dict[view]
[image_index],
view=view,
best_center=datum["best_center"][view]
[image_index],
random_number_generator=random_number_generator,
augmentation=parameters["augmentation"],
max_crop_noise=parameters["max_crop_noise"],
max_crop_size_noise=parameters[
"max_crop_size_noise"],
)
if loaded_heatmaps_dict[view][image_index] is None:
batch_dict[view].append(cropped_image[:, :,
np.newaxis])
else:
batch_dict[view].append(
np.concatenate([
cropped_image[:, :, np.newaxis],
cropped_heatmaps,
],
axis=2))
tensor_batch = {
view: torch.tensor(np.stack(batch_dict[view])).permute(
0, 3, 1, 2).to(device)
for view in VIEWS.LIST
}
output = model(tensor_batch)
batch_predictions = compute_batch_predictions(
output, mode=parameters["model_mode"])
pred_df = pd.DataFrame(
{k: v[:, 1]
for k, v in batch_predictions.items()})
pred_df.columns.names = ["label", "view_angle"]
predictions = pred_df.T.reset_index().groupby(
"label").mean().T[LABELS.LIST].values
predictions_for_datum.append(predictions)
predictions_ls.append(
np.mean(np.concatenate(predictions_for_datum, axis=0), axis=0))
return np.array(predictions_ls)
def run_sub_model(model, exam_list, parameters):
"""
Returns predictions of image only model or image+heatmaps model.
Prediction for each exam is averaged for a given number of epochs.
"""
if (parameters["device_type"] == "gpu") and torch.has_cudnn:
device = torch.device(f"cuda:{parameters["gpu_number"]}")
else:
device = torch.device("cpu")
sub_model = sub_model.to(device)
# F: sets model in evaluation mode. It has an effect in certain modules: e.g. Dropout or BatchNorm Layers
sub_model.eval()
random_number_generator = np.random.RandomState(parameters["seed"])
image_extension = ".hdf5" if parameters["use_hdf5"] else ".png"
with torch.no_grad():
predictions_ls = []
for datum in tqdm.tqdm(exam_list):
predictions_for_datum = []
loaded_image_dict = {view: [] for view in VIEWS.LIST}
loaded_heatmaps_dict = {view: [] for view in VIEWS.LIST}
for view in VIEWS.LIST:
# F: for one exam, all images of a specific view
for short_file_path in datum[view]:
loaded_image = loading.load_image(
image_path=os.path.join(parameters["image_path"], short_file_path + image_extension),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
loaded_image_dict[view].append(loaded_image)
print(f"length loaded_image: {len(loaded_image_dict)}")
for data_batch in tools.partition_batch(range(parameters["num_epochs"]), parameters["batch_size"]):
print(f"num_epochs: {parameters['num_epochs']}")
print(f"batch_size: {parameters['batch_size']}")
tmp = tools.partition_batch(range(parameters["num_epochs"]), parameters["batch_size"])
print(f"partition_batch: {tmp}")
batch_dict = {view: [] for view in VIEWS.LIST}
for _ in data_batch:
for view in VIEWS.LIST:
image_index = 0
# F: they use different augmentation for each view
if parameters["augmentation"]:
image_index = random_number_generator.randint(low=0, high=len(datum[view]))
cropped_image, cropped_heatmaps = loading.augment_and_normalize_image(
image=loaded_image_dict[view][image_index],
auxiliary_image=loaded_heatmaps_dict[view][image_index],
view=view,
best_center=datum["best_center"][view][image_index],
random_number_generator=random_number_generator,
augmentation=parameters["augmentation"],
max_crop_noise=parameters["max_crop_noise"],
max_crop_size_noise=parameters["max_crop_size_noise"],
)
# print(f"cropped_image: {image_index} of m in minibatch: {_} size: {cropped_image.shape}")
else:
# F: e.g. batch_dict[view][:,:,1] is the first heatmap
batch_dict[view].append(np.concatenate([
cropped_image[:, :, np.newaxis],
cropped_heatmaps,
], axis=2))
# print(f"batch_dict_view: {len(batch_dict[view])}")
# print(f"batch_img_size: {batch_dict[view][_].shape}")
tensor_batch = {
# F: result of np.stack has one more dimension:
# F: 4 dimensions: batch_data_i, y_pixels, x_pixels, channels
view: torch.tensor(np.stack(batch_dict[view])).permute(0, 3, 1, 2).to(device)
for view in VIEWS.LIST
}
# print(f"layer_names: {model.state_dict().keys()}")
# Print model's state_dict
output = model(tensor_batch)
batch_predictions = compute_batch_predictions(output)
print(f"batch_predictions: \n {batch_predictions}")
print(len(batch_predictions.keys()))
# F: they pick value 1, disregarding value 0 which is the complement of that (prob = 1)
pred_df = pd.DataFrame({k: v[:, 1] for k, v in batch_predictions.items()})
pred_df.columns.names = ["label", "view_angle"]
# print(f"pred_df.head: {pred_df.head()}")
# F: complicated way of grouping by label and calculating the mean
predictions = pred_df.T.reset_index().groupby("label").mean().T[LABELS.LIST].values
predictions_for_datum.append(predictions)
print(f"predictions: {predictions}")
exit()
predictions_ls.append(np.mean(np.concatenate(predictions_for_datum, axis=0), axis=0))
def __getitem__(self, idx):
batch_x_dict = self.x[idx * self.batch_size:(idx + 1) *
self.batch_size]
batch_y = []
batch_x = []
batch_x_feed = {}
batch_x_feed_new = {}
for view in VIEWS.LIST:
batch_x_feed_new[view] = []
test_x_var = []
test_y_var = []
image_extension = ".hdf5" if self.parameters["use_hdf5"] else ".png"
image_index = 0
for datum in (
batch_x_dict
): # THIS AUGMENTATION PART IS PARTIALLY TAKEN FROM run_model.py
patient = datum['L-CC'][0].split('_')[1] # get name of patient
patient_screens = []
patient_screens_dict = {}
patient_dict = {
view: []
for view in VIEWS.LIST
} # create structure that will be filled with images
for view in VIEWS.LIST:
patient_screens_dict[view] = []
short_file_path = datum[view][
image_index] # name of image file associated to that view
test_x_var.append(short_file_path)
loaded_image = loading.load_image(
image_path=os.path.join(self.parameters["image_path"],
short_file_path + image_extension),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
if self.parameters["use_heatmaps"]:
loaded_heatmaps = loading.load_heatmaps(
benign_heatmap_path=os.path.join(
self.parameters["heatmaps_path"], "heatmap_benign",
short_file_path + ".hdf5"),
malignant_heatmap_path=os.path.join(
self.parameters["heatmaps_path"],
"heatmap_malignant", short_file_path + ".hdf5"),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
else:
loaded_heatmaps = None
if self.parameters["augmentation"]:
image_index = self.random_number_generator.randint(
low=0, high=len(datum[view]))
cropped_image, cropped_heatmaps = loading.augment_and_normalize_image( # producing cropped image
image=loaded_image,
auxiliary_image=loaded_heatmaps,
view=view,
best_center=datum["best_center"][view][image_index],
random_number_generator=self.random_number_generator,
augmentation=self.parameters["augmentation"],
max_crop_noise=self.parameters["max_crop_noise"],
max_crop_size_noise=self.parameters["max_crop_size_noise"],
)
if loaded_heatmaps is None:
patient_dict[view].append(
cropped_image[:, :, np.newaxis]
) # adding the image to the list by specific view (i.e L-MLO, R-CC...)
else:
patient_dict[view].append(
np.concatenate(
[ # adding the image and heatmap to the list by specific view (i.e L-MLO, R-CC...)
cropped_image[:, :, np.newaxis],
cropped_heatmaps,
],
axis=2))
batch_x_feed_new[view].append(
patient_dict[view][-1]
) # adding image of specific view to dictionary of patient
batch_y.append(self.y[patient]) # output related to the patient
batch_y = np.stack(np.array(batch_y), axis=0)
# IF U WANT (BATCH, 4, 2) DECOMMENT THIS
# for ni, i in enumerate(batch_y):
# if ni==0:
# #batch_y_tf = tf.expand_dims(tf.one_hot(i,2), 0)
# batch_y_tf = tf.expand_dims(tf.cast(tf.math.logical_not(tf.cast(tf.one_hot(i,2), dtype=tf.bool)), dtype='float32'), 0)
# #print(batch_y_tf)
# else:
# new_patient = tf.expand_dims(tf.cast(tf.math.logical_not(tf.cast(tf.one_hot(i,2), dtype=tf.bool)), dtype='float32'), 0)
# batch_y_tf = tf.concat([batch_y_tf, new_patient] , axis =0)
# #print(batch_y_tf)
# for n, view in enumerate(VIEWS.LIST):
# batch_x_feed[view] = np.moveaxis(np.stack(np.array(batch_x)[:, n], axis=0), -1,1) # dictionary: key=scan(i.e. 'L-CC', 'R-CC'...), value=tensor of scans (1 image per patient)
for n, view in enumerate(VIEWS.LIST):
batch_x_feed_new[view] = np.moveaxis(
np.stack(np.array(batch_x_feed_new[view]), axis=0), -1,
1) # to obtain (1, width, height)
# returns AS X a dict with all images for all patient patient organized by keys
# that are the views (i.e. {'L-CC: [all L-CC iamges of patients per batch], 'L-MLO': ...})
# returns as y a tensor of [batch, classes]
return batch_x_feed_new, batch_y #, [None]
def run_model(model, device, exam_list, parameters):
"""
Returns predictions of image only model or image+heatmaps model.
Prediction for each exam is averaged for a given number of epochs.
"""
random_number_generator = np.random.RandomState(parameters["seed"])
image_extension = ".hdf5" if parameters["use_hdf5"] else ".png"
with torch.no_grad():
predictions_ls = []
for datum in tqdm.tqdm(exam_list):
predictions_for_datum = []
# F: VIEWS is an adhoc class
# F: VIEWS.LIST : list of views as string
loaded_image_dict = {view: [] for view in VIEWS.LIST}
loaded_heatmaps_dict = {view: [] for view in VIEWS.LIST}
for view in VIEWS.LIST:
# F: for one exam, all images of a specific view
for short_file_path in datum[view]:
loaded_image = loading.load_image(
image_path=os.path.join(
parameters["image_path"],
short_file_path + image_extension),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
if parameters["use_heatmaps"]:
loaded_heatmaps = loading.load_heatmaps(
benign_heatmap_path=os.path.join(
parameters["heatmaps_path"], "heatmap_benign",
short_file_path + ".hdf5"),
malignant_heatmap_path=os.path.join(
parameters["heatmaps_path"],
"heatmap_malignant",
short_file_path + ".hdf5"),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
else:
loaded_heatmaps = None
loaded_image_dict[view].append(loaded_image)
loaded_heatmaps_dict[view].append(loaded_heatmaps)
# print(f"length loaded_image: {len(loaded_image_dict)}")
for data_batch in tools.partition_batch(
range(parameters["num_epochs"]), parameters["batch_size"]):
# print(f"num_epochs: {parameters['num_epochs']}")
# print(f"batch_size: {parameters['batch_size']}")
tmp = tools.partition_batch(range(parameters["num_epochs"]),
parameters["batch_size"])
# print(f"partition_batch: {tmp}")
batch_dict = {view: [] for view in VIEWS.LIST}
for _ in data_batch:
for view in VIEWS.LIST:
image_index = 0
# F: they use different augmentation for each view
if parameters["augmentation"]:
image_index = random_number_generator.randint(
low=0, high=len(datum[view]))
cropped_image, cropped_heatmaps = loading.augment_and_normalize_image(
image=loaded_image_dict[view][image_index],
auxiliary_image=loaded_heatmaps_dict[view]
[image_index],
view=view,
best_center=datum["best_center"][view]
[image_index],
random_number_generator=random_number_generator,
augmentation=parameters["augmentation"],
max_crop_noise=parameters["max_crop_noise"],
max_crop_size_noise=parameters[
"max_crop_size_noise"],
)
# print(f"cropped_image: {image_index} of m in minibatch: {_} size: {cropped_image.shape}")
if loaded_heatmaps_dict[view][image_index] is None:
batch_dict[view].append(cropped_image[:, :,
np.newaxis])
# F: e.g. batch_dict[view][_].shape = (2974, 1748, 1)
else:
# F: e.g. batch_dict[view][:,:,1] is the first heatmap
batch_dict[view].append(
np.concatenate([
cropped_image[:, :, np.newaxis],
cropped_heatmaps,
],
axis=2))
# print(f"batch_dict_view: {len(batch_dict[view])}")
# print(f"batch_img_size: {batch_dict[view][_].shape}")
tensor_batch = {
# F: result of np.stack has one more dimension:
# F: 4 dimensions: batch_data_i, y_pixels, x_pixels, channels
view: torch.tensor(np.stack(batch_dict[view])).permute(
0, 3, 1, 2).to(device)
for view in VIEWS.LIST
}
# print(f"layer_names: {model.state_dict().keys()}")
# Print model's state_dict
# print("Model's state_dict:")
# for param_tensor in model.state_dict():
# print(param_tensor, "\t", model.state_dict()[param_tensor].size())
output = model(tensor_batch)
batch_predictions = compute_batch_predictions(
output, mode=parameters["model_mode"])
# print(f"batch_predictions: \n {batch_predictions}")
# print(len(batch_predictions.keys()))
# F: they pick value 1, disregarding value 0 which is the complement of that (prob = 1)
pred_df = pd.DataFrame(
{k: v[:, 1]
for k, v in batch_predictions.items()})
pred_df.columns.names = ["label", "view_angle"]
# print(f"pred_df.head: {pred_df.head()}")
# F: complicated way of grouping by label and calculating the mean
predictions = pred_df.T.reset_index().groupby(
"label").mean().T[LABELS.LIST].values
predictions_for_datum.append(predictions)
# print(f"predictions: {predictions}")
predictions_ls.append(
np.mean(np.concatenate(predictions_for_datum, axis=0), axis=0))
return np.array(predictions_ls)
