def load_inputs(image_path,
metadata_path,
use_heatmaps,
benign_heatmap_path=None,
malignant_heatmap_path=None):
"""
Load a single input example, optionally with heatmaps
"""
if use_heatmaps:
assert benign_heatmap_path is not None
assert malignant_heatmap_path is not None
else:
assert benign_heatmap_path is None
assert malignant_heatmap_path is None
metadata = pickling.unpickle_from_file(metadata_path)
image = loading.load_image(
image_path=image_path,
view=metadata["full_view"],
horizontal_flip=metadata["horizontal_flip"],
)
if use_heatmaps:
heatmaps = loading.load_heatmaps(
benign_heatmap_path=benign_heatmap_path,
malignant_heatmap_path=malignant_heatmap_path,
view=metadata["full_view"],
horizontal_flip=metadata["horizontal_flip"],
)
else:
heatmaps = None
return ModelInput(image=image, heatmaps=heatmaps, metadata=metadata)
def run(parameters):
"""
Outputs the predictions as csv file
"""
random_number_generator = np.random.RandomState(parameters["seed"])
model, device = load_model(parameters)
model_input = load_inputs(
image_path=parameters["cropped_mammogram_path"],
metadata_path=parameters["metadata_path"],
use_heatmaps=parameters["use_heatmaps"],
benign_heatmap_path=parameters["heatmap_path_benign"],
malignant_heatmap_path=parameters["heatmap_path_malignant"],
)
assert model_input.metadata["full_view"] == parameters["view"]
all_predictions = []
# set up hook
activation = {'out_resnet': []}
# out_shape = [0,256]
# activation = {'out_resnet': torch.empty(out_shape)}
handle = model.all_views_avg_pool.register_forward_hook(
tools.get_activation(activation, 'out_resnet'))
# handle = model.view_resnet.layer_list[4][1].conv2.register_forward_hook(tools.get_activation(activation, 'out_resnet'))
for datum in tqdm.tqdm(exam_list):
for short_file_path in datum[view]:
loaded_image = loading.load_image(
image_path=os.path.join(parameters["image_path"],
short_file_path + image_extension),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
loaded_image_dict[view].append(loaded_image)
for data_batch in tools.partition_batch(range(parameters["num_epochs"]),
parameters["batch_size"]):
batch = []
for _ in data_batch:
batch.append(
process_augment_inputs(
model_input=model_input,
random_number_generator=random_number_generator,
parameters=parameters,
))
tensor_batch = batch_to_tensor(batch, device)
y_hat = model(tensor_batch)
predictions = np.exp(y_hat.cpu().detach().numpy())[:, :2, 1]
all_predictions.append(predictions)
agg_predictions = np.concatenate(all_predictions, axis=0).mean(0)
predictions_dict = {
"benign": float(agg_predictions[0]),
"malignant": float(agg_predictions[1]),
}
print(json.dumps(predictions_dict))
def run_model(model, parameters, data_path):
"""
Run the model over images in sample_data.
Save the predictions as csv and visualizations as png.
"""
exam_list = os.listdir(data_path)
if (parameters["device_type"] == "gpu") and torch.has_cudnn:
device = torch.device("cuda:{}".format(parameters["gpu_number"]))
else:
device = torch.device("cpu")
model = model.to(device)
model.eval()
with torch.no_grad():
# load image
# the image is already flipped so no need to do it again
for image in tqdm.tqdm(exam_list):
loaded_image = loading.load_image(
image_path=os.path.join(parameters["image_path"], image),
horizontal_flip=False,
)
# convert python 2D array into 4D torch tensor in N,C,H,W format
loaded_image = np.expand_dims(np.expand_dims(loaded_image, 0),
0).copy()
tensor_batch = torch.Tensor(loaded_image).to(device)
# forward propagation
output = model(tensor_batch)
#turn_on_visualization:
saliency_maps = model.saliency_map.data.cpu().numpy()
print('>>>>>>>>>>>>', saliency_maps.shape)
patch_locations = model.patch_locations
patch_imgs = model.patches
patch_attentions = model.patch_attns[0, :].data.cpu().numpy()
# create directories
output_path = "/content/gdrive/My Drive/chestxray/chestnet_results/sample_images_final_0.1_final/"
os.makedirs(output_path, exist_ok=True)
os.makedirs(os.path.join(output_path, "visualization"),
exist_ok=True)
short_file_path = image.split('.')[0]
save_dir = os.path.join(parameters["output_path"], "visualization",
"{0}.png".format(short_file_path))
#print(save_dir)
visualize_example(loaded_image, saliency_maps, patch_locations,
patch_imgs, patch_attentions, save_dir,
parameters)
def ori_image_prepare(image_path, view, horizontal_flip, parameters):
"""
Loads an image and creates stride_lists
"""
patch_size = parameters['patch_size']
more_patches = parameters['more_patches']
stride_fixed = parameters['stride_fixed']
image = loading.load_image(image_path, view, horizontal_flip)
image = image.astype(float)
loading.standard_normalize_single_image(image)
img_width, img_length = image.shape
width_stride_list = stride_list_generator(img_width, patch_size, more_patches, stride_fixed)
length_stride_list = stride_list_generator(img_length, patch_size, more_patches, stride_fixed)
return image, width_stride_list, length_stride_list
def ori_image_prepare(short_file_path, view, horizontal_flip, parameters):
"""
Loads an image and creates stride_lists
"""
orginal_image_path = parameters['orginal_image_path']
patch_size = parameters['patch_size']
more_patches = parameters['more_patches']
stride_fixed = parameters['stride_fixed']
image_extension = '.hdf5' if parameters['use_hdf5'] else '.png'
image_path = os.path.join(orginal_image_path, short_file_path + image_extension)
image = loading.load_image(image_path, view, horizontal_flip)
image = image.astype(float)
loading.standard_normalize_single_image(image)
img_width, img_length = image.shape
width_stride_list = stride_list_generator(img_width, patch_size, more_patches, stride_fixed)
length_stride_list = stride_list_generator(img_length, patch_size, more_patches, stride_fixed)
return image, width_stride_list, length_stride_list
def run_model(model, device, exam_list, parameters):
"""
Returns predictions of image only model or image+heatmaps model.
Prediction for each exam is averaged for a given number of epochs.
"""
random_number_generator = np.random.RandomState(parameters["seed"])
image_extension = ".hdf5" if parameters["use_hdf5"] else ".png"
with torch.no_grad():
predictions_ls = []
for datum in tqdm.tqdm(exam_list):
predictions_for_datum = []
loaded_image_dict = {view: [] for view in VIEWS.LIST}
loaded_heatmaps_dict = {view: [] for view in VIEWS.LIST}
for view in VIEWS.LIST:
for short_file_path in datum[view]:
loaded_image = loading.load_image(
image_path=os.path.join(
parameters["image_path"],
short_file_path + image_extension),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
if parameters["use_heatmaps"]:
loaded_heatmaps = loading.load_heatmaps(
benign_heatmap_path=os.path.join(
parameters["heatmaps_path"], "heatmap_benign",
short_file_path + ".hdf5"),
malignant_heatmap_path=os.path.join(
parameters["heatmaps_path"],
"heatmap_malignant",
short_file_path + ".hdf5"),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
else:
loaded_heatmaps = None
loaded_image_dict[view].append(loaded_image)
loaded_heatmaps_dict[view].append(loaded_heatmaps)
for data_batch in tools.partition_batch(
range(parameters["num_epochs"]), parameters["batch_size"]):
batch_dict = {view: [] for view in VIEWS.LIST}
for _ in data_batch:
for view in VIEWS.LIST:
image_index = 0
if parameters["augmentation"]:
image_index = random_number_generator.randint(
low=0, high=len(datum[view]))
cropped_image, cropped_heatmaps = loading.augment_and_normalize_image(
image=loaded_image_dict[view][image_index],
auxiliary_image=loaded_heatmaps_dict[view]
[image_index],
view=view,
best_center=datum["best_center"][view]
[image_index],
random_number_generator=random_number_generator,
augmentation=parameters["augmentation"],
max_crop_noise=parameters["max_crop_noise"],
max_crop_size_noise=parameters[
"max_crop_size_noise"],
)
if loaded_heatmaps_dict[view][image_index] is None:
batch_dict[view].append(cropped_image[:, :,
np.newaxis])
else:
batch_dict[view].append(
np.concatenate([
cropped_image[:, :, np.newaxis],
cropped_heatmaps,
],
axis=2))
tensor_batch = {
view: torch.tensor(np.stack(batch_dict[view])).permute(
0, 3, 1, 2).to(device)
for view in VIEWS.LIST
}
output = model(tensor_batch)
batch_predictions = compute_batch_predictions(
output, mode=parameters["model_mode"])
pred_df = pd.DataFrame(
{k: v[:, 1]
for k, v in batch_predictions.items()})
pred_df.columns.names = ["label", "view_angle"]
predictions = pred_df.T.reset_index().groupby(
"label").mean().T[LABELS.LIST].values
predictions_for_datum.append(predictions)
predictions_ls.append(
np.mean(np.concatenate(predictions_for_datum, axis=0), axis=0))
return np.array(predictions_ls)
def run_model(model, exam_list, parameters, turn_on_visualization):
"""
Run the model over images in sample_data.
Save the predictions as csv and visualizations as png.
"""
if (parameters["device_type"] == "gpu") and torch.has_cudnn:
device = torch.device("cuda:{}".format(
parameters["gpu_number"]))
else:
device = torch.device("cpu")
model = model.to(device)
model.eval()
# initialize data holders
pred_dict = {
"image_index": [],
"benign_pred": [],
"malignant_pred": [],
"benign_label": 'no input',
"malignant_label": 'no input'
}
with torch.no_grad():
# iterate through each exam
for datum in tqdm.tqdm(exam_list):
for view in VIEWS.LIST:
short_file_path = datum[view][0]
# load image
# the image is already flipped so no need to do it again
loaded_image = loading.load_image(
image_path=os.path.join(
parameters["image_path"],
short_file_path + ".png"),
view=view,
horizontal_flip=False,
)
loading.standard_normalize_single_image(
loaded_image)
# load segmentation if available
benign_seg_path = os.path.join(
parameters["segmentation_path"],
"{0}_{1}".format(short_file_path,
"benign.png"))
malignant_seg_path = os.path.join(
parameters["segmentation_path"],
"{0}_{1}".format(short_file_path,
"malignant.png"))
benign_seg = np.zeros([1920, 2944], dtype=int)
#benign_seg = None
malignant_seg = np.zeros([1920, 2944], dtype=int)
#malignant_seg = None
if os.path.exists(benign_seg_path):
loaded_seg = loading.load_image(
image_path=benign_seg_path,
view=view,
horizontal_flip=False,
)
benign_seg = loaded_seg
if os.path.exists(malignant_seg_path):
loaded_seg = loading.load_image(
image_path=malignant_seg_path,
view=view,
horizontal_flip=False,
)
malignant_seg = loaded_seg
# convert python 2D array into 4D torch tensor in N,C,H,W format
loaded_image = np.expand_dims(
np.expand_dims(loaded_image, 0), 0).copy()
tensor_batch = torch.Tensor(loaded_image).to(
device)
# forward propagation
output = model(tensor_batch)
pred_numpy = output.data.cpu().numpy()
benign_pred, malignant_pred = pred_numpy[
0, 0], pred_numpy[0, 1]
# save visualization
if turn_on_visualization:
saliency_maps = model.saliency_map.data.cpu(
).numpy()
patch_locations = model.patch_locations
patch_imgs = model.patches
patch_attentions = model.patch_attns[
0, :].data.cpu().numpy()
save_dir = os.path.join(
parameters["output_path"], "visualization",
"{0}.png".format(short_file_path))
#b = np.zeros([2560 , 3328], dtype = int)
visualize_example(loaded_image, saliency_maps,
[benign_seg, malignant_seg],
patch_locations, patch_imgs,
patch_attentions, save_dir,
parameters)
# propagate holders
#benign_label, malignant_label = fetch_cancer_label_by_view(view, datum["cancer_label"])
pred_dict["image_index"].append(short_file_path)
pred_dict["benign_pred"].append(benign_pred)
pred_dict["malignant_pred"].append(malignant_pred)
#pred_dict["benign_label"].append(benign_label)
#pred_dict["malignant_label"].append(malignant_label)
return pd.DataFrame(pred_dict)
def __getitem__(self, idx):
batch_x_dict = self.x[idx * self.batch_size:(idx + 1) *
self.batch_size]
batch_y = []
batch_x = []
batch_x_feed = {}
batch_x_feed_new = {}
for view in VIEWS.LIST:
batch_x_feed_new[view] = []
test_x_var = []
test_y_var = []
image_extension = ".hdf5" if self.parameters["use_hdf5"] else ".png"
image_index = 0
for datum in (
batch_x_dict
): # THIS AUGMENTATION PART IS PARTIALLY TAKEN FROM run_model.py
patient = datum['L-CC'][0].split('_')[1] # get name of patient
patient_screens = []
patient_screens_dict = {}
patient_dict = {
view: []
for view in VIEWS.LIST
} # create structure that will be filled with images
for view in VIEWS.LIST:
patient_screens_dict[view] = []
short_file_path = datum[view][
image_index] # name of image file associated to that view
test_x_var.append(short_file_path)
loaded_image = loading.load_image(
image_path=os.path.join(self.parameters["image_path"],
short_file_path + image_extension),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
if self.parameters["use_heatmaps"]:
loaded_heatmaps = loading.load_heatmaps(
benign_heatmap_path=os.path.join(
self.parameters["heatmaps_path"], "heatmap_benign",
short_file_path + ".hdf5"),
malignant_heatmap_path=os.path.join(
self.parameters["heatmaps_path"],
"heatmap_malignant", short_file_path + ".hdf5"),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
else:
loaded_heatmaps = None
if self.parameters["augmentation"]:
image_index = self.random_number_generator.randint(
low=0, high=len(datum[view]))
cropped_image, cropped_heatmaps = loading.augment_and_normalize_image( # producing cropped image
image=loaded_image,
auxiliary_image=loaded_heatmaps,
view=view,
best_center=datum["best_center"][view][image_index],
random_number_generator=self.random_number_generator,
augmentation=self.parameters["augmentation"],
max_crop_noise=self.parameters["max_crop_noise"],
max_crop_size_noise=self.parameters["max_crop_size_noise"],
)
if loaded_heatmaps is None:
patient_dict[view].append(
cropped_image[:, :, np.newaxis]
) # adding the image to the list by specific view (i.e L-MLO, R-CC...)
else:
patient_dict[view].append(
np.concatenate(
[ # adding the image and heatmap to the list by specific view (i.e L-MLO, R-CC...)
cropped_image[:, :, np.newaxis],
cropped_heatmaps,
],
axis=2))
batch_x_feed_new[view].append(
patient_dict[view][-1]
) # adding image of specific view to dictionary of patient
batch_y.append(self.y[patient]) # output related to the patient
batch_y = np.stack(np.array(batch_y), axis=0)
# IF U WANT (BATCH, 4, 2) DECOMMENT THIS
# for ni, i in enumerate(batch_y):
# if ni==0:
# #batch_y_tf = tf.expand_dims(tf.one_hot(i,2), 0)
# batch_y_tf = tf.expand_dims(tf.cast(tf.math.logical_not(tf.cast(tf.one_hot(i,2), dtype=tf.bool)), dtype='float32'), 0)
# #print(batch_y_tf)
# else:
# new_patient = tf.expand_dims(tf.cast(tf.math.logical_not(tf.cast(tf.one_hot(i,2), dtype=tf.bool)), dtype='float32'), 0)
# batch_y_tf = tf.concat([batch_y_tf, new_patient] , axis =0)
# #print(batch_y_tf)
# for n, view in enumerate(VIEWS.LIST):
# batch_x_feed[view] = np.moveaxis(np.stack(np.array(batch_x)[:, n], axis=0), -1,1) # dictionary: key=scan(i.e. 'L-CC', 'R-CC'...), value=tensor of scans (1 image per patient)
for n, view in enumerate(VIEWS.LIST):
batch_x_feed_new[view] = np.moveaxis(
np.stack(np.array(batch_x_feed_new[view]), axis=0), -1,
1) # to obtain (1, width, height)
# returns AS X a dict with all images for all patient patient organized by keys
# that are the views (i.e. {'L-CC: [all L-CC iamges of patients per batch], 'L-MLO': ...})
# returns as y a tensor of [batch, classes]
return batch_x_feed_new, batch_y #, [None]
def run_sub_model(model, exam_list, parameters):
"""
Returns predictions of image only model or image+heatmaps model.
Prediction for each exam is averaged for a given number of epochs.
"""
if (parameters["device_type"] == "gpu") and torch.has_cudnn:
device = torch.device(f"cuda:{parameters["gpu_number"]}")
else:
device = torch.device("cpu")
sub_model = sub_model.to(device)
# F: sets model in evaluation mode. It has an effect in certain modules: e.g. Dropout or BatchNorm Layers
sub_model.eval()
random_number_generator = np.random.RandomState(parameters["seed"])
image_extension = ".hdf5" if parameters["use_hdf5"] else ".png"
with torch.no_grad():
predictions_ls = []
for datum in tqdm.tqdm(exam_list):
predictions_for_datum = []
loaded_image_dict = {view: [] for view in VIEWS.LIST}
loaded_heatmaps_dict = {view: [] for view in VIEWS.LIST}
for view in VIEWS.LIST:
# F: for one exam, all images of a specific view
for short_file_path in datum[view]:
loaded_image = loading.load_image(
image_path=os.path.join(parameters["image_path"], short_file_path + image_extension),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
loaded_image_dict[view].append(loaded_image)
print(f"length loaded_image: {len(loaded_image_dict)}")
for data_batch in tools.partition_batch(range(parameters["num_epochs"]), parameters["batch_size"]):
print(f"num_epochs: {parameters['num_epochs']}")
print(f"batch_size: {parameters['batch_size']}")
tmp = tools.partition_batch(range(parameters["num_epochs"]), parameters["batch_size"])
print(f"partition_batch: {tmp}")
batch_dict = {view: [] for view in VIEWS.LIST}
for _ in data_batch:
for view in VIEWS.LIST:
image_index = 0
# F: they use different augmentation for each view
if parameters["augmentation"]:
image_index = random_number_generator.randint(low=0, high=len(datum[view]))
cropped_image, cropped_heatmaps = loading.augment_and_normalize_image(
image=loaded_image_dict[view][image_index],
auxiliary_image=loaded_heatmaps_dict[view][image_index],
view=view,
best_center=datum["best_center"][view][image_index],
random_number_generator=random_number_generator,
augmentation=parameters["augmentation"],
max_crop_noise=parameters["max_crop_noise"],
max_crop_size_noise=parameters["max_crop_size_noise"],
)
# print(f"cropped_image: {image_index} of m in minibatch: {_} size: {cropped_image.shape}")
else:
# F: e.g. batch_dict[view][:,:,1] is the first heatmap
batch_dict[view].append(np.concatenate([
cropped_image[:, :, np.newaxis],
cropped_heatmaps,
], axis=2))
# print(f"batch_dict_view: {len(batch_dict[view])}")
# print(f"batch_img_size: {batch_dict[view][_].shape}")
tensor_batch = {
# F: result of np.stack has one more dimension:
# F: 4 dimensions: batch_data_i, y_pixels, x_pixels, channels
view: torch.tensor(np.stack(batch_dict[view])).permute(0, 3, 1, 2).to(device)
for view in VIEWS.LIST
}
# print(f"layer_names: {model.state_dict().keys()}")
# Print model's state_dict
output = model(tensor_batch)
batch_predictions = compute_batch_predictions(output)
print(f"batch_predictions: \n {batch_predictions}")
print(len(batch_predictions.keys()))
# F: they pick value 1, disregarding value 0 which is the complement of that (prob = 1)
pred_df = pd.DataFrame({k: v[:, 1] for k, v in batch_predictions.items()})
pred_df.columns.names = ["label", "view_angle"]
# print(f"pred_df.head: {pred_df.head()}")
# F: complicated way of grouping by label and calculating the mean
predictions = pred_df.T.reset_index().groupby("label").mean().T[LABELS.LIST].values
predictions_for_datum.append(predictions)
print(f"predictions: {predictions}")
exit()
predictions_ls.append(np.mean(np.concatenate(predictions_for_datum, axis=0), axis=0))
def run_model(model, device, exam_list, parameters):
"""
Returns predictions of image only model or image+heatmaps model.
Prediction for each exam is averaged for a given number of epochs.
"""
random_number_generator = np.random.RandomState(parameters["seed"])
image_extension = ".hdf5" if parameters["use_hdf5"] else ".png"
with torch.no_grad():
predictions_ls = []
for datum in tqdm.tqdm(exam_list):
predictions_for_datum = []
# F: VIEWS is an adhoc class
# F: VIEWS.LIST : list of views as string
loaded_image_dict = {view: [] for view in VIEWS.LIST}
loaded_heatmaps_dict = {view: [] for view in VIEWS.LIST}
for view in VIEWS.LIST:
# F: for one exam, all images of a specific view
for short_file_path in datum[view]:
loaded_image = loading.load_image(
image_path=os.path.join(
parameters["image_path"],
short_file_path + image_extension),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
if parameters["use_heatmaps"]:
loaded_heatmaps = loading.load_heatmaps(
benign_heatmap_path=os.path.join(
parameters["heatmaps_path"], "heatmap_benign",
short_file_path + ".hdf5"),
malignant_heatmap_path=os.path.join(
parameters["heatmaps_path"],
"heatmap_malignant",
short_file_path + ".hdf5"),
view=view,
horizontal_flip=datum["horizontal_flip"],
)
else:
loaded_heatmaps = None
loaded_image_dict[view].append(loaded_image)
loaded_heatmaps_dict[view].append(loaded_heatmaps)
# print(f"length loaded_image: {len(loaded_image_dict)}")
for data_batch in tools.partition_batch(
range(parameters["num_epochs"]), parameters["batch_size"]):
# print(f"num_epochs: {parameters['num_epochs']}")
# print(f"batch_size: {parameters['batch_size']}")
tmp = tools.partition_batch(range(parameters["num_epochs"]),
parameters["batch_size"])
# print(f"partition_batch: {tmp}")
batch_dict = {view: [] for view in VIEWS.LIST}
for _ in data_batch:
for view in VIEWS.LIST:
image_index = 0
# F: they use different augmentation for each view
if parameters["augmentation"]:
image_index = random_number_generator.randint(
low=0, high=len(datum[view]))
cropped_image, cropped_heatmaps = loading.augment_and_normalize_image(
image=loaded_image_dict[view][image_index],
auxiliary_image=loaded_heatmaps_dict[view]
[image_index],
view=view,
best_center=datum["best_center"][view]
[image_index],
random_number_generator=random_number_generator,
augmentation=parameters["augmentation"],
max_crop_noise=parameters["max_crop_noise"],
max_crop_size_noise=parameters[
"max_crop_size_noise"],
)
# print(f"cropped_image: {image_index} of m in minibatch: {_} size: {cropped_image.shape}")
if loaded_heatmaps_dict[view][image_index] is None:
batch_dict[view].append(cropped_image[:, :,
np.newaxis])
# F: e.g. batch_dict[view][_].shape = (2974, 1748, 1)
else:
# F: e.g. batch_dict[view][:,:,1] is the first heatmap
batch_dict[view].append(
np.concatenate([
cropped_image[:, :, np.newaxis],
cropped_heatmaps,
],
axis=2))
# print(f"batch_dict_view: {len(batch_dict[view])}")
# print(f"batch_img_size: {batch_dict[view][_].shape}")
tensor_batch = {
# F: result of np.stack has one more dimension:
# F: 4 dimensions: batch_data_i, y_pixels, x_pixels, channels
view: torch.tensor(np.stack(batch_dict[view])).permute(
0, 3, 1, 2).to(device)
for view in VIEWS.LIST
}
# print(f"layer_names: {model.state_dict().keys()}")
# Print model's state_dict
# print("Model's state_dict:")
# for param_tensor in model.state_dict():
# print(param_tensor, "\t", model.state_dict()[param_tensor].size())
output = model(tensor_batch)
batch_predictions = compute_batch_predictions(
output, mode=parameters["model_mode"])
# print(f"batch_predictions: \n {batch_predictions}")
# print(len(batch_predictions.keys()))
# F: they pick value 1, disregarding value 0 which is the complement of that (prob = 1)
pred_df = pd.DataFrame(
{k: v[:, 1]
for k, v in batch_predictions.items()})
pred_df.columns.names = ["label", "view_angle"]
# print(f"pred_df.head: {pred_df.head()}")
# F: complicated way of grouping by label and calculating the mean
predictions = pred_df.T.reset_index().groupby(
"label").mean().T[LABELS.LIST].values
predictions_for_datum.append(predictions)
# print(f"predictions: {predictions}")
predictions_ls.append(
np.mean(np.concatenate(predictions_for_datum, axis=0), axis=0))
return np.array(predictions_ls)
