def create_contour_plots(conf, num_images=3, rand=False):
model_path = conf["model_path"]
path = str(Path(model_path).parent / "evaluation")
path += "/predictions.h5"
out_path = Path(model_path).parent / "evaluation"
if not conf["fourier"]:
click.echo("\n This is not a fourier dataset.\n")
pred, img_test, img_true = read_pred(path)
# inverse fourier transformation for prediction
ifft_pred = get_ifft(pred, amp_phase=conf["amp_phase"])
# inverse fourier transform for truth
ifft_truth = get_ifft(img_true, amp_phase=conf["amp_phase"])
for i, (pred, truth) in enumerate(zip(ifft_pred, ifft_truth)):
plot_contour(
pred,
truth,
out_path,
i,
plot_format=conf["format"],
)
def test_contour(self):
from radionets.evaluation.utils import (
read_config,
get_ifft,
read_pred,
)
from radionets.evaluation.contour import (
area_of_contour, )
import toml
import numpy as np
config = toml.load("./tests/evaluate.toml")
conf = read_config(config)
pred, img_test, img_true = read_pred(
"./tests/build/test_training/evaluation/predictions_model_eval.h5")
ifft_pred = get_ifft(pred, amp_phase=conf["amp_phase"])
ifft_truth = get_ifft(img_true, amp_phase=conf["amp_phase"])
assert ~np.isnan([ifft_pred, ifft_truth]).any()
assert ifft_pred[0].shape == (64, 64)
assert ifft_truth[0].shape == (64, 64)
val = area_of_contour(ifft_pred[0], ifft_truth[0])
assert isinstance(val, np.float64)
assert ~np.isnan(val).any()
assert val > 0
def test_calc_blobs_and_crop_first_comp(self):
import torch
import toml
import numpy as np
from radionets.evaluation.utils import read_config, get_ifft, read_pred
from radionets.evaluation.blob_detection import calc_blobs, crop_first_component
config = toml.load("./tests/evaluate.toml")
conf = read_config(config)
pred, _, img_true = read_pred(
"./tests/build/test_training/evaluation/predictions_model_eval.h5")
ifft_pred = get_ifft(torch.tensor(pred[0]),
conf["amp_phase"]).reshape(64, 64)
ifft_truth = get_ifft(torch.tensor(img_true[0]),
conf["amp_phase"]).reshape(64, 64)
blobs_pred, blobs_truth = calc_blobs(ifft_pred, ifft_truth)
assert ~np.isnan(blobs_pred).any()
assert ~np.isnan(blobs_truth).any()
assert blobs_pred.all() >= 0
assert blobs_truth.all() >= 0
assert len(blobs_truth[0]) == 3
flux_pred, flux_truth = crop_first_component(ifft_pred, ifft_truth,
blobs_truth[0])
assert ~np.isnan(flux_pred).any()
assert ~np.isnan(flux_truth).any()
assert flux_pred.all() > 0
assert flux_truth.all() > 0
def evaluate_ms_ssim(conf):
# create DataLoader
loader = create_databunch(conf["data_path"], conf["fourier"],
conf["source_list"], conf["batch_size"])
model_path = conf["model_path"]
out_path = Path(model_path).parent / "evaluation"
out_path.mkdir(parents=True, exist_ok=True)
img_size = loader.dataset[0][0][0].shape[-1]
model = load_pretrained_model(conf["arch_name"], conf["model_path"],
img_size)
if conf["model_path_2"] != "none":
model_2 = load_pretrained_model(conf["arch_name_2"],
conf["model_path_2"], img_size)
vals = []
if img_size < 160:
click.echo("\nThis is only a placeholder!\
Images too small for meaningful ms ssim calculations.\n")
# iterate trough DataLoader
for i, (img_test, img_true) in enumerate(tqdm(loader)):
pred = eval_model(img_test, model)
if conf["model_path_2"] != "none":
pred_2 = eval_model(img_test, model_2)
pred = torch.cat((pred, pred_2), dim=1)
ifft_truth = get_ifft(img_true, amp_phase=conf["amp_phase"])
ifft_pred = get_ifft(pred, amp_phase=conf["amp_phase"])
if img_size < 160:
ifft_truth = pad_unsqueeze(torch.tensor(ifft_truth))
ifft_pred = pad_unsqueeze(torch.tensor(ifft_pred))
vals.extend([
ms_ssim(pred.unsqueeze(0),
truth.unsqueeze(0),
data_range=truth.max())
for pred, truth in zip(ifft_pred, ifft_truth)
])
click.echo("\nCreating ms-ssim histogram.\n")
vals = torch.tensor(vals)
histogram_ms_ssim(
vals,
out_path,
plot_format=conf["format"],
)
click.echo(f"\nThe mean ms-ssim value is {vals.mean()}.\n")
def evaluate_point(conf):
# create DataLoader
loader = create_databunch(conf["data_path"], conf["fourier"],
conf["source_list"], conf["batch_size"])
model_path = conf["model_path"]
out_path = Path(model_path).parent / "evaluation"
out_path.mkdir(parents=True, exist_ok=True)
img_size = loader.dataset[0][0][0].shape[-1]
model = load_pretrained_model(conf["arch_name"], conf["model_path"],
img_size)
if conf["model_path_2"] != "none":
model_2 = load_pretrained_model(conf["arch_name_2"],
conf["model_path_2"], img_size)
vals = []
lengths = []
for i, (img_test, img_true, source_list) in enumerate(tqdm(loader)):
pred = eval_model(img_test, model)
if conf["model_path_2"] != "none":
pred_2 = eval_model(img_test, model_2)
pred = torch.cat((pred, pred_2), dim=1)
ifft_truth = get_ifft(img_true, amp_phase=conf["amp_phase"])
ifft_pred = get_ifft(pred, amp_phase=conf["amp_phase"])
fluxes_pred, fluxes_truth, length = flux_comparison(
ifft_pred, ifft_truth, source_list)
val = ((fluxes_pred - fluxes_truth) / fluxes_truth) * 100
vals += list(val)
lengths += list(length)
vals = np.concatenate(vals).ravel()
lengths = np.array(lengths, dtype="object")
mask = lengths < 10
click.echo("\nCreating pointsources histogram.\n")
hist_point(vals, mask, out_path, plot_format=conf["format"])
click.echo(f"\nThe mean flux difference is {vals.mean()}.\n")
click.echo("\nCreating linear extent-mean flux diff plot.\n")
plot_length_point(lengths,
vals,
mask,
out_path,
plot_format=conf["format"])
def evaluate_mean_diff(conf):
# create DataLoader
loader = create_databunch(conf["data_path"], conf["fourier"],
conf["source_list"], conf["batch_size"])
model_path = conf["model_path"]
out_path = Path(model_path).parent / "evaluation"
out_path.mkdir(parents=True, exist_ok=True)
img_size = loader.dataset[0][0][0].shape[-1]
model = load_pretrained_model(conf["arch_name"], conf["model_path"],
img_size)
if conf["model_path_2"] != "none":
model_2 = load_pretrained_model(conf["arch_name_2"],
conf["model_path_2"], img_size)
vals = []
# iterate trough DataLoader
for i, (img_test, img_true) in enumerate(tqdm(loader)):
pred = eval_model(img_test, model)
if conf["model_path_2"] != "none":
pred_2 = eval_model(img_test, model_2)
pred = torch.cat((pred, pred_2), dim=1)
ifft_truth = get_ifft(img_true, amp_phase=conf["amp_phase"])
ifft_pred = get_ifft(pred, amp_phase=conf["amp_phase"])
for pred, truth in zip(ifft_pred, ifft_truth):
blobs_pred, blobs_truth = calc_blobs(pred, truth)
flux_pred, flux_truth = crop_first_component(
pred, truth, blobs_truth[0])
vals.extend([
(flux_pred.mean() - flux_truth.mean()) / flux_truth.mean()
])
click.echo("\nCreating mean_diff histogram.\n")
vals = torch.tensor(vals) * 100
histogram_mean_diff(
vals,
out_path,
plot_format=conf["format"],
)
click.echo(f"\nThe mean difference is {vals.mean()}.\n")
def evaluate_dynamic_range(conf):
# create Dataloader
loader = create_databunch(conf["data_path"], conf["fourier"],
conf["source_list"], conf["batch_size"])
model_path = conf["model_path"]
out_path = Path(model_path).parent / "evaluation"
out_path.mkdir(parents=True, exist_ok=True)
img_size = loader.dataset[0][0][0].shape[-1]
model = load_pretrained_model(conf["arch_name"], conf["model_path"],
img_size)
if conf["model_path_2"] != "none":
model_2 = load_pretrained_model(conf["arch_name_2"],
conf["model_path_2"], img_size)
dr_truths = np.array([])
dr_preds = np.array([])
# iterate trough DataLoader
for i, (img_test, img_true) in enumerate(tqdm(loader)):
pred = eval_model(img_test, model)
if conf["model_path_2"] != "none":
pred_2 = eval_model(img_test, model_2)
pred = torch.cat((pred, pred_2), dim=1)
ifft_truth = get_ifft(img_true, amp_phase=conf["amp_phase"])
ifft_pred = get_ifft(pred, amp_phase=conf["amp_phase"])
dr_truth, dr_pred, _, _ = calc_dr(ifft_truth, ifft_pred)
dr_truths = np.append(dr_truths, dr_truth)
dr_preds = np.append(dr_preds, dr_pred)
click.echo(f"\nMean dynamic range for true source distributions:\
{round(dr_truths.mean())}\n")
click.echo(f"\nMean dynamic range for predicted source distributions:\
{round(dr_preds.mean())}\n")
click.echo("\nCreating histogram of dynamic ranges.\n")
histogram_dynamic_ranges(
dr_truths,
dr_preds,
out_path,
plot_format=conf["format"],
)
def evaluate_viewing_angle(conf):
# create DataLoader
loader = create_databunch(conf["data_path"], conf["fourier"],
conf["source_list"], conf["batch_size"])
model_path = conf["model_path"]
out_path = Path(model_path).parent / "evaluation"
out_path.mkdir(parents=True, exist_ok=True)
img_size = loader.dataset[0][0][0].shape[-1]
model = load_pretrained_model(conf["arch_name"], conf["model_path"],
img_size)
if conf["model_path_2"] != "none":
model_2 = load_pretrained_model(conf["arch_name_2"],
conf["model_path_2"], img_size)
alpha_truths = []
alpha_preds = []
# iterate trough DataLoader
for i, (img_test, img_true) in enumerate(tqdm(loader)):
pred = eval_model(img_test, model)
if conf["model_path_2"] != "none":
pred_2 = eval_model(img_test, model_2)
pred = torch.cat((pred, pred_2), dim=1)
ifft_truth = get_ifft(img_true, amp_phase=conf["amp_phase"])
ifft_pred = get_ifft(pred, amp_phase=conf["amp_phase"])
m_truth, n_truth, alpha_truth = calc_jet_angle(
torch.tensor(ifft_truth))
m_pred, n_pred, alpha_pred = calc_jet_angle(torch.tensor(ifft_pred))
alpha_truths.extend(abs(alpha_truth))
alpha_preds.extend(abs(alpha_pred))
alpha_truths = torch.tensor(alpha_truths)
alpha_preds = torch.tensor(alpha_preds)
click.echo("\nCreating histogram of jet angles.\n")
histogram_jet_angles(
alpha_truths,
alpha_preds,
out_path,
plot_format=conf["format"],
)
def evaluate_area(conf):
# create DataLoader
loader = create_databunch(conf["data_path"], conf["fourier"],
conf["source_list"], conf["batch_size"])
model_path = conf["model_path"]
out_path = Path(model_path).parent / "evaluation"
out_path.mkdir(parents=True, exist_ok=True)
img_size = loader.dataset[0][0][0].shape[-1]
model = load_pretrained_model(conf["arch_name"], conf["model_path"],
img_size)
if conf["model_path_2"] != "none":
model_2 = load_pretrained_model(conf["arch_name_2"],
conf["model_path_2"], img_size)
vals = []
# iterate trough DataLoader
for i, (img_test, img_true) in enumerate(tqdm(loader)):
pred = eval_model(img_test, model)
if conf["model_path_2"] != "none":
pred_2 = eval_model(img_test, model_2)
pred = torch.cat((pred, pred_2), dim=1)
ifft_truth = get_ifft(img_true, amp_phase=conf["amp_phase"])
ifft_pred = get_ifft(pred, amp_phase=conf["amp_phase"])
for pred, truth in zip(ifft_pred, ifft_truth):
val = area_of_contour(pred, truth)
vals.extend([val])
click.echo("\nCreating eval_area histogram.\n")
vals = torch.tensor(vals)
histogram_area(
vals,
out_path,
plot_format=conf["format"],
)
click.echo(f"\nThe mean area ratio is {vals.mean()}.\n")
def create_source_plots(conf, num_images=3, rand=False):
"""
function for visualizing the output of a inverse fourier transform. For now, it is
necessary to take the absolute of the result of the inverse fourier transform,
because the output is complex.
i: current index of the loop, just used for saving
real_pred: real part of the prediction computed in visualize with fourier
imag_pred: imaginary part of the prediction computed in visualize with fourier
real_truth: real part of the truth computed in visualize with fourier
imag_truth: imaginary part of the truth computed in visualize with fourier
"""
model_path = conf["model_path"]
path = str(Path(model_path).parent / "evaluation")
path += "/predictions.h5"
out_path = Path(model_path).parent / "evaluation"
pred, img_test, img_true = read_pred(path)
# inverse fourier transformation for prediction
ifft_pred = get_ifft(pred, amp_phase=conf["amp_phase"])
# inverse fourier transform for truth
ifft_truth = get_ifft(img_true, amp_phase=conf["amp_phase"])
for i, (pred, truth) in enumerate(zip(ifft_pred, ifft_truth)):
visualize_source_reconstruction(
pred,
truth,
out_path,
i,
dr=conf["vis_dr"],
blobs=conf["vis_blobs"],
msssim=conf["vis_ms_ssim"],
plot_format=conf["format"],
)
return np.abs(ifft_pred), np.abs(ifft_truth)
def test_pca(self):
import torch
import toml
from radionets.evaluation.jet_angle import im_to_array_value, bmul, pca
from radionets.evaluation.utils import read_pred, get_ifft, read_config
config = toml.load("./tests/evaluate.toml")
conf = read_config(config)
torch.set_printoptions(precision=16)
pred, img_test, img_true = read_pred(
"./tests/build/test_training/evaluation/predictions_model_eval.h5")
ifft_pred = get_ifft(pred, conf["amp_phase"])
assert ifft_pred.shape == (10, 64, 64)
pix_x, pix_y, image = im_to_array_value(torch.tensor(ifft_pred))
cog_x = (torch.sum(pix_x * image, axis=1) /
torch.sum(image, axis=1)).unsqueeze(-1)
cog_y = (torch.sum(pix_y * image, axis=1) /
torch.sum(image, axis=1)).unsqueeze(-1)
assert cog_x.shape == (10, 1)
assert cog_y.shape == (10, 1)
delta_x = pix_x - cog_x
delta_y = pix_y - cog_y
inp = torch.cat([delta_x.unsqueeze(1), delta_y.unsqueeze(1)], dim=1)
cov_w = bmul(
(cog_x - 1 * torch.sum(image * image, axis=1).unsqueeze(-1) /
cog_x).squeeze(1),
(torch.matmul(image.unsqueeze(1) * inp, inp.transpose(1, 2))),
)
eig_vals_torch, eig_vecs_torch = torch.linalg.eigh(cov_w, UPLO='U')
assert eig_vals_torch.shape == (10, 2)
assert eig_vecs_torch.shape == (10, 2, 2)
_, _, psi_torch = pca(torch.tensor(ifft_pred))
assert len(psi_torch) == 10
assert len(psi_torch[psi_torch > 360]) == 0
def test_calc_jet_angle(self):
import torch
import toml
from radionets.evaluation.jet_angle import calc_jet_angle
from radionets.evaluation.utils import read_config, read_pred, get_ifft
config = toml.load("./tests/evaluate.toml")
conf = read_config(config)
pred, _, _ = read_pred(
"./tests/build/test_training/evaluation/predictions_model_eval.h5")
image = get_ifft(pred, conf["amp_phase"])
assert image.shape == (10, 64, 64)
if not isinstance(image, torch.Tensor):
image = torch.tensor(image)
image = image.clone()
img_size = image.shape[-1]
# ignore negative pixels, which can appear in predictions
image[image < 0] = 0
if len(image.shape) == 2:
image = image.unsqueeze(0)
bs = image.shape[0]
# only use brightest pixel
max_val = torch.tensor([(i.max() * 0.4) for i in image])
max_arr = (torch.ones(img_size, img_size, bs) * max_val).permute(
2, 0, 1)
image[image < max_arr] = 0
assert image.shape == (10, 64, 64)
m, n, alpha = calc_jet_angle(image)
assert len(n) == 10
assert len(alpha) == 10
assert len(alpha[alpha > 360]) == 0
