def eval(self):
self.metric.reset()
print(f"Length of classes: {len(self.classes)}")
temp_weights = torch.eye(len(self.classes), device = "cuda")
torch.nn.init.xavier_uniform_(temp_weights, gain=1.0)
print(temp_weights)
temp_weights.requires_grad= True
# temp_weights.requires_grad= True
temp_bias = torch.zeros(len(self.classes), device = "cuda")
# torch.nn.init.xavier_uniform_(temp_bias, gain=1.0)
temp_bias.requires_grad= True
# temp_weights = torch.rand(len(self.classes), len(self.classes), device="cuda", requires_grad=True)
# temp_bias = torch.rand(len(self.classes), device="cuda", requires_grad=True)
logging.info("Start training of temprature weights, Total sample: {:d}".format(len(self.val_loader)))
cce_criterion = CCELoss(len(self.classes)).to(self.device)
cross_criterion = torch.nn.CrossEntropyLoss(ignore_index=-1)
info_entropy_criterion = InfoEntropyLoss()
optimizer = torch.optim.SGD([temp_weights, temp_bias], lr=self.lr)
import time
time_start = time.time()
num_epochs = 300
for epoch in range(num_epochs):
eceEvaluator_perimage = perimageCCE(n_classes = len(self.classes))
epoch_loss_cce_total = 0
epoch_loss_cross_entropy_total = 0
epoch_loss_total = 0
epoch_loss_info_entropy_total = 0
for i, (images, targets, filenames) in enumerate(self.val_loader):
# import pdb; pdb.set_trace()
optimizer.zero_grad()
images = images.to(self.device)
targets = targets.to(self.device)
# print(image.shape)
with torch.no_grad():
# outputs = model.evaluate(images)
# outputs = torch.rand(1,3,300,400)
outputs = torch.ones(1,2,300,400)*(torch.Tensor([0.51,0.49]).reshape(1,-1,1,1))
# outputs = torch.ones(1,4,300,400)*(torch.Tensor([0.5,0.25,0.15, 0.1]).reshape(1,-1,1,1))
outputs = outputs.cuda()
outputs[0,0,:, :200] = 0.49
outputs[0,1,:, 200:] = 0.51
# outputs = torch.ones(1,3,300,400)*(torch.Tensor([0.7,0.2,0.1]).reshape(1,-1,1,1))
# # outputs = torch.ones(1,4,300,400)*(torch.Tensor([0.5,0.25,0.15, 0.1]).reshape(1,-1,1,1))
# outputs = outputs.cuda()
# outputs[0,0,100:200, 50:150] = 0.1
# outputs[0,0,100:150, 250:300] = 0.2
# outputs[0,1,100:200, 50:150] = 0.7
# outputs[0,1,100:150, 250:300] = 0.1
# outputs[0,2,100:200, 50:150] = 0.2
# outputs[0,2,100:150, 250:300] = 0.7
# Converting back to logits
outputs = torch.log(outputs)
outputs = outputs.permute(0, 2, 3, 1).contiguous()
outputs = torch.matmul(outputs, temp_weights)
outputs = outputs + temp_bias
outputs = outputs.permute(0, 3, 1, 2).contiguous()
# Add image stuff
save_imgs = torch.softmax(outputs, dim =1).squeeze(0)
# analyse(outputs = save_imgs.unsqueeze(0))
# accuracy(outputs = outputs)
for class_no, class_distri in enumerate(save_imgs):
plt.clf()
class_distri[0][0] = 0
class_distri[0][1] = 1
im = plt.imshow(class_distri.detach().cpu().numpy(),cmap="Greens")
plt.colorbar(im)
plt.savefig("temp_files/temp.jpg")
plt.clf()
import cv2
img_dif = cv2.imread("temp_files/temp.jpg")
self.writer.add_image(f"Class_{class_no}", img_dif, epoch, dataformats="HWC")
loss_cce = cce_criterion.forward(outputs, targets)
loss_cross_entropy = cross_criterion.forward(outputs, targets)
loss_info_entropy = info_entropy_criterion.forward(outputs)
alpha = 1
total_loss = loss_cce + alpha * loss_info_entropy
# total_loss = loss_cross_entropy
epoch_loss_info_entropy_total += loss_info_entropy
epoch_loss_cce_total += loss_cce.item()
epoch_loss_cross_entropy_total += loss_cross_entropy.item()
epoch_loss_total += total_loss.item()
total_loss.backward()
optimizer.step()
with torch.no_grad():
for output, target in zip(outputs,targets.detach()):
# older ece requires softmax and size output=[class,w,h] target=[w,h]
eceEvaluator_perimage.update(output.softmax(dim=0), target)
# print(outputs.shape)
# print(eceEvaluator_perimage.get_overall_CCELoss())
print(f"batch :{i+1}/{len(self.val_loader)}" + "loss cce : {:.5f} | loss cls : {:.5f} | loss tot : {:.5f}".format(loss_cce, loss_cross_entropy, total_loss))
print(temp_weights)
print(temp_bias)
epoch_loss_cce_total /= len(self.val_loader)
epoch_loss_cross_entropy_total /= len(self.val_loader)
epoch_loss_total /= len(self.val_loader)
count_table_image, _ = eceEvaluator_perimage.get_count_table_img(self.classes)
cce_table_image, dif_map= eceEvaluator_perimage.get_perc_table_img(self.classes)
self.writer.add_image("CCE_table", cce_table_image, epoch, dataformats="HWC")
self.writer.add_image("Count table", count_table_image, epoch, dataformats="HWC")
self.writer.add_image("DifMap", dif_map, epoch, dataformats="HWC")
self.writer.add_scalar(f"Cross EntropyLoss_LR", epoch_loss_cross_entropy_total, epoch)
self.writer.add_scalar(f"Info EntropyLoss_LR", epoch_loss_info_entropy_total, epoch)
self.writer.add_scalar(f"CCELoss_LR", epoch_loss_cce_total, epoch)
self.writer.add_scalar(f"Total Loss_LR", epoch_loss_total, epoch)
self.writer.add_histogram("Weights", temp_weights, epoch)
self.writer.add_histogram("Bias", temp_bias, epoch)
# output = output/temp_weights
# print(output.shape)
# print(temp_weights, temp_bias)
if epoch > 0 and epoch % 10 == 0:
print("saving weights.")
np.save("weights/toy/wt_{}_{}.npy".format(epoch, self.prefix), temp_weights.cpu().detach().numpy())
np.save("weights/toy/b{}_{}.npy".format(epoch, self.prefix), temp_bias.cpu().detach().numpy())
# print("epoch {} : loss {:.5f}".format(epoch, epoch_loss))
# import pdb; pdb.set_trace()
self.writer.close()
from calibration_library.metrics import CCELoss as perimageCCE
from calibration_library.cce_loss import CCELoss
a = CCELoss(19, 10)
b = perimageCCE(19, 10)
import torch
img = torch.rand(4, 19, 769, 769).cuda()
target = torch.randint(0, 20, (4, 769, 769)).cuda()
a.forward(img, target)
with torch.no_grad():
for output, target in zip(img, target.detach()):
#older ece requires softmax and size output=[class,w,h] target=[w,h]
b.update(output.softmax(dim=0), target)
b.get_overall_CCELoss()
img = torch.rand(4, 19, 769, 769).cuda()
target = torch.randint(0, 20, (4, 769, 769)).cuda()
a.forward(img, target)
with torch.no_grad():
for output, target in zip(img, target.detach()):
#older ece requires softmax and size output=[class,w,h] target=[w,h]
b.update(output.softmax(dim=0), target)
(b.get_overall_CCELoss())
a.get_overall_CCELoss()
def eval(self):
self.metric.reset()
self.model.eval()
model = self.model
temp_weights = torch.eye(len(self.classes), device="cuda")
temp_weights /= 1.5
# temp_weights.requires_grad= True
# temp_weights.requires_grad= True
temp_bias = torch.zeros(len(self.classes),
device="cuda",
requires_grad=False)
# temp_weights = torch.rand(len(self.classes), len(self.classes), device="cuda", requires_grad=True)
# temp_bias = torch.rand(len(self.classes), device="cuda", requires_grad=True)
logging.info(
"Start training of temprature weights, Total sample: {:d}".format(
len(self.val_loader)))
cce_criterion = CCELoss(len(self.classes)).to(self.device)
cross_criterion = torch.nn.CrossEntropyLoss(ignore_index=-1)
# optimizer = torch.optim.SGD([temp_weights, temp_bias], lr=self.lr)
import time
time_start = time.time()
num_epochs = 1
for epoch in range(num_epochs):
eceEvaluator_perimage = perimageCCE(n_classes=len(self.classes))
epoch_loss_cce_total = 0
epoch_loss_cross_entropy_total = 0
epoch_loss_total = 0
for i, (images, targets, filenames) in enumerate(self.val_loader):
# import pdb; pdb.set_trace()
# optimizer.zero_grad()
images = images.to(self.device)
targets = targets.to(self.device)
# print(image.shape)
with torch.no_grad():
outputs = model.evaluate(images)
outputs = outputs.permute(0, 2, 3, 1).contiguous()
outputs = torch.matmul(outputs, temp_weights)
outputs = outputs + temp_bias
outputs = outputs.permute(0, 3, 1, 2).contiguous()
loss_cce = cce_criterion.forward(outputs, targets)
loss_cross_entropy = cross_criterion.forward(outputs, targets)
total_loss = loss_cce + loss_cross_entropy
epoch_loss_cce_total += loss_cce.item()
epoch_loss_cross_entropy_total += loss_cross_entropy.item()
epoch_loss_total += total_loss.item()
# total_loss.backward()
# optimizer.step()
with torch.no_grad():
for output, target in zip(outputs, targets.detach()):
#older ece requires softmax and size output=[class,w,h] target=[w,h]
eceEvaluator_perimage.update(output.softmax(dim=0),
target)
# print(outputs.shape)
# print(eceEvaluator_perimage.get_overall_CCELoss())
print(
f"batch :{i+1}/{len(self.val_loader)}" +
"loss cce : {:.5f} | loss cls : {:.5f} | loss tot : {:.5f}"
.format(loss_cce, loss_cross_entropy, total_loss))
print(temp_weights)
print(temp_bias)
epoch_loss_cce_total /= len(self.val_loader)
epoch_loss_cross_entropy_total /= len(self.val_loader)
epoch_loss_total /= len(self.val_loader)
cce_table_image, dif_map = eceEvaluator_perimage.get_perc_table_img(
self.classes)
self.writer.add_image("CCE_table",
cce_table_image,
epoch,
dataformats="HWC")
self.writer.add_image("DifMap", dif_map, epoch, dataformats="HWC")
self.writer.add_scalar(f"Cross EntropyLoss_LR",
epoch_loss_cross_entropy_total, epoch)
self.writer.add_scalar(f"CCELoss_LR", epoch_loss_cce_total, epoch)
self.writer.add_scalar(f"Total Loss_LR", epoch_loss_total, epoch)
self.writer.add_histogram("Weights", temp_weights, epoch)
self.writer.add_histogram("Bias", temp_bias, epoch)
# output = output/temp_weights
# print(output.shape)
# print(temp_weights, temp_bias)
if epoch > 0 and epoch % 10 == 0:
print("saving weights.")
np.save(
"weights/foggy_cityscapes/wt_{}_{}.npy".format(
epoch, self.prefix),
temp_weights.cpu().detach().numpy())
np.save(
"weights/foggy_cityscapes/b{}_{}.npy".format(
epoch, self.prefix),
temp_bias.cpu().detach().numpy())
# print("epoch {} : loss {:.5f}".format(epoch, epoch_loss))
# import pdb; pdb.set_trace()
self.writer.close()
def eval(self):
self.metric.reset()
self.model.eval()
model = self.model
temp_weights = torch.eye(len(self.classes), device = "cuda")
torch.nn.init.xavier_uniform_(temp_weights, gain=1.0)
temp_weights.requires_grad= True
# temp_weights.requires_grad= True
temp_bias = torch.zeros(len(self.classes), device = "cuda")
# torch.nn.init.xavier_uniform_(temp_bias, gain=1.0)
temp_bias.requires_grad= True
# temp_weights = torch.rand(len(self.classes), len(self.classes), device="cuda", requires_grad=True)
# temp_bias = torch.rand(len(self.classes), device="cuda", requires_grad=True)
logging.info("Start training of temprature weights, Total sample: {:d}".format(len(self.val_loader)))
cce_criterion = CCELoss(len(self.classes)).to(self.device)
cross_criterion = torch.nn.CrossEntropyLoss(ignore_index=-1)
info_entropy_criterion = InfoEntropyLoss()
optimizer = torch.optim.SGD(self.transformation_network.parameters(), lr=self.lr)
import time
time_start = time.time()
num_epochs = 200
for epoch in range(num_epochs):
eceEvaluator_perimage = perimageCCE(n_classes = len(self.classes))
epoch_loss_cce_total = 0
epoch_loss_cross_entropy_total = 0
epoch_loss_total = 0
epoch_loss_info_entropy = 0
for i, (images, targets, filenames) in enumerate(self.val_loader):
# import pdb; pdb.set_trace()
optimizer.zero_grad()
images = images.to(self.device)
targets = targets.to(self.device)
# print(image.shape)
with torch.no_grad():
outputs = model.evaluate(images)
outputs = self.transformation_network(outputs)
# print(outputs.shape)
# Image saving and stuff
save_imgs = torch.softmax(outputs, dim =1).squeeze(0)
for class_no, class_distri in enumerate(save_imgs):
plt.clf()
class_distri[0][0] = 0
class_distri[0][1] = 1
im = plt.imshow(class_distri.detach().cpu().numpy(),cmap="Greens")
plt.colorbar(im)
plt.savefig("temp_files/temp.jpg")
plt.clf()
import cv2
img_dif = cv2.imread("temp_files/temp.jpg")
self.writer.add_image(f"Class_{self.classes[class_no]}", img_dif, epoch, dataformats="HWC")
loss_cce = cce_criterion.forward(outputs, targets)
loss_cross_entropy = cross_criterion.forward(outputs, targets)
loss_info_entropy = info_entropy_criterion.forward(outputs)
alpha = 1
# total_loss = loss_cce + alpha * loss_info_entropy
total_loss = loss_cce
epoch_loss_info_entropy += loss_info_entropy
epoch_loss_cce_total += loss_cce.item()
epoch_loss_cross_entropy_total += loss_cross_entropy.item()
epoch_loss_total += total_loss.item()
total_loss.backward()
optimizer.step()
with torch.no_grad():
for output, target in zip(outputs,targets.detach()):
#older ece requires softmax and size output=[class,w,h] target=[w,h]
eceEvaluator_perimage.update(output.softmax(dim=0), target)
# print(outputs.shape)
# print(eceEvaluator_perimage.get_overall_CCELoss())
print(f"batch :{i+1}/{len(self.val_loader)}" + "loss cce : {:.5f} | loss cls : {:.5f} | loss tot : {:.5f}".format(loss_cce, loss_cross_entropy, total_loss))
epoch_loss_cce_total /= len(self.val_loader)
epoch_loss_cross_entropy_total /= len(self.val_loader)
epoch_loss_total /= len(self.val_loader)
epoch_loss_cross_entropy_total /= len(self.val_loader)
count_table_image, _ = eceEvaluator_perimage.get_count_table_img(self.classes)
cce_table_image, dif_map= eceEvaluator_perimage.get_perc_table_img(self.classes)
self.writer.add_image("CCE_table", cce_table_image, epoch, dataformats="HWC")
self.writer.add_image("Count table", count_table_image, epoch, dataformats="HWC")
self.writer.add_image("DifMap", dif_map, epoch, dataformats="HWC")
self.writer.add_scalar(f"Cross EntropyLoss_LR", epoch_loss_cross_entropy_total, epoch)
self.writer.add_scalar(f"CCELoss_LR", epoch_loss_cce_total, epoch)
self.writer.add_scalar(f"Info EntropyLoss_LR", epoch_loss_info_entropy, epoch)
self.writer.add_scalar(f"Total Loss_LR", epoch_loss_total, epoch)
self.writer.add_histogram("Weights", temp_weights, epoch)
self.writer.add_histogram("Bias", temp_bias, epoch)
# output = output/temp_weights
# print(output.shape)
# print(temp_weights, temp_bias)
if epoch > 0 and epoch % 10 == 0:
print("saving weights.")
np.save("weights/foggy_cityscapes/wt_{}_{}.npy".format(epoch, self.prefix), temp_weights.cpu().detach().numpy())
np.save("weights/foggy_cityscapes/b{}_{}.npy".format(epoch, self.prefix), temp_bias.cpu().detach().numpy())
# print("epoch {} : loss {:.5f}".format(epoch, epoch_loss))
# import pdb; pdb.set_trace()
self.writer.close()
def validation(self, val_loader, writer, temp = 1):
self.metric.reset()
self.ece_evaluator.reset()
self.cce_evaluator.reset()
eceEvaluator_perimage = perimageCCE(n_classes = len(self.classes))
if self.args.distributed:
model = self.model.module
else:
model = self.model
torch.cuda.empty_cache()
model.eval()
for i, (image, target, filename) in enumerate(val_loader):
image = image.to(self.device)
target = target.to(self.device)
with torch.no_grad():
if cfg.DATASET.MODE == 'val' or cfg.TEST.CROP_SIZE is None:
output = model(image)[0]
else:
size = image.size()[2:]
pad_height = cfg.TEST.CROP_SIZE[0] - size[0]
pad_width = cfg.TEST.CROP_SIZE[1] - size[1]
image = F.pad(image, (0, pad_height, 0, pad_width))
output = model(image)[0]
output = output[..., :size[0], :size[1]]
# output is [1, 19, 1024, 2048] logits
# target is [1, 1024, 2048]
output /= temp
self.metric.update(output, target)
if (i==0):
import cv2
image_read = cv2.imread(filename[0])
writer.add_image("Image[0] Read", image_read, temp*10, dataformats="HWC")
save_imgs = torch.softmax(output, dim =1)[0]
for class_no, class_distri in enumerate(save_imgs):
plt.clf()
class_distri[0][0] = 0
class_distri[0][1] = 1
im = plt.imshow(class_distri.detach().cpu().numpy(),cmap="Greens")
plt.colorbar(im)
plt.savefig("temp_files/temp.jpg")
plt.clf()
import cv2
img_dif = cv2.imread("temp_files/temp.jpg")
writer.add_image(f"Class_{self.classes[class_no]}", img_dif, temp*10, dataformats="HWC")
with torch.no_grad():
self.ece_evaluator.forward(output,target)
self.cce_evaluator.forward(output,target)
# for output, target in zip(output,target.detach()):
# #older ece requires softmax and size output=[class,w,h] target=[w,h]
# eceEvaluator_perimage.update(output.softmax(dim=0), target)
pixAcc, mIoU = self.metric.get()
logging.info("[EVAL] Sample: {:d}, pixAcc: {:.3f}, mIoU: {:.3f}".format(i + 1, pixAcc * 100, mIoU * 100))
pixAcc, mIoU = self.metric.get()
logging.info("[EVAL END] temp: {:f}, pixAcc: {:.3f}, mIoU: {:.3f}".format(temp, pixAcc * 100, mIoU * 100))
writer.add_scalar("Temp [EVAL END] pixAcc", pixAcc * 100, temp*10 )
writer.add_scalar("Temp [EVAL END] mIoU", mIoU * 100, temp*10 )
# count_table_image, _ = eceEvaluator_perimage.get_count_table_img(self.classes)
# cce_table_image, dif_map = eceEvaluator_perimage.get_perc_table_img(self.classes)
ece_count_table_image, _ = self.ece_evaluator.get_count_table_img(self.classes)
ece_table_image, ece_dif_map = self.ece_evaluator.get_perc_table_img(self.classes)
cce_count_table_image, _ = self.cce_evaluator.get_count_table_img(self.classes)
cce_table_image, cce_dif_map = self.cce_evaluator.get_perc_table_img(self.classes)
# writer.add_image("Temp CCE_table", cce_table_image, temp, dataformats="HWC")
# writer.add_image("Temp CCE Count table", count_table_image, temp, dataformats="HWC")
# writer.add_image("Temp CCE DifMap", dif_map, temp, dataformats="HWC")
# writer.add_scalar("Temp CCE Score", eceEvaluator_perimage.get_overall_CCELoss(), temp)
writer.add_image("Temp ece_table", ece_table_image, temp*10, dataformats="HWC")
writer.add_image("Temp ece Count table", ece_count_table_image, temp*10, dataformats="HWC")
writer.add_image("Temp ece DifMap", ece_dif_map, temp*10, dataformats="HWC")
writer.add_scalar("Temp ece Score", self.ece_evaluator.get_overall_ECELoss(), temp*10)
writer.add_image("Temp cce_table", cce_table_image, temp*10, dataformats="HWC")
writer.add_image("Temp cce Count table", cce_count_table_image, temp*10, dataformats="HWC")
writer.add_image("Temp cce DifMap", cce_dif_map, temp*10, dataformats="HWC")
writer.add_scalar("Temp cce Score", self.cce_evaluator.get_overall_CCELoss(), temp*10)
synchronize()
