def run(net,
net_var,
loader,
optimizer,
optimizer_var,
tracker,
train=False,
prefix='',
epoch=0):
count = 0
COUNT = 20000 # calculating classification error and entropy of 20,000 randomly sampled questions
N_MC = 50 # no. of Monte-carlo simulations
cps = []
cvs = []
wps = []
wvs = []
net.train()
net_var.train()
tq = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
for v, q, a, idx, image_id, q_len in tq:
p_miss = []
if count > COUNT:
break
var_params = {
'volatile': train,
'requires_grad': True,
}
v = Variable(v.cuda(async=True), **var_params)
q = Variable(q.type(torch.FloatTensor).cuda(async=True), **var_params)
a = Variable(a.type(torch.FloatTensor).cuda(async=True), **var_params)
q_len = Variable(
q_len.type(torch.FloatTensor).cuda(async=True), **var_params)
a_temp = a
a_temp = a_temp.detach().cpu().numpy()
a_indices = np.argmax(a_temp, axis=1)
out, p_at = net(v, q, q_len)
sum = np.zeros(tuple(out.shape))
for j in range(N_MC):
out, p_at = net(v, q, q_len)
preds = F.softmax(out, dim=1)
sum += preds.data
avg = sum / N_MC
entropy = -1 * np.sum(avg * np.log(avg), axis=-1)
for k, an_index in enumerate(a_indices):
p_miss.append(
1 - avg[k][an_index]) # probability of mis-classification
acc = utils.batch_accuracy(out.data, a.data).cpu()
for i, imgIdx in enumerate(image_id):
if math.isnan(entropy[i]):
continue
if count > COUNT:
break
count += 1
p = p_miss[i]
error = np.log(1 / (1 - p))
if acc[i] == 0:
wps.append(error)
wvs.append(entropy[i])
else:
cps.append(error)
cvs.append(entropy[i])
with open("classification_error_of_correct_samples.txt", "w") as file:
file.write(str(cps))
with open("entropy_of_correct_samples.txt", "w") as file:
file.write(str(cvs))
with open("classification_error_of_incorrect_samples.txt.txt",
"w") as file:
file.write(str(wps))
with open("entropy_of_incorrect_samples.txt", "w") as file:
file.write(str(wvs))
def run(net,net_dis, loader, optimizer,optimizer_dis, tracker, train=False, prefix='', epoch=0):
""" Run an epoch over the given loader """
if train:
net.train()
tracker_class, tracker_params = tracker.MovingMeanMonitor, {'momentum': 0.99}
else:
net.train()
tracker_class, tracker_params = tracker.MeanMonitor, {}
answ = []
idxs = []
accs = []
tq = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0, file=sys.stdout)
loss_tracker = tracker.track('{}_loss'.format(prefix), tracker_class(**tracker_params))
loss_grad_tracker = tracker.track('{}_loss_grad'.format(prefix), tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix), tracker_class(**tracker_params))
log_softmax = nn.LogSoftmax().cuda()
d_loss = nn.BCELoss().cuda()
cnt = 0
avg_loss_dis=0
for v, q, a, idx, image_id, q_len in tq:
var_params = {
'volatile': False,
'requires_grad': True,
}
# var_params = {
# 'volatile': not train,
# 'requires_grad': False,
# }
v = Variable(v.cuda(async=True), **var_params)
q = Variable(q.type(torch.FloatTensor).cuda(async=True), **var_params)
a = Variable(a.type(torch.FloatTensor).cuda(async=True), **var_params)
q_len = Variable(q_len.type(torch.FloatTensor).cuda(async=True), **var_params)
out,p_att = net(v, q, q_len)
nll = -log_softmax(out)
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(out.data, a.data).cpu()
if train:
global total_iterations
update_learning_rate(optimizer, total_iterations)
optimizer.zero_grad()
loss.backward(retain_graph=True)
optimizer.step()
gradients = get_p_gradcam(v.grad.cpu().data.numpy(), v.cpu().data.numpy())
real_input=torch.Tensor(gradients).view(-1,196).cuda()# to convert numpy to tensor
gt_real=torch.ones(real_input.size(0)).cuda()
gt_fake=torch.zeros(real_input.size(0)).cuda()
#for real
output_real=net_dis(real_input)
loss_r=d_loss(output_real,gt_real)
optimizer_dis.zero_grad()
loss_r.backward(retain_graph=True)
optimizer_dis.step()
#for fake
output_fake=net_dis(p_att)
loss_f=d_loss(output_fake,gt_fake)
optimizer_dis.zero_grad()
loss_f.backward(retain_graph=True)
optimizer_dis.step()
#for generator
output_fake_grl=net_dis(p_att)
loss_grl=d_loss(output_fake_grl,gt_real)
optimizer.zero_grad()
loss_grl.backward()
optimizer.step()
loss_dis=loss_r+loss_f
avg_loss_dis +=loss_dis.item()
if cnt%20 ==0:
print('avg_loss_dis',avg_loss_dis/20)
avg_loss_dis=0
total_iterations += 1
loss_tracker.append(loss.item())
loss_grad_tracker.append(loss_dis.item())
acc_tracker.append(acc.mean())
fmt = '{:.4f}'.format
tq.set_postfix(loss=fmt(loss_tracker.mean.value),loss_dis=fmt(loss_grad_tracker.mean.value), acc=fmt(acc_tracker.mean.value))
else:
# Uncomment the following lines when you need to get the attention and gradcam visualizations
# ----------------------------------------
loss.backward()
gradients = get_p_gradcam(v.grad.cpu().data.numpy(), v.cpu().data.numpy())
for i, imgIdx in enumerate(image_id):
imgIdx = "COCO_" + prefix + "2014_000000" + "0" * (6-len(str(imgIdx.numpy()))) + str(imgIdx.numpy()) + ".jpg"
rawImg = scipy.misc.imread(os.path.join('/VQA/Images/mscoco', prefix + '2014/' + imgIdx), mode='RGB')
rawImg = scipy.misc.imresize(rawImg, (448, 448), interp='bicubic')
plt.imsave("Results/RawImages/ep" + str(epoch) + "_" + "cnt" + str(cnt) + "_" + str(i) + "raw.png", rawImg)
plt.imsave("Results/AttImages/ep" + str(epoch) + "_" + "cnt" + str(cnt) + "_" + str(i) + "att.png", get_blend_map_att(rawImg/255.0, p_att[i].cpu().data.numpy()))
cv2.imwrite("Results/GradcamImages/ep" + str(epoch) + "_" + "cnt" + str(cnt) + "_" + str(i) + "gradcam.png", np.uint8(255 * get_blend_map_gradcam(rawImg/255.0, gradients[i])))
# ----------------------------------------
# store information about evaluation of this minibatch
_, answer = out.data.cpu().max(dim=1)
answ.append(answer.view(-1))
accs.append(acc.view(-1))
idxs.append(idx.view(-1).clone())
loss_tracker.append(loss.item())
acc_tracker.append(acc.mean())
fmt = '{:.4f}'.format
tq.set_postfix(loss=fmt(loss_tracker.mean.value), acc=fmt(acc_tracker.mean.value))
cnt = cnt + 1
if not train:
answ = list(torch.cat(answ, dim=0))
accs = list(torch.cat(accs, dim=0))
idxs = list(torch.cat(idxs, dim=0))
return answ, accs, idxs
def run(net,
net_dis,
loader,
optimizer,
optimizer_dis,
tracker,
train=False,
prefix='',
epoch=0):
""" Run an epoch over the given loader """
if train:
net.train()
tracker_class, tracker_params = tracker.MovingMeanMonitor, {
'momentum': 0.99
}
else:
net.train()
tracker_class, tracker_params = tracker.MeanMonitor, {}
answ = []
idxs = []
accs = []
tq = tqdm(loader,
desc='{} E{:03d}'.format(prefix, epoch),
ncols=0,
file=sys.stdout)
loss_tracker = tracker.track('{}_loss'.format(prefix),
tracker_class(**tracker_params))
loss_grad_tracker = tracker.track('{}_loss_grad'.format(prefix),
tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix),
tracker_class(**tracker_params))
log_softmax = nn.LogSoftmax().cuda()
d_loss = nn.BCELoss().cuda()
cnt = 0
hat_id = np.array(np.genfromtxt('../pytorch-vqa_att_new/hat_id.txt'),
np.int)
hat_id_wrong = np.array(
np.genfromtxt('../pytorch-vqa_att_new/hat_id_wrong.txt'), np.int)
rankCorr = 0
pVal = 0
# emd = 0
avg_loss_dis = 0
for v, q, q_id, a, atxt, idx, image_id, q_len in tq:
var_params = {
'volatile': False,
'requires_grad': True,
}
# var_params = {
# 'volatile': not train,
# 'requires_grad': False,
# }
v = Variable(v.cuda(async=True), **var_params)
q = Variable(q.type(torch.FloatTensor).cuda(async=True), **var_params)
a = Variable(a.type(torch.FloatTensor).cuda(async=True), **var_params)
q_len = Variable(
q_len.type(torch.FloatTensor).cuda(async=True), **var_params)
out, p_att = net(v, q, q_len)
nll = -log_softmax(out)
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(out.data, a.data).cpu()
if train:
global total_iterations
update_learning_rate(optimizer, total_iterations)
optimizer.zero_grad()
loss.backward(retain_graph=True)
optimizer.step()
gradients = get_p_gradcam(v.grad.cpu().data.numpy(),
v.cpu().data.numpy())
real_input = torch.Tensor(gradients).view(
-1, 196).cuda() # to convert numpy to tensor
gt_real = torch.ones(real_input.size(0)).cuda()
gt_fake = torch.zeros(real_input.size(0)).cuda()
#for real
output_real = net_dis(real_input)
loss_r = d_loss(output_real, gt_real)
optimizer_dis.zero_grad()
loss_r.backward(retain_graph=True)
optimizer_dis.step()
#for fake
output_fake = net_dis(p_att)
loss_f = d_loss(output_fake, gt_fake)
optimizer_dis.zero_grad()
loss_f.backward(retain_graph=True)
optimizer_dis.step()
#for generator
output_fake_grl = net_dis(p_att)
loss_grl = d_loss(output_fake_grl, gt_real)
optimizer.zero_grad()
loss_grl.backward()
optimizer.step()
loss_dis = loss_r + loss_f
avg_loss_dis += loss_dis.item()
if cnt % 20 == 0:
print('avg_loss_dis', avg_loss_dis / 20)
avg_loss_dis = 0
total_iterations += 1
loss_tracker.append(loss.item())
loss_grad_tracker.append(loss_dis.item())
acc_tracker.append(acc.mean())
fmt = '{:.4f}'.format
tq.set_postfix(loss=fmt(loss_tracker.mean.value),
loss_dis=fmt(loss_grad_tracker.mean.value),
acc=fmt(acc_tracker.mean.value))
else:
# Uncomment the following lines when you need to get the attention and gradcam visualizations
# ----------------------------------------
loss.backward()
gradients = get_p_gradcam(v.grad.cpu().data.numpy(),
v.cpu().data.numpy())
for i, imgIdx in enumerate(image_id):
imgIdx = "COCO_" + prefix + "2014_000000" + "0" * (6 - len(
str(imgIdx.numpy()))) + str(imgIdx.numpy()) + ".jpg"
rawImg = scipy.misc.imread(os.path.join(
'/VQA/Images/mscoco', prefix + '2014/' + imgIdx),
mode='RGB')
rawImg = scipy.misc.imresize(rawImg, (448, 448),
interp='bicubic')
# plt.imsave("Results/RawImages/ep" + str(epoch) + "_" + "cnt" + str(cnt) + "_" + str(i) + "raw.png", rawImg)
plt.imsave(
"Results/AttImages/ep" + str(epoch) + "_" + "cnt" +
str(cnt) + "_" + str(i) + "att.png",
get_blend_map_att(rawImg / 255.0,
p_att[i].cpu().data.numpy()))
cv2.imwrite(
"Results/GradcamImages/ep" + str(epoch) + "_" + "cnt" +
str(cnt) + "_" + str(i) + "gradcam.png",
np.uint8(
255 *
get_blend_map_gradcam(rawImg / 255.0, gradients[i])))
# ----------------------------------------
for i, quesId in enumerate(q_id):
# quesIndices.append(quesId)
if quesId.numpy() not in hat_id:
continue
else:
cnt = cnt + 1
hatIdx = str(quesId.numpy())[:-1] + "0" + str(
quesId.numpy())[-1]
if int(hatIdx) not in hat_id_wrong:
hatIdx = str(quesId.numpy())[:-1] + "1" + str(
quesId.numpy())[-1]
if int(hatIdx) not in hat_id_wrong:
hatIdx = str(quesId.numpy())[:-1] + "2" + str(
quesId.numpy())[-1]
hatIdx = hatIdx[:-1] + "_" + str(int(hatIdx[-1]) +
1) + ".png"
hatImg = scipy.misc.imread('/HAT_dataset/vqahat_val/' +
hatIdx)
hatAtt = transform.resize(hatImg, ((196, 1)), order=3)
hatAtt = np.reshape(hatAtt, (196))
hatAtt -= hatAtt.min()
if hatAtt.max() > 0:
hatAtt /= hatAtt.max()
# print(hatAtt)
# print(hatAtt.shape)
# print(p_att[i].cpu().data.numpy())
# print(p_att[i].cpu().data.numpy().shape)
nowRankCorr, nowPVal = scipy.stats.spearmanr(
hatAtt, p_att[i].cpu().data.numpy())
rankCorr += nowRankCorr
pVal += nowPVal
# emd += emd_samples(hatAtt, p_att[i].cpu().data.numpy())
print("update cnt: ", cnt, " ", quesId.numpy())
print("update rankcorrelation : ", rankCorr / cnt)
print("update p Value : ", pVal / cnt)
# print("update emd : ", emd/cnt)
# store information about evaluation of this minibatch
_, answer = out.data.cpu().max(dim=1)
answ.append(answer.view(-1))
accs.append(acc.view(-1))
idxs.append(idx.view(-1).clone())
loss_tracker.append(loss.item())
acc_tracker.append(acc.mean())
fmt = '{:.4f}'.format
tq.set_postfix(loss=fmt(loss_tracker.mean.value),
acc=fmt(acc_tracker.mean.value))
# cnt = cnt + 1
print("total match : ", cnt)
print("rankcorrelation : ", rankCorr / cnt)
print("p Value : ", pVal / cnt)
# print("emd : ", emd/cnt)
if not train:
answ = list(torch.cat(answ, dim=0))
accs = list(torch.cat(accs, dim=0))
idxs = list(torch.cat(idxs, dim=0))
return answ, accs, idxs