def run(net, net_var, loader, optimizer, optimizer_var, tracker, train=False, prefix='', epoch=0): count = 0 COUNT = 20000 # calculating classification error and entropy of 20,000 randomly sampled questions N_MC = 50 # no. of Monte-carlo simulations cps = [] cvs = [] wps = [] wvs = [] net.train() net_var.train() tq = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0) for v, q, a, idx, image_id, q_len in tq: p_miss = [] if count > COUNT: break var_params = { 'volatile': train, 'requires_grad': True, } v = Variable(v.cuda(async=True), **var_params) q = Variable(q.type(torch.FloatTensor).cuda(async=True), **var_params) a = Variable(a.type(torch.FloatTensor).cuda(async=True), **var_params) q_len = Variable( q_len.type(torch.FloatTensor).cuda(async=True), **var_params) a_temp = a a_temp = a_temp.detach().cpu().numpy() a_indices = np.argmax(a_temp, axis=1) out, p_at = net(v, q, q_len) sum = np.zeros(tuple(out.shape)) for j in range(N_MC): out, p_at = net(v, q, q_len) preds = F.softmax(out, dim=1) sum += preds.data avg = sum / N_MC entropy = -1 * np.sum(avg * np.log(avg), axis=-1) for k, an_index in enumerate(a_indices): p_miss.append( 1 - avg[k][an_index]) # probability of mis-classification acc = utils.batch_accuracy(out.data, a.data).cpu() for i, imgIdx in enumerate(image_id): if math.isnan(entropy[i]): continue if count > COUNT: break count += 1 p = p_miss[i] error = np.log(1 / (1 - p)) if acc[i] == 0: wps.append(error) wvs.append(entropy[i]) else: cps.append(error) cvs.append(entropy[i]) with open("classification_error_of_correct_samples.txt", "w") as file: file.write(str(cps)) with open("entropy_of_correct_samples.txt", "w") as file: file.write(str(cvs)) with open("classification_error_of_incorrect_samples.txt.txt", "w") as file: file.write(str(wps)) with open("entropy_of_incorrect_samples.txt", "w") as file: file.write(str(wvs))
def run(net,net_dis, loader, optimizer,optimizer_dis, tracker, train=False, prefix='', epoch=0): """ Run an epoch over the given loader """ if train: net.train() tracker_class, tracker_params = tracker.MovingMeanMonitor, {'momentum': 0.99} else: net.train() tracker_class, tracker_params = tracker.MeanMonitor, {} answ = [] idxs = [] accs = [] tq = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0, file=sys.stdout) loss_tracker = tracker.track('{}_loss'.format(prefix), tracker_class(**tracker_params)) loss_grad_tracker = tracker.track('{}_loss_grad'.format(prefix), tracker_class(**tracker_params)) acc_tracker = tracker.track('{}_acc'.format(prefix), tracker_class(**tracker_params)) log_softmax = nn.LogSoftmax().cuda() d_loss = nn.BCELoss().cuda() cnt = 0 avg_loss_dis=0 for v, q, a, idx, image_id, q_len in tq: var_params = { 'volatile': False, 'requires_grad': True, } # var_params = { # 'volatile': not train, # 'requires_grad': False, # } v = Variable(v.cuda(async=True), **var_params) q = Variable(q.type(torch.FloatTensor).cuda(async=True), **var_params) a = Variable(a.type(torch.FloatTensor).cuda(async=True), **var_params) q_len = Variable(q_len.type(torch.FloatTensor).cuda(async=True), **var_params) out,p_att = net(v, q, q_len) nll = -log_softmax(out) loss = (nll * a / 10).sum(dim=1).mean() acc = utils.batch_accuracy(out.data, a.data).cpu() if train: global total_iterations update_learning_rate(optimizer, total_iterations) optimizer.zero_grad() loss.backward(retain_graph=True) optimizer.step() gradients = get_p_gradcam(v.grad.cpu().data.numpy(), v.cpu().data.numpy()) real_input=torch.Tensor(gradients).view(-1,196).cuda()# to convert numpy to tensor gt_real=torch.ones(real_input.size(0)).cuda() gt_fake=torch.zeros(real_input.size(0)).cuda() #for real output_real=net_dis(real_input) loss_r=d_loss(output_real,gt_real) optimizer_dis.zero_grad() loss_r.backward(retain_graph=True) optimizer_dis.step() #for fake output_fake=net_dis(p_att) loss_f=d_loss(output_fake,gt_fake) optimizer_dis.zero_grad() loss_f.backward(retain_graph=True) optimizer_dis.step() #for generator output_fake_grl=net_dis(p_att) loss_grl=d_loss(output_fake_grl,gt_real) optimizer.zero_grad() loss_grl.backward() optimizer.step() loss_dis=loss_r+loss_f avg_loss_dis +=loss_dis.item() if cnt%20 ==0: print('avg_loss_dis',avg_loss_dis/20) avg_loss_dis=0 total_iterations += 1 loss_tracker.append(loss.item()) loss_grad_tracker.append(loss_dis.item()) acc_tracker.append(acc.mean()) fmt = '{:.4f}'.format tq.set_postfix(loss=fmt(loss_tracker.mean.value),loss_dis=fmt(loss_grad_tracker.mean.value), acc=fmt(acc_tracker.mean.value)) else: # Uncomment the following lines when you need to get the attention and gradcam visualizations # ---------------------------------------- loss.backward() gradients = get_p_gradcam(v.grad.cpu().data.numpy(), v.cpu().data.numpy()) for i, imgIdx in enumerate(image_id): imgIdx = "COCO_" + prefix + "2014_000000" + "0" * (6-len(str(imgIdx.numpy()))) + str(imgIdx.numpy()) + ".jpg" rawImg = scipy.misc.imread(os.path.join('/VQA/Images/mscoco', prefix + '2014/' + imgIdx), mode='RGB') rawImg = scipy.misc.imresize(rawImg, (448, 448), interp='bicubic') plt.imsave("Results/RawImages/ep" + str(epoch) + "_" + "cnt" + str(cnt) + "_" + str(i) + "raw.png", rawImg) plt.imsave("Results/AttImages/ep" + str(epoch) + "_" + "cnt" + str(cnt) + "_" + str(i) + "att.png", get_blend_map_att(rawImg/255.0, p_att[i].cpu().data.numpy())) cv2.imwrite("Results/GradcamImages/ep" + str(epoch) + "_" + "cnt" + str(cnt) + "_" + str(i) + "gradcam.png", np.uint8(255 * get_blend_map_gradcam(rawImg/255.0, gradients[i]))) # ---------------------------------------- # store information about evaluation of this minibatch _, answer = out.data.cpu().max(dim=1) answ.append(answer.view(-1)) accs.append(acc.view(-1)) idxs.append(idx.view(-1).clone()) loss_tracker.append(loss.item()) acc_tracker.append(acc.mean()) fmt = '{:.4f}'.format tq.set_postfix(loss=fmt(loss_tracker.mean.value), acc=fmt(acc_tracker.mean.value)) cnt = cnt + 1 if not train: answ = list(torch.cat(answ, dim=0)) accs = list(torch.cat(accs, dim=0)) idxs = list(torch.cat(idxs, dim=0)) return answ, accs, idxs
def run(net, net_dis, loader, optimizer, optimizer_dis, tracker, train=False, prefix='', epoch=0): """ Run an epoch over the given loader """ if train: net.train() tracker_class, tracker_params = tracker.MovingMeanMonitor, { 'momentum': 0.99 } else: net.train() tracker_class, tracker_params = tracker.MeanMonitor, {} answ = [] idxs = [] accs = [] tq = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0, file=sys.stdout) loss_tracker = tracker.track('{}_loss'.format(prefix), tracker_class(**tracker_params)) loss_grad_tracker = tracker.track('{}_loss_grad'.format(prefix), tracker_class(**tracker_params)) acc_tracker = tracker.track('{}_acc'.format(prefix), tracker_class(**tracker_params)) log_softmax = nn.LogSoftmax().cuda() d_loss = nn.BCELoss().cuda() cnt = 0 hat_id = np.array(np.genfromtxt('../pytorch-vqa_att_new/hat_id.txt'), np.int) hat_id_wrong = np.array( np.genfromtxt('../pytorch-vqa_att_new/hat_id_wrong.txt'), np.int) rankCorr = 0 pVal = 0 # emd = 0 avg_loss_dis = 0 for v, q, q_id, a, atxt, idx, image_id, q_len in tq: var_params = { 'volatile': False, 'requires_grad': True, } # var_params = { # 'volatile': not train, # 'requires_grad': False, # } v = Variable(v.cuda(async=True), **var_params) q = Variable(q.type(torch.FloatTensor).cuda(async=True), **var_params) a = Variable(a.type(torch.FloatTensor).cuda(async=True), **var_params) q_len = Variable( q_len.type(torch.FloatTensor).cuda(async=True), **var_params) out, p_att = net(v, q, q_len) nll = -log_softmax(out) loss = (nll * a / 10).sum(dim=1).mean() acc = utils.batch_accuracy(out.data, a.data).cpu() if train: global total_iterations update_learning_rate(optimizer, total_iterations) optimizer.zero_grad() loss.backward(retain_graph=True) optimizer.step() gradients = get_p_gradcam(v.grad.cpu().data.numpy(), v.cpu().data.numpy()) real_input = torch.Tensor(gradients).view( -1, 196).cuda() # to convert numpy to tensor gt_real = torch.ones(real_input.size(0)).cuda() gt_fake = torch.zeros(real_input.size(0)).cuda() #for real output_real = net_dis(real_input) loss_r = d_loss(output_real, gt_real) optimizer_dis.zero_grad() loss_r.backward(retain_graph=True) optimizer_dis.step() #for fake output_fake = net_dis(p_att) loss_f = d_loss(output_fake, gt_fake) optimizer_dis.zero_grad() loss_f.backward(retain_graph=True) optimizer_dis.step() #for generator output_fake_grl = net_dis(p_att) loss_grl = d_loss(output_fake_grl, gt_real) optimizer.zero_grad() loss_grl.backward() optimizer.step() loss_dis = loss_r + loss_f avg_loss_dis += loss_dis.item() if cnt % 20 == 0: print('avg_loss_dis', avg_loss_dis / 20) avg_loss_dis = 0 total_iterations += 1 loss_tracker.append(loss.item()) loss_grad_tracker.append(loss_dis.item()) acc_tracker.append(acc.mean()) fmt = '{:.4f}'.format tq.set_postfix(loss=fmt(loss_tracker.mean.value), loss_dis=fmt(loss_grad_tracker.mean.value), acc=fmt(acc_tracker.mean.value)) else: # Uncomment the following lines when you need to get the attention and gradcam visualizations # ---------------------------------------- loss.backward() gradients = get_p_gradcam(v.grad.cpu().data.numpy(), v.cpu().data.numpy()) for i, imgIdx in enumerate(image_id): imgIdx = "COCO_" + prefix + "2014_000000" + "0" * (6 - len( str(imgIdx.numpy()))) + str(imgIdx.numpy()) + ".jpg" rawImg = scipy.misc.imread(os.path.join( '/VQA/Images/mscoco', prefix + '2014/' + imgIdx), mode='RGB') rawImg = scipy.misc.imresize(rawImg, (448, 448), interp='bicubic') # plt.imsave("Results/RawImages/ep" + str(epoch) + "_" + "cnt" + str(cnt) + "_" + str(i) + "raw.png", rawImg) plt.imsave( "Results/AttImages/ep" + str(epoch) + "_" + "cnt" + str(cnt) + "_" + str(i) + "att.png", get_blend_map_att(rawImg / 255.0, p_att[i].cpu().data.numpy())) cv2.imwrite( "Results/GradcamImages/ep" + str(epoch) + "_" + "cnt" + str(cnt) + "_" + str(i) + "gradcam.png", np.uint8( 255 * get_blend_map_gradcam(rawImg / 255.0, gradients[i]))) # ---------------------------------------- for i, quesId in enumerate(q_id): # quesIndices.append(quesId) if quesId.numpy() not in hat_id: continue else: cnt = cnt + 1 hatIdx = str(quesId.numpy())[:-1] + "0" + str( quesId.numpy())[-1] if int(hatIdx) not in hat_id_wrong: hatIdx = str(quesId.numpy())[:-1] + "1" + str( quesId.numpy())[-1] if int(hatIdx) not in hat_id_wrong: hatIdx = str(quesId.numpy())[:-1] + "2" + str( quesId.numpy())[-1] hatIdx = hatIdx[:-1] + "_" + str(int(hatIdx[-1]) + 1) + ".png" hatImg = scipy.misc.imread('/HAT_dataset/vqahat_val/' + hatIdx) hatAtt = transform.resize(hatImg, ((196, 1)), order=3) hatAtt = np.reshape(hatAtt, (196)) hatAtt -= hatAtt.min() if hatAtt.max() > 0: hatAtt /= hatAtt.max() # print(hatAtt) # print(hatAtt.shape) # print(p_att[i].cpu().data.numpy()) # print(p_att[i].cpu().data.numpy().shape) nowRankCorr, nowPVal = scipy.stats.spearmanr( hatAtt, p_att[i].cpu().data.numpy()) rankCorr += nowRankCorr pVal += nowPVal # emd += emd_samples(hatAtt, p_att[i].cpu().data.numpy()) print("update cnt: ", cnt, " ", quesId.numpy()) print("update rankcorrelation : ", rankCorr / cnt) print("update p Value : ", pVal / cnt) # print("update emd : ", emd/cnt) # store information about evaluation of this minibatch _, answer = out.data.cpu().max(dim=1) answ.append(answer.view(-1)) accs.append(acc.view(-1)) idxs.append(idx.view(-1).clone()) loss_tracker.append(loss.item()) acc_tracker.append(acc.mean()) fmt = '{:.4f}'.format tq.set_postfix(loss=fmt(loss_tracker.mean.value), acc=fmt(acc_tracker.mean.value)) # cnt = cnt + 1 print("total match : ", cnt) print("rankcorrelation : ", rankCorr / cnt) print("p Value : ", pVal / cnt) # print("emd : ", emd/cnt) if not train: answ = list(torch.cat(answ, dim=0)) accs = list(torch.cat(accs, dim=0)) idxs = list(torch.cat(idxs, dim=0)) return answ, accs, idxs