def final_expectation_dataset(sq):
sq.model.eval()
sq.model = sq.model.module
sq.model.cpu()
sq.model.eval()
print sq.model
eb.use_eb(True)
for batch_idx, (sequence, target) in enumerate(sq.exciteloader_train):
np.save("./data/full_frame_data_2016_orig_trinary/"+str(batch_idx)+"_seq.npy", sequence.numpy())
np.save("./data/full_frame_data_2016_orig_trinary/"+str(batch_idx)+"_tar.npy", target.numpy())
continue
timer_init = time.time()
picnum = sq.length_of_sequence
mean=[0.3886, 0.4391, 0.3203]
std=[0.2294, 0.2396, 0.2175]
unnormalize = transforms.Normalize(
mean=[-0.3886/0.2294, -0.4391/0.2396, -0.3203/0.2175],
std=[1/0.2294, 1/0.2396, 1/0.2175]
)
# save sequence and target here
sequence = Variable(sequence)
target = Variable(target)
mylogger.log("Making a prediction")
output_model = sq.model(sequence)
print output_model, target
mylogger.log("Made the prediction")
pred = output_model.data.max(1)[1]
prediction = pred.numpy()
mytarget = target.data.cpu().numpy()
if prediction != mytarget:
continue
layer_top = list(sq.model.modules())[0].decoder_to_classifier[6]
layer_second = list(sq.model.modules())[0].decoder_to_classifier[4]
target_layer = list(sq.model.modules())[0].features[2]
top_dh = torch.zeros(sequence.size()[0], layer_top.out_features)
for i in range(sequence.size()[0]):
top_dh[i,mytarget[i][0]] = 1 # ground truth based contrastive signal
print top_dh
timer = time.time()
mylogger.log("Using eb")
sq.optimizer.zero_grad()
grad = eb.contrastive_eb(sq.model, sequence, layer_top, layer_second, target=target_layer, top_dh=top_dh)
mylogger.log("Time void from contrastive EB: {}".format(time.time()-timer))
print grad.numpy().shape
grad_ = grad.numpy()[:,:,:,:]
grad_ = np.mean(grad_,axis=1)
np.save("./data/full_frame_data_2016_npy_trinary/"+str(batch_idx)+".npy", grad_)
print time.time()-timer_init
'''
def final_expectation(sq):
sq.model = sq.model.module
sq.model.cpu()
sq.model.eval()
eb.use_eb(True)
for batch_idx, (sequence, target) in enumerate(sq.exciteloader):
picnum = sq.length_of_sequence
mean=[0.3886, 0.4391, 0.3203]
std=[0.2294, 0.2396, 0.2175]
unnormalize = transforms.Normalize(
mean=[-0.3886/0.2294, -0.4391/0.2396, -0.3203/0.2175],
std=[1/0.2294, 1/0.2396, 1/0.2175]
)
sequence = Variable(sequence)
target = Variable(target)
'''
mysequence = sequence[0].cpu().data
for i in range(picnum):
img = mysequence[i,:,:,:]
img = unnormalize(img)
img = img.permute(1,2,0)
plt.subplot(math.sqrt(picnum),math.sqrt(picnum),i+1)
plt.imshow(img.numpy())
plt.draw()
plt.axis('off')
plt.show()
'''
output_model = sq.model(sequence)
pred = output_model.data.max(1)[1]
print(pred)
print(target)
if sq.model.decode:
layer_top = list(sq.model.modules())[0].batch_to_classifier[0]
layer_second = list(sq.model.modules())[0].final_classifier[1]
target_layer = list(sq.model.modules())[0].features[2]
else:
layer_top = list(sq.model.modules())[0].decoder_to_classifier[3]
layer_second = list(sq.model.modules())[0].decoder_to_classifier[1]
target_layer = list(sq.model.modules())[0].features[2]
for label_choice in [0,1]:
top_dh = torch.zeros(1, layer_top.out_features)
top_dh[0,label_choice] = 1
# since sequence is a series of images, it does not batch and send one signal but sends multiple signals
timer = time.time()
sq.optimizer.zero_grad()
grad = eb.contrastive_eb(sq.model, sequence, layer_top, layer_second, target=target_layer, top_dh=top_dh)
#grad = eb.eb(sq.model, sequence, layer_top, target=target_layer, top_dh=top_dh)
mylogger.log("Time void from contrastive EB: {}".format(time.time()-timer))
#print(grad)
#print(torch.max(grad))
grad_ = grad.numpy()[:,:,:,:]
grad_ = np.mean(grad_,axis=1)
mylogger.log(pred)
mylogger.log(target)
plt.figure(1)
for i in range(picnum):
img = grad_[i,:,:]
plt.subplot(math.sqrt(picnum),math.sqrt(picnum),i+1)
plt.imshow(img)
plt.draw()
plt.axis('off')
plt.show()
mysequence = sequence[0].cpu().data
for i in range(picnum):
img = mysequence[i,:,:,:]
img = unnormalize(img)
img = img.permute(1,2,0)
plt.subplot(math.sqrt(picnum),math.sqrt(picnum),i+1)
plt.imshow(img.numpy())
plt.draw()
plt.axis('off')
plt.show()
# it kind off works
exit()
pred = output.data.max(1)[1]
layer_top = list(vp.VGG.modules())[0].classifier[6] # output layer of network
layer_second = list(vp.VGG.modules())[0].classifier[4]
target_layer = list(vp.VGG.modules())[0].features[2]
prediction = pred.cpu().numpy()[0][0]
# excite the prediction neuron
#self.mean = [ 0.54005252 ,0.51546471 ,0.48611783]
#self.std = [ 0.42398592 ,0.4154406 ,0.43313607]
unnormalize = transforms.Normalize(mean=[-0.53/0.4,-0.50/0.4,-0.47/0.4],std=[1/0.4,1/0.4,1/0.4])
topil = transforms.ToPILImage()
chunk_of_tensor = unnormalize(first_image[0])
img = topil(chunk_of_tensor)
plt.figure(1)
plt.subplot(4,4,1)
plt.title("Prediction: "+str(my_keys[prediction]))
plt.imshow(img)
plt.draw()
plt.axis('off')
for i in range(30,40):
top_dh = torch.zeros(1, layer_top.out_features)
idd = -1
if i != 37:
idd = i ; top_dh[0, idd] = 1
else:
idd = first_target; top_dh[0, idd] = 1
grad = eb.contrastive_eb(vp.VGG, datum, layer_top, layer_second ,target=target_layer, top_dh=top_dh)
grad_ = grad.numpy()[0,:,:,:]
grad_ = np.mean(grad_,axis=0)
ix = i-28
# ix from 0 to 8
plt.subplot(4,4,ix)
plt.axis('off')
plt.title("Contrast: "+str(my_keys[idd]))
plt.imshow(grad_)
plt.draw()
plt.show()
def EB_extraction(vp):
print("EB extraction, 3 levels, sequence and non-sequence")
for loader in [vp.testloader_acc]:
vp.model_VGG.eval()
eb.use_eb(True)
print("Starting EB extraction")
print(len(loader))
counter = 0
all_examples = 0
last_video = "-"
last_batch = 0
n_image = 0
for batch_idx, (data, target, video_name) in enumerate(loader):
#if video_name[0] == "2016090800": # already did it before
# continue
if last_video != video_name[0]:
print("video ", video_name[0])
last_batch = 0
last_video = video_name[0]
if last_batch >= 32:
continue
else:
last_batch += 1
print(last_batch, batch_idx, video_name[0])
# remove continue and exit()
timer = time.time()
target.squeeze_()
target = Variable(target)
data = data[0]
data = Variable(data)
#print data.size(), target.size(), video_name
#features = vp.model_VGG(data)
#print time.time()-timer
output = vp.model_VGG(data)
pred = output.data.max(1)[1]
all_examples += data.size()[0]
flat_pred = pred.cpu().numpy().flatten()
np_target = target.data.cpu().numpy()
correct = pred.eq(target.data).sum()
#print(output) # don't even need output
if correct == 32:
counter += 1
print("all correct here")
else:
print(correct,"/",32 , "not correct...")
#print time.time()-timer
layer_top = list(vp.model_VGG.modules())[0].classifier[6]
layer_second = list(vp.model_VGG.modules())[0].classifier[4]
target_layer = list(vp.model_VGG.modules())[0].features[2] # 3,4,5 or 6
top_dh = torch.zeros(32,3)
#print np_target
for i in range(32):
top_dh[i,np_target[i]] = 1 # ground truth based contrastive signal
#print top_dh
mylogger.log("Using eb")
grad = eb.contrastive_eb(vp.model_VGG, data, layer_top, layer_second, target=target_layer, top_dh=top_dh)
mylogger.log("Time void from contrastive EB: {}".format(time.time()-timer))
#print grad.numpy().shape
grad_ = grad.numpy()[:,:,:,:]
grad_ = np.mean(grad_,axis=1)
prediction = pred.cpu().numpy()
for j in range(32):
mytensor = torch.from_numpy(grad_[j]).unsqueeze(0).unsqueeze(0)
print(mytensor.size())
vutils.save_image(mytensor,"./experiments/viewpoint_EB_final_data_proper/sample_"+str(batch_idx).zfill(4)+"_"+str(j+n_image*32).zfill(4)+"_"+str(int(prediction[j]))+".png", normalize=True)
print(data.size())
vutils.save_image(data[j].data,"./experiments/viewpoint_EB_final_data_proper/sample_orig_"+str(batch_idx).zfill(4)+"_"+str(j+n_image*32).zfill(4)+"_"+str(int(prediction[j]))+".png", normalize=False)
n_image += 1
continue
np.save("./experiments/viewpoint_EB_final_data_proper/"+str(last_batch)+"_"+str(video_name[0])+"_attentions.npy", grad_)
np.save("./experiments/viewpoint_EB_final_data_proper/"+str(last_batch)+"_"+str(video_name[0])+"_original.npy", data.data.numpy())
np.save("./experiments/viewpoint_EB_final_data_proper/"+str(last_batch)+"_"+str(video_name[0])+"_targets.npy", target.data.numpy())
'''
plt.figure(1)
for i in range(32):
img = grad_[i,:,:]
plt.subplot(6,6,i+1)
plt.imshow(img)
plt.draw()
plt.axis('off')
plt.show()
'''
#print time.time()-timer
print("Correct total sequences from 9128 to ", counter)
def EB_extraction(vp):
print("EB extraction, 3 levels, sequence and non-sequence")
for loader in [vp.testloader_acc]:
vp.model_ED.eval()
eb.use_eb(True)
print("Starting EB extraction")
print(len(loader))
counter = 0
all_examples = 0
for batch_idx, (data, target, video_name) in enumerate(loader):
timer = time.time()
target.squeeze_()
target = Variable(target)
data = data[0]
data = Variable(data)
#print data.size(), target.size(), video_name
#features = vp.model_VGG(data)
print time.time() - timer
output = vp.model_ED(data)
pred = output.data.max(1)[1]
all_examples += data.size()[0]
flat_pred = pred.cpu().numpy().flatten()
np_target = target.data.cpu().numpy()
correct = pred.eq(target.data).sum()
#print(output) # don't even need output
if correct == 32:
counter += 1
print("all correct here")
else:
print(correct, "/", 32, "not correct...")
print time.time() - timer
layer_top = list(vp.model_ED.modules())[0].classifier[6]
layer_second = list(vp.model_ED.modules())[0].classifier[4]
target_layer = list(
vp.model_ED.modules())[0].features[0] # 3,4,5 or 6
top_dh = torch.zeros(32, 3)
print np_target
for i in range(32):
top_dh[
i,
np_target[i]] = 1 # ground truth based contrastive signal
print top_dh
mylogger.log("Using eb")
grad = eb.contrastive_eb(vp.model_ED,
data,
layer_top,
layer_second,
target=target_layer,
top_dh=top_dh)
mylogger.log(
"Time void from contrastive EB: {}".format(time.time() -
timer))
print grad.numpy().shape
grad_ = grad.numpy()[:, :, :, :]
grad_ = np.mean(grad_, axis=1)
np.save(
"./experiments/viewpoint_EB_data/" + str(batch_idx) + "_" +
str(video_name[0]) + "_attentions.npy", grad_)
np.save(
"./experiments/viewpoint_EB_data/" + str(batch_idx) + "_" +
str(video_name[0]) + "_original.npy", data.data.numpy())
np.save(
"./experiments/viewpoint_EB_data/" + str(batch_idx) + "_" +
str(video_name[0]) + "_targets.npy", target.data.numpy())
'''
plt.figure(1)
for i in range(32):
img = grad_[i,:,:]
plt.subplot(6,6,i+1)
plt.imshow(img)
plt.draw()
plt.axis('off')
plt.show()
'''
print time.time() - timer
print("Correct total sequences from 9128 to ", counter)
def final_expectation_dataset(pt):
pt.model.eval()
pt.model = pt.model.module
pt.model.cpu()
pt.model.eval()
print pt.model
eb.use_eb(True)
pt.model.eval()
for batch_idx, (imgs, play_type, _) in enumerate(pt.exciteloader_train):
counter = 0
all_examples = 0
last_video = "-"
last_batch = 0
if batch_idx == len(pt.exciteloader_train) - 1:
continue
mypredictions = [0, 0, 0]
n_image = 0
for t in range(
0, int(
(imgs.size()[1] - pt.proper_length) / (pt.proper_length))):
#print(int(imgs.size()[1]/(pt.proper_length/4)), remainder)
imgs_temp = imgs[:, t * (pt.proper_length):t * (pt.proper_length) +
pt.proper_length]
_temp = _[:][t * (pt.proper_length):t * (pt.proper_length) +
pt.proper_length]
#vutils.save_image(imgs_temp[0],"./myfigure{}.jpg".format(t))
#print(imgs.size())
batch_size = play_type.cpu().size()[0]
play_type_fragment = Variable(play_type,
volatile=True,
requires_grad=False)
imgs_fragment = Variable(imgs_temp,
volatile=True,
requires_grad=False)
#print(imgs_temp.size(), imgs.size())
output = pt.model(imgs_fragment)
play_type_fragment = play_type_fragment.squeeze(1)
prediction = output.data.cpu().max(1)[1].numpy()
prediction = np.squeeze(prediction)
play_type_target = play_type_fragment.data.cpu().numpy()
mypredictions[int(prediction)] += 1
#print("Done for t=",t," and batch id=",batch_idx," and video ", _[t*(pt.proper_length)])
layer_top = list(pt.model.modules())[0].decoder_to_classifier[6]
layer_second = list(pt.model.modules())[0].decoder_to_classifier[4]
target_layer = list(pt.model.modules())[0].features[1]
top_dh = torch.zeros(imgs.size()[0], layer_top.out_features)
top_dh[0, play_type_target[
0]] = 1 # ground truth based contrastive signal
#print(top_dh)
#prediction_final = mypredictions.index(max(mypredictions))
if prediction != play_type_target:
#print("not a match",prediction, play_type_target, int(prediction), int(play_type_target[0]))
n_image += 1
continue
else:
pass
#print(int(prediction),"matched",int(play_type_target[0]))
imgs_temp = Variable(imgs_temp)
pt.optimizer.zero_grad()
grad = eb.contrastive_eb(pt.model,
imgs_temp,
layer_top,
layer_second,
target=target_layer,
top_dh=top_dh)
print grad.numpy().shape
grad_ = grad.numpy()[:, :, :, :]
grad_ = np.mean(grad_, axis=1)
for j in range(pt.proper_length):
mytensor = torch.from_numpy(grad_[j]).unsqueeze(0).unsqueeze(0)
print(mytensor.size())
vutils.save_image(
mytensor,
"./experiments/playtype_EB_final_data/sample_" +
str(batch_idx).zfill(4) + "_" +
str(j + n_image * pt.proper_length).zfill(4) + "_" +
str(int(prediction)) + ".png",
normalize=True)
copyfile(
str(_temp[j][0]),
"./experiments/playtype_EB_final_data/sample_orig_" +
str(batch_idx).zfill(4) + "_" +
str(j + n_image * pt.proper_length).zfill(4) + "_" +
str(int(prediction)) + ".jpg")
n_image += 1
continue
if prediction_final == play_type_target:
print("CORRECT!", prediction_final, play_type_target)
exit()
np.save(
"./data/full_frame_data_2016_orig_trinary/" + str(batch_idx) +
"_seq.npy", sequence.numpy())
np.save(
"./data/full_frame_data_2016_orig_trinary/" + str(batch_idx) +
"_tar.npy", target.numpy())
continue
timer_init = time.time()
picnum = sq.length_of_sequence
mean = [0.3886, 0.4391, 0.3203]
std = [0.2294, 0.2396, 0.2175]
unnormalize = transforms.Normalize(
mean=[-0.3886 / 0.2294, -0.4391 / 0.2396, -0.3203 / 0.2175],
std=[1 / 0.2294, 1 / 0.2396, 1 / 0.2175])
# save sequence and target here
sequence = Variable(sequence)
target = Variable(target)
mylogger.log("Making a prediction")
output_model = sq.model(sequence)
print output_model, target
mylogger.log("Made the prediction")
pred = output_model.data.max(1)[1]
prediction = pred.numpy()
mytarget = target.data.cpu().numpy()
if prediction != mytarget:
continue
layer_top = list(sq.model.modules())[0].decoder_to_classifier[6]
layer_second = list(sq.model.modules())[0].decoder_to_classifier[4]
target_layer = list(sq.model.modules())[0].features[2]
top_dh = torch.zeros(sequence.size()[0], layer_top.out_features)
for i in range(sequence.size()[0]):
top_dh[i,
mytarget[i][0]] = 1 # ground truth based contrastive signal
print top_dh
timer = time.time()
mylogger.log("Using eb")
sq.optimizer.zero_grad()
grad = eb.contrastive_eb(sq.model,
sequence,
layer_top,
layer_second,
target=target_layer,
top_dh=top_dh)
mylogger.log("Time void from contrastive EB: {}".format(time.time() -
timer))
print grad.numpy().shape
grad_ = grad.numpy()[:, :, :, :]
grad_ = np.mean(grad_, axis=1)
np.save(
"./data/full_frame_data_2016_npy_trinary/" + str(batch_idx) +
".npy", grad_)
print time.time() - timer_init