def get_model(feature_type='F'):
# extract the files corresponding to each epochs
model_path = os.path.join(args.test_dir, "model")
# use original DPC weights
model_filename = "k400_128_r18_dpc-rnn.pth.tar"
model_file = os.path.join(model_path, model_filename)
try:
checkpoint = torch.load(model_file)
except:
print('model failed to load')
return
model = model_visualize(sample_size=args.img_dim,
seq_len=args.seq_len,
pred_steps=1,
network=args.net,
feature_type=feature_type,
distance_type=args.distance_type)
model = model.to(cuda)
model.eval()
model = neq_load_customized(model, checkpoint['state_dict'])
model = nn.DataParallel(model)
return model
def main():
global args
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu)
global cuda
cuda = torch.device('cuda')
# get validation data with label for plotting the embedding
transform = transforms.Compose([
# centercrop
CenterCrop(size=224),
RandomSizedCrop(consistent=True, size=224, p=0.0),
Scale(size=(args.img_dim, args.img_dim)),
ToTensor(),
Normalize()
])
data_loader = get_data(transform, 'test')
if args.output == 'embedding':
# write the embeddings to writer
try: # old version
writer = SummaryWriter(log_dir=args.test_dir + 'embeddings')
except: # v1.7
writer = SummaryWriter(logdir=args.test_dir + 'embeddings')
elif args.output == 'neighbours':
base_path = args.test_dir + 'neighbours'
try:
if not os.path.exists(base_path):
os.mkdir(base_path)
except OSError as error:
print(error)
# assuming that model is saved after every 10 epochs
# extract the files corresponding to each epochs
model_path = os.path.join(args.test_dir, "model")
model_filename = "epoch" + str(args.epoch) + ".pth.tar"
model_file = os.path.join(model_path, model_filename)
# print(model_file)
try:
checkpoint = torch.load(model_file)
except:
return
# print values corresponding to the keys
if args.output == 'embedding':
model = model_visualize(sample_size=args.img_dim,
seq_len=args.seq_len,
pred_steps=1,
network=args.net,
feature_type='Phi',
distance_type=args.distance_type)
elif args.output == 'neighbours':
model = model_visualize(sample_size=args.img_dim,
seq_len=args.seq_len,
pred_steps=1,
network=args.net,
feature_type='Phi',
distance_type=args.distance_type)
model = model.to(cuda)
model.eval()
model = neq_load_customized(model, checkpoint['state_dict'])
model = nn.DataParallel(model)
input_embed_all = torch.Tensor()
output_embed_all = torch.Tensor().to(cuda)
target_embed_all = torch.LongTensor()
if args.distance_type == 'uncertain':
radius_embed_all = torch.Tensor().to(cuda)
with torch.no_grad():
for idx, (input_seq, target) in enumerate(data_loader):
# get the middle frame of the middle sequence to represent with visualization
input_embed = input_seq.permute(0, 2, 1, 3, 4, 5)[:, :, 2, 2, :, :]
input_seq = input_seq.to(cuda)
output = model(input_seq)
# get a 256-dim embedding from F+G
output_embed = output.squeeze()
# consider only first 256 dimensions in case we use radii
# Note that radii will not be displayed in the embeddings
if args.distance_type == 'uncertain':
output_embed = output_embed[:, :-1]
# radius_embed = output_embed[:, -1].expand(1, -1)
# print(radius_embed.shape)
elif args.distance_type == 'certain':
output_embed = output_embed
# actionlabels for each video
target_embed = target.squeeze()
target_embed = target_embed.reshape(-1)
# concatenate all pertaining to current batch
input_embed_all = torch.cat((input_embed_all, input_embed), 0)
output_embed_all = torch.cat((output_embed_all, output_embed), 0)
target_embed_all = torch.cat((target_embed_all, target_embed), 0)
# if args.distance_type == 'uncertain':
# radius_embed_all = torch.cat(
# (radius_embed_all, radius_embed), 1)
# print("Here")
if args.output == 'embedding':
# store the embedding as tensorboard projection to visualize
writer.add_embedding(
output_embed_all,
metadata=target_embed_all,
# label_img=input_embed_all,
global_step=i)
print("Right here")
elif args.output == 'neighbours':
# store all the inputs, outputs and targets as a single file
# output = output.squeeze()
# output_embed = output[:, :-1, :, :]
# radius = output[:, -1, :, :]
# target = target.squeeze()
# input_embed = input_seq.squeeze().detach().cpu()
# print(input_embed.shape)
# choose middle step and middle frame as representation
# input_embed = input_embed[:, :, 2, :, :].squeeze()
path = os.path.join(base_path, "%0.5d" % args.epoch)
try:
os.mkdir(path)
except OSError as error:
print(error)
torch.save(input_embed_all, os.path.join(base_path, 'input_embed.pt'))
torch.save(output_embed_all.detach().cpu(),
os.path.join(path, 'output_embed.pt'))
torch.save(target_embed_all.detach().cpu(),
os.path.join(path, 'target_embed.pt'))
# store radius in case of uncertain weights
# if args.distance_type == 'uncertain':
# torch.save(radius_embed_all.detach().cpu(),
# os.path.join(path, 'radius.pt'))
print(input_embed_all.shape, output_embed_all.shape,
target_embed_all.shape)
generate_distance_matrix(embed_dir=base_path,
epoch=args.epoch,
gpu=args.gpu,
distance=args.score_type)
KNN_retrieval(embed_dir=base_path,
distance=args.score_type,
img_dim=256,
dataset=args.dataset,
mode='test',
seq_len=args.seq_len,
num_seq=args.num_seq,
ds=args.ds,
epoch=args.epoch,
K=args.K)
def main():
global args
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu)
global cuda
cuda = torch.device('cuda')
# get validation data with label for plotting the embedding
transform = transforms.Compose([
# centercrop
# CenterCrop(size=224),
# RandomSizedCrop(consistent=True, size=224, p=0.0),
Scale(size=(args.img_dim, args.img_dim)),
ToTensor(),
Normalize()
])
data_loader = get_data(transform, args.mode, 0)
if args.output == 'embedding':
# write the embeddings to writer
try: # old version
writer = SummaryWriter(
log_dir=args.test_dir + 'embeddings')
except: # v1.7
writer = SummaryWriter(
logdir=args.test_dir + 'embeddings')
elif args.output == 'neighbours':
base_path = os.path.join(args.test_dir, 'radius')
try:
if not os.path.exists(base_path):
os.mkdir(base_path)
except OSError as error:
print(error)
# assuming that model is saved after every 10 epochs
# extract the files corresponding to each epochs
model_path = os.path.join(args.test_dir, "model")
model_filename = args.model
model_file = os.path.join(model_path, model_filename)
print(model_file)
try:
checkpoint = torch.load(model_file)
except:
return
# print values corresponding to the keys
if args.output == 'embedding':
model = model_visualize(sample_size=args.img_dim,
seq_len=args.seq_len,
pred_steps=args.pred_steps,
network=args.net,
feature_type='Phi',
distance_type=args.distance_type)
elif args.output == 'neighbours':
model = model_visualize(sample_size=args.img_dim,
seq_len=args.seq_len,
pred_steps=args.pred_steps,
network=args.net,
feature_type='Phi',
distance_type=args.distance_type)
model = model.to(cuda)
model.eval()
model = neq_load_customized(model, checkpoint['state_dict'])
model = nn.DataParallel(model)
radius_embed_all = torch.Tensor().to(cuda)
vpath_list = []
verb_list = torch.LongTensor()
noun_list = torch.LongTensor()
first_list = torch.LongTensor()
last_list = torch.LongTensor()
with torch.no_grad():
for idx, input_dict in tqdm(enumerate(data_loader), total=len(data_loader)):
# get the middle frame of the middle sequence to represent with visualization
input_seq = input_dict['t_seq']
vpath = input_dict['vpath']
verb = input_dict['verb']
noun = input_dict['noun']
idx_block = input_dict['idx_block']
first_frame = idx_block[:, 0]
last_frame = idx_block[:, 1]
input_embed = input_seq.permute(0, 2, 1, 3, 4, 5)[:, :, 2, 2, :, :]
input_seq = input_seq.to(cuda)
output = model(input_seq)
# get a 256-dim embedding from F+G
output_embed = output.squeeze()
# consider only first 256 dimensions in case we use radii
# Note that radii will not be displayed in the embeddings
radius_embed = output_embed[:, -1].expand(1, -1)
radius_embed_all = torch.cat(
(radius_embed_all, radius_embed), 1)
# print("Here")
vpath_list.extend(vpath)
verb_list = torch.cat((verb_list, verb), 0)
noun_list = torch.cat((noun_list, noun), 0)
first_list = torch.cat((first_list, first_frame), 0)
last_list = torch.cat((last_list, last_frame), 0)
if args.output == 'embedding':
# store the embedding as tensorboard projection to visualize
writer.add_embedding(
output_embed_all,
metadata=verb_embed_all,
# label_img=input_embed_all,
global_step=i)
print("Right here")
elif args.output == 'neighbours':
torch.save(radius_embed_all.detach().cpu(), 'radius/radius.pt')
torch.save(verb_list, 'radius/verb.pt')
torch.save(noun_list, 'radius/noun.pt')
torch.save(first_list, 'radius/first_ind.pt')
torch.save(last_list, 'radius/last_ind.pt')
with open("radius/vpath.txt", 'w') as f:
for s in vpath_list:
f.write(str(s) + '\n')
def main():
global args
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu)
global cuda
cuda = torch.device('cuda')
# get validation data with label for plotting the embedding
transform = transforms.Compose([
# centercrop
CenterCrop(size=224),
RandomSizedCrop(consistent=True, size=224, p=0.0),
Scale(size=(args.img_dim, args.img_dim)),
ToTensor(),
Normalize()
])
data_loader_pred = get_data(transform, 'test', 0, args.num_seq)
# dataloader to generate GT embeddings
data_loader_GT = get_data(transform, 'test', 0, args.num_seq)
if args.output == 'embedding':
# write the embeddings to writer
try: # old version
writer = SummaryWriter(log_dir=args.test_dir + 'embeddings')
except: # v1.7
writer = SummaryWriter(logdir=args.test_dir + 'embeddings')
elif args.output == 'neighbours':
base_path = os.path.join(
args.test_dir,
'neighbours_%s_%s' % (args.score_type, args.dataset))
try:
if not os.path.exists(base_path):
os.mkdir(base_path)
except OSError as error:
print(error)
# assuming that model is saved after every 10 epochs
# extract the files corresponding to each epochs
model_path = os.path.join(args.test_dir, "model")
model_filename = "epoch" + str(args.epoch) + ".pth.tar"
model_file = os.path.join(model_path, model_filename)
try:
checkpoint = torch.load(model_file)
except:
print('model failed to load')
return
# print values corresponding to the keys
if args.output == 'embedding':
model = model_visualize(sample_size=args.img_dim,
seq_len=args.seq_len,
pred_steps=args.pred_steps,
network=args.net,
feature_type='Phi',
distance_type=args.distance_type)
elif args.output == 'neighbours':
model = model_visualize(sample_size=args.img_dim,
seq_len=args.seq_len,
pred_steps=args.pred_steps,
network=args.net,
feature_type='F',
distance_type=args.distance_type)
model = model.to(cuda)
model.eval()
model_F = model_visualize(sample_size=args.img_dim,
seq_len=args.seq_len,
pred_steps=1,
network=args.net,
feature_type='F',
distance_type=args.distance_type)
model_F = model_F.to(cuda)
model_F.eval()
model_F = neq_load_customized(model_F, checkpoint['state_dict'])
model_F = nn.DataParallel(model_F)
model = neq_load_customized(model, checkpoint['state_dict'])
model = nn.DataParallel(model)
# print('here')
# process ground truth embeddings
output_embed_all = torch.Tensor().to(cuda)
target_embed_all = torch.LongTensor()
GT_input_embed_all = torch.Tensor()
GT_output_embed_all = torch.Tensor().to(cuda)
GT_target_embed_all = torch.LongTensor()
jj = 0
with torch.no_grad():
for idx, (input_seq, target) in enumerate(data_loader_GT):
print('Processing batch #', jj)
jj += 1
# input_seq: (B, N, C, SL, H, W) || (B, C, N, SL, H, W)
input_embed = torch.Tensor()
# print(input_seq.shape)
print(input_seq.permute(0, 2, 1, 3, 4, 5)[:, :, -1, 0, :, :].shape)
for i in range(args.num_seq):
input_embed = torch.cat(
(input_embed, input_seq.permute(0, 2, 1, 3, 4,
5)[:, :, i, -1, :, :]),
0) # concatenate batch
# print(input_embed.shape)
input_seq = input_seq.to(cuda)
output = model_F(input_seq)
# print('output shape: ', output.shape)
output_embed = torch.Tensor().to(cuda)
for i in range(args.num_seq):
output_embed = torch.cat((output_embed, output[:, i, :]),
0) # concatenate batch
output_embed = output_embed.squeeze()
if args.distance_type == 'uncertain':
output_embed = output_embed[:, :-1]
# radius_embed = output_embed[:, -1].expand(1, -1)
# print(radius_embed.shape)
elif args.distance_type == 'certain':
output_embed = output_embed
target_embed = target.squeeze()
target_embed = target_embed.reshape(-1)
# concatenate all pertaining to current batch
GT_input_embed_all = torch.cat((GT_input_embed_all, input_embed),
0)
GT_output_embed_all = torch.cat(
(GT_output_embed_all, output_embed), 0)
GT_target_embed_all = torch.cat(
(GT_target_embed_all, target_embed), 0)
input_embed_all = torch.Tensor()
output_embed_all = torch.Tensor().to(cuda)
target_embed_all = torch.LongTensor()
if args.distance_type == 'uncertain':
radius_embed_all = torch.Tensor().to(cuda)
# print('here')
with torch.no_grad():
for idx, (input_seq, target) in tqdm(enumerate(data_loader_pred)):
# get the middle frame of the middle sequence to represent with visualization
# input_seq: (B, N, C, SL, H, W)
# input_embed: (B, N, SL, C, H, W)
input_embed = torch.Tensor()
print(input_seq.permute(0, 1, 3, 2, 4, 5)[:, :, -1, :, :, :].shape)
input_embed = torch.cat(
(input_embed, input_seq.permute(0, 1, 3, 2, 4,
5)[:, :, -1, :, :, :]), 0)
input_seq = input_seq.to(cuda)
output = model(input_seq)
# print(output.shape)
# get a 256-dim embedding from F+G
output_embed = output.squeeze()
# consider only first 256 dimensions in case we use radii
# Note that radii will not be displayed in the embeddings
if args.distance_type == 'uncertain':
radius_embed = output_embed[:, -1].expand(1, -1)
output_embed = output_embed[:, :-1]
# print(radius_embed.shape)
elif args.distance_type == 'certain':
output_embed = output_embed
# actionlabels for each video
target_embed = target.squeeze()
target_embed = target_embed.reshape(-1)
# concatenate all pertaining to current batch
input_embed_all = torch.cat((input_embed_all, input_embed), 0)
output_embed_all = torch.cat((output_embed_all, output_embed), 0)
target_embed_all = torch.cat((target_embed_all, target_embed), 0)
if args.distance_type == 'uncertain':
radius_embed_all = torch.cat((radius_embed_all, radius_embed),
1)
# print("Here")
if args.output == 'embedding':
# store the embedding as tensorboard projection to visualize
writer.add_embedding(
output_embed_all,
metadata=target_embed_all,
# label_img=input_embed_all,
global_step=i)
print("Right here")
elif args.output == 'neighbours':
# store all the inputs, outputs and targets as a single file
# output = output.squeeze()
# output_embed = output[:, :-1, :, :]
# radius = output[:, -1, :, :]
# target = target.squeeze()
# input_embed = input_seq.squeeze().detach().cpu()
# print(input_embed.shape)
# choose middle step and middle frame as representation
# input_embed = input_embed[:, :, 2, :, :].squeeze()
path = os.path.join(base_path, "%0.5d" % args.epoch)
print(path)
try:
os.mkdir(path)
except OSError as error:
print(error)
torch.save(GT_input_embed_all,
os.path.join(base_path, 'GT_input_embed.pt'))
torch.save(GT_output_embed_all.detach().cpu(),
os.path.join(path, 'GT_output_embed.pt'))
torch.save(GT_target_embed_all.detach().cpu(),
os.path.join(path, 'GT_target_embed.pt'))
torch.save(input_embed_all, os.path.join(base_path, 'input_embed.pt'))
torch.save(output_embed_all.detach().cpu(),
os.path.join(path, 'output_embed.pt'))
torch.save(target_embed_all.detach().cpu(),
os.path.join(path, 'target_embed.pt'))
# store radius in case of uncertain weights
if args.distance_type == 'uncertain':
torch.save(radius_embed_all.detach().cpu(),
os.path.join(path, 'radius.pt'))
print('input_embed:', input_embed_all.shape, ' output_embed: ',
output_embed_all.shape, ' target_embed: ',
target_embed_all.shape)
print('GT_input_embed:', GT_input_embed_all.shape,
' GT_output_embed: ', GT_output_embed_all.shape,
' GT_target_embed: ', GT_target_embed_all.shape)