def main():
args = parser.parse_args()
data_const = HicoConstants(exp_ver=args.exp_ver)
print('Creating output dir ...')
io.mkdir_if_not_exists(data_const.result_dir + '/map', recursive=True)
# Load hoi_list
hoi_list_json = os.path.join(data_const.proc_dir, 'hoi_list.json')
hoi_list = io.load_json_object(hoi_list_json)
# Load subset ids to eval on
split_ids_json = os.path.join(data_const.proc_dir, 'split_ids.json')
split_ids = io.load_json_object(split_ids_json)
global_ids = split_ids[args.subset]
global_ids_set = set(global_ids)
# Create gt_dets
print('Creating GT dets ...')
gt_dets = load_gt_dets(data_const.proc_dir, global_ids_set)
eval_inputs = []
for hoi in hoi_list:
eval_inputs.append((hoi['id'], global_ids, gt_dets,
data_const.result_dir + '/pred_hoi_dets.hdf5',
data_const.result_dir + '/map'))
# import ipdb; ipdb.set_trace()
# eval_hoi(*eval_inputs[0])
print(f'Starting a pool of {args.num_processes} workers ...')
p = Pool(args.num_processes)
print(f'Begin mAP computation ...')
output = p.starmap(eval_hoi, eval_inputs)
#output = eval_hoi('003',global_ids,gt_dets,args.pred_hoi_dets_hdf5,args.out_dir)
p.close()
p.join()
mAP = {
'AP': {},
'mAP': 0,
'invalid': 0,
}
map_ = 0
count = 0
for ap, hoi_id in output:
mAP['AP'][hoi_id] = ap
if not np.isnan(ap):
count += 1
map_ += ap
mAP['mAP'] = map_ / count
mAP['invalid'] = len(output) - count
mAP_json = os.path.join(data_const.result_dir + '/map', 'mAP.json')
io.dump_json_object(mAP, mAP_json)
print(f'APs have been saved to {data_const.result_dir}/map')
def main():
args = parser.parse_args()
data_const = HicoConstants(exp_ver=args.exp_ver)
out_dir = data_const.result_dir+'/map'
bin_to_hoi_ids = io.load_json_object(data_const.bin_to_hoi_ids_json)
mAP_json = os.path.join(out_dir,'mAP.json')
APs = io.load_json_object(mAP_json)['AP']
bin_map = {}
bin_count = {}
for bin_id,hoi_ids in bin_to_hoi_ids.items():
bin_map[bin_id] = compute_mAP(APs,hoi_ids)
non_rare_hoi_ids = []
for ul in bin_to_hoi_ids.keys():
if ul=='10':
continue
non_rare_hoi_ids += bin_to_hoi_ids[ul]
sample_complexity_analysis = {
'bin': bin_map,
'full': compute_mAP(APs,APs.keys()),
'rare': bin_map['10'],
'non_rare': compute_mAP(APs,non_rare_hoi_ids)
}
sample_complexity_analysis_json = os.path.join(
out_dir,
f'sample_complexity_analysis.json')
io.dump_json_object(
sample_complexity_analysis,
sample_complexity_analysis_json)
bin_names = sorted([int(ul) for ul in bin_map.keys()])
bin_names = [str(ul) for ul in bin_names]
bin_headers = ['0'] + bin_names
bin_headers = [bin_headers[i]+'-'+str(int(ul)-1) for i,ul in enumerate(bin_headers[1:])]
headers = ['Full','Rare','Non-Rare'] + bin_headers
sca = sample_complexity_analysis
values = [sca['full'],sca['rare'],sca['non_rare']] + \
[bin_map[name] for name in bin_names]
values = [str(round(v*100,2)) for v in values]
print('Space delimited values that can be copied to spreadsheet and split by space')
print(' '.join(headers))
print(' '.join(values))
def __init__(self,
data_const=HicoConstants(),
subset='train',
data_aug=False,
sampler=None,
test=False):
super(HicoDataset, self).__init__()
self.data_aug = data_aug
self.data_const = data_const
self.test = test
self.subset_ids = self._load_subset_ids(subset, sampler)
self.sub_app_data = self._load_subset_app_data(subset)
self.sub_spatial_data = self._load_subset_spatial_data(subset)
self.word2vec = h5py.File(self.data_const.word2vec, 'r')
def main():
data_const = HicoConstants()
hico_list = io.load_json_object(data_const.anno_list_json)
global_ids = [anno['global_id'] for anno in hico_list]
# Create and save splits
split_ids = split(global_ids,0.2)
split_ids_json = os.path.join(
data_const.proc_dir,
'split_ids.json')
io.dump_json_object(split_ids,split_ids_json)
# Create and save split stats
split_stats = {}
for subset_name,subset_ids in split_ids.items():
split_stats[subset_name] = len(subset_ids)
print(f'{subset_name}: {len(subset_ids)}')
split_stats_json = os.path.join(
data_const.proc_dir,
'split_ids_stats.json')
io.dump_json_object(split_stats,split_stats_json)
def main():
data_const = HicoConstants()
anno_list = io.load_json_object(data_const.anno_list_json)
hoi_cls_count = {}
for anno in tqdm(anno_list):
if 'test' in anno['global_id']:
continue
for hoi in anno['hois']:
hoi_id = hoi['id']
if hoi_id not in hoi_cls_count:
hoi_cls_count[hoi_id] = 0
hoi_cls_count[hoi_id] += len(hoi['connections'])
upper_limits = [10, 50, 100, 500, 1000, 10000]
bin_to_hoi_ids = bin_hoi_ids(hoi_cls_count, upper_limits)
hoi_cls_count_json = os.path.join(data_const.proc_dir,
'hoi_cls_count.json')
io.dump_json_object(hoi_cls_count, hoi_cls_count_json)
bin_to_hoi_ids_json = os.path.join(data_const.proc_dir,
'bin_to_hoi_ids.json')
io.dump_json_object(bin_to_hoi_ids, bin_to_hoi_ids_json)
return anno_list
def convert(self):
print('Creating anno list ...')
anno_list = self.create_anno_list()
io.dump_json_object(anno_list, self.const.anno_list_json)
print('Creating hoi list ...')
hoi_list = self.create_hoi_list()
io.dump_json_object(hoi_list, self.const.hoi_list_json)
print('Creating object list ...')
object_list = sorted(list(set([hoi['object'] for hoi in hoi_list])))
for i, obj in enumerate(object_list):
object_list[i] = {'id': str(i + 1).zfill(3), 'name': obj}
io.dump_json_object(object_list, self.const.object_list_json)
print('Creating verb list ...')
verb_list = sorted(list(set([hoi['verb'] for hoi in hoi_list])))
for i, verb in enumerate(verb_list):
verb_list[i] = {'id': str(i + 1).zfill(3), 'name': verb}
io.dump_json_object(verb_list, self.const.verb_list_json)
if __name__ == '__main__':
# import ipdb; ipdb.set_trace()
hico_const = HicoConstants()
converter = ConvertMat2Json(hico_const)
converter.convert()
single_feat = []
box1_wrt_img = box_with_respect_to_img(det_boxes[i], im_wh)
box2_wrt_img = box_with_respect_to_img(det_boxes[j], im_wh)
box1_wrt_box2 = box1_with_respect_to_box2(
det_boxes[i], det_boxes[j])
offset = center_offset(det_boxes[i], det_boxes[j], im_wh)
single_feat = single_feat + box1_wrt_img + box2_wrt_img + box1_wrt_box2 + offset.tolist(
)
# ipdb.set_trace()
spatial_feats.append(single_feat)
spatial_feats = np.array(spatial_feats)
return spatial_feats
if __name__ == "__main__":
data_const = HicoConstants()
boxes_scores_rpn_ids_labels = h5py.File(
data_const.boxes_scores_rpn_ids_labels, 'r')
print('Load seleced boxes data file successfully...')
split_id = io.load_json_object(data_const.split_ids_json)
anno_list = io.load_json_object(data_const.anno_list_json)
anno_dict = {item['global_id']: item for item in anno_list}
print('Load original data successfully!')
for subset in ['train_val', 'test']:
# create saving file
if subset == 'train_val':
print('Creating trainval_spatial_feat.hdf5 file....')
save_data = h5py.File(data_const.trainval_spatial_feat, 'w')
else:
def main(args):
# use GPU if available else revert to CPU
device = torch.device(
'cuda' if torch.cuda.is_available() and args.gpu else 'cpu')
print("Testing on", device)
# Load checkpoint and set up model
try:
# load checkpoint
checkpoint = torch.load(args.main_pretrained, map_location=device)
print('vsgats Checkpoint loaded!')
pg_checkpoint = torch.load(args.pretrained, map_location=device)
# set up model and initialize it with uploaded checkpoint
# ipdb.set_trace()
if not args.exp_ver:
args.exp_ver = args.pretrained.split(
"/")[-2] + "_" + args.pretrained.split("/")[-1].split("_")[-2]
# import ipdb; ipdb.set_trace()
data_const = HicoConstants(feat_type=checkpoint['feat_type'],
exp_ver=args.exp_ver)
vs_gats = AGRNN(feat_type=checkpoint['feat_type'],
bias=checkpoint['bias'],
bn=checkpoint['bn'],
dropout=checkpoint['dropout'],
multi_attn=checkpoint['multi_head'],
layer=checkpoint['layers'],
diff_edge=checkpoint['diff_edge']) #2 )
vs_gats.load_state_dict(checkpoint['state_dict'])
vs_gats.to(device)
vs_gats.eval()
print(pg_checkpoint['o_c_l'], pg_checkpoint['b_l'],
pg_checkpoint['attn'], pg_checkpoint['lr'],
pg_checkpoint['dropout'])
pgception = PGception(action_num=pg_checkpoint['a_n'],
layers=1,
classifier_mod=pg_checkpoint['classifier_mod'],
o_c_l=pg_checkpoint['o_c_l'],
last_h_c=pg_checkpoint['last_h_c'],
bias=pg_checkpoint['bias'],
drop=pg_checkpoint['dropout'],
bn=pg_checkpoint['bn'],
agg_first=pg_checkpoint['agg_first'],
attn=pg_checkpoint['attn'],
b_l=pg_checkpoint['b_l'])
pgception.load_state_dict(pg_checkpoint['state_dict'])
pgception.to(device)
pgception.eval()
print('Constructed model successfully!')
except Exception as e:
print('Failed to load checkpoint or construct model!', e)
sys.exit(1)
print('Creating hdf5 file for predicting hoi dets ...')
if not os.path.exists(data_const.result_dir):
os.mkdir(data_const.result_dir)
pred_hoi_dets_hdf5 = os.path.join(data_const.result_dir,
'pred_hoi_dets.hdf5')
pred_hois = h5py.File(pred_hoi_dets_hdf5, 'w')
test_dataset = HicoDataset(data_const=data_const, subset='test', test=True)
test_dataloader = DataLoader(dataset=test_dataset,
batch_size=1,
shuffle=False,
collate_fn=collate_fn)
# for global_id in tqdm(test_list):
for data in tqdm(test_dataloader):
global_id = data['global_id'][0]
det_boxes = data['det_boxes'][0]
roi_scores = data['roi_scores'][0]
roi_labels = data['roi_labels'][0]
node_num = data['node_num']
features = data['features']
spatial_feat = data['spatial_feat']
word2vec = data['word2vec']
pose_normalized = data["pose_to_human"]
pose_to_obj_offset = data["pose_to_obj_offset"]
# referencing
pose_to_obj_offset, pose_normalized, features, spatial_feat, word2vec = pose_to_obj_offset.to(
device), pose_normalized.to(device), features.to(
device), spatial_feat.to(device), word2vec.to(device)
outputs, attn, attn_lang = vs_gats(
node_num, features, spatial_feat, word2vec,
[roi_labels]) # !NOTE: it is important to set [roi_labels]
pg_outputs = pgception(pose_normalized, pose_to_obj_offset)
action_scores = nn.Sigmoid()(outputs + pg_outputs)
action_scores = action_scores.cpu().detach().numpy()
# save detection result
pred_hois.create_group(global_id)
det_data_dict = {}
h_idxs = np.where(roi_labels == 1)[0]
for h_idx in h_idxs:
for i_idx in range(len(roi_labels)):
if i_idx == h_idx:
continue
if h_idx > i_idx:
edge_idx = h_idx * (node_num[0] - 1) + i_idx
else:
edge_idx = h_idx * (node_num[0] - 1) + i_idx - 1
score = roi_scores[h_idx] * roi_scores[i_idx] * action_scores[
edge_idx]
try:
hoi_ids = metadata.obj_hoi_index[roi_labels[i_idx]]
except Exception as e:
ipdb.set_trace()
for hoi_idx in range(hoi_ids[0] - 1, hoi_ids[1]):
hoi_pair_score = np.concatenate(
(det_boxes[h_idx], det_boxes[i_idx],
np.expand_dims(score[metadata.hoi_to_action[hoi_idx]],
0)),
axis=0)
if str(hoi_idx + 1).zfill(3) not in det_data_dict.keys():
det_data_dict[str(hoi_idx + 1).zfill(
3)] = hoi_pair_score[None, :]
else:
det_data_dict[str(hoi_idx + 1).zfill(3)] = np.vstack(
(det_data_dict[str(hoi_idx + 1).zfill(3)],
hoi_pair_score[None, :]))
for k, v in det_data_dict.items():
pred_hois[global_id].create_dataset(k, data=v)
pred_hois.close()
def main(args):
# Load checkpoint and set up model
try:
# use GPU if available else revert to CPU
device = torch.device(
'cuda:0' if torch.cuda.is_available() and args.gpu else 'cpu')
print("Testing on", device)
# set up model and initialize it with uploaded checkpoint
if args.dataset == 'hico':
# load checkpoint
checkpoint = torch.load(args.main_pretrained_hico,
map_location=device)
print('vsgats Checkpoint loaded!')
pg_checkpoint = torch.load(args.pretrained_hico,
map_location=device)
data_const = HicoConstants(feat_type=checkpoint['feat_type'])
vs_gats = vsgat_hico(feat_type=checkpoint['feat_type'],
bias=checkpoint['bias'],
bn=checkpoint['bn'],
dropout=checkpoint['dropout'],
multi_attn=checkpoint['multi_head'],
layer=checkpoint['layers'],
diff_edge=checkpoint['diff_edge']) #2 )
if args.dataset == 'vcoco':
# load checkpoint
checkpoint = torch.load(args.main_pretrained_vcoco,
map_location=device)
print('vsgats Checkpoint loaded!')
pg_checkpoint = torch.load(args.pretrained_vcoco,
map_location=device)
data_const = VcocoConstants()
vs_gats = vsgat_vcoco(feat_type=checkpoint['feat_type'],
bias=checkpoint['bias'],
bn=checkpoint['bn'],
dropout=checkpoint['dropout'],
multi_attn=checkpoint['multi_head'],
layer=checkpoint['layers'],
diff_edge=checkpoint['diff_edge']) #2 )
vs_gats.load_state_dict(checkpoint['state_dict'])
vs_gats.to(device)
vs_gats.eval()
print(pg_checkpoint['o_c_l'], pg_checkpoint['b_l'],
pg_checkpoint['attn'], pg_checkpoint['lr'],
pg_checkpoint['dropout'])
# pgception = PGception(action_num=24, classifier_mod='cat', o_c_l=[64,64,128,128], last_h_c=256, bias=pg_checkpoint['bias'], drop=pg_checkpoint['dropout'], bn=pg_checkpoint['bn'])
pgception = PGception(action_num=pg_checkpoint['a_n'],
layers=1,
classifier_mod=pg_checkpoint['classifier_mod'],
o_c_l=pg_checkpoint['o_c_l'],
last_h_c=pg_checkpoint['last_h_c'],
bias=pg_checkpoint['bias'],
drop=pg_checkpoint['dropout'],
bn=pg_checkpoint['bn'],
agg_first=pg_checkpoint['agg_first'],
attn=pg_checkpoint['attn'],
b_l=pg_checkpoint['b_l'])
# pgception = PGception(action_num=pg_checkpoint['a_n'], drop=pg_checkpoint['dropout'])
pgception.load_state_dict(pg_checkpoint['state_dict'])
pgception.to(device)
pgception.eval()
print('Constructed model successfully!')
except Exception as e:
print('Failed to load checkpoint or construct model!', e)
sys.exit(1)
# prepare for data
if args.dataset == 'hico':
original_imgs_dir = os.path.join(data_const.infer_dir,
'original_imgs/hico')
# original_imgs_dir = './datasets/hico/images/test2015'
save_path = os.path.join(data_const.infer_dir, 'processed_imgs/hico')
test_dataset = HicoDataset(data_const=data_const, subset='test')
dataloader = sorted(os.listdir(original_imgs_dir))
# dataloader = ['HICO_test2015_00000128.jpg']
else:
original_imgs_dir = os.path.join(data_const.infer_dir,
'original_imgs/vcoco')
# original_imgs_dir = './datasets/vcoco/coco/images/val2014'
save_path = os.path.join(data_const.infer_dir, 'processed_imgs/vcoco')
test_dataset = VcocoDataset(data_const=data_const,
subset='vcoco_test',
pg_only=False)
# dataloader = DataLoader(dataset=test_dataset, batch_size=1, shuffle=False, collate_fn=vcoco_collate_fn)
dataloader = sorted(os.listdir(original_imgs_dir))
dataloader = ['COCO_val2014_000000150361.jpg']
if not os.path.exists(original_imgs_dir):
os.makedirs(original_imgs_dir)
if not os.path.exists(save_path):
os.mkdir(save_path)
print('result images will be kept here{}'.format(save_path))
# ipdb.set_trace()
for data in tqdm(dataloader):
# load corresponding data
# print("Testing on image named {}".format(img))
if args.dataset == 'hico':
img = data
global_id = data.split('.')[0]
test_data = test_dataset.sample_date(global_id)
test_data = collate_fn([test_data])
det_boxes = test_data['det_boxes'][0]
roi_scores = test_data['roi_scores'][0]
roi_labels = test_data['roi_labels'][0]
keypoints = test_data['keypoints'][0]
edge_labels = test_data['edge_labels']
node_num = test_data['node_num']
features = test_data['features']
spatial_feat = test_data['spatial_feat']
word2vec = test_data['word2vec']
pose_normalized = test_data["pose_to_human"]
pose_to_obj_offset = test_data["pose_to_obj_offset"]
else:
# global_id = data['global_id'][0]
img = data
global_id = str(int((data.split('.')[0].split('_')[-1])))
test_data = test_dataset.sample_date(global_id)
test_data = vcoco_collate_fn([test_data])
# img = data['img_name'][0][:].astype(np.uint8).tostring().decode('ascii').split("/")[-1]
# test_data = data
det_boxes = test_data['det_boxes'][0]
roi_scores = test_data['roi_scores'][0]
roi_labels = test_data['roi_labels'][0]
edge_labels = test_data['edge_labels']
node_num = test_data['node_num']
features = test_data['features']
spatial_feat = test_data['spatial_feat']
word2vec = test_data['word2vec']
pose_normalized = test_data["pose_to_human"]
pose_to_obj_offset = test_data["pose_to_obj_offset"]
# inference
pose_to_obj_offset, pose_normalized, features, spatial_feat, word2vec = pose_to_obj_offset.to(
device), pose_normalized.to(device), features.to(
device), spatial_feat.to(device), word2vec.to(device)
outputs, attn, attn_lang = vs_gats(
node_num, features, spatial_feat, word2vec,
[roi_labels]) # !NOTE: it is important to set [roi_labels]
pg_outputs = pgception(pose_normalized, pose_to_obj_offset)
# action_score = nn.Sigmoid()(outputs+pg_outputs)
# action_score = action_score.cpu().detach().numpy()
det_outputs = nn.Sigmoid()(outputs + pg_outputs)
det_outputs = det_outputs.cpu().detach().numpy()
# show result
# import ipdb; ipdb.set_trace()
if args.dataset == 'hico':
image = Image.open(
os.path.join('datasets/hico/images/test2015',
img)).convert('RGB')
image_temp = image.copy()
gt_img = vis_img(image,
det_boxes,
roi_labels,
roi_scores,
edge_labels.cpu().numpy(),
score_thresh=0.5)
det_img = vis_img(image_temp,
det_boxes,
roi_labels,
roi_scores,
det_outputs,
score_thresh=0.5)
if args.dataset == 'vcoco':
image = Image.open(
os.path.join(data_const.original_image_dir, 'val2014',
img)).convert('RGB')
image_temp = image.copy()
gt_img = vis_img_vcoco(image,
det_boxes,
roi_labels,
roi_scores,
edge_labels.cpu().numpy(),
score_thresh=0.1)
det_img = vis_img_vcoco(image_temp,
det_boxes,
roi_labels,
roi_scores,
det_outputs,
score_thresh=0.5)
# det_img.save('/home/birl/ml_dl_projects/bigjun/hoi/VS_GATs/inference_imgs/original_imgs'+'/'+img)
det_img.save(save_path + '/' + img.split("/")[-1])
def main():
# exp_name = 'factors_rcnn_det_prob_appearance_boxes_and_object_label_human_pose'
# exp_dir = os.path.join(
# os.getcwd(),
# f'data_symlinks/hico_exp/hoi_classifier/{exp_name}')
# map_json = os.path.join(
# exp_dir,
# 'mAP_eval/test_30000/mAP.json')
# map_json = '/home/birl/ml_dl_projects/bigjun/hoi/agrnn/result/hico/final_ver/map/mAP.json'
map_json = '/home/birl/ml_dl_projects/bigjun/hoi/VS_GATs/result/hico/vsgat_trainval2_260/map/mAP.json'
hoi_aps = io.load_json_object(map_json)['AP']
data_const = HicoConstants()
hoi_list = io.load_json_object(data_const.hoi_list_json)
verb_to_hoi_id = {}
for hoi in hoi_list:
hoi_id = hoi['id']
verb = hoi['verb']
if verb not in verb_to_hoi_id:
verb_to_hoi_id[verb] = []
verb_to_hoi_id[verb].append(hoi_id)
per_verb_hoi_aps = []
for verb, hoi_ids in verb_to_hoi_id.items():
verb_obj_aps = []
for hoi_id in hoi_ids:
verb_obj_aps.append(hoi_aps[hoi_id] * 100)
per_verb_hoi_aps.append((verb, verb_obj_aps))
# import ipdb; ipdb.set_trace()
per_verb_hoi_aps = sorted(per_verb_hoi_aps, key=lambda x: np.median(x[1]))
N = len(per_verb_hoi_aps)
c = ['hsl(' + str(h) + ',50%' + ',50%)' for h in np.linspace(0, 360, N)]
data = []
for i, (verb, aps) in enumerate(per_verb_hoi_aps):
trace = go.Box(
y=aps,
name=" ".join(verb.split("_")),
boxpoints=False, #"outliers"
marker={'color': c[i]},
line={'width': 1})
data.append(trace)
layout = go.Layout(
plot_bgcolor='#FFFFFF',
# paper_bgcolor='#DBDBDB',
yaxis=dict(
title='AP of HOI Categories',
range=[0, 80],
titlefont=dict(size=15),
showgrid=True,
gridcolor='#DBDBDB',
# showline=True,
# linecolor='#666666'
),
xaxis=dict(title='Interactions',
titlefont=dict(size=15),
tickangle=45,
tickfont=dict(size=8, ),
showline=True,
linecolor='#666666'),
height=500,
margin=go.Margin(
l=100,
r=100,
b=150,
t=50,
),
)
filename = os.path.join('./inference_imgs', 'obj_aps_per_interaction.html')
plotly.offline.plot({
'data': data,
'layout': layout
},
filename=filename,
auto_open=False)
def main(args):
# use GPU if available else revert to CPU
device = torch.device('cuda' if torch.cuda.is_available() and args.gpu else 'cpu')
print("Testing on", device)
# Load checkpoint and set up model
try:
# load checkpoint
checkpoint = torch.load(args.pretrained, map_location=device)
print('Checkpoint loaded!')
# set up model and initialize it with uploaded checkpoint
# ipdb.set_trace()
if not args.exp_ver:
args.exp_ver = args.pretrained.split("/")[-3]+"_"+args.pretrained.split("/")[-1].split("_")[-2]
data_const = HicoConstants(feat_type=checkpoint['feat_type'], exp_ver=args.exp_ver)
model = AGRNN(feat_type=checkpoint['feat_type'], bias=checkpoint['bias'], bn=checkpoint['bn'], dropout=checkpoint['dropout'], multi_attn=checkpoint['multi_head'], layer=checkpoint['layers'], diff_edge=checkpoint['diff_edge']) #2 )
# ipdb.set_trace()
model.load_state_dict(checkpoint['state_dict'])
model.to(device)
model.eval()
print('Constructed model successfully!')
except Exception as e:
print('Failed to load checkpoint or construct model!', e)
sys.exit(1)
print('Creating hdf5 file for predicting hoi dets ...')
if not os.path.exists(data_const.result_dir):
os.mkdir(data_const.result_dir)
pred_hoi_dets_hdf5 = os.path.join(data_const.result_dir, 'pred_hoi_dets.hdf5')
pred_hois = h5py.File(pred_hoi_dets_hdf5,'w')
test_dataset = HicoDataset(data_const=data_const, subset='test', test=True)
test_dataloader = DataLoader(dataset=test_dataset, batch_size=1, shuffle=False, collate_fn=collate_fn)
# for global_id in tqdm(test_list):
for data in tqdm(test_dataloader):
train_data = data
global_id = train_data['global_id'][0]
# img_name = train_data['img_name'][0]
det_boxes = train_data['det_boxes'][0]
roi_scores = train_data['roi_scores'][0]
roi_labels = train_data['roi_labels'][0]
node_num = train_data['node_num']
features = train_data['features']
spatial_feat = train_data['spatial_feat']
word2vec = train_data['word2vec']
# referencing
features, spatial_feat, word2vec = features.to(device), spatial_feat.to(device), word2vec.to(device)
outputs, attn, attn_lang = model(node_num, features, spatial_feat, word2vec, [roi_labels]) # !NOTE: it is important to set [roi_labels]
action_score = nn.Sigmoid()(outputs)
action_score = action_score.cpu().detach().numpy()
attn = attn.cpu().detach().numpy()
attn_lang = attn_lang.cpu().detach().numpy()
# save detection result
pred_hois.create_group(global_id)
det_data_dict = {}
h_idxs = np.where(roi_labels == 1)[0]
labeled_edge_list = np.cumsum(node_num - np.arange(len(h_idxs)) - 1)
labeled_edge_list[-1] = 0
for h_idx in h_idxs:
for i_idx in range(len(roi_labels)):
if i_idx <= h_idx:
continue
# import ipdb; ipdb.set_trace()
edge_idx = labeled_edge_list[h_idx-1] + (i_idx-h_idx-1)
# score = roi_scores[h_idx] * roi_scores[i_idx] * action_score[edge_idx] * (attn[h_idx][i_idx-1]+attn_lang[h_idx][i_idx-1])
score = roi_scores[h_idx] * roi_scores[i_idx] * action_score[edge_idx]
try:
hoi_ids = metadata.obj_hoi_index[roi_labels[i_idx]]
except Exception as e:
ipdb.set_trace()
for hoi_idx in range(hoi_ids[0]-1, hoi_ids[1]):
hoi_pair_score = np.concatenate((det_boxes[h_idx], det_boxes[i_idx], np.expand_dims(score[metadata.hoi_to_action[hoi_idx]], 0)), axis=0)
if str(hoi_idx+1).zfill(3) not in det_data_dict.keys():
det_data_dict[str(hoi_idx+1).zfill(3)] = hoi_pair_score[None,:]
else:
det_data_dict[str(hoi_idx+1).zfill(3)] = np.vstack((det_data_dict[str(hoi_idx+1).zfill(3)], hoi_pair_score[None,:]))
for k, v in det_data_dict.items():
pred_hois[global_id].create_dataset(k, data=v)
pred_hois.close()
'--diff_edge',
type=str2bool,
default='false',
required=True,
help='h_h edge, h_o edge, o_o edge are different with each other')
parser.add_argument(
'--sampler',
type=float,
default=0,
help='h_h edge, h_o edge, o_o edge are different with each other')
args = parser.parse_args()
if __name__ == "__main__":
data_const = HicoConstants(feat_type=args.feat_type)
run_model(args, data_const)
# # import ipdb; ipdb.set_trace()
# if args.data_aug:
# # filter ROIs
# keep_inds = list(set(np.where(node_labels.cpu().numpy() == 1)[0]))
# original_inds = np.arange(node_num[0])
# remain_inds = np.delete(original_inds, keep_inds, axis=0)
# random_select_inds = np.array(random.sample(remain_inds.tolist(), int(remain_inds.shape[0]/2)))
# choose_inds = sorted(np.hstack((keep_inds,random_select_inds)))
# # remove_inds = [x for x in original_inds if x not in choose_inds]
# if len(keep_inds)==0 or len(choose_inds)==1:
# continue
# # re-construct the data
