def main(exp_const, data_const):
print(f'Reading split_ids.json ...')
split_ids = io.load_json_object(data_const.split_ids_json)
print(f'Creating a human_candidates_pose_{exp_const.subset}.hdf5 file ...')
human_cand_pose_hdf5 = os.path.join(
exp_const.exp_dir, f'human_candidates_pose_{exp_const.subset}.hdf5')
human_cand_pose = h5py.File(human_cand_pose_hdf5, 'w')
print(f'Reading hoi_candidates_{exp_const.subset}.hdf5 file ...')
hoi_cand = h5py.File(data_const.hoi_cand_hdf5, 'r')
count_assignments = 0
for global_id in tqdm(split_ids[exp_const.subset]):
boxes_scores_rpn_ids_hoi_idx = \
hoi_cand[global_id]['boxes_scores_rpn_ids_hoi_idx']
human_boxes = boxes_scores_rpn_ids_hoi_idx[:, :4]
human_rpn_ids = boxes_scores_rpn_ids_hoi_idx[:, 10]
num_cand = human_boxes.shape[0]
if 'val' in global_id:
pose_prefix = 'val2014/'
else:
pose_prefix = 'train2014/'
pose_json = os.path.join(data_const.human_pose_dir,
f'{pose_prefix}{global_id}_keypoints.json')
human_poses = [
np.reshape(np.array(pose['pose_keypoints_2d']), (-1, 3))
for pose in io.load_json_object(pose_json)['people']
]
pose_boxes = [get_pose_box(pose) for pose in human_poses]
rpn_id_to_pose = {}
for i in range(num_cand):
rpn_id = str(int(human_rpn_ids[i]))
if rpn_id in rpn_id_to_pose:
continue
else:
rpn_id_to_pose[rpn_id], match_status = assign_pose(
human_boxes[i], pose_boxes, human_poses,
data_const.num_keypoints)
if match_status:
count_assignments += 1
human_cand_pose.create_group(global_id)
for rpn_id, pose in rpn_id_to_pose.items():
human_cand_pose[global_id].create_dataset(rpn_id, data=pose)
print(f'Number of assignments: {count_assignments}')
human_cand_pose.close()
hoi_cand.close()
def __init__(self,const):
self.const = deepcopy(const)
self.annos = io.load_json_object(self.const.annos_json)
if self.const.read_noun_adj_tokens is True:
self.noun_adj_token_ids = io.load_json_object(
self.const.noun_adj_tokens_json)
if self.const.read_neg_noun_samples is True:
self.neg_noun_samples = io.load_json_object(
self.const.neg_noun_samples_json)
os.environ['HDF5_USE_FILE_LOCKING']="FALSE"
def __init__(self, const):
super().__init__()
self.const = deepcopy(const)
self.image_ids = self.read_image_ids()
self.phrase_boxes = io.load_json_object(self.const.phrase_boxes_json)
self.sentences = io.load_json_object(self.const.sentences_json)
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
self.transforms = transforms.Compose([
transforms.Resize(self.const.image_size),
transforms.ToTensor(), normalize
])
os.environ['HDF5_USE_FILE_LOCKING'] = "FALSE"
def __init__(self, const):
super().__init__()
self.const = deepcopy(const)
self.image_ids = self.read_image_ids()
self.phrase_boxes = io.load_json_object(self.const.phrase_boxes_json)
self.sentences = io.load_json_object(self.const.sentences_json)
if self.const.read_noun_adj_tokens is True:
self.noun_adj_token_ids = io.load_json_object(
self.const.noun_adj_tokens_json)
if self.const.read_neg_noun_samples is True:
self.neg_noun_samples = io.load_json_object(
self.const.neg_noun_samples_json)
os.environ['HDF5_USE_FILE_LOCKING'] = "FALSE"
def main():
args = parser.parse_args()
data_const = HicoConstants(exp_ver=args.exp_ver)
out_dir = data_const.result_dir+'/map'
bin_to_hoi_ids = io.load_json_object(data_const.bin_to_hoi_ids_json)
mAP_json = os.path.join(out_dir,'mAP.json')
APs = io.load_json_object(mAP_json)['AP']
bin_map = {}
bin_count = {}
for bin_id,hoi_ids in bin_to_hoi_ids.items():
bin_map[bin_id] = compute_mAP(APs,hoi_ids)
non_rare_hoi_ids = []
for ul in bin_to_hoi_ids.keys():
if ul=='10':
continue
non_rare_hoi_ids += bin_to_hoi_ids[ul]
sample_complexity_analysis = {
'bin': bin_map,
'full': compute_mAP(APs,APs.keys()),
'rare': bin_map['10'],
'non_rare': compute_mAP(APs,non_rare_hoi_ids)
}
sample_complexity_analysis_json = os.path.join(
out_dir,
f'sample_complexity_analysis.json')
io.dump_json_object(
sample_complexity_analysis,
sample_complexity_analysis_json)
bin_names = sorted([int(ul) for ul in bin_map.keys()])
bin_names = [str(ul) for ul in bin_names]
bin_headers = ['0'] + bin_names
bin_headers = [bin_headers[i]+'-'+str(int(ul)-1) for i,ul in enumerate(bin_headers[1:])]
headers = ['Full','Rare','Non-Rare'] + bin_headers
sca = sample_complexity_analysis
values = [sca['full'],sca['rare'],sca['non_rare']] + \
[bin_map[name] for name in bin_names]
values = [str(round(v*100,2)) for v in values]
print('Space delimited values that can be copied to spreadsheet and split by space')
print(' '.join(headers))
print(' '.join(values))
def select(exp_const,data_const):
io.mkdir_if_not_exists(exp_const.exp_dir)
select_boxes_dir = exp_const.exp_dir
# Print where the boxes are coming from and where the output is written
print(f'Boxes will be read from: {data_const.faster_rcnn_boxes}')
print(f'Boxes will be written to: {select_boxes_dir}')
print('Writing constants to exp dir ...')
data_const_json = os.path.join(exp_const.exp_dir,'data_const.json')
data_const.to_json(data_const_json)
exp_const_json = os.path.join(exp_const.exp_dir,'exp_const.json')
exp_const.to_json(exp_const_json)
print('Loading anno_list.json ...')
anno_list = io.load_json_object(data_const.anno_list_json)
print('Creating selected_coco_cls_dets.hdf5 file ...')
hdf5_file = os.path.join(select_boxes_dir,'selected_coco_cls_dets.hdf5')
f = h5py.File(hdf5_file,'w')
print('Selecting boxes ...')
for anno in tqdm(anno_list):
global_id = anno['global_id']
boxes_npy = os.path.join(
data_const.faster_rcnn_boxes,
f'{global_id}_boxes.npy')
boxes = np.load(boxes_npy)
scores_npy = os.path.join(
data_const.faster_rcnn_boxes,
f'{global_id}_scores.npy')
scores = np.load(scores_npy)
nms_keep_indices_json = os.path.join(
data_const.faster_rcnn_boxes,
f'{global_id}_nms_keep_indices.json')
nms_keep_indices = io.load_json_object(nms_keep_indices_json)
selected_dets, start_end_ids = select_dets(boxes,scores,nms_keep_indices,exp_const)
f.create_group(global_id)
f[global_id].create_dataset('boxes_scores_rpn_ids',data=selected_dets)
f[global_id].create_dataset('start_end_ids',data=start_end_ids)
f.close()
def select(data_const):
io.mkdir_if_not_exists(data_const.proc_dir)
select_boxes_dir = data_const.proc_dir
# Print where the boxes are coming from and where the output is written
print(f'Boxes will be written to: {select_boxes_dir}')
print('Loading anno_list.json ...')
anno_list = io.load_json_object(data_const.anno_list_json)
print('Creating selected_coco_cls_dets.hdf5 file ...')
# hdf5_file = os.path.join(select_boxes_dir,'selected_coco_cls_dets_0.1eval.hdf5')
hdf5_file = os.path.join(select_boxes_dir, 'selected_coco_cls_dets.hdf5')
f = h5py.File(hdf5_file, 'w')
# Load faster-rcnn detection results
all_faster_rcnn_det_data = h5py.File(data_const.faster_det_fc7_feat, 'r')
all_nms_keep_indices = io.load_json_object(
os.path.join(data_const.proc_dir, 'nms_keep_indices.json'))
print('Selecting boxes ...')
for anno in tqdm(anno_list):
global_id = anno['global_id']
# # get more detection for evaluation
# if 'test' in global_id:
# data_const.human_score_thresh = 0.1
# data_const.object_score_thresh = 0.1
boxes = all_faster_rcnn_det_data[global_id]['boxes']
scores = all_faster_rcnn_det_data[global_id]['scores']
features = all_faster_rcnn_det_data[global_id]['fc7_feat']
nms_keep_indices = all_nms_keep_indices[global_id]
# import ipdb; ipdb.set_trace()
selected_dets, start_end_ids = select_dets(boxes, scores,
nms_keep_indices,
data_const)
selected_feat = []
for rpn_id in selected_dets[:, 5]:
selected_feat.append(np.expand_dims(features[rpn_id, :], 0))
selected_feat = np.concatenate(selected_feat, axis=0)
f.create_group(global_id)
f[global_id].create_dataset('boxes_scores_rpn_ids', data=selected_dets)
f[global_id].create_dataset('start_end_ids', data=start_end_ids)
f[global_id].create_dataset('features', data=selected_feat)
f.close()
def main(exp_const, data_const):
hoi_cands = h5py.File(data_const.hoi_cand_hdf5, 'r')
human_cands_pose = h5py.File(data_const.human_cands_pose_hdf5, 'r')
human_pose_feats_hdf5 = os.path.join(
exp_const.exp_dir, f'human_pose_feats_{exp_const.subset}.hdf5')
human_pose_feats = h5py.File(human_pose_feats_hdf5, 'w')
anno_list = io.load_json_object(data_const.anno_list_json)
anno_dict = {anno['global_id']: anno for anno in anno_list}
pose_feat_computer = PoseFeatures(num_keypts=data_const.num_keypoints)
for global_id in tqdm(hoi_cands.keys()):
img_hoi_cands = hoi_cands[global_id]
human_boxes = img_hoi_cands['boxes_scores_rpn_ids_hoi_idx'][:, :4]
object_boxes = img_hoi_cands['boxes_scores_rpn_ids_hoi_idx'][:, 4:8]
human_rpn_ids = img_hoi_cands['boxes_scores_rpn_ids_hoi_idx'][:, 10]
rpn_id_to_pose = pose_feat_computer.rpn_id_to_pose_h5py_to_npy(
human_cands_pose[global_id])
img_size = anno_dict[global_id]['image_size'][:2]
imh, imw = [float(v) for v in img_size[:2]]
im_wh = np.array([[imw, imh]], dtype=np.float32)
num_cand = human_boxes.shape[0]
im_wh = np.tile(im_wh, (num_cand, 1))
feats = pose_feat_computer.compute_pose_feats(human_boxes,
object_boxes,
human_rpn_ids,
rpn_id_to_pose, im_wh)
human_pose_feats.create_group(global_id)
human_pose_feats[global_id].create_dataset('absolute_pose',
data=feats['absolute_pose'])
human_pose_feats[global_id].create_dataset('relative_pose',
data=feats['relative_pose'])
human_pose_feats.close()
def _load_subset_ids(self, subset, sampler):
global_ids = io.load_json_object(self.data_const.split_ids_json)
bad_det_ids = io.load_json_object(
self.data_const.bad_faster_rcnn_det_ids)
# skip bad instance detection image with 0-1 det
# !NOTE: How to reduce the number of bad instance detection images
subset_ids = [
id for id in global_ids[subset]
if id not in bad_det_ids['0'] + bad_det_ids["1"]
]
if sampler:
# import ipdb; ipdb.set_trace()
''' when changing the model, use sub-dataset to quickly show if there is something wrong '''
subset_ids = random.sample(subset_ids,
int(len(subset_ids) * sampler))
return subset_ids
def prepare_data(exp_const, data_const):
io.mkdir_if_not_exists(exp_const.exp_dir)
print('Writing constants to exp dir ...')
data_const_json = os.path.join(exp_const.exp_dir, 'data_const.json')
data_const.to_json(data_const_json)
exp_const_json = os.path.join(exp_const.exp_dir, 'exp_const.json')
exp_const.to_json(exp_const_json)
print('Loading anno_list.json ...')
anno_list = io.load_json_object(data_const.anno_list_json)
print('Creating input json for faster rcnn ...')
images_in_out = [None] * len(anno_list)
for i, anno in enumerate(anno_list):
global_id = anno['global_id']
image_in_out = dict()
image_in_out['in_path'] = os.path.join(data_const.images_dir,
anno['image_path_postfix'])
image_in_out['out_dir'] = os.path.join(data_const.proc_dir,
'faster_rcnn_boxes')
image_in_out['prefix'] = f'{global_id}_'
images_in_out[i] = image_in_out
images_in_out_json = os.path.join(exp_const.exp_dir,
'faster_rcnn_im_in_out.json')
io.dump_json_object(images_in_out, images_in_out_json)
def load_gt_dets(anno_list_json,global_ids):
global_ids_set = set(global_ids)
# Load anno_list
print('Loading anno_list.json ...')
anno_list = io.load_json_object(anno_list_json)
gt_dets = {}
for anno in anno_list:
if anno['global_id'] not in global_ids_set:
continue
global_id = anno['global_id']
gt_dets[global_id] = {}
for hoi in anno['hois']:
hoi_id = hoi['id']
gt_dets[global_id][hoi_id] = []
for human_box_num, object_box_num in hoi['connections']:
human_box = hoi['human_bboxes'][human_box_num]
object_box = hoi['object_bboxes'][object_box_num]
det = {
'human_box': human_box,
'object_box': object_box,
}
gt_dets[global_id][hoi_id].append(det)
return gt_dets
def load_embeddings(self, labels):
embed_h5py = io.load_h5py_object(self.const.embed_h5py)['embeddings']
word_to_idx = io.load_json_object(self.const.embed_word_to_idx_json)
embeddings = np.zeros([len(labels), self.const.embed_dims])
word_to_label = {}
for i, label in enumerate(labels):
if ' ' in label:
words = label.split(' ')
elif '_' in label:
words = label.split('_')
else:
words = [label]
denom = len(words)
for word in words:
if word == 'tree':
denom = len(words) - 1
continue
if word not in word_to_label:
word_to_label[word] = set()
word_to_label[word].add(label)
idx = word_to_idx[word]
embeddings[i] += embed_h5py[idx][()]
embeddings[i] /= denom
if self.const.no_glove == True:
embeddings[:, :self.const.glove_dim] = 0
self.embed.weight.data.copy_(torch.from_numpy(embeddings))
def main(exp_const, data_const):
hoi_cands = h5py.File(data_const.hoi_cand_hdf5, 'r')
box_feats_hdf5 = os.path.join(
exp_const.exp_dir, f'hoi_candidates_box_feats_{exp_const.subset}.hdf5')
box_feats = h5py.File(box_feats_hdf5, 'w')
anno_list = io.load_json_object(data_const.anno_list_json)
anno_dict = {anno['global_id']: anno for anno in anno_list}
for global_id in tqdm(hoi_cands.keys()):
img_hoi_cands = hoi_cands[global_id]
human_boxes = img_hoi_cands['boxes_scores_rpn_ids_hoi_idx'][:, :4]
object_boxes = img_hoi_cands['boxes_scores_rpn_ids_hoi_idx'][:, 4:8]
# 查找无object的pair
index = np.where(
img_hoi_cands['boxes_scores_rpn_ids_hoi_idx'][:, 11] != -1)[0]
chosen_humans = human_boxes[index]
chosen_objects = object_boxes[index]
img_size = anno_dict[global_id]['image_size'][:2]
feats = compute_box_feats(chosen_humans, chosen_objects, img_size)
features = np.zeros([human_boxes.shape[0], feats.shape[1]])
features[index] = feats
box_feats.create_dataset(global_id, data=features)
box_feats.close()
def main(**kwargs):
exp_base_dir = coco_paths['exp_dir']
if kwargs['dataset'] == 'flickr':
exp_base_dir = flickr_paths['exp_dir']
exp_const = ExpConstants(kwargs['exp_name'], exp_base_dir)
exp_const.model_dir = os.path.join(exp_const.exp_dir, 'models')
exp_const.seed = 0
exp_const.contextualize = not kwargs['no_context']
exp_const.random_lang = kwargs['random_lang']
data_const = FlickrDatasetConstants(kwargs['subset'])
model_const = Constants()
model_const.model_num = kwargs['model_num']
model_const.object_encoder = ObjectEncoderConstants()
model_const.object_encoder.context_layer.output_attentions = True
model_const.object_encoder.object_feature_dim = 2048
model_const.cap_encoder = CapEncoderConstants()
model_const.cap_encoder.output_attentions = True
model_const.cap_info_nce_layers = kwargs['cap_info_nce_layers']
if model_const.model_num == -100:
filename = os.path.join(exp_const.exp_dir, f'results_val_best.json')
results = io.load_json_object(filename)
model_const.model_num = results['model_num']
print('Selected model num:', model_const.model_num)
model_const.object_encoder_path = os.path.join(
exp_const.model_dir, f'object_encoder_{model_const.model_num}')
model_const.lang_sup_criterion_path = os.path.join(
exp_const.model_dir, f'lang_sup_criterion_{model_const.model_num}')
if exp_const.random_lang is True:
model_const.cap_encoder_path = os.path.join(
exp_const.model_dir, f'cap_encoder_{model_const.model_num}')
eval_flickr_phrase_loc.main(exp_const, data_const, model_const)
def get_verb_to_id(self, verb_list_json):
verb_list = io.load_json_object(verb_list_json)
verb_to_id = {
verb['verb'] + "_" + verb['role']: verb['id']
for verb in verb_list
}
return verb_to_id
def select(data_const):
io.mkdir_if_not_exists(data_const.proc_dir)
select_boxes_dir = data_const.proc_dir
# Print where the boxes are coming from and where the output is written
print(f'Boxes will be read from: {data_const.faster_rcnn_boxes}')
print(f'Boxes will be written to: {select_boxes_dir}')
print('Loading anno_list.json ...')
anno_list = io.load_json_object(data_const.anno_list_json)
print('Creating selected_coco_cls_dets.hdf5 file ...')
# hdf5_file = os.path.join(select_boxes_dir,'selected_coco_cls_dets_0.1eval.hdf5')
hdf5_file = os.path.join(select_boxes_dir, 'selected_coco_cls_dets.hdf5')
f = h5py.File(hdf5_file, 'w')
print('Selecting boxes ...')
for anno in tqdm(anno_list):
global_id = anno['global_id']
# # get more detection for evaluation
# if 'test' in global_id:
# data_const.human_score_thresh = 0.1
# data_const.object_score_thresh = 0.1
boxes_npy = os.path.join(data_const.faster_rcnn_boxes,
f'{global_id}_boxes.npy')
boxes = np.load(boxes_npy)
scores_npy = os.path.join(data_const.faster_rcnn_boxes,
f'{global_id}_scores.npy')
scores = np.load(scores_npy)
nms_keep_indices_json = os.path.join(
data_const.faster_rcnn_boxes, f'{global_id}_nms_keep_indices.json')
nms_keep_indices = io.load_json_object(nms_keep_indices_json)
# import ipdb; ipdb.set_trace()
selected_dets, start_end_ids = select_dets(boxes, scores,
nms_keep_indices,
data_const)
f.create_group(global_id)
f[global_id].create_dataset('boxes_scores_rpn_ids', data=selected_dets)
f[global_id].create_dataset('start_end_ids', data=start_end_ids)
f.close()
def load_subset_ids(self, subset):
split_ids = io.load_json_object(
"data/vcoco/annotations/split_ids.json")
subset_list_ = split_ids[subset]
subset_list = []
for id in subset_list_:
subset_list.append(subset + "_" + id)
return sorted(subset_list)
def get_wnid_and_img_paths(self):
wnid_to_urls = io.load_json_object(self.const.wnid_to_urls_json)
wnid_and_img_paths = []
for wnid, urls in wnid_to_urls.items():
for img_id, url in urls.items():
wnid_and_img_paths.append((wnid, url))
return wnid_and_img_paths
def assign(exp_const, data_const):
io.mkdir_if_not_exists(exp_const.exp_dir)
print('Saving constants ...')
save_constants({'exp': exp_const, 'data':data_const}, exp_const.exp_dir)
print(f'Reading hoi_candidates_{exp_const.subset}.hdf5 ...')
hoi_cand_hdf5 = h5py.File(data_const.hoi_cand_hdf5, 'r')
print(f'Creating hoi_candidate_labels_{exp_const.subset}.hdf5 ...')
filename = os.path.join(
exp_const.exp_dir,
f'hoi_candidate_labels_{exp_const.subset}.hdf5')
hoi_cand_label_hdf5 = h5py.File(filename, 'w')
print('Loading gt hoi detections ...')
split_ids = io.load_json_object(data_const.split_ids_json)
global_ids = split_ids[exp_const.subset]
gt_dets = load_gt_dets(data_const.anno_list_json, global_ids)
print('Loading hoi_list.json ...')
hoi_list = io.load_json_object(data_const.hoi_list_json)
hoi_ids = [hoi['id'] for hoi in hoi_list]
for global_id in tqdm(global_ids):
boxes_scores_rpn_ids_hoi_idx = \
hoi_cand_hdf5[global_id]['boxes_scores_rpn_ids_hoi_idx']
start_end_ids = hoi_cand_hdf5[global_id]['start_end_ids']
num_cand = boxes_scores_rpn_ids_hoi_idx.shape[0]
labels = np.zeros([num_cand])
for hoi_id in gt_dets[global_id]:
start_id, end_id = start_end_ids[int(hoi_id)-1]
for i in range(start_id, end_id):
cand_det = {
'human_box': boxes_scores_rpn_ids_hoi_idx[i, :4],
'object_box': boxes_scores_rpn_ids_hoi_idx[i, 4:8],
}
# 查看检测结果ho候选对与gt中ho候选对的匹配情况，如果有所匹配(iou>0.5)则label置1
is_match = match_hoi(cand_det, gt_dets[global_id][hoi_id])
if is_match:
labels[i] = 1.0
hoi_cand_label_hdf5.create_dataset(global_id, data=labels)
hoi_cand_label_hdf5.close()
def main():
const = ImagenetConstants()
io.mkdir_if_not_exists(const.img_dir)
print('Loading urls ...')
wnid_to_urls = io.load_json_object(const.wnid_to_urls_json)
print('Starting pool ...')
with Pool(40) as p:
p.starmap(downloader,product([const.img_dir],wnid_to_urls.items()))
def read_samples(self,subset):
if subset=='train':
filename = self.const.train_json
elif subset=='val':
filename = self.const.val_json
elif subset=='test':
filename = self.const.truth_json
else:
msg = f'subset {subset} not supported'
assert(False), msg
return io.load_json_object(filename)
def main(exp_const, data_const, model_const):
print('Loading pred dets ...')
pred_hois = h5py.File(data_const.pred_hoi_dets_h5py, 'r')
human_pose_feats = h5py.File(data_const.human_pose_feats_hdf5, 'r')
print('Reading anno_list.json ...')
anno_list = io.load_json_object(data_const.anno_list_json)
anno_dict = {anno['global_id']: anno for anno in anno_list}
print('Selecting top box configurations for each hoi ...')
top_boxes = select_best_boxes_across_dataset(pred_hois, anno_dict,
human_pose_feats, data_const,
exp_const)
hoi_list = io.load_json_object(data_const.hoi_list_json)
hoi_dict = {hoi['id']: hoi for hoi in hoi_list}
print('Creating visualization images ...')
vis_dir = os.path.join(exp_const.exp_dir, 'vis/top_boxes_per_hoi')
create_html(top_boxes, anno_dict, hoi_dict, data_const.images_dir, vis_dir)
def get_acc_data(acc_dir, iters):
accs = [None] * len(iters)
for i, it in enumerate(iters):
results_json = os.path.join(acc_dir, f'results_val_{it}.json')
if not os.path.exists(results_json):
continue
accs[i] = io.load_json_object(results_json)['pt_recall']
return accs
def __init__(self,const):
super(SemEval201810Dataset,self).__init__()
self.const = copy.deepcopy(const)
self.embeddings = h5py.File(self.const.embeddings_h5py,'r')
if self.const.random==True:
n,d = self.embeddings['embeddings'].shape
self.embeddings = {
'embeddings': 2*(np.random.rand(n,d)-0.5)
}
self.word_to_idx = io.load_json_object(self.const.word_to_idx_json)
self.samples = self.read_samples(self.const.subset)
def main(**kwargs):
print('Creating Caption Encoder (tokenizer) ...')
cap_encoder = CapEncoder(CapEncoderConstants())
nltk.download('punkt')
data_const = DetFeatDatasetConstants(kwargs['subset'])
annos = io.load_json_object(data_const.annos_json)['annotations']
noun_token_ids = [None] * len(annos)
noun_vocab = set()
num_human_captions = 0
num_noun_captions = 0
for i, anno in enumerate(tqdm(annos)):
image_id = anno['image_id']
cap_id = anno['id']
caption = anno['caption']
token_ids, tokens = cap_encoder.tokenize(caption)
nltk_tokens = nltk.word_tokenize(caption.lower())
pos_tags = nltk.pos_tag(nltk_tokens)
pos_tags = ignore_words_from_pos(pos_tags,
['is', 'has', 'have', 'had', 'be'])
alignment = align_pos_tokens(pos_tags, tokens)
noun_token_ids_, noun_words = get_noun_token_ids(pos_tags, alignment)
noun_token_ids_ = group_token_ids(noun_token_ids_, tokens)
if len(noun_token_ids_) > 0:
num_noun_captions += 1
noun_token_ids[i] = {
'image_id': image_id,
'cap_id': cap_id,
'token_ids': noun_token_ids_,
'words': list(noun_words)
}
noun_vocab.update(noun_words)
for human_word in [
'man', 'person', 'human', 'woman', 'boy', 'girl', 'men',
'women', 'boys', 'girls', 'child', 'children'
]:
if human_word in tokens:
num_human_captions += 1
break
io.dump_json_object(noun_token_ids, data_const.noun_tokens_json)
io.dump_json_object(sorted(list(noun_vocab)), data_const.noun_vocab_json)
print('Number of human captions:', num_human_captions)
print('Number of noun captions:', num_noun_captions)
print('Total number of captions:', len(annos))
print('Size of noun vocabulary:', len(noun_vocab))
def get_infonce_data(infonce_dir, layers):
infonce_data = io.load_json_object(
os.path.join(infonce_dir, f'infonce_{layers}_layer.json'))
iters = []
losses = []
for time, it, loss in infonce_data:
if it == 0:
continue
iters.append(it)
losses.append(round(loss, 2))
return iters, losses
def main():
data_const = HicoConstants()
anno_list = io.load_json_object(data_const.anno_list_json)
global_ids = [anno['global_id'] for anno in anno_list]
feats_hdf5 = os.path.join(data_const.proc_dir, 'faster_rcnn_fc7.hdf5')
feats = h5py.File(feats_hdf5, 'w')
for global_id in tqdm(global_ids):
fc7_npy = os.path.join(data_const.faster_rcnn_boxes,
f'{global_id}_fc7.npy')
fc7 = np.load(fc7_npy)
feats.create_dataset(global_id, data=fc7)
feats.close()
def eval_hoi_ap(args, hoi_id, pred_dets_hdf5, global_ids_list, subset='test'):
print('Creating output dir ...')
io.mkdir_if_not_exists(args.out_dir, recursive=True)
# Load hoi_list
hoi_list_json = os.path.join(args.proc_dir, 'hoi_list.json')
hoi_list = io.load_json_object(hoi_list_json)
# Load subset ids to eval on
split_ids_json = os.path.join(args.proc_dir, 'split_ids.json')
split_ids = io.load_json_object(split_ids_json)
global_ids = split_ids[subset]
global_ids_set = set(global_ids)
# Create gt_dets
print('Creating GT dets ...')
gt_dets = load_gt_dets(args.proc_dir, global_ids_set)
ap, _ = eval_hoi_my(hoi_id, global_ids, gt_dets, pred_dets_hdf5,
args.out_dir, global_ids_list)
return ap
def generate(exp_const, data_const, data_sign):
print(f'Creating exp_dir: {exp_const.exp_dir}')
io.mkdir_if_not_exists(exp_const.exp_dir)
save_constants({'exp': exp_const, 'data': data_const}, exp_const.exp_dir)
print(f'Reading split_ids.json ...')
split_ids = io.load_json_object(data_const.split_ids_json)
print('Creating an object-detector-only HOI detector ...')
hoi_cand_gen = HoiCandidatesGenerator(data_const, data_sign)
print(f'Creating a hoi_candidates_{exp_const.subset}.hdf5 file ...')
hoi_cand_hdf5 = os.path.join(exp_const.exp_dir,
f'hoi_candidates_{exp_const.subset}.hdf5')
f = h5py.File(hoi_cand_hdf5, 'w')
# 从Faster RCNN的所有预测结果中选择的高分预测
print('Reading selected dets from hdf5 file ...')
all_selected_dets = h5py.File(data_const.selected_dets_hdf5, 'r')
for global_id in tqdm(split_ids[exp_const.subset]):
selected_dets = {
'boxes': {},
'scores': {},
'rpn_ids': {},
'obj_cls': {}
}
start_end_ids = all_selected_dets[global_id]['start_end_ids'][()]
boxes_scores_rpn_ids = \
all_selected_dets[global_id]['boxes_scores_rpn_ids'][()]
for cls_ind, cls_name in enumerate(COCO_CLASSES):
start_id, end_id = start_end_ids[cls_ind]
boxes = boxes_scores_rpn_ids[start_id:end_id, :4]
scores = boxes_scores_rpn_ids[start_id:end_id, 4]
rpn_ids = boxes_scores_rpn_ids[start_id:end_id, 5]
object_cls = np.full((end_id - start_id, ), cls_ind)
selected_dets['boxes'][cls_name] = boxes
selected_dets['scores'][cls_name] = scores
selected_dets['rpn_ids'][cls_name] = rpn_ids
selected_dets['obj_cls'][cls_name] = object_cls
pred_dets, start_end_ids = hoi_cand_gen.predict(selected_dets)
f.create_group(global_id)
f[global_id].create_dataset('boxes_scores_rpn_ids_hoi_idx',
data=pred_dets)
f[global_id].create_dataset('start_end_ids', data=start_end_ids)
f.close()
def main(exp_const,data_const):
io.mkdir_if_not_exists(exp_const.exp_dir)
print('Reading anno_list.json ...')
anno_list = io.load_json_object(data_const.anno_list_json)
anno_dict = {anno['global_id']:anno for anno in anno_list}
print('Reading box and pose features ...')
human_pose_feats = h5py.File(data_const.human_pose_feats_h5py,'r')
hoi_cand = h5py.File(data_const.hoi_cand_h5py,'r')
for count,global_id in enumerate(tqdm(human_pose_feats.keys())):
if count>=exp_const.max_count:
break
human_boxes = hoi_cand[global_id]['boxes_scores_rpn_ids_hoi_idx'][:,:4]
human_rpn_ids = hoi_cand[global_id]['boxes_scores_rpn_ids_hoi_idx'][:,10]
B = human_boxes.shape[0]
absolute_pose = human_pose_feats[global_id]['absolute_pose'][()]
absolute_pose = np.reshape(absolute_pose,(B,data_const.num_keypts,3))
x1y1 = human_boxes[:,:2] # Bx2
wh = 0*x1y1 # Bx2
wh[:,0] = (human_boxes[:,2] - human_boxes[:,0])
wh[:,1] = (human_boxes[:,3] - human_boxes[:,1])
x1y1 = np.tile(x1y1[:,np.newaxis,:],(1,data_const.num_keypts,1)) # Bx18x2
wh = np.tile(wh[:,np.newaxis,:],(1,data_const.num_keypts,1)) # Bx18x2
keypts = 0*absolute_pose
keypts[:,:,:2] = absolute_pose[:,:,:2]*wh + x1y1
keypts[:,:,2] = absolute_pose[:,:,2]
img_path = os.path.join(
data_const.images_dir,
anno_dict[global_id]['image_path_postfix'])
img = skio.imread(img_path)
if len(img.shape)==2:
img = np.tile(img[:,:,np.newaxis],(1,1,3))
seen_rpn_ids = set()
for i in range(B):
rpn_id = human_rpn_ids[i]
if rpn_id in seen_rpn_ids:
continue
else:
seen_rpn_ids.add(rpn_id)
img = bbox_utils.vis_human_keypts(img,keypts[i],modify=True)
img_out_path = os.path.join(
exp_const.exp_dir,
f'{global_id}.png')
skio.imsave(img_out_path,img)
