def main(exp_const, data_const): print(f'Reading split_ids.json ...') split_ids = io.load_json_object(data_const.split_ids_json) print(f'Creating a human_candidates_pose_{exp_const.subset}.hdf5 file ...') human_cand_pose_hdf5 = os.path.join( exp_const.exp_dir, f'human_candidates_pose_{exp_const.subset}.hdf5') human_cand_pose = h5py.File(human_cand_pose_hdf5, 'w') print(f'Reading hoi_candidates_{exp_const.subset}.hdf5 file ...') hoi_cand = h5py.File(data_const.hoi_cand_hdf5, 'r') count_assignments = 0 for global_id in tqdm(split_ids[exp_const.subset]): boxes_scores_rpn_ids_hoi_idx = \ hoi_cand[global_id]['boxes_scores_rpn_ids_hoi_idx'] human_boxes = boxes_scores_rpn_ids_hoi_idx[:, :4] human_rpn_ids = boxes_scores_rpn_ids_hoi_idx[:, 10] num_cand = human_boxes.shape[0] if 'val' in global_id: pose_prefix = 'val2014/' else: pose_prefix = 'train2014/' pose_json = os.path.join(data_const.human_pose_dir, f'{pose_prefix}{global_id}_keypoints.json') human_poses = [ np.reshape(np.array(pose['pose_keypoints_2d']), (-1, 3)) for pose in io.load_json_object(pose_json)['people'] ] pose_boxes = [get_pose_box(pose) for pose in human_poses] rpn_id_to_pose = {} for i in range(num_cand): rpn_id = str(int(human_rpn_ids[i])) if rpn_id in rpn_id_to_pose: continue else: rpn_id_to_pose[rpn_id], match_status = assign_pose( human_boxes[i], pose_boxes, human_poses, data_const.num_keypoints) if match_status: count_assignments += 1 human_cand_pose.create_group(global_id) for rpn_id, pose in rpn_id_to_pose.items(): human_cand_pose[global_id].create_dataset(rpn_id, data=pose) print(f'Number of assignments: {count_assignments}') human_cand_pose.close() hoi_cand.close()
def __init__(self,const): self.const = deepcopy(const) self.annos = io.load_json_object(self.const.annos_json) if self.const.read_noun_adj_tokens is True: self.noun_adj_token_ids = io.load_json_object( self.const.noun_adj_tokens_json) if self.const.read_neg_noun_samples is True: self.neg_noun_samples = io.load_json_object( self.const.neg_noun_samples_json) os.environ['HDF5_USE_FILE_LOCKING']="FALSE"
def __init__(self, const): super().__init__() self.const = deepcopy(const) self.image_ids = self.read_image_ids() self.phrase_boxes = io.load_json_object(self.const.phrase_boxes_json) self.sentences = io.load_json_object(self.const.sentences_json) normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) self.transforms = transforms.Compose([ transforms.Resize(self.const.image_size), transforms.ToTensor(), normalize ]) os.environ['HDF5_USE_FILE_LOCKING'] = "FALSE"
def __init__(self, const): super().__init__() self.const = deepcopy(const) self.image_ids = self.read_image_ids() self.phrase_boxes = io.load_json_object(self.const.phrase_boxes_json) self.sentences = io.load_json_object(self.const.sentences_json) if self.const.read_noun_adj_tokens is True: self.noun_adj_token_ids = io.load_json_object( self.const.noun_adj_tokens_json) if self.const.read_neg_noun_samples is True: self.neg_noun_samples = io.load_json_object( self.const.neg_noun_samples_json) os.environ['HDF5_USE_FILE_LOCKING'] = "FALSE"
def main(): args = parser.parse_args() data_const = HicoConstants(exp_ver=args.exp_ver) out_dir = data_const.result_dir+'/map' bin_to_hoi_ids = io.load_json_object(data_const.bin_to_hoi_ids_json) mAP_json = os.path.join(out_dir,'mAP.json') APs = io.load_json_object(mAP_json)['AP'] bin_map = {} bin_count = {} for bin_id,hoi_ids in bin_to_hoi_ids.items(): bin_map[bin_id] = compute_mAP(APs,hoi_ids) non_rare_hoi_ids = [] for ul in bin_to_hoi_ids.keys(): if ul=='10': continue non_rare_hoi_ids += bin_to_hoi_ids[ul] sample_complexity_analysis = { 'bin': bin_map, 'full': compute_mAP(APs,APs.keys()), 'rare': bin_map['10'], 'non_rare': compute_mAP(APs,non_rare_hoi_ids) } sample_complexity_analysis_json = os.path.join( out_dir, f'sample_complexity_analysis.json') io.dump_json_object( sample_complexity_analysis, sample_complexity_analysis_json) bin_names = sorted([int(ul) for ul in bin_map.keys()]) bin_names = [str(ul) for ul in bin_names] bin_headers = ['0'] + bin_names bin_headers = [bin_headers[i]+'-'+str(int(ul)-1) for i,ul in enumerate(bin_headers[1:])] headers = ['Full','Rare','Non-Rare'] + bin_headers sca = sample_complexity_analysis values = [sca['full'],sca['rare'],sca['non_rare']] + \ [bin_map[name] for name in bin_names] values = [str(round(v*100,2)) for v in values] print('Space delimited values that can be copied to spreadsheet and split by space') print(' '.join(headers)) print(' '.join(values))
def select(exp_const,data_const): io.mkdir_if_not_exists(exp_const.exp_dir) select_boxes_dir = exp_const.exp_dir # Print where the boxes are coming from and where the output is written print(f'Boxes will be read from: {data_const.faster_rcnn_boxes}') print(f'Boxes will be written to: {select_boxes_dir}') print('Writing constants to exp dir ...') data_const_json = os.path.join(exp_const.exp_dir,'data_const.json') data_const.to_json(data_const_json) exp_const_json = os.path.join(exp_const.exp_dir,'exp_const.json') exp_const.to_json(exp_const_json) print('Loading anno_list.json ...') anno_list = io.load_json_object(data_const.anno_list_json) print('Creating selected_coco_cls_dets.hdf5 file ...') hdf5_file = os.path.join(select_boxes_dir,'selected_coco_cls_dets.hdf5') f = h5py.File(hdf5_file,'w') print('Selecting boxes ...') for anno in tqdm(anno_list): global_id = anno['global_id'] boxes_npy = os.path.join( data_const.faster_rcnn_boxes, f'{global_id}_boxes.npy') boxes = np.load(boxes_npy) scores_npy = os.path.join( data_const.faster_rcnn_boxes, f'{global_id}_scores.npy') scores = np.load(scores_npy) nms_keep_indices_json = os.path.join( data_const.faster_rcnn_boxes, f'{global_id}_nms_keep_indices.json') nms_keep_indices = io.load_json_object(nms_keep_indices_json) selected_dets, start_end_ids = select_dets(boxes,scores,nms_keep_indices,exp_const) f.create_group(global_id) f[global_id].create_dataset('boxes_scores_rpn_ids',data=selected_dets) f[global_id].create_dataset('start_end_ids',data=start_end_ids) f.close()
def select(data_const): io.mkdir_if_not_exists(data_const.proc_dir) select_boxes_dir = data_const.proc_dir # Print where the boxes are coming from and where the output is written print(f'Boxes will be written to: {select_boxes_dir}') print('Loading anno_list.json ...') anno_list = io.load_json_object(data_const.anno_list_json) print('Creating selected_coco_cls_dets.hdf5 file ...') # hdf5_file = os.path.join(select_boxes_dir,'selected_coco_cls_dets_0.1eval.hdf5') hdf5_file = os.path.join(select_boxes_dir, 'selected_coco_cls_dets.hdf5') f = h5py.File(hdf5_file, 'w') # Load faster-rcnn detection results all_faster_rcnn_det_data = h5py.File(data_const.faster_det_fc7_feat, 'r') all_nms_keep_indices = io.load_json_object( os.path.join(data_const.proc_dir, 'nms_keep_indices.json')) print('Selecting boxes ...') for anno in tqdm(anno_list): global_id = anno['global_id'] # # get more detection for evaluation # if 'test' in global_id: # data_const.human_score_thresh = 0.1 # data_const.object_score_thresh = 0.1 boxes = all_faster_rcnn_det_data[global_id]['boxes'] scores = all_faster_rcnn_det_data[global_id]['scores'] features = all_faster_rcnn_det_data[global_id]['fc7_feat'] nms_keep_indices = all_nms_keep_indices[global_id] # import ipdb; ipdb.set_trace() selected_dets, start_end_ids = select_dets(boxes, scores, nms_keep_indices, data_const) selected_feat = [] for rpn_id in selected_dets[:, 5]: selected_feat.append(np.expand_dims(features[rpn_id, :], 0)) selected_feat = np.concatenate(selected_feat, axis=0) f.create_group(global_id) f[global_id].create_dataset('boxes_scores_rpn_ids', data=selected_dets) f[global_id].create_dataset('start_end_ids', data=start_end_ids) f[global_id].create_dataset('features', data=selected_feat) f.close()
def main(exp_const, data_const): hoi_cands = h5py.File(data_const.hoi_cand_hdf5, 'r') human_cands_pose = h5py.File(data_const.human_cands_pose_hdf5, 'r') human_pose_feats_hdf5 = os.path.join( exp_const.exp_dir, f'human_pose_feats_{exp_const.subset}.hdf5') human_pose_feats = h5py.File(human_pose_feats_hdf5, 'w') anno_list = io.load_json_object(data_const.anno_list_json) anno_dict = {anno['global_id']: anno for anno in anno_list} pose_feat_computer = PoseFeatures(num_keypts=data_const.num_keypoints) for global_id in tqdm(hoi_cands.keys()): img_hoi_cands = hoi_cands[global_id] human_boxes = img_hoi_cands['boxes_scores_rpn_ids_hoi_idx'][:, :4] object_boxes = img_hoi_cands['boxes_scores_rpn_ids_hoi_idx'][:, 4:8] human_rpn_ids = img_hoi_cands['boxes_scores_rpn_ids_hoi_idx'][:, 10] rpn_id_to_pose = pose_feat_computer.rpn_id_to_pose_h5py_to_npy( human_cands_pose[global_id]) img_size = anno_dict[global_id]['image_size'][:2] imh, imw = [float(v) for v in img_size[:2]] im_wh = np.array([[imw, imh]], dtype=np.float32) num_cand = human_boxes.shape[0] im_wh = np.tile(im_wh, (num_cand, 1)) feats = pose_feat_computer.compute_pose_feats(human_boxes, object_boxes, human_rpn_ids, rpn_id_to_pose, im_wh) human_pose_feats.create_group(global_id) human_pose_feats[global_id].create_dataset('absolute_pose', data=feats['absolute_pose']) human_pose_feats[global_id].create_dataset('relative_pose', data=feats['relative_pose']) human_pose_feats.close()
def _load_subset_ids(self, subset, sampler): global_ids = io.load_json_object(self.data_const.split_ids_json) bad_det_ids = io.load_json_object( self.data_const.bad_faster_rcnn_det_ids) # skip bad instance detection image with 0-1 det # !NOTE: How to reduce the number of bad instance detection images subset_ids = [ id for id in global_ids[subset] if id not in bad_det_ids['0'] + bad_det_ids["1"] ] if sampler: # import ipdb; ipdb.set_trace() ''' when changing the model, use sub-dataset to quickly show if there is something wrong ''' subset_ids = random.sample(subset_ids, int(len(subset_ids) * sampler)) return subset_ids
def prepare_data(exp_const, data_const): io.mkdir_if_not_exists(exp_const.exp_dir) print('Writing constants to exp dir ...') data_const_json = os.path.join(exp_const.exp_dir, 'data_const.json') data_const.to_json(data_const_json) exp_const_json = os.path.join(exp_const.exp_dir, 'exp_const.json') exp_const.to_json(exp_const_json) print('Loading anno_list.json ...') anno_list = io.load_json_object(data_const.anno_list_json) print('Creating input json for faster rcnn ...') images_in_out = [None] * len(anno_list) for i, anno in enumerate(anno_list): global_id = anno['global_id'] image_in_out = dict() image_in_out['in_path'] = os.path.join(data_const.images_dir, anno['image_path_postfix']) image_in_out['out_dir'] = os.path.join(data_const.proc_dir, 'faster_rcnn_boxes') image_in_out['prefix'] = f'{global_id}_' images_in_out[i] = image_in_out images_in_out_json = os.path.join(exp_const.exp_dir, 'faster_rcnn_im_in_out.json') io.dump_json_object(images_in_out, images_in_out_json)
def load_gt_dets(anno_list_json,global_ids): global_ids_set = set(global_ids) # Load anno_list print('Loading anno_list.json ...') anno_list = io.load_json_object(anno_list_json) gt_dets = {} for anno in anno_list: if anno['global_id'] not in global_ids_set: continue global_id = anno['global_id'] gt_dets[global_id] = {} for hoi in anno['hois']: hoi_id = hoi['id'] gt_dets[global_id][hoi_id] = [] for human_box_num, object_box_num in hoi['connections']: human_box = hoi['human_bboxes'][human_box_num] object_box = hoi['object_bboxes'][object_box_num] det = { 'human_box': human_box, 'object_box': object_box, } gt_dets[global_id][hoi_id].append(det) return gt_dets
def load_embeddings(self, labels): embed_h5py = io.load_h5py_object(self.const.embed_h5py)['embeddings'] word_to_idx = io.load_json_object(self.const.embed_word_to_idx_json) embeddings = np.zeros([len(labels), self.const.embed_dims]) word_to_label = {} for i, label in enumerate(labels): if ' ' in label: words = label.split(' ') elif '_' in label: words = label.split('_') else: words = [label] denom = len(words) for word in words: if word == 'tree': denom = len(words) - 1 continue if word not in word_to_label: word_to_label[word] = set() word_to_label[word].add(label) idx = word_to_idx[word] embeddings[i] += embed_h5py[idx][()] embeddings[i] /= denom if self.const.no_glove == True: embeddings[:, :self.const.glove_dim] = 0 self.embed.weight.data.copy_(torch.from_numpy(embeddings))
def main(exp_const, data_const): hoi_cands = h5py.File(data_const.hoi_cand_hdf5, 'r') box_feats_hdf5 = os.path.join( exp_const.exp_dir, f'hoi_candidates_box_feats_{exp_const.subset}.hdf5') box_feats = h5py.File(box_feats_hdf5, 'w') anno_list = io.load_json_object(data_const.anno_list_json) anno_dict = {anno['global_id']: anno for anno in anno_list} for global_id in tqdm(hoi_cands.keys()): img_hoi_cands = hoi_cands[global_id] human_boxes = img_hoi_cands['boxes_scores_rpn_ids_hoi_idx'][:, :4] object_boxes = img_hoi_cands['boxes_scores_rpn_ids_hoi_idx'][:, 4:8] # 查找无object的pair index = np.where( img_hoi_cands['boxes_scores_rpn_ids_hoi_idx'][:, 11] != -1)[0] chosen_humans = human_boxes[index] chosen_objects = object_boxes[index] img_size = anno_dict[global_id]['image_size'][:2] feats = compute_box_feats(chosen_humans, chosen_objects, img_size) features = np.zeros([human_boxes.shape[0], feats.shape[1]]) features[index] = feats box_feats.create_dataset(global_id, data=features) box_feats.close()
def main(**kwargs): exp_base_dir = coco_paths['exp_dir'] if kwargs['dataset'] == 'flickr': exp_base_dir = flickr_paths['exp_dir'] exp_const = ExpConstants(kwargs['exp_name'], exp_base_dir) exp_const.model_dir = os.path.join(exp_const.exp_dir, 'models') exp_const.seed = 0 exp_const.contextualize = not kwargs['no_context'] exp_const.random_lang = kwargs['random_lang'] data_const = FlickrDatasetConstants(kwargs['subset']) model_const = Constants() model_const.model_num = kwargs['model_num'] model_const.object_encoder = ObjectEncoderConstants() model_const.object_encoder.context_layer.output_attentions = True model_const.object_encoder.object_feature_dim = 2048 model_const.cap_encoder = CapEncoderConstants() model_const.cap_encoder.output_attentions = True model_const.cap_info_nce_layers = kwargs['cap_info_nce_layers'] if model_const.model_num == -100: filename = os.path.join(exp_const.exp_dir, f'results_val_best.json') results = io.load_json_object(filename) model_const.model_num = results['model_num'] print('Selected model num:', model_const.model_num) model_const.object_encoder_path = os.path.join( exp_const.model_dir, f'object_encoder_{model_const.model_num}') model_const.lang_sup_criterion_path = os.path.join( exp_const.model_dir, f'lang_sup_criterion_{model_const.model_num}') if exp_const.random_lang is True: model_const.cap_encoder_path = os.path.join( exp_const.model_dir, f'cap_encoder_{model_const.model_num}') eval_flickr_phrase_loc.main(exp_const, data_const, model_const)
def get_verb_to_id(self, verb_list_json): verb_list = io.load_json_object(verb_list_json) verb_to_id = { verb['verb'] + "_" + verb['role']: verb['id'] for verb in verb_list } return verb_to_id
def select(data_const): io.mkdir_if_not_exists(data_const.proc_dir) select_boxes_dir = data_const.proc_dir # Print where the boxes are coming from and where the output is written print(f'Boxes will be read from: {data_const.faster_rcnn_boxes}') print(f'Boxes will be written to: {select_boxes_dir}') print('Loading anno_list.json ...') anno_list = io.load_json_object(data_const.anno_list_json) print('Creating selected_coco_cls_dets.hdf5 file ...') # hdf5_file = os.path.join(select_boxes_dir,'selected_coco_cls_dets_0.1eval.hdf5') hdf5_file = os.path.join(select_boxes_dir, 'selected_coco_cls_dets.hdf5') f = h5py.File(hdf5_file, 'w') print('Selecting boxes ...') for anno in tqdm(anno_list): global_id = anno['global_id'] # # get more detection for evaluation # if 'test' in global_id: # data_const.human_score_thresh = 0.1 # data_const.object_score_thresh = 0.1 boxes_npy = os.path.join(data_const.faster_rcnn_boxes, f'{global_id}_boxes.npy') boxes = np.load(boxes_npy) scores_npy = os.path.join(data_const.faster_rcnn_boxes, f'{global_id}_scores.npy') scores = np.load(scores_npy) nms_keep_indices_json = os.path.join( data_const.faster_rcnn_boxes, f'{global_id}_nms_keep_indices.json') nms_keep_indices = io.load_json_object(nms_keep_indices_json) # import ipdb; ipdb.set_trace() selected_dets, start_end_ids = select_dets(boxes, scores, nms_keep_indices, data_const) f.create_group(global_id) f[global_id].create_dataset('boxes_scores_rpn_ids', data=selected_dets) f[global_id].create_dataset('start_end_ids', data=start_end_ids) f.close()
def load_subset_ids(self, subset): split_ids = io.load_json_object( "data/vcoco/annotations/split_ids.json") subset_list_ = split_ids[subset] subset_list = [] for id in subset_list_: subset_list.append(subset + "_" + id) return sorted(subset_list)
def get_wnid_and_img_paths(self): wnid_to_urls = io.load_json_object(self.const.wnid_to_urls_json) wnid_and_img_paths = [] for wnid, urls in wnid_to_urls.items(): for img_id, url in urls.items(): wnid_and_img_paths.append((wnid, url)) return wnid_and_img_paths
def assign(exp_const, data_const): io.mkdir_if_not_exists(exp_const.exp_dir) print('Saving constants ...') save_constants({'exp': exp_const, 'data':data_const}, exp_const.exp_dir) print(f'Reading hoi_candidates_{exp_const.subset}.hdf5 ...') hoi_cand_hdf5 = h5py.File(data_const.hoi_cand_hdf5, 'r') print(f'Creating hoi_candidate_labels_{exp_const.subset}.hdf5 ...') filename = os.path.join( exp_const.exp_dir, f'hoi_candidate_labels_{exp_const.subset}.hdf5') hoi_cand_label_hdf5 = h5py.File(filename, 'w') print('Loading gt hoi detections ...') split_ids = io.load_json_object(data_const.split_ids_json) global_ids = split_ids[exp_const.subset] gt_dets = load_gt_dets(data_const.anno_list_json, global_ids) print('Loading hoi_list.json ...') hoi_list = io.load_json_object(data_const.hoi_list_json) hoi_ids = [hoi['id'] for hoi in hoi_list] for global_id in tqdm(global_ids): boxes_scores_rpn_ids_hoi_idx = \ hoi_cand_hdf5[global_id]['boxes_scores_rpn_ids_hoi_idx'] start_end_ids = hoi_cand_hdf5[global_id]['start_end_ids'] num_cand = boxes_scores_rpn_ids_hoi_idx.shape[0] labels = np.zeros([num_cand]) for hoi_id in gt_dets[global_id]: start_id, end_id = start_end_ids[int(hoi_id)-1] for i in range(start_id, end_id): cand_det = { 'human_box': boxes_scores_rpn_ids_hoi_idx[i, :4], 'object_box': boxes_scores_rpn_ids_hoi_idx[i, 4:8], } # 查看检测结果ho候选对与gt中ho候选对的匹配情况,如果有所匹配(iou>0.5)则label置1 is_match = match_hoi(cand_det, gt_dets[global_id][hoi_id]) if is_match: labels[i] = 1.0 hoi_cand_label_hdf5.create_dataset(global_id, data=labels) hoi_cand_label_hdf5.close()
def main(): const = ImagenetConstants() io.mkdir_if_not_exists(const.img_dir) print('Loading urls ...') wnid_to_urls = io.load_json_object(const.wnid_to_urls_json) print('Starting pool ...') with Pool(40) as p: p.starmap(downloader,product([const.img_dir],wnid_to_urls.items()))
def read_samples(self,subset): if subset=='train': filename = self.const.train_json elif subset=='val': filename = self.const.val_json elif subset=='test': filename = self.const.truth_json else: msg = f'subset {subset} not supported' assert(False), msg return io.load_json_object(filename)
def main(exp_const, data_const, model_const): print('Loading pred dets ...') pred_hois = h5py.File(data_const.pred_hoi_dets_h5py, 'r') human_pose_feats = h5py.File(data_const.human_pose_feats_hdf5, 'r') print('Reading anno_list.json ...') anno_list = io.load_json_object(data_const.anno_list_json) anno_dict = {anno['global_id']: anno for anno in anno_list} print('Selecting top box configurations for each hoi ...') top_boxes = select_best_boxes_across_dataset(pred_hois, anno_dict, human_pose_feats, data_const, exp_const) hoi_list = io.load_json_object(data_const.hoi_list_json) hoi_dict = {hoi['id']: hoi for hoi in hoi_list} print('Creating visualization images ...') vis_dir = os.path.join(exp_const.exp_dir, 'vis/top_boxes_per_hoi') create_html(top_boxes, anno_dict, hoi_dict, data_const.images_dir, vis_dir)
def get_acc_data(acc_dir, iters): accs = [None] * len(iters) for i, it in enumerate(iters): results_json = os.path.join(acc_dir, f'results_val_{it}.json') if not os.path.exists(results_json): continue accs[i] = io.load_json_object(results_json)['pt_recall'] return accs
def __init__(self,const): super(SemEval201810Dataset,self).__init__() self.const = copy.deepcopy(const) self.embeddings = h5py.File(self.const.embeddings_h5py,'r') if self.const.random==True: n,d = self.embeddings['embeddings'].shape self.embeddings = { 'embeddings': 2*(np.random.rand(n,d)-0.5) } self.word_to_idx = io.load_json_object(self.const.word_to_idx_json) self.samples = self.read_samples(self.const.subset)
def main(**kwargs): print('Creating Caption Encoder (tokenizer) ...') cap_encoder = CapEncoder(CapEncoderConstants()) nltk.download('punkt') data_const = DetFeatDatasetConstants(kwargs['subset']) annos = io.load_json_object(data_const.annos_json)['annotations'] noun_token_ids = [None] * len(annos) noun_vocab = set() num_human_captions = 0 num_noun_captions = 0 for i, anno in enumerate(tqdm(annos)): image_id = anno['image_id'] cap_id = anno['id'] caption = anno['caption'] token_ids, tokens = cap_encoder.tokenize(caption) nltk_tokens = nltk.word_tokenize(caption.lower()) pos_tags = nltk.pos_tag(nltk_tokens) pos_tags = ignore_words_from_pos(pos_tags, ['is', 'has', 'have', 'had', 'be']) alignment = align_pos_tokens(pos_tags, tokens) noun_token_ids_, noun_words = get_noun_token_ids(pos_tags, alignment) noun_token_ids_ = group_token_ids(noun_token_ids_, tokens) if len(noun_token_ids_) > 0: num_noun_captions += 1 noun_token_ids[i] = { 'image_id': image_id, 'cap_id': cap_id, 'token_ids': noun_token_ids_, 'words': list(noun_words) } noun_vocab.update(noun_words) for human_word in [ 'man', 'person', 'human', 'woman', 'boy', 'girl', 'men', 'women', 'boys', 'girls', 'child', 'children' ]: if human_word in tokens: num_human_captions += 1 break io.dump_json_object(noun_token_ids, data_const.noun_tokens_json) io.dump_json_object(sorted(list(noun_vocab)), data_const.noun_vocab_json) print('Number of human captions:', num_human_captions) print('Number of noun captions:', num_noun_captions) print('Total number of captions:', len(annos)) print('Size of noun vocabulary:', len(noun_vocab))
def get_infonce_data(infonce_dir, layers): infonce_data = io.load_json_object( os.path.join(infonce_dir, f'infonce_{layers}_layer.json')) iters = [] losses = [] for time, it, loss in infonce_data: if it == 0: continue iters.append(it) losses.append(round(loss, 2)) return iters, losses
def main(): data_const = HicoConstants() anno_list = io.load_json_object(data_const.anno_list_json) global_ids = [anno['global_id'] for anno in anno_list] feats_hdf5 = os.path.join(data_const.proc_dir, 'faster_rcnn_fc7.hdf5') feats = h5py.File(feats_hdf5, 'w') for global_id in tqdm(global_ids): fc7_npy = os.path.join(data_const.faster_rcnn_boxes, f'{global_id}_fc7.npy') fc7 = np.load(fc7_npy) feats.create_dataset(global_id, data=fc7) feats.close()
def eval_hoi_ap(args, hoi_id, pred_dets_hdf5, global_ids_list, subset='test'): print('Creating output dir ...') io.mkdir_if_not_exists(args.out_dir, recursive=True) # Load hoi_list hoi_list_json = os.path.join(args.proc_dir, 'hoi_list.json') hoi_list = io.load_json_object(hoi_list_json) # Load subset ids to eval on split_ids_json = os.path.join(args.proc_dir, 'split_ids.json') split_ids = io.load_json_object(split_ids_json) global_ids = split_ids[subset] global_ids_set = set(global_ids) # Create gt_dets print('Creating GT dets ...') gt_dets = load_gt_dets(args.proc_dir, global_ids_set) ap, _ = eval_hoi_my(hoi_id, global_ids, gt_dets, pred_dets_hdf5, args.out_dir, global_ids_list) return ap
def generate(exp_const, data_const, data_sign): print(f'Creating exp_dir: {exp_const.exp_dir}') io.mkdir_if_not_exists(exp_const.exp_dir) save_constants({'exp': exp_const, 'data': data_const}, exp_const.exp_dir) print(f'Reading split_ids.json ...') split_ids = io.load_json_object(data_const.split_ids_json) print('Creating an object-detector-only HOI detector ...') hoi_cand_gen = HoiCandidatesGenerator(data_const, data_sign) print(f'Creating a hoi_candidates_{exp_const.subset}.hdf5 file ...') hoi_cand_hdf5 = os.path.join(exp_const.exp_dir, f'hoi_candidates_{exp_const.subset}.hdf5') f = h5py.File(hoi_cand_hdf5, 'w') # 从Faster RCNN的所有预测结果中选择的高分预测 print('Reading selected dets from hdf5 file ...') all_selected_dets = h5py.File(data_const.selected_dets_hdf5, 'r') for global_id in tqdm(split_ids[exp_const.subset]): selected_dets = { 'boxes': {}, 'scores': {}, 'rpn_ids': {}, 'obj_cls': {} } start_end_ids = all_selected_dets[global_id]['start_end_ids'][()] boxes_scores_rpn_ids = \ all_selected_dets[global_id]['boxes_scores_rpn_ids'][()] for cls_ind, cls_name in enumerate(COCO_CLASSES): start_id, end_id = start_end_ids[cls_ind] boxes = boxes_scores_rpn_ids[start_id:end_id, :4] scores = boxes_scores_rpn_ids[start_id:end_id, 4] rpn_ids = boxes_scores_rpn_ids[start_id:end_id, 5] object_cls = np.full((end_id - start_id, ), cls_ind) selected_dets['boxes'][cls_name] = boxes selected_dets['scores'][cls_name] = scores selected_dets['rpn_ids'][cls_name] = rpn_ids selected_dets['obj_cls'][cls_name] = object_cls pred_dets, start_end_ids = hoi_cand_gen.predict(selected_dets) f.create_group(global_id) f[global_id].create_dataset('boxes_scores_rpn_ids_hoi_idx', data=pred_dets) f[global_id].create_dataset('start_end_ids', data=start_end_ids) f.close()
def main(exp_const,data_const): io.mkdir_if_not_exists(exp_const.exp_dir) print('Reading anno_list.json ...') anno_list = io.load_json_object(data_const.anno_list_json) anno_dict = {anno['global_id']:anno for anno in anno_list} print('Reading box and pose features ...') human_pose_feats = h5py.File(data_const.human_pose_feats_h5py,'r') hoi_cand = h5py.File(data_const.hoi_cand_h5py,'r') for count,global_id in enumerate(tqdm(human_pose_feats.keys())): if count>=exp_const.max_count: break human_boxes = hoi_cand[global_id]['boxes_scores_rpn_ids_hoi_idx'][:,:4] human_rpn_ids = hoi_cand[global_id]['boxes_scores_rpn_ids_hoi_idx'][:,10] B = human_boxes.shape[0] absolute_pose = human_pose_feats[global_id]['absolute_pose'][()] absolute_pose = np.reshape(absolute_pose,(B,data_const.num_keypts,3)) x1y1 = human_boxes[:,:2] # Bx2 wh = 0*x1y1 # Bx2 wh[:,0] = (human_boxes[:,2] - human_boxes[:,0]) wh[:,1] = (human_boxes[:,3] - human_boxes[:,1]) x1y1 = np.tile(x1y1[:,np.newaxis,:],(1,data_const.num_keypts,1)) # Bx18x2 wh = np.tile(wh[:,np.newaxis,:],(1,data_const.num_keypts,1)) # Bx18x2 keypts = 0*absolute_pose keypts[:,:,:2] = absolute_pose[:,:,:2]*wh + x1y1 keypts[:,:,2] = absolute_pose[:,:,2] img_path = os.path.join( data_const.images_dir, anno_dict[global_id]['image_path_postfix']) img = skio.imread(img_path) if len(img.shape)==2: img = np.tile(img[:,:,np.newaxis],(1,1,3)) seen_rpn_ids = set() for i in range(B): rpn_id = human_rpn_ids[i] if rpn_id in seen_rpn_ids: continue else: seen_rpn_ids.add(rpn_id) img = bbox_utils.vis_human_keypts(img,keypts[i],modify=True) img_out_path = os.path.join( exp_const.exp_dir, f'{global_id}.png') skio.imsave(img_out_path,img)