def test_integration_single_timeout(self, mock_davis): dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS') with DavisInteractiveSession( davis_root=dataset_dir, subset='train', max_nb_interactions=None, max_time=1, report_save_dir=tempfile.mkdtemp()) as session: count = 0 while session.next(): seq, scribble, new_seq = session.get_scribbles(only_last=True) assert new_seq == (count == 0) assert seq == 'bear' with dataset_dir.joinpath('Scribbles', 'bear', '001.json').open() as fp: sc = json.load(fp) assert sc == scribble assert not is_empty(scribble) # Simulate model predicting masks pred_masks = np.zeros((2, 480, 854)) time.sleep(1.2) session.submit_masks(pred_masks) count += 1 assert count == 1 assert mock_davis.call_count == 0
def test_workflow(self, mock_service): mock_service.return_value = EvaluationService( 'train', storage=DBStorage, davis_root=os.path.join(settings.BASE_DIR, 'evaluation', 'test_data', 'DAVIS'), max_t=settings.EVALUATION_MAX_TIME, max_i=settings.EVALUATION_MAX_INTERACTIONS, time_threshold=settings.EVALUATION_TIME_THRESHOLD) RemoteConnector.VALID_SUBSETS.append('train') with DavisInteractiveSession( host='/', user_key='1234', subset='train', report_save_dir='/tmp') as session: while session.next(): sequence, _, _ = session.get_scribbles() nb_frames = Davis.dataset[sequence]['num_frames'] w, h = Davis.dataset[sequence]['image_size'] pred_masks = np.zeros((nb_frames, h, w), dtype=np.int) pred_masks[:, 50:250, 150:350] = 1 session.submit_masks(pred_masks) summary = session.get_global_summary() self.assertEqual(summary, {}) summary = session.get_global_summary() self.assertNotEqual(summary, {})
def test_report_folder_creation(self, mock_davis): dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS') tmp_dir = Path(tempfile.mkdtemp()) / 'test' assert not tmp_dir.exists() session = DavisInteractiveSession( davis_root=dataset_dir, subset='train', report_save_dir=tmp_dir) assert tmp_dir.exists() assert mock_davis.call_count == 0
def test_integration_single(self, mock_davis): dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS') tmp_dir = Path(tempfile.mkdtemp()) with DavisInteractiveSession( davis_root=dataset_dir, subset='train', max_nb_interactions=5, report_save_dir=tmp_dir, max_time=None) as session: count = 0 temp_csv = tmp_dir / ("%s.tmp.csv" % session.report_name) final_csv = tmp_dir / ("%s.csv" % session.report_name) while session.next(): assert not final_csv.exists() assert temp_csv.exists() df = pd.read_csv(temp_csv, index_col=0) assert df.shape == (count * 2, 10) seq, scribble, new_seq = session.get_scribbles(only_last=True) assert new_seq == (count == 0) assert seq == 'bear' if count == 0: with dataset_dir.joinpath('Scribbles', 'bear', '001.json').open() as fp: sc = json.load(fp) assert sc == scribble else: assert annotated_frames(scribble) == [1] assert not is_empty(scribble) assert len(scribble['scribbles']) == 2 assert len(scribble['scribbles'][1]) > 0 assert len(scribble['scribbles'][0]) == 0 # Simulate model predicting masks pred_masks = np.zeros((2, 480, 854)) session.submit_masks( pred_masks, next_scribble_frame_candidates=[1]) if count > 0: assert df.sequence.unique() == ['bear'] assert np.all(df.interaction.unique() == [i + 1 for i in range(count)]) assert np.all(df.object_id.unique() == [1]) count += 1 assert count == 5 assert final_csv.exists() assert not temp_csv.exists() assert mock_davis.call_count == 0
def test_shuffle(self, mock_davis): dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS') with DavisInteractiveSession( davis_root=dataset_dir, subset='train', shuffle=True, report_save_dir=tempfile.mkdtemp()) as session: assert ('bear', 1) in session.samples assert ('bear', 2) in session.samples assert ('tennis', 1) in session.samples assert mock_davis.call_count == 0
def test_integration_single_only_last(self, mock_davis): dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS') with DavisInteractiveSession( davis_root=dataset_dir, subset='train', max_nb_interactions=4, report_save_dir=tempfile.mkdtemp(), max_time=None) as session: count = 0 annotated_frames_list = [] while session.next(): seq, scribble, new_seq = session.get_scribbles(only_last=True) assert new_seq == (count == 0) assert seq == 'blackswan' if count == 0: with dataset_dir.joinpath('Scribbles', 'blackswan', '001.json').open() as fp: sc = json.load(fp) assert sc == scribble else: assert len(annotated_frames(scribble)) == 1 a_fr = annotated_frames(scribble)[0] assert a_fr not in annotated_frames_list annotated_frames_list.append(a_fr) assert not is_empty(scribble) # Simulate model predicting masks pred_masks = np.zeros((6, 480, 854)) session.submit_masks(pred_masks) count += 1 assert count == 4 assert mock_davis.call_count == 0
def test_interactions_limit(self, mock_start_session, mock_get_scribble, mock_get_report, mock_submit_masks): davis_root = '/tmp/DAVIS' with DavisInteractiveSession( davis_root=davis_root, max_nb_interactions=5, report_save_dir=tempfile.mkdtemp(), max_time=None) as session: assert mock_start_session.call_count == 1 for i in range(7): assert session.next() seq, scribbles, new_seq = session.get_scribbles() assert seq == 'bear' assert is_empty(scribbles) if i % 5 == 0: assert new_seq, i session.submit_masks(None) assert mock_get_scribble.call_count == 2 assert mock_submit_masks.call_count == 7
def test_subset(self, mock_davis): davis_root = '/tmp/DAVIS' session = DavisInteractiveSession(subset='val', davis_root=davis_root) session.__enter__() session = DavisInteractiveSession(subset='train', davis_root=davis_root) session.__enter__() session1 = DavisInteractiveSession( subset='test-dev', davis_root=davis_root) session2 = DavisInteractiveSession(subset='xxxx', davis_root=davis_root) with pytest.raises(ValueError): session1.__enter__() with pytest.raises(ValueError): session2.__enter__() assert mock_davis.call_count == 2
def test_integration_multiple_sequences(self, mock_davis): dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS') with DavisInteractiveSession( davis_root=dataset_dir, subset='train', max_nb_interactions=4, report_save_dir=tempfile.mkdtemp(), max_time=None) as session: count = 0 for seq, scribble, new_seq in session.scribbles_iterator(): assert new_seq == (count == 0 or count == 4 or count == 8) if count < 8: assert seq == 'bear', count else: assert seq == 'tennis', count if count == 0: with dataset_dir.joinpath('Scribbles', 'bear', '001.json').open() as fp: sc = json.load(fp) assert sc == scribble if count == 4: with dataset_dir.joinpath('Scribbles', 'bear', '002.json').open() as fp: sc = json.load(fp) assert sc == scribble if count == 8: with dataset_dir.joinpath('Scribbles', 'tennis', '001.json').open() as fp: sc = json.load(fp) assert sc == scribble assert not is_empty(scribble) # Simulate model predicting masks pred_masks = np.zeros((2, 480, 854)) session.submit_masks(pred_masks) count += 1 assert count == 12 df = session.get_report() assert mock_davis.call_count == 0 assert df.shape == (2 * 4 * 2 * 1 + 4 * 2 * 2, 8) assert np.all(df.jaccard == 0.) global_summary_file = os.path.join(tempfile.mkdtemp(), 'summary.json') summary = session.get_global_summary() self.assertFalse(os.path.exists(global_summary_file)) self.assertTrue('auc' in summary) self.assertTrue('jaccard_at_threshold' in summary) self.assertEqual(summary['jaccard_at_threshold']['threshold'], 60) self.assertTrue('curve' in summary) curve = summary['curve'] self.assertEqual(len(curve['jaccard']), 6) self.assertEqual(len(curve['time']), 6) summary = session.get_global_summary(save_file=global_summary_file) self.assertTrue(os.path.exists(global_summary_file))
def main(): # General parameters gpu_id = 0 # Configuration used in the challenges max_nb_interactions = 8 # Maximum number of interactions max_time_per_interaction = 30 # Maximum time per interaction per object # Total time available to interact with a sequence and an initial set of scribbles max_time = max_nb_interactions * max_time_per_interaction # Maximum time per object # Interactive parameters subset = 'val' host = 'localhost' # 'localhost' for subsets train and val. # ScribbleNet parameters time_budget_per_object = 20 parent_model = 'scribblenet_parent.pth' prev_mask = True # Use previous mask as no-care area when fine-tuning save_model_dir = Path.models_dir() report_save_dir = Path.save_root_dir() save_result_dir = report_save_dir model = ScribbleNetMain(parent_model, save_model_dir, gpu_id, time_budget_per_object, save_result_dir=save_result_dir) seen_seq = {} with DavisInteractiveSession(host=host, davis_root=Path.db_root_dir(), subset=subset, report_save_dir=report_save_dir, max_nb_interactions=max_nb_interactions, max_time=max_time) as sess: while sess.next(): t_total = timeit.default_timer() # Get the current iteration scribbles sequence, scribbles, first_scribble = sess.get_scribbles() if first_scribble: n_interaction = 1 n_objects = Davis.dataset[sequence]['num_objects'] first_frame = interactive_utils.scribbles.annotated_frames( scribbles)[0] seen_seq[sequence] = 1 if sequence not in seen_seq.keys( ) else seen_seq[sequence] + 1 else: n_interaction += 1 pred_masks = [] print( '\nRunning sequence {} in interaction {} and scribble iteration {}' .format(sequence, n_interaction, seen_seq[sequence])) for obj_id in range(1, n_objects + 1): model.train(first_frame, n_interaction, obj_id, scribbles, seen_seq[sequence], subset=subset, use_previous_mask=prev_mask) pred_masks.append( model.test(sequence, n_interaction, obj_id, subset=subset, scribble_iter=seen_seq[sequence])) final_masks = interactive_utils.mask.combine_masks(pred_masks) # Submit your prediction sess.submit_masks(final_masks) t_end = timeit.default_timer() print( 'Total time (training and testing) for single interaction: ' + str(t_end - t_total)) # Get the DataFrame report report = sess.get_report() # Get the global summary summary = sess.get_global_summary( save_file=os.path.join(report_save_dir, 'summary.json'))
def main(): total_frame_num_dic={} ################# seqs = [] with open(os.path.join(cfg.DATA_ROOT, 'ImageSets', '2017', 'val' + '.txt')) as f: seqs_tmp = f.readlines() seqs_tmp = list(map(lambda elem: elem.strip(), seqs_tmp)) seqs.extend(seqs_tmp) for seq_name in seqs: images = np.sort(os.listdir(os.path.join(cfg.DATA_ROOT, 'JPEGImages/480p/', seq_name.strip()))) total_frame_num_dic[seq_name]=len(images) _seq_list_file=os.path.join(cfg.DATA_ROOT, 'ImageSets', '2017', 'v_a_l' + '_instances.txt') seq_dict = json.load(open(_seq_list_file, 'r')) ################## is_save_image=True report_save_dir= cfg.RESULT_ROOT save_res_dir = './saved_model_inter_net_new_re' # Configuration used in the challenges max_nb_interactions = 8 # Maximum number of interactions max_time_per_interaction = 10000 # Maximum time per interaction per object # Total time available to interact with a sequence and an initial set of scribbles max_time = max_nb_interactions * max_time_per_interaction # Maximum time per object # Interactive parameters subset = 'val' host = 'localhost' # 'localhost' for subsets train and val. feature_extracter = DeepLab(backbone='resnet',freeze_bn=False) model = IntVOS(cfg,feature_extracter) model= model.cuda() print('model loading...') saved_model_dict = os.path.join(save_res_dir,'save_step_80000.pth') pretrained_dict = torch.load(saved_model_dict) load_network(model,pretrained_dict) print('model loading finished!') model.eval() seen_seq={} with torch.no_grad(): with DavisInteractiveSession(host=host, davis_root=cfg.DATA_ROOT, subset=subset, report_save_dir=report_save_dir, max_nb_interactions=max_nb_interactions, max_time=max_time, metric_to_optimize='J' ) as sess: while sess.next(): t_total=timeit.default_timer() # Get the current iteration scribbles sequence, scribbles, first_scribble = sess.get_scribbles(only_last=True) print(sequence) start_annotated_frame=annotated_frames(scribbles)[0] pred_masks=[] pred_masks_reverse=[] if first_scribble: anno_frame_list=[] n_interaction=1 eval_global_map_tmp_dic={} local_map_dics=({},{}) total_frame_num=total_frame_num_dic[sequence] obj_nums = seq_dict[sequence][-1] eval_data_manager=DAVIS2017_Test_Manager(split='val',root=cfg.DATA_ROOT,transform=tr.ToTensor(), seq_name=sequence) else: n_interaction+=1 ##########################Reference image process scr_f = start_annotated_frame anno_frame_list.append(start_annotated_frame) print(start_annotated_frame) scr_f = str(scr_f) while len(scr_f)!=5: scr_f='0'+scr_f ref_img = os.path.join(cfg.DATA_ROOT,'JPEGImages/480p',sequence,scr_f+'.jpg') ref_img = cv2.imread(ref_img) h_,w_ = ref_img.shape[:2] ref_img = np.array(ref_img,dtype=np.float32) #ref_img = tr.ToTensor()(ref_img) #ref_img = ref_img.unsqueeze(0) scribble_masks=scribbles2mask(scribbles,(h_,w_)) scribble_label=scribble_masks[start_annotated_frame] sample = {'ref_img':ref_img,'scribble_label':scribble_label} sample = tr.ToTensor()(sample) ref_img = sample['ref_img'] scribble_label = sample['scribble_label'] ref_img = ref_img.unsqueeze(0) scribble_label =scribble_label.unsqueeze(0) ref_img= ref_img.cuda() scribble_label = scribble_label.cuda() ###### ref_scribble_to_show = scribble_label.cpu().squeeze().numpy() im_ = Image.fromarray(ref_scribble_to_show.astype('uint8')).convert('P') im_.putpalette(_palette) ref_img_name= scr_f if not os.path.exists(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction))): os.makedirs(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction))) im_.save(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction),'inter_'+ref_img_name+'.png')) #ref_frame_embedding = model.extract_feature(ref_img) if first_scribble: ref_frame_embedding = model.extract_feature(ref_img) _,channel,emb_h,emb_w = ref_frame_embedding.size() embedding_memory=torch.zeros((total_frame_num,channel,emb_h,emb_w)) embedding_memory = embedding_memory.cuda() embedding_memory[start_annotated_frame]=ref_frame_embedding[0] else: ref_frame_embedding = embedding_memory[start_annotated_frame] ref_frame_embedding = ref_frame_embedding.unsqueeze(0) ######## if first_scribble: prev_label= None else: prev_label = os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction-1),scr_f+'.png') prev_label = Image.open(prev_label) prev_label = np.array(prev_label,dtype=np.uint8) prev_label = tr.ToTensor()({'label':prev_label}) prev_label = prev_label['label'].unsqueeze(0) prev_label = prev_label.cuda() ############### #tmp_dic, eval_global_map_tmp_dic= model.before_seghead_process(ref_frame_embedding,ref_frame_embedding, # ref_frame_embedding,scribble_label,prev_label, # normalize_nearest_neighbor_distances=True,use_local_map=False, # seq_names=[sequence],gt_ids=torch.Tensor([obj_nums]),k_nearest_neighbors=cfg.KNNS, # global_map_tmp_dic=eval_global_map_tmp_dic,frame_num=[start_annotated_frame],dynamic_seghead=model.dynamic_seghead) tmp_dic,local_map_dics = model.int_seghead(ref_frame_embedding=ref_frame_embedding,ref_scribble_label=scribble_label,prev_round_label=prev_label, global_map_tmp_dic=eval_global_map_tmp_dic,local_map_dics=local_map_dics,interaction_num=n_interaction, seq_names=[sequence],gt_ids=torch.Tensor([obj_nums]),frame_num=[start_annotated_frame],first_inter=first_scribble) pred_label = tmp_dic[sequence] pred_label = nn.functional.interpolate(pred_label,size=(h_,w_),mode = 'bilinear',align_corners=True) pred_label=torch.argmax(pred_label,dim=1) pred_masks.append(pred_label.float()) #### if is_save_image: pred_label_to_save=pred_label.squeeze(0).cpu().numpy() im = Image.fromarray(pred_label_to_save.astype('uint8')).convert('P') im.putpalette(_palette) imgname = str(start_annotated_frame) while len(imgname)<5: imgname='0'+imgname if not os.path.exists(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction))): os.makedirs(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction))) im.save(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction),imgname+'.png')) ####################################### ############################## ref_prev_label = pred_label.unsqueeze(0) prev_label = pred_label.unsqueeze(0) prev_img = ref_img prev_embedding = ref_frame_embedding for ii in range(start_annotated_frame+1,total_frame_num): print('evaluating sequence:{} frame:{}'.format(sequence,ii)) sample = eval_data_manager.get_image(ii) img = sample['img'] img = img.unsqueeze(0) _,_,h,w = img.size() img = img.cuda() # current_embedding= model.extract_feature(img) if first_scribble: current_embedding= model.extract_feature(img) embedding_memory[ii]=current_embedding[0] else: current_embedding=embedding_memory[ii] current_embedding = current_embedding.unsqueeze(0) prev_img =prev_img.cuda() prev_label =prev_label.cuda() tmp_dic, eval_global_map_tmp_dic,local_map_dics= model.before_seghead_process(ref_frame_embedding,prev_embedding, current_embedding,scribble_label,prev_label, normalize_nearest_neighbor_distances=True,use_local_map=True, seq_names=[sequence],gt_ids=torch.Tensor([obj_nums]),k_nearest_neighbors=cfg.KNNS, global_map_tmp_dic=eval_global_map_tmp_dic,local_map_dics=local_map_dics, interaction_num=n_interaction,start_annotated_frame=start_annotated_frame, frame_num=[ii],dynamic_seghead=model.dynamic_seghead) pred_label = tmp_dic[sequence] pred_label = nn.functional.interpolate(pred_label,size=(h,w),mode = 'bilinear',align_corners=True) pred_label=torch.argmax(pred_label,dim=1) pred_masks.append(pred_label.float()) prev_label = pred_label.unsqueeze(0) prev_img = img prev_embedding = current_embedding #### if is_save_image: pred_label_to_save=pred_label.squeeze(0).cpu().numpy() im = Image.fromarray(pred_label_to_save.astype('uint8')).convert('P') im.putpalette(_palette) imgname = str(ii) while len(imgname)<5: imgname='0'+imgname if not os.path.exists(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction))): os.makedirs(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction))) im.save(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction),imgname+'.png')) ####################################### prev_label = ref_prev_label prev_img = ref_img prev_embedding = ref_frame_embedding ###################################### for ii in range(start_annotated_frame): current_frame_num=start_annotated_frame-1-ii print('evaluating sequence:{} frame:{}'.format(sequence,current_frame_num)) sample = eval_data_manager.get_image(current_frame_num) img = sample['img'] img = img.unsqueeze(0) _,_,h,w = img.size() img = img.cuda() #current_embedding= model.extract_feature(img) if first_scribble: current_embedding= model.extract_feature(img) embedding_memory[current_frame_num]=current_embedding[0] else: current_embedding = embedding_memory[current_frame_num] current_embedding = current_embedding.unsqueeze(0) prev_img =prev_img.cuda() prev_label =prev_label.cuda() tmp_dic, eval_global_map_tmp_dic,local_map_dics= model.before_seghead_process(ref_frame_embedding,prev_embedding, current_embedding,scribble_label,prev_label, normalize_nearest_neighbor_distances=True,use_local_map=True, seq_names=[sequence],gt_ids=torch.Tensor([obj_nums]),k_nearest_neighbors=cfg.KNNS, global_map_tmp_dic=eval_global_map_tmp_dic,local_map_dics=local_map_dics,interaction_num=n_interaction,start_annotated_frame=start_annotated_frame,frame_num=[current_frame_num],dynamic_seghead=model.dynamic_seghead) pred_label = tmp_dic[sequence] pred_label = nn.functional.interpolate(pred_label,size=(h,w),mode = 'bilinear',align_corners=True) pred_label=torch.argmax(pred_label,dim=1) pred_masks_reverse.append(pred_label.float()) prev_label = pred_label.unsqueeze(0) prev_img = img prev_embedding = current_embedding #### if is_save_image: pred_label_to_save=pred_label.squeeze(0).cpu().numpy() im = Image.fromarray(pred_label_to_save.astype('uint8')).convert('P') im.putpalette(_palette) imgname = str(current_frame_num) while len(imgname)<5: imgname='0'+imgname if not os.path.exists(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction))): os.makedirs(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction))) im.save(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction),imgname+'.png')) pred_masks_reverse.reverse() pred_masks_reverse.extend(pred_masks) final_masks = torch.cat(pred_masks_reverse,0) sess.submit_masks(final_masks.cpu().numpy()) t_end = timeit.default_timer() print('Total time for single interaction: ' + str(t_end - t_total)) report = sess.get_report() summary = sess.get_global_summary(save_file=os.path.join(report_save_dir, 'summary.json'))
save_model_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'models_test') report_save_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'results_test') save_result_dir = report_save_dir # 'None' to not save the results model = OsvosScribble(parent_model, save_model_dir, gpu_id, time_budget_per_object, save_result_dir=save_result_dir) seen_seq = {} with DavisInteractiveSession(host=host, davis_root=Path.db_root_dir(), subset=subset, report_save_dir=report_save_dir, user_key=key) as sess: while sess.next(): # Get the current iteration scribbles sequence, scribbles, first_scribble = sess.get_scribbles() if first_scribble: n_interaction = 1 n_objects = Davis.dataset[sequence]['num_objects'] first_frame = interactive_utils.scribbles.annotated_frames( scribbles)[0] seen_seq[sequence] = 1 if sequence not in seen_seq.keys( ) else seen_seq[sequence] + 1 else: n_interaction += 1 pred_masks = []
return np.zeros([len(files), img_height, img_width]), len(files) pass def compute_req_indices(scribbles, nb_frame): index = 0 for i, l_obj in enumerate(scribbles['scribbles']): if len(l_obj) > 0: index = i break return [(index + int(np.ceil(nb_frame/8)) * i) % nb_frame for i in range(8)] pass with DavisInteractiveSession(host='https://server.davischallenge.org', user_key='ed3db2783547b034398e89f6163b955dc1a0135b33d5ae346ce61aa8dcf3d356', # ntan # user_key='fae1b878c12cec0857a64b13154a45cd9dd4be1f450a0064e4a2fde32020f144', # nhdang # user_key='d028a4c4bc13459359b63c9d948e91e95c1998c1a474cac07cdbfd4be811dadc', # qckan davis_root='/home/ntan/DAVIS2020/data/test-dev', subset='test-dev', max_nb_interactions=max_nb_interactions, max_time=max_time) as sess: while sess.next(): st = time.time() # Get the current interaction's scribbles sequence, scribbles, _ = sess.get_scribbles() saved_masks = glob.glob('output/masks_test-dev_chiet/%s_*'%(sequence)) saved_masks.sort() if len(saved_masks) > 0: pred_masks = np.load(saved_masks[-1]) sess.submit_masks(pred_masks) print('submitted...') continue
def run_IVOS(self, metric): seen_seq = {} numseq, tmpseq = 0, '' with open(self.save_csvsummary_dir, mode='a') as csv_file: writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) writer.writerow(['sequence', 'obj_idx', 'scr_idx'] + [ 'round-' + str(i + 1) for i in range(self.max_nb_interactions) ]) with DavisInteractiveSession( host=self.config.test_host, user_key=self.config.test_userkey, davis_root=self.config.davis_dataset_dir, subset=self.config.test_subset, report_save_dir=self.save_res_dir, max_nb_interactions=self.max_nb_interactions, max_time=self.max_time, metric_to_optimize=metric) as sess: sess.connector.service.robot.min_nb_nodes = self.config.test_min_nb_nodes sess.samples = self.scr_samples while sess.next(): # Get the current iteration scribbles self.sequence, scribbles, first_scribble = sess.get_scribbles( only_last=False) if first_scribble: anno_dict = { 'frames': [], 'annotated_masks': [], 'masks_tobe_modified': [] } n_interaction = 1 info = Davis.dataset[self.sequence] self.img_size = info['image_size'][::-1] self.num_frames = info['num_frames'] self.n_objects = info['num_objects'] info = None seen_seq[ self. sequence] = 1 if self.sequence not in seen_seq.keys( ) else seen_seq[self.sequence] + 1 scr_id = seen_seq[self.sequence] self.final_masks = np.zeros( [self.num_frames, self.img_size[0], self.img_size[1]]) self.pad_info = utils_custom.apply_pad( self.final_masks[0])[1] self.hpad1, self.wpad1 = self.pad_info[0][ 0], self.pad_info[1][0] self.hpad2, self.wpad2 = self.pad_info[0][ 1], self.pad_info[1][1] self.h_ds, self.w_ds = int( (self.img_size[0] + sum(self.pad_info[0])) / 4), int( (self.img_size[1] + sum(self.pad_info[1])) / 4) self.anno_6chEnc_r4_list = [] self.anno_3chEnc_r4_list = [] self.prob_map_of_frames = torch.zeros( (self.num_frames, self.n_objects + 1, 4 * self.h_ds, 4 * self.w_ds)).cuda() self.gt_masks = self.Davisclass.load_annotations( self.sequence) self.scores_ni_nf = np.zeros([8, self.num_frames]) IoU_over_eobj = [] else: n_interaction += 1 self.save_logger.printNlog( '\nRunning sequence {} in (scribble index: {}) (round: {})' .format(self.sequence, sess.samples[sess.sample_idx][1], n_interaction)) annotated_now = interactive_utils.scribbles.annotated_frames( sess.sample_last_scribble)[0] anno_dict['frames'].append( annotated_now) # Where we save annotated frames anno_dict['masks_tobe_modified'].append( self.final_masks[annotated_now] ) # mask before modefied at the annotated frame # Get Predicted mask & Mask decision from pred_mask self.final_masks = self.run_VOS_singleiact( n_interaction, scribbles, anno_dict['frames']) # self.final_mask changes if self.config.test_save_pngs_option: utils_custom.mkdir( os.path.join( self.save_res_dir, 'result_video', '{}-scr{:02d}/round{:02d}'.format( self.sequence, scr_id, n_interaction))) for fr in range(self.num_frames): savefname = os.path.join( self.save_res_dir, 'result_video', '{}-scr{:02d}/round{:02d}'.format( self.sequence, scr_id, n_interaction), '{:05d}.png'.format(fr)) tmpPIL = Image.fromarray( self.final_masks[fr].astype(np.uint8), 'P') tmpPIL.putpalette(self._palette) tmpPIL.save(savefname) # Limit candidate frames if n_interaction != self.max_nb_interactions: self.scores_ni_nf[n_interaction] = self.scores_ni_nf[ n_interaction - 1] current_score_np = self.scores_ni_nf[n_interaction - 1] if self.config.test_guide_method == 'RS1': next_scribble_frame_candidates = list( np.argsort(current_score_np)[:1]) elif self.config.test_guide_method == 'RS4': sorted_score_idx = np.argsort(current_score_np) exclude_range = self.num_frames / 10 excluded_next_candidates = [] next_scribble_frame_candidates = [] for i in range(self.num_frames): if not sorted_score_idx[i] in excluded_next_candidates: next_scribble_frame_candidates.append( sorted_score_idx[i]) excluded_next_candidates += list( range( int(sorted_score_idx[i] - (exclude_range / 2) + 0.5), int(sorted_score_idx[i] + (exclude_range / 2) + 0.5))) if len(next_scribble_frame_candidates) == 4: break elif self.config.test_guide_method == 'wo_RS': next_scribble_frame_candidates = None else: raise NotImplementedError # Submit your prediction sess.submit_masks(self.final_masks, next_scribble_frame_candidates) # F, H, W # print sequence name if tmpseq != self.sequence: tmpseq, numseq = self.sequence, numseq + 1 print( str(numseq) + ':' + str(self.sequence) + '-' + str(seen_seq[self.sequence]) + '\n') ## Visualizers and Saver # IoU estimation jaccard = batched_jaccard( self.gt_masks, self.final_masks, average_over_objects=False, nb_objects=self.n_objects) # frames, objid IoU_over_eobj.append(jaccard) # save final mask in anno_dict anno_dict['annotated_masks'].append( self.final_masks[annotated_now] ) # mask after modefied at the annotated frame if self.max_nb_interactions == len( anno_dict['frames'] ): # After Lastround -> total 90 iter seq_scrid_name = self.sequence + str(scr_id) # IoU manager IoU_over_eobj = np.stack(IoU_over_eobj, axis=0) # niact,frames,n_obj IoUeveryround_perobj = np.mean(IoU_over_eobj, axis=1) # niact,n_obj # show scribble input and output savefiledir = os.path.join(self.save_res_dir, 'debug_scribble') utils_custom.mkdir(savefiledir) savefilename = os.path.join(savefiledir, seq_scrid_name + '.jpg') utils_visualize.visualize_scrib_interaction( scribbles, anno_dict, self.sequence, savefilename) # plot IoU if self.config.test_save_pngs_option: savefiledir = os.path.join(self.save_res_dir, 'plot_IoU_perObj') utils_custom.mkdir(savefiledir) for obj_idx in range(self.n_objects): savefilename = os.path.join( savefiledir, seq_scrid_name + '-obj' + str(obj_idx + 1) + '_first{:03d}final{:03d}.png'.format( int(1000 * IoUeveryround_perobj[0, obj_idx]), int(1000 * IoUeveryround_perobj[-1, obj_idx]))) utils_visualize.visualize_interactionIoU( IoU_over_eobj[:, :, obj_idx], seq_scrid_name + '-obj' + str(obj_idx + 1), anno_dict['frames'], save_dir=savefilename, show_propagated_region=True) # write csv for obj_idx in range(self.n_objects): with open(self.save_csvsummary_dir, mode='a') as csv_file: writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) writer.writerow( [self.sequence, str(obj_idx + 1), str(scr_id)] + list(IoUeveryround_perobj[:, obj_idx])) summary = sess.get_global_summary(save_file=self.save_res_dir + '/summary_' + sess.report_name[7:] + '.json') analyze_summary(self.save_res_dir + '/summary_' + sess.report_name[7:] + '.json', metric=metric) fps = self.n_operated_frames_accum / self.total_taken_times_accum self.save_logger.printNlog('n_operated_frames_accum={}'.format( str(self.n_operated_frames_accum))) self.save_logger.printNlog('total_taken_times_accum={}'.format( str(self.total_taken_times_accum))) self.save_logger.printNlog('fps={}'.format(str(fps))) # final_IOU = summary['curve'][metric][-1] average_IoU_per_round = summary['curve'][metric][1:-1] torch.cuda.empty_cache() model = None return average_IoU_per_round