def test_integration_single_timeout(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=None,
max_time=1,
report_save_dir=tempfile.mkdtemp()) as session:
count = 0
while session.next():
seq, scribble, new_seq = session.get_scribbles(only_last=True)
assert new_seq == (count == 0)
assert seq == 'bear'
with dataset_dir.joinpath('Scribbles', 'bear',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
assert not is_empty(scribble)
# Simulate model predicting masks
pred_masks = np.zeros((2, 480, 854))
time.sleep(1.2)
session.submit_masks(pred_masks)
count += 1
assert count == 1
assert mock_davis.call_count == 0
def test_workflow(self, mock_service):
mock_service.return_value = EvaluationService(
'train',
storage=DBStorage,
davis_root=os.path.join(settings.BASE_DIR, 'evaluation',
'test_data', 'DAVIS'),
max_t=settings.EVALUATION_MAX_TIME,
max_i=settings.EVALUATION_MAX_INTERACTIONS,
time_threshold=settings.EVALUATION_TIME_THRESHOLD)
RemoteConnector.VALID_SUBSETS.append('train')
with DavisInteractiveSession(
host='/', user_key='1234', subset='train',
report_save_dir='/tmp') as session:
while session.next():
sequence, _, _ = session.get_scribbles()
nb_frames = Davis.dataset[sequence]['num_frames']
w, h = Davis.dataset[sequence]['image_size']
pred_masks = np.zeros((nb_frames, h, w), dtype=np.int)
pred_masks[:, 50:250, 150:350] = 1
session.submit_masks(pred_masks)
summary = session.get_global_summary()
self.assertEqual(summary, {})
summary = session.get_global_summary()
self.assertNotEqual(summary, {})
def test_report_folder_creation(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
tmp_dir = Path(tempfile.mkdtemp()) / 'test'
assert not tmp_dir.exists()
session = DavisInteractiveSession(
davis_root=dataset_dir, subset='train', report_save_dir=tmp_dir)
assert tmp_dir.exists()
assert mock_davis.call_count == 0
def test_integration_single(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
tmp_dir = Path(tempfile.mkdtemp())
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=5,
report_save_dir=tmp_dir,
max_time=None) as session:
count = 0
temp_csv = tmp_dir / ("%s.tmp.csv" % session.report_name)
final_csv = tmp_dir / ("%s.csv" % session.report_name)
while session.next():
assert not final_csv.exists()
assert temp_csv.exists()
df = pd.read_csv(temp_csv, index_col=0)
assert df.shape == (count * 2, 10)
seq, scribble, new_seq = session.get_scribbles(only_last=True)
assert new_seq == (count == 0)
assert seq == 'bear'
if count == 0:
with dataset_dir.joinpath('Scribbles', 'bear',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
else:
assert annotated_frames(scribble) == [1]
assert not is_empty(scribble)
assert len(scribble['scribbles']) == 2
assert len(scribble['scribbles'][1]) > 0
assert len(scribble['scribbles'][0]) == 0
# Simulate model predicting masks
pred_masks = np.zeros((2, 480, 854))
session.submit_masks(
pred_masks, next_scribble_frame_candidates=[1])
if count > 0:
assert df.sequence.unique() == ['bear']
assert np.all(df.interaction.unique() ==
[i + 1 for i in range(count)])
assert np.all(df.object_id.unique() == [1])
count += 1
assert count == 5
assert final_csv.exists()
assert not temp_csv.exists()
assert mock_davis.call_count == 0
def test_shuffle(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
shuffle=True,
report_save_dir=tempfile.mkdtemp()) as session:
assert ('bear', 1) in session.samples
assert ('bear', 2) in session.samples
assert ('tennis', 1) in session.samples
assert mock_davis.call_count == 0
def test_integration_single_only_last(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=4,
report_save_dir=tempfile.mkdtemp(),
max_time=None) as session:
count = 0
annotated_frames_list = []
while session.next():
seq, scribble, new_seq = session.get_scribbles(only_last=True)
assert new_seq == (count == 0)
assert seq == 'blackswan'
if count == 0:
with dataset_dir.joinpath('Scribbles', 'blackswan',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
else:
assert len(annotated_frames(scribble)) == 1
a_fr = annotated_frames(scribble)[0]
assert a_fr not in annotated_frames_list
annotated_frames_list.append(a_fr)
assert not is_empty(scribble)
# Simulate model predicting masks
pred_masks = np.zeros((6, 480, 854))
session.submit_masks(pred_masks)
count += 1
assert count == 4
assert mock_davis.call_count == 0
def test_interactions_limit(self, mock_start_session, mock_get_scribble,
mock_get_report, mock_submit_masks):
davis_root = '/tmp/DAVIS'
with DavisInteractiveSession(
davis_root=davis_root,
max_nb_interactions=5,
report_save_dir=tempfile.mkdtemp(),
max_time=None) as session:
assert mock_start_session.call_count == 1
for i in range(7):
assert session.next()
seq, scribbles, new_seq = session.get_scribbles()
assert seq == 'bear'
assert is_empty(scribbles)
if i % 5 == 0:
assert new_seq, i
session.submit_masks(None)
assert mock_get_scribble.call_count == 2
assert mock_submit_masks.call_count == 7
def test_subset(self, mock_davis):
davis_root = '/tmp/DAVIS'
session = DavisInteractiveSession(subset='val', davis_root=davis_root)
session.__enter__()
session = DavisInteractiveSession(subset='train', davis_root=davis_root)
session.__enter__()
session1 = DavisInteractiveSession(
subset='test-dev', davis_root=davis_root)
session2 = DavisInteractiveSession(subset='xxxx', davis_root=davis_root)
with pytest.raises(ValueError):
session1.__enter__()
with pytest.raises(ValueError):
session2.__enter__()
assert mock_davis.call_count == 2
def test_integration_multiple_sequences(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=4,
report_save_dir=tempfile.mkdtemp(),
max_time=None) as session:
count = 0
for seq, scribble, new_seq in session.scribbles_iterator():
assert new_seq == (count == 0 or count == 4 or count == 8)
if count < 8:
assert seq == 'bear', count
else:
assert seq == 'tennis', count
if count == 0:
with dataset_dir.joinpath('Scribbles', 'bear',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
if count == 4:
with dataset_dir.joinpath('Scribbles', 'bear',
'002.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
if count == 8:
with dataset_dir.joinpath('Scribbles', 'tennis',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
assert not is_empty(scribble)
# Simulate model predicting masks
pred_masks = np.zeros((2, 480, 854))
session.submit_masks(pred_masks)
count += 1
assert count == 12
df = session.get_report()
assert mock_davis.call_count == 0
assert df.shape == (2 * 4 * 2 * 1 + 4 * 2 * 2, 8)
assert np.all(df.jaccard == 0.)
global_summary_file = os.path.join(tempfile.mkdtemp(), 'summary.json')
summary = session.get_global_summary()
self.assertFalse(os.path.exists(global_summary_file))
self.assertTrue('auc' in summary)
self.assertTrue('jaccard_at_threshold' in summary)
self.assertEqual(summary['jaccard_at_threshold']['threshold'], 60)
self.assertTrue('curve' in summary)
curve = summary['curve']
self.assertEqual(len(curve['jaccard']), 6)
self.assertEqual(len(curve['time']), 6)
summary = session.get_global_summary(save_file=global_summary_file)
self.assertTrue(os.path.exists(global_summary_file))
def main():
# General parameters
gpu_id = 0
# Configuration used in the challenges
max_nb_interactions = 8 # Maximum number of interactions
max_time_per_interaction = 30 # Maximum time per interaction per object
# Total time available to interact with a sequence and an initial set of scribbles
max_time = max_nb_interactions * max_time_per_interaction # Maximum time per object
# Interactive parameters
subset = 'val'
host = 'localhost' # 'localhost' for subsets train and val.
# ScribbleNet parameters
time_budget_per_object = 20
parent_model = 'scribblenet_parent.pth'
prev_mask = True # Use previous mask as no-care area when fine-tuning
save_model_dir = Path.models_dir()
report_save_dir = Path.save_root_dir()
save_result_dir = report_save_dir
model = ScribbleNetMain(parent_model,
save_model_dir,
gpu_id,
time_budget_per_object,
save_result_dir=save_result_dir)
seen_seq = {}
with DavisInteractiveSession(host=host,
davis_root=Path.db_root_dir(),
subset=subset,
report_save_dir=report_save_dir,
max_nb_interactions=max_nb_interactions,
max_time=max_time) as sess:
while sess.next():
t_total = timeit.default_timer()
# Get the current iteration scribbles
sequence, scribbles, first_scribble = sess.get_scribbles()
if first_scribble:
n_interaction = 1
n_objects = Davis.dataset[sequence]['num_objects']
first_frame = interactive_utils.scribbles.annotated_frames(
scribbles)[0]
seen_seq[sequence] = 1 if sequence not in seen_seq.keys(
) else seen_seq[sequence] + 1
else:
n_interaction += 1
pred_masks = []
print(
'\nRunning sequence {} in interaction {} and scribble iteration {}'
.format(sequence, n_interaction, seen_seq[sequence]))
for obj_id in range(1, n_objects + 1):
model.train(first_frame,
n_interaction,
obj_id,
scribbles,
seen_seq[sequence],
subset=subset,
use_previous_mask=prev_mask)
pred_masks.append(
model.test(sequence,
n_interaction,
obj_id,
subset=subset,
scribble_iter=seen_seq[sequence]))
final_masks = interactive_utils.mask.combine_masks(pred_masks)
# Submit your prediction
sess.submit_masks(final_masks)
t_end = timeit.default_timer()
print(
'Total time (training and testing) for single interaction: ' +
str(t_end - t_total))
# Get the DataFrame report
report = sess.get_report()
# Get the global summary
summary = sess.get_global_summary(
save_file=os.path.join(report_save_dir, 'summary.json'))
def main():
total_frame_num_dic={}
#################
seqs = []
with open(os.path.join(cfg.DATA_ROOT, 'ImageSets', '2017', 'val' + '.txt')) as f:
seqs_tmp = f.readlines()
seqs_tmp = list(map(lambda elem: elem.strip(), seqs_tmp))
seqs.extend(seqs_tmp)
for seq_name in seqs:
images = np.sort(os.listdir(os.path.join(cfg.DATA_ROOT, 'JPEGImages/480p/', seq_name.strip())))
total_frame_num_dic[seq_name]=len(images)
_seq_list_file=os.path.join(cfg.DATA_ROOT, 'ImageSets', '2017',
'v_a_l' + '_instances.txt')
seq_dict = json.load(open(_seq_list_file, 'r'))
##################
is_save_image=True
report_save_dir= cfg.RESULT_ROOT
save_res_dir = './saved_model_inter_net_new_re'
# Configuration used in the challenges
max_nb_interactions = 8 # Maximum number of interactions
max_time_per_interaction = 10000 # Maximum time per interaction per object
# Total time available to interact with a sequence and an initial set of scribbles
max_time = max_nb_interactions * max_time_per_interaction # Maximum time per object
# Interactive parameters
subset = 'val'
host = 'localhost' # 'localhost' for subsets train and val.
feature_extracter = DeepLab(backbone='resnet',freeze_bn=False)
model = IntVOS(cfg,feature_extracter)
model= model.cuda()
print('model loading...')
saved_model_dict = os.path.join(save_res_dir,'save_step_80000.pth')
pretrained_dict = torch.load(saved_model_dict)
load_network(model,pretrained_dict)
print('model loading finished!')
model.eval()
seen_seq={}
with torch.no_grad():
with DavisInteractiveSession(host=host,
davis_root=cfg.DATA_ROOT,
subset=subset,
report_save_dir=report_save_dir,
max_nb_interactions=max_nb_interactions,
max_time=max_time,
metric_to_optimize='J'
) as sess:
while sess.next():
t_total=timeit.default_timer()
# Get the current iteration scribbles
sequence, scribbles, first_scribble = sess.get_scribbles(only_last=True)
print(sequence)
start_annotated_frame=annotated_frames(scribbles)[0]
pred_masks=[]
pred_masks_reverse=[]
if first_scribble:
anno_frame_list=[]
n_interaction=1
eval_global_map_tmp_dic={}
local_map_dics=({},{})
total_frame_num=total_frame_num_dic[sequence]
obj_nums = seq_dict[sequence][-1]
eval_data_manager=DAVIS2017_Test_Manager(split='val',root=cfg.DATA_ROOT,transform=tr.ToTensor(),
seq_name=sequence)
else:
n_interaction+=1
##########################Reference image process
scr_f = start_annotated_frame
anno_frame_list.append(start_annotated_frame)
print(start_annotated_frame)
scr_f = str(scr_f)
while len(scr_f)!=5:
scr_f='0'+scr_f
ref_img = os.path.join(cfg.DATA_ROOT,'JPEGImages/480p',sequence,scr_f+'.jpg')
ref_img = cv2.imread(ref_img)
h_,w_ = ref_img.shape[:2]
ref_img = np.array(ref_img,dtype=np.float32)
#ref_img = tr.ToTensor()(ref_img)
#ref_img = ref_img.unsqueeze(0)
scribble_masks=scribbles2mask(scribbles,(h_,w_))
scribble_label=scribble_masks[start_annotated_frame]
sample = {'ref_img':ref_img,'scribble_label':scribble_label}
sample = tr.ToTensor()(sample)
ref_img = sample['ref_img']
scribble_label = sample['scribble_label']
ref_img = ref_img.unsqueeze(0)
scribble_label =scribble_label.unsqueeze(0)
ref_img= ref_img.cuda()
scribble_label = scribble_label.cuda()
######
ref_scribble_to_show = scribble_label.cpu().squeeze().numpy()
im_ = Image.fromarray(ref_scribble_to_show.astype('uint8')).convert('P')
im_.putpalette(_palette)
ref_img_name= scr_f
if not os.path.exists(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction))):
os.makedirs(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction)))
im_.save(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction),'inter_'+ref_img_name+'.png'))
#ref_frame_embedding = model.extract_feature(ref_img)
if first_scribble:
ref_frame_embedding = model.extract_feature(ref_img)
_,channel,emb_h,emb_w = ref_frame_embedding.size()
embedding_memory=torch.zeros((total_frame_num,channel,emb_h,emb_w))
embedding_memory = embedding_memory.cuda()
embedding_memory[start_annotated_frame]=ref_frame_embedding[0]
else:
ref_frame_embedding = embedding_memory[start_annotated_frame]
ref_frame_embedding = ref_frame_embedding.unsqueeze(0)
########
if first_scribble:
prev_label= None
else:
prev_label = os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction-1),scr_f+'.png')
prev_label = Image.open(prev_label)
prev_label = np.array(prev_label,dtype=np.uint8)
prev_label = tr.ToTensor()({'label':prev_label})
prev_label = prev_label['label'].unsqueeze(0)
prev_label = prev_label.cuda()
###############
#tmp_dic, eval_global_map_tmp_dic= model.before_seghead_process(ref_frame_embedding,ref_frame_embedding,
# ref_frame_embedding,scribble_label,prev_label,
# normalize_nearest_neighbor_distances=True,use_local_map=False,
# seq_names=[sequence],gt_ids=torch.Tensor([obj_nums]),k_nearest_neighbors=cfg.KNNS,
# global_map_tmp_dic=eval_global_map_tmp_dic,frame_num=[start_annotated_frame],dynamic_seghead=model.dynamic_seghead)
tmp_dic,local_map_dics = model.int_seghead(ref_frame_embedding=ref_frame_embedding,ref_scribble_label=scribble_label,prev_round_label=prev_label,
global_map_tmp_dic=eval_global_map_tmp_dic,local_map_dics=local_map_dics,interaction_num=n_interaction,
seq_names=[sequence],gt_ids=torch.Tensor([obj_nums]),frame_num=[start_annotated_frame],first_inter=first_scribble)
pred_label = tmp_dic[sequence]
pred_label = nn.functional.interpolate(pred_label,size=(h_,w_),mode = 'bilinear',align_corners=True)
pred_label=torch.argmax(pred_label,dim=1)
pred_masks.append(pred_label.float())
####
if is_save_image:
pred_label_to_save=pred_label.squeeze(0).cpu().numpy()
im = Image.fromarray(pred_label_to_save.astype('uint8')).convert('P')
im.putpalette(_palette)
imgname = str(start_annotated_frame)
while len(imgname)<5:
imgname='0'+imgname
if not os.path.exists(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction))):
os.makedirs(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction)))
im.save(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction),imgname+'.png'))
#######################################
##############################
ref_prev_label = pred_label.unsqueeze(0)
prev_label = pred_label.unsqueeze(0)
prev_img = ref_img
prev_embedding = ref_frame_embedding
for ii in range(start_annotated_frame+1,total_frame_num):
print('evaluating sequence:{} frame:{}'.format(sequence,ii))
sample = eval_data_manager.get_image(ii)
img = sample['img']
img = img.unsqueeze(0)
_,_,h,w = img.size()
img = img.cuda()
# current_embedding= model.extract_feature(img)
if first_scribble:
current_embedding= model.extract_feature(img)
embedding_memory[ii]=current_embedding[0]
else:
current_embedding=embedding_memory[ii]
current_embedding = current_embedding.unsqueeze(0)
prev_img =prev_img.cuda()
prev_label =prev_label.cuda()
tmp_dic, eval_global_map_tmp_dic,local_map_dics= model.before_seghead_process(ref_frame_embedding,prev_embedding,
current_embedding,scribble_label,prev_label,
normalize_nearest_neighbor_distances=True,use_local_map=True,
seq_names=[sequence],gt_ids=torch.Tensor([obj_nums]),k_nearest_neighbors=cfg.KNNS,
global_map_tmp_dic=eval_global_map_tmp_dic,local_map_dics=local_map_dics,
interaction_num=n_interaction,start_annotated_frame=start_annotated_frame,
frame_num=[ii],dynamic_seghead=model.dynamic_seghead)
pred_label = tmp_dic[sequence]
pred_label = nn.functional.interpolate(pred_label,size=(h,w),mode = 'bilinear',align_corners=True)
pred_label=torch.argmax(pred_label,dim=1)
pred_masks.append(pred_label.float())
prev_label = pred_label.unsqueeze(0)
prev_img = img
prev_embedding = current_embedding
####
if is_save_image:
pred_label_to_save=pred_label.squeeze(0).cpu().numpy()
im = Image.fromarray(pred_label_to_save.astype('uint8')).convert('P')
im.putpalette(_palette)
imgname = str(ii)
while len(imgname)<5:
imgname='0'+imgname
if not os.path.exists(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction))):
os.makedirs(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction)))
im.save(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction),imgname+'.png'))
#######################################
prev_label = ref_prev_label
prev_img = ref_img
prev_embedding = ref_frame_embedding
######################################
for ii in range(start_annotated_frame):
current_frame_num=start_annotated_frame-1-ii
print('evaluating sequence:{} frame:{}'.format(sequence,current_frame_num))
sample = eval_data_manager.get_image(current_frame_num)
img = sample['img']
img = img.unsqueeze(0)
_,_,h,w = img.size()
img = img.cuda()
#current_embedding= model.extract_feature(img)
if first_scribble:
current_embedding= model.extract_feature(img)
embedding_memory[current_frame_num]=current_embedding[0]
else:
current_embedding = embedding_memory[current_frame_num]
current_embedding = current_embedding.unsqueeze(0)
prev_img =prev_img.cuda()
prev_label =prev_label.cuda()
tmp_dic, eval_global_map_tmp_dic,local_map_dics= model.before_seghead_process(ref_frame_embedding,prev_embedding,
current_embedding,scribble_label,prev_label,
normalize_nearest_neighbor_distances=True,use_local_map=True,
seq_names=[sequence],gt_ids=torch.Tensor([obj_nums]),k_nearest_neighbors=cfg.KNNS,
global_map_tmp_dic=eval_global_map_tmp_dic,local_map_dics=local_map_dics,interaction_num=n_interaction,start_annotated_frame=start_annotated_frame,frame_num=[current_frame_num],dynamic_seghead=model.dynamic_seghead)
pred_label = tmp_dic[sequence]
pred_label = nn.functional.interpolate(pred_label,size=(h,w),mode = 'bilinear',align_corners=True)
pred_label=torch.argmax(pred_label,dim=1)
pred_masks_reverse.append(pred_label.float())
prev_label = pred_label.unsqueeze(0)
prev_img = img
prev_embedding = current_embedding
####
if is_save_image:
pred_label_to_save=pred_label.squeeze(0).cpu().numpy()
im = Image.fromarray(pred_label_to_save.astype('uint8')).convert('P')
im.putpalette(_palette)
imgname = str(current_frame_num)
while len(imgname)<5:
imgname='0'+imgname
if not os.path.exists(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction))):
os.makedirs(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction)))
im.save(os.path.join(cfg.RESULT_ROOT,sequence,'interactive'+str(n_interaction),imgname+'.png'))
pred_masks_reverse.reverse()
pred_masks_reverse.extend(pred_masks)
final_masks = torch.cat(pred_masks_reverse,0)
sess.submit_masks(final_masks.cpu().numpy())
t_end = timeit.default_timer()
print('Total time for single interaction: ' + str(t_end - t_total))
report = sess.get_report()
summary = sess.get_global_summary(save_file=os.path.join(report_save_dir, 'summary.json'))
save_model_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'models_test')
report_save_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'results_test')
save_result_dir = report_save_dir # 'None' to not save the results
model = OsvosScribble(parent_model,
save_model_dir,
gpu_id,
time_budget_per_object,
save_result_dir=save_result_dir)
seen_seq = {}
with DavisInteractiveSession(host=host,
davis_root=Path.db_root_dir(),
subset=subset,
report_save_dir=report_save_dir,
user_key=key) as sess:
while sess.next():
# Get the current iteration scribbles
sequence, scribbles, first_scribble = sess.get_scribbles()
if first_scribble:
n_interaction = 1
n_objects = Davis.dataset[sequence]['num_objects']
first_frame = interactive_utils.scribbles.annotated_frames(
scribbles)[0]
seen_seq[sequence] = 1 if sequence not in seen_seq.keys(
) else seen_seq[sequence] + 1
else:
n_interaction += 1
pred_masks = []
return np.zeros([len(files), img_height, img_width]), len(files)
pass
def compute_req_indices(scribbles, nb_frame):
index = 0
for i, l_obj in enumerate(scribbles['scribbles']):
if len(l_obj) > 0:
index = i
break
return [(index + int(np.ceil(nb_frame/8)) * i) % nb_frame for i in range(8)]
pass
with DavisInteractiveSession(host='https://server.davischallenge.org',
user_key='ed3db2783547b034398e89f6163b955dc1a0135b33d5ae346ce61aa8dcf3d356', # ntan
# user_key='fae1b878c12cec0857a64b13154a45cd9dd4be1f450a0064e4a2fde32020f144', # nhdang
# user_key='d028a4c4bc13459359b63c9d948e91e95c1998c1a474cac07cdbfd4be811dadc', # qckan
davis_root='/home/ntan/DAVIS2020/data/test-dev',
subset='test-dev',
max_nb_interactions=max_nb_interactions,
max_time=max_time) as sess:
while sess.next():
st = time.time()
# Get the current interaction's scribbles
sequence, scribbles, _ = sess.get_scribbles()
saved_masks = glob.glob('output/masks_test-dev_chiet/%s_*'%(sequence))
saved_masks.sort()
if len(saved_masks) > 0:
pred_masks = np.load(saved_masks[-1])
sess.submit_masks(pred_masks)
print('submitted...')
continue
def run_IVOS(self, metric):
seen_seq = {}
numseq, tmpseq = 0, ''
with open(self.save_csvsummary_dir, mode='a') as csv_file:
writer = csv.writer(csv_file,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
writer.writerow(['sequence', 'obj_idx', 'scr_idx'] + [
'round-' + str(i + 1) for i in range(self.max_nb_interactions)
])
with DavisInteractiveSession(
host=self.config.test_host,
user_key=self.config.test_userkey,
davis_root=self.config.davis_dataset_dir,
subset=self.config.test_subset,
report_save_dir=self.save_res_dir,
max_nb_interactions=self.max_nb_interactions,
max_time=self.max_time,
metric_to_optimize=metric) as sess:
sess.connector.service.robot.min_nb_nodes = self.config.test_min_nb_nodes
sess.samples = self.scr_samples
while sess.next():
# Get the current iteration scribbles
self.sequence, scribbles, first_scribble = sess.get_scribbles(
only_last=False)
if first_scribble:
anno_dict = {
'frames': [],
'annotated_masks': [],
'masks_tobe_modified': []
}
n_interaction = 1
info = Davis.dataset[self.sequence]
self.img_size = info['image_size'][::-1]
self.num_frames = info['num_frames']
self.n_objects = info['num_objects']
info = None
seen_seq[
self.
sequence] = 1 if self.sequence not in seen_seq.keys(
) else seen_seq[self.sequence] + 1
scr_id = seen_seq[self.sequence]
self.final_masks = np.zeros(
[self.num_frames, self.img_size[0], self.img_size[1]])
self.pad_info = utils_custom.apply_pad(
self.final_masks[0])[1]
self.hpad1, self.wpad1 = self.pad_info[0][
0], self.pad_info[1][0]
self.hpad2, self.wpad2 = self.pad_info[0][
1], self.pad_info[1][1]
self.h_ds, self.w_ds = int(
(self.img_size[0] + sum(self.pad_info[0])) / 4), int(
(self.img_size[1] + sum(self.pad_info[1])) / 4)
self.anno_6chEnc_r4_list = []
self.anno_3chEnc_r4_list = []
self.prob_map_of_frames = torch.zeros(
(self.num_frames, self.n_objects + 1, 4 * self.h_ds,
4 * self.w_ds)).cuda()
self.gt_masks = self.Davisclass.load_annotations(
self.sequence)
self.scores_ni_nf = np.zeros([8, self.num_frames])
IoU_over_eobj = []
else:
n_interaction += 1
self.save_logger.printNlog(
'\nRunning sequence {} in (scribble index: {}) (round: {})'
.format(self.sequence, sess.samples[sess.sample_idx][1],
n_interaction))
annotated_now = interactive_utils.scribbles.annotated_frames(
sess.sample_last_scribble)[0]
anno_dict['frames'].append(
annotated_now) # Where we save annotated frames
anno_dict['masks_tobe_modified'].append(
self.final_masks[annotated_now]
) # mask before modefied at the annotated frame
# Get Predicted mask & Mask decision from pred_mask
self.final_masks = self.run_VOS_singleiact(
n_interaction, scribbles,
anno_dict['frames']) # self.final_mask changes
if self.config.test_save_pngs_option:
utils_custom.mkdir(
os.path.join(
self.save_res_dir, 'result_video',
'{}-scr{:02d}/round{:02d}'.format(
self.sequence, scr_id, n_interaction)))
for fr in range(self.num_frames):
savefname = os.path.join(
self.save_res_dir, 'result_video',
'{}-scr{:02d}/round{:02d}'.format(
self.sequence, scr_id,
n_interaction), '{:05d}.png'.format(fr))
tmpPIL = Image.fromarray(
self.final_masks[fr].astype(np.uint8), 'P')
tmpPIL.putpalette(self._palette)
tmpPIL.save(savefname)
# Limit candidate frames
if n_interaction != self.max_nb_interactions:
self.scores_ni_nf[n_interaction] = self.scores_ni_nf[
n_interaction - 1]
current_score_np = self.scores_ni_nf[n_interaction - 1]
if self.config.test_guide_method == 'RS1':
next_scribble_frame_candidates = list(
np.argsort(current_score_np)[:1])
elif self.config.test_guide_method == 'RS4':
sorted_score_idx = np.argsort(current_score_np)
exclude_range = self.num_frames / 10
excluded_next_candidates = []
next_scribble_frame_candidates = []
for i in range(self.num_frames):
if not sorted_score_idx[i] in excluded_next_candidates:
next_scribble_frame_candidates.append(
sorted_score_idx[i])
excluded_next_candidates += list(
range(
int(sorted_score_idx[i] -
(exclude_range / 2) + 0.5),
int(sorted_score_idx[i] +
(exclude_range / 2) + 0.5)))
if len(next_scribble_frame_candidates) == 4:
break
elif self.config.test_guide_method == 'wo_RS':
next_scribble_frame_candidates = None
else:
raise NotImplementedError
# Submit your prediction
sess.submit_masks(self.final_masks,
next_scribble_frame_candidates) # F, H, W
# print sequence name
if tmpseq != self.sequence:
tmpseq, numseq = self.sequence, numseq + 1
print(
str(numseq) + ':' + str(self.sequence) + '-' +
str(seen_seq[self.sequence]) + '\n')
## Visualizers and Saver
# IoU estimation
jaccard = batched_jaccard(
self.gt_masks,
self.final_masks,
average_over_objects=False,
nb_objects=self.n_objects) # frames, objid
IoU_over_eobj.append(jaccard)
# save final mask in anno_dict
anno_dict['annotated_masks'].append(
self.final_masks[annotated_now]
) # mask after modefied at the annotated frame
if self.max_nb_interactions == len(
anno_dict['frames']
): # After Lastround -> total 90 iter
seq_scrid_name = self.sequence + str(scr_id)
# IoU manager
IoU_over_eobj = np.stack(IoU_over_eobj,
axis=0) # niact,frames,n_obj
IoUeveryround_perobj = np.mean(IoU_over_eobj,
axis=1) # niact,n_obj
# show scribble input and output
savefiledir = os.path.join(self.save_res_dir,
'debug_scribble')
utils_custom.mkdir(savefiledir)
savefilename = os.path.join(savefiledir,
seq_scrid_name + '.jpg')
utils_visualize.visualize_scrib_interaction(
scribbles, anno_dict, self.sequence, savefilename)
# plot IoU
if self.config.test_save_pngs_option:
savefiledir = os.path.join(self.save_res_dir,
'plot_IoU_perObj')
utils_custom.mkdir(savefiledir)
for obj_idx in range(self.n_objects):
savefilename = os.path.join(
savefiledir,
seq_scrid_name + '-obj' + str(obj_idx + 1) +
'_first{:03d}final{:03d}.png'.format(
int(1000 *
IoUeveryround_perobj[0, obj_idx]),
int(1000 *
IoUeveryround_perobj[-1, obj_idx])))
utils_visualize.visualize_interactionIoU(
IoU_over_eobj[:, :, obj_idx],
seq_scrid_name + '-obj' + str(obj_idx + 1),
anno_dict['frames'],
save_dir=savefilename,
show_propagated_region=True)
# write csv
for obj_idx in range(self.n_objects):
with open(self.save_csvsummary_dir,
mode='a') as csv_file:
writer = csv.writer(csv_file,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
writer.writerow(
[self.sequence,
str(obj_idx + 1),
str(scr_id)] +
list(IoUeveryround_perobj[:, obj_idx]))
summary = sess.get_global_summary(save_file=self.save_res_dir +
'/summary_' + sess.report_name[7:] +
'.json')
analyze_summary(self.save_res_dir + '/summary_' +
sess.report_name[7:] + '.json',
metric=metric)
fps = self.n_operated_frames_accum / self.total_taken_times_accum
self.save_logger.printNlog('n_operated_frames_accum={}'.format(
str(self.n_operated_frames_accum)))
self.save_logger.printNlog('total_taken_times_accum={}'.format(
str(self.total_taken_times_accum)))
self.save_logger.printNlog('fps={}'.format(str(fps)))
# final_IOU = summary['curve'][metric][-1]
average_IoU_per_round = summary['curve'][metric][1:-1]
torch.cuda.empty_cache()
model = None
return average_IoU_per_round