def get_sixd_gt_train_crops(obj_id,
hw_ae,
pad_factor=1.2,
dataset='tless',
cam_type='primesense'):
data_params = dataset_params.get_dataset_params(dataset,
model_type='',
train_type='',
test_type=cam_type,
cam_type=cam_type)
eval_args = configparser.ConfigParser()
eval_args.add_section("DATA")
eval_args.add_section("BBOXES")
eval_args.add_section("EVALUATION")
eval_args.set('BBOXES', 'ESTIMATE_BBS', "False")
eval_args.set('EVALUATION', 'ICP', "False")
eval_args.set('BBOXES', 'PAD_FACTOR', str(pad_factor))
eval_args.set('BBOXES', 'ESTIMATE_MASKS', "False")
eval_args.set('DATA', 'OBJ_ID', str(obj_id))
gt = inout.load_gt(data_params['obj_gt_mpath'].format(obj_id))
imgs = []
for im_id in range(504):
imgs.append(
cv2.imread(data_params['train_rgb_mpath'].format(obj_id, im_id)))
test_img_crops, _, _, _, _ = generate_scene_crops(np.array(imgs), None, gt,
eval_args, hw_ae)
return test_img_crops
def get_gt_scene_crops(scene_id, eval_args, train_args):
dataset_name = eval_args.get('DATA','DATASET')
cam_type = eval_args.get('DATA','CAM_TYPE')
icp = eval_args.getboolean('EVALUATION','ICP')
delta = eval_args.get('METRIC', 'VSD_DELTA')
workspace_path = os.environ.get('AE_WORKSPACE_PATH')
dataset_path = u.get_dataset_path(workspace_path)
H = train_args.getint('Dataset','H')
# linemod with 13 objects and linemod with 15 objects may use the same md5 file!
# for example, 7 in 13 is duck, but 7 in 15 is cup!
cfg_string = str([scene_id] + eval_args.items('DATA') + eval_args.items('BBOXES') + [H])
current_config_hash = hashlib.md5(cfg_string).hexdigest()
# current_config_hash = cfg_string
current_file_name = os.path.join(dataset_path, current_config_hash + '.npz')
if os.path.exists(current_file_name):
data = np.load(current_file_name)
test_img_crops = data['test_img_crops'].item()
test_img_depth_crops = data['test_img_depth_crops'].item()
bb_scores = data['bb_scores'].item()
bb_vis = data['visib_gt'].item()
bbs = data['bbs'].item()
if not os.path.exists(current_file_name) or len(test_img_crops) == 0 or len(test_img_depth_crops) == 0:
test_imgs = load_scenes(scene_id, eval_args)
test_imgs_depth = load_scenes(scene_id, eval_args, depth=True) if icp else None
data_params = dataset_params.get_dataset_params(dataset_name, model_type='', train_type='', test_type=cam_type, cam_type=cam_type)
# only available for primesense, sixdtoolkit can generate. calc_gt_stats.py
visib_gt = inout.load_yaml(data_params['scene_gt_stats_mpath'].format(scene_id, delta))
bb_gt = inout.load_gt(data_params['scene_gt_mpath'].format(scene_id))
test_img_crops, test_img_depth_crops, bbs, bb_scores, bb_vis = generate_scene_crops(test_imgs, test_imgs_depth, bb_gt, eval_args,
train_args, visib_gt=visib_gt)
np.savez(current_file_name, test_img_crops=test_img_crops, test_img_depth_crops=test_img_depth_crops, bbs = bbs, bb_scores=bb_scores, visib_gt=bb_vis)
current_cfg_file_name = os.path.join(dataset_path, current_config_hash + '.cfg')
with open(current_cfg_file_name, 'w') as f:
f.write(cfg_string)
print 'created new ground truth crops!'
else:
print 'loaded previously generated ground truth crops!'
print len(test_img_crops), len(test_img_depth_crops)
return (test_img_crops, test_img_depth_crops, bbs, bb_scores, bb_vis)
def plot_R_err_hist_vis(eval_args, eval_dir, scene_ids, bins=20):
top_n_eval = eval_args.getint('EVALUATION','TOP_N_EVAL')
top_n = eval_args.getint('METRIC','TOP_N')
cam_type = eval_args.get('DATA','cam_type')
dataset_name = eval_args.get('DATA','dataset')
obj_id = eval_args.getint('DATA','obj_id')
# if top_n_eval < 1:
# return
data_params = dataset_params.get_dataset_params(dataset_name, model_type='', train_type='', test_type=cam_type, cam_type=cam_type)
angle_errs = []
for scene_id in scene_ids:
error_file_path = os.path.join(eval_dir,'error=re_ntop=%s' % top_n,'errors_{:02d}.yml'.format(scene_id))
if not os.path.exists(error_file_path):
print(('WARNING: ' + error_file_path + ' not found'))
continue
# angle_errs_dict = inout.load_yaml(error_file_path)
# angle_errs += [angle_e['errors'].values()[0] for angle_e in angle_errs_dict]
gts = inout.load_gt(data_params['scene_gt_mpath'].format(scene_id))
visib_gts = inout.load_yaml(data_params['scene_gt_stats_mpath'].format(scene_id, 15))
re_dict = inout.load_yaml(error_file_path)
for view in range(len(gts)):
res = re_dict[view*top_n:(view+1)*top_n]
for gt,visib_gt in zip(gts[view],visib_gts[view]):
if gt['obj_id'] == obj_id:
if visib_gt['visib_fract'] > 0.1:
for re_e in res:
angle_errs += [list(re_e['errors'].values())[0]]
if len(angle_errs) == 0:
return
angle_errs = np.array(angle_errs)
plot_R_err_hist2(angle_errs, eval_dir, bins=bins)
def plot_scene_with_3DBoxes(scene_res_dirs,dataset_name='tless',scene_id=1,save=False):
# sixd_img_path = '/home_local/sund_ma/data/linemod_dataset/test'
# model_path = '/home_local/sund_ma/data/linemod_dataset/models'
# inout.save_results_sixd17(res_path, preds, run_time=run_time)
# obj_gts = []
# obj_infos = []
# for object_id in xrange(1,noofobjects+1):
# obj_gts.append(inout.load_gt(os.path.join(sixd_img_path,'{:02d}'.format(object_id),'gt.yml')))
# obj_infos.append(inout.load_info(os.path.join(sixd_img_path,'{:02d}'.format(object_id),'info.yml')))
# print len(obj_gts)
# dataset_name = eval_args.get('DATA','DATASET')
# cam_type = eval_args.get('DATA','CAM_TYPE')
# data_params = dataset_params.get_dataset_params(dataset_name, model_type='', train_type='', test_type='primesense', cam_type='primesense')
data_params = dataset_params.get_dataset_params(dataset_name, model_type='', train_type='')
models_cad_files = sorted(glob.glob(os.path.join(os.path.dirname(data_params['model_mpath']),'*.ply')))
W,H = data_params['test_im_size']
from auto_pose.meshrenderer import box3d_renderer
renderer_line = box3d_renderer.Renderer(
models_cad_files,
1,
W,
H
)
scene_result_dirs = sorted(glob.glob(scene_res_dirs))
print data_params['test_rgb_mpath']
print data_params['scene_gt_mpath']
# for scene_id in xrange(1,21):
# sixd_img_path = data_params['test_rgb_mpath'].format(scene_id)
scene_gts = inout.load_gt(data_params['scene_gt_mpath'].format(scene_id))
scene_infos = inout.load_info(data_params['scene_info_mpath'].format(scene_id))
scene_dirs = [d for d in scene_result_dirs if '%02d' % scene_id == d.split('/')[-1]]
print scene_dirs
for view in xrange(len(scene_infos)):
sixd_img_path = data_params['test_rgb_mpath'].format(scene_id,view)
img = cv2.imread(sixd_img_path)
box_img = img.copy()
# cv2.imshow('',img)
# cv2.waitKey(0)
K = scene_infos[view]['cam_K']
for bb in scene_gts[view]:
xmin = int(bb['obj_bb'][0])
ymin = int(bb['obj_bb'][1])
xmax = int(bb['obj_bb'][0]+bb['obj_bb'][2])
ymax = int(bb['obj_bb'][1]+bb['obj_bb'][3])
cv2.rectangle(box_img, (xmin,ymin),(xmax,ymax), (0,255,0), 2)
cv2.putText(box_img, '%s' % (bb['obj_id']), (xmin, ymax+20), cv2.FONT_ITALIC, .5, (0,255,0), 2)
# for gt in scene_gts[view]:
# if gt['obj_id'] not in [1,8,9]:
# lines_gt = renderer_line.render(gt['obj_id']-1,K,gt['cam_R_m2c'],gt['cam_t_m2c'],10,5000)
# lines_gt_mask = (np.sum(lines_gt,axis=2) < 20)[:,:,None]
# print lines_gt.shape
# lines_gt = lines_gt[:,:,[1,0,2]]
# img = lines_gt_mask*img + lines_gt
for scene_dir in scene_dirs:
try:
res_path = glob.glob(os.path.join(scene_dir,'%04d_*.yml' % (view)))
print res_path
res_path = res_path[0]
# print 'here', res_path
obj_id = int(res_path.split('_')[-1].split('.')[0])
results = inout.load_results_sixd17(res_path)
print results
e = results['ests'][0]
R_est = e['R']
t_est = e['t']
K = scene_infos[view]['cam_K']
lines = renderer_line.render(obj_id-1,K,R_est,t_est,10,5000)
lines_mask = (np.sum(lines,axis=2) < 20)[:,:,None]
# img[lines>0] = lines[lines>0]
if obj_id % 7 == 1:
lines[:,:,0] = lines[:,:,1]
elif obj_id % 7 == 2:
lines[:,:,2] = lines[:,:,1]
elif obj_id % 7 == 3:
lines[:,:,0] = lines[:,:,1]
lines[:,:,1] = lines[:,:,2]
img = lines_mask*img + lines
except:
print 'undeteceted obj: ', scene_dir
cv2.imshow('',img)
if cv2.waitKey(1) == 32:
cv2.waitKey(0)
if save:
if 'icp' in scene_res_dirs:
if not os.path.exists('%02d' % scene_id):
os.makedirs('%02d' % scene_id)
cv2.imwrite(os.path.join('%02d' % scene_id,'%04d.png' % view), img)
else:
if not os.path.exists('%02d_rgb' % scene_id):
os.makedirs('%02d_rgb' % scene_id)
cv2.imwrite(os.path.join('%02d_rgb' % scene_id,'%04d.png' % view), img)
def plot_vsd_err_hist(eval_args, eval_dir, scene_ids):
# THIS DOES NOT WORK WELL FOR MULTIPLE INSTANCE
# E.G. TEJANI 04 MILK. lxc
top_n_eval = eval_args.getint('EVALUATION','TOP_N_EVAL')
top_n = eval_args.getint('METRIC','TOP_N')
delta = eval_args.getint('METRIC','VSD_DELTA')
tau = eval_args.getint('METRIC','VSD_TAU')
cost = eval_args.get('METRIC','VSD_COST')
cam_type = eval_args.get('DATA','cam_type')
dataset_name = eval_args.get('DATA','dataset')
obj_id = eval_args.getint('DATA','obj_id')
# if top_n_eval < 1:
# return
data_params = dataset_params.get_dataset_params(dataset_name, model_type='', train_type='', test_type=cam_type, cam_type=cam_type)
vsd_errs = []
for scene_id in scene_ids:
error_file_path = os.path.join(eval_dir,'error=vsd_ntop=%s_delta=%s_tau=%s_cost=%s' % (top_n, delta, tau, cost), 'errors_{:02d}.yml'.format(scene_id))
if not os.path.exists(error_file_path):
print 'WARNING: ' + error_file_path + ' not found'
continue
gts = inout.load_gt(data_params['scene_gt_mpath'].format(scene_id))
visib_gts = inout.load_yaml(data_params['scene_gt_stats_mpath'].format(scene_id, 15))
vsd_dict = inout.load_yaml(error_file_path)
for view,vsd_e in enumerate(vsd_dict):
vsds = vsd_dict[view*top_n:(view+1)*top_n]
for gt,visib_gt in zip(gts[view],visib_gts[view]):
if gt['obj_id'] == obj_id:
if visib_gt['visib_fract'] > 0.1:
for vsd_e in vsds:
vsd_errs += [vsd_e['errors'].values()[0]]
if len(vsd_errs) == 0:
return
vsd_errs = np.array(vsd_errs)
print len(vsd_errs)
fig = plt.figure()
ax = plt.gca()
ax.set_xlim((0.0,1.0))
plt.grid()
plt.xlabel('vsd err')
plt.ylabel('recall')
plt.title('VSD Error vs Recall')
legend=[]
for n in np.unique(np.array([top_n, 1])):
total_views = len(vsd_errs)/top_n
min_vsd_errs = np.empty((total_views,))
for view in xrange(total_views):
top_n_errors = vsd_errs[view*top_n:(view+1)*top_n]
if n == 1:
top_n_errors = top_n_errors[np.newaxis,0]
min_vsd_errs[view] = np.min(top_n_errors)
min_vsd_errs_sorted = np.sort(min_vsd_errs)
recall = np.float32(np.arange(total_views)+1.)/total_views
# fill curve
min_vsd_errs_sorted = np.hstack((min_vsd_errs_sorted, np.array([1.])))
recall = np.hstack((recall,np.array([1.])))
AUC_vsd = np.trapz(recall, min_vsd_errs_sorted)
plt.plot(min_vsd_errs_sorted,recall)
legend += ['top {0} vsd err, AUC = {1:.4f}'.format(n,AUC_vsd)]
plt.legend(legend)
plt.savefig(os.path.join(eval_dir,'figures','vsd_err_hist_{:02d}.png'.format(obj_id)), dpi=300)
tikz_save(os.path.join(eval_dir,'latex','vsd_err_hist.tex'), figurewidth ='0.45\\textheight', figureheight='0.45\\textheight', show_info=False)
def plot_R_err_hist(eval_args, eval_dir, scene_ids):
top_n_eval = eval_args.getint('EVALUATION','TOP_N_EVAL')
top_n = eval_args.getint('METRIC','TOP_N')
cam_type = eval_args.get('DATA','cam_type')
dataset_name = eval_args.get('DATA','dataset')
obj_id = eval_args.getint('DATA','obj_id')
# if top_n_eval < 1: # why?
# return
data_params = dataset_params.get_dataset_params(dataset_name, model_type='', train_type='', test_type=cam_type, cam_type=cam_type)
angle_errs = []
for scene_id in scene_ids:
error_file_path = os.path.join(eval_dir,'error=re_ntop=%s' % top_n,'errors_{:02d}.yml'.format(scene_id))
if not os.path.exists(error_file_path):
print 'WARNING: ' + error_file_path + ' not found'
continue
# angle_errs_dict = inout.load_yaml(error_file_path)
# angle_errs += [angle_e['errors'].values()[0] for angle_e in angle_errs_dict]
gts = inout.load_gt(data_params['scene_gt_mpath'].format(scene_id))
visib_gts = inout.load_yaml(data_params['scene_gt_stats_mpath'].format(scene_id, 15))
re_dict = inout.load_yaml(error_file_path)
for view in xrange(len(gts)):
res = re_dict[view*top_n:(view+1)*top_n]
for gt,visib_gt in zip(gts[view],visib_gts[view]):
if gt['obj_id'] == obj_id:
if visib_gt['visib_fract'] > 0.1:
for re_e in res:
angle_errs += [re_e['errors'].values()[0]]
if len(angle_errs) == 0:
return
angle_errs = np.array(angle_errs)
fig = plt.figure()
plt.grid()
plt.xlabel('angle err [deg]')
plt.ylabel('recall')
plt.title('Angle Error vs Recall')
legend=[]
for n in np.unique(np.array([top_n, 1])):
total_views = len(angle_errs)/top_n
min_angle_errs = np.empty((total_views,))
min_angle_errs_rect = np.empty((total_views,))
for view in xrange(total_views):
top_n_errors = angle_errs[view*top_n:(view+1)*top_n]
if n == 1:
top_n_errors = top_n_errors[np.newaxis,0]
min_angle_errs[view] = np.min(top_n_errors)
min_angle_errs_rect[view] = np.min(np.hstack((top_n_errors, 180-top_n_errors)))
min_angle_errs_sorted = np.sort(min_angle_errs)
min_angle_errs_rect_sorted = np.sort(min_angle_errs_rect)
recall = (np.arange(total_views)+1.)/total_views
# fill curve
min_angle_errs_sorted = np.hstack((min_angle_errs_sorted, np.array([180.])))
min_angle_errs_rect_sorted = np.hstack((min_angle_errs_rect_sorted, np.array([90.])))
recall = np.hstack((recall,np.array([1.])))
AUC_angle = np.trapz(recall,min_angle_errs_sorted/180.)
AUC_angle_rect = np.trapz(recall,min_angle_errs_rect_sorted/90.)
plt.plot(min_angle_errs_sorted,recall)
plt.plot(min_angle_errs_rect_sorted,recall)
legend += ['top {0} angle err, AUC = {1:.4f}'.format(n,AUC_angle), 'top {0} rectified angle err, AUC = {1:.4f}'.format(n,AUC_angle_rect)]
plt.legend(legend)
plt.savefig(os.path.join(eval_dir,'figures','R_err_hist_{:02d}.png'.format(obj_id)), dpi=300)
tikz_save(os.path.join(eval_dir,'latex','R_err_hist.tex'), figurewidth ='0.45\\textheight', figureheight='0.45\\textheight', show_info=False)
def main():
'''
lxc:
use_euclidean means the similarity between test embedding and template embedding
are computed using Euclidean Distance
'''
#use_euclidean = False
parser = argparse.ArgumentParser()
parser.add_argument('experiment_name')
parser.add_argument('evaluation_name')
parser.add_argument('--eval_cfg', default='eval.cfg', required=False)
parser.add_argument('--at_step', default=None, required=False)
arguments = parser.parse_args()
full_name = arguments.experiment_name.split('/')
experiment_name = full_name.pop()
experiment_group = full_name.pop() if len(full_name) > 0 else ''
evaluation_name = arguments.evaluation_name
eval_cfg = arguments.eval_cfg
at_step = arguments.at_step
workspace_path = os.environ.get('AE_WORKSPACE_PATH')
train_cfg_file_path = u.get_config_file_path(workspace_path,
experiment_name,
experiment_group)
eval_cfg_file_path = u.get_eval_config_file_path(workspace_path,
eval_cfg=eval_cfg)
train_args = configparser.ConfigParser()
eval_args = configparser.ConfigParser()
train_args.read(train_cfg_file_path)
eval_args.read(eval_cfg_file_path)
#[DATA]
# target data params
dataset_name = eval_args.get('DATA', 'DATASET')
obj_id = eval_args.getint('DATA', 'OBJ_ID')
scenes = eval(eval_args.get(
'DATA', 'SCENES')) if len(eval(eval_args.get(
'DATA',
'SCENES'))) > 0 else eval_utils.get_all_scenes_for_obj(eval_args)
cam_type = eval_args.get('DATA', 'cam_type')
model_type = 'reconst' if dataset_name == 'tless' else '' # model_type set to reconst only for tless.
data_params = dataset_params.get_dataset_params(dataset_name,
model_type=model_type,
train_type='',
test_type=cam_type,
cam_type=cam_type)
target_models_info = inout.load_yaml(
data_params['models_info_path']) # lxc
# source data params, lxc
source_dataset_name = 'toyotalight'
# source_dataset_name = train_args.get('DATA','DATASET') # TODO train args no section DATA
# source_obj_id = train_args.getint('DATA','OBJ_ID') # TODO train args no section DATA
source_obj_id = int(train_cfg_file_path[-6:-4]) # TODO workaround
source_data_params = dataset_params.get_dataset_params(source_dataset_name,
model_type='',
train_type='',
test_type='',
cam_type='')
# for tless temporarily.
# source_data_params = dataset_params.get_dataset_params(source_dataset_name, model_type='', train_type='', test_type='kinect', cam_type='kinect')
source_models_info = inout.load_yaml(
source_data_params['models_info_path'])
print("source_models_info_path:", source_data_params['models_info_path'])
# 'diameter' is not equal to sqrt(x^2+y^2+z^2) for hinterstoisser, rutgers, tless, tejaniDB. etc.
# for toyotalight, 'diameter' == sqrt(...).
target_models_3Dlength = np.linalg.norm([
target_models_info[obj_id][key]
for key in ['size_x', 'size_y', 'size_z']
])
source_models_3Dlength = np.linalg.norm([
source_models_info[source_obj_id][key]
for key in ['size_x', 'size_y', 'size_z']
])
target_source_length_ratio = target_models_3Dlength / source_models_3Dlength
print("target_source_length_ratio:", target_source_length_ratio)
print("source id {:02d}, target id {:02d}".format(source_obj_id, obj_id))
print('basepath: ', data_params['base_path'])
#[BBOXES]
estimate_bbs = eval_args.getboolean('BBOXES', 'ESTIMATE_BBS')
#[METRIC]
top_nn = eval_args.getint('METRIC', 'TOP_N')
#[EVALUATION]
icp = eval_args.getboolean('EVALUATION', 'ICP')
evaluation_name = evaluation_name + '_icp' if icp else evaluation_name
evaluation_name = evaluation_name + '_bbest' if estimate_bbs else evaluation_name
data = dataset_name + '_' + cam_type if len(cam_type) > 0 else dataset_name
log_dir = u.get_log_dir(workspace_path, experiment_name, experiment_group)
ckpt_dir = u.get_checkpoint_dir(log_dir)
eval_dir = u.get_eval_dir(log_dir, evaluation_name, data)
# if eval_args.getboolean('EVALUATION','EVALUATE_ERRORS'):
# eval_loc.match_and_eval_performance_scores(eval_args, eval_dir)
# exit()
if not os.path.exists(eval_dir):
os.makedirs(eval_dir)
shutil.copy2(eval_cfg_file_path, eval_dir)
print "eval_args: ", eval_args
codebook, dataset, decoder = factory.build_codebook_from_name(
experiment_name,
experiment_group,
return_dataset=True,
return_decoder=True)
dataset.renderer
gpu_options = tf.GPUOptions(allow_growth=True,
per_process_gpu_memory_fraction=0.5)
config = tf.ConfigProto(gpu_options=gpu_options)
sess = tf.Session(config=config)
factory.restore_checkpoint(sess,
tf.train.Saver(),
ckpt_dir,
at_step=at_step)
if estimate_bbs:
#Object Detection, seperate from main
# sys.path.append('/net/rmc-lx0050/home_local/sund_ma/src/SSD_Tensorflow')
# from ssd_detector import SSD_detector
# #TODO: set num_classes, network etc.
# ssd = SSD_detector(sess, num_classes=31, net_shape=(300,300))
from rmcssd.bin import detector
ssd = detector.Detector(eval_args.get('BBOXES', 'CKPT'))
t_errors = []
R_errors = []
all_test_visibs = []
test_embeddings = []
for scene_id in scenes:
test_imgs = eval_utils.load_scenes(scene_id, eval_args)
test_imgs_depth = eval_utils.load_scenes(
scene_id, eval_args, depth=True) if icp else None
if estimate_bbs:
print eval_args.get('BBOXES', 'EXTERNAL')
if eval_args.get('BBOXES', 'EXTERNAL') == 'False':
bb_preds = {}
for i, img in enumerate(test_imgs):
print img.shape
bb_preds[i] = ssd.detectSceneBBs(img,
min_score=.2,
nms_threshold=.45)
# inout.save_yaml(os.path.join(scene_res_dir,'bb_preds.yml'), bb_preds)
print bb_preds
else:
bb_preds = inout.load_yaml(
os.path.join(eval_args.get('BBOXES', 'EXTERNAL'),
'{:02d}.yml'.format(scene_id)))
test_img_crops, test_img_depth_crops, bbs, bb_scores, visibilities = eval_utils.generate_scene_crops(
test_imgs, test_imgs_depth, bb_preds, eval_args, train_args)
else:
# test_img_crops: each crop contains some bbox(es) for specified object id.
test_img_crops, test_img_depth_crops, bbs, bb_scores, visibilities = eval_utils.get_gt_scene_crops(
scene_id, eval_args, train_args)
if len(test_img_crops) == 0:
print 'ERROR: object %s not in scene %s' % (obj_id, scene_id)
exit()
info = inout.load_info(
data_params['scene_info_mpath'].format(scene_id))
Ks_test = [np.array(v['cam_K']).reshape(3, 3) for v in info.values()]
######remove
gts = inout.load_gt(data_params['scene_gt_mpath'].format(scene_id))
visib_gts = inout.load_yaml(data_params['scene_gt_stats_mpath'].format(
scene_id, 15))
#######
W_test, H_test = data_params['test_im_size']
icp_renderer = icp_utils.SynRenderer(train_args) if icp else None
noof_scene_views = eval_utils.noof_scene_views(scene_id, eval_args)
test_embeddings.append([])
scene_res_dir = os.path.join(
eval_dir, '{scene_id:02d}'.format(scene_id=scene_id))
if not os.path.exists(scene_res_dir):
os.makedirs(scene_res_dir)
for view in xrange(
noof_scene_views
): # for example, LINEMOD ape noof_scene_views = 1236
try:
# only a specified object id is selected throughout the whole scene views.
test_crops, test_crops_depth, test_bbs, test_scores, test_visibs = eval_utils.select_img_crops(
test_img_crops[view][obj_id],
test_img_depth_crops[view][obj_id] if icp else None,
bbs[view][obj_id], bb_scores[view][obj_id],
visibilities[view][obj_id], eval_args)
except:
print 'no detections'
continue
print view
preds = {}
pred_views = []
all_test_visibs.append(test_visibs[0])
t_errors_crop = []
R_errors_crop = []
for i, (test_crop, test_bb, test_score) in enumerate(
zip(test_crops, test_bbs, test_scores)):
# each test_crop is a ground truth patch
if train_args.getint('Dataset', 'C') == 1:
test_crop = cv2.cvtColor(test_crop,
cv2.COLOR_BGR2GRAY)[:, :, None]
start = time.time()
'''modify here to change the pose estimation algorithm. lxc'''
Rs_est, ts_est = codebook.auto_pose6d(
sess,
test_crop,
test_bb,
Ks_test[view].copy(),
top_nn,
train_args,
target_source_length_ratio=target_source_length_ratio)
ae_time = time.time() - start
run_time = ae_time + bb_preds[view][0][
'det_time'] if estimate_bbs else ae_time
if eval_args.getboolean('PLOT', 'EMBEDDING_PCA'):
test_embeddings[-1].append(
codebook.test_embedding(sess,
test_crop,
normalized=True))
# icp = False if view<350 else True
#TODO:
Rs_est_old, ts_est_old = Rs_est.copy(), ts_est.copy()
for p in xrange(top_nn):
if icp:
start = time.time()
# icp only along tz
R_est_refined, t_est_refined = icp_utils.icp_refinement(
test_crops_depth[i],
icp_renderer,
Rs_est[p],
ts_est[p],
Ks_test[view].copy(), (W_test, H_test),
depth_only=True,
max_mean_dist_factor=5.0)
print ts_est[p]
print t_est_refined
# x,y update,does not change tz:
_, ts_est_refined = codebook.auto_pose6d(
sess,
test_crop,
test_bb,
Ks_test[view].copy(),
top_nn,
train_args,
depth_pred=t_est_refined[2])
# commented by lxc
# _, ts_est_refined, _ = codebook.auto_pose6d(sess, test_crop, test_bb, Ks_test[view].copy(), top_nn, train_args,depth_pred=t_est_refined[2])
t_est_refined = ts_est_refined[p]
# rotation icp, only accepted if below 20 deg change
R_est_refined, _ = icp_utils.icp_refinement(
test_crops_depth[i],
icp_renderer,
R_est_refined,
t_est_refined,
Ks_test[view].copy(), (W_test, H_test),
no_depth=True)
print Rs_est[p]
print R_est_refined
icp_time = time.time() - start
Rs_est[p], ts_est[p] = R_est_refined, t_est_refined
preds.setdefault('ests', []).append({
'score': test_score,
'R': Rs_est[p],
't': ts_est[p]
})
run_time = run_time + icp_time if icp else run_time
min_t_err, min_R_err = eval_plots.print_trans_rot_errors(
gts[view], obj_id, ts_est, ts_est_old, Rs_est, Rs_est_old)
t_errors_crop.append(min_t_err)
R_errors_crop.append(min_R_err)
if eval_args.getboolean('PLOT', 'RECONSTRUCTION'):
eval_plots.plot_reconstruction_test(
sess, codebook._encoder, decoder, test_crop)
# eval_plots.plot_reconstruction_train(sess, decoder, nearest_train_codes[0])
if eval_args.getboolean('PLOT',
'NEAREST_NEIGHBORS') and not icp:
for R_est, t_est in zip(Rs_est, ts_est):
pred_views.append(
dataset.render_rot(R_est, downSample=2))
eval_plots.show_nearest_rotation(pred_views, test_crop,
view)
if eval_args.getboolean('PLOT', 'SCENE_WITH_ESTIMATE'):
eval_plots.plot_scene_with_estimate(
test_imgs[view].copy(),
icp_renderer.renderer if icp else dataset.renderer,
Ks_test[view].copy(), Rs_est_old[0], ts_est_old[0],
Rs_est[0], ts_est[0], test_bb, test_score, obj_id,
gts[view], bb_preds[view] if estimate_bbs else None)
if cv2.waitKey(1) == 32:
cv2.waitKey(0)
t_errors.append(t_errors_crop[np.argmin(
np.linalg.norm(np.array(t_errors_crop), axis=1))])
R_errors.append(R_errors_crop[np.argmin(
np.linalg.norm(np.array(t_errors_crop), axis=1))])
# save predictions in sixd format
res_path = os.path.join(scene_res_dir,
'%04d_%02d.yml' % (view, obj_id))
inout.save_results_sixd17(res_path, preds, run_time=run_time)
if not os.path.exists(os.path.join(eval_dir, 'latex')):
os.makedirs(os.path.join(eval_dir, 'latex'))
if not os.path.exists(os.path.join(eval_dir, 'figures')):
os.makedirs(os.path.join(eval_dir, 'figures'))
'''evaluation code
dataset_renderer renders source object model for evaluation;
If we need target object model for evaluation, go get a new renderer.
'''
if eval_args.getboolean('EVALUATION', 'COMPUTE_ERRORS'):
eval_calc_errors.eval_calc_errors(eval_args,
eval_dir,
dataset_renderer=dataset.renderer)
if eval_args.getboolean('EVALUATION', 'EVALUATE_ERRORS'):
eval_loc.match_and_eval_performance_scores(eval_args, eval_dir)
'''plot code'''
cyclo = train_args.getint('Embedding', 'NUM_CYCLO')
if eval_args.getboolean('PLOT', 'EMBEDDING_PCA'):
embedding = sess.run(codebook.embedding_normalized)
eval_plots.compute_pca_plot_embedding(eval_dir,
embedding[::cyclo],
np.array(test_embeddings[0]),
obj_id=obj_id)
if eval_args.getboolean('PLOT', 'VIEWSPHERE'):
eval_plots.plot_viewsphere_for_embedding(
dataset.viewsphere_for_embedding[::cyclo], eval_dir, obj_id=obj_id)
if eval_args.getboolean('PLOT', 'CUM_T_ERROR_HIST'):
eval_plots.plot_t_err_hist(np.array(t_errors), eval_dir, obj_id=obj_id)
eval_plots.plot_t_err_hist2(np.array(t_errors),
eval_dir,
obj_id=obj_id)
if eval_args.getboolean('PLOT', 'CUM_R_ERROR_HIST'):
eval_plots.plot_R_err_hist(eval_args, eval_dir, scenes)
eval_plots.plot_R_err_hist2(np.array(R_errors),
eval_dir,
obj_id=obj_id)
if eval_args.getboolean('PLOT', 'CUM_VSD_ERROR_HIST'):
eval_plots.plot_vsd_err_hist(eval_args, eval_dir, scenes)
if eval_args.getboolean('PLOT', 'VSD_OCCLUSION'):
eval_plots.plot_vsd_occlusion(eval_args, eval_dir, scenes,
np.array(all_test_visibs))
if eval_args.getboolean('PLOT', 'R_ERROR_OCCLUSION'):
eval_plots.plot_re_rect_occlusion(eval_args, eval_dir, scenes,
np.array(all_test_visibs))
if eval_args.getboolean('PLOT', 'ANIMATE_EMBEDDING_PCA'):
eval_plots.animate_embedding_path(test_embeddings[0])
if eval_args.getboolean('PLOT', 'RECONSTRUCTION_TEST_BATCH'):
eval_plots.plot_reconstruction_test_batch(sess,
codebook,
decoder,
test_img_crops,
noof_scene_views,
obj_id,
eval_dir=eval_dir)
# plt.show()
# calculate 6D pose errors
# print 'exiting ...'
# eval_calc_errors.eval_calc_errors(eval_args, eval_dir)
# calculate 6D pose errors
report = latex_report.Report(eval_dir, log_dir)
report.write_configuration(train_cfg_file_path, eval_cfg_file_path)
report.merge_all_tex_files()
report.include_all_figures()
report.save(open_pdf=False)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('experiment_name')
parser.add_argument('evaluation_name')
parser.add_argument('--eval_cfg', default='eval.cfg', required=False)
parser.add_argument('--at_step', default=None, type=str, required=False)
parser.add_argument('--model_path', default=None, required=True)
arguments = parser.parse_args()
full_name = arguments.experiment_name.split('/')
experiment_name = full_name.pop()
experiment_group = full_name.pop() if len(full_name) > 0 else ''
evaluation_name = arguments.evaluation_name
eval_cfg = arguments.eval_cfg
at_step = arguments.at_step
model_path = arguments.model_path
workspace_path = os.environ.get('AE_WORKSPACE_PATH')
log_dir = u.get_log_dir(workspace_path, experiment_name, experiment_group)
train_cfg_file_path = u.get_train_config_exp_file_path(
log_dir, experiment_name)
eval_cfg_file_path = u.get_eval_config_file_path(workspace_path,
eval_cfg=eval_cfg)
train_args = configparser.ConfigParser(inline_comment_prefixes="#")
eval_args = configparser.ConfigParser(inline_comment_prefixes="#")
train_args.read(train_cfg_file_path)
eval_args.read(eval_cfg_file_path)
#[DATA]
dataset_name = eval_args.get('DATA', 'DATASET')
obj_id = eval_args.getint('DATA', 'OBJ_ID')
scenes = eval(eval_args.get(
'DATA', 'SCENES')) if len(eval(eval_args.get(
'DATA',
'SCENES'))) > 0 else eval_utils.get_all_scenes_for_obj(eval_args)
cam_type = eval_args.get('DATA', 'cam_type')
data_params = dataset_params.get_dataset_params(dataset_name,
model_type='',
train_type='',
test_type=cam_type,
cam_type=cam_type)
#[BBOXES]
estimate_bbs = eval_args.getboolean('BBOXES', 'ESTIMATE_BBS')
gt_masks = eval_args.getboolean('BBOXES', 'gt_masks')
estimate_masks = eval_args.getboolean('BBOXES', 'estimate_masks')
#[METRIC]
top_nn = eval_args.getint('METRIC', 'TOP_N')
#[EVALUATION]
icp = eval_args.getboolean('EVALUATION', 'ICP')
gt_trans = eval_args.getboolean('EVALUATION', 'gt_trans')
iterative_code_refinement = eval_args.getboolean(
'EVALUATION', 'iterative_code_refinement')
H_AE = train_args.getint('Dataset', 'H')
W_AE = train_args.getint('Dataset', 'W')
evaluation_name = evaluation_name + '_icp' if icp else evaluation_name
evaluation_name = evaluation_name + '_bbest' if estimate_bbs else evaluation_name
evaluation_name = evaluation_name + '_maskest' if estimate_masks else evaluation_name
evaluation_name = evaluation_name + '_gttrans' if gt_trans else evaluation_name
evaluation_name = evaluation_name + '_gtmasks' if gt_masks else evaluation_name
evaluation_name = evaluation_name + '_refined' if iterative_code_refinement else evaluation_name
data = dataset_name + '_' + cam_type if len(cam_type) > 0 else dataset_name
if at_step is None:
checkpoint_file = u.get_checkpoint_basefilename(
log_dir,
False,
latest=train_args.getint('Training', 'NUM_ITER'),
joint=True)
else:
checkpoint_file = u.get_checkpoint_basefilename(log_dir,
False,
latest=at_step,
joint=True)
print(checkpoint_file)
eval_dir = u.get_eval_dir(log_dir, evaluation_name, data)
if not os.path.exists(eval_dir):
os.makedirs(eval_dir)
shutil.copy2(eval_cfg_file_path, eval_dir)
codebook, dataset = factory.build_codebook_from_name(experiment_name,
experiment_group,
return_dataset=True,
joint=True)
dataset._kw['model_path'] = [model_path]
dataset._kw['model'] = 'cad' if 'cad' in model_path else 'reconst'
dataset._kw['model'] = 'reconst' if 'reconst' in model_path else 'cad'
gpu_options = tf.GPUOptions(allow_growth=True,
per_process_gpu_memory_fraction=0.5)
config = tf.ConfigProto(gpu_options=gpu_options)
sess = tf.Session(config=config)
saver = tf.train.Saver()
saver.restore(sess, checkpoint_file)
t_errors = []
R_errors = []
all_test_visibs = []
external_path = eval_args.get('BBOXES', 'EXTERNAL')
test_embeddings = []
for scene_id in scenes:
test_imgs = eval_utils.load_scenes(scene_id, eval_args)
test_imgs_depth = eval_utils.load_scenes(
scene_id, eval_args, depth=True) if icp else None
if estimate_bbs:
print(external_path)
if external_path == 'False':
bb_preds = {}
for i, img in enumerate(test_imgs):
print((img.shape))
bb_preds[i] = ssd.detectSceneBBs(img,
min_score=.05,
nms_threshold=.45)
print(bb_preds)
else:
if estimate_masks:
bb_preds = inout.load_yaml(
os.path.join(
external_path,
'{:02d}/mask_rcnn_predict.yml'.format(scene_id)))
print(list(bb_preds[0][0].keys()))
mask_paths = glob.glob(
os.path.join(external_path,
'{:02d}/masks/*.npy'.format(scene_id)))
maskrcnn_scene_masks = [np.load(mp) for mp in mask_paths]
else:
maskrcnn_scene_masks = None
bb_preds = inout.load_yaml(
os.path.join(external_path,
'{:02d}.yml'.format(scene_id)))
test_img_crops, test_img_depth_crops, bbs, bb_scores, visibilities = eval_utils.generate_scene_crops(
test_imgs,
test_imgs_depth,
bb_preds,
eval_args, (H_AE, W_AE),
inst_masks=maskrcnn_scene_masks)
else:
test_img_crops, test_img_depth_crops, bbs, bb_scores, visibilities = eval_utils.get_gt_scene_crops(
scene_id,
eval_args,
train_args,
load_gt_masks=external_path if gt_masks else gt_masks)
if len(test_img_crops) == 0:
print(('ERROR: object %s not in scene %s' % (obj_id, scene_id)))
exit()
info = inout.load_info(
data_params['scene_info_mpath'].format(scene_id))
Ks_test = [
np.array(v['cam_K']).reshape(3, 3) for v in list(info.values())
]
######remove
gts = inout.load_gt(data_params['scene_gt_mpath'].format(scene_id))
visib_gts = inout.load_yaml(data_params['scene_gt_stats_mpath'].format(
scene_id, 15))
#######
W_test, H_test = data_params['test_im_size']
icp_renderer = icp_utils.SynRenderer(
train_args, dataset._kw['model_path'][0]) if icp else None
noof_scene_views = eval_utils.noof_scene_views(scene_id, eval_args)
test_embeddings.append([])
scene_res_dir = os.path.join(
eval_dir, '{scene_id:02d}'.format(scene_id=scene_id))
if not os.path.exists(scene_res_dir):
os.makedirs(scene_res_dir)
for view in range(noof_scene_views):
try:
test_crops, test_crops_depth, test_bbs, test_scores, test_visibs = eval_utils.select_img_crops(
test_img_crops[view][obj_id],
test_img_depth_crops[view][obj_id] if icp else None,
bbs[view][obj_id], bb_scores[view][obj_id],
visibilities[view][obj_id], eval_args)
except:
print('no detections')
continue
print(view)
preds = {}
pred_views = []
all_test_visibs.append(test_visibs[0])
t_errors_crop = []
R_errors_crop = []
for i, (test_crop, test_bb, test_score) in enumerate(
zip(test_crops, test_bbs, test_scores)):
start = time.time()
if train_args.getint('Dataset', 'C') == 1:
test_crop = cv2.cvtColor(test_crop,
cv2.COLOR_BGR2GRAY)[:, :, None]
Rs_est, ts_est, _ = codebook.auto_pose6d(
sess,
test_crop,
test_bb,
Ks_test[view].copy(),
top_nn,
train_args,
codebook._get_codebook_name(model_path),
refine=iterative_code_refinement)
Rs_est_old, ts_est_old = Rs_est.copy(), ts_est.copy()
ae_time = time.time() - start
if eval_args.getboolean('PLOT', 'EMBEDDING_PCA'):
test_embeddings[-1].append(
codebook.test_embedding(sess,
test_crop,
normalized=True))
if eval_args.getboolean('EVALUATION', 'gt_trans'):
ts_est = np.empty((top_nn, 3))
for n in range(top_nn):
smallest_diff = np.inf
for visib_gt, gt in zip(visib_gts[view], gts[view]):
if gt['obj_id'] == obj_id:
diff = np.sum(
np.abs(gt['obj_bb'] -
np.array(visib_gt['bbox_obj'])))
if diff < smallest_diff:
smallest_diff = diff
gt_obj = gt.copy()
print('Im there')
ts_est[n] = np.array(gt_obj['cam_t_m2c']).reshape(-1)
try:
run_time = ae_time + bb_preds[view][0][
'det_time'] if estimate_bbs else ae_time
except:
run_time = ae_time
for p in range(top_nn):
if icp:
# note: In the CVPR paper a different ICP was used
start = time.time()
# depth icp
R_est_refined, t_est_refined = icp_utils.icp_refinement(
test_crops_depth[i],
icp_renderer,
Rs_est[p],
ts_est[p],
Ks_test[view].copy(), (W_test, H_test),
depth_only=True,
max_mean_dist_factor=5.0)
print(t_est_refined)
# x,y update,does not change tz:
_, ts_est_refined, _ = codebook.auto_pose6d(
sess,
test_crop,
test_bb,
Ks_test[view].copy(),
top_nn,
train_args,
codebook._get_codebook_name(model_path),
depth_pred=t_est_refined[2],
refine=iterative_code_refinement)
t_est_refined = ts_est_refined[p]
# rotation icp, only accepted if below 20 deg change
R_est_refined, _ = icp_utils.icp_refinement(
test_crops_depth[i],
icp_renderer,
R_est_refined,
t_est_refined,
Ks_test[view].copy(), (W_test, H_test),
no_depth=True)
print((Rs_est[p]))
print(R_est_refined)
icp_time = time.time() - start
Rs_est[p], ts_est[p] = R_est_refined, t_est_refined
preds.setdefault('ests', []).append({
'score': test_score,
'R': Rs_est[p],
't': ts_est[p]
})
run_time = run_time + icp_time if icp else run_time
min_t_err, min_R_err = eval_plots.print_trans_rot_errors(
gts[view], obj_id, ts_est, ts_est_old, Rs_est, Rs_est_old)
t_errors_crop.append(min_t_err)
R_errors_crop.append(min_R_err)
if eval_args.getboolean('PLOT',
'NEAREST_NEIGHBORS') and not icp:
for R_est, t_est in zip(Rs_est, ts_est):
pred_views.append(
dataset.render_rot(R_est, downSample=2))
eval_plots.show_nearest_rotation(pred_views, test_crop,
view)
if eval_args.getboolean('PLOT', 'SCENE_WITH_ESTIMATE'):
eval_plots.plot_scene_with_estimate(
test_imgs[view].copy(),
icp_renderer.renderer if icp else dataset.renderer,
Ks_test[view].copy(), Rs_est_old[0], ts_est_old[0],
Rs_est[0], ts_est[0], test_bb, test_score, obj_id,
gts[view], bb_preds[view] if estimate_bbs else None)
if cv2.waitKey(1) == 32:
cv2.waitKey(0)
t_errors.append(t_errors_crop[np.argmin(
np.linalg.norm(np.array(t_errors_crop), axis=1))])
R_errors.append(R_errors_crop[np.argmin(
np.linalg.norm(np.array(t_errors_crop), axis=1))])
# save predictions in sixd format
res_path = os.path.join(scene_res_dir,
'%04d_%02d.yml' % (view, obj_id))
inout.save_results_sixd17(res_path, preds, run_time=run_time)
if not os.path.exists(os.path.join(eval_dir, 'latex')):
os.makedirs(os.path.join(eval_dir, 'latex'))
if not os.path.exists(os.path.join(eval_dir, 'figures')):
os.makedirs(os.path.join(eval_dir, 'figures'))
if eval_args.getboolean('EVALUATION', 'COMPUTE_ERRORS'):
eval_calc_errors.eval_calc_errors(eval_args, eval_dir)
if eval_args.getboolean('EVALUATION', 'EVALUATE_ERRORS'):
eval_loc.match_and_eval_performance_scores(eval_args, eval_dir)
cyclo = train_args.getint('Embedding', 'NUM_CYCLO')
if eval_args.getboolean('PLOT', 'EMBEDDING_PCA'):
embedding = sess.run(codebook.embedding_normalized)
eval_plots.compute_pca_plot_embedding(eval_dir, embedding[::cyclo],
np.array(test_embeddings[0]))
if eval_args.getboolean('PLOT', 'VIEWSPHERE'):
eval_plots.plot_viewsphere_for_embedding(
dataset.viewsphere_for_embedding[::cyclo], eval_dir)
if eval_args.getboolean('PLOT', 'CUM_T_ERROR_HIST'):
eval_plots.plot_t_err_hist(np.array(t_errors), eval_dir)
eval_plots.plot_t_err_hist2(np.array(t_errors), eval_dir)
if eval_args.getboolean('PLOT', 'CUM_R_ERROR_HIST'):
eval_plots.plot_R_err_recall(eval_args, eval_dir, scenes)
eval_plots.plot_R_err_hist2(np.array(R_errors), eval_dir)
if eval_args.getboolean('PLOT', 'CUM_VSD_ERROR_HIST'):
try:
eval_plots.plot_vsd_err_hist(eval_args, eval_dir, scenes)
except:
pass
if eval_args.getboolean('PLOT', 'VSD_OCCLUSION'):
try:
eval_plots.plot_vsd_occlusion(eval_args, eval_dir, scenes,
np.array(all_test_visibs))
except:
pass
if eval_args.getboolean('PLOT', 'R_ERROR_OCCLUSION'):
try:
eval_plots.plot_re_rect_occlusion(eval_args, eval_dir, scenes,
np.array(all_test_visibs))
except:
pass
if eval_args.getboolean('PLOT', 'ANIMATE_EMBEDDING_PCA'):
eval_plots.animate_embedding_path(test_embeddings[0])
report = latex_report.Report(eval_dir, log_dir)
report.write_configuration(train_cfg_file_path, eval_cfg_file_path)
report.merge_all_tex_files()
report.include_all_figures()
report.save(open_pdf=True)
def get_gt_scene_crops(scene_id, eval_args, train_args, load_gt_masks=False):
dataset_name = eval_args.get('DATA', 'DATASET')
cam_type = eval_args.get('DATA', 'CAM_TYPE')
icp = eval_args.getboolean('EVALUATION', 'ICP')
delta = eval_args.get('METRIC', 'VSD_DELTA')
workspace_path = os.environ.get('AE_WORKSPACE_PATH')
dataset_path = u.get_dataset_path(workspace_path)
H_AE = train_args.getint('Dataset', 'H')
W_AE = train_args.getint('Dataset', 'W')
cfg_string = str([scene_id] + eval_args.items('DATA') +
eval_args.items('BBOXES') + [H_AE])
cfg_string = cfg_string.encode('utf-8')
current_config_hash = hashlib.md5(cfg_string).hexdigest()
current_file_name = os.path.join(dataset_path,
current_config_hash + '.npz')
if os.path.exists(current_file_name):
data = np.load(current_file_name)
test_img_crops = data['test_img_crops'].item()
test_img_depth_crops = data['test_img_depth_crops'].item()
bb_scores = data['bb_scores'].item()
bb_vis = data['visib_gt'].item()
bbs = data['bbs'].item()
if not os.path.exists(current_file_name) or len(
test_img_crops) == 0 or len(test_img_depth_crops) == 0:
test_imgs = load_scenes(scene_id, eval_args)
test_imgs_depth = load_scenes(scene_id, eval_args,
depth=True) if icp else None
data_params = dataset_params.get_dataset_params(dataset_name,
model_type='',
train_type='',
test_type=cam_type,
cam_type=cam_type)
# only available for primesense, sixdtoolkit can generate
visib_gt = inout.load_yaml(data_params['scene_gt_stats_mpath'].format(
scene_id, delta))
gt = inout.load_gt(data_params['scene_gt_mpath'].format(scene_id))
gt_inst_masks = None
if load_gt_masks:
mask_paths = glob.glob(
os.path.join(load_gt_masks,
'{:02d}/masks/*.npy'.format(scene_id)))
gt_inst_masks = [np.load(mp) for mp in mask_paths]
test_img_crops, test_img_depth_crops, bbs, bb_scores, bb_vis = generate_scene_crops(
test_imgs,
test_imgs_depth,
gt,
eval_args, (H_AE, W_AE),
visib_gt=visib_gt,
inst_masks=gt_inst_masks)
np.savez(current_file_name,
test_img_crops=test_img_crops,
test_img_depth_crops=test_img_depth_crops,
bbs=bbs,
bb_scores=bb_scores,
visib_gt=bb_vis)
current_cfg_file_name = os.path.join(dataset_path,
current_config_hash + '.cfg')
with open(current_cfg_file_name, 'w') as f:
f.write(cfg_string)
print('created new ground truth crops!')
else:
print('loaded previously generated ground truth crops!')
print((len(test_img_crops), len(test_img_depth_crops)))
return (test_img_crops, test_img_depth_crops, bbs, bb_scores, bb_vis)
def eval_calc_errors(eval_args, eval_dir):
# Results for which the errors will be calculated
#-------------------------------------------------------------------------------
# result_base = '/path/to/results/'
# result_paths = [
# pjoin(result_base, 'hodan-iros15_hinterstoisser'),
# # pjoin(result_base, 'hodan-iros15_tless_primesense'),
# ]
#[METHOD]
method = eval_args.get('METHOD', 'METHOD')
#[DATA]
dataset = eval_args.get('DATA', 'DATASET')
test_type = eval_args.get('DATA', 'CAM_TYPE')
#[METRIC]
# Top N pose estimates (with the highest score) to be evaluated for each
# object in each image
n_top = eval_args.getint(
'EVALUATION', 'TOP_N_EVAL'
) # 0 = all estimates, -1 = given by the number of GT poses
n_top_str = eval_args.getint('METRIC', 'TOP_N')
# Pose error function
error_types = eval(eval_args.get(
'METRIC', 'ERROR_TYPE')) # 'vsd', 'adi', 'add', 'cou', 're', 'te'
# VSD parameters
vsd_delta = eval_args.getint('METRIC', 'VSD_DELTA')
vsd_tau = eval_args.getint('METRIC', 'VSD_TAU')
vsd_cost = eval_args.get('METRIC', 'VSD_COST') # 'step', 'tlinear'
result_path = eval_dir
print('Processing: ' + result_path)
# Other paths
#-------------------------------------------------------------------------------
for error_type in error_types:
# Mask of path to the output file with calculated errors
# errors_mpath = pjoin('{result_path}', '..', '..', 'eval', '{result_name}',
# '{error_sign}', 'errors_{scene_id:02d}.yml')
errors_mpath = '{result_path}/{error_sign}/errors_{scene_id:02d}.yml'
# Error signature
error_sign = 'error=' + error_type + '_ntop=' + str(n_top_str)
if error_type == 'vsd':
error_sign += '_delta={}_tau={}_cost={}'.format(
vsd_delta, vsd_tau, vsd_cost)
# Error calculation
#-------------------------------------------------------------------------------
# Select data type
if dataset == 'tless':
cam_type = test_type
if error_type in ['adi', 'add']:
model_type = 'cad_subdivided'
else:
model_type = 'cad'
else:
model_type = ''
cam_type = ''
# Load dataset parameters
dp = get_dataset_params(dataset,
model_type=model_type,
test_type=test_type,
cam_type=cam_type)
# Load object models
if error_type in ['vsd', 'add', 'adi', 'cou', 'proj', 'projamb']:
print('Loading object models...')
models = {}
for obj_id in range(1, dp['obj_count'] + 1):
models[obj_id] = inout.load_ply(
dp['model_mpath'].format(obj_id))
test_sensor = pjoin(dp['base_path'], dp['test_dir'])
# Directories with results for individual scenes
scene_dirs = sorted([
d for d in glob.glob(os.path.join(result_path, '*'))
if os.path.isdir(d) and os.path.basename(d).isdigit()
])
print scene_dirs
for scene_dir in scene_dirs:
scene_id = int(os.path.basename(scene_dir))
# Load info and GT poses for the current scene
scene_info = inout.load_info(
dp['scene_info_mpath'].format(scene_id))
scene_gt = inout.load_gt(dp['scene_gt_mpath'].format(scene_id))
res_paths = sorted(glob.glob(os.path.join(scene_dir, '*.yml')))
errs = []
im_id = -1
depth_im = None
for res_id, res_path in enumerate(res_paths):
# t = time.time()
# Parse image ID and object ID from the filename
filename = os.path.basename(res_path).split('.')[0]
im_id_prev = im_id
im_id, obj_id = map(int, filename.split('_'))
if res_id % 10 == 0:
dataset_str = dataset
if test_type != '':
dataset_str += ' - {}'.format(test_type)
print('Calculating error: {}, {}, {}, {}, {}, {}'.format(
error_type, method, dataset_str, scene_id, im_id,
obj_id))
# Load depth image if VSD is selected
if error_type == 'vsd' and im_id != im_id_prev:
depth_path = dp['test_depth_mpath'].format(scene_id, im_id)
# depth_im = inout.load_depth(depth_path)
depth_im = inout.load_depth2(depth_path) # Faster
depth_im *= dp['cam']['depth_scale'] # to [mm]
# Load camera matrix
if error_type in ['vsd', 'cou', 'proj', 'projamb']:
K = scene_info[im_id]['cam_K']
# Load pose estimates
res = inout.load_results_sixd17(res_path)
ests = res['ests']
# Sort the estimates by score (in descending order)
ests_sorted = sorted(enumerate(ests),
key=lambda x: x[1]['score'],
reverse=True)
# Select the required number of top estimated poses
if n_top == 0: # All estimates are considered
n_top_curr = None
elif n_top == -1: # Given by the number of GT poses
n_gt = sum(
[gt['obj_id'] == obj_id for gt in scene_gt[im_id]])
n_top_curr = n_gt
else:
n_top_curr = n_top
ests_sorted = ests_sorted[slice(0, n_top_curr)]
for est_id, est in ests_sorted:
est_errs = []
R_e = est['R']
t_e = est['t']
errs_gts = {} # Errors w.r.t. GT poses of the same object
for gt_id, gt in enumerate(scene_gt[im_id]):
if gt['obj_id'] != obj_id:
continue
e = -1.0
R_g = gt['cam_R_m2c']
t_g = gt['cam_t_m2c']
if error_type == 'vsd':
e = pose_error.vsd(R_e, t_e, R_g, t_g,
models[obj_id], depth_im, K,
vsd_delta, vsd_tau, vsd_cost)
elif error_type == 'add':
e = pose_error.add(R_e, t_e, R_g, t_g,
models[obj_id])
elif error_type == 'adi':
e = pose_error.adi(R_e, t_e, R_g, t_g,
models[obj_id])
elif error_type == 'proj':
e = pose_error.arp_2d(R_e, t_e, R_g, t_g,
models[obj_id], K)
elif error_type == 'projamb':
e = pose_error.arpi_2d(R_e, t_e, R_g, t_g,
models[obj_id], K)
elif error_type == 'cou':
e = pose_error.cou(R_e, t_e, R_g, t_g,
models[obj_id],
dp['test_im_size'], K)
elif error_type == 're':
e = pose_error.re(R_e, R_g)
elif error_type == 'te':
e = pose_error.te(t_e, t_g)
errs_gts[gt_id] = e
errs.append({
'im_id': im_id,
'obj_id': obj_id,
'est_id': est_id,
'score': est['score'],
'errors': errs_gts
})
# print('Evaluation time: {}s'.format(time.time() - t))
print('Saving errors...')
errors_path = errors_mpath.format(result_path=result_path,
error_sign=error_sign,
scene_id=scene_id)
misc.ensure_dir(os.path.dirname(errors_path))
inout.save_errors(errors_path, errs)
print('')
print('Done.')
return True
def main():
visualize = True
gt_masks = False
dataset = 'tless'
num_train_imgs = 80000
max_objects_in_scene = 6
noofvoc_imgs = 15000
min_visib = 0.6
blackness_thres = 16
vocpath = '/home_local_nvme/sund_ma/data/VOCdevkit/VOC2012/JPEGImages/*.jpg'
voc_img_pathes = glob.glob(vocpath)
output_path = '/home_local_nvme/sund_ma/data/scene_renderings/linemod_real_imgs_voc_rotated/01/rgb'
if dataset == 'linemod':
sixd_train_path = '/home_local_nvme/sund_ma/data/train_linemod'
cad_path = '/home_local/sund_ma/data/linemod_dataset/models'
W, H = 640,480
noofobjects = 15
elif dataset == 'tless':
sixd_train_path = '/home_local/sund_ma/data/t-less/t-less_v2/train_primesense'
cad_path = '/home_local/sund_ma/data/t-less/t-less_v2/models_reconst'
W, H = 720,540
noofobjects = 30
# with ground truth masks
if gt_masks:
from auto_pose.meshrenderer import meshrenderer_phong
models_cad_files = sorted(glob.glob(os.path.join(cad_path,'*.ply')))
renderer = meshrenderer_phong.Renderer(
models_cad_files,
1
)
obj_gts = []
for obj_id in xrange(1,noofobjects+1):
obj_gts.append(inout.load_gt(os.path.join(sixd_train_path,'{:02d}'.format(obj_id),'gt.yml')))
if not os.path.exists(output_path):
os.makedirs(output_path)
voc_imgs = []
print 'loading bg'
for i,path in enumerate(voc_img_pathes[:noofvoc_imgs]):
voc_imgs.append(cv2.resize(cv2.imread(path),(W,H)))
print i,voc_imgs[-1].shape
obj_infos = []
for obj_id in xrange(1,noofobjects+1):
obj_infos.append(inout.load_info(os.path.join(sixd_train_path,'{:02d}'.format(obj_id),'info.yml')))
augmenters = Sequential([
Sometimes(0.2, GaussianBlur(0.4)),
Sometimes(0.1, AdditiveGaussianNoise(scale=10, per_channel=True)),
Sometimes(0.4, Add((-15, 15), per_channel=0.5)),
#Sometimes(0.3, Invert(0.2, per_channel=True)),
Sometimes(0.5, Multiply((0.6, 1.4), per_channel=0.5)),
# Sometimes(0.5, Multiply((0.6, 1.4))),
Sometimes(0.5, ContrastNormalization((0.5, 2.2), per_channel=0.3)),
#Sometimes(0.2, CoarseDropout( p=0.1, size_px = 10, size_percent=0.001) )
], random_order=True)
new_scene_gt = {}
bar = pb.ProgressBar(
maxval=num_train_imgs,
widgets=[' [', pb.Timer(), ' | ', pb.Counter('%0{}d / {}'.format(len(str(num_train_imgs)), num_train_imgs)), ' ] ',
pb.Bar(), ' (', pb.ETA(), ') ']
)
for i in bar(xrange(num_train_imgs)):
new_scene_gt[i] = []
new_train_img = np.zeros((H,W,3),dtype=np.uint8)
new_train_mask = np.zeros((H,W,1),dtype=np.uint8)
random_imgs = []
orig_bbs = []
random_trans = []
for k in xrange(max_objects_in_scene):
rand_obj_id = np.random.randint(0,noofobjects)
rand_view_id = np.random.randint(0,len(obj_infos[rand_obj_id]))
img_path = os.path.join(sixd_train_path,'{:02d}'.format(rand_obj_id+1),'rgb','{:04d}.png'.format(rand_view_id))
rand_img = cv2.imread(img_path)
# rand_depth_img = inout.load_depth2(os.path.join(sixd_train_path,'{:02d}'.format(rand_obj_id+1),'depth','{:04d}.png'.format(rand_view_id)))
# random rotate in-plane
rot_angle= np.random.rand()*360
M = cv2.getRotationMatrix2D((int(rand_img.shape[1]/2),int(rand_img.shape[0]/2)),rot_angle,1)
rand_img = cv2.warpAffine(rand_img, M, (rand_img.shape[1],rand_img.shape[0]))
# with ground truth masks
if gt_masks:
gt = obj_gts[rand_obj_id][rand_view_id][0]
K = obj_infos[rand_obj_id][rand_view_id]['cam_K']
_, depth = renderer.render(rand_obj_id,rand_img.shape[1],rand_img.shape[0],K,gt['cam_R_m2c'],gt['cam_t_m2c'],10,5000)
depth = cv2.warpAffine(depth, M, (depth.shape[1],depth.shape[0]))
mask = depth > 0
rand_img[mask == False] = 0
else:
rand_img[(rand_img[:,:,0] < blackness_thres) & (rand_img[:,:,1] < blackness_thres) & (rand_img[:,:,2] < blackness_thres)] = 0
mask = np.all(rand_img > 0,axis=2)
# print rand_img.shape,mask.shape
# cv2.imshow('mask2',mask.astype(np.float32))
# cv2.imshow('rand_img',rand_img)
# cv2.waitKey(0)
ys, xs = np.nonzero(mask)
new_bb = misc.calc_2d_bbox(xs, ys, mask.shape[:2])
if dataset == 'tless':
#tless specific
crop_x = np.array([20,380]) + np.random.randint(-15,15)
crop_y = np.array([20,380]) + np.random.randint(-15,15)
elif dataset == 'linemod':
#linemod specific
crop_x = np.array([80,560]) + np.random.randint(-20,20)
crop_y = np.array([0,480])# + np.random.randint(-20,20)
mask = mask[crop_y[0]:crop_y[1],crop_x[0]:crop_x[1]]
rand_img = rand_img[crop_y[0]:crop_y[1],crop_x[0]:crop_x[1]]
orig_H, orig_W = rand_img.shape[:2]
s = 0.5*np.random.rand()+0.5
new_H, new_W = int(s*orig_H),int(s*orig_W)
scaled_img = cv2.resize(rand_img,(new_W,new_H), interpolation=cv2.INTER_NEAREST)
scaled_mask = cv2.resize(mask.astype(np.int32).reshape(orig_W,orig_H,1),(new_W,new_H), interpolation=cv2.INTER_NEAREST)
y_offset = np.random.randint(0,H-scaled_img.shape[0])
x_offset = np.random.randint(0,W-scaled_img.shape[1])
y1, y2 = y_offset, y_offset + scaled_img.shape[0]
x1, x2 = x_offset, x_offset + scaled_img.shape[1]
alpha_s = np.dstack((scaled_mask,scaled_mask,scaled_mask)) > 0
alpha_l = 1.0 - alpha_s
old_train_mask = new_train_mask.copy()
new_train_mask[y1:y2, x1:x2, 0] = alpha_s[:,:,0] * scaled_mask + alpha_l[:,:,0] * new_train_mask[y1:y2, x1:x2, 0]
old_scene_pix = len(old_train_mask[y1:y2, x1:x2, 0]>0)
new_scene_pix = len(new_train_mask>0)
new_mask_pix = len(scaled_mask>0)
if (new_scene_pix-old_scene_pix)/float(new_mask_pix) < min_visib:
new_train_mask = old_train_mask.copy()
continue
new_train_img[y1:y2, x1:x2, :] = alpha_s * scaled_img + alpha_l * new_train_img[y1:y2, x1:x2, :]
x,y,w,h = np.round((np.array(new_bb)+np.array([-crop_x[0],-crop_y[0],0,0]))*s+np.array([x_offset,y_offset,0,0])).astype(np.int32)
# x,y,w,h = np.round(np.array(gt['obj_bb'])*s+np.array([x_offset,y_offset,0,0])).astype(np.int32)
new_scene_gt[i].append({'obj_id':rand_obj_id+1,'obj_bb':[x,y,w,h]})
bg = voc_imgs[np.random.randint(0,noofvoc_imgs)]
stacked_new_train_mask = np.dstack((new_train_mask,new_train_mask,new_train_mask))
new_train_img[stacked_new_train_mask==0] = bg[stacked_new_train_mask==0]
new_train_img = augmenters.augment_image(new_train_img)
if visualize:
print new_scene_gt[i]
for sc_gt in new_scene_gt[i]:
x,y,w,h = sc_gt['obj_bb']
cv2.rectangle(new_train_img, (x, y), (x+w, y+h), color=(32, 192, 192))
cv2.imshow('new_train_img', new_train_img)
cv2.imshow('new_train_mask', new_train_mask.astype(np.float32))
cv2.waitKey(0)
cv2.imwrite(os.path.join(output_path,'%s.png' % i), new_train_img)
with open(os.path.join(os.path.dirname(output_path),'gt.yml'), 'w') as f:
yaml.dump(new_scene_gt, f, Dumper=yaml.CDumper, width=10000)
def match_and_eval_performance_scores(eval_args, eval_dir):
# Paths to pose errors (calculated using eval_calc_errors.py)
#---------------------------------------------------------------------------
# error_bpath = '/path/to/eval/'
# error_paths = [
# pjoin(error_bpath, 'hodan-iros15_hinterstoisser'),
# # pjoin(error_bpath, 'hodan-iros15_tless_primesense'),
# ]
#[METHOD]
method = eval_args.get('METHOD', 'METHOD')
#[DATA]
dataset = eval_args.get('DATA', 'DATASET')
test_type = eval_args.get('DATA', 'CAM_TYPE')
#[METRIC]
# Top N pose estimates (with the highest score) to be evaluated for each
# object in each image
n_top = eval_args.getint(
'EVALUATION', 'TOP_N_EVAL'
) # 0 = all estimates, -1 = given by the number of GT poses
n_top_str = eval_args.getint('METRIC', 'TOP_N')
# Pose error function
error_types = eval(eval_args.get(
'METRIC', 'ERROR_TYPE')) # 'vsd', 'adi', 'add', 'cou', 're', 'te'
error_thresh = eval(eval_args.get('METRIC', 'ERROR_THRES'))
error_thresh_fact = eval(eval_args.get('METRIC', 'ERROR_THRESH_FACT'))
# VSD parameters
vsd_delta = eval_args.getint('METRIC', 'VSD_DELTA')
vsd_tau = eval_args.getint('METRIC', 'VSD_TAU')
vsd_cost = eval_args.get('METRIC', 'VSD_COST') # 'step', 'tlinear'
idx_th = 0
idx_thf = 0
for error_type in error_types:
# Error signature
error_sign = 'error=' + error_type + '_ntop=' + str(n_top_str)
if error_type == 'vsd':
error_sign += '_delta={}_tau={}_cost={}'.format(
vsd_delta, vsd_tau, vsd_cost)
error_path = os.path.join(eval_dir, error_sign)
# error_dir = 'error=vsd_ntop=1_delta=15_tau=20_cost=step'
# Other paths
#---------------------------------------------------------------------------
# Mask of path to the input file with calculated errors
errors_mpath = pjoin('{error_path}', 'errors_{scene_id:02d}.yml')
# Mask of path to the output file with established matches and calculated scores
matches_mpath = pjoin('{error_path}', 'matches_{eval_sign}.yml')
scores_mpath = pjoin('{error_path}', 'scores_{eval_sign}.yml')
# Parameters
#---------------------------------------------------------------------------
require_all_errors = False # Whether to break if some errors are missing
visib_gt_min = 0.1 # Minimum visible surface fraction of valid GT pose
visib_delta = 15 # [mm]
# # Threshold of correctness
# error_thresh = {
# 'vsd': 0.3,
# 'cou': 0.5,
# 'te': 5.0, # [cm]
# 're': 5.0 # [deg]
# }
# # Factor k; threshold of correctness = k * d, where d is the object diameter
# error_thresh_fact = {
# 'add': 0.1,
# 'adi': 0.1
# }
# Evaluation
#---------------------------------------------------------------------------
# Evaluation signature
if error_type in ['add', 'adi']:
if type(error_thresh_fact[error_type]) is list:
cur_thres_f = error_thresh_fact[error_type][idx_thf]
idx_thf += 1
else:
cur_thres_f = error_thresh_fact[error_type]
eval_sign = 'thf=' + str(cur_thres_f)
else:
if type(error_thresh[error_type]) is list:
cur_thres = error_thresh[error_type][idx_th]
idx_th += 1
else:
cur_thres = error_thresh[error_type]
eval_sign = 'th=' + str(cur_thres)
eval_sign += '_min-visib=' + str(visib_gt_min)
print('--- Processing: {}, {}, {}'.format(method, dataset, error_type))
# Load dataset parameters
dp = get_dataset_params(dataset, test_type=test_type)
obj_ids = range(1, dp['obj_count'] + 1)
scene_ids = range(1, dp['scene_count'] + 1)
# Set threshold of correctness (might be different for each object)
error_threshs = {}
if error_type in ['add', 'adi']:
# Relative to object diameter
models_info = inout.load_yaml(dp['models_info_path'])
for obj_id in obj_ids:
obj_diameter = models_info[obj_id]['diameter']
error_threshs[obj_id] = cur_thres_f * obj_diameter
else:
# The same threshold for all objects
for obj_id in obj_ids:
error_threshs[obj_id] = cur_thres
# Go through the test scenes and match estimated poses to GT poses
#-----------------------------------------------------------------------
matches = [] # Stores info about the matching estimate for each GT
for scene_id in scene_ids:
# Load GT poses
gts = inout.load_gt(dp['scene_gt_mpath'].format(scene_id))
# Load statistics (e.g. visibility fraction) of the GT poses
gt_stats_path = dp['scene_gt_stats_mpath'].format(
scene_id, visib_delta)
gt_stats = inout.load_yaml(gt_stats_path)
# Load pre-calculated errors of the pose estimates
scene_errs_path = errors_mpath.format(error_path=error_path,
scene_id=scene_id)
if os.path.isfile(scene_errs_path):
errs = inout.load_errors(scene_errs_path)
matches += match_poses(gts, gt_stats, errs, scene_id,
visib_gt_min, error_threshs, n_top)
elif require_all_errors:
raise IOError(
'{} is missing, but errors for all scenes are required'
' (require_all_results = True).'.format(scene_errs_path))
# Calculate the performance scores
#-----------------------------------------------------------------------
# Split the dataset of Hinterstoisser to the original LINEMOD dataset
# and the Occlusion dataset by TUD (i.e. the extended GT for scene #2)
if dataset == 'hinterstoisser':
print('-- LINEMOD dataset')
eval_sign_lm = 'linemod_' + eval_sign
matches_lm = [m for m in matches if m['scene_id'] == m['obj_id']]
scores_lm = calc_scores(scene_ids, obj_ids, matches_lm, n_top)
# Save scores
scores_lm_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign_lm)
inout.save_yaml(scores_lm_path, scores_lm)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign_lm)
inout.save_yaml(matches_path, matches_lm)
print('-- Occlusion dataset')
eval_sign_occ = 'occlusion_' + eval_sign
matches_occ = [m for m in matches if m['scene_id'] == 2]
scene_ids_occ = [2]
obj_ids_occ = [1, 2, 5, 6, 8, 9, 10, 11, 12]
scores_occ = calc_scores(scene_ids_occ, obj_ids_occ, matches_occ,
n_top)
# Save scores
scores_occ_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign_occ)
inout.save_yaml(scores_occ_path, scores_occ)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign_occ)
inout.save_yaml(matches_path, matches_occ)
else:
scores = calc_scores(scene_ids, obj_ids, matches, n_top)
# Save scores
scores_path = scores_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(scores_path, scores)
# Save matches
matches_path = matches_mpath.format(error_path=error_path,
eval_sign=eval_sign)
inout.save_yaml(matches_path, matches)
print('Done.')
