ests = inout.load_bop_results(
os.path.join(p['results_path'], result_filename))
# Organize the pose estimates by scene, image and object.
misc.log('Organizing pose estimates...')
ests_org = {}
for est in ests:
ests_org.setdefault(est['scene_id'],
{}).setdefault(est['im_id'],
{}).setdefault(est['obj_id'],
[]).append(est)
for scene_id, scene_targets in targets_org.items():
# Load camera and GT poses for the current scene.
scene_camera = inout.load_scene_camera(
dp_split['scene_camera_tpath'].format(scene_id=scene_id))
scene_gt = inout.load_scene_gt(
dp_split['scene_gt_tpath'].format(scene_id=scene_id))
scene_errs = []
for im_ind, (im_id, im_targets) in enumerate(scene_targets.items()):
if im_ind % 10 == 0:
misc.log(
'Calculating error {} - method: {}, dataset: {}{}, scene: {}, '
'im: {}'.format(p['error_type'], method, dataset,
split_type_str, scene_id, im_ind))
# Intrinsic camera matrix.
K = scene_camera[im_id]['cam_K']
model_type = None
if p['dataset'] == 'tless':
model_type = 'cad'
dp_model = dataset_params.get_model_params(p['datasets_path'], p['dataset'],
model_type)
for scene_id in dp_split['scene_ids']:
# Load scene GT.
scene_gt_path = dp_split['scene_gt_tpath'].format(scene_id=scene_id)
scene_gt = inout.load_scene_gt(scene_gt_path)
# Load scene camera.
scene_camera_path = dp_split['scene_camera_tpath'].format(
scene_id=scene_id)
scene_camera = inout.load_scene_camera(scene_camera_path)
# Create folders for the output masks (if they do not exist yet).
mask_dir_path = os.path.dirname(dp_split['mask_tpath'].format(
scene_id=scene_id, im_id=0, gt_id=0))
misc.ensure_dir(mask_dir_path)
mask_visib_dir_path = os.path.dirname(dp_split['mask_visib_tpath'].format(
scene_id=scene_id, im_id=0, gt_id=0))
misc.ensure_dir(mask_visib_dir_path)
# Initialize a renderer.
misc.log('Initializing renderer...')
width, height = dp_split['im_size']
ren = renderer.create_renderer(width,
height,
if(dummy_run):
#to activate networks before atual recognition
#since only the first run takes longer time
image_t = np.zeros((im_height,im_width,3),np.uint8)
if(detect_type=='rcnn'):
rois,obj_orders,obj_ids,scores,masks = get_rcnn_detection(image_t,model)
elif(detect_type=='retinanet'):
rois,obj_orders,obj_ids,scores = get_retinanet_detection(image_t,model)
for scene_id,im_id,obj_id_targets,inst_counts in target_list:
print("Recognizing scene_id:{}, im_id:{}".format(scene_id,im_id))
if(prev_sid!=scene_id):
cam_path = test_dir+"/{:06d}/scene_camera.json".format(scene_id)
cam_info = inout.load_scene_camera(cam_path)
if(dummy_run):
image_t = np.zeros((im_height,im_width,3),np.uint8)
for obj_id_target in obj_id_targets: #refreshing
_,_,_,_,_,_ = obj_pix2pose[model_ids_list.index(obj_id_target)].est_pose(image_t,np.array([0,0,128,128],np.int))
prev_sid=scene_id #to avoid re-load scene_camera.json
cam_param = cam_info[im_id]
cam_K = cam_param['cam_K']
depth_scale = cam_param['depth_scale'] #depth/1000 * depth_scale
if(img_type=='gray'):
rgb_path = test_dir+"/{:06d}/".format(scene_id)+img_type+\
"/{:06d}.tif".format(im_id)
image_gray = inout.load_im(rgb_path)
#copy gray values to three channels
def get_dataset(cfg,dataset,train=True,incl_param=False,eval=False,eval_model=False):
#return serialized datset information
bop_dir = cfg['dataset_dir']
if eval_model:
postfix_model = '_eval'
else:
postfix_model = ''
if(dataset=='lmo'):
bop_dataset_dir = os.path.join(bop_dir,"lmo")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.001
elif(dataset=='ruapc'):
bop_dataset_dir = os.path.join(bop_dir,"ruapc")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.001
elif(dataset=='hb'):
bop_dataset_dir = os.path.join(bop_dir,"hb")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.0001
elif(dataset=='icbin'):
bop_dataset_dir = os.path.join(bop_dir,"icbin")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.001
elif(dataset=='itodd'):
bop_dataset_dir = os.path.join(bop_dir,"itodd")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.001
elif(dataset=='tudl'):
bop_dataset_dir = os.path.join(bop_dir,"tudl")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train_real"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.001
elif(dataset=='tless'):
bop_dataset_dir = os.path.join(bop_dir,"tless")
test_dir = bop_dataset_dir+"/test_primesense"
train_dir = bop_dataset_dir+"/train_primesense"
if not(train) and not(eval_model):
model_dir = bop_dataset_dir+"/models_reconst" #use this only for vis
elif eval_model:
model_dir = bop_dataset_dir+"/models_eval"
else:
model_dir = bop_dataset_dir+"/models_cad"
model_scale=0.001
elif(dataset=='ycbv'):
bop_dataset_dir = os.path.join(bop_dir,"ycbv")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.001
elif(dataset=='lm'):
bop_dataset_dir = os.path.join(bop_dir,"lm")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_dir = "/home/kiru/media/hdd_linux/PoseDataset/hinterstoisser/model_eval"
model_scale=0.001
model_info = inout.load_json(os.path.join(model_dir,"models_info.json"))
if(dataset=='ycbv'):
cam_param_global = inout.load_cam_params(os.path.join(bop_dataset_dir,"camera_uw.json"))
else:
cam_param_global = inout.load_cam_params(os.path.join(bop_dataset_dir,"camera.json"))
im_size=np.array(cam_param_global['im_size'])[::-1]
model_plys=[]
rgb_files=[]
depth_files=[]
mask_files=[]
gts=[]
params=[]
model_ids = []
for model_id in model_info.keys():
ply_fn = os.path.join(model_dir,"obj_{:06d}.ply".format(int(model_id)))
if(os.path.exists(ply_fn)): model_ids.append(int(model_id)) #add model id only if the model.ply file exists
model_ids = np.sort(np.array(model_ids))
for model_id in model_ids:
ply_fn = os.path.join(model_dir,"obj_{:06d}.ply".format(int(model_id)))
model_plys.append(ply_fn)
print(model_id,ply_fn)
print("if models are not fully listed above, please make sure there are ply files available")
if(train):
target_dir =train_dir
if(os.path.exists(target_dir)):
for dir in os.listdir(target_dir): #loop over a seqeunce
current_dir = target_dir+"/"+dir
if os.path.exists(os.path.join(current_dir,"scene_camera.json")):
scene_params = inout.load_scene_camera(os.path.join(current_dir,"scene_camera.json"))
scene_gt_fn = os.path.join(current_dir,"scene_gt.json")
has_gt=False
if os.path.exists(scene_gt_fn):
scene_gts = inout.load_scene_gt(scene_gt_fn)
has_gt=True
for img_id in sorted(scene_params.keys()):
im_id = int(img_id)
if(dataset=="itodd" and not(train)):
rgb_fn = os.path.join(current_dir+"/gray","{:06d}.tif".format(im_id))
else:
rgb_fn = os.path.join(current_dir+"/rgb","{:06d}.png".format(im_id))
depth_fn = os.path.join(current_dir+"/depth","{:06d}.png".format(im_id))
if(train):
if(dataset=='hb' or dataset=='itodd' or dataset=='ycbv'):
mask_fn = os.path.join(current_dir+"/mask","{:06d}.png".format(im_id))
else:
mask_fn = os.path.join(current_dir+"/mask","{:06d}_000000.png".format(im_id))
mask_files.append(mask_fn)
rgb_files.append(rgb_fn)
depth_files.append(depth_fn)
if(has_gt):gts.append(scene_gts[im_id])
params.append(scene_params[im_id])
else:
target_dir =test_dir
if(incl_param):
return bop_dataset_dir,target_dir,model_plys,model_info,model_ids,rgb_files,depth_files,mask_files,gts,cam_param_global,params
else:
return bop_dataset_dir,target_dir,model_plys,model_info,model_ids,rgb_files,depth_files,mask_files,gts,cam_param_global
def preprocess_data():
# Create directory
Path(f'{SAVE_DIR}/image').mkdir(parents=True, exist_ok=True)
Path(f'{SAVE_DIR}/model').mkdir(parents=True, exist_ok=True)
# Parse metadata and load information of images with the target object
img_info = []
if not IS_TARGET:
for sc_id in range(*SCENE_ID_RANGE):
assert Path(f'{DATASET_DIR}/{sc_id:06d}/scene_gt.json').is_file()
scene_gt = io.load_scene_gt(
f'{DATASET_DIR}/{sc_id:06d}/scene_gt.json')
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/scene_camera.json').is_file()
scene_cam = io.load_scene_camera(
f'{DATASET_DIR}/{sc_id:06d}/scene_camera.json')
for (im_id, im_gt), im_cam in zip(scene_gt.items(),
scene_cam.values()):
for gt_id, gt in enumerate(im_gt):
if int(gt['obj_id']) == OBJ_ID:
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/depth/{im_id:06d}.png'
).is_file()
depth_path = f'{DATASET_DIR}/{sc_id:06d}/depth/{im_id:06d}.png'
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/mask_visib/{im_id:06d}_{gt_id:06d}.png'
).is_file()
mask_path = f'{DATASET_DIR}/{sc_id:06d}/mask_visib/{im_id:06d}_{gt_id:06d}.png'
save_name = f'{sc_id:06d}_{im_id:06d}_{gt_id:06d}'
img_info.append(
[depth_path, mask_path, im_cam, gt, save_name])
else:
targets = io.load_json(
f'{Path(DATASET_DIR).parent}/test_targets_bop19.json')
for target in targets:
if int(target['obj_id']) == OBJ_ID:
sc_id = target['scene_id']
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/scene_gt.json').is_file()
scene_gt = io.load_scene_gt(
f'{DATASET_DIR}/{sc_id:06d}/scene_gt.json')
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/scene_camera.json').is_file()
scene_cam = io.load_scene_camera(
f'{DATASET_DIR}/{sc_id:06d}/scene_camera.json')
im_id = int(target['im_id'])
im_gt, im_cam = scene_gt[im_id], scene_cam[im_id]
for gt_id, gt in enumerate(im_gt):
if int(gt['obj_id']) == OBJ_ID:
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/depth/{im_id:06d}.png'
).is_file()
depth_path = f'{DATASET_DIR}/{sc_id:06d}/depth/{im_id:06d}.png'
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/mask_visib/{im_id:06d}_{gt_id:06d}.png'
).is_file()
mask_path = f'{DATASET_DIR}/{sc_id:06d}/mask_visib/{im_id:06d}_{gt_id:06d}.png'
save_name = f'{sc_id:06d}_{im_id:06d}_{gt_id:06d}'
img_info.append(
[depth_path, mask_path, im_cam, gt, save_name])
# Read model point cloud
model_file = f'{Path(DATASET_DIR).parent}/models/obj_{OBJ_ID:06d}.ply'
assert Path(model_file).is_file()
model_pcd = read_point_cloud(model_file).voxel_down_sample(VOXEL_SIZE)
md_pcd_pts = np.asarray(model_pcd.points)
# Create point cloud from image information
t1 = time()
with get_context('spawn').Pool(8) as pool:
jobs = [
pool.apply_async(create_patch_pair, (*info, md_pcd_pts))
for info in img_info
]
df = pd.DataFrame([j.get() for j in jobs]).dropna(axis=0)
if Path(f'{SAVE_DIR}/labels.csv').is_file():
old_df = pd.read_csv(f'{SAVE_DIR}/labels.csv', header=None)
df = pd.concat([old_df, df])
df.to_csv(f'{SAVE_DIR}/labels.csv',
header=['filename', 'num_patches'],
index=False)
t2 = time()
print(
f'Created patch_pairs from {len(df)} images. Time elapsed: {t2 - t1 :.3f}'
)
step=augment_inplane)
xyz_id += 1
if (dataset == "ycbv"):
#for ycbv, exceptionally extract patches
#from cluttered training images.
print("using real images for training")
train_real_dir = bop_dir + "/train_real"
for folder in sorted(os.listdir(train_real_dir)):
sub_dir = os.path.join(train_real_dir, folder)
print("Processing:", sub_dir)
scene_gt_fn = os.path.join(sub_dir, "scene_gt.json")
scene_gts = inout.load_scene_gt(scene_gt_fn)
scene_gt_info = inout.load_json(
os.path.join(sub_dir, "scene_gt_info.json"))
scene_cam_fn = os.path.join(sub_dir, "scene_camera.json")
scene_cam_real = inout.load_scene_camera(scene_cam_fn)
scene_keys = scene_gts.keys()
n_scene = len(scene_keys)
for img_id in np.arange(1, n_scene, 100):
im_id = img_id
rgb_fn = os.path.join(sub_dir + "/rgb",
"{:06d}.png".format(im_id))
print(rgb_fn)
gts_real = scene_gts[im_id]
scene_info = scene_gt_info["{}".format(im_id)]
for gt_id in range(len(gts_real)):
gt = gts_real[gt_id]
obj_id = int(gt['obj_id'])
bbox = scene_info[gt_id]['bbox_obj']
visib_fract = scene_info[gt_id]['visib_fract']
#skip objects in the boundary
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
# Load the list examples.
examples_path = os.path.join(config.TF_DATA_PATH, 'example_lists',
FLAGS.examples_filename)
tf.logging.info(
'Loading a list of examples from: {}'.format(examples_path))
examples_list = tfrecord.load_example_list(examples_path)
# Load dataset parameters.
dp_split = dataset_params.get_split_params(config.BOP_PATH, FLAGS.dataset,
FLAGS.split, FLAGS.split_type)
# Pre-load camera parameters and ground-truth annotations.
scene_gt = {}
scene_gt_info = {}
scene_camera = {}
scene_ids = set([e['scene_id'] for e in examples_list])
for scene_id in scene_ids:
scene_camera[scene_id] = inout.load_scene_camera(
dp_split['scene_camera_tpath'].format(scene_id=scene_id))
if FLAGS.add_gt:
scene_gt[scene_id] = inout.load_scene_gt(
dp_split['scene_gt_tpath'].format(scene_id=scene_id))
scene_gt_info[scene_id] = inout.load_json(
dp_split['scene_gt_info_tpath'].format(scene_id=scene_id),
keys_to_int=True)
# Check the name of the file with examples.
examples_end = '_examples.txt'
if not FLAGS.examples_filename.endswith(examples_end):
raise ValueError(
'Name of the file with examples must end with {}.'.format(
examples_end))
# Prepare writer of the TFRecord file.
output_name = FLAGS.examples_filename.split(examples_end)[0]
output_path = os.path.join(FLAGS.output_dir, output_name + '.tfrecord')
writer = tf.python_io.TFRecordWriter(output_path)
tf.logging.info('File to be created: {}'.format(output_path))
# Optionally shuffle the examples.
if FLAGS.shuffle:
random.shuffle(examples_list)
# Write the examples to the TFRecord file.
w_start_t = time.time()
create_tf_example_partial = partial(create_tf_example,
dp_split=dp_split,
scene_camera=scene_camera,
scene_gt=scene_gt,
scene_gt_info=scene_gt_info)
for example_id, example in enumerate(examples_list):
if example_id % 50 == 0:
tf.logging.info('Processing example {}/{}'.format(
example_id + 1, len(examples_list)))
tf_example, _ = create_tf_example_partial(example)
writer.write(tf_example)
# Close the writer.
writer.close()
w_total_t = time.time() - w_start_t
tf.logging.info('Writing took {} s.'.format(w_total_t))
import matplotlib.pyplot as plt
import transforms3d as tf3d
import numpy as np
import time
from bop_toolkit_lib import inout
from tools import bop_io
import copy
#YCB(have to check) - > LMO
#HB, ITODD -> T-LESS
ref_gt = inout.load_scene_gt(
os.path.join("/home/kiru/media/hdd/bop/tless/train_render_reconst/000001",
"scene_gt.json"))
ref_camera = inout.load_scene_camera(
os.path.join("/home/kiru/media/hdd/bop/tless/train_render_reconst/000001",
"scene_camera.json"))
#bop_dir,source_dir,model_plys,model_info,model_ids,rgb_files,depth_files,mask_files,gts,cam_param_global = bop_io.get_dataset('hb',train=True)
#bop_dir,source_dir,model_plys,model_info,model_ids,rgb_files,depth_files,mask_files,gts,cam_param_global = bop_io.get_dataset('itodd',train=True)
bop_dir, source_dir, model_plys, model_info, model_ids, rgb_files, depth_files, mask_files, gts, cam_param_global = bop_io.get_dataset(
'ycbv', train=True)
im_width, im_height = cam_param_global['im_size']
camK = cam_param_global['K']
cam_K_list = np.array(camK).reshape(-1)
ren = Renderer((im_width, im_height), camK)
source_dir = bop_dir + "/train"
if not (os.path.exists(source_dir)): os.makedirs(source_dir)
for i in range(len(model_plys)):
