def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
if FLAGS.scene_ids is not None and FLAGS.targets_filename is not None:
raise ValueError(
'Only up to one of scene_ids and targets_filename can be specified.')
# Load dataset parameters.
dp_split = dataset_params.get_split_params(
config.BOP_PATH, FLAGS.dataset, FLAGS.split, FLAGS.split_type)
output_suffix = None
if FLAGS.targets_filename:
output_suffix = 'targets'
test_targets = inout.load_json(
os.path.join(config.BOP_PATH, FLAGS.dataset, FLAGS.targets_filename))
example_list = []
for trg in test_targets:
example = {'scene_id': trg['scene_id'], 'im_id': trg['im_id']}
if example not in example_list:
example_list.append(example)
else:
if FLAGS.scene_ids is None:
FLAGS.scene_ids = dataset_params.get_present_scene_ids(dp_split)
else:
FLAGS.scene_ids = list(map(int, FLAGS.scene_ids))
output_suffix = 'scenes-' + '-'.join(
map(lambda x: '{:01d}'.format(x), FLAGS.scene_ids))
tf.logging.info('Collecting examples...')
example_list = []
for scene_id in FLAGS.scene_ids:
scene_gt_fpath = dp_split['scene_gt_tpath'].format(scene_id=scene_id)
im_ids = inout.load_scene_gt(scene_gt_fpath).keys()
for im_id in sorted(im_ids):
example_list.append({'scene_id': scene_id, 'im_id': im_id})
tf.logging.info('Collected {} examples.'.format(len(example_list)))
assert(len(example_list) > 0)
split_name = FLAGS.split
if FLAGS.split_type is not None:
split_name += '-' + FLAGS.split_type
if output_suffix is not None:
output_suffix = '_' + output_suffix
else:
output_suffix = ''
output_fname = '{}_{}{}_examples.txt'.format(
FLAGS.dataset, split_name, output_suffix)
output_fpath = os.path.join(FLAGS.output_dir, output_fname)
tf.logging.info('Saving the list to: {}'.format(output_fpath))
if not os.path.exists(FLAGS.output_dir):
os.makedirs(FLAGS.output_dir)
tfrecord.save_example_list(output_fpath, example_list)
# Organize the pose estimates by scene, image and object.
misc.log('Organizing pose estimates...')
ests_org = {}
for est in ests:
ests_org.setdefault(est['scene_id'],
{}).setdefault(est['im_id'],
{}).setdefault(est['obj_id'],
[]).append(est)
for scene_id, scene_targets in targets_org.items():
# Load camera and GT poses for the current scene.
scene_camera = inout.load_scene_camera(
dp_split['scene_camera_tpath'].format(scene_id=scene_id))
scene_gt = inout.load_scene_gt(
dp_split['scene_gt_tpath'].format(scene_id=scene_id))
scene_errs = []
for im_ind, (im_id, im_targets) in enumerate(scene_targets.items()):
if im_ind % 10 == 0:
misc.log(
'Calculating error {} - method: {}, dataset: {}{}, scene: {}, '
'im: {}'.format(p['error_type'], method, dataset,
split_type_str, scene_id, im_ind))
# Intrinsic camera matrix.
K = scene_camera[im_id]['cam_K']
# Load the depth image if VSD is selected as the pose error function.
# Load dataset parameters.
dp_split = dataset_params.get_split_params(p['datasets_path'], p['dataset'],
p['dataset_split'],
p['dataset_split_type'])
model_type = None
if p['dataset'] == 'tless':
model_type = 'cad'
dp_model = dataset_params.get_model_params(p['datasets_path'], p['dataset'],
model_type)
for scene_id in dp_split['scene_ids']:
# Load scene GT.
scene_gt_path = dp_split['scene_gt_tpath'].format(scene_id=scene_id)
scene_gt = inout.load_scene_gt(scene_gt_path)
# Load scene camera.
scene_camera_path = dp_split['scene_camera_tpath'].format(
scene_id=scene_id)
scene_camera = inout.load_scene_camera(scene_camera_path)
# Create folders for the output masks (if they do not exist yet).
mask_dir_path = os.path.dirname(dp_split['mask_tpath'].format(
scene_id=scene_id, im_id=0, gt_id=0))
misc.ensure_dir(mask_dir_path)
mask_visib_dir_path = os.path.dirname(dp_split['mask_visib_tpath'].format(
scene_id=scene_id, im_id=0, gt_id=0))
misc.ensure_dir(mask_visib_dir_path)
def get_dataset(cfg,dataset,train=True,incl_param=False,eval=False,eval_model=False):
#return serialized datset information
bop_dir = cfg['dataset_dir']
if eval_model:
postfix_model = '_eval'
else:
postfix_model = ''
if(dataset=='lmo'):
bop_dataset_dir = os.path.join(bop_dir,"lmo")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.001
elif(dataset=='ruapc'):
bop_dataset_dir = os.path.join(bop_dir,"ruapc")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.001
elif(dataset=='hb'):
bop_dataset_dir = os.path.join(bop_dir,"hb")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.0001
elif(dataset=='icbin'):
bop_dataset_dir = os.path.join(bop_dir,"icbin")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.001
elif(dataset=='itodd'):
bop_dataset_dir = os.path.join(bop_dir,"itodd")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.001
elif(dataset=='tudl'):
bop_dataset_dir = os.path.join(bop_dir,"tudl")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train_real"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.001
elif(dataset=='tless'):
bop_dataset_dir = os.path.join(bop_dir,"tless")
test_dir = bop_dataset_dir+"/test_primesense"
train_dir = bop_dataset_dir+"/train_primesense"
if not(train) and not(eval_model):
model_dir = bop_dataset_dir+"/models_reconst" #use this only for vis
elif eval_model:
model_dir = bop_dataset_dir+"/models_eval"
else:
model_dir = bop_dataset_dir+"/models_cad"
model_scale=0.001
elif(dataset=='ycbv'):
bop_dataset_dir = os.path.join(bop_dir,"ycbv")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_scale=0.001
elif(dataset=='lm'):
bop_dataset_dir = os.path.join(bop_dir,"lm")
test_dir = bop_dataset_dir+"/test"
train_dir = bop_dataset_dir+"/train"
model_dir = bop_dataset_dir+"/models"+postfix_model
model_dir = "/home/kiru/media/hdd_linux/PoseDataset/hinterstoisser/model_eval"
model_scale=0.001
model_info = inout.load_json(os.path.join(model_dir,"models_info.json"))
if(dataset=='ycbv'):
cam_param_global = inout.load_cam_params(os.path.join(bop_dataset_dir,"camera_uw.json"))
else:
cam_param_global = inout.load_cam_params(os.path.join(bop_dataset_dir,"camera.json"))
im_size=np.array(cam_param_global['im_size'])[::-1]
model_plys=[]
rgb_files=[]
depth_files=[]
mask_files=[]
gts=[]
params=[]
model_ids = []
for model_id in model_info.keys():
ply_fn = os.path.join(model_dir,"obj_{:06d}.ply".format(int(model_id)))
if(os.path.exists(ply_fn)): model_ids.append(int(model_id)) #add model id only if the model.ply file exists
model_ids = np.sort(np.array(model_ids))
for model_id in model_ids:
ply_fn = os.path.join(model_dir,"obj_{:06d}.ply".format(int(model_id)))
model_plys.append(ply_fn)
print(model_id,ply_fn)
print("if models are not fully listed above, please make sure there are ply files available")
if(train):
target_dir =train_dir
if(os.path.exists(target_dir)):
for dir in os.listdir(target_dir): #loop over a seqeunce
current_dir = target_dir+"/"+dir
if os.path.exists(os.path.join(current_dir,"scene_camera.json")):
scene_params = inout.load_scene_camera(os.path.join(current_dir,"scene_camera.json"))
scene_gt_fn = os.path.join(current_dir,"scene_gt.json")
has_gt=False
if os.path.exists(scene_gt_fn):
scene_gts = inout.load_scene_gt(scene_gt_fn)
has_gt=True
for img_id in sorted(scene_params.keys()):
im_id = int(img_id)
if(dataset=="itodd" and not(train)):
rgb_fn = os.path.join(current_dir+"/gray","{:06d}.tif".format(im_id))
else:
rgb_fn = os.path.join(current_dir+"/rgb","{:06d}.png".format(im_id))
depth_fn = os.path.join(current_dir+"/depth","{:06d}.png".format(im_id))
if(train):
if(dataset=='hb' or dataset=='itodd' or dataset=='ycbv'):
mask_fn = os.path.join(current_dir+"/mask","{:06d}.png".format(im_id))
else:
mask_fn = os.path.join(current_dir+"/mask","{:06d}_000000.png".format(im_id))
mask_files.append(mask_fn)
rgb_files.append(rgb_fn)
depth_files.append(depth_fn)
if(has_gt):gts.append(scene_gts[im_id])
params.append(scene_params[im_id])
else:
target_dir =test_dir
if(incl_param):
return bop_dataset_dir,target_dir,model_plys,model_info,model_ids,rgb_files,depth_files,mask_files,gts,cam_param_global,params
else:
return bop_dataset_dir,target_dir,model_plys,model_info,model_ids,rgb_files,depth_files,mask_files,gts,cam_param_global
def load(bop_dataset_path: str,
temp_dir: str,
sys_paths: list,
model_type: str = "",
cam_type: str = "",
split: str = "test",
scene_id: int = -1,
obj_ids: list = [],
sample_objects: bool = False,
num_of_objs_to_sample: int = None,
obj_instances_limit: int = -1,
move_origin_to_x_y_plane: bool = False,
source_frame: list = ["X", "-Y", "-Z"],
mm2m: bool = False) -> List[MeshObject]:
""" Loads the 3D models of any BOP dataset and allows replicating BOP scenes
- Interfaces with the bob_toolkit, allows loading of train, val and test splits
- Relative camera poses are loaded/computed with respect to a reference model
- Sets real camera intrinsics
:param bop_dataset_path: Full path to a specific bop dataset e.g. /home/user/bop/tless.
:param temp_dir: A temp directory which is used for writing the temporary .ply file.
:param sys_paths: System paths to append.
:param model_type: Optionally, specify type of BOP model. Available: [reconst, cad or eval].
:param cam_type: Camera type. If not defined, dataset-specific default camera type is used.
:param split: Optionally, test or val split depending on BOP dataset.
:param scene_id: Optionally, specify BOP dataset scene to synthetically replicate. Default: -1 (no scene is replicated,
only BOP Objects are loaded).
:param obj_ids: List of object ids to load. Default: [] (all objects from the given BOP dataset if scene_id is not
specified).
:param sample_objects: Toggles object sampling from the specified dataset.
:param num_of_objs_to_sample: Amount of objects to sample from the specified dataset. If this amount is bigger than the dataset
actually contains, then all objects will be loaded.
:param obj_instances_limit: Limits the amount of object copies when sampling. Default: -1 (no limit).
:param move_origin_to_x_y_plane: Move center of the object to the lower side of the object, this will not work when used in combination with
pose estimation tasks! This is designed for the use-case where BOP objects are used as filler objects in
the background.
:param source_frame: Can be used if the given positions and rotations are specified in frames different from the blender
frame. Has to be a list of three strings. Example: ['X', '-Z', 'Y']: Point (1,2,3) will be transformed
to (1, -3, 2). Available: ['X', 'Y', 'Z', '-X', '-Y', '-Z'].
:param mm2m: Specify whether to convert poses and models to meters.
:return: The list of loaded mesh objects.
"""
for sys_path in sys_paths:
if 'bop_toolkit' in sys_path:
sys.path.append(sys_path)
scale = 0.001 if mm2m else 1
bop_dataset_name = os.path.basename(bop_dataset_path)
has_external_texture = bop_dataset_name in ["ycbv", "ruapc"]
if obj_ids or sample_objects:
allow_duplication = True
else:
allow_duplication = False
datasets_path = os.path.dirname(bop_dataset_path)
dataset = os.path.basename(bop_dataset_path)
print("bob: {}, dataset_path: {}".format(bop_dataset_path,
datasets_path))
print("dataset: {}".format(dataset))
try:
from bop_toolkit_lib import dataset_params, inout
except ImportError as error:
print(
'ERROR: Please download the bop_toolkit package and add it to sys_paths in config!'
)
print('https://github.com/thodan/bop_toolkit')
raise error
model_p = dataset_params.get_model_params(
datasets_path,
dataset,
model_type=model_type if model_type else None)
cam_p = dataset_params.get_camera_params(
datasets_path, dataset, cam_type=cam_type if cam_type else None)
try:
split_p = dataset_params.get_split_params(datasets_path,
dataset,
split=split)
except ValueError:
raise Exception(
"Wrong path or {} split does not exist in {}.".format(
split, dataset))
bpy.context.scene.world["category_id"] = 0
bpy.context.scene.render.resolution_x = cam_p['im_size'][0]
bpy.context.scene.render.resolution_y = cam_p['im_size'][1]
loaded_objects = []
# only load all/selected objects here, use other modules for setting poses
# e.g. camera.CameraSampler / object.ObjectPoseSampler
if scene_id == -1:
# TLESS exception because images are cropped
if bop_dataset_name in ['tless']:
cam_p['K'][0, 2] = split_p['im_size'][0] / 2
cam_p['K'][1, 2] = split_p['im_size'][1] / 2
# set camera intrinsics
CameraUtility.set_intrinsics_from_K_matrix(cam_p['K'],
split_p['im_size'][0],
split_p['im_size'][1])
obj_ids = obj_ids if obj_ids else model_p['obj_ids']
# if sampling is enabled
if sample_objects:
loaded_ids = {}
loaded_amount = 0
if obj_instances_limit != -1 and len(
obj_ids) * obj_instances_limit < num_of_objs_to_sample:
raise RuntimeError(
"{}'s {} split contains {} objects, {} object where requested to sample with "
"an instances limit of {}. Raise the limit amount or decrease the requested "
"amount of objects.".format(bop_dataset_path, split,
len(obj_ids),
num_of_objs_to_sample,
obj_instances_limit))
while loaded_amount != num_of_objs_to_sample:
random_id = choice(obj_ids)
if random_id not in loaded_ids.keys():
loaded_ids.update({random_id: 0})
# if there is no limit or if there is one, but it is not reached for this particular object
if obj_instances_limit == -1 or loaded_ids[
random_id] < obj_instances_limit:
cur_obj = BopLoader._load_mesh(random_id, model_p,
bop_dataset_name,
has_external_texture,
temp_dir,
allow_duplication,
scale)
loaded_ids[random_id] += 1
loaded_amount += 1
loaded_objects.append(cur_obj)
else:
print(
"ID {} was loaded {} times with limit of {}. Total loaded amount {} while {} are "
"being requested".format(random_id,
loaded_ids[random_id],
obj_instances_limit,
loaded_amount,
num_of_objs_to_sample))
else:
for obj_id in obj_ids:
cur_obj = BopLoader._load_mesh(obj_id, model_p,
bop_dataset_name,
has_external_texture,
temp_dir, allow_duplication,
scale)
loaded_objects.append(cur_obj)
# replicate scene: load scene objects, object poses, camera intrinsics and camera poses
else:
sc_gt = inout.load_scene_gt(
split_p['scene_gt_tpath'].format(**{'scene_id': scene_id}))
sc_camera = inout.load_json(
split_p['scene_camera_tpath'].format(**{'scene_id': scene_id}))
for i, (cam_id, insts) in enumerate(sc_gt.items()):
cam_K, cam_H_m2c_ref = BopLoader._get_ref_cam_extrinsics_intrinsics(
sc_camera, cam_id, insts, scale)
if i == 0:
# define world = first camera
cam_H_m2w_ref = cam_H_m2c_ref.copy()
cur_objs = []
# load scene objects and set their poses
for inst in insts:
cur_objs.append(
BopLoader._load_mesh(inst['obj_id'], model_p,
bop_dataset_name,
has_external_texture,
temp_dir, allow_duplication,
scale))
BopLoader.set_object_pose(cur_objs[-1], inst, scale)
cam_H_c2w = BopLoader._compute_camera_to_world_trafo(
cam_H_m2w_ref, cam_H_m2c_ref, source_frame)
# set camera intrinsics
CameraUtility.set_intrinsics_from_K_matrix(
cam_K, split_p['im_size'][0], split_p['im_size'][1])
# set camera extrinsics as next frame
frame_id = CameraUtility.add_camera_pose(cam_H_c2w)
# Add key frame for camera shift, as it changes from frame to frame in the tless replication
cam = bpy.context.scene.camera.data
cam.keyframe_insert(data_path='shift_x', frame=frame_id)
cam.keyframe_insert(data_path='shift_y', frame=frame_id)
# Copy object poses to key frame (to be sure)
for cur_obj in cur_objs:
BopLoader._insert_key_frames(cur_obj, frame_id)
# move the origin of the object to the world origin and on top of the X-Y plane
# makes it easier to place them later on, this does not change the `.location`
# This is only useful if the BOP objects are not used in a pose estimation scenario.
if move_origin_to_x_y_plane:
for obj in loaded_objects:
obj.move_origin_to_bottom_mean_point()
return loaded_objects
def run(self):
""" Load BOP data """
bop_dataset_path = self.config.get_string("bop_dataset_path")
scene_id = self.config.get_int("scene_id", -1)
obj_ids = self.config.get_list("obj_ids", [])
split = self.config.get_string("split", "test")
model_type = self.config.get_string("model_type", "")
cam_type = self.config.get_string("cam_type", "")
scale = 0.001 if self.config.get_bool("mm2m", False) else 1
datasets_path = os.path.dirname(bop_dataset_path)
dataset = os.path.basename(bop_dataset_path)
print("bob: {}, dataset_path: {}".format(bop_dataset_path,
datasets_path))
print("dataset: {}".format(dataset))
try:
from bop_toolkit_lib import dataset_params, inout
except ImportError as error:
print(
'ERROR: Please download the bop_toolkit package and add it to sys_paths in config!'
)
print('https://github.com/thodan/bop_toolkit')
raise error
model_p = dataset_params.get_model_params(
datasets_path,
dataset,
model_type=model_type if model_type else None)
cam_p = dataset_params.get_camera_params(
datasets_path, dataset, cam_type=cam_type if cam_type else None)
bpy.data.scenes["Scene"]["num_labels"] = len(model_p['obj_ids'])
try:
split_p = dataset_params.get_split_params(datasets_path,
dataset,
split=split)
except ValueError:
raise Exception(
"Wrong path or {} split does not exist in {}.".format(
split, dataset))
bpy.context.scene.world["category_id"] = 0
bpy.context.scene.render.resolution_x = self.config.get_int(
"resolution_x", split_p['im_size'][0])
bpy.context.scene.render.resolution_y = self.config.get_int(
"resolution_y", split_p['im_size'][1])
# Collect camera and camera object
cam_ob = bpy.context.scene.camera
cam = cam_ob.data
cam['loaded_resolution'] = bpy.context.scene.render.resolution_x, bpy.context.scene.render.resolution_y
cam['loaded_intrinsics'] = cam_p[
'K'] # load default intrinsics from camera.json
config = Config({})
camera_module = CameraModule(config)
camera_module._set_cam_intrinsics(cam, config)
#only load all/selected objects here, use other modules for setting poses, e.g. camera.CameraSampler / object.ObjectPoseSampler
if scene_id == -1:
obj_ids = obj_ids if obj_ids else model_p['obj_ids']
for obj_id in obj_ids:
self._load_mesh(obj_id, model_p, scale=scale)
# replicate scene: load scene objects, object poses, camera intrinsics and camera poses
else:
sc_gt = inout.load_scene_gt(
split_p['scene_gt_tpath'].format(**{'scene_id': scene_id}))
sc_camera = inout.load_json(
split_p['scene_camera_tpath'].format(**{'scene_id': scene_id}))
for i, (cam_id, insts) in enumerate(sc_gt.items()):
cam_K, cam_H_m2c_ref = self._get_ref_cam_extrinsics_intrinsics(
sc_camera, cam_id, insts, scale)
if i == 0:
# define world = first camera
cam_H_m2w_ref = cam_H_m2c_ref.copy()
# load scene objects
for inst in insts:
cur_obj = self._load_mesh(inst['obj_id'], model_p)
self.set_object_pose(cur_obj, inst, scale)
cam_H_c2w = self._compute_camera_to_world_trafo(
cam_H_m2w_ref, cam_H_m2c_ref)
#set camera intrinsics and extrinsics
config = Config({
"cam2world_matrix": list(cam_H_c2w.flatten()),
"camK": list(cam_K.flatten())
})
camera_module._set_cam_intrinsics(cam, config)
camera_module._set_cam_extrinsics(cam_ob, config)
# Store new cam pose as next frame
frame_id = bpy.context.scene.frame_end
camera_module._insert_key_frames(cam, cam_ob, frame_id)
bpy.context.scene.frame_end = frame_id + 1
def preprocess_data():
# Create directory
Path(f'{SAVE_DIR}/image').mkdir(parents=True, exist_ok=True)
Path(f'{SAVE_DIR}/model').mkdir(parents=True, exist_ok=True)
# Parse metadata and load information of images with the target object
img_info = []
if not IS_TARGET:
for sc_id in range(*SCENE_ID_RANGE):
assert Path(f'{DATASET_DIR}/{sc_id:06d}/scene_gt.json').is_file()
scene_gt = io.load_scene_gt(
f'{DATASET_DIR}/{sc_id:06d}/scene_gt.json')
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/scene_camera.json').is_file()
scene_cam = io.load_scene_camera(
f'{DATASET_DIR}/{sc_id:06d}/scene_camera.json')
for (im_id, im_gt), im_cam in zip(scene_gt.items(),
scene_cam.values()):
for gt_id, gt in enumerate(im_gt):
if int(gt['obj_id']) == OBJ_ID:
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/depth/{im_id:06d}.png'
).is_file()
depth_path = f'{DATASET_DIR}/{sc_id:06d}/depth/{im_id:06d}.png'
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/mask_visib/{im_id:06d}_{gt_id:06d}.png'
).is_file()
mask_path = f'{DATASET_DIR}/{sc_id:06d}/mask_visib/{im_id:06d}_{gt_id:06d}.png'
save_name = f'{sc_id:06d}_{im_id:06d}_{gt_id:06d}'
img_info.append(
[depth_path, mask_path, im_cam, gt, save_name])
else:
targets = io.load_json(
f'{Path(DATASET_DIR).parent}/test_targets_bop19.json')
for target in targets:
if int(target['obj_id']) == OBJ_ID:
sc_id = target['scene_id']
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/scene_gt.json').is_file()
scene_gt = io.load_scene_gt(
f'{DATASET_DIR}/{sc_id:06d}/scene_gt.json')
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/scene_camera.json').is_file()
scene_cam = io.load_scene_camera(
f'{DATASET_DIR}/{sc_id:06d}/scene_camera.json')
im_id = int(target['im_id'])
im_gt, im_cam = scene_gt[im_id], scene_cam[im_id]
for gt_id, gt in enumerate(im_gt):
if int(gt['obj_id']) == OBJ_ID:
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/depth/{im_id:06d}.png'
).is_file()
depth_path = f'{DATASET_DIR}/{sc_id:06d}/depth/{im_id:06d}.png'
assert Path(
f'{DATASET_DIR}/{sc_id:06d}/mask_visib/{im_id:06d}_{gt_id:06d}.png'
).is_file()
mask_path = f'{DATASET_DIR}/{sc_id:06d}/mask_visib/{im_id:06d}_{gt_id:06d}.png'
save_name = f'{sc_id:06d}_{im_id:06d}_{gt_id:06d}'
img_info.append(
[depth_path, mask_path, im_cam, gt, save_name])
# Read model point cloud
model_file = f'{Path(DATASET_DIR).parent}/models/obj_{OBJ_ID:06d}.ply'
assert Path(model_file).is_file()
model_pcd = read_point_cloud(model_file).voxel_down_sample(VOXEL_SIZE)
md_pcd_pts = np.asarray(model_pcd.points)
# Create point cloud from image information
t1 = time()
with get_context('spawn').Pool(8) as pool:
jobs = [
pool.apply_async(create_patch_pair, (*info, md_pcd_pts))
for info in img_info
]
df = pd.DataFrame([j.get() for j in jobs]).dropna(axis=0)
if Path(f'{SAVE_DIR}/labels.csv').is_file():
old_df = pd.read_csv(f'{SAVE_DIR}/labels.csv', header=None)
df = pd.concat([old_df, df])
df.to_csv(f'{SAVE_DIR}/labels.csv',
header=['filename', 'num_patches'],
index=False)
t2 = time()
print(
f'Created patch_pairs from {len(df)} images. Time elapsed: {t2 - t1 :.3f}'
)
def run(self):
""" Load BOP data """
datasets_path = os.path.dirname(self.bop_dataset_path)
dataset = os.path.basename(self.bop_dataset_path)
print("bob: {}, dataset_path: {}".format(self.bop_dataset_path,
datasets_path))
print("dataset: {}".format(dataset))
try:
from bop_toolkit_lib import dataset_params, inout
except ImportError as error:
print(
'ERROR: Please download the bop_toolkit package and add it to sys_paths in config!'
)
print('https://github.com/thodan/bop_toolkit')
raise error
model_p = dataset_params.get_model_params(
datasets_path,
dataset,
model_type=self.model_type if self.model_type else None)
cam_p = dataset_params.get_camera_params(
datasets_path,
dataset,
cam_type=self.cam_type if self.cam_type else None)
bpy.data.scenes["Scene"]["num_labels"] = len(model_p['obj_ids'])
try:
split_p = dataset_params.get_split_params(datasets_path,
dataset,
split=self.split)
except ValueError:
raise Exception(
"Wrong path or {} split does not exist in {}.".format(
self.split, dataset))
bpy.context.scene.world["category_id"] = 0
bpy.context.scene.render.resolution_x = split_p['im_size'][0]
bpy.context.scene.render.resolution_y = split_p['im_size'][1]
# Collect camera and camera object
cam_ob = bpy.context.scene.camera
cam = cam_ob.data
cam['loaded_resolution'] = split_p['im_size'][0], split_p['im_size'][1]
# load default intrinsics from camera.json
cam['loaded_intrinsics'] = cam_p['K']
config = Config({})
camera_module = CameraModule(config)
camera_module._set_cam_intrinsics(cam, config)
loaded_objects = []
# only load all/selected objects here, use other modules for setting poses
# e.g. camera.CameraSampler / object.ObjectPoseSampler
if self.scene_id == -1:
obj_ids = self.obj_ids if self.obj_ids else model_p['obj_ids']
# if sampling is enabled
if self.sample_objects:
loaded_ids = {}
loaded_amount = 0
if self.obj_instances_limit != -1 and len(
obj_ids
) * self.obj_instances_limit < self.num_of_objs_to_sample:
raise RuntimeError(
"{}'s {} split contains {} objects, {} object where requested to sample with "
"an instances limit of {}. Raise the limit amount or decrease the requested "
"amount of objects.".format(self.bop_dataset_path,
self.split, len(obj_ids),
self.num_of_objs_to_sample,
self.obj_instances_limit))
while loaded_amount != self.num_of_objs_to_sample:
random_id = choice(obj_ids)
if random_id not in loaded_ids.keys():
loaded_ids.update({random_id: 0})
# if there is no limit or if there is one, but it is not reached for this particular object
if self.obj_instances_limit == -1 or loaded_ids[
random_id] < self.obj_instances_limit:
cur_obj = self._load_mesh(random_id,
model_p,
scale=self.scale)
loaded_ids[random_id] += 1
loaded_amount += 1
loaded_objects.append(cur_obj)
else:
print(
"ID {} was loaded {} times with limit of {}. Total loaded amount {} while {} are "
"being requested".format(
random_id, loaded_ids[random_id],
self.obj_instances_limit, loaded_amount,
self.num_of_objs_to_sample))
else:
for obj_id in obj_ids:
cur_obj = self._load_mesh(obj_id,
model_p,
scale=self.scale)
loaded_objects.append(cur_obj)
self._set_properties(loaded_objects)
# replicate scene: load scene objects, object poses, camera intrinsics and camera poses
else:
sc_gt = inout.load_scene_gt(split_p['scene_gt_tpath'].format(
**{'scene_id': self.scene_id}))
sc_camera = inout.load_json(split_p['scene_camera_tpath'].format(
**{'scene_id': self.scene_id}))
for i, (cam_id, insts) in enumerate(sc_gt.items()):
cam_K, cam_H_m2c_ref = self._get_ref_cam_extrinsics_intrinsics(
sc_camera, cam_id, insts, self.scale)
if i == 0:
# define world = first camera
cam_H_m2w_ref = cam_H_m2c_ref.copy()
cur_objs = []
# load scene objects and set their poses
for inst in insts:
cur_objs.append(
self._load_mesh(inst['obj_id'],
model_p,
scale=self.scale))
self.set_object_pose(cur_objs[-1], inst, self.scale)
cam_H_c2w = self._compute_camera_to_world_trafo(
cam_H_m2w_ref, cam_H_m2c_ref)
#set camera intrinsics and extrinsics
config = Config({
"cam2world_matrix": list(cam_H_c2w.flatten()),
"cam_K": list(cam_K.flatten())
})
camera_module._set_cam_intrinsics(cam, config)
camera_module._set_cam_extrinsics(cam_ob, config)
# Store new cam pose as next frame
frame_id = bpy.context.scene.frame_end
# Copy object poses to next key frame (to be sure)
for cur_obj in cur_objs:
self._insert_key_frames(cur_obj, frame_id)
camera_module._insert_key_frames(cam, cam_ob, frame_id)
bpy.context.scene.frame_end = frame_id + 1
def run(self):
bop_dataset_path = self.config.get_string("bop_dataset_path")
scene_id = self.config.get_int("scene_id")
split = self.config.get_string("split", "test")
model_type = self.config.get_string("model_type", "")
mm2m = 0.001 if self.config.get_bool("mm2m") else 1
datasets_path = os.path.dirname(bop_dataset_path)
dataset = os.path.basename(bop_dataset_path)
print("bob: {}, dataset_path: {}".format(bop_dataset_path,
datasets_path))
print("dataset: {}".format(dataset))
model_p = dataset_params.get_model_params(
datasets_path,
dataset,
model_type=model_type if model_type else None)
camera_p = dataset_params.get_camera_params(datasets_path, dataset)
try:
split_p = dataset_params.get_split_params(datasets_path,
dataset,
split=split)
except ValueError:
raise Exception(
"Wrong path or {} split does not exist in {}.".format(
split, dataset))
sc_gt = inout.load_scene_gt(
split_p['scene_gt_tpath'].format(**{'scene_id': scene_id}))
sc_camera = inout.load_json(
split_p['scene_camera_tpath'].format(**{'scene_id': scene_id}))
bpy.context.scene.render.resolution_x = self.config.get_int(
"resolution_x", split_p['im_size'][0])
bpy.context.scene.render.resolution_y = self.config.get_int(
"resolution_y", split_p['im_size'][1])
#bpy.context.scene.render.pixel_aspect_x = self.config.get_float("pixel_aspect_x", 1) #split_p['im_size'][0] / split_p['im_size'][1])
cm = CameraModule(self.config)
for i, (cam_id, insts) in enumerate(sc_gt.items()):
cam_K = np.array(sc_camera[str(cam_id)]['cam_K']).reshape(3, 3)
cam_H_m2c_ref = np.eye(4)
cam_H_m2c_ref[:3, :3] = np.array(insts[0]['cam_R_m2c']).reshape(
3, 3)
cam_H_m2c_ref[:3, 3] = np.array(
insts[0]['cam_t_m2c']).reshape(3) * mm2m
if i == 0:
# define world = first camera
cam_H_m2w_ref = cam_H_m2c_ref.copy()
for inst in insts:
bpy.ops.import_mesh.ply(
filepath=model_p['model_tpath'].format(
**{'obj_id': inst['obj_id']}))
cam_H_m2c = np.eye(4)
cam_H_m2c[:3, :3] = np.array(inst['cam_R_m2c']).reshape(
3, 3)
cam_H_m2c[:3, 3] = np.array(
inst['cam_t_m2c']).reshape(3) * mm2m
# world = camera @ i=0
cam_H_m2w = cam_H_m2c
print('-----------------------------')
print("Model: {}".format(cam_H_m2w))
print('-----------------------------')
cur_obj = bpy.context.selected_objects[-1]
cur_obj.matrix_world = Matrix(cam_H_m2w)
cur_obj.scale = Vector((mm2m, mm2m, mm2m))
mat = self._load_materials(cur_obj)
self._link_col_node(mat)
cam_H_c2w = np.dot(cam_H_m2w_ref, np.linalg.inv(cam_H_m2c_ref))
print('-----------------------------')
print("Cam: {}".format(cam_H_c2w))
print('-----------------------------')
config = {"location": [0, 0, 0], "rotation": list([0, 0, 0])}
cm._add_cam_pose(Config(config), Matrix(cam_H_c2w), cam_K)
def run(self):
""" Load BOP data """
datasets_path = os.path.dirname(self.bop_dataset_path)
dataset = os.path.basename(self.bop_dataset_path)
print("bob: {}, dataset_path: {}".format(self.bop_dataset_path,
datasets_path))
print("dataset: {}".format(dataset))
try:
from bop_toolkit_lib import dataset_params, inout
except ImportError as error:
print(
'ERROR: Please download the bop_toolkit package and add it to sys_paths in config!'
)
print('https://github.com/thodan/bop_toolkit')
raise error
model_p = dataset_params.get_model_params(
datasets_path,
dataset,
model_type=self.model_type if self.model_type else None)
cam_p = dataset_params.get_camera_params(
datasets_path,
dataset,
cam_type=self.cam_type if self.cam_type else None)
try:
split_p = dataset_params.get_split_params(datasets_path,
dataset,
split=self.split)
except ValueError:
raise Exception(
"Wrong path or {} split does not exist in {}.".format(
self.split, dataset))
bpy.context.scene.world["category_id"] = 0
bpy.context.scene.render.resolution_x = cam_p['im_size'][0]
bpy.context.scene.render.resolution_y = cam_p['im_size'][1]
loaded_objects = []
# only load all/selected objects here, use other modules for setting poses
# e.g. camera.CameraSampler / object.ObjectPoseSampler
if self.scene_id == -1:
# TLESS exception because images are cropped
if self.bop_dataset_name in ['tless']:
cam_p['K'][0, 2] = split_p['im_size'][0] / 2
cam_p['K'][1, 2] = split_p['im_size'][1] / 2
# set camera intrinsics
CameraUtility.set_intrinsics_from_K_matrix(cam_p['K'],
split_p['im_size'][0],
split_p['im_size'][1])
obj_ids = self.obj_ids if self.obj_ids else model_p['obj_ids']
# if sampling is enabled
if self.sample_objects:
loaded_ids = {}
loaded_amount = 0
if self.obj_instances_limit != -1 and len(
obj_ids
) * self.obj_instances_limit < self.num_of_objs_to_sample:
raise RuntimeError(
"{}'s {} split contains {} objects, {} object where requested to sample with "
"an instances limit of {}. Raise the limit amount or decrease the requested "
"amount of objects.".format(self.bop_dataset_path,
self.split, len(obj_ids),
self.num_of_objs_to_sample,
self.obj_instances_limit))
while loaded_amount != self.num_of_objs_to_sample:
random_id = choice(obj_ids)
if random_id not in loaded_ids.keys():
loaded_ids.update({random_id: 0})
# if there is no limit or if there is one, but it is not reached for this particular object
if self.obj_instances_limit == -1 or loaded_ids[
random_id] < self.obj_instances_limit:
cur_obj = self._load_mesh(random_id,
model_p,
scale=self.scale)
loaded_ids[random_id] += 1
loaded_amount += 1
loaded_objects.append(cur_obj)
else:
print(
"ID {} was loaded {} times with limit of {}. Total loaded amount {} while {} are "
"being requested".format(
random_id, loaded_ids[random_id],
self.obj_instances_limit, loaded_amount,
self.num_of_objs_to_sample))
else:
for obj_id in obj_ids:
cur_obj = self._load_mesh(obj_id,
model_p,
scale=self.scale)
loaded_objects.append(cur_obj)
self._set_properties(loaded_objects)
# replicate scene: load scene objects, object poses, camera intrinsics and camera poses
else:
sc_gt = inout.load_scene_gt(split_p['scene_gt_tpath'].format(
**{'scene_id': self.scene_id}))
sc_camera = inout.load_json(split_p['scene_camera_tpath'].format(
**{'scene_id': self.scene_id}))
for i, (cam_id, insts) in enumerate(sc_gt.items()):
cam_K, cam_H_m2c_ref = self._get_ref_cam_extrinsics_intrinsics(
sc_camera, cam_id, insts, self.scale)
if i == 0:
# define world = first camera
cam_H_m2w_ref = cam_H_m2c_ref.copy()
cur_objs = []
# load scene objects and set their poses
for inst in insts:
cur_objs.append(
self._load_mesh(inst['obj_id'],
model_p,
scale=self.scale))
self.set_object_pose(cur_objs[-1], inst, self.scale)
cam_H_c2w = self._compute_camera_to_world_trafo(
cam_H_m2w_ref, cam_H_m2c_ref)
# set camera intrinsics
CameraUtility.set_intrinsics_from_K_matrix(
cam_K, split_p['im_size'][0], split_p['im_size'][1])
# set camera extrinsics as next frame
frame_id = CameraUtility.add_camera_pose(cam_H_c2w)
# Add key frame for camera shift, as it changes from frame to frame in the tless replication
cam = bpy.context.scene.camera.data
cam.keyframe_insert(data_path='shift_x', frame=frame_id)
cam.keyframe_insert(data_path='shift_y', frame=frame_id)
# Copy object poses to key frame (to be sure)
for cur_obj in cur_objs:
self._insert_key_frames(cur_obj, frame_id)
# move the origin of the object to the world origin and on top of the X-Y plane
# makes it easier to place them later on, this does not change the `.location`
# This is only useful if the BOP objects are not used in a pose estimation scenario.
move_to_origin = self.config.get_bool("move_origin_to_x_y_plane",
False)
if move_to_origin:
LoaderInterface.move_obj_origin_to_bottom_mean_point(
loaded_objects)
im_height = 480
train_syn_dir = bop_dir + "/train_synt"
train_real_dir = bop_dir + "/train_real"
if not (os.path.exists(target_dir)): os.makedirs(target_dir)
if not (os.path.exists(target_dir + "/mask")):
os.makedirs(target_dir + "/mask")
s_id = 0
for train_type in ["syn", "real"]:
if (train_type == "syn"): train_dir = train_syn_dir
else: train_dir = train_real_dir
for folder in os.listdir(train_dir):
sub_dir = os.path.join(train_dir, folder)
print("Processing:", sub_dir)
scene_gt_fn = os.path.join(sub_dir, "scene_gt.json")
scene_gts = inout.load_scene_gt(scene_gt_fn)
for img_id in sorted(scene_gts.keys()):
im_id = int(img_id)
rgb_fn = os.path.join(sub_dir + "/rgb",
"{:06d}.png".format(im_id))
gts = scene_gts[im_id]
mask_gt = np.zeros((im_height, im_width), np.int8)
mask_gt[:, :] = 0
for gt_id, gt in enumerate(gts):
mask_img_fn = os.path.join(
os.path.join(sub_dir, "mask_visib"),
"{:06d}_{:06d}.png".format(im_id, gt_id))
mask_img = io.imread(mask_img_fn)
obj_id = gt['obj_id'] #real 1~
mask_obj = mask_img > 0
mask_gt[mask_obj] = obj_id
def run(self):
bop_dataset_path = self.config.get_string("bop_dataset_path")
scene_id = self.config.get_int("scene_id", -1)
obj_ids = self.config.get_list("obj_ids", [])
split = self.config.get_string("split", "test")
model_type = self.config.get_string("model_type", "")
cam_type = self.config.get_string("cam_type", "")
mm2m = 0.001 if self.config.get_bool("mm2m", False) else 1
datasets_path = os.path.dirname(bop_dataset_path)
dataset = os.path.basename(bop_dataset_path)
print("bob: {}, dataset_path: {}".format(bop_dataset_path,
datasets_path))
print("dataset: {}".format(dataset))
try:
from bop_toolkit_lib import dataset_params, inout
except ImportError as error:
print(
'ERROR: Please download the bop_toolkit package and add it to sys_paths in config!'
)
print('https://github.com/thodan/bop_toolkit')
raise error
model_p = dataset_params.get_model_params(
datasets_path,
dataset,
model_type=model_type if model_type else None)
cam_p = dataset_params.get_camera_params(
datasets_path, dataset, cam_type=cam_type if cam_type else None)
bpy.data.scenes["Scene"]["num_labels"] = len(model_p['obj_ids'])
try:
split_p = dataset_params.get_split_params(datasets_path,
dataset,
split=split)
except ValueError:
raise Exception(
"Wrong path or {} split does not exist in {}.".format(
split, dataset))
bpy.context.scene.world["category_id"] = 0
bpy.context.scene.render.resolution_x = self.config.get_int(
"resolution_x", split_p['im_size'][0])
bpy.context.scene.render.resolution_y = self.config.get_int(
"resolution_y", split_p['im_size'][1])
# Collect camera and camera object
cam_ob = bpy.context.scene.camera
cam = cam_ob.data
cam['loaded_resolution'] = bpy.context.scene.render.resolution_x, bpy.context.scene.render.resolution_y
cam['loaded_intrinsics'] = cam_p[
'K'] # load default intrinsics from camera.json
#only load all/selected objects here, use other modules for setting poses, e.g. camera.CameraSampler / object.ObjectPoseSampler
if scene_id == -1:
obj_ids = obj_ids if obj_ids else model_p['obj_ids']
for obj_id in obj_ids:
self._load_mesh(obj_id, model_p, mm2m=mm2m)
# replicate scene: load scene objects, object poses, camera intrinsics and camera poses
else:
sc_gt = inout.load_scene_gt(
split_p['scene_gt_tpath'].format(**{'scene_id': scene_id}))
sc_camera = inout.load_json(
split_p['scene_camera_tpath'].format(**{'scene_id': scene_id}))
cm = CameraModule(self.config)
for i, (cam_id, insts) in enumerate(sc_gt.items()):
cam_K = np.array(sc_camera[str(cam_id)]['cam_K']).reshape(3, 3)
cam_H_m2c_ref = np.eye(4)
cam_H_m2c_ref[:3, :3] = np.array(
insts[0]['cam_R_m2c']).reshape(3, 3)
cam_H_m2c_ref[:3, 3] = np.array(
insts[0]['cam_t_m2c']).reshape(3) * mm2m
if i == 0:
# define world = first camera
cam_H_m2w_ref = cam_H_m2c_ref.copy()
for inst in insts:
cur_obj = self._load_mesh(inst['obj_id'],
model_p,
mm2m=mm2m)
cam_H_m2c = np.eye(4)
cam_H_m2c[:3, :3] = np.array(
inst['cam_R_m2c']).reshape(3, 3)
cam_H_m2c[:3, 3] = np.array(
inst['cam_t_m2c']).reshape(3) * mm2m
# world = camera @ i=0
cam_H_m2w = cam_H_m2c
print('-----------------------------')
print("Model: {}".format(cam_H_m2w))
print('-----------------------------')
cur_obj.matrix_world = Matrix(cam_H_m2w)
cam_H_c2w = np.dot(cam_H_m2w_ref, np.linalg.inv(cam_H_m2c_ref))
print('-----------------------------')
print("Cam: {}".format(cam_H_c2w))
print('-----------------------------')
config = {"location": [0, 0, 0], "rotation": list([0, 0, 0])}
cm._add_cam_pose(Config(config), Matrix(cam_H_c2w), cam_K)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
# Load the list examples.
examples_path = os.path.join(config.TF_DATA_PATH, 'example_lists',
FLAGS.examples_filename)
tf.logging.info(
'Loading a list of examples from: {}'.format(examples_path))
examples_list = tfrecord.load_example_list(examples_path)
# Load dataset parameters.
dp_split = dataset_params.get_split_params(config.BOP_PATH, FLAGS.dataset,
FLAGS.split, FLAGS.split_type)
# Pre-load camera parameters and ground-truth annotations.
scene_gt = {}
scene_gt_info = {}
scene_camera = {}
scene_ids = set([e['scene_id'] for e in examples_list])
for scene_id in scene_ids:
scene_camera[scene_id] = inout.load_scene_camera(
dp_split['scene_camera_tpath'].format(scene_id=scene_id))
if FLAGS.add_gt:
scene_gt[scene_id] = inout.load_scene_gt(
dp_split['scene_gt_tpath'].format(scene_id=scene_id))
scene_gt_info[scene_id] = inout.load_json(
dp_split['scene_gt_info_tpath'].format(scene_id=scene_id),
keys_to_int=True)
# Check the name of the file with examples.
examples_end = '_examples.txt'
if not FLAGS.examples_filename.endswith(examples_end):
raise ValueError(
'Name of the file with examples must end with {}.'.format(
examples_end))
# Prepare writer of the TFRecord file.
output_name = FLAGS.examples_filename.split(examples_end)[0]
output_path = os.path.join(FLAGS.output_dir, output_name + '.tfrecord')
writer = tf.python_io.TFRecordWriter(output_path)
tf.logging.info('File to be created: {}'.format(output_path))
# Optionally shuffle the examples.
if FLAGS.shuffle:
random.shuffle(examples_list)
# Write the examples to the TFRecord file.
w_start_t = time.time()
create_tf_example_partial = partial(create_tf_example,
dp_split=dp_split,
scene_camera=scene_camera,
scene_gt=scene_gt,
scene_gt_info=scene_gt_info)
for example_id, example in enumerate(examples_list):
if example_id % 50 == 0:
tf.logging.info('Processing example {}/{}'.format(
example_id + 1, len(examples_list)))
tf_example, _ = create_tf_example_partial(example)
writer.write(tf_example)
# Close the writer.
writer.close()
w_total_t = time.time() - w_start_t
tf.logging.info('Writing took {} s.'.format(w_total_t))
from rendering.renderer import Renderer
from rendering.model import Model3D
import matplotlib.pyplot as plt
import transforms3d as tf3d
import numpy as np
import time
from bop_toolkit_lib import inout
from tools import bop_io
import copy
#YCB(have to check) - > LMO
#HB, ITODD -> T-LESS
ref_gt = inout.load_scene_gt(
os.path.join("/home/kiru/media/hdd/bop/tless/train_render_reconst/000001",
"scene_gt.json"))
ref_camera = inout.load_scene_camera(
os.path.join("/home/kiru/media/hdd/bop/tless/train_render_reconst/000001",
"scene_camera.json"))
#bop_dir,source_dir,model_plys,model_info,model_ids,rgb_files,depth_files,mask_files,gts,cam_param_global = bop_io.get_dataset('hb',train=True)
#bop_dir,source_dir,model_plys,model_info,model_ids,rgb_files,depth_files,mask_files,gts,cam_param_global = bop_io.get_dataset('itodd',train=True)
bop_dir, source_dir, model_plys, model_info, model_ids, rgb_files, depth_files, mask_files, gts, cam_param_global = bop_io.get_dataset(
'ycbv', train=True)
im_width, im_height = cam_param_global['im_size']
camK = cam_param_global['K']
cam_K_list = np.array(camK).reshape(-1)
ren = Renderer((im_width, im_height), camK)
source_dir = bop_dir + "/train"