def test_vis():
dset_name = sys.argv[1]
assert dset_name in DatasetCatalog.list()
meta = MetadataCatalog.get(dset_name)
dprint("MetadataCatalog: ", meta)
objs = meta.objs
t_start = time.perf_counter()
dicts = DatasetCatalog.get(dset_name)
logger.info("Done loading {} samples with {:.3f}s.".format(len(dicts), time.perf_counter() - t_start))
dirname = "output/{}-data-vis".format(dset_name)
os.makedirs(dirname, exist_ok=True)
for d in dicts:
img = read_image_cv2(d["file_name"], format="BGR")
depth = mmcv.imread(d["depth_file"], "unchanged") / 1000.0
anno = d["annotations"][0] # only one instance per image
imH, imW = img.shape[:2]
mask = cocosegm2mask(anno["segmentation"], imH, imW)
bbox = anno["bbox"]
bbox_mode = anno["bbox_mode"]
bbox_xyxy = np.array(BoxMode.convert(bbox, bbox_mode, BoxMode.XYXY_ABS))
kpt3d = anno["bbox3d_and_center"]
quat = anno["quat"]
trans = anno["trans"]
R = quat2mat(quat)
# 0-based label
cat_id = anno["category_id"]
K = d["cam"]
kpt_2d = misc.project_pts(kpt3d, K, R, trans)
# # TODO: visualize pose and keypoints
label = objs[cat_id]
# img_vis = vis_image_bboxes_cv2(img, bboxes=bboxes_xyxy, labels=labels)
img_vis = vis_image_mask_bbox_cv2(img, [mask], bboxes=[bbox_xyxy], labels=[label])
img_vis_kpt2d = img.copy()
img_vis_kpt2d = misc.draw_projected_box3d(
img_vis_kpt2d, kpt_2d, middle_color=None, bottom_color=(128, 128, 128)
)
xyz_info = mmcv.load(anno["xyz_path"])
xyz = np.zeros((imH, imW, 3), dtype=np.float32)
xyz_crop = xyz_info["xyz_crop"].astype(np.float32)
x1, y1, x2, y2 = xyz_info["xyxy"]
xyz[y1 : y2 + 1, x1 : x2 + 1, :] = xyz_crop
xyz_show = get_emb_show(xyz)
grid_show(
[img[:, :, [2, 1, 0]], img_vis[:, :, [2, 1, 0]], img_vis_kpt2d[:, :, [2, 1, 0]], depth, xyz_show],
["img", "vis_img", "img_vis_kpts2d", "depth", "emb_show"],
row=2,
col=3,
)
def register_dataset(config , annot_df , images_df , annot):
"""
Register dataset (detectron2 register coco instance)
folds included
"""
fold = config.general["fold"]
train_dataset_name=f"my_dataset_train_{fold}"
test_dataset_name=f"my_dataset_test_{fold}"
train_dataset_file=os.path.join(DATASET_PATH,f"my_dataset_train_{fold}.json")
test_dataset_file=os.path.join(DATASET_PATH,f"my_dataset_test_{fold}.json")
train_annot_df=annot_df[annot_df["folds"]!=fold]
test_annot_df=annot_df[annot_df["folds"]==fold]
train_annot_df=train_annot_df.drop(["normal_category","normal_category_id"],axis=1)
test_annot_df=test_annot_df.drop(["normal_category","normal_category_id"],axis=1)
train_images_df=images_df[images_df["id"].apply(lambda i:True if i in list(train_annot_df["image_id"].unique()) else False)]
test_images_df=images_df[images_df["id"].apply(lambda i:True if i in list(test_annot_df["image_id"].unique()) else False)]
train_annot=annot.copy()
test_annot=annot.copy()
train_annot["annotations"]=train_annot_df.reset_index(drop=True).to_dict("records")
train_annot["images"]=train_images_df.reset_index(drop=True).to_dict("records")
test_annot["annotations"]=test_annot_df.reset_index(drop=True).to_dict("records")
test_annot["images"]=test_images_df.reset_index(drop=True).to_dict("records")
json.dump(train_annot,open(train_dataset_file,"w"))
json.dump(test_annot,open(test_dataset_file,"w"))
if train_dataset_name in DatasetCatalog.list():
DatasetCatalog.remove(train_dataset_name)
MetadataCatalog.remove(train_dataset_name)
if test_dataset_name in DatasetCatalog.list():
DatasetCatalog.remove(test_dataset_name)
MetadataCatalog.remove(test_dataset_name)
register_coco_instances(train_dataset_name, {}, train_dataset_file, os.path.join(DATASET_PATH,"data"))
register_coco_instances(test_dataset_name, {}, test_dataset_file, os.path.join(DATASET_PATH,"data"))
def register_datasets(dataset_names):
for name in dataset_names:
if name in DatasetCatalog.list():
continue
registered = False
# try to find in pre-defined datasets
# NOTE: it is better to let all datasets pre-refined
for _mod_name in _DSET_MOD_NAMES:
if name in get_available_datasets(_mod_name):
register_dataset(_mod_name, name, data_cfg=None)
registered = True
break
# not in pre-defined; not recommend
if not registered:
raise ValueError(f"dataset {name} is not defined")
def __init__(self, params):
self.base_path = params['base_path']
self.min_points_threshold = params['min_points_threshold']
self.n_jobs = params['n_jobs']
self.overwrite = params['overwrite']
self.n_sample_hard = params['n_sample_hard']
self.n_sample_per_label = params['n_sample_per_label']
self.input_format = params['input_format']
assert os.path.exists(
self.base_path +
'/data/train.csv'), f"No train CSV file in data folder."
self.data = pd.read_csv('./data/train.csv')
self.preprocess()
# load label and assigned idx
self.unique_labels = json.load(
open(self.base_path + f'/data/labels.json', 'r'))
self.unique_labels['other'] = 0
self.labels = list(self.unique_labels.keys())
self.labels.insert(0,
self.labels.pop()) # need "other" first in the list
# idx to labels for inference
self.bond_labels = [self.unique_labels[b] for b in ['-', '=', '#']]
self.idx_to_labels = {v: k for k, v in self.unique_labels.items()}
for l, b in zip(self.bond_labels, ['SINGLE', 'DOUBLE', 'TRIPLE']):
self.idx_to_labels[l] = b
# preparing datasets for training
for mode in ["train", "val"]:
dataset_name = f"smilesdetect_{mode}"
if dataset_name in DatasetCatalog.list():
DatasetCatalog.remove(dataset_name)
DatasetCatalog.register(dataset_name,
lambda mode=mode: self.get_metadata(mode))
MetadataCatalog.get(dataset_name).set(thing_classes=self.labels)
self.smiles_metadata = MetadataCatalog.get("smilesdetect_val")
self.cfg = self.create_cfg()
self.predictor = None
self.inference_metadata = MetadataCatalog.get("smilesdetect_val")
def _register_if_not(dataset_name: str, data_func: callable):
"""
Register data if it is not already registered
Args:
dataset_name: data-set name
data_func: data initialization function
Returns:
data catalog with registered dataset
"""
if dataset_name in DatasetCatalog.list():
print(f'Data-set {dataset_name} is already registered')
else:
DatasetCatalog.register(dataset_name, data_func)
MetadataCatalog.get(dataset_name).set(thing_classes=class_names)
print(
f'Registration of the data-set {dataset_name} is done with labels {LABEL_NAMES}'
)
return DatasetCatalog
def pseudo_label_train(cfg, out_folder):
# data prepare
all_datasets = DatasetCatalog.list()
if "wheat_coco_pseudo" not in all_datasets:
register_pseudo_datasets("wheat_coco_pseudo",
image_root=cfg.INPUT_DIR,
json_file=cfg.OUTPUT_DIR +
'/pseudo_label.json')
# set training cfg
if cfg.INPUT_DIR:
img_list = glob.glob(cfg.INPUT_DIR + "/*.jpg")
out_dir = os.path.join(cfg.OUTPUT_DIR, out_folder)
if not os.path.exists(out_dir):
os.mkdirs(out_dir)
set_pseudo_cfg(cfg, len(img_list), os.path.join(cfg.OUTPUT_DIR,
out_folder))
# trainer
trainer = PseudoTrainer(cfg)
trainer.resume_or_load(resume=False)
trainer.train()
def register_imagenet_instances(name, metadata, json_file):
assert isinstance(name, str), name
assert isinstance(json_file, (str, os.PathLike)), json_file
logger = logging.getLogger("detectron2.data.imagenet")
# check if dataset already exists
if name not in DatasetCatalog.list():
logger.info("Adding dataset '%s' to dataset catalog" % name)
# 1. register a function which returns dicts
DatasetCatalog.register(name, lambda: json.load(open(json_file)))
# 2. Optionally, add metadata about this dataset,
# since they might be useful in evaluation, visualization or logging
MetadataCatalog.get(name).set(json_file=json_file,
evaluator_type="imagenet",
**metadata)
else:
logger.warning(
"Dataset with name '%s' is already registered, ignoring...")
def register_all_oid():
# register dataset
for tv in ["train", "validation"]:
is_train = tv == 'train'
paths = get_paths(OID_DIR, tv)
ds_name = "oid_" + tv
if ds_name in DatasetCatalog.list():
DatasetCatalog.remove(ds_name)
# make sure validation dataset compliant to Detectron2 Format.
if is_train:
pipeline = DescPipeline([
partial(get_oid_descs, is_train=is_train),
],
cache_paths=[
paths.descs_pkl,
])
else:
pipeline = DescPipeline(
[
partial(get_oid_descs, is_train=is_train),
partial(oid_descs_to_detectron2_dicts,
masks_dir=paths.masks_dir,
cache_path=paths.dicts_pkl)
],
cache_paths=[paths.descs_pkl, paths.dicts_pkl])
# register oid dataset dicts.
DatasetCatalog.register("oid_" + tv, pipeline)
# set oid metadata.
MetadataCatalog.get(ds_name).set(
images_dir=paths.images_dir,
masks_dir=paths.masks_dir,
# json_file=json_file,
image_root=paths.images_dir,
evaluator_type="tfod",
thing_classes=KLASS_NAMES,
no_to_mid=NO_TO_MID)
def test_vis():
dset_name = sys.argv[1]
assert dset_name in DatasetCatalog.list()
meta = MetadataCatalog.get(dset_name)
dprint("MetadataCatalog: ", meta)
objs = meta.objs
t_start = time.perf_counter()
dicts = DatasetCatalog.get(dset_name)
logger.info("Done loading {} samples with {:.3f}s.".format(
len(dicts),
time.perf_counter() - t_start))
dirname = "output/{}-data-vis".format(dset_name)
os.makedirs(dirname, exist_ok=True)
for d in dicts:
img = read_image_cv2(d["file_name"], format="BGR")
depth = mmcv.imread(d["depth_file"], "unchanged") / 1000.0
imH, imW = img.shape[:2]
annos = d["annotations"]
masks = [
cocosegm2mask(anno["segmentation"], imH, imW) for anno in annos
]
bboxes = [anno["bbox"] for anno in annos]
bbox_modes = [anno["bbox_mode"] for anno in annos]
bboxes_xyxy = np.array([
BoxMode.convert(box, box_mode, BoxMode.XYXY_ABS)
for box, box_mode in zip(bboxes, bbox_modes)
])
kpts_3d_list = [anno["bbox3d_and_center"] for anno in annos]
quats = [anno["quat"] for anno in annos]
transes = [anno["trans"] for anno in annos]
Rs = [quat2mat(quat) for quat in quats]
# 0-based label
cat_ids = [anno["category_id"] for anno in annos]
K = d["cam"]
kpts_2d = [
misc.project_pts(kpt3d, K, R, t)
for kpt3d, R, t in zip(kpts_3d_list, Rs, transes)
]
# # TODO: visualize pose and keypoints
labels = [objs[cat_id] for cat_id in cat_ids]
for _i in range(len(annos)):
img_vis = vis_image_mask_bbox_cv2(img,
masks[_i:_i + 1],
bboxes=bboxes_xyxy[_i:_i + 1],
labels=labels[_i:_i + 1])
img_vis_kpts2d = misc.draw_projected_box3d(img_vis.copy(),
kpts_2d[_i])
if "test" not in dset_name:
xyz_path = annos[_i]["xyz_path"]
xyz_info = mmcv.load(xyz_path)
x1, y1, x2, y2 = xyz_info["xyxy"]
xyz_crop = xyz_info["xyz_crop"].astype(np.float32)
xyz = np.zeros((imH, imW, 3), dtype=np.float32)
xyz[y1:y2 + 1, x1:x2 + 1, :] = xyz_crop
xyz_show = get_emb_show(xyz)
xyz_crop_show = get_emb_show(xyz_crop)
img_xyz = img.copy() / 255.0
mask_xyz = ((xyz[:, :, 0] != 0) | (xyz[:, :, 1] != 0) |
(xyz[:, :, 2] != 0)).astype("uint8")
fg_idx = np.where(mask_xyz != 0)
img_xyz[fg_idx[0], fg_idx[1], :] = xyz_show[fg_idx[0],
fg_idx[1], :3]
img_xyz_crop = img_xyz[y1:y2 + 1, x1:x2 + 1, :]
img_vis_crop = img_vis[y1:y2 + 1, x1:x2 + 1, :]
# diff mask
diff_mask_xyz = np.abs(masks[_i] - mask_xyz)[y1:y2 + 1,
x1:x2 + 1]
grid_show(
[
img[:, :, [2, 1, 0]],
img_vis[:, :, [2, 1, 0]],
img_vis_kpts2d[:, :, [2, 1, 0]],
depth,
# xyz_show,
diff_mask_xyz,
xyz_crop_show,
img_xyz[:, :, [2, 1, 0]],
img_xyz_crop[:, :, [2, 1, 0]],
img_vis_crop,
],
[
"img",
"vis_img",
"img_vis_kpts2d",
"depth",
"diff_mask_xyz",
"xyz_crop_show",
"img_xyz",
"img_xyz_crop",
"img_vis_crop",
],
row=3,
col=3,
)
else:
grid_show(
[
img[:, :, [2, 1, 0]], img_vis[:, :, [2, 1, 0]],
img_vis_kpts2d[:, :, [2, 1, 0]], depth
],
["img", "vis_img", "img_vis_kpts2d", "depth"],
row=2,
col=2,
)
grid_show(
[
img[:, :, [2, 1, 0]], img_vis[:, :, [2, 1, 0]],
img_vis_kpts2d[:, :, [2, 1, 0]], depth
],
["img", "vis_img", "img_vis_kpts2d", "depth"],
row=2,
col=2,
)
if __name__ == "__main__":
"""Test the dataset loader.
python this_file.py dataset_name
"""
from lib.vis_utils.image import grid_show
from lib.utils.setup_logger import setup_logger
import detectron2.data.datasets # noqa # add pre-defined metadata
from lib.vis_utils.image import vis_image_mask_bbox_cv2
from core.utils.utils import get_emb_show
from core.utils.data_utils import read_image_cv2
print("sys.argv:", sys.argv)
setup_logger()
register_with_name_cfg(sys.argv[1])
print("dataset catalog: ", DatasetCatalog.list())
test_vis()
if objs:
record["annotations"] = objs
if objs or box_cnt == 0:
dataset_dicts.append(record)
print(
f'dataset_dicts={len(dataset_dicts)} with fold:{fold}, box_cnt:{box_cnt}'
)
return dataset_dicts
from detectron2.data import DatasetCatalog, MetadataCatalog
DatasetCatalog.clear()
for d in ["train", "val_0", "val_1", "val_2", "val_3", "val_4"]:
name = "lung_" + d
if name not in DatasetCatalog.list():
print(d)
DatasetCatalog.register(name, lambda d=d: get_anotation_dicts(d))
MetadataCatalog.get(name).set(thing_classes=["bz"])
metadata = MetadataCatalog.get("lung_train")
def save_stack_feature(train: pd.DataFrame, test: pd.DataFrame, file_path):
train.to_hdf(file_path, 'train', mode='a')
test.to_hdf(file_path, 'test', mode='a')
logger.info(f'OOF file save to :{file_path}')
return train, test
class _Trainer(DefaultTrainer):
def __init__(self, cfg):
def train(model_name: str,
results_path: str,
train_idx: List[int],
test_idx: List[int],
train_annotations: OrderedDict,
test_annotations: OrderedDict,
lr: float = 0.0025,
max_it: int = 500,
img_per_batch: int = 16,
batch_size: int = 512,
num_freeze: int = 1) -> NoReturn:
if Path(results_path).exists():
shutil.rmtree(results_path)
os.makedirs(results_path, exist_ok=True)
for catalog_type in ['train', 'test']:
catalog = f'aic19_{catalog_type}'
if catalog in DatasetCatalog.list():
DatasetCatalog.remove(catalog)
if catalog_type == 'train':
DatasetCatalog.register(
catalog,
lambda d=catalog_type: get_dicts(train_idx, train_annotations))
else:
DatasetCatalog.register(
catalog,
lambda d=catalog_type: get_dicts(test_idx, test_annotations))
MetadataCatalog.get(catalog).set(thing_classes=['Car'])
cfg = get_cfg()
cfg.OUTPUT_DIR = results_path
cfg.merge_from_file(model_zoo.get_config_file(model_name))
cfg.DATASETS.TRAIN = (f'aic19_train', )
cfg.DATASETS.TEST = ()
cfg.DATALOADER.NUM_WORKERS = 16
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(model_name)
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5
#cfg.MODEL.BACKBONE.FREEZE_AT = 1
cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = batch_size
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1
cfg.INPUT.MAX_SIZE_TEST = 1200
cfg.INPUT.MAX_SIZE_TRAIN = 1200
cfg.SOLVER.IMS_PER_BATCH = img_per_batch
cfg.SOLVER.BASE_LR = lr
cfg.SOLVER.MAX_ITER = max_it
cfg.SOLVER.STEPS = []
trainer = DefaultTrainer(cfg)
trainer.resume_or_load(resume=False)
trainer.train()
cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, 'model_final.pth')
evaluator = COCOEvaluator('aic19_test',
cfg,
False,
output_dir=results_path)
val_loader = build_detection_test_loader(cfg, "aic19_test")
print(inference_on_dataset(trainer.model, val_loader, evaluator))
Usage:
python -m detectron2.data.datasets.coco_panoptic \
path/to/image_root path/to/panoptic_root path/to/panoptic_json dataset_name 10
"dataset_name" can be "coco_2017_train_panoptic", or other
pre-registered ones
"""
from detectron2.utils.logger import setup_logger
from detectron2.utils.visualizer import Visualizer
import detectron2.data.datasets # noqa # add pre-defined metadata
import sys
from PIL import Image
import numpy as np
logger = setup_logger(name=__name__)
assert sys.argv[4] in DatasetCatalog.list()
meta = MetadataCatalog.get(sys.argv[4])
dicts = load_coco_panoptic_json(sys.argv[3], sys.argv[1], sys.argv[2],
meta.as_dict())
logger.info("Done loading {} samples.".format(len(dicts)))
dirname = "coco-data-vis"
os.makedirs(dirname, exist_ok=True)
num_imgs_to_vis = int(sys.argv[5])
for i, d in enumerate(dicts):
img = np.array(Image.open(d["file_name"]))
visualizer = Visualizer(img, metadata=meta)
vis = visualizer.draw_dataset_dict(d)
fpath = os.path.join(dirname, os.path.basename(d["file_name"]))
vis.save(fpath)
def test_vis():
dset_name = sys.argv[1]
assert dset_name in DatasetCatalog.list()
meta = MetadataCatalog.get(dset_name)
dprint("MetadataCatalog: ", meta)
objs = meta.objs
t_start = time.perf_counter()
dicts = DatasetCatalog.get(dset_name)
logger.info("Done loading {} samples with {:.3f}s.".format(
len(dicts),
time.perf_counter() - t_start))
dirname = "output/{}-data-vis".format(dset_name)
os.makedirs(dirname, exist_ok=True)
for d in dicts:
img = read_image_cv2(d["file_name"], format="BGR")
depth = mmcv.imread(d["depth_file"], "unchanged") / 1000.0
imH, imW = img.shape[:2]
annos = d["annotations"]
masks = [
cocosegm2mask(anno["segmentation"], imH, imW) for anno in annos
]
bboxes = [anno["bbox"] for anno in annos]
bbox_modes = [anno["bbox_mode"] for anno in annos]
bboxes_xyxy = np.array([
BoxMode.convert(box, box_mode, BoxMode.XYXY_ABS)
for box, box_mode in zip(bboxes, bbox_modes)
])
kpts_3d_list = [anno["bbox3d_and_center"] for anno in annos]
quats = [anno["quat"] for anno in annos]
transes = [anno["trans"] for anno in annos]
Rs = [quat2mat(quat) for quat in quats]
# 0-based label
cat_ids = [anno["category_id"] for anno in annos]
K = d["cam"]
kpts_2d = [
misc.project_pts(kpt3d, K, R, t)
for kpt3d, R, t in zip(kpts_3d_list, Rs, transes)
]
# # TODO: visualize pose and keypoints
labels = [objs[cat_id] for cat_id in cat_ids]
# img_vis = vis_image_bboxes_cv2(img, bboxes=bboxes_xyxy, labels=labels)
img_vis = vis_image_mask_bbox_cv2(img,
masks,
bboxes=bboxes_xyxy,
labels=labels)
img_vis_kpts2d = img.copy()
for anno_i in range(len(annos)):
img_vis_kpts2d = misc.draw_projected_box3d(img_vis_kpts2d,
kpts_2d[anno_i])
grid_show(
[
img[:, :, [2, 1, 0]], img_vis[:, :, [2, 1, 0]],
img_vis_kpts2d[:, :, [2, 1, 0]], depth
],
[f"img:{d['file_name']}", "vis_img", "img_vis_kpts2d", "depth"],
row=2,
col=2,
)
