def read(self, config_path: str) -> None:
"""read an oid config file"""
try:
with open(config_path) as f:
config_obj = json.load(f)
self._check(config_obj)
self.class_desc_path = config_obj['class_descriptions']
self.oid_class = OidClass(self.class_desc_path)
self.oid_class.read()
for ds_type in DATASET_TYPE_ALL:
ds_str = dataset_type2str(ds_type)
self.objs[ds_type].set(ds_type, config_obj['images'][ds_str],
config_obj['bounding_boxes'][ds_str],
self.oid_class,
self._class_filter(config_obj, ds_type),
self._required_class(config_obj))
self.output['images'][ds_type] = config_obj['output'][
'images'][ds_str]
self.output['bounding_boxes'][ds_type] = config_obj['output'][
'bounding_boxes'][ds_str]
if 'mixed_dataset' in config_obj:
self.mixed_dataset = config_obj['mixed_dataset']
except Exception as e:
logger.error(f'action=read error={e}')
raise
def test_oid_class_conv2label_single(self, global_variables):
oid_class = OidClass()
oid_class.read(global_variables.class_desc_path)
class_names_01 = ["Vehicle registration plate"]
label_names_01 = oid_class.conv2label(class_names_01)
assert len(label_names_01) == 1
assert type(label_names_01) == list
assert label_names_01[0] == "/m/01jfm_"
def test_oid_class_conv2label_multiple(self, global_variables):
oid_class = OidClass()
oid_class.read(global_variables.class_desc_path)
class_names_02 = ["Human face", "Person", "Car"]
label_names_02 = oid_class.conv2label(class_names_02)
assert len(label_names_02) == 3
assert type(label_names_02) == list
assert label_names_02[0] == "/m/01g317"
assert label_names_02[1] == "/m/0dzct"
assert label_names_02[2] == "/m/0k4j"
def test_coco_conv_bbox(self, global_variables):
coco = Coco()
assert coco
oid = Oid()
assert oid
oid.build_config(global_variables.test_config_path)
conv = OidConv()
assert conv
ds_type = DATASET_TYPE_VAL
oid.build_bbox(ds_type)
assert oid.bbox[ds_type]
oid.build_image(ds_type)
assert oid.images[ds_type]
assert type(oid.images[ds_type]) == OidImages
coco_images = conv.conv_images(oid_images=oid.images[ds_type])
assert coco_images
assert conv.oid_images
assert conv.coco_images
oid_class = OidClass()
assert oid_class
oid_class.read(global_variables.class_desc_path)
lable_name = "/m/01jfm_"
class_name = "Vehicle registration plate"
assert oid_class.conv2label([class_name]) == [lable_name]
assert oid_class.label2class(lable_name) == class_name
coco_category = CocoCategories()
coco_category.build(global_variables.required_class)
conv.set(oid_class, coco_category)
assert conv.oid_class
assert conv.coco_categories
print(f"oid.bbox[ds_type].df lengh: {len(oid.bbox[ds_type].df)}")
conv.conv_bbox(oid_bbox=oid.bbox[ds_type])
print(f"conv.coco_bbox lengh: {len(conv.coco_bbox)}")
assert len(conv.coco_bbox) == len(oid.bbox[ds_type].df)
print(conv.coco_bbox.head(5))
print(conv.coco_bbox.tail(5))
annotations_json = conv.annotations_json()
assert annotations_json
assert len(annotations_json["annotations"]) == len(conv.coco_bbox)
print(f'the number of bbox: {len(annotations_json["annotations"])}')
for i in range(5):
print(annotations_json["annotations"][i])
for i in range(1, 6):
print(annotations_json["annotations"][-i])
def test_coco_conv_id(self, global_variables):
oid = Oid()
oid.build_config(global_variables.test_config_path)
ds_type = DATASET_TYPE_VAL
oid.build_bbox(ds_type)
oid.build_image(ds_type)
conv = OidConv()
assert conv
coco_images = conv.conv_images(oid_images=oid.images[ds_type])
assert coco_images
assert conv.oid_images
assert coco_images == conv.coco_images
assert coco_images.images
assert coco_images.images_fileid
file_id = "0ba4e129511b237b"
image_id = coco_images.conv_id(file_id)
print(f"file_id: {file_id}, image_id: {image_id}")
assert image_id > 0
oid_class = OidClass()
assert oid_class
oid_class.read(global_variables.class_desc_path)
label_name = global_variables.required_labels[0]
class_name = global_variables.required_class[0]
assert oid_class.conv2label([class_name]) == [label_name]
assert oid_class.label2class(label_name) == class_name
print(f"required_class: {global_variables.required_class}")
coco_category = CocoCategories()
coco_category.build(global_variables.required_class)
conv.set(oid_class, coco_category)
assert conv.oid_class
assert conv.coco_categories
for label_name in global_variables.required_labels:
category_id = conv.conv_label2category_id(label_name)
print(f"label_name: {label_name}, category_id: {category_id}")
def test_oid_class_constructor(self, global_variables):
oid_class = OidClass(global_variables.class_desc_path)
assert oid_class is not None
def test_oid_class_label2class(self, global_variables):
oid_class = OidClass()
oid_class.read(global_variables.class_desc_path)
class_names_01 = ["Vehicle registration plate", "Human face"]
label_names_01 = oid_class.conv2label(class_names_01)
assert oid_class.label2class(label_names_01[0]) == class_names_01[0]
def test_oid_class_constructor_without_path(self):
oid_class = OidClass()
assert oid_class is not None
def test_oid_class_conv2label_error(self):
oid_class = OidClass()
with pytest.raises(ValueError):
class_names = ["Vehicle registration plate"]
label_names = oid_class.conv2label(class_names)
print(label_names)
def test_oid_class_read_path_is_invalid(self):
oid_class = OidClass()
with pytest.raises(ValueError):
oid_class.read()
def test_oid_class_read_path_not_set(self):
oid_class = OidClass()
with pytest.raises(ValueError):
oid_class.read()
def test_oid_class_read(self, global_variables):
oid_class = OidClass()
oid_class.read(global_variables.class_desc_path)
assert oid_class.class_desc_path == global_variables.class_desc_path
class OidConvConfig(object):
"""read an oid config file and check"""
def __init__(self):
""" Initializer """
self.objs = {
DATASET_TYPE_TRAIN: OidConvConfigObj(DATASET_TYPE_TRAIN),
DATASET_TYPE_VAL: OidConvConfigObj(DATASET_TYPE_VAL),
DATASET_TYPE_TEST: OidConvConfigObj(DATASET_TYPE_TEST)
}
self.class_desc_path: str = ""
self.oid_class: Optional[OidClass] = None
self.mixed_dataset: Optional[Dict] = None
self.output: Dict = dict()
self.output['images']: Dict = dict()
self.output['bounding_boxes']: Dict = dict()
@staticmethod
def _check(config_obj: dict) -> None:
"""check the syntax of the config file"""
mandatory_keys = [
'images', 'bounding_boxes', 'class_descriptions', 'output'
]
mandatory_keys_with_dataset = ['images', 'bounding_boxes']
dataset_keys = ['train', 'val', 'test']
class_filter_keys = ['train', 'val', 'train_and_val', 'test']
for m_key in mandatory_keys:
if m_key not in config_obj.keys():
raise TypeError(f'{m_key} not found')
for m_key in mandatory_keys_with_dataset:
for ds_key in dataset_keys:
if ds_key not in config_obj[m_key].keys():
raise TypeError(f'{m_key}.{ds_key} not found')
for m_key in mandatory_keys_with_dataset:
output_key = 'output'
if not config_obj[output_key][m_key]:
raise TypeError(f'{output_key}.{m_key} not found')
for ds_key in dataset_keys:
if ds_key not in config_obj[output_key][m_key].keys():
raise TypeError(f'{output_key}.{m_key}.{ds_key} not found')
class_filter_key = 'class_filter'
for cf_key in config_obj[class_filter_key].keys():
if cf_key not in class_filter_keys:
raise TypeError(f'{class_filter_key}.{cf_key} is invalid')
if 'mixed_dataset' in config_obj:
sum_proportion = 0
for k, v in config_obj['mixed_dataset'].items():
if k == 'train' or k == 'val' or k == 'test':
sum_proportion += v
else:
raise ValueError(f'{k} must be train, val or test.')
if sum_proportion != 1.0:
raise ValueError(f'{sum_proportion} must be 1.0.')
@staticmethod
def _class_filter(config_obj: dict, ds_type: int) -> Optional[List]:
""" fetch the class filter from the config object """
ds_str = dataset_type2str(ds_type)
for l2_keys in config_obj['class_filter'].keys():
if ds_str in l2_keys:
return config_obj['class_filter'][l2_keys]
return None
@staticmethod
def _required_class(config_obj: dict) -> Optional[List]:
""" return the required class """
return config_obj['required_class']
def read(self, config_path: str) -> None:
"""read an oid config file"""
try:
with open(config_path) as f:
config_obj = json.load(f)
self._check(config_obj)
self.class_desc_path = config_obj['class_descriptions']
self.oid_class = OidClass(self.class_desc_path)
self.oid_class.read()
for ds_type in DATASET_TYPE_ALL:
ds_str = dataset_type2str(ds_type)
self.objs[ds_type].set(ds_type, config_obj['images'][ds_str],
config_obj['bounding_boxes'][ds_str],
self.oid_class,
self._class_filter(config_obj, ds_type),
self._required_class(config_obj))
self.output['images'][ds_type] = config_obj['output'][
'images'][ds_str]
self.output['bounding_boxes'][ds_type] = config_obj['output'][
'bounding_boxes'][ds_str]
if 'mixed_dataset' in config_obj:
self.mixed_dataset = config_obj['mixed_dataset']
except Exception as e:
logger.error(f'action=read error={e}')
raise
def get_bbox_path(self, ds_type: int) -> str:
""" return the path for bounding box csv file """
return self.objs[ds_type].bbox_path
def get_label_filter(self, ds_type: int) -> List[str]:
""" return the filter constructed of label names """
return self.objs[ds_type].create_label_filter()
def get_required_labels(self, ds_type: int) -> List[str]:
""" return the requid labels """
return self.objs[ds_type].create_required_labels()
def get_image_dir(self, ds_type: int) -> str:
""" return the path for images of specified the dataset type """
return self.objs[ds_type].images_path
def get_image_all_dirs(self) -> Dict:
""" return pathes for all images """
image_dirs = dict()
for ds_type in DATASET_TYPE_ALL:
image_dirs[ds_type] = self.objs[ds_type].images_path
return image_dirs
def get_output_images_dir(self, dst_type: int) -> str:
""" return the output directory for images """
return self.output['images'][dst_type]
def get_output_bbox_path(self, ds_type: int) -> str:
""" return the output path for the bounding box """
return self.output['bounding_boxes'][ds_type]
def get_proportion(self, ds_type: int) -> Optional[float]:
return self.mixed_dataset.get(dataset_type2str(ds_type), None)