def __update_list(self, data_type):
assert((data_type == "train") or \
(data_type == "val" ) or \
(data_type == "test" ) or \
(data_type == "debug" ))
max_num = None
if data_type == "debug":
max_num = 100
data_type = "val"
if not data_type in self.__json_path_dict.keys():
json_dir_path = os.path.join(self.__data_path, "labels", data_type)
if os.path.exists(json_dir_path):
json_path_list_ = []
for json_path in fileio.get_file_list(tgt_dir=json_dir_path,
tgt_ext=".json",
max_num=max_num):
json_path_list_.append(json_path)
rgb_dir_path = os.path.join(self.__data_path, "images", data_type)
if os.path.exists(rgb_dir_path):
rgb_path_list_ = []
for rgb_path in fileio.get_file_list(tgt_dir=rgb_dir_path,
tgt_ext=".jpg"):
rgb_path_list_.append(rgb_path)
assert (len(json_path_list_) > 0)
assert (len(rgb_path_list_) > 0)
json_key_list = []
rgb_key_list = []
for json_path in json_path_list_:
key = fileio.get_file_name(json_path)[:-len(".json")]
json_key_list.append(key)
for rgb_path in rgb_path_list_:
key = fileio.get_file_name(rgb_path)[:-len(".jpg")]
rgb_key_list.append(key)
assert (len(json_key_list) > 0)
assert (len(rgb_key_list) > 0)
json_path_list = []
rgb_path_list = []
for j in range(len(json_key_list)):
json_key = json_key_list[j]
if json_key in rgb_key_list:
json_path_list.append(json_path_list_[j])
i = rgb_key_list.index(json_key)
rgb_path_list.append(rgb_path_list_[i])
assert (len(json_path_list) == len(rgb_path_list))
assert (len(json_path_list) > 0)
assert (len(rgb_path_list) > 0)
self.__rgb_path_dict[data_type] = rgb_path_list
self.__json_path_dict[data_type] = json_path_list
return data_type
def split_json(self):
for data_type in ["val", "train"]:
dst_dir = os.path.join(self.__data_path, "labels", data_type)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
for json_path in fileio.get_file_list(tgt_dir = os.path.join(self.__data_path, "labels"),
name_txt = data_type,
tgt_ext = ".json"):
with open(json_path, "r") as fin:
line = fin.readline()
read_flg = False
while line:
if line == " {\n":
# start reading
read_flg = True
out = line
elif line.find(" }") == 0:
# end reading
out = out + line.replace(",", "")
cont = json.loads(out)
dst_path = os.path.join(dst_dir, cont["name"][:-len(".jpg")] + ".json")
with open(dst_path, 'w') as fout:
json.dump(cont, fout, ensure_ascii=False, indent=4, sort_keys=True, separators=(',', ': '))
read_flg = False
else:
if read_flg:
out = out + line
line = fin.readline()
def __update_seg_list(self, data_type):
assert((data_type == "train") or \
(data_type == "test" ) or \
(data_type == "debug" ))
max_num = None
if data_type == "debug":
max_num = 100
data_type = "test"
# RGB image
if not data_type in self.__seg_rgb_path_dict.keys():
seg_rgb_dir_path = os.path.join(self.__seg_path, "seg_{}_images".format(data_type))
assert(os.path.exists(seg_rgb_dir_path))
assert(os.path.isdir(seg_rgb_dir_path))
seg_rgb_path_list = fileio.get_file_list(tgt_dir = seg_rgb_dir_path,
tgt_ext = ".jpg",
max_num = max_num)
seg_rgb_path_list.sort()
self.__seg_rgb_path_dict[data_type] = seg_rgb_path_list
# annotation
if not data_type in self.__seg_lbl_path_dict.keys():
seg_lbl_dir_path = os.path.join(self.__seg_path, "seg_{}_annotations".format(data_type))
if os.path.exists(seg_lbl_dir_path):
assert(os.path.exists(seg_lbl_dir_path))
assert(os.path.isdir(seg_lbl_dir_path))
seg_lbl_path_list = fileio.get_file_list(tgt_dir = seg_lbl_dir_path,
tgt_ext = ".png",
max_num = max_num)
seg_lbl_path_list.sort()
self.__seg_lbl_path_dict[data_type] = seg_lbl_path_list
# matching check
assert(len(self.__seg_rgb_path_dict[data_type]) == len(self.__seg_lbl_path_dict[data_type]))
for i, lbl_path in enumerate(self.__seg_lbl_path_dict[data_type]):
lbl_name = fileio.get_file_name(lbl_path)
lbl_key = lbl_name[:-len(".png")]
rgb_path = self.__seg_rgb_path_dict[data_type][i]
rgb_name = fileio.get_file_name(rgb_path)
rgb_key = rgb_name[:-len(".jpg")]
assert(rgb_key == lbl_key)
return data_type
def __update_seg_list(self, data_type):
assert((data_type == "train") or \
(data_type == "val" ) or \
(data_type == "test" ))
if not data_type in self.__seg_path_dict.keys():
seg_dir_path = os.path.join(self.__data_path, "seg", "labels",
data_type)
if os.path.exists(seg_dir_path):
seg_path_list_ = []
for seg_path in fileio.get_file_list(tgt_dir=seg_dir_path,
tgt_ext=".png"):
seg_path_list_.append(seg_path)
segimg_dir_path = os.path.join(self.__data_path, "seg", "images",
data_type)
if os.path.exists(segimg_dir_path):
segimg_path_list_ = []
for segimg_path in fileio.get_file_list(
tgt_dir=segimg_dir_path, tgt_ext=".jpg"):
segimg_path_list_.append(segimg_path)
seg_key_list = []
segimg_key_list = []
for seg_path in seg_path_list_:
name = fileio.get_file_name(seg_path)
key = name[:name.find("_")]
seg_key_list.append(key)
for img_path in segimg_path_list_:
name = fileio.get_file_name(img_path)
key = name[:name.rfind(".")]
segimg_key_list.append(key)
seg_path_list = []
segimg_path_list = []
for s, seg_key in enumerate(seg_key_list):
if seg_key in segimg_key_list:
seg_path_list.append(seg_path_list_[s])
segimg_path_list.append(
segimg_path_list_[segimg_key_list.index(seg_key)])
assert (len(seg_path_list) > 0)
assert (len(seg_path_list) == len(segimg_path_list))
self.__seg_path_dict[data_type] = seg_path_list
self.__segimg_path_dict[data_type] = segimg_path_list
def __update_list(self, data_type):
assert((data_type == "train") or \
(data_type == "test" ) or \
(data_type == "debug" ))
max_num = None
if data_type == "debug":
max_num = 100
data_type = "test"
# RGB image
if not data_type in self.__rgb_path_dict.keys():
rgb_dir_path = os.path.join(self.__dtc_path, "dtc_{}_images".format(data_type))
assert(os.path.exists(rgb_dir_path))
assert(os.path.isdir(rgb_dir_path))
rgb_path_list = fileio.get_file_list(tgt_dir = rgb_dir_path,
tgt_ext = ".jpg",
max_num = max_num)
rgb_path_list.sort()
self.__rgb_path_dict[data_type] = rgb_path_list
# JSON
if not data_type in self.__json_path_dict.keys():
json_dir_path = os.path.join(self.__dtc_path, "dtc_{}_annotations".format(data_type))
if os.path.exists(json_dir_path):
assert(os.path.isdir(json_dir_path))
json_path_list = fileio.get_file_list(tgt_dir = json_dir_path,
tgt_ext = ".json",
max_num = max_num)
json_path_list.sort()
self.__json_path_dict[data_type] = json_path_list
# matching check
assert(len(self.__rgb_path_dict[data_type]) == len(self.__json_path_dict[data_type]))
for i, json_path in enumerate(self.__json_path_dict[data_type]):
json_name = fileio.get_file_name(json_path)
json_key = json_name[:-len(".json")]
rgb_path = self.__rgb_path_dict[data_type][i]
rgb_name = fileio.get_file_name(rgb_path)
rgb_key = rgb_name[:-len(".jpg")]
assert(rgb_key == json_key)
return data_type
def __update_seg_list(self, data_type):
assert(data_type in ["road01_ins", "road02_ins", "road03_ins", "road04_ins"])
if not data_type in self.__seg_path_dict.keys():
seg_dir_path = os.path.join(self.__data_path, data_type, "Label")
if os.path.exists(seg_dir_path):
seg_path_list_ = []
for seg_path in fileio.get_file_list(tgt_dir = seg_dir_path,
tgt_ext = ".json",
recursive = True):
seg_path_list_.append(seg_path)
segimg_dir_path = os.path.join(self.__data_path, data_type, "ColorImage")
if os.path.exists(segimg_dir_path):
segimg_path_list_ = []
for segimg_path in fileio.get_file_list(tgt_dir = segimg_dir_path,
tgt_ext = ".jpg",
recursive = True):
segimg_path_list_.append(segimg_path)
seg_key_list = []
segimg_key_list = []
for seg_path in seg_path_list_:
name = fileio.get_file_name(seg_path)
key = name[:name.rfind(".")]
seg_key_list.append(key)
for img_path in segimg_path_list_:
name = fileio.get_file_name(img_path)
key = name[:name.rfind(".")]
segimg_key_list.append(key)
seg_path_list = []
segimg_path_list = []
for s, seg_key in enumerate(seg_key_list):
if seg_key in segimg_key_list:
seg_path_list.append(seg_path_list_[s])
segimg_path_list.append(segimg_path_list_[segimg_key_list.index(seg_key)])
assert(len(seg_path_list) > 0)
assert(len(seg_path_list) == len(segimg_path_list))
self.__seg_path_dict[data_type] = seg_path_list
self.__segimg_path_dict[data_type] = segimg_path_list
def list_all_seg_val(self):
seg_val = []
for data_type in ["val", "train"]:
for seg_path in fileio.get_file_list(tgt_dir = os.path.join(self.__data_path, "seg", "labels", data_type),
tgt_ext = ".png"):
img = Image.open(seg_path)
for new_val in np.unique(np.asarray(img)):
if not new_val in seg_val:
seg_val.append(new_val)
for val in seg_val:
print(val)
def check_matching(json_dir_path, img_dir_path, ext = ".jpg"):
assert(os.path.exists(json_dir_path))
assert(os.path.isdir(json_dir_path))
assert(os.path.exists(img_dir_path))
assert(os.path.isdir(img_dir_path))
for json_path in fileio.get_file_list(json_dir_path):
json_name = fileio.get_file_name(json_path)
img_name = json_name.replace(".json", ext)
img_path = os.path.join(img_dir_path, img_name)
if not os.path.exists(img_path):
print(img_path)
def convert_to_npy(self, src_dir_path, dst_dir_path, src_ext):
assert (os.path.exists(src_dir_path))
assert (os.path.isdir(src_dir_path))
assert (os.path.exists(dst_dir_path))
assert (os.path.isdir(dst_dir_path))
for img_path in fio.get_file_list(src_dir_path, tgt_ext=src_ext):
dst_path = os.path.join(
dst_dir_path,
fio.get_file_name(img_path))[:-len(src_ext)] + ".npy"
np.save(dst_path,
np.asarray(Image.open(img_path)).astype(np.uint8))
def list_new_category(self):
bdd_cat = []
for json_path in fileio.get_file_list(tgt_dir = os.path.join(self.__data_path, "labels"),
name_txt = "val",
tgt_ext = ".json"):
imgs = json.loads(open(json_path, "r").read())
for img in imgs:
for obj in img["labels"]:
if not obj["category"] in bdd_cat:
bdd_cat.append(obj["category"])
for bdd in bdd_cat:
if not bdd in self.label_dict.keys():
print(bdd)
def list_new_category(self):
data_type = "train"
json_cat = []
for json_path in fileio.get_file_list(tgt_dir = os.path.join(self.__data_path, "dtc_{}_annotations".format(data_type)),
name_txt = data_type,
tgt_ext = ".json"):
info = json.load(open(json_path, "r"))
for obj in info["labels"]:
if not obj["category"] in json_cat:
json_cat.append(obj["category"])
for label in json_cat:
if not label in self.label_dict.keys():
print(label)
def __update_list(self, data_path):
json_path_list_ = fileio.get_file_list(
tgt_dir=os.path.join(data_path, "gtFine_trainvaltest"), # Fineだけ,
exclude_path_txt=os.sep + "test" + os.sep,
tgt_ext=".json")
img_path_list_ = fileio.get_file_list(tgt_dir=os.path.join(
data_path, "leftImg8bit_trainvaltest"),
tgt_ext=".png",
name_txt="leftImg8bit.png")
json_key_list = []
for json_path in json_path_list_:
name = fileio.get_file_name(json_path)
idx = name.find("_")
idx = idx + 1 + name[idx + 1:].find("_")
idx = idx + 1 + name[idx + 1:].find("_")
key = name[:idx]
json_key_list.append(key)
img_key_list = []
for img_path in img_path_list_:
name = fileio.get_file_name(img_path)
idx = name.find("_")
idx = idx + 1 + name[idx + 1:].find("_")
idx = idx + 1 + name[idx + 1:].find("_")
key = name[:idx]
img_key_list.append(key)
img_path_list = []
json_path_list = []
for j in range(len(json_key_list)):
json_key = json_key_list[j]
if json_key in img_key_list:
idx = img_key_list.index(json_key)
img_path_list.append(img_path_list_[idx])
json_path_list.append(json_path_list_[j])
return img_path_list, json_path_list
