def __update_list(self, data_type): assert((data_type == "train") or \ (data_type == "val" ) or \ (data_type == "test" ) or \ (data_type == "debug" )) max_num = None if data_type == "debug": max_num = 100 data_type = "val" if not data_type in self.__json_path_dict.keys(): json_dir_path = os.path.join(self.__data_path, "labels", data_type) if os.path.exists(json_dir_path): json_path_list_ = [] for json_path in fileio.get_file_list(tgt_dir=json_dir_path, tgt_ext=".json", max_num=max_num): json_path_list_.append(json_path) rgb_dir_path = os.path.join(self.__data_path, "images", data_type) if os.path.exists(rgb_dir_path): rgb_path_list_ = [] for rgb_path in fileio.get_file_list(tgt_dir=rgb_dir_path, tgt_ext=".jpg"): rgb_path_list_.append(rgb_path) assert (len(json_path_list_) > 0) assert (len(rgb_path_list_) > 0) json_key_list = [] rgb_key_list = [] for json_path in json_path_list_: key = fileio.get_file_name(json_path)[:-len(".json")] json_key_list.append(key) for rgb_path in rgb_path_list_: key = fileio.get_file_name(rgb_path)[:-len(".jpg")] rgb_key_list.append(key) assert (len(json_key_list) > 0) assert (len(rgb_key_list) > 0) json_path_list = [] rgb_path_list = [] for j in range(len(json_key_list)): json_key = json_key_list[j] if json_key in rgb_key_list: json_path_list.append(json_path_list_[j]) i = rgb_key_list.index(json_key) rgb_path_list.append(rgb_path_list_[i]) assert (len(json_path_list) == len(rgb_path_list)) assert (len(json_path_list) > 0) assert (len(rgb_path_list) > 0) self.__rgb_path_dict[data_type] = rgb_path_list self.__json_path_dict[data_type] = json_path_list return data_type
def split_json(self): for data_type in ["val", "train"]: dst_dir = os.path.join(self.__data_path, "labels", data_type) if not os.path.exists(dst_dir): os.makedirs(dst_dir) for json_path in fileio.get_file_list(tgt_dir = os.path.join(self.__data_path, "labels"), name_txt = data_type, tgt_ext = ".json"): with open(json_path, "r") as fin: line = fin.readline() read_flg = False while line: if line == " {\n": # start reading read_flg = True out = line elif line.find(" }") == 0: # end reading out = out + line.replace(",", "") cont = json.loads(out) dst_path = os.path.join(dst_dir, cont["name"][:-len(".jpg")] + ".json") with open(dst_path, 'w') as fout: json.dump(cont, fout, ensure_ascii=False, indent=4, sort_keys=True, separators=(',', ': ')) read_flg = False else: if read_flg: out = out + line line = fin.readline()
def __update_seg_list(self, data_type): assert((data_type == "train") or \ (data_type == "test" ) or \ (data_type == "debug" )) max_num = None if data_type == "debug": max_num = 100 data_type = "test" # RGB image if not data_type in self.__seg_rgb_path_dict.keys(): seg_rgb_dir_path = os.path.join(self.__seg_path, "seg_{}_images".format(data_type)) assert(os.path.exists(seg_rgb_dir_path)) assert(os.path.isdir(seg_rgb_dir_path)) seg_rgb_path_list = fileio.get_file_list(tgt_dir = seg_rgb_dir_path, tgt_ext = ".jpg", max_num = max_num) seg_rgb_path_list.sort() self.__seg_rgb_path_dict[data_type] = seg_rgb_path_list # annotation if not data_type in self.__seg_lbl_path_dict.keys(): seg_lbl_dir_path = os.path.join(self.__seg_path, "seg_{}_annotations".format(data_type)) if os.path.exists(seg_lbl_dir_path): assert(os.path.exists(seg_lbl_dir_path)) assert(os.path.isdir(seg_lbl_dir_path)) seg_lbl_path_list = fileio.get_file_list(tgt_dir = seg_lbl_dir_path, tgt_ext = ".png", max_num = max_num) seg_lbl_path_list.sort() self.__seg_lbl_path_dict[data_type] = seg_lbl_path_list # matching check assert(len(self.__seg_rgb_path_dict[data_type]) == len(self.__seg_lbl_path_dict[data_type])) for i, lbl_path in enumerate(self.__seg_lbl_path_dict[data_type]): lbl_name = fileio.get_file_name(lbl_path) lbl_key = lbl_name[:-len(".png")] rgb_path = self.__seg_rgb_path_dict[data_type][i] rgb_name = fileio.get_file_name(rgb_path) rgb_key = rgb_name[:-len(".jpg")] assert(rgb_key == lbl_key) return data_type
def __update_seg_list(self, data_type): assert((data_type == "train") or \ (data_type == "val" ) or \ (data_type == "test" )) if not data_type in self.__seg_path_dict.keys(): seg_dir_path = os.path.join(self.__data_path, "seg", "labels", data_type) if os.path.exists(seg_dir_path): seg_path_list_ = [] for seg_path in fileio.get_file_list(tgt_dir=seg_dir_path, tgt_ext=".png"): seg_path_list_.append(seg_path) segimg_dir_path = os.path.join(self.__data_path, "seg", "images", data_type) if os.path.exists(segimg_dir_path): segimg_path_list_ = [] for segimg_path in fileio.get_file_list( tgt_dir=segimg_dir_path, tgt_ext=".jpg"): segimg_path_list_.append(segimg_path) seg_key_list = [] segimg_key_list = [] for seg_path in seg_path_list_: name = fileio.get_file_name(seg_path) key = name[:name.find("_")] seg_key_list.append(key) for img_path in segimg_path_list_: name = fileio.get_file_name(img_path) key = name[:name.rfind(".")] segimg_key_list.append(key) seg_path_list = [] segimg_path_list = [] for s, seg_key in enumerate(seg_key_list): if seg_key in segimg_key_list: seg_path_list.append(seg_path_list_[s]) segimg_path_list.append( segimg_path_list_[segimg_key_list.index(seg_key)]) assert (len(seg_path_list) > 0) assert (len(seg_path_list) == len(segimg_path_list)) self.__seg_path_dict[data_type] = seg_path_list self.__segimg_path_dict[data_type] = segimg_path_list
def __update_list(self, data_type): assert((data_type == "train") or \ (data_type == "test" ) or \ (data_type == "debug" )) max_num = None if data_type == "debug": max_num = 100 data_type = "test" # RGB image if not data_type in self.__rgb_path_dict.keys(): rgb_dir_path = os.path.join(self.__dtc_path, "dtc_{}_images".format(data_type)) assert(os.path.exists(rgb_dir_path)) assert(os.path.isdir(rgb_dir_path)) rgb_path_list = fileio.get_file_list(tgt_dir = rgb_dir_path, tgt_ext = ".jpg", max_num = max_num) rgb_path_list.sort() self.__rgb_path_dict[data_type] = rgb_path_list # JSON if not data_type in self.__json_path_dict.keys(): json_dir_path = os.path.join(self.__dtc_path, "dtc_{}_annotations".format(data_type)) if os.path.exists(json_dir_path): assert(os.path.isdir(json_dir_path)) json_path_list = fileio.get_file_list(tgt_dir = json_dir_path, tgt_ext = ".json", max_num = max_num) json_path_list.sort() self.__json_path_dict[data_type] = json_path_list # matching check assert(len(self.__rgb_path_dict[data_type]) == len(self.__json_path_dict[data_type])) for i, json_path in enumerate(self.__json_path_dict[data_type]): json_name = fileio.get_file_name(json_path) json_key = json_name[:-len(".json")] rgb_path = self.__rgb_path_dict[data_type][i] rgb_name = fileio.get_file_name(rgb_path) rgb_key = rgb_name[:-len(".jpg")] assert(rgb_key == json_key) return data_type
def __update_seg_list(self, data_type): assert(data_type in ["road01_ins", "road02_ins", "road03_ins", "road04_ins"]) if not data_type in self.__seg_path_dict.keys(): seg_dir_path = os.path.join(self.__data_path, data_type, "Label") if os.path.exists(seg_dir_path): seg_path_list_ = [] for seg_path in fileio.get_file_list(tgt_dir = seg_dir_path, tgt_ext = ".json", recursive = True): seg_path_list_.append(seg_path) segimg_dir_path = os.path.join(self.__data_path, data_type, "ColorImage") if os.path.exists(segimg_dir_path): segimg_path_list_ = [] for segimg_path in fileio.get_file_list(tgt_dir = segimg_dir_path, tgt_ext = ".jpg", recursive = True): segimg_path_list_.append(segimg_path) seg_key_list = [] segimg_key_list = [] for seg_path in seg_path_list_: name = fileio.get_file_name(seg_path) key = name[:name.rfind(".")] seg_key_list.append(key) for img_path in segimg_path_list_: name = fileio.get_file_name(img_path) key = name[:name.rfind(".")] segimg_key_list.append(key) seg_path_list = [] segimg_path_list = [] for s, seg_key in enumerate(seg_key_list): if seg_key in segimg_key_list: seg_path_list.append(seg_path_list_[s]) segimg_path_list.append(segimg_path_list_[segimg_key_list.index(seg_key)]) assert(len(seg_path_list) > 0) assert(len(seg_path_list) == len(segimg_path_list)) self.__seg_path_dict[data_type] = seg_path_list self.__segimg_path_dict[data_type] = segimg_path_list
def list_all_seg_val(self): seg_val = [] for data_type in ["val", "train"]: for seg_path in fileio.get_file_list(tgt_dir = os.path.join(self.__data_path, "seg", "labels", data_type), tgt_ext = ".png"): img = Image.open(seg_path) for new_val in np.unique(np.asarray(img)): if not new_val in seg_val: seg_val.append(new_val) for val in seg_val: print(val)
def check_matching(json_dir_path, img_dir_path, ext = ".jpg"): assert(os.path.exists(json_dir_path)) assert(os.path.isdir(json_dir_path)) assert(os.path.exists(img_dir_path)) assert(os.path.isdir(img_dir_path)) for json_path in fileio.get_file_list(json_dir_path): json_name = fileio.get_file_name(json_path) img_name = json_name.replace(".json", ext) img_path = os.path.join(img_dir_path, img_name) if not os.path.exists(img_path): print(img_path)
def convert_to_npy(self, src_dir_path, dst_dir_path, src_ext): assert (os.path.exists(src_dir_path)) assert (os.path.isdir(src_dir_path)) assert (os.path.exists(dst_dir_path)) assert (os.path.isdir(dst_dir_path)) for img_path in fio.get_file_list(src_dir_path, tgt_ext=src_ext): dst_path = os.path.join( dst_dir_path, fio.get_file_name(img_path))[:-len(src_ext)] + ".npy" np.save(dst_path, np.asarray(Image.open(img_path)).astype(np.uint8))
def list_new_category(self): bdd_cat = [] for json_path in fileio.get_file_list(tgt_dir = os.path.join(self.__data_path, "labels"), name_txt = "val", tgt_ext = ".json"): imgs = json.loads(open(json_path, "r").read()) for img in imgs: for obj in img["labels"]: if not obj["category"] in bdd_cat: bdd_cat.append(obj["category"]) for bdd in bdd_cat: if not bdd in self.label_dict.keys(): print(bdd)
def list_new_category(self): data_type = "train" json_cat = [] for json_path in fileio.get_file_list(tgt_dir = os.path.join(self.__data_path, "dtc_{}_annotations".format(data_type)), name_txt = data_type, tgt_ext = ".json"): info = json.load(open(json_path, "r")) for obj in info["labels"]: if not obj["category"] in json_cat: json_cat.append(obj["category"]) for label in json_cat: if not label in self.label_dict.keys(): print(label)
def __update_list(self, data_path): json_path_list_ = fileio.get_file_list( tgt_dir=os.path.join(data_path, "gtFine_trainvaltest"), # Fineだけ, exclude_path_txt=os.sep + "test" + os.sep, tgt_ext=".json") img_path_list_ = fileio.get_file_list(tgt_dir=os.path.join( data_path, "leftImg8bit_trainvaltest"), tgt_ext=".png", name_txt="leftImg8bit.png") json_key_list = [] for json_path in json_path_list_: name = fileio.get_file_name(json_path) idx = name.find("_") idx = idx + 1 + name[idx + 1:].find("_") idx = idx + 1 + name[idx + 1:].find("_") key = name[:idx] json_key_list.append(key) img_key_list = [] for img_path in img_path_list_: name = fileio.get_file_name(img_path) idx = name.find("_") idx = idx + 1 + name[idx + 1:].find("_") idx = idx + 1 + name[idx + 1:].find("_") key = name[:idx] img_key_list.append(key) img_path_list = [] json_path_list = [] for j in range(len(json_key_list)): json_key = json_key_list[j] if json_key in img_key_list: idx = img_key_list.index(json_key) img_path_list.append(img_path_list_[idx]) json_path_list.append(json_path_list_[j]) return img_path_list, json_path_list