def get_config_file(name: str): CONFIG_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "default_configs") COCOCONFIG_DIR = os.path.join(CONFIG_DIR, "coco") MODCONFIG_DIR = os.path.join(CONFIG_DIR, "mnistod") MODGEOCONFIG_DIR = os.path.join(CONFIG_DIR, "mnistodgeo") RESEARCH_DIR = os.path.join(CONFIG_DIR, "research") SEMANTIC_DIR = os.path.join(CONFIG_DIR, "semantic") KEYPOINTS = os.path.join(CONFIG_DIR, "keypoints") MOT = os.path.join(CONFIG_DIR, "MOT") search_dirs = [ SEMANTIC_DIR, COCOCONFIG_DIR, MODCONFIG_DIR, CONFIG_DIR, RESEARCH_DIR, MODGEOCONFIG_DIR, KEYPOINTS, MOT ] if os.path.exists(name): return name if not name.endswith(".yaml"): name = name + ".yaml" for dir in search_dirs: path = os.path.join(dir, name) if os.path.exists(path): return path ALL_FILES = wmlu.recurse_get_filepath_in_dir(CONFIG_DIR, suffix="yaml") for file in ALL_FILES: if os.path.basename(file) == name: return file return name
def get_data(data_dir, batch_size=4, num_samples=1, num_classes=3, id_to_label={}): dataset = get_database(dataset_dir=data_dir, num_classes=num_classes, num_samples=num_samples) files = wmlu.recurse_get_filepath_in_dir(data_dir, suffix=".tfrecord") print("tfrecords:") wmlu.show_list(files) with tf.name_scope('data_provider'): provider = slim.dataset_data_provider.DatasetDataProvider( dataset, num_readers=2, common_queue_capacity=20 * batch_size, common_queue_min=10 * batch_size, seed=int(time.time()), shuffle=True) [image, glabels, bboxes] = provider.get(["image", "object/label", "object/bbox"]) label = 1 if len(id_to_label) == 0: for i in range(1, 80): id_to_label[i] = label label += 1 table = tf.contrib.lookup.HashTable( tf.contrib.lookup.KeyValueTensorInitializer( np.array(list(id_to_label.keys()), dtype=np.int64), np.array(list(id_to_label.values()), dtype=np.int64)), -1) glabels = table.lookup(glabels) wmlt.add_to_hash_table_collection(table.init) return image, glabels, bboxes
def get_datas(): if not os.path.exists(dir_path): print("path {} not exists.".format(dir_path)) files = wml_utils.recurse_get_filepath_in_dir(dir_path,suffix=".xml") print("\ntotal file size {}.".format(len(files))) for file in files: shape, bboxes, labels_text, difficult, truncated = read_voc_xml(file,aspect_range=aspect_range) yield bboxes,labels_text,os.path.basename(file)
def getVOCFilesV2(dir_path): img_files = wmlu.recurse_get_filepath_in_dir(dir_path, ".jpg") res = [] for f in img_files: xml_path = wmlu.change_suffix(f, "xml") if os.path.exists(xml_path) and os.path.exists(f): res.append([f, xml_path]) return res
def add_foler_for_images(dir_path, save_dir): files = wmlu.recurse_get_filepath_in_dir(dir_path, suffix=".jpg;;..png;;.jpeg") for file in files: base_name = wmlu.base_name(file) cur_save_dir = os.path.join(save_dir, base_name) if not os.path.exists(cur_save_dir): os.makedirs(cur_save_dir) os.link(file, os.path.join(cur_save_dir, os.path.basename(file)))
def get_files(data_dir, img_suffix="jpg"): files = wmlu.recurse_get_filepath_in_dir(data_dir, suffix=".json") res = [] for file in files: img_file = wmlu.change_suffix(file, img_suffix) img_file1 = wmlu.change_suffix(file, "jpeg") if os.path.exists(img_file): res.append((img_file, file)) elif os.path.exists(img_file1): res.append((img_file1, file)) return res
def removeUnmatchVOCFiles(dir_path, image_sub_dir="JPEGImages", xml_sub_dir="Annotations", img_suffix=".jpg", shuffe=False): if image_sub_dir is not None: jpeg_dir = os.path.join(dir_path, image_sub_dir) else: jpeg_dir = dir_path if xml_sub_dir is not None: xml_dir = os.path.join(dir_path, xml_sub_dir) else: xml_dir = dir_path inputfilenames = wml_utils.recurse_get_filepath_in_dir(jpeg_dir, suffix=img_suffix) total_removed_jpgs = 0 total_removed_xmls = 0 good_xml_names = [] for file in inputfilenames: base_name = wmlu.base_name(file) xml_path = wmlu.change_suffix(file, "xml") if os.path.exists(xml_path): good_xml_names.append(base_name) else: print(f"remove {file}") total_removed_jpgs += 1 os.remove(file) for file in wml_utils.recurse_get_filepath_in_dir(xml_dir, suffix="xml"): base_name = wmlu.base_name(file) if base_name not in good_xml_names: total_removed_xmls += 1 print(f"remove {file}") os.remove(file) print( f"Total remove {total_removed_jpgs} images, total remove {total_removed_xmls} xmls." )
def get_files(dir_path, sub_dir_name): img_dir = os.path.join(dir_path, sub_dir_name,'images') label_dir = os.path.join(dir_path, sub_dir_name,"v2.0","polygons") res = [] json_files = wmlu.recurse_get_filepath_in_dir(label_dir,suffix=".json") for jf in json_files: base_name = wmlu.base_name(jf) igf = os.path.join(img_dir, base_name + ".jpg") if os.path.exists(igf): res.append((igf, jf)) else: print(f"ERROR: Find {igf} faild, json file is {jf}") return res
def forward(self,inputs): if self.tmp_path.startswith("/tmp"): wmlu.create_empty_dir(self.tmp_path,remove_if_exists=True,yes_to_all=True) print("inputs shape:",inputs.shape) raw_path = os.path.join(self.tmp_path,"input.raw") input_list = os.path.join(self.tmp_path,"file_list.txt") output_dir = os.path.join(self.tmp_path,"output") if not os.path.isdir(output_dir): os.makedirs(output_dir) with open(input_list, "w") as f: if self.output_layers is not None: v = f"#{self.output_layers[0]}" for x in self.output_layers[1:]: v += f" {x}" v += "\n" f.write(v) f.write(raw_path) self.to_snpe_raw(inputs,raw_path) cmd = "{} --container {} --input_list {} --output_dir {}".format(self.bin, self.model_path, input_list, output_dir) print(f"CMD:{cmd}") print(f"All output files.") os.system(cmd) all_files = wmlu.recurse_get_filepath_in_dir(output_dir,suffix=".raw") wmlu.show_list(all_files) print("-------------------------------") res_data = [] output_dir = "/home/wj/0day/output" #for DEBUG if self.output_names is not None: for name,shape in zip(self.output_names,self.output_shapes): path = os.path.join(output_dir,"Result_0",name+self.output_suffix) if not os.path.exists(path): print(f"{path} not exits") res_data.append(None) else: print(f"Read from {path}") td = np.fromfile(path,dtype=np.float32) if shape is not None: td = np.reshape(td,shape) res_data.append(td) else: path = all_files[0] print(f"Use ") td = np.fromfile(path, dtype=np.float32) res_data.append(td) return res_data
def main(_): is_training = False args = default_argument_parser().parse_args() cfg = setup(args) data_loader = DataLoader(cfg=cfg, is_training=is_training) data_args = DATASETS_REGISTRY[cfg.DATASETS.TEST] data, num_classes = data_loader.load_data(*data_args, batch_size=1, is_training=False) cfg.MODEL.ROI_HEADS.NUM_CLASSES = num_classes cfg.MODEL.SSD.NUM_CLASSES = num_classes cfg.MODEL.RETINANET.NUM_CLASSES = num_classes cfg.MODEL.CENTERNET.NUM_CLASSES = num_classes cfg.MODEL.YOLACT.NUM_CLASSES = num_classes cfg.MODEL.FCOS.NUM_CLASSES = num_classes cfg.MODEL.NUM_CLASSES = num_classes cfg.DATASETS.NUM_CLASSES = num_classes cfg.freeze() config.set_global_cfg(cfg) model = PredictModel(cfg=cfg,is_remove_batch=True) model.restoreVariables() data_path = args.test_data_dir if os.path.isdir(data_path): files = wmlu.recurse_get_filepath_in_dir(data_path,suffix=".jpg") else: files = [data_path] save_path = args.save_data_dir wmlu.create_empty_dir(save_path,remove_if_exists=True) for file in files: img = wmli.imread(file) imgs = np.expand_dims(img,axis=0) res = model.predictImages(imgs) if RD_MASKS in res: r_img = odv.draw_bboxes_and_mask(img,res[RD_LABELS],res[RD_PROBABILITY],res[RD_BOXES], res[RD_MASKS], show_text=True) else: r_img = odv.bboxes_draw_on_imgv2(img,res[RD_LABELS],res[RD_PROBABILITY],res[RD_BOXES], text_fn=text_fn, show_text=True) name = wmlu.base_name(file) img_save_path = os.path.join(save_path,name+".png") wmli.imwrite(img_save_path,r_img)
def extract_data(self, data_path): if os.path.exists(self.tmp_dir): shutil.rmtree(self.tmp_dir) os.makedirs(self.tmp_dir) os.system(f"tar xvf {data_path} -C {self.tmp_dir}") files = wmlu.recurse_get_filepath_in_dir(self.tmp_dir, suffix=".index") if len(files) == 0: print(f"Error data {data_path}") return None file_name = wmlu.base_name(files[0]) '''ckpt_file = os.path.join(self.tmp_dir,"checkpoint") with open(ckpt_file,"w") as f: f.write(f"model_checkpoint_path: \"{file_name}\"\n") f.write(f"all_model_checkpoint_paths: \"{file_name}\"\n")''' return os.path.join(self.tmp_dir, file_name)
def getVOCFiles(dir_path, image_sub_dir="JPEGImages", xml_sub_dir="Annotations", img_suffix=".jpg", shuffe=False, auto_sub_dir=False, silent=False, check_xml_file=True): if auto_sub_dir: jpeg_dir = os.path.join(dir_path, "JPEGImages") if not os.path.exists(jpeg_dir): jpeg_dir = dir_path xml_dir = os.path.join(dir_path, "Annotations") if not os.path.exists(xml_dir): xml_dir = dir_path else: if image_sub_dir is not None: jpeg_dir = os.path.join(dir_path, image_sub_dir) else: jpeg_dir = dir_path if xml_sub_dir is not None: xml_dir = os.path.join(dir_path, xml_sub_dir) else: xml_dir = dir_path inputfilenames = wml_utils.recurse_get_filepath_in_dir(jpeg_dir, suffix=img_suffix) img_file_paths = [] xml_file_paths = [] for file in inputfilenames: base_name = os.path.basename(file)[:-4] + ".xml" if xml_sub_dir is not None: xml_path = os.path.join(xml_dir, base_name) else: xml_path = os.path.join(os.path.dirname(file), base_name) if (check_xml_file and os.path.exists(xml_path)) or not check_xml_file: img_file_paths.append(file) xml_file_paths.append(xml_path) elif not silent: print("ERROR, xml file dosen't exists: ", file, xml_path) res = [] for x in zip(img_file_paths, xml_file_paths): res.append(list(x)) if shuffe: random.shuffle(res) return res
def read_data(self, dir_path): img_dir = dir_path if self.img_sub_dir is None else os.path.join( dir_path, self.img_sub_dir) label_dir = dir_path if self.label_sub_dir is None else os.path.join( dir_path, self.label_sub_dir) image_files = wmlu.recurse_get_filepath_in_dir(img_dir, suffix=self.img_suffix) self.files = [] for ifn in image_files: base_name = wmlu.base_name(ifn) label_path = os.path.join(label_dir, base_name + self.label_suffix) if not os.path.exists(label_path): print( f"ERROR: Find label file {label_path} for image {ifn} faild." ) continue else: self.files.append([ifn, label_path])
def get_files(dir_path, sub_dir_name): img_dir = os.path.join(dir_path, 'leftImg8bit', sub_dir_name) label_dir = os.path.join(dir_path, 'gtFine', sub_dir_name) dir_names = wmlu.get_subdir_in_dir(label_dir) res = [] for dir_name in dir_names: l_label_dir = os.path.join(label_dir, dir_name) l_img_dir = os.path.join(img_dir, dir_name) json_files = wmlu.recurse_get_filepath_in_dir(l_label_dir, suffix=".json") for jf in json_files: base_name = wmlu.base_name(jf)[:-15] + "leftImg8bit" igf = os.path.join(l_img_dir, base_name + ".png") if os.path.exists(igf): res.append((igf, jf)) else: print(f"ERROR: Find {igf} faild, json file is {jf}") return res
def __call__(self, dir_path): dir_path = os.path.abspath(dir_path) while True: files = wmlu.recurse_get_filepath_in_dir(dir_path, suffix=".tar.gz") wmlu.show_list(files) process_nr = 0 for file in files: if file in self.history: continue ckpt_file = self.extract_data(file) if ckpt_file is None: continue print("process file {}.".format(file)) self.history.append(file) if self.evaler is not None: result, info = self.evaler(ckpt_file) else: result, info = self.eval(ckpt_file) if result < 0.01: print("Unnormal result {}, ignored.".format(result)) continue if result < self.best_result: print( "{} not the best result, best result is {}/{}, achieved at {}, skip backup." .format(file, self.best_result, self.best_file, self.best_result_time)) continue print("RESULT:", self.best_result, result) print("New best result {}, {}.".format(file, info)) self.best_file = file self.best_result = result self.best_result_time = time.strftime("%m-%d %H:%M:%S", time.localtime()) self.best_result_t = time.time() if process_nr == 0: print("sleep for 30 seconds.") sys.stdout.flush() time.sleep(30)
remove_nr = 0 remove_labels = [] bboxes2remove = [] for i,l in enumerate(labels_names): if l in labels: remove_nr += 1 bboxes2remove.append(bboxes[i]) remove_labels.append(l) continue _bboxes.append(bboxes[i]) _labels_name.append(l) if remove_nr==0: wmlu.try_link(img_file, save_dir) shutil.copy(xml_file,save_dir) else: print(f"{wmlu.base_name(xml_file)} remove {remove_nr} labels, labels is {remove_labels}") img_save_path = osp.join(save_dir,osp.basename(img_file)) xml_save_path = osp.join(save_dir,osp.basename(xml_file)) img = wmli.imread(img_file) img = wmli.remove_boxes_of_img(img,np.array(bboxes2remove).astype(np.int32)) wmli.imwrite(img_save_path,img) write_voc_xml(xml_save_path,img_save_path,shape,_bboxes,_labels_name,is_relative_coordinate=False) if __name__ == "__main__": args = parse_args() files = wmlu.recurse_get_filepath_in_dir(args.src_dir,suffix=".xml") wmlu.create_empty_dir(args.out_dir,remove_if_exists=False) for xml_file in files: trans_one_file(xml_file,args.out_dir,args.labels,args.img_ext)
default="/home/wj/ai/mldata/drive_and_act/phone/training/images/2_np", type=str, help='output rawframe directory') parser.add_argument('--ext', default=".gif", type=str, help='file ext') parser.add_argument("--reverse", default=False, action="store_true", help="resume training") args = parser.parse_args() return args if __name__ == "__main__": args = parse_args() files = wmlu.recurse_get_filepath_in_dir(args.src_file, suffix=args.ext) base_names = [wmlu.base_name(x) for x in files] subdirs = wmlu.get_subdir_in_dir(args.src_dir, append_self=False, absolute_path=False) dirs_need_to_move = [] move_data = [] dirs_dont_move = [] for sd in subdirs: rdir = osp.join(args.src_dir, sd) ddir = args.out_dir tdir = wmlu.base_name(sd, process_suffix=False) move = False if args.reverse:
def multi_thread_to_tfrecords(self,dataset_dir, output_dir, shuffling=False,fidx=0): files = wmlu.recurse_get_filepath_in_dir(dataset_dir,suffix=".jpg") return self.multi_thread_to_tfrecords_by_files(files,output_dir,shuffling,fidx)
def sample_in_one_dir(dir_path, nr): print(f"Sample in {dir_path}") files = wmlu.recurse_get_filepath_in_dir(dir_path) random.shuffle(files) return files[:nr]
import os import wml_utils as wmlu from object_detection2.snpe_toolkit.toolkit import * save_dir = "/home/wj/0day/data" src_dir = "/home/wj/ai/mldata/MOT/MOT15/test/ETH-Crossing/img1" dir_path_in_target_file = None max_file_nr = 20 input_size = (256, 480) if dir_path_in_target_file is None: dir_path_in_target_file = save_dir wmlu.create_empty_dir(save_dir) images = wmlu.recurse_get_filepath_in_dir(src_dir, suffix=".jpg") if max_file_nr is not None: images = images[:max_file_nr] #output_layers = [] output_layers = [ "shared_head/l2_normalize/Square", "shared_head/hw_regr/Conv_1/Conv2D", "shared_head/ct_regr/Conv_1/Conv2D", "shared_head/heat_ct/Conv_1/Conv2D" ], input_file_list = os.path.join(save_dir, "input.txt") with open(input_file_list, "w") as f: if output_layers is not None and len(output_layers) > 0: v = f"#{output_layers[0]}" for x in output_layers[1:]: v += f" {x}" v += "\n" f.write(v)
#return os.remove(file) img_file = wmlu.change_suffix(file,"jpg") if os.path.exists(img_file): print(f"Remove {img_file}") os.remove(img_file) if __name__ == "__main__": args = parse_args() #要处理的文件夹 src_dir = args.src_dir #如果在exclude_dir和src_dir文件夹中同时出现,则从src_dir中删除 exclude_dir = args.exc_dir suffix = args.ext files0 = wmlu.recurse_get_filepath_in_dir(src_dir,suffix=suffix) files1 = wmlu.recurse_get_filepath_in_dir(exclude_dir,suffix=suffix) files1 = [os.path.basename(file) for file in files1] total_skip = 0 total_remove = 0 files_to_remove = [] for file in files0: base_name = os.path.basename(file) if base_name not in files1: print(f"Skip {base_name}") total_skip += 1 else: print(f"Remove {file}") total_remove += 1 files_to_remove.append(file)
frames.append(reader[i]) else: frames = [x for x in reader] save_name = wmlu.base_name(src_data, process_suffix=False) + ".gif" save_path = osp.join(out_dir, save_name) if osp.exists(save_path): print(f"Error {save_path} exists.") print(f"Save {save_path}") imageio.mimsave(save_path, frames, fps=fps) if __name__ == "__main__": args = parse_args() wmlu.create_empty_dir(args.out_dir, remove_if_exists=False) if args.video_ext is not None: files = wmlu.recurse_get_filepath_in_dir(args.src_dir, suffix=args.video_ext) if args.total_nr > 1: print(f"ERROR: can't specify total nr for video.") for file in files: trans_one(file, args.out_dir, args.fps, args.beg_idx, args.total_nr, args.step) else: _sub_dirs = wmlu.recurse_get_subdir_in_dir(args.src_dir, append_self=True) sub_dirs = [] for sd in _sub_dirs: rd = osp.join(args.src_dir, sd) files = glob.glob(osp.join(rd, "*" + args.img_ext)) if len(files) > 3: sub_dirs.append(rd) else:
def main(_): is_training = False args = default_argument_parser().parse_args() cfg = setup(args) data_args = DATASETS_REGISTRY[cfg.DATASETS.TEST] cfg.MODEL.NUM_CLASSES = data_args[2] data_loader = DataLoader(cfg=cfg, is_training=is_training) data, num_classes = data_loader.load_data(*data_args, batch_size=1, is_training=False) cfg.MODEL.ROI_HEADS.NUM_CLASSES = num_classes cfg.MODEL.SSD.NUM_CLASSES = num_classes cfg.MODEL.RETINANET.NUM_CLASSES = num_classes cfg.MODEL.CENTERNET.NUM_CLASSES = num_classes cfg.MODEL.DEEPLAB.NUM_CLASSES = num_classes cfg.MODEL.YOLACT.NUM_CLASSES = num_classes cfg.MODEL.FCOS.NUM_CLASSES = num_classes cfg.MODEL.NUM_CLASSES = num_classes cfg.DATASETS.NUM_CLASSES = num_classes cfg.freeze() config.set_global_cfg(cfg) #model = PredictModel(cfg=cfg,is_remove_batch=False,input_shape=[1,256,480,3],input_name="input",input_dtype=tf.float32) model = PredictModel(cfg=cfg, is_remove_batch=False, input_shape=[1, None, None, 3], input_name="input", input_dtype=tf.float32) #model = PredictModel(cfg=cfg,is_remove_batch=False,input_shape=[1,None,None,3],input_name="input",input_dtype=tf.float32) rename_dict = {RD_BOXES: "bboxes", RD_PROBABILITY: "probs", RD_ID: "id"} model.remove_batch_and_rename(rename_dict) model.restoreVariables() names = [ "shared_head/heat_ct/Conv_1/BiasAdd", "shared_head/ct_regr/Conv_1/BiasAdd", "shared_head/hw_regr/Conv_1/BiasAdd", "shared_head/l2_normalize" ] #model.savePBFile("/home/wj/0day/test.pb",["bboxes","probs","id"]) #model.savePBFile("/home/wj/0day/mot_large.pb",names) model.savePBFile("/home/wj/0day/motv3.pb", names) #return tracker = build_mot(cfg, model) #model.savePBFile("/home/wj/0day/test.pb",names) #return #tracker = JDETracker(model) #tracker = CPPTracker(model) path = '/home/wj/ai/mldata/MOT/MOT20/test/MOT20-04' path = '/home/wj/ai/mldata/MOT/MOT20/test_1img' path = '/home/wj/ai/mldata/MOT/MOT15/test2/TUD-Crossing/img1' path = '/home/wj/ai/mldata/pose3d/basketball2.mp4' path = '/home/wj/ai/mldata/pose3d/tennis1.mp4' #files = wmlu.recurse_get_filepath_in_dir(args.test_data_dir,suffix=".jpg") files = wmlu.recurse_get_filepath_in_dir(path, suffix=".jpg") #save_path = args.save_data_dir save_dir = '/home/wj/ai/mldata/MOT/output2' save_path = osp.join(save_dir, osp.basename(path)) wmlu.create_empty_dir(save_dir, remove_if_exists=True, yes_to_all=True) writer = wmli.VideoWriter(save_path) reader = wmli.VideoReader(path) for img in reader: img = wmli.resize_width(img, 960) objs = tracker.update(img) img = tracker.draw_tracks(img, objs) writer.write(img) writer.release()
def trans_dirs(dir_path, classes_names): txt_paths = wmlu.recurse_get_filepath_in_dir(dir_path, suffix=".txt") for txt_path in txt_paths: trans_yolotxt(txt_path, classes_names)