def conf_map_frame(iou_conf, conf_list): cuda = True specific_conf = 0.9 cfg_path = "../4Others/color_ball.cfg" test_root_dir = "../1TestData" test_label_csv_mame = '../1TestData/label.csv' classes = load_classes('../4Others/color_ball.names') blocks = parse_cfg(cfg_path) model = yolo_v3(blocks) checkpoint_path = "../4TrainingWeights/experiment/2018-11-30_05_19_48.404150/2018-11-30_05_59_33.719706_model.pth" checkpoint = torch.load(checkpoint_path) model.load_state_dict(checkpoint) model = model.cuda() test_transform = transforms.Compose([ transforms.Resize(model.net["height"]), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) test_data = CustData(test_label_csv_mame, test_root_dir, transform=test_transform) test_loader = DataLoader(test_data, shuffle=False, batch_size=model.net["batch"], collate_fn=my_collate, num_workers=6) best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\ map_frame = get_map(model, test_loader, cuda, conf_list, iou_conf, classes, False, specific_conf, True) return best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap, \ map_frame
def compare(): date_time_now = str(datetime.datetime.now()).replace(" ", "_").replace( ":", "_") compare_path = f"../5Compare/individual_train/{date_time_now}/" if not os.path.exists(compare_path): os.makedirs(compare_path) config_name = "exp_config.p" conf_list = np.arange(start=0.1, stop=0.95, step=0.025) seed_range = range(420, 425) classes = load_classes('../4Others/color_ball.names') cfg_path = "../4Others/color_ball.cfg" blocks = parse_cfg(cfg_path) model = yolo_v3(blocks) model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True) with open(compare_path + config_name, "wb") as fp: pickle.dump(model.net, fp, protocol=pickle.HIGHEST_PROTOCOL) time_taken_df = pd.DataFrame(columns=list(seed_range)) for cls_index, cls in enumerate(classes): to_path = f"../{cls}/" for index, seed in enumerate(seed_range): model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True) x_start = time.time() random.seed(seed) if not os.path.exists(to_path): os.makedirs(to_path) sub_name = f"{cls}_seed_{seed}_" name_list = ["img_name", "c", "gx", "gy", "gw", "gh"] # Original label names label_csv_mame = '../color_balls/label.csv' img_txt_path = "../color_balls/*.txt" prep_labels(img_txt_path, name_list, label_csv_mame) # sub sampled label names sub_sample_csv_name = to_path + "label.csv" sub_sample_txt_path = to_path + "*.txt" prep_labels(sub_sample_txt_path, name_list, sub_sample_csv_name) # label_csv_mame = '../1TestData/label.csv' # img_txt_path = "../1TestData/*.txt" move_images(label_name=label_csv_mame, to_path=to_path, action_fn=action_fn, cat_nums=[cls_index]) best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\ map_frame = main(model, classes, conf_list, sub_sample_csv_name, sub_sample_txt_path, to_path, cuda=True, specific_conf=0.5, sub_name=sub_name, selected_cls=[str(x) for x in [cls_index]]) map_frame.to_csv(f"{compare_path+sub_name}.csv", index=True) shutil.rmtree(to_path) time_taken_df.loc[cls, seed] = time.time() - x_start # if you change the csv_name, pls change accordingly in the # visualization part time_taken_df.to_csv(compare_path + 'time_taken.csv', index=True)
def compare(): date_time_now = str(datetime.datetime.now()).replace(" ", "_").replace( ":", "_") compare_path = f"../5Compare/input_size/{date_time_now}/" if not os.path.exists(compare_path): os.makedirs(compare_path) config_name = "exp_config.p" num_anchors = 3 conf_list = np.arange(start=0.1, stop=0.95, step=0.025) seed_range = range(424, 428) input_sizes = [448, 480] classes = load_classes('../4Others/color_ball.names') cfg_path = "../4Others/color_ball.cfg" blocks = parse_cfg(cfg_path) model = yolo_v3(blocks) model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True) model.net['seed_range'] = seed_range label_csv_mame = '../color_balls/label.csv' img_txt_path = "../color_balls/*.txt" root_dir = "../color_balls" # save the list of input sizes into the conf dictionary model.net['width'] = input_sizes model.net['height'] = input_sizes # save the dictionary into local pickle file with open(compare_path + config_name, "wb") as fp: pickle.dump(model.net, fp, protocol=pickle.HIGHEST_PROTOCOL) time_taken_df = pd.DataFrame(columns=list(seed_range)) for input_size in input_sizes: # now parse the size into the model model.net['width'] = input_size model.net['height'] = input_size set_anchors_to_model(model, num_anchors, label_csv_mame, input_size, input_size) for index, seed in enumerate(seed_range): model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True) x_start = time.time() random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) sub_name = f"{input_size}_seed_{seed}_" # label_csv_mame = '../1TestData/label.csv' # img_txt_path = "../1TestData/*.txt" best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\ map_frame = main(model, classes, conf_list, label_csv_mame, img_txt_path, root_dir, cuda=True, specific_conf=0.5, sub_name=sub_name) map_frame.to_csv(f"{compare_path+sub_name}.csv", index=True) time_taken_df.loc[input_size, seed] = time.time() - x_start time_taken_df.to_csv(compare_path + 'time_taken.csv', index=True)
def compare(): date_time_now = str(datetime.datetime.now()).replace(" ", "_").replace( ":", "_") compare_path = f"../5Compare/one_anchor/448_480/{date_time_now}/" if not os.path.exists(compare_path): os.makedirs(compare_path) num_anchors = 3 conf_list = np.arange(start=0.1, stop=0.95, step=0.025) seed_range = list(range(1227, 1242)) input_sizes = [448, 480] classes = load_classes('../4Others/color_ball.names') cfg_path = "../4Others/color_ball_one_anchor.cfg" blocks = parse_cfg(cfg_path) model = yolo_v3(blocks) model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True) model.net['seed_range'] = seed_range label_csv_mame = '../color_balls/label.csv' img_txt_path = "../color_balls/*.txt" root_dir = "../color_balls" time_taken_df = pd.DataFrame(columns=list(seed_range)) for input_size in input_sizes: # now parse the size into the model model.net['width'] = input_size model.net['height'] = input_size set_anchors_to_model(model, num_anchors, label_csv_mame, input_size, input_size) yolo_layer = [(model.layer_type_dic['yolo'][i]) for i in range(3)] model.net['anchors'] = [ model.module_list[yolo_layer[i]][0].anchors.tolist() for i in range(3) ] for index, seed in enumerate(seed_range): model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True) x_start = time.time() random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) sub_name = f"{input_size}_seed_{seed}_" # label_csv_mame = '../1TestData/label.csv' # img_txt_path = "../1TestData/*.txt" best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\ map_frame = main(model, classes, conf_list, label_csv_mame, img_txt_path, root_dir, cuda=True, specific_conf=0.5, sub_name=sub_name) map_frame.to_csv(f"{compare_path+sub_name}.csv", index=True) time_taken_df.loc[input_size, seed] = time.time() - x_start time_taken_df.to_csv(compare_path + 'time_taken.csv', index=True)
def compare(): date_time_now = str(datetime.datetime.now()).replace(" ", "_").replace( ":", "_") compare_path = f"../5Compare/multiple_train/{date_time_now}/" if not os.path.exists(compare_path): os.makedirs(compare_path) config_name = "exp_config.p" conf_list = np.arange(start=0.1, stop=0.95, step=0.025) seed_range = range(1218, 1221) classes = load_classes('../4Others/color_ball.names') cfg_path = "../4Others/color_ball_one_anchor.cfg" blocks = parse_cfg(cfg_path) model = yolo_v3(blocks) model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True) model.net['seed_used'] = list(seed_range) with open(compare_path + config_name, "wb") as fp: pickle.dump(model.net, fp, protocol=pickle.HIGHEST_PROTOCOL) time_taken_df = pd.DataFrame(columns=list(seed_range)) for index, seed in enumerate(seed_range): to_path = f"../{seed}/" model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True) x_start = time.time() random.seed(seed) sub_name = f"_seed_{seed}_" # Original label names label_csv_mame = '../color_balls/label.csv' img_txt_path = "../color_balls/*.txt" # sub sampled label names # label_csv_mame = '../1TestData/label.csv' # img_txt_path = "../1TestData/*.txt" best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\ map_frame = main(model, classes, conf_list, label_csv_mame, img_txt_path, to_path, cuda=True, specific_conf=0.5, sub_name=sub_name) map_frame.to_csv(f"{compare_path+sub_name}.csv", index=True) time_taken_df.loc[0, seed] = time.time() - x_start # if you change the csv_name, pls change accordingly in the # visualization part time_taken_df.to_csv(compare_path + 'time_taken.csv', index=True)
def main(): cuda = True cfg_path = "../4Others/color_ball.cfg" test_root_dir = "../1TestData" test_label_csv_mame = '../1TestData/label.csv' classes = load_classes('../4Others/color_ball.names') blocks = parse_cfg(cfg_path) model = yolo_v3(blocks) conf_list = np.arange(start=0.2, stop=0.95, step=0.025) checkpoint_path = "../4TrainingWeights/2018-11-07_23_13_38.391465/2018-11-08_02_45_20.195250_model.pth" checkpoint = torch.load(checkpoint_path) model.load_state_dict(checkpoint) model = model.cuda() test_transform = transforms.Compose([ transforms.Resize(model.net["height"]), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) test_data = CustData(test_label_csv_mame, test_root_dir, transform=test_transform) test_loader = DataLoader(test_data, shuffle=True, batch_size=1, collate_fn=my_collate, num_workers=4) start = time.time() sum_ap, sum_map, ap, mAP, map_frame = get_map(model, test_loader, cuda, conf_list, 0.5, classes, train=False) print(time.time() - start) return sum_ap, sum_map, ap, mAP, map_frame
# list_FilenamesToMove = list_FilenamesToMove[:50] MoveImagesForProcessing.MoveImagesForProcessing(list_FilenamesToMove, path_ImagesToProcess) # %% # layer_type_dic, module_list = create_module(blocks) # it might imporve performance # torch.backends.cudnn.benchmark = True # change cuda to True if you wish to use gpu cuda = False colors = pkl.load(open("../4Others/pallete", "rb")) # images = "../1RawData/" # det = "../2ProcessedData/" classes = load_classes('../4Others/coco.names') batch_size = 12 # Used in filter_results to apply a confidence mask confidence = float(0.5) # Used in filter_results to remove points with IoU < nms_thesh nms_thesh = float(0.4) start = 0 num_classes = 80 cfg_path = "../4Others/yolov3.cfg" blocks = parse_cfg(cfg_path) # setup our model object from the yolo_v3.py file model = yolo_v3(blocks) model.load_weights("../4Weights/yolov3.weights") if cuda: model = model.cuda()
def compare(): date_time_now = str(datetime.datetime.now()).replace(" ", "_").replace( ":", "_") compare_path = f"../5Compare/epoch_experiment/{date_time_now}/" if not os.path.exists(compare_path): os.makedirs(compare_path) config_name = "exp_config.p" conf_list = np.arange(start=0.2, stop=0.95, step=0.025) seed_range = range(420, 425) to_path_list = { "../250_to_300_imgs/": 1.05 / 7, "../400_to_450_imgs/": 1 / 4, "../550_to_600_imgs/": 3 / 7, "../700_to_750_imgs/": 4.7 / 7 } epochs_list = [25, 30, 35, 40] classes = load_classes('../4Others/color_ball.names') cfg_path = "../4Others/color_ball.cfg" blocks = parse_cfg(cfg_path) model = yolo_v3(blocks) model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True) model.net['epochs_list'] = epochs_list model.net['img_sampling_info'] = to_path_list with open(compare_path + config_name, "wb") as fp: pickle.dump(model.net, fp, protocol=pickle.HIGHEST_PROTOCOL) for to_path, train_size in to_path_list.items(): time_taken_df = pd.DataFrame(columns=list(epochs_list)) file_name = to_path.strip(".").strip("/") for index, seed in enumerate(seed_range): x_start = time.time() random.seed(seed) if not os.path.exists(to_path): os.makedirs(to_path) name_list = ["img_name", "c", "gx", "gy", "gw", "gh"] # Original label names label_csv_mame = '../color_balls/label.csv' img_txt_path = "../color_balls/*.txt" prep_labels(img_txt_path, name_list, label_csv_mame) # sub sampled label names sub_sample_csv_name = to_path + "label.csv" sub_sample_txt_path = to_path + "*.txt" # label_csv_mame = '../1TestData/label.csv' # img_txt_path = "../1TestData/*.txt" move_images(label_name=label_csv_mame, to_path=to_path, action_fn=split_train, train_size=train_size, random_state=seed) for epoch in epochs_list: model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True) model.net['epochs'] = epoch sub_name = f"{file_name}_seed_{seed}_epoch_{epoch}_" best_map, best_ap, best_conf, specific_conf_map,\ specific_conf_ap, map_frame = main(model, classes, conf_list, sub_sample_csv_name, sub_sample_txt_path, to_path, cuda=True, specific_conf=0.5, sub_name=sub_name) map_frame.to_csv(f"{compare_path+sub_name}.csv", index=True) time_taken_df.loc[file_name, epoch] = time.time() - x_start # if you change the csv_name, pls change accordingly in the # visualization part time_taken_df.to_csv(compare_path + f"time_taken_{seed}.csv", index=True) shutil.rmtree(to_path)
# change cuda to True if you wish to use gpu cuda = True images = "../1RawData/" det = "../2ProcessedData/" batch_size = 1 confidence = float(0.5) nms_thesh = float(0.40) start = 0 num_classes = 4 #colors = pkl.load(open("../4Others/pallete", "rb")) #cfg_path = "../4Others/yolov3.cfg" cfg_path = "../4Others/color_ball.cfg" blocks = parse_cfg(cfg_path) model = yolo_v3(blocks) classes = load_classes('../4Others/color_ball.names') #model.load_weights("../4Weights/yolov3.weights") #checkpoint_path = "../4TrainingWeights/2018-11-07_02_53_39.276476/2018-11-07_04_46_47.912754_model.pth" #checkpoint_path = "../4TrainingWeights/2018-11-07_23_13_38.391465/2018-11-08_02_45_20.195250_model.pth" checkpoint_path = "../4TrainingWeights/input_size_experiment/416_seed_425_2018-12-09_02_27_00.453122/2018-12-09_02_37_45.648345_model.pth" #checkpoint_path = "../4TrainingWeights/2018-11-07_23_13_38.391465/2018-11-08_03_43_34.979783_model.pth" checkpoint = torch.load(checkpoint_path) model.load_state_dict(checkpoint) # colors = pkl.load(open("../4Others/pallete", "rb")) for params in model.parameters(): params.requires_grad = False if cuda: model = model.cuda() # torch.set_num_threads(8)