def conf_map_frame(iou_conf, conf_list):
cuda = True
specific_conf = 0.9
cfg_path = "../4Others/color_ball.cfg"
test_root_dir = "../1TestData"
test_label_csv_mame = '../1TestData/label.csv'
classes = load_classes('../4Others/color_ball.names')
blocks = parse_cfg(cfg_path)
model = yolo_v3(blocks)
checkpoint_path = "../4TrainingWeights/experiment/2018-11-30_05_19_48.404150/2018-11-30_05_59_33.719706_model.pth"
checkpoint = torch.load(checkpoint_path)
model.load_state_dict(checkpoint)
model = model.cuda()
test_transform = transforms.Compose([
transforms.Resize(model.net["height"]),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
test_data = CustData(test_label_csv_mame,
test_root_dir,
transform=test_transform)
test_loader = DataLoader(test_data,
shuffle=False,
batch_size=model.net["batch"],
collate_fn=my_collate,
num_workers=6)
best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\
map_frame = get_map(model, test_loader, cuda, conf_list, iou_conf,
classes, False, specific_conf, True)
return best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap, \
map_frame
def compare():
date_time_now = str(datetime.datetime.now()).replace(" ", "_").replace(
":", "_")
compare_path = f"../5Compare/individual_train/{date_time_now}/"
if not os.path.exists(compare_path):
os.makedirs(compare_path)
config_name = "exp_config.p"
conf_list = np.arange(start=0.1, stop=0.95, step=0.025)
seed_range = range(420, 425)
classes = load_classes('../4Others/color_ball.names')
cfg_path = "../4Others/color_ball.cfg"
blocks = parse_cfg(cfg_path)
model = yolo_v3(blocks)
model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True)
with open(compare_path + config_name, "wb") as fp:
pickle.dump(model.net, fp, protocol=pickle.HIGHEST_PROTOCOL)
time_taken_df = pd.DataFrame(columns=list(seed_range))
for cls_index, cls in enumerate(classes):
to_path = f"../{cls}/"
for index, seed in enumerate(seed_range):
model.load_weights("../4Weights/yolov3.weights",
cust_train_zero=True)
x_start = time.time()
random.seed(seed)
if not os.path.exists(to_path):
os.makedirs(to_path)
sub_name = f"{cls}_seed_{seed}_"
name_list = ["img_name", "c", "gx", "gy", "gw", "gh"]
# Original label names
label_csv_mame = '../color_balls/label.csv'
img_txt_path = "../color_balls/*.txt"
prep_labels(img_txt_path, name_list, label_csv_mame)
# sub sampled label names
sub_sample_csv_name = to_path + "label.csv"
sub_sample_txt_path = to_path + "*.txt"
prep_labels(sub_sample_txt_path, name_list, sub_sample_csv_name)
# label_csv_mame = '../1TestData/label.csv'
# img_txt_path = "../1TestData/*.txt"
move_images(label_name=label_csv_mame,
to_path=to_path,
action_fn=action_fn,
cat_nums=[cls_index])
best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\
map_frame = main(model,
classes, conf_list,
sub_sample_csv_name,
sub_sample_txt_path,
to_path,
cuda=True,
specific_conf=0.5,
sub_name=sub_name,
selected_cls=[str(x) for x in [cls_index]])
map_frame.to_csv(f"{compare_path+sub_name}.csv", index=True)
shutil.rmtree(to_path)
time_taken_df.loc[cls, seed] = time.time() - x_start
# if you change the csv_name, pls change accordingly in the
# visualization part
time_taken_df.to_csv(compare_path + 'time_taken.csv', index=True)
def compare():
date_time_now = str(datetime.datetime.now()).replace(" ", "_").replace(
":", "_")
compare_path = f"../5Compare/input_size/{date_time_now}/"
if not os.path.exists(compare_path):
os.makedirs(compare_path)
config_name = "exp_config.p"
num_anchors = 3
conf_list = np.arange(start=0.1, stop=0.95, step=0.025)
seed_range = range(424, 428)
input_sizes = [448, 480]
classes = load_classes('../4Others/color_ball.names')
cfg_path = "../4Others/color_ball.cfg"
blocks = parse_cfg(cfg_path)
model = yolo_v3(blocks)
model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True)
model.net['seed_range'] = seed_range
label_csv_mame = '../color_balls/label.csv'
img_txt_path = "../color_balls/*.txt"
root_dir = "../color_balls"
# save the list of input sizes into the conf dictionary
model.net['width'] = input_sizes
model.net['height'] = input_sizes
# save the dictionary into local pickle file
with open(compare_path + config_name, "wb") as fp:
pickle.dump(model.net, fp, protocol=pickle.HIGHEST_PROTOCOL)
time_taken_df = pd.DataFrame(columns=list(seed_range))
for input_size in input_sizes:
# now parse the size into the model
model.net['width'] = input_size
model.net['height'] = input_size
set_anchors_to_model(model, num_anchors, label_csv_mame, input_size,
input_size)
for index, seed in enumerate(seed_range):
model.load_weights("../4Weights/yolov3.weights",
cust_train_zero=True)
x_start = time.time()
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
sub_name = f"{input_size}_seed_{seed}_"
# label_csv_mame = '../1TestData/label.csv'
# img_txt_path = "../1TestData/*.txt"
best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\
map_frame = main(model,
classes, conf_list,
label_csv_mame,
img_txt_path,
root_dir,
cuda=True,
specific_conf=0.5,
sub_name=sub_name)
map_frame.to_csv(f"{compare_path+sub_name}.csv", index=True)
time_taken_df.loc[input_size, seed] = time.time() - x_start
time_taken_df.to_csv(compare_path + 'time_taken.csv', index=True)
def compare():
date_time_now = str(datetime.datetime.now()).replace(" ", "_").replace(
":", "_")
compare_path = f"../5Compare/one_anchor/448_480/{date_time_now}/"
if not os.path.exists(compare_path):
os.makedirs(compare_path)
num_anchors = 3
conf_list = np.arange(start=0.1, stop=0.95, step=0.025)
seed_range = list(range(1227, 1242))
input_sizes = [448, 480]
classes = load_classes('../4Others/color_ball.names')
cfg_path = "../4Others/color_ball_one_anchor.cfg"
blocks = parse_cfg(cfg_path)
model = yolo_v3(blocks)
model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True)
model.net['seed_range'] = seed_range
label_csv_mame = '../color_balls/label.csv'
img_txt_path = "../color_balls/*.txt"
root_dir = "../color_balls"
time_taken_df = pd.DataFrame(columns=list(seed_range))
for input_size in input_sizes:
# now parse the size into the model
model.net['width'] = input_size
model.net['height'] = input_size
set_anchors_to_model(model, num_anchors, label_csv_mame, input_size,
input_size)
yolo_layer = [(model.layer_type_dic['yolo'][i]) for i in range(3)]
model.net['anchors'] = [
model.module_list[yolo_layer[i]][0].anchors.tolist()
for i in range(3)
]
for index, seed in enumerate(seed_range):
model.load_weights("../4Weights/yolov3.weights",
cust_train_zero=True)
x_start = time.time()
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
sub_name = f"{input_size}_seed_{seed}_"
# label_csv_mame = '../1TestData/label.csv'
# img_txt_path = "../1TestData/*.txt"
best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\
map_frame = main(model,
classes, conf_list,
label_csv_mame,
img_txt_path,
root_dir,
cuda=True,
specific_conf=0.5,
sub_name=sub_name)
map_frame.to_csv(f"{compare_path+sub_name}.csv", index=True)
time_taken_df.loc[input_size, seed] = time.time() - x_start
time_taken_df.to_csv(compare_path + 'time_taken.csv', index=True)
def compare():
date_time_now = str(datetime.datetime.now()).replace(" ", "_").replace(
":", "_")
compare_path = f"../5Compare/multiple_train/{date_time_now}/"
if not os.path.exists(compare_path):
os.makedirs(compare_path)
config_name = "exp_config.p"
conf_list = np.arange(start=0.1, stop=0.95, step=0.025)
seed_range = range(1218, 1221)
classes = load_classes('../4Others/color_ball.names')
cfg_path = "../4Others/color_ball_one_anchor.cfg"
blocks = parse_cfg(cfg_path)
model = yolo_v3(blocks)
model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True)
model.net['seed_used'] = list(seed_range)
with open(compare_path + config_name, "wb") as fp:
pickle.dump(model.net, fp, protocol=pickle.HIGHEST_PROTOCOL)
time_taken_df = pd.DataFrame(columns=list(seed_range))
for index, seed in enumerate(seed_range):
to_path = f"../{seed}/"
model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True)
x_start = time.time()
random.seed(seed)
sub_name = f"_seed_{seed}_"
# Original label names
label_csv_mame = '../color_balls/label.csv'
img_txt_path = "../color_balls/*.txt"
# sub sampled label names
# label_csv_mame = '../1TestData/label.csv'
# img_txt_path = "../1TestData/*.txt"
best_map, best_ap, best_conf, specific_conf_map, specific_conf_ap,\
map_frame = main(model,
classes, conf_list,
label_csv_mame,
img_txt_path,
to_path,
cuda=True,
specific_conf=0.5,
sub_name=sub_name)
map_frame.to_csv(f"{compare_path+sub_name}.csv", index=True)
time_taken_df.loc[0, seed] = time.time() - x_start
# if you change the csv_name, pls change accordingly in the
# visualization part
time_taken_df.to_csv(compare_path + 'time_taken.csv', index=True)
def main():
cuda = True
cfg_path = "../4Others/color_ball.cfg"
test_root_dir = "../1TestData"
test_label_csv_mame = '../1TestData/label.csv'
classes = load_classes('../4Others/color_ball.names')
blocks = parse_cfg(cfg_path)
model = yolo_v3(blocks)
conf_list = np.arange(start=0.2, stop=0.95, step=0.025)
checkpoint_path = "../4TrainingWeights/2018-11-07_23_13_38.391465/2018-11-08_02_45_20.195250_model.pth"
checkpoint = torch.load(checkpoint_path)
model.load_state_dict(checkpoint)
model = model.cuda()
test_transform = transforms.Compose([
transforms.Resize(model.net["height"]),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
test_data = CustData(test_label_csv_mame,
test_root_dir,
transform=test_transform)
test_loader = DataLoader(test_data,
shuffle=True,
batch_size=1,
collate_fn=my_collate,
num_workers=4)
start = time.time()
sum_ap, sum_map, ap, mAP, map_frame = get_map(model,
test_loader,
cuda,
conf_list,
0.5,
classes,
train=False)
print(time.time() - start)
return sum_ap, sum_map, ap, mAP, map_frame
# list_FilenamesToMove = list_FilenamesToMove[:50]
MoveImagesForProcessing.MoveImagesForProcessing(list_FilenamesToMove,
path_ImagesToProcess)
# %%
# layer_type_dic, module_list = create_module(blocks)
# it might imporve performance
# torch.backends.cudnn.benchmark = True
# change cuda to True if you wish to use gpu
cuda = False
colors = pkl.load(open("../4Others/pallete", "rb"))
# images = "../1RawData/"
# det = "../2ProcessedData/"
classes = load_classes('../4Others/coco.names')
batch_size = 12
# Used in filter_results to apply a confidence mask
confidence = float(0.5)
# Used in filter_results to remove points with IoU < nms_thesh
nms_thesh = float(0.4)
start = 0
num_classes = 80
cfg_path = "../4Others/yolov3.cfg"
blocks = parse_cfg(cfg_path)
# setup our model object from the yolo_v3.py file
model = yolo_v3(blocks)
model.load_weights("../4Weights/yolov3.weights")
if cuda:
model = model.cuda()
def compare():
date_time_now = str(datetime.datetime.now()).replace(" ", "_").replace(
":", "_")
compare_path = f"../5Compare/epoch_experiment/{date_time_now}/"
if not os.path.exists(compare_path):
os.makedirs(compare_path)
config_name = "exp_config.p"
conf_list = np.arange(start=0.2, stop=0.95, step=0.025)
seed_range = range(420, 425)
to_path_list = {
"../250_to_300_imgs/": 1.05 / 7,
"../400_to_450_imgs/": 1 / 4,
"../550_to_600_imgs/": 3 / 7,
"../700_to_750_imgs/": 4.7 / 7
}
epochs_list = [25, 30, 35, 40]
classes = load_classes('../4Others/color_ball.names')
cfg_path = "../4Others/color_ball.cfg"
blocks = parse_cfg(cfg_path)
model = yolo_v3(blocks)
model.load_weights("../4Weights/yolov3.weights", cust_train_zero=True)
model.net['epochs_list'] = epochs_list
model.net['img_sampling_info'] = to_path_list
with open(compare_path + config_name, "wb") as fp:
pickle.dump(model.net, fp, protocol=pickle.HIGHEST_PROTOCOL)
for to_path, train_size in to_path_list.items():
time_taken_df = pd.DataFrame(columns=list(epochs_list))
file_name = to_path.strip(".").strip("/")
for index, seed in enumerate(seed_range):
x_start = time.time()
random.seed(seed)
if not os.path.exists(to_path):
os.makedirs(to_path)
name_list = ["img_name", "c", "gx", "gy", "gw", "gh"]
# Original label names
label_csv_mame = '../color_balls/label.csv'
img_txt_path = "../color_balls/*.txt"
prep_labels(img_txt_path, name_list, label_csv_mame)
# sub sampled label names
sub_sample_csv_name = to_path + "label.csv"
sub_sample_txt_path = to_path + "*.txt"
# label_csv_mame = '../1TestData/label.csv'
# img_txt_path = "../1TestData/*.txt"
move_images(label_name=label_csv_mame,
to_path=to_path,
action_fn=split_train,
train_size=train_size,
random_state=seed)
for epoch in epochs_list:
model.load_weights("../4Weights/yolov3.weights",
cust_train_zero=True)
model.net['epochs'] = epoch
sub_name = f"{file_name}_seed_{seed}_epoch_{epoch}_"
best_map, best_ap, best_conf, specific_conf_map,\
specific_conf_ap, map_frame = main(model,
classes, conf_list,
sub_sample_csv_name,
sub_sample_txt_path,
to_path,
cuda=True,
specific_conf=0.5,
sub_name=sub_name)
map_frame.to_csv(f"{compare_path+sub_name}.csv", index=True)
time_taken_df.loc[file_name, epoch] = time.time() - x_start
# if you change the csv_name, pls change accordingly in the
# visualization part
time_taken_df.to_csv(compare_path + f"time_taken_{seed}.csv",
index=True)
shutil.rmtree(to_path)
# change cuda to True if you wish to use gpu
cuda = True
images = "../1RawData/"
det = "../2ProcessedData/"
batch_size = 1
confidence = float(0.5)
nms_thesh = float(0.40)
start = 0
num_classes = 4
#colors = pkl.load(open("../4Others/pallete", "rb"))
#cfg_path = "../4Others/yolov3.cfg"
cfg_path = "../4Others/color_ball.cfg"
blocks = parse_cfg(cfg_path)
model = yolo_v3(blocks)
classes = load_classes('../4Others/color_ball.names')
#model.load_weights("../4Weights/yolov3.weights")
#checkpoint_path = "../4TrainingWeights/2018-11-07_02_53_39.276476/2018-11-07_04_46_47.912754_model.pth"
#checkpoint_path = "../4TrainingWeights/2018-11-07_23_13_38.391465/2018-11-08_02_45_20.195250_model.pth"
checkpoint_path = "../4TrainingWeights/input_size_experiment/416_seed_425_2018-12-09_02_27_00.453122/2018-12-09_02_37_45.648345_model.pth"
#checkpoint_path = "../4TrainingWeights/2018-11-07_23_13_38.391465/2018-11-08_03_43_34.979783_model.pth"
checkpoint = torch.load(checkpoint_path)
model.load_state_dict(checkpoint)
# colors = pkl.load(open("../4Others/pallete", "rb"))
for params in model.parameters():
params.requires_grad = False
if cuda:
model = model.cuda()
# torch.set_num_threads(8)
