def combine_voc_files_pipeline(folders, combine_folder, possible_wrong_folder):
"""
We need to abandon these with wrong bounding box, https://github.com/thtrieu/darkflow/issues/151
:return:
"""
assert not os.path.isdir(combine_folder)
assert not os.path.isdir(possible_wrong_folder)
os.makedirs(combine_folder)
os.makedirs(possible_wrong_folder)
combine_voc_files(folders, combine_folder)
all_voc_files = preprocess.generate_all_abs_filenames(combine_folder)
wrong_files = []
for voc_f in all_voc_files:
print(voc_f)
ok = assert_voc_xml_file_valid(voc_f)
if not ok:
wrong_files.append(voc_f)
from pprint import pprint
pprint(wrong_files)
print(len(wrong_files), len(wrong_files) / len(all_voc_files))
for wrong_f in wrong_files:
f_basename, f_no_suffix = preprocess.split_the_abs_filename(wrong_f)
shutil.move(wrong_f, possible_wrong_folder+'/'+f_basename)
all_voc_files = preprocess.generate_all_abs_filenames(combine_folder)
wrong_files = []
for voc_f in all_voc_files:
print(voc_f)
ok = assert_voc_xml_file_valid(voc_f)
if not ok:
wrong_files.append(voc_f)
assert len(wrong_files) == 0
def combine_voc_files(floders, new_folder):
if not os.path.isdir(new_folder):
os.makedirs(new_folder)
for i, folder in enumerate(floders):
voc_xml_files = preprocess.generate_all_abs_filenames(folder)
for voc_xml_f in voc_xml_files:
print(voc_xml_f)
f_basename, f_no_suffix = preprocess.split_the_abs_filename(voc_xml_f)
new_file = "{}/{:02}__{}".format(new_folder, i, f_basename)
shutil.copyfile(voc_xml_f, new_file)
def change_des_base_folder(des_folder, old_base_folder, new_base_folder, win2linux=True):
des_all_files = preprocess.generate_all_abs_filenames(des_folder)
for des_f in des_all_files:
print("{} ...".format(des_f))
with open(des_f, 'r') as f:
info = json_tricks.load(f)
abs_file_name = info['abs_file_name'] # type: str
abs_file_name = change_file_prefix(abs_file_name, old_base_folder, new_base_folder, win2linux=True)
info['abs_file_name'] = abs_file_name
with open(des_f, 'w') as f:
json_tricks.dump(info, f, sort_keys=True, indent=4)
def update_des_with_manual_or_track_labeled_bbox(manual_or_track_labeled_dir,
des_dir,
manual=True):
all_labeled_file = preprocess.generate_all_abs_filenames(
manual_or_track_labeled_dir)
for labeled_file in all_labeled_file:
print("NOW labeled file:{}".format(labeled_file))
f_basename, f_no_suffix = preprocess.split_the_abs_filename(
labeled_file)
des_file = des_dir + '/' + f_no_suffix + '.json'
with open(des_file, 'r') as f_r:
info = json_tricks.load(f_r)
with open(labeled_file, 'r') as f_r:
labeled_info = json_tricks.load(f_r)
info['label_list'] = []
info['BBox_list'] = []
if manual:
info['manual_label'] = True
info['track_label'] = False
for shape in labeled_info['shapes']:
info['label_list'].append(shape["label"])
info['BBox_list'].append(shape["points"])
else:
info['manual_label'] = False
info['track_label'] = True
for shape in labeled_info: # ['shapes'], note there is no shape here
info['label_list'].append(shape["label"])
info['BBox_list'].append(shape["points"])
with open(des_file, 'w') as f_w:
json_tricks.dump(info, f_w, sort_keys=True, indent=4)
def generate_dlib_and_voc_detection_file_from_des(
des_dir,
sample_num=100000000,
only_manual_label=False,
resize=False,
resize_ratio=1.0,
resized_dir='resized_frame',
voc_xml_dir='voc_xml_dir',
dlib_detection_file='dlib_detection.xml'):
assert os.path.isdir(des_dir)
assert not os.path.isdir(resized_dir)
assert not os.path.isdir(voc_xml_dir)
assert not os.path.isfile(dlib_detection_file)
if resize:
os.makedirs(resized_dir)
all_des_file = preprocess.generate_all_abs_filenames(des_dir)
im_file_list = []
BBox_list_list = []
label_list_list = []
for des_f in all_des_file:
print("{} ...".format(des_f))
with open(des_f, 'r') as f_r:
info = json_tricks.load(f_r)
if "manual_label" not in info and "track_label" not in info:
continue
abs_file_name = info['abs_file_name']
bbox_list = np.array(info['BBox_list'])
label_list = info['label_list']
label_list_list.append(label_list)
f_basename, f_no_suffix = preprocess.split_the_abs_filename(
abs_file_name)
if resize:
im = cv2.imread(abs_file_name)
im = cv2.resize(im, None, None, resize_ratio, resize_ratio)
new_file = resized_dir + '/' + f_basename
cv2.imwrite(new_file, im)
abs_file_name = os.path.abspath(new_file) #overwrite abs file here
bbox_list *= resize_ratio
if only_manual_label:
if info['manual_label']:
im_file_list.append(abs_file_name)
BBox_list_list.append(bbox_list)
else:
im_file_list.append(abs_file_name)
BBox_list_list.append(bbox_list)
zip_imfile_bbox = list(zip(im_file_list, BBox_list_list))
import random
random.shuffle(zip_imfile_bbox)
zip_imfile_bbox = zip_imfile_bbox[:sample_num]
zip_imfile_bbox = sorted(zip_imfile_bbox) # sort by im file name
im_file_list, BBox_list_list = list(zip(*zip_imfile_bbox))
content_list = []
for BBox_list, im_file in zip(BBox_list_list, im_file_list):
content_list.append(
generate_dlib_content_for_one_img(BBox_list,
im_file,
visualize=True))
cv2.destroyAllWindows()
write_content_to_file(content_list, dlib_detection_file)
os.makedirs(voc_xml_dir)
for BBox_list, label_list, im_file in zip(BBox_list_list, label_list_list,
im_file_list):
generate_voc_for_one(BBox_list, label_list, im_file, voc_xml_dir)