def save_images_with_cv(source_dir_name,
target_dir_name,
max_dim=None,
recursive=False):
if not os.path.isdir(target_dir_name):
file_helper.create_dir(target_dir_name)
i = 0
for fn in file_helper.enumerate_files(source_dir_name,
recursive=recursive):
try:
i += 1
print("{} - {}".format(i, fn), end=end_txt)
dir_name, name, extension = file_helper.get_file_name_extension(fn)
if string_helper.equals_case_insensitive(extension, ".txt"):
new_fn = file_helper.path_join(target_dir_name,
name + extension)
file_helper.copy_file(fn, new_fn)
else:
mat = cv2.imread(fn)
if max_dim is not None:
mat = image_helper.resize_if_larger(mat, max_dim)
new_fn = file_helper.path_join(target_dir_name, name + ".jpg")
cv2.imwrite(new_fn, mat)
except Exception as e:
print("error - save_images_with_cv: {}".format(fn))
print("save_images_with_cv finished")
def testFileHelperDir(self):
dir = opj(self.tempdir, "testdir")
# create dir:
file_helper.create_dir(dir)
# test if it can be destroyed:
file_helper.destroy_dir_recursive(dir)
self.assertEqual(file_helper.dir_exists(dir), False)
def testFileHelperDir(self):
dir = opj(self.tempdir,"testdir")
# create dir:
file_helper.create_dir(dir)
# test if it can be destroyed:
file_helper.destroy_dir_recursive(dir)
self.assertEqual(file_helper.dir_exists(dir),False)
def oidv6_to_yolo(yolo_dir, classes, images_dir_pattern, labels_dir_pattern):
if not os.path.isdir(yolo_dir):
file_helper.create_dir(yolo_dir)
classes_txt_fn = file_helper.path_join(yolo_dir, "classes.txt")
for class_name in classes:
file_helper.append_line(classes_txt_fn, class_name)
for class_id, class_name in enumerate(classes):
images_dir = images_dir_pattern.format(class_name)
labels_dir = labels_dir_pattern.format(class_name)
_oidv6_to_yolo(class_id, class_name, images_dir, labels_dir, yolo_dir)
def testFileHelperDir(self):
dir = opj(self.tempdir,"testdir")
# create dir:
file_helper.create_dir(dir)
# get stat:
stat1 = file_helper.get_stat(dir)
# stat1 could be "0755" or so
self.assertEqual(len(stat1),4)
self.assertEqual(stat1[0],"0")
def testFileHelperDir(self):
dir = opj(self.tempdir, "testdir")
# create dir:
file_helper.create_dir(dir)
# get stat:
stat1 = file_helper.get_stat(dir)
# stat1 could be "0755" or so
self.assertEqual(len(stat1), 4)
self.assertEqual(stat1[0], "0")
def coco_separate_classes(input_dir, output_dir):
classes_fn = file_helper.path_join(input_dir, "_classes.txt")
classes = []
for line in file_helper.read_lines(classes_fn):
classes.append(line)
i = 0
ann_fn = file_helper.path_join(input_dir, "_annotations.txt")
img_fn = None
for line in file_helper.read_lines(ann_fn):
try:
items = line.split(" ")
img_fn = file_helper.path_join(input_dir, items[0])
dir_name, name, extension = file_helper.get_file_name_extension(
img_fn)
i += 1
print("{} - {}".format(i, img_fn), end=end_txt)
mat = cv2.imread(img_fn)
h, w = image_helper.image_h_w(mat)
for j in range(1, len(items)):
item = items[j]
values_str = item.split(",")
values = []
for v in values_str:
values.append(int(v))
class_name = classes[values[4]]
write_dir = file_helper.path_join(output_dir, class_name)
if not os.path.isdir(write_dir):
file_helper.create_dir(write_dir)
write_img_fn = file_helper.path_join(write_dir,
name + extension)
if not os.path.isfile(write_img_fn):
file_helper.copy_file(img_fn, write_img_fn)
x1, y1, x2, y2 = list(values[:4])
w_ = (x2 - x1)
h_ = (y2 - y1)
cx = (x1 + w_ * 0.5) / w
cy = (y1 + h_ * 0.5) / h
line = "0 {} {} {} {}".format(cx, cy, w_ / w, h_ / h)
write_lbl_fn = file_helper.path_join(write_dir, name + ".txt")
if os.path.isfile(write_lbl_fn):
file_helper.append_line(write_lbl_fn, line)
else:
file_helper.write_lines(write_lbl_fn, [line])
except Exception as e:
print('Error - file:{} msg:{}'.format(img_fn, str(e)))
def testFileHelperDirError(self):
# create dir at not existing location:
try:
exception = False
file_helper.create_dir(opj(self.tempdir,"notexisting","testdir"))
except:
exception = True
# should have raised exception (OSError)
self.assertEqual(exception,True)
# should not exists:
self.assertEqual(file_helper.dir_exists(opj(self.tempdir,"testfile")),False)
def testFileHelperDirError(self):
# create dir at not existing location:
try:
exception = False
file_helper.create_dir(opj(self.tempdir, "notexisting", "testdir"))
except:
exception = True
# should have raised exception (OSError)
self.assertEqual(exception, True)
# should not exists:
self.assertEqual(file_helper.dir_exists(opj(self.tempdir, "testfile")),
False)
def check_single_files(images_dir):
bad_dir = images_dir + "/bad"
counts = {}
fnames = {}
names = {}
for file_full_name in file_helper.enumerate_files(images_dir):
dir_name, name, extension = file_helper.get_file_name_extension(
file_full_name)
if name not in counts:
counts[name] = 1
fnames[name] = file_full_name
names[name] = name + extension
else:
counts[name] += 1
fn = None
for name, count in counts.items():
if count == 1:
try:
fn = fnames[name]
if name == ".DS_Store":
os.remove(fn)
else:
name_with_ext = names[name]
if name_with_ext == "classes.txt":
continue
if not os.path.isdir(bad_dir):
file_helper.create_dir(bad_dir)
new_fn = file_helper.path_join(bad_dir, name_with_ext)
os.rename(fn, new_fn)
print("check_single_files - bad file: " + fn)
except Exception as e:
print(f'ERROR:{fn} - {str(e)} ')
elif count != 2:
print(name)
else:
name_with_ext = names[name]
name, ext = os.path.splitext(name_with_ext)
if ext != ".txt":
txt_fn = os.path.join(images_dir, name + ".txt")
if not os.path.isfile(txt_fn):
if not os.path.isdir(bad_dir):
file_helper.create_dir(bad_dir)
new_fn = os.path.join(bad_dir, name_with_ext)
fn = fnames[name]
os.rename(fn, new_fn)
print("check_single_files - bad file: " + fn)
print("check_single_files finished")
def oidv6_to_yolo2(input_images_dir, input_labels_dir,
output_labels_and_images_dir, class_id):
i = 0
if not os.path.isdir(output_labels_and_images_dir):
file_helper.create_dir(output_labels_and_images_dir)
for label_fn in file_helper.enumerate_files(input_labels_dir,
recursive=False,
wildcard_pattern="*.txt"):
try:
i += 1
print("{} - {}".format(i, label_fn), end=end_txt)
dir_name, name, extension = file_helper.get_file_name_extension(
label_fn)
image_fn = file_helper.path_join(input_images_dir, name + ".jpg")
if os.path.isfile(image_fn):
mat = cv2.imread(image_fn)
h, w = image_helper.image_h_w(mat)
lines = []
for line in file_helper.read_lines(label_fn):
line = line.replace("\t", " ").replace(" ", " ").replace(
" ", " ").replace(" ", " ").replace(" ", " ")
arr0 = line.split(" ")
arr = [class_id]
x1 = float(arr0[1])
y1 = float(arr0[2])
x2 = float(arr0[3])
y2 = float(arr0[4])
arr.append((x1 + x2) * 0.5 / w)
arr.append((y1 + y2) * 0.5 / h)
arr.append((x2 - x1) / w)
arr.append((y2 - y1) / h)
line = ' '.join(str(e) for e in arr)
lines.append(line)
out_img_fn = file_helper.path_join(
output_labels_and_images_dir, name + ".jpg")
out_lbl_fn = file_helper.path_join(
output_labels_and_images_dir, name + ".txt")
if os.path.isfile(out_img_fn):
os.remove(out_img_fn)
if os.path.isfile(out_lbl_fn):
os.remove(out_lbl_fn)
file_helper.write_lines(out_lbl_fn, lines)
file_helper.copy_file(image_fn, out_img_fn)
else:
print("no image: " + image_fn)
except Exception as e:
print('Error - file:{} msg:{}'.format(label_fn, str(e)))
def mirror_images(source_dir_name, target_dir_name):
if not os.path.isdir(target_dir_name):
file_helper.create_dir(target_dir_name)
i = 0
for fn in file_helper.enumerate_files(source_dir_name):
try:
i += 1
print("{} - {}".format(i, fn), end=end_txt)
dir_name, name, extension = file_helper.get_file_name_extension(fn)
if string_helper.equals_case_insensitive(extension, ".txt"):
if name == "classes" and extension == ".txt":
new_fn = file_helper.path_join(target_dir_name,
name + extension)
if not os.path.isfile(new_fn):
file_helper.copy_file(fn, new_fn)
continue
new_fn = file_helper.path_join(target_dir_name,
name + "_mirror" + extension)
if os.path.isfile(new_fn):
raise Exception()
for line in file_helper.read_lines(fn):
lst = line.split(" ")
lst[1] = str(1 - float(lst[1]))
new_line = string_helper.join(lst, " ")
file_helper.append_line(new_fn, new_line)
else:
mat = cv2.imread(fn)
new_fn = file_helper.path_join(target_dir_name,
name + "_mirror" + ".jpg")
mat = cv2.flip(mat, 1)
cv2.imwrite(new_fn, mat)
except Exception as e:
print("error - mirror_images: {}".format(fn))
print("mirror_images finished")
def combine_classes(class_items,
out_images_dir,
out_labels_dir,
out_classes_txt_fn,
out_style="yolo"):
# a typical class_item
# {
# "class_name": "vehicle license plate",
# "dirs":
# [
# {
# "images": "C:/_koray/train_datasets/yolo_oidv6_class0/vehicle_registration_plate",
# "labels": "C:/_koray/train_datasets/yolo_oidv6_class0/vehicle_registration_plate"
# },
# {
# "images": "C:/_koray/train_datasets/yolo_misc/vehicle_registration_plate/class0",
# "labels": "C:/_koray/train_datasets/yolo_misc/vehicle_registration_plate/class0"
# }
# ],
# "resize_style": None, "if_larger" or "crop" - if "crop" -> make the image unique, don't combine, other combine with other classes
# "image_size": 640, - if None no resize, combine all
# "style": "yolo"
# }
class_names = []
if out_style == "yolo":
if not os.path.isdir(out_images_dir):
file_helper.create_dir(out_images_dir)
if not os.path.isdir(out_labels_dir):
file_helper.create_dir(out_labels_dir)
if os.path.isfile(out_classes_txt_fn):
for class_name in file_helper.read_lines(out_classes_txt_fn):
class_names.append(class_name)
else:
raise Exception("Bad out_style: " + out_style)
for class_item in class_items:
class_name = class_item["class_name"]
resize_style = class_item["resize_style"]
image_size = class_item["image_size"]
if image_size is not None:
image_size_txt = "{}_{}".format(image_size[0], image_size[1])
else:
image_size_txt = None
input_style = class_item["input_style"]
if class_name in class_names:
class_index = class_names.index(class_name)
else:
class_index = len(class_names)
class_names.append(class_name)
file_helper.append_line(out_classes_txt_fn, class_name)
i = 0
for dir_item in class_item["dirs"]:
images_dir = dir_item["images"]
labels_dir = dir_item["labels"]
if input_style == "yolo":
for label_fn in file_helper.enumerate_files(
labels_dir, wildcard_pattern="*.txt"):
try:
_dir_name, name, _extension = file_helper.get_file_name_extension(
label_fn)
i += 1
print("{} - {}".format(i, label_fn), end=end_txt)
for line in file_helper.read_lines(label_fn):
line_items = line.split(" ")
cx_norm = float(line_items[1])
cy_norm = float(line_items[2])
w_norm = float(line_items[3])
h_norm = float(line_items[4])
line_items[0] = str(class_index)
out_lbl_fn = None
out_img_fn = None
mat = None
# for image_fn in file_helper.enumerate_files(images_dir, wildcard_pattern=name + ".*"):
image_fn = find_image_file(images_dir, name)
if image_fn is not None:
_dir_name, _name, extension = file_helper.get_file_name_extension(
image_fn)
if extension != ".txt":
try:
mat = cv2.imread(image_fn)
except Exception as e:
print(
'Error reading image file: {} msg:{}'
.format(image_fn, str(e)))
if mat is not None:
if image_size is None or resize_style is None:
out_img_fn = file_helper.path_join(
out_images_dir, name + ".jpg")
out_lbl_fn = file_helper.path_join(
out_labels_dir, name + ".txt")
elif resize_style == "if_larger":
out_img_fn = file_helper.path_join(
out_images_dir, name + "_" +
image_size_txt + ".jpg")
out_lbl_fn = file_helper.path_join(
out_labels_dir, name + "_" +
image_size_txt + ".txt")
mat = image_helper.resize_if_larger(
mat,
max(image_size[0],
image_size[1]))
elif resize_style == "crop":
new_name = file_helper.get_unique_file_name(
)
out_img_fn = file_helper.path_join(
out_images_dir, name +
"_crop_" + new_name + ".jpg")
out_lbl_fn = file_helper.path_join(
out_labels_dir, name +
"_crop_" + new_name + ".txt")
mat, cx, cy, w, h = crop_rect(
mat, cx_norm, cy_norm, w_norm,
h_norm, image_size)
line_items[1] = str(cx)
line_items[2] = str(cy)
line_items[3] = str(w)
line_items[4] = str(h)
else:
raise Exception(
"Bad resize_style: " +
resize_style)
else:
raise Exception("Cannot find image file")
if out_lbl_fn is not None:
line = string_helper.join(line_items, " ")
if os.path.isfile(out_lbl_fn):
file_helper.append_line(out_lbl_fn, line)
else:
file_helper.write_lines(out_lbl_fn, [line])
if out_img_fn is not None and mat is not None:
if not os.path.isfile(out_img_fn):
cv2.imwrite(out_img_fn, mat)
except Exception as e:
print('Error - file:{} msg:{}'.format(
label_fn, str(e)))
else:
raise Exception("Bad input_style: " + out_style)
def mirror_images_of_classes(input_images_dir,
input_labels_dir,
output_dir,
class_ids,
copy_other_classes=True):
out_images_dir = file_helper.path_join(output_dir, "images")
out_labels_dir = file_helper.path_join(output_dir, "labels")
for dir_name in [out_images_dir, out_labels_dir]:
if not os.path.isdir(dir_name):
file_helper.create_dir(dir_name)
i = 0
for fn_label in file_helper.enumerate_files(input_labels_dir,
recursive=False,
wildcard_pattern="*.txt"):
try:
i += 1
print("{} - {}".format(i, fn_label), end=end_txt)
dir_name, name, extension = file_helper.get_file_name_extension(
fn_label)
mirror = False
for line in file_helper.read_lines(fn_label):
class_id = int(line.split(" ")[0])
if class_id in class_ids:
mirror = True
break
if mirror:
fn_image = file_helper.path_join(input_images_dir,
name + ".jpg")
if not os.path.isfile(fn_image):
print("No image: {}".format(fn_image))
else:
new_image_fn = file_helper.path_join(
out_images_dir, name + "_mirror" + ".jpg")
cv2.imwrite(new_image_fn,
image_helper.mirror(cv2.imread(fn_image)))
new_label_fn1 = file_helper.path_join(
out_labels_dir, name + "_mirror" + ".txt")
new_label_fn2 = file_helper.path_join(
out_images_dir, name + "_mirror" + ".txt")
new_label_file_names = [new_label_fn1, new_label_fn2]
for new_label_fn in new_label_file_names:
if os.path.isfile(new_label_fn):
raise Exception()
for line in file_helper.read_lines(fn_label):
lst = line.split(" ")
lst[1] = str(1 - float(lst[1]))
new_line = string_helper.join(lst, " ")
file_helper.append_line(new_label_fn, new_line)
if mirror or copy_other_classes:
fn_image = file_helper.path_join(input_images_dir,
name + ".jpg")
if not os.path.isfile(fn_image):
print("No image: {}".format(fn_image))
else:
new_image_fn = file_helper.path_join(
out_images_dir, name + ".jpg")
cv2.imwrite(new_image_fn, cv2.imread(fn_image))
new_label_fn1 = file_helper.path_join(
out_labels_dir, name + ".txt")
new_label_fn2 = file_helper.path_join(
out_images_dir, name + ".txt")
new_label_file_names = [new_label_fn1, new_label_fn2]
for new_label_fn in new_label_file_names:
if os.path.isfile(new_label_fn):
raise Exception()
file_helper.copy_file(fn_label, new_label_fn)
except Exception as e:
print("error - mirror_images_of_classes: {}".format(fn_label))
print("mirror_images_of_classes {} finished".format(class_ids))
def testFileHelperDir(self): # create dir: file_helper.create_dir(opj(self.tempdir,"testdir")) # test if it exists: self.assertEqual(file_helper.dir_exists(opj(self.tempdir,"testdir")),True)
def testFileHelperDir(self):
# create dir:
file_helper.create_dir(opj(self.tempdir, "testdir"))
# test if it exists:
self.assertEqual(file_helper.dir_exists(opj(self.tempdir, "testdir")),
True)
def oidv6_to_yolo_multi(input_multi_oidv6_dir, output_yolo_dir):
if not os.path.isdir(output_yolo_dir):
file_helper.create_dir(output_yolo_dir)
output_images_dir = file_helper.path_join(output_yolo_dir, "images")
if not os.path.isdir(output_images_dir):
file_helper.create_dir(output_images_dir)
output_labels_dir = file_helper.path_join(output_yolo_dir, "labels")
if not os.path.isdir(output_labels_dir):
file_helper.create_dir(output_labels_dir)
classes = []
print("started.... oidv6_to_yolo_multi - {}".format(input_multi_oidv6_dir))
i = 0
for sub_dir in ["test", "train", "validation"]:
images_dir = file_helper.path_join(input_multi_oidv6_dir, sub_dir)
labels_dir = file_helper.path_join(images_dir, "labels")
for image_fn in file_helper.enumerate_files(images_dir,
recursive=False):
try:
dir_name, name, extension = file_helper.get_file_name_extension(
image_fn)
label_fn = file_helper.path_join(labels_dir, name + ".txt")
if not os.path.isfile(label_fn):
print("!!! File has no label: {}".format(image_fn))
else:
i += 1
print("processing {} - {}".format(str(i), image_fn),
end=" \r")
key_name = name[name.rfind("_") + 1:]
out_image_fn = file_helper.path_join(
output_yolo_dir, "images", key_name + extension)
out_label_fn_1 = file_helper.path_join(
output_yolo_dir, "images", key_name + ".txt")
out_label_fn_2 = file_helper.path_join(
output_yolo_dir, "labels", key_name + ".txt")
class_name = name[:name.rfind("_")]
if class_name not in classes:
classes.append(class_name)
print(class_name)
class_id = classes.index(class_name)
if os.path.isfile(out_image_fn) and os.path.isfile(
out_label_fn_2):
exists = False
for line in file_helper.read_lines(out_label_fn_2):
line = line.replace("\t", " ").replace(
" ", " ").replace(" ", " ").replace(
" ", " ").replace(" ", " ")
if class_id == int(line.split(" ")[0]):
exists = True
break
if exists:
continue
mat = cv2.imread(image_fn)
h, w = image_helper.image_h_w(mat)
lines = []
for line in file_helper.read_lines(label_fn):
line = line.replace("\t", " ").replace(
" ",
" ").replace(" ",
" ").replace(" ",
" ").replace(" ", " ")
arr0 = line.split(" ")
arr = [class_id]
x1 = float(arr0[1])
y1 = float(arr0[2])
x2 = float(arr0[3])
y2 = float(arr0[4])
arr.append((x1 + x2) * 0.5 / w)
arr.append((y1 + y2) * 0.5 / h)
arr.append((x2 - x1) / w)
arr.append((y2 - y1) / h)
line = ' '.join(str(e) for e in arr)
lines.append(line)
if len(lines) > 0:
cv2.imwrite(out_image_fn, mat)
# file_helper.copy_file(image_fn, out_image_fn)
# if os.path.isfile(out_label_fn_2):
# print("merged: " + out_label_fn_2)
for line in lines:
file_helper.append_line(out_label_fn_1, line)
file_helper.append_line(out_label_fn_2, line)
else:
print("nok: " + label_fn)
except Exception as e:
print('Error - file:{} msg:{}'.format(image_fn, str(e)))
classes_txt_fn = file_helper.path_join(output_yolo_dir, "classes.txt")
file_helper.write_lines(classes_txt_fn, classes)
print("finished: oidv6_to_yolo_multi")
def generate_train_txt(output_dir,
model_name,
class_names,
images_dir,
ratio_train=0.7,
ratio_val=0.2,
ratio_test=0.1):
yaml = file_helper.path_join(output_dir, model_name + ".yaml")
if os.path.isfile(yaml):
os.remove(yaml)
with open(yaml, "a") as file:
train_fn = file_helper.path_join(output_dir, model_name + "_train.txt")
file.write(f"train: {train_fn}\n")
val_fn = file_helper.path_join(output_dir, model_name + "_val.txt")
file.write(f"val: {val_fn}\n")
if ratio_test > 0:
test_fn = file_helper.path_join(output_dir,
model_name + "_test.txt")
file.write(f"test: {test_fn}\n")
file.write(f"nc: {str(len(class_names))}" + "\n")
file.write("names: " + str(class_names))
all_data = file_helper.path_join(output_dir, model_name + "_all_data.txt")
train = file_helper.path_join(output_dir, model_name + "_train.txt")
val = file_helper.path_join(output_dir, model_name + "_val.txt")
test = file_helper.path_join(output_dir, model_name, "_test.txt")
if os.path.isfile(all_data):
os.remove(all_data)
if os.path.isfile(train):
os.remove(train)
if os.path.isfile(val):
os.remove(val)
if os.path.isfile(test):
os.remove(test)
labels_dir = os.path.join(
os.path.abspath(os.path.join(images_dir, os.pardir)), "labels")
if not os.path.isdir(labels_dir):
file_helper.create_dir(labels_dir)
for fn in file_helper.enumerate_files(labels_dir, recursive=False):
if str.endswith(fn, ".txt"):
os.remove(fn)
with open(all_data, "a") as file:
for file_full_name in file_helper.enumerate_files(images_dir):
dir_name, name, extension = file_helper.get_file_name_extension(
file_full_name)
if extension != ".txt":
file.write(file_full_name + "\n")
else:
new_file_name = file_helper.path_join(labels_dir,
name + extension)
file_helper.copy_file(file_full_name, new_file_name)
with open(all_data) as f:
content = f.readlines()
content = [x.strip() for x in content]
from random import shuffle
shuffle(content)
train_count = int(len(content) * ratio_train)
val_count = train_count + int(len(content) * ratio_val)
# test %10
if ratio_test > 0:
i = 0
for line in content:
i += 1
if i < train_count:
fn = train
elif i < val_count:
fn = val
else:
fn = test
with open(fn, "a") as file:
file.write(line + "\n")
# print(line)
else:
i = 0
for line in content:
i += 1
if i < train_count:
fn = train
else:
fn = val
with open(fn, "a") as file:
file.write(line + "\n")
# print(line)
print("generate_train_txt finished: " + output_dir)
def setUp(self): self.keep = False # set this to true to keep a resulting temp dir after testing #get a temp dir: self.tempdir = file_helper.get_random_temp_dir_name() #create the temp dir: file_helper.create_dir(self.tempdir)
def setUp(self):
self.keep = False # set this to true to keep a resulting temp dir after testing
#get a temp dir:
self.tempdir = file_helper.get_random_temp_dir_name()
#create the temp dir:
file_helper.create_dir(self.tempdir)