def __npy_copy(src_img_path_list, dst_path):
for img in src_img_path_list:
label = img.replace('img', 'label', 1)
ext = bz_path.get_file_name(label, return_ext=True)[1]
label = label.replace(ext, 'npy')
shutil.copy(label, dst_path)
def imwrite(file_path, image_data):
"""
:param file_path:存储的路径
:param image_data:图像数据
:return:
"""
if not isinstance(image_data, list) and not isinstance(
image_data, np.ndarray):
raise ValueError("image_data的类型为list或者ndarray类型!")
if len(image_data) == 0:
raise ValueError("image_data是空列表，请检查!")
if isinstance(image_data, list):
image_data = np.array(image_data)
if ((len(image_data.shape) != 3 and len(image_data.shape) != 2)) or \
(len(image_data.shape) == 3 and image_data.shape[2] != 3):
raise ValueError('image_data shape只能为heightXwidth 或者heightXwidthX3')
if len(image_data.shape) == 3 and image_data.shape[2] == 3:
image_data = cv2.cvtColor(image_data, cv2.COLOR_BGR2RGB)
folder_path = os.path.split(file_path)[0]
if not os.path.exists(folder_path):
os.makedirs(folder_path)
_, ext = get_file_name(file_path, return_ext=True)
cv2.imencode('.' + ext, image_data)[1].tofile(file_path)
def __npy_save(src_img_path_list, src_label_path_list, dst_path):
for img, label in zip(src_img_path_list, src_label_path_list):
np.save(
dst_path + ''.join(bz_path.get_file_name(img, return_ext=False)) +
'.npy', label)
def __img_copy(src_path_list, dst_path):
for img in src_path_list:
shutil.copy(
img,
dst_path + '.'.join(bz_path.get_file_name(img, return_ext=True)))
def __check_param(img_list, label_list, ratio, task):
# img_list,label_list的参数检查
if not isinstance(img_list, list) and not isinstance(img_list, np.ndarray):
bz_log.error("img_list的类型为list或者ndarray类型!")
bz_log.error("传入的img_list类型：", img_list.dtype)
raise ValueError("img_list的类型为list或者ndarray类型!")
if len(img_list) == 0:
bz_log.error("img_list是空列表，请检查!")
raise ValueError("img_list是空列表，请检查!")
if not os.path.exists(img_list[0]):
bz_log.error("img_list中的文件不存在，可能是非全路径，请检查！")
bz_log.error("传入的错误img_list路径：", img_list[0])
raise ValueError("img_list中的文件不存在，可能是非全路径，请检查！")
if isinstance(img_list, list):
img_list = np.array(img_list)
# 对label_list的参数检查
if label_list is not None:
# list类型转换ndarray
if isinstance(label_list, list):
label_list = np.array(label_list)
if not isinstance(label_list, list) and not isinstance(
label_list, np.ndarray):
bz_log.error("label_list的类型为list或者ndarray类型！")
bz_log.error("label_list：", label_list.dtype)
raise ValueError("label_list的类型为list或者ndarray类型！")
# img_list,label_list的长度检查
if len(img_list) != len(label_list):
bz_log.error("img_list,label_list的长度需要保持一致！")
bz_log.error("img_list和label_list长度分别为：", len(img_list),
len(label_list))
raise ValueError("img_list,label_list的长度需要保持一致！")
# ratio的参数检查
ratio = np.array(ratio)
if ratio.sum() != 1 or len(ratio) != 3 or np.any(ratio < 0) or np.any(
ratio > 1):
bz_log.error(" 'ratio'元素取值为[0,1],其加和需为1，请输入正确比例或取值！")
bz_log.error("ratio值为：", ratio)
raise ValueError(" 'ratio'元素取值为[0,1],其加和需为1，请输入正确比例或取值！")
# 任务类型的参数检查
task = task.lower()
if task not in ('classification', 'segmentation', 'detection'):
bz_log.error(
"task 必须为'classification','Classification','CLASSIFICATION','segmentation',"
"'SEGMENTATION','Segmentation','detection','detection'")
bz_log.error("传入的task为：", task)
raise ValueError(
"task 必须为'classification','Classification','CLASSIFICATION','segmentation',"
"'SEGMENTATION','Segmentation','detection','detection'")
# segmentation特有的img_list,label_list的一致性检查(长度一样，但是名称不一致的情况)
if task == 'segmentation':
check_img_list = list(map(lambda x: bz_path.get_file_name(x),
img_list))
check_label_list = list(
map(lambda x: bz_path.get_file_name(x), label_list))
diff = list(set(check_img_list) ^ set(check_label_list))
if len(diff) != 0:
bz_log.error('img_list、label_list必须保持对应关系！')
raise ValueError('img_list、label_list必须保持对应关系！')
return img_list, label_list, ratio, task
def copy_and_split_train_val_data(original_data_path,
out_path,
min_example_num=20,
ext_list=(['jpg'], ['npy']),
task='classification'):
if not os.path.exists(out_path):
os.makedirs(out_path)
for sub_folder in bz_path.get_subfolder_path(original_data_path,
ret_full_path=False,
is_recursion=False):
img_path = original_data_path + sub_folder + '/img/'
label_path = original_data_path + sub_folder + '/label/'
print(img_path)
img_copy_path = out_path + '/original_data_copy/' + sub_folder + '/img/'
label_copy_path = out_path + '/original_data_copy/' + sub_folder + '/label/'
if not os.path.exists(img_copy_path):
os.makedirs(out_path + '/original_data_copy/' + sub_folder +
'/img/')
os.makedirs(out_path + '/original_data_copy/' + sub_folder +
'/label/')
img_file_path_list = np.sort(
np.array(bz_path.get_file_path(img_path, ret_full_path=True)))
label_file_path_list = np.sort(
np.array(bz_path.get_file_path(label_path, ret_full_path=True)))
img_list_num = len(img_file_path_list)
if img_list_num < min_example_num:
if task == 'classification' or task == 'segmentation':
data_aug = DataAugmentation(
img_file_path_list,
label_file_path_list,
augmentation_ratio=np.ceil(min_example_num / img_list_num),
generate_data_folder=out_path + '/generate_data/',
task=task,
out_file_extension_list=ext_list)
data_aug.augment_data()
for file_path in bz_path.get_file_path(
out_path + '/generate_data/augmentation_img/',
ret_full_path=True):
file_name, ext = bz_path.get_file_name(file_path,
return_ext=True)
# img = cv2.imread(file_path)
# cv2.imwrite(out_path + '/original_data_copy/' + sub_folder + '/img/' + file_name + '.jpg', img)
img = imread(file_path, 'RGB')
imwrite(
out_path + '/original_data_copy/' + sub_folder +
'/img/' + file_name + '.jpg', img)
if task.lower() == 'classification':
for file_path in bz_path.get_file_path(
out_path + '/generate_data/augmentation_label/',
ret_full_path=False):
shutil.copy(
out_path + '/generate_data/augmentation_label/' +
file_path, out_path + '/original_data_copy/' +
sub_folder + '/label/' + file_path)
elif task.lower() == 'segmentation':
for file_path in bz_path.get_file_path(
out_path + '/generate_data/augmentation_label/',
ret_full_path=True):
file_name, ext = bz_path.get_file_name(file_path,
return_ext=True)
# label = cv2.imread(file_path, 0)
# cv2.imwrite(out_path + '/original_data_copy/' + sub_folder + '/label/' + file_name + ".png", label)
label = imread(file_path, 'gray')
imwrite(
out_path + '/original_data_copy/' + sub_folder +
'/label/' + file_name + ".png", label)
elif task == 'detection':
print('样本小于min_example_num，进行预增强...')
bz_log.info('样本小于min_example_num，进行预增强...')
#进行txt的格式转换
txt2npy_path = out_path + '/txt2npy/' + sub_folder + '/label/'
if os.path.exists(txt2npy_path):
shutil.rmtree(txt2npy_path)
os.makedirs(txt2npy_path)
for file_path in bz_path.get_file_path(label_path,
ret_full_path=True):
file_name, ext = bz_path.get_file_name(file_path,
return_ext=True)
if ext == 'txt': # 进行格式转换
with open(file_path, 'r') as f:
lines = f.readlines()
data = []
for line in lines:
temp = list(map(int, line.strip().split(',')))
data.append(
[temp[1], temp[0], temp[3], temp[2], temp[4]])
np.save(txt2npy_path + file_name + ".npy", data)
if len(os.listdir(txt2npy_path)) != 0:
label_file_path_list = np.sort(
np.array(
bz_path.get_file_path(txt2npy_path,
ret_full_path=True)))
yolo_min_example_augmentation_data = DataAugmentation(
img_list=img_file_path_list,
label_list=label_file_path_list,
channel=3,
augmentation_ratio=np.ceil(min_example_num / img_list_num),
# 增强倍数
generate_data_folder=out_path + '/generate_data/' +
sub_folder + os.sep,
task='detection')
yolo_min_example_augmentation_data.augment_data()
for file_path in bz_path.get_file_path(
out_path + '/generate_data/' + sub_folder +
'/augmentation_img/',
ret_full_path=True):
file_name, ext = bz_path.get_file_name(file_path,
return_ext=True)
img = cv2.imread(file_path)
cv2.imwrite(
out_path + '/original_data_copy/' + sub_folder +
'/img/' + file_name + '.jpg', img)
for file_path in bz_path.get_file_path(
out_path + '/generate_data/' + sub_folder +
'/augmentation_label/',
ret_full_path=True):
file_name, ext = bz_path.get_file_name(file_path,
return_ext=True)
shutil.copy(
file_path, out_path + '/original_data_copy/' +
sub_folder + '/label/' + file_name + ".npy")
else:
for file_path in bz_path.get_file_path(img_path,
ret_full_path=True):
file_name, ext = bz_path.get_file_name(file_path,
return_ext=True)
# img = cv2.imread(file_path)
# cv2.imwrite(out_path + '/original_data_copy/' + sub_folder + '/img/' + file_name + '.jpg', img)
img = imread(file_path, 'rgb')
imwrite(
out_path + '/original_data_copy/' + sub_folder + '/img/' +
file_name + '.jpg', img)
if task.lower() == 'classification':
for file_path in label_file_path_list:
file_name, ext = bz_path.get_file_name(file_path,
return_ext=True)
shutil.copy(
file_path, out_path + '/original_data_copy/' +
sub_folder + '/label/' + file_name + ".npy")
elif task.lower() == 'segmentation':
for file_path in bz_path.get_file_path(label_path,
ret_full_path=True):
file_name, ext = bz_path.get_file_name(file_path,
return_ext=True)
# label = cv2.imread(file_path, 0)
# cv2.imwrite(out_path + '/original_data_copy/' + sub_folder + '/label/' + file_name + ".png", label)
label = imread(file_path, 'gray')
imwrite(
out_path + '/original_data_copy/' + sub_folder +
'/label/' + file_name + ".png", label)
elif task.lower() == 'detection':
for file_path in bz_path.get_file_path(label_path,
ret_full_path=True):
file_name, ext = bz_path.get_file_name(file_path,
return_ext=True)
if ext == 'txt': #进行格式转换
with open(file_path, 'r') as f:
lines = f.readlines()
data = []
for line in lines:
temp = list(map(int, line.strip().split(',')))
data.append(
[temp[1], temp[0], temp[3], temp[2], temp[4]])
np.save(
out_path + '/original_data_copy/' + sub_folder +
'/label/' + file_name + ".npy", data)
elif ext == 'npy':
shutil.copy(
file_path, out_path + '/original_data_copy/' +
sub_folder + '/label/' + file_name + ".npy")
img_list, label_list = bz_path.get_img_label_path_list(
img_copy_path, label_copy_path, ret_full_path=True)
if task.lower() != 'segmentation' and task.lower() != 'detection':
label_list = np.array(
[np.loadtxt(label_path) for label_path in label_list])
split_train_eval_test_data(img_list,
label_list,
out_path + '/train/' + sub_folder,
out_path + '/val/' + sub_folder,
out_path + '/test/' + sub_folder,
task=task)