def del_empty_ending_files(root_dir: str, ending: str, display=False):
assertor.path_exist_assert(root_dir)
if display:
print('Begin to get ending file path list')
ending_file_path_list = yy.io.get_all_ending_files_under_dir(
root_dir, [ending], display=display)
if display:
print('Done get ending file path list')
empty_count = 0
total_count = 0
for file_path in ending_file_path_list:
if display:
print('processing %d/%d ...' %
(total_count, len(ending_file_path_list)))
total_count += 1
size = os.path.getsize(file_path)
if size == 0:
yy.io.del_file('', file_path)
if display:
print('Done del empty file %s' % (file_path))
empty_count += 1
if display:
print('Total delete %d empty files' % (empty_count))
def get_target_name_dir_path_list_under_dir(file_dir: str,
name_list: list,
display=False):
"""
获得所有名字列表中的路径列表
:param file_dir:
:param name_list:
:param display:
:return:
"""
assertor.path_exist_assert(file_dir)
assertor.list_type_assert(name_list, str)
if display:
print('Begin get all target name dir under dir')
file_dir = judge_add_os_sep(file_dir)
path_set = set()
count = 0
for root, dirs, files in os.walk(file_dir, topdown=False):
if len(dirs) != 0:
for dir_name in dirs:
if dir_name in name_list:
dir_path = join(root, dir_name)
path_set.add(dir_path)
count += 1
if display:
print(count)
print(dir_path)
if display:
print('Done get all target name dir under dir')
return list(path_set)
def get_all_ending_files_under_dir(file_dir: str,
ending_list: list,
display=False):
assertor.path_exist_assert(file_dir)
assertor.list_type_assert(ending_list, str)
if display:
print('Begin get all ending files under dir')
file_dir = judge_add_os_sep(file_dir)
path_set = set()
count = 0
for root, dirs, files in os.walk(file_dir, topdown=False):
if len(files) != 0:
for file in files:
ending = os.path.splitext(file)[-1]
if ending in ending_list:
file_path = join(root, file)
path_set.add(file_path)
count += 1
if display:
print(count)
print(file_path)
if display:
print('Done get all ending files under dir')
return list(path_set)
def copy_file(old_dir, old_name, new_dir, new_name):
old_path = join(old_dir, old_name)
# 断言保证
assertor.file_exist_assert(old_path)
assertor.path_exist_assert(new_dir)
new_path = join(new_dir, new_name)
shutil.copyfile(old_path, new_path)
def file_dir(self) -> str:
"""
判别file_dir是否存在,并返回file_dir
:return: str
"""
# 断言保证
assertor.path_exist_assert(self._file_dir)
return self._file_dir
def move_dir_or_file(ori_dir, ori_name, tar_dir, tar_name):
ori_path = join(ori_dir, ori_name)
assertor.path_exist_assert(ori_path)
judge_mkdir(tar_dir)
tar_path = join(tar_dir, tar_name)
shutil.move(ori_path, tar_path)
def get_files_path(root_dir):
assertor.path_exist_assert(root_dir)
path_set = set()
for root, dirs, files in os.walk(root_dir, topdown=False):
if len(files) != 0:
for file in files:
path_set.add(join(root, file))
return list(path_set)
def file_dir(self, value: str):
"""
判别value的path是否存在,若存在则设置file_dir为value
:param value: str
:return:
"""
# 断言保证
assertor.path_exist_assert(value)
self._file_dir = value
def get_all_file_ending_under_dir(file_dir: str):
assertor.path_exist_assert(file_dir)
file_ending = set()
for root, dirs, files in os.walk(file_dir, topdown=False):
if len(files) != 0:
for file in files:
ending = os.path.splitext(file)[-1]
file_ending.add(ending)
return file_ending
def make_same_empty_sub_dirs(root_dir,
save_dir,
anti_list=['lost+found', '.Trash']):
assertor.path_exist_assert(root_dir)
yy.io.judge_mkdir(save_dir)
sub_dir_list = yy.io.get_dir_list_under_root_dir(root_dir, anti_list)
for sub_dir in sub_dir_list:
sub_dir_name = sub_dir.split(os.sep)[-1]
save_path = yy.io.join(save_dir, sub_dir_name)
yy.io.judge_mkdir(save_path)
def un_compression_all_ending_files_under_dir(file_dir: str,
ending_list: list):
assertor.path_exist_assert(file_dir)
assertor.list_type_assert(ending_list, str)
file_dir = judge_add_os_sep(file_dir)
file_list = get_all_ending_files_under_dir(file_dir, ending_list)
print("Total %d files need uncompression\n" % len(file_list))
for i in range(len(file_list)):
print('%d/%d: %s un_compression' %
(i + 1, len(file_list), file_list[i]))
un_compression(file_list[i])
def zip_dir(dir_path: str, target_dir: str, target_name):
assertor.path_exist_assert(dir_path)
judge_mkdir(target_dir)
files = get_files_path(dir_path)
with zipfile.ZipFile(join(target_dir, target_name), 'w',
zipfile.ZIP_DEFLATED) as f:
print('begin to zip file')
for file in files:
f.write(file)
print('done')
def save_dict_as_json(save_file_dir: str, save_file_name: str,
input_dict: dict):
"""
保存dict为json文件
:param save_file_dir: str
:param save_file_name: str
:param input_dict: dict
:return: None
"""
# 断言保证
assertor.path_exist_assert(save_file_dir)
with open(yy.io.join(save_file_dir, save_file_name), 'w') as f:
f.write(json.dumps(input_dict, indent=4))
def judge_del_copy_all_to_target_dir(ori_dir: str, tar_dir: str):
assertor.path_exist_assert(ori_dir)
judge_mkdir(tar_dir)
for file in os.listdir(ori_dir):
file_path = join(ori_dir, file)
if os.path.isfile(file_path):
copy_file(ori_dir, file, tar_dir, file)
if os.path.isdir(file_path):
tar_path = join(tar_dir, file)
judge_del_copy_dir(file_path, tar_path)
def compress_dir_by_shutil(ori_dir: str,
tar_dir: str,
tar_name: str,
display=True):
assertor.path_exist_assert(ori_dir)
judge_mkdir(tar_dir)
tar_path = join(tar_dir, tar_name)
tar_path = os.path.splitext(tar_path)[0]
shutil.make_archive(tar_path, 'zip', ori_dir)
if display:
print('Done compression: %s to %s' % (ori_dir, tar_path))
def get_dir_list_under_root_dir(root_dir: str, anti_list=[]):
assertor.path_exist_assert(root_dir)
dir_list = []
for file in os.listdir(root_dir):
file_path = join(root_dir, file)
if os.path.isdir(file_path):
res = True
for anti_word in anti_list:
if anti_word in file_path:
res = False
if res:
dir_list.append(file_path)
return dir_list
def _get_match_num(cls, dir_path: str, rules: list) -> int:
"""
得到匹配的数量
:param dir_path: str,文件夹的路径
:param rules: list,匹配的规则
:return: match_num: int,匹配的数量
"""
# 断言保证
assertor.path_exist_assert(dir_path)
assertor.list_type_assert(rules, str)
match_num = 0
for file in os.listdir(dir_path):
for rule in rules:
res = cls._check_label_exist(rule, file)
match_num += res
return match_num
def get_all_include_name_files_list_under_dir(file_dir: str,
name_list: list,
ignore_capital=True,
display=False):
"""
获得所有名字列表中的路径列表
:param file_dir:
:param name_list:
:param display:
:return:
"""
assertor.path_exist_assert(file_dir)
assertor.list_type_assert(name_list, str)
if display:
print('Begin get all include name files under dir')
file_dir = judge_add_os_sep(file_dir)
path_set = set()
count = 0
# 遍历所有的数据
for root, dirs, files in os.walk(file_dir, topdown=False):
# 如果文件列表不是空
if len(files) != 0:
# 遍历所有文件名
for file_name in files:
# 遍历所有包含名的部分
for include_name in name_list:
# 是否忽略大小写
if ignore_capital:
temp_include_name = include_name.lower()
temp_file_name = file_name.lower()
else:
temp_include_name = include_name
temp_file_name = file_name
# 如果include name在文件名中
if temp_include_name in temp_file_name:
file_path = join(root, file_name)
path_set.add(file_path)
count += 1
if display:
print(count)
print(file_path)
if display:
print('Done get all include name files under dir')
return list(path_set)
def transfer_npy_to_hdf5(npy_file_path: str, save_file_dir: str,
save_file_name: str, key_name: str) -> None:
"""
读取npy文件, 转换为hdf5文件保存
:param npy_file_path: str
:param save_file_dir: str
:param save_file_name: str
:param key_name: str
:return: None
"""
# 断言保证
assertor.file_exist_assert(npy_file_path)
assertor.path_exist_assert(save_file_dir)
# 读取npy数据
npy_data = np.load(npy_file_path)
# 变为dict
rst_dict = {key_name: npy_data}
# 存储为hdf5
save_dict_as_hdf5(save_file_dir, save_file_name, rst_dict)
def check_labels_exists(cls,
dir_path: str,
rules: list,
label_dirname='1') -> None:
"""
检查labels是否存在且是否齐全
:param dir_path: str,文件夹的路径
:param rules: list,匹配的规则
:param label_dirname: str,label文件夹的名字
:return: None
"""
# 断言保证
assertor.path_exist_assert(dir_path)
assertor.list_type_assert(rules, str)
assertor.type_assert(label_dirname, str)
dir_end = dir_path.split(os.sep)[-1]
for (root, dirs, files) in os.walk(dir_path):
if label_dirname in dirs:
path_list = root.split(os.sep)
for i in range(len(path_list)):
if dir_end == path_list[i]:
print('Dir %s' % path_list[i + 1])
break
sub_path = os.path.join(root, label_dirname)
print(os.path.join(root, sub_path).split('/')[-2])
match_num = cls._get_match_num(sub_path, rules)
if match_num != len(rules):
print(
'---------------------------------------------------------------------'
)
print(
'---------------------------------------------------------------------'
)
print('Warning: dir %s Not have all of the mask' %
str(dir_path))
print(
'---------------------------------------------------------------------'
)
print(
'---------------------------------------------------------------------'
)
def del_all_ending_files_under_dir(file_dir: str,
ending_list: str,
display=False):
assertor.path_exist_assert(file_dir)
assertor.list_type_assert(ending_list, str)
file_dir = judge_add_os_sep(file_dir)
if display:
print("Begin to get all ending files")
file_list = get_all_ending_files_under_dir(file_dir, ending_list, display)
if display:
print("Done get all ending files")
if display:
print("Total %d files need del\n" % len(file_list))
for i in range(len(file_list)):
if display:
print('%d/%d: %s deleting' % (i + 1, len(file_list), file_list[i]))
del_file('', file_list[i])
def save_dict_as_hdf5(save_file_dir: str,
save_file_name: str,
input_dict: dict,
useCompression=False) -> None:
"""
保存dict为hdf5文件
:param save_file_dir: str
:param save_file_name: str
:param input_dict: dict
:param useCompression: bool
:return: None
"""
# 断言保证
assertor.path_exist_assert(save_file_dir)
with h5py.File(yy.io.join(save_file_dir, save_file_name), 'w') as f:
for key, value in input_dict.items():
if useCompression:
f.create_dataset(key, data=value, compression='gzip')
else:
f.create_dataset(key, data=value)
def compress_dir_by_7z(cmp_dir: str,
target_path: str,
password='',
print_msg=True):
"""
压缩文件夹到目标位置
:param cmp_dir: 要压缩的文件夹
:param target_path: 要存储的路径
:param password: 密码
:param print_msg: 是否打印细节
:return:
"""
# 断言保证
assertor.path_exist_assert(cmp_dir)
if password != '':
cmd = '7za a -t7z -p%s -r %s %s' % (target_path, password, cmp_dir)
else:
cmd = '7za a -t7z -r %s %s' % (target_path, cmp_dir)
log = run_command(cmd, print_msg=print_msg)
return log
def del_dir(file_dir):
# 断言保证
assertor.path_exist_assert(file_dir)
shutil.rmtree(file_dir)
def tank_dir(self, value):
# 断言保证
assertor.path_exist_assert(value)
self._tank_dir = value
def tank_dir(self):
# 断言保证
assertor.path_exist_assert(self._tank_dir)
return self._tank_dir
