def del_empty_ending_files(root_dir: str, ending: str, display=False): assertor.path_exist_assert(root_dir) if display: print('Begin to get ending file path list') ending_file_path_list = yy.io.get_all_ending_files_under_dir( root_dir, [ending], display=display) if display: print('Done get ending file path list') empty_count = 0 total_count = 0 for file_path in ending_file_path_list: if display: print('processing %d/%d ...' % (total_count, len(ending_file_path_list))) total_count += 1 size = os.path.getsize(file_path) if size == 0: yy.io.del_file('', file_path) if display: print('Done del empty file %s' % (file_path)) empty_count += 1 if display: print('Total delete %d empty files' % (empty_count))
def get_target_name_dir_path_list_under_dir(file_dir: str, name_list: list, display=False): """ 获得所有名字列表中的路径列表 :param file_dir: :param name_list: :param display: :return: """ assertor.path_exist_assert(file_dir) assertor.list_type_assert(name_list, str) if display: print('Begin get all target name dir under dir') file_dir = judge_add_os_sep(file_dir) path_set = set() count = 0 for root, dirs, files in os.walk(file_dir, topdown=False): if len(dirs) != 0: for dir_name in dirs: if dir_name in name_list: dir_path = join(root, dir_name) path_set.add(dir_path) count += 1 if display: print(count) print(dir_path) if display: print('Done get all target name dir under dir') return list(path_set)
def get_all_ending_files_under_dir(file_dir: str, ending_list: list, display=False): assertor.path_exist_assert(file_dir) assertor.list_type_assert(ending_list, str) if display: print('Begin get all ending files under dir') file_dir = judge_add_os_sep(file_dir) path_set = set() count = 0 for root, dirs, files in os.walk(file_dir, topdown=False): if len(files) != 0: for file in files: ending = os.path.splitext(file)[-1] if ending in ending_list: file_path = join(root, file) path_set.add(file_path) count += 1 if display: print(count) print(file_path) if display: print('Done get all ending files under dir') return list(path_set)
def copy_file(old_dir, old_name, new_dir, new_name): old_path = join(old_dir, old_name) # 断言保证 assertor.file_exist_assert(old_path) assertor.path_exist_assert(new_dir) new_path = join(new_dir, new_name) shutil.copyfile(old_path, new_path)
def file_dir(self) -> str: """ 判别file_dir是否存在,并返回file_dir :return: str """ # 断言保证 assertor.path_exist_assert(self._file_dir) return self._file_dir
def move_dir_or_file(ori_dir, ori_name, tar_dir, tar_name): ori_path = join(ori_dir, ori_name) assertor.path_exist_assert(ori_path) judge_mkdir(tar_dir) tar_path = join(tar_dir, tar_name) shutil.move(ori_path, tar_path)
def get_files_path(root_dir): assertor.path_exist_assert(root_dir) path_set = set() for root, dirs, files in os.walk(root_dir, topdown=False): if len(files) != 0: for file in files: path_set.add(join(root, file)) return list(path_set)
def file_dir(self, value: str): """ 判别value的path是否存在,若存在则设置file_dir为value :param value: str :return: """ # 断言保证 assertor.path_exist_assert(value) self._file_dir = value
def get_all_file_ending_under_dir(file_dir: str): assertor.path_exist_assert(file_dir) file_ending = set() for root, dirs, files in os.walk(file_dir, topdown=False): if len(files) != 0: for file in files: ending = os.path.splitext(file)[-1] file_ending.add(ending) return file_ending
def make_same_empty_sub_dirs(root_dir, save_dir, anti_list=['lost+found', '.Trash']): assertor.path_exist_assert(root_dir) yy.io.judge_mkdir(save_dir) sub_dir_list = yy.io.get_dir_list_under_root_dir(root_dir, anti_list) for sub_dir in sub_dir_list: sub_dir_name = sub_dir.split(os.sep)[-1] save_path = yy.io.join(save_dir, sub_dir_name) yy.io.judge_mkdir(save_path)
def un_compression_all_ending_files_under_dir(file_dir: str, ending_list: list): assertor.path_exist_assert(file_dir) assertor.list_type_assert(ending_list, str) file_dir = judge_add_os_sep(file_dir) file_list = get_all_ending_files_under_dir(file_dir, ending_list) print("Total %d files need uncompression\n" % len(file_list)) for i in range(len(file_list)): print('%d/%d: %s un_compression' % (i + 1, len(file_list), file_list[i])) un_compression(file_list[i])
def zip_dir(dir_path: str, target_dir: str, target_name): assertor.path_exist_assert(dir_path) judge_mkdir(target_dir) files = get_files_path(dir_path) with zipfile.ZipFile(join(target_dir, target_name), 'w', zipfile.ZIP_DEFLATED) as f: print('begin to zip file') for file in files: f.write(file) print('done')
def save_dict_as_json(save_file_dir: str, save_file_name: str, input_dict: dict): """ 保存dict为json文件 :param save_file_dir: str :param save_file_name: str :param input_dict: dict :return: None """ # 断言保证 assertor.path_exist_assert(save_file_dir) with open(yy.io.join(save_file_dir, save_file_name), 'w') as f: f.write(json.dumps(input_dict, indent=4))
def judge_del_copy_all_to_target_dir(ori_dir: str, tar_dir: str): assertor.path_exist_assert(ori_dir) judge_mkdir(tar_dir) for file in os.listdir(ori_dir): file_path = join(ori_dir, file) if os.path.isfile(file_path): copy_file(ori_dir, file, tar_dir, file) if os.path.isdir(file_path): tar_path = join(tar_dir, file) judge_del_copy_dir(file_path, tar_path)
def compress_dir_by_shutil(ori_dir: str, tar_dir: str, tar_name: str, display=True): assertor.path_exist_assert(ori_dir) judge_mkdir(tar_dir) tar_path = join(tar_dir, tar_name) tar_path = os.path.splitext(tar_path)[0] shutil.make_archive(tar_path, 'zip', ori_dir) if display: print('Done compression: %s to %s' % (ori_dir, tar_path))
def get_dir_list_under_root_dir(root_dir: str, anti_list=[]): assertor.path_exist_assert(root_dir) dir_list = [] for file in os.listdir(root_dir): file_path = join(root_dir, file) if os.path.isdir(file_path): res = True for anti_word in anti_list: if anti_word in file_path: res = False if res: dir_list.append(file_path) return dir_list
def _get_match_num(cls, dir_path: str, rules: list) -> int: """ 得到匹配的数量 :param dir_path: str,文件夹的路径 :param rules: list,匹配的规则 :return: match_num: int,匹配的数量 """ # 断言保证 assertor.path_exist_assert(dir_path) assertor.list_type_assert(rules, str) match_num = 0 for file in os.listdir(dir_path): for rule in rules: res = cls._check_label_exist(rule, file) match_num += res return match_num
def get_all_include_name_files_list_under_dir(file_dir: str, name_list: list, ignore_capital=True, display=False): """ 获得所有名字列表中的路径列表 :param file_dir: :param name_list: :param display: :return: """ assertor.path_exist_assert(file_dir) assertor.list_type_assert(name_list, str) if display: print('Begin get all include name files under dir') file_dir = judge_add_os_sep(file_dir) path_set = set() count = 0 # 遍历所有的数据 for root, dirs, files in os.walk(file_dir, topdown=False): # 如果文件列表不是空 if len(files) != 0: # 遍历所有文件名 for file_name in files: # 遍历所有包含名的部分 for include_name in name_list: # 是否忽略大小写 if ignore_capital: temp_include_name = include_name.lower() temp_file_name = file_name.lower() else: temp_include_name = include_name temp_file_name = file_name # 如果include name在文件名中 if temp_include_name in temp_file_name: file_path = join(root, file_name) path_set.add(file_path) count += 1 if display: print(count) print(file_path) if display: print('Done get all include name files under dir') return list(path_set)
def transfer_npy_to_hdf5(npy_file_path: str, save_file_dir: str, save_file_name: str, key_name: str) -> None: """ 读取npy文件, 转换为hdf5文件保存 :param npy_file_path: str :param save_file_dir: str :param save_file_name: str :param key_name: str :return: None """ # 断言保证 assertor.file_exist_assert(npy_file_path) assertor.path_exist_assert(save_file_dir) # 读取npy数据 npy_data = np.load(npy_file_path) # 变为dict rst_dict = {key_name: npy_data} # 存储为hdf5 save_dict_as_hdf5(save_file_dir, save_file_name, rst_dict)
def check_labels_exists(cls, dir_path: str, rules: list, label_dirname='1') -> None: """ 检查labels是否存在且是否齐全 :param dir_path: str,文件夹的路径 :param rules: list,匹配的规则 :param label_dirname: str,label文件夹的名字 :return: None """ # 断言保证 assertor.path_exist_assert(dir_path) assertor.list_type_assert(rules, str) assertor.type_assert(label_dirname, str) dir_end = dir_path.split(os.sep)[-1] for (root, dirs, files) in os.walk(dir_path): if label_dirname in dirs: path_list = root.split(os.sep) for i in range(len(path_list)): if dir_end == path_list[i]: print('Dir %s' % path_list[i + 1]) break sub_path = os.path.join(root, label_dirname) print(os.path.join(root, sub_path).split('/')[-2]) match_num = cls._get_match_num(sub_path, rules) if match_num != len(rules): print( '---------------------------------------------------------------------' ) print( '---------------------------------------------------------------------' ) print('Warning: dir %s Not have all of the mask' % str(dir_path)) print( '---------------------------------------------------------------------' ) print( '---------------------------------------------------------------------' )
def del_all_ending_files_under_dir(file_dir: str, ending_list: str, display=False): assertor.path_exist_assert(file_dir) assertor.list_type_assert(ending_list, str) file_dir = judge_add_os_sep(file_dir) if display: print("Begin to get all ending files") file_list = get_all_ending_files_under_dir(file_dir, ending_list, display) if display: print("Done get all ending files") if display: print("Total %d files need del\n" % len(file_list)) for i in range(len(file_list)): if display: print('%d/%d: %s deleting' % (i + 1, len(file_list), file_list[i])) del_file('', file_list[i])
def save_dict_as_hdf5(save_file_dir: str, save_file_name: str, input_dict: dict, useCompression=False) -> None: """ 保存dict为hdf5文件 :param save_file_dir: str :param save_file_name: str :param input_dict: dict :param useCompression: bool :return: None """ # 断言保证 assertor.path_exist_assert(save_file_dir) with h5py.File(yy.io.join(save_file_dir, save_file_name), 'w') as f: for key, value in input_dict.items(): if useCompression: f.create_dataset(key, data=value, compression='gzip') else: f.create_dataset(key, data=value)
def compress_dir_by_7z(cmp_dir: str, target_path: str, password='', print_msg=True): """ 压缩文件夹到目标位置 :param cmp_dir: 要压缩的文件夹 :param target_path: 要存储的路径 :param password: 密码 :param print_msg: 是否打印细节 :return: """ # 断言保证 assertor.path_exist_assert(cmp_dir) if password != '': cmd = '7za a -t7z -p%s -r %s %s' % (target_path, password, cmp_dir) else: cmd = '7za a -t7z -r %s %s' % (target_path, cmp_dir) log = run_command(cmd, print_msg=print_msg) return log
def del_dir(file_dir): # 断言保证 assertor.path_exist_assert(file_dir) shutil.rmtree(file_dir)
def tank_dir(self, value): # 断言保证 assertor.path_exist_assert(value) self._tank_dir = value
def tank_dir(self): # 断言保证 assertor.path_exist_assert(self._tank_dir) return self._tank_dir