def _add_images_doc(self, path: str, import_path: str, url_prefix: str): """ 为文件夹下的图片添加信息字典文件(id为不含扩展的文件名) @param {str} path - 要处理的文件目录 """ _import_path = os.path.realpath(import_path) # 处理当前文件夹 _file_list = FileTool.get_filelist(path, regex_str=r'^((?!\.json$).)*$', is_fullname=True) for _file in _file_list: _ext = FileTool.get_file_ext(_file) _json_file = _file[0:-len(_ext)] + 'json' if os.path.exists(_json_file): # 字典文件已存在,无需处理 continue # 生成并写入字典 _file_name = FileTool.get_file_name_no_ext(_file) _url = os.path.realpath(_file)[len(_import_path):].replace( '\\', '/').lstrip('/') _url = '%s/%s' % (url_prefix, _url) _image_doc = {'id': _file_name, 'url': _url, 'path': _file} _json_str = json.dumps(_image_doc, ensure_ascii=False) with open(_json_file, 'wb') as _fid: _fid.write(_json_str.encode(encoding='utf-8')) # 处理子文件夹 _sub_dir_list = FileTool.get_dirlist(path) for _sub_dir in _sub_dir_list: self._add_images_doc(_sub_dir, import_path, url_prefix)
def change_info_file(cls, image_file: str, prop_name: str, prop_value: str) -> bool: """ 修改指定图片的info文件 @param {str} image_file - 传入图片文件 @param {str} prop_name - 属性名 @param {str} prop_value - 属性值 @returns {bool} - 处理结果 """ _path = os.path.split(image_file)[0] _file_no_ext = FileTool.get_file_name_no_ext(image_file) _info_file = os.path.join(_path, _file_no_ext + ".info") if not os.path.exists(_info_file): _info_file = os.path.join(_path, 'info.json') if os.path.exists(_info_file): # 有信息文件才处理 _info = dict() with open(_info_file, 'rb') as f: _eval = str(f.read(), encoding='utf-8') _info = eval(_eval) _info[prop_name] = prop_value # 保存JSON文件 _json = str(_info) with open(_info_file, 'wb') as f: f.write(str.encode(_json, encoding='utf-8')) return True else: return False
def get_info_dict(cls, image_file: str, key_dict: dict) -> dict: """ 通过图片文件路径获取信息字典 @param {str} image_file - 图片文件 @param {dict} key_dict - 需要展示信息的模板字典 @returns {dict} """ _info_dict = dict() _path = os.path.split(image_file)[0] _file_no_ext = FileTool.get_file_name_no_ext(image_file) _info_file = os.path.join(_path, _file_no_ext + ".info") if not os.path.exists(_info_file): _info_file = os.path.join(_path, 'info.json') if os.path.exists(_info_file): with open(_info_file, 'rb') as f: _eval = str(f.read(), encoding='utf-8') _info = eval(_eval) _info_dict = copy.deepcopy(key_dict) for _key in _info_dict: if _key in _info.keys(): _info_dict[_key] = _info[_key] return _info_dict
def labelimg_flags_count(cls, input_path: str, mapping: dict): """ 统计指定目录中的labelimg标记对应标签的数量 @param {str} input_path - 要统计的目录 @param {dict} mapping - mapping.json的字典 @returns {iter_list} - 通过yield返回的处理进度信息清单 [总文件数int, 当前已处理文件数int, 是否成功, 统计结果字典(标签名, 数量)] """ try: # 遍历所有文件夹,获取需要处理的文件数量 _file_list = cls._get_labelimg_annotation_file_list(input_path) _total = len(_file_list) _deal_num = 0 _flags_count = dict() # 先返回进度情况 if _total == 0: yield [_deal_num, _total, True, _flags_count] return # 遍历文件进行处理 for _file in _file_list: # 当前进展 yield [_deal_num, _total, True, _flags_count] # 统计当前文件 _tree = ET.parse(_file) _root = _tree.getroot() _image_file = os.path.join( os.path.split(_file)[0], FileTool.get_file_name_no_ext(_file) + '.jpg') _info_dict = ExtendLib.get_info_dict(_image_file, mapping['info_key_dict']) # 逐个标签处理 for _member in _root.findall('object'): _member_class = _member[0].text if _member_class == mapping['set_by_info']['class_name']: # 需要转换为当前类型 if mapping['set_by_info'][ 'info_tag'] in _info_dict.keys(): _member_class = _info_dict[mapping['set_by_info'] ['info_tag']] if _member_class in _flags_count.keys(): _flags_count[_member_class] += 1 else: _flags_count[_member_class] = 1 _deal_num += 1 # 返回结果 yield [_total, _total, True, _flags_count] except: print('labelimg_flags_count error: %s\r\n%s' % (input_path, traceback.format_exc())) yield [-1, -1, False]
def labelimg_rename_filename(cls, path: str, fix_len: int = 10): """ 重名名labelimg对应目录下的文件名(图片文件和标注文件同步修改) @param {str} path - 要修改文件名的路径 @param {int} fix_len=10 - 文件名长度 """ _path = os.path.realpath(path) _files = FileTool.get_filelist(path=_path, is_fullname=False) _index = 1 for _file in _files: _file_ext = FileTool.get_file_ext(_file) if _file_ext == 'xml': # 标签文件不处理 continue _file_no_ext = FileTool.get_file_name_no_ext(_file) # 获取最新的文件名 while True: _new_name = StringTool.fill_fix_string(str(_index), fix_len, '0', left=True) _new_file = _new_name + '.' + _file_ext _index += 1 if os.path.exists(os.path.join(path, _new_file)): # 文件名已存在 _index += 1 continue # 文件名不存在,跳出循环 break # 修改文件名 os.rename(os.path.join(_path, _file), os.path.join(_path, _new_file)) if os.path.exists(os.path.join(_path, _file_no_ext + '.xml')): # 需要修改标签文件 _xml_file = _new_name + '.xml' os.rename(os.path.join(_path, _file_no_ext + '.xml'), os.path.join(_path, _xml_file)) # 修改标签文件内容 _tree = ET.parse(os.path.join(_path, _xml_file)) _root = _tree.getroot() _root.find('filename').text = _new_file _root.find('path').text = os.path.join(_path, _new_file) _tree.write(os.path.join(_path, _xml_file), encoding='utf-8', method="xml", xml_declaration=None)
def labelimg_del_not_rgb_pic(cls, path: str): """ 删除位深不为RGB三通道的图片 (解决image_size must contain 3 elements[4]报错) @param {str} path - 要处理的路径 """ _path = os.path.realpath(path) # 遍历所有子目录 _sub_dirs = FileTool.get_dirlist(path=_path, is_fullpath=True) for _dir in _sub_dirs: # 递归删除子目录的信息 cls.labelimg_del_not_rgb_pic(_dir) # 检查自己目录下的图片 _files = FileTool.get_filelist(path=_path, is_fullname=False) for _file in _files: _file_ext = FileTool.get_file_ext(_file) if _file_ext == 'xml': # 标签文件不处理 continue # 打开图片判断位深 _fp = open(os.path.join(_path, _file), 'rb') _img = Image.open(_fp) if _img.mode != 'RGB': # 需要删除掉 _fp.close() _img_file = os.path.join(_path, _file) _xml_file = os.path.join( _path, FileTool.get_file_name_no_ext(_file) + '.xml') print('delete %s' % _img_file) FileTool.remove_file(_img_file) if os.path.exists(_xml_file): FileTool.remove_file(_xml_file) else: _fp.close()
def _graph_cmd_dealfun(self, message='', cmd='', cmd_para='', prompt_obj=None, **kwargs): """ 生成APP网络关系图 @param {string} message='' - prompt提示信息 @param {string} cmd - 执行的命令key值 @param {string} cmd_para - 传入的命令参数(命令后的字符串,去掉第一个空格) @param {PromptPlus} prompt_obj=None - 传入调用函数的PromptPlus对象,可以通过该对象的一些方法控制输出显示 @param {kwargs} - 传入的主进程的初始化kwargs对象 @returns {CResult} - 命令执行结果,可通过返回错误码10101通知框架退出命令行, 同时也可以通过CResult对象的 print_str属性要求框架进行打印处理 """ _ok_result = CResult(code='00000') try: # 获取参数及处理参数 _execute_path = self._console_global_para['execute_file_path'] _run_para = { 'file': '', 'formatter': 'col-list', 'comment': 'graph', 'outformat': 'pdf', 'direction': 'forward', 'trace_id': '', 'display_no_trace': 'false', 'down_flow_first': 'true', 'trace_relations': None, 'save': '', 'temp': os.path.join(_execute_path, 'temp'), 'view': 'true', 'cleanup': 'false', 'graph_config': os.path.join(_execute_path, 'conf/graph_config.xml'), 'template': 'default' } _in_para = self._cmd_para_to_dict(cmd_para, name_with_sign=False) _run_para.update(_in_para) if '{para}1' not in _run_para.keys(): prompt_obj.prompt_print(_('you must give the $1 para!', 'file')) return CResult(code='20999') _run_para['file'] = _run_para['{para}1'] if _run_para['save'] == '': _run_para['save'] = 'graph.pdf' # 样式模板信息处理 _style_xml = SimpleXml(_run_para['graph_config'], encoding='utf-8') _style_dict = _style_xml.to_dict()['graphviz']['templates'].get( _run_para['template'], { 'graph': {}, 'node': {}, 'edge': {}, 'edge_color': {}, 'trace': {} } ) # 关联关系线颜色 _name_mapping = _style_dict['edge_color'].get('name_mapping', {}) _use_round_color = _style_dict['edge_color'].get('use_round_color', False) _round_color = _style_dict['edge_color'].get( 'round_color', '').replace(' ', '').split(',') # 生成关系字典 _graph_json = self._formatter_col_list(_run_para['file'], prompt_obj=prompt_obj) # 处理生成调用链的情况 _display_no_trace = _run_para['display_no_trace'] == 'true' if _run_para['trace_id'] != '': # 参与追踪的关系名 if _run_para['trace_relations'] is None: # 全部关系 _run_para['trace_relations'] = _graph_json['relation_name'] else: _run_para['trace_relations'] = _run_para['trace_relations'].split(',') # 获取追踪信息字典 _trace_info = self._get_trace_info( _graph_json, _run_para['trace_id'], _run_para['down_flow_first'] == 'true', _run_para['direction'], _run_para['trace_relations'] ) else: _trace_info = { 'trace_id': '', 'up_nodes': list(), 'down_nodes': list(), 'up_edges': list(), 'down_edges': list() } # 调用链的颜色配置 _attr_center_node = _style_dict.get('trace', {}).get('center_node', {}) _attr_up_node = _style_dict.get('trace', {}).get('up_node', {}) _attr_down_node = _style_dict.get('trace', {}).get('down_node', {}) _attr_up_edge = _style_dict.get('trace', {}).get('up_edge', {}) _attr_down_edge = _style_dict.get('trace', {}).get('down_edge', {}) # 处理关系线条的颜色映射 _relation_color = dict() _index = 0 for _relation_name in _graph_json['relation_name']: if _relation_name in _name_mapping: _relation_color[_relation_name] = _name_mapping[_relation_name] elif _use_round_color: _color_index = _index % len(_round_color) _relation_color[_relation_name] = _round_color[_color_index] _index += 1 else: _relation_color[_relation_name] = '' # 开始处理画图 _dot = Digraph( comment=_run_para['comment'], format=_run_para['outformat'], encoding='utf-8', directory=_run_para['temp'], graph_attr=_style_dict['graph'], node_attr=_style_dict['node'], edge_attr=_style_dict['edge'] ) # 开始画节点 _added_rank = list() # 已添加的分组 for _id, _info in _graph_json['list'].items(): if _info['rank'] != '': # 有分组处理 if _info['rank'] not in _added_rank: _added_rank.append(_info['rank']) with _dot.subgraph() as _sub_dot: _sub_dot.attr(rank='same') for _sub_id in _graph_json['rank'][_info['rank']]: # 节点颜色设置 if _sub_id == _trace_info['trace_id']: # 中心节点 _attr = _attr_center_node elif _sub_id in _trace_info['up_nodes']: _attr = _attr_up_node elif _sub_id in _trace_info['down_nodes']: _attr = _attr_down_node elif _display_no_trace and _trace_info['trace_id'] != '': # 不在链中且不应显示 continue else: _attr = {} _sub_dot.node( _sub_id, label=_graph_json['list'][_sub_id]['name'], **_attr ) else: # 节点颜色设置 if _id == _trace_info['trace_id']: # 中心节点 _attr = _attr_center_node elif _id in _trace_info['up_nodes']: _attr = _attr_up_node elif _id in _trace_info['down_nodes']: _attr = _attr_down_node elif _display_no_trace and _trace_info['trace_id'] != '': # 不在链中且不应显示 continue else: _attr = {} _dot.node(_id, label=_info['name'], **_attr) # 开始画关联线 for _id, _info in _graph_json['list'].items(): for _key, _list in _info.items(): if _key in ('name', 'rank'): continue # 判断颜色 _edge_color = _relation_color[_key] for _temp_id in _list: if _temp_id == '': continue if _run_para['direction'] == 'reverse': _head_id = _temp_id _tail_id = _id else: _head_id = _id _tail_id = _temp_id # 处理颜色 _findstr = '%s&&%s' % (_head_id, _tail_id) if _findstr in _trace_info['up_edges']: _attr = _attr_up_edge elif _findstr in _trace_info['down_edges']: _attr = _attr_down_edge elif _display_no_trace and _trace_info['trace_id'] != '': continue else: _attr = {'color': _edge_color} _dot.edge(_head_id, _tail_id, **_attr) # 保存图片 _dir, _filename = os.path.split(os.path.abspath(_run_para['save'])) _filename = FileTool.get_file_name_no_ext(_filename) _dot.render( filename=_filename, directory=_dir, format=_run_para['outformat'], view=(_run_para['view'] == 'true'), cleanup=(_run_para['cleanup'] == 'true') ) except Exception as e: _prin_str = '%s (%s):\n%s' % ( _('execution exception'), str(e), traceback.format_exc() ) prompt_obj.prompt_print(_prin_str) return CResult(code='20999') # 结束 return _ok_result
def labelimg_pic_deal(cls, path: str): """ TFRecord图片兼容处理 1.删除位深不为RGB三通道的图片 (解决image_size must contain 3 elements[4]报错) 2.转换图片格式为jpg 3.检查xml文件的文件名和路径是否正确 @param {str} path - 要处理的路径 """ _path = os.path.realpath(path) # 遍历所有子目录 _sub_dirs = FileTool.get_dirlist(path=_path, is_fullpath=True) for _dir in _sub_dirs: # 递归删除子目录的信息 cls.labelimg_pic_deal(_dir) # 检查自己目录下的图片 _files = FileTool.get_filelist(path=_path, is_fullname=False) for _file in _files: _file_ext = FileTool.get_file_ext(_file) if _file_ext == 'xml': # 标签文件不处理 continue _img_file = os.path.join(_path, _file) _file_no_ext = FileTool.get_file_name_no_ext(_file) if _file_ext in ('png', 'gif'): # 转换图片格式 _fp = open(_img_file, 'rb') _img = Image.open(_fp) _rgb_im = _img.convert('RGB') _rgb_im.save(os.path.join(_path, _file_no_ext + '.jpg')) _fp.close() # 删除原文件,修改xml中的文件名 FileTool.remove_file(_img_file) _xml_file = os.path.join(_path, _file_no_ext + '.xml') if os.path.exists(_xml_file): _tree = ET.parse(os.path.join(_path, _xml_file)) _root = _tree.getroot() _root.find('filename').text = _file_no_ext + '.jpg' _root.find('path').text = os.path.join( _path, _file_no_ext + '.jpg') _tree.write(os.path.join(_path, _xml_file), encoding='utf-8', method="xml", xml_declaration=None) # 修改文件名变量 _img_file = os.path.join(_path, _file_no_ext + '.jpg') # 打开图片判断位深 _fp = open(_img_file, 'rb') _img = Image.open(_fp) if _img.mode != 'RGB': # 需要删除掉 _fp.close() _xml_file = os.path.join( _path, FileTool.get_file_name_no_ext(_file) + '.xml') print('delete %s' % _img_file) FileTool.remove_file(_img_file) if os.path.exists(_xml_file): FileTool.remove_file(_xml_file) else: _fp.close() # 检查xml文件 _xml_file = os.path.join(_path, _file_no_ext + '.xml') if os.path.exists(_xml_file): _tree = ET.parse(os.path.join(_path, _xml_file)) _root = _tree.getroot() if _root.find('filename' ).text != _file_no_ext + '.jpg' or os.path.split( _root.find('path').text)[0] != _path: _root.find('filename').text = _file_no_ext + '.jpg' _root.find('path').text = os.path.join( _path, _file_no_ext + '.jpg') _tree.write(os.path.join(_path, _xml_file), encoding='utf-8', method="xml", xml_declaration=None)
def labelimg_copy_flags_pics(cls, input_path: str, output_path: str, use_mapping: bool = False, mapping: dict = None): """ 按类别复制图片和标注文件到指定目录 @param {str} input_path - 图片路径 @param {str} output_path - 输出路径 @param {bool} use_mapping=False - 是否使用mapping处理映射 @param {dict} mapping=None - mapping.josn字典 @returns {iter_list} - 通过yield返回的处理进度信息清单 [总文件数int, 当前已处理文件数int, 是否成功] """ try: # 遍历所有文件夹,获取需要处理的文件数量 _file_list = cls._get_labelimg_annotation_file_list(input_path) _total = len(_file_list) _deal_num = 0 # 先返回进度情况 if _total == 0: yield [_deal_num, _total, True] return # 创建复制文件夹 FileTool.create_dir(output_path, exist_ok=True) # 遍历处理 for _file in _file_list: # 当前进展 yield [_deal_num, _total, True] # 逐个标注文件进行处理 _tree = ET.parse(_file) _root = _tree.getroot() _annotations = dict() _annotations['filename'] = _root.find('filename').text _annotations['file_path'] = os.path.join( os.path.split(_file)[0], _annotations['filename']) # 获取信息字典 _info_dict = ExtendLib.get_info_dict(_annotations['file_path'], mapping['info_key_dict']) # 逐个标签处理 _save_class_path = '' # 要保存到的分类路径 _is_copy = False # 标注是否已复制文件 _new_xml_name = '' # 新的xml名 for _member in _root.findall('object'): _member_class = _member[0].text if use_mapping: # 使用映射处理 if _member_class == mapping['set_by_info'][ 'class_name']: # 需要获取真实的信息 if mapping['set_by_info'][ 'info_tag'] in _info_dict.keys(): _member_class = _info_dict[ mapping['set_by_info']['info_tag']] # 变更分类名 _member[0].text = _member_class # 过滤不需要的类别 if _member_class not in mapping['class_int'].keys(): _deal_num += 1 continue # 保存分类路径 _save_class_path = os.path.join( output_path, mapping['class'][_member_class]) else: # 普通分类 _save_class_path = os.path.join( output_path, _member_class) # 复制文件 if not _is_copy: # 处理重复文件名 _file_name = FileTool.get_file_name_no_ext( _annotations['filename']) _file_ext = FileTool.get_file_ext( _annotations['filename']) _rename_num = 1 _new_file_name = '%s.%s' % (_file_name, _file_ext) _new_xml_name = '%s.xml' % (_file_name, ) while os.path.exists( os.path.join(_save_class_path, _new_file_name)): _new_file_name = '%s_%d.%s' % ( _file_name, _rename_num, _file_ext) _new_xml_name = '%s_%d.xml' % (_file_name, _rename_num) _rename_num += 1 # 创建文件夹 FileTool.create_dir(_save_class_path, exist_ok=True) shutil.copyfile( _annotations['file_path'], os.path.join(_save_class_path, _new_file_name)) # 修改xml里面的文件名和文件路径 _root.find('filename').text = _new_file_name _root.find('path').text = os.path.join( _save_class_path, _new_file_name) _is_copy = True if _is_copy: # 有修改xml内容 _tree.write(os.path.join(_save_class_path, _new_xml_name), encoding='utf-8', method="xml", xml_declaration=None) # 继续循环处理 _deal_num += 1 # 返回结果 yield [_total, _total, True] except: print('labelimg_copy_flags_pics error: %s\r\n%s' % (input_path, traceback.format_exc())) yield [-1, -1, False]
def labelimg_copy_flags_pics(cls, input_path: str, output_path: str, is_cc: bool = False): """ 按类别复制图片和标注文件到指定目录 @param {str} input_path - 图片路径 @param {str} output_path - 输出路径 @param {bool} is_cc=False - 是否CC项目 @returns {iter_list} - 通过yield返回的处理进度信息清单 [总文件数int, 当前已处理文件数int, 是否成功] """ try: # 遍历所有文件夹,获取需要处理的文件数量 _file_list = cls._get_labelimg_annotation_file_list(input_path) _total = len(_file_list) _deal_num = 0 # 先返回进度情况 if _total == 0: yield [_deal_num, _total, True] return # 创建复制文件夹 FileTool.create_dir(output_path, exist_ok=True) # 遍历处理 for _file in _file_list: # 当前进展 yield [_deal_num, _total, True] # 逐个标注文件进行处理 _tree = ET.parse(_file) _root = _tree.getroot() _annotations = dict() _annotations['filename'] = _root.find('filename').text _annotations['file_path'] = os.path.join( os.path.split(_file)[0], _annotations['filename']) # 逐个标签处理 _save_class_path = '' # 要保存到的分类路径 _is_copy = False # 标注是否已复制文件 _is_change_class = False # 标注是否有修改分类名 _new_xml_name = '' # 新的xml名 for _member in _root.findall('object'): _member_class = _member[0].text if is_cc: # CC专属的类型转换 if _member_class == '翡翠': # 需要获取真实的信息 _info_file = os.path.join( os.path.split(_file)[0], 'info.json') _info = dict() with open(_info_file, 'rb') as f: _eval = str(f.read(), encoding='utf-8') _info = eval(_eval) if _info['款式'] == '挂件': # 挂件,需要二级分类 _member_class = _info['挂件类型'] else: # 一级分类 _member_class = _info['款式'] # 变更分类名 _member[0].text = _member_class _is_change_class = True # 过滤不需要的类别 if _member_class not in USE_CLASS_TEXT_LIST: _deal_num += 1 continue # 保存分类路径 _save_class_path = os.path.join( output_path, cls._cc_get_class_text(_member_class)) else: # 普通分类 _save_class_path = os.path.join( output_path, _member_class) # 复制文件 if not _is_copy: # 处理重复文件名 _file_name = FileTool.get_file_name_no_ext( _annotations['filename']) _file_ext = FileTool.get_file_ext( _annotations['filename']) _rename_num = 1 _new_file_name = '%s.%s' % (_file_name, _file_ext) _new_xml_name = '%s.xml' % (_file_name, ) while os.path.exists( os.path.join(_save_class_path, _new_file_name)): _new_file_name = '%s_%d.%s' % ( _file_name, _rename_num, _file_ext) _new_xml_name = '%s_%d.xml' % (_file_name, _rename_num) _rename_num += 1 # 创建文件夹 FileTool.create_dir(_save_class_path, exist_ok=True) shutil.copyfile( _annotations['file_path'], os.path.join(_save_class_path, _new_file_name)) _is_copy = True if _is_copy: # 有复制文件 if _is_change_class: # 有修改xml内容 _tree.write(os.path.join(_save_class_path, _new_xml_name), encoding='utf-8', method="xml", xml_declaration=None) else: shutil.copyfile( _file, os.path.join(_save_class_path, _new_xml_name)) # 继续循环处理 _deal_num += 1 # 返回结果 yield [_total, _total, True] except: print('labelimg_copy_flags_pics error: %s\r\n%s' % (input_path, traceback.format_exc())) yield [-1, -1, False]