def _add_images_doc(self, path: str, import_path: str, url_prefix: str):
"""
为文件夹下的图片添加信息字典文件(id为不含扩展的文件名)
@param {str} path - 要处理的文件目录
"""
_import_path = os.path.realpath(import_path)
# 处理当前文件夹
_file_list = FileTool.get_filelist(path,
regex_str=r'^((?!\.json$).)*$',
is_fullname=True)
for _file in _file_list:
_ext = FileTool.get_file_ext(_file)
_json_file = _file[0:-len(_ext)] + 'json'
if os.path.exists(_json_file):
# 字典文件已存在，无需处理
continue
# 生成并写入字典
_file_name = FileTool.get_file_name_no_ext(_file)
_url = os.path.realpath(_file)[len(_import_path):].replace(
'\\', '/').lstrip('/')
_url = '%s/%s' % (url_prefix, _url)
_image_doc = {'id': _file_name, 'url': _url, 'path': _file}
_json_str = json.dumps(_image_doc, ensure_ascii=False)
with open(_json_file, 'wb') as _fid:
_fid.write(_json_str.encode(encoding='utf-8'))
# 处理子文件夹
_sub_dir_list = FileTool.get_dirlist(path)
for _sub_dir in _sub_dir_list:
self._add_images_doc(_sub_dir, import_path, url_prefix)
def change_info_file(cls, image_file: str, prop_name: str,
prop_value: str) -> bool:
"""
修改指定图片的info文件
@param {str} image_file - 传入图片文件
@param {str} prop_name - 属性名
@param {str} prop_value - 属性值
@returns {bool} - 处理结果
"""
_path = os.path.split(image_file)[0]
_file_no_ext = FileTool.get_file_name_no_ext(image_file)
_info_file = os.path.join(_path, _file_no_ext + ".info")
if not os.path.exists(_info_file):
_info_file = os.path.join(_path, 'info.json')
if os.path.exists(_info_file):
# 有信息文件才处理
_info = dict()
with open(_info_file, 'rb') as f:
_eval = str(f.read(), encoding='utf-8')
_info = eval(_eval)
_info[prop_name] = prop_value
# 保存JSON文件
_json = str(_info)
with open(_info_file, 'wb') as f:
f.write(str.encode(_json, encoding='utf-8'))
return True
else:
return False
def get_info_dict(cls, image_file: str, key_dict: dict) -> dict:
"""
通过图片文件路径获取信息字典
@param {str} image_file - 图片文件
@param {dict} key_dict - 需要展示信息的模板字典
@returns {dict}
"""
_info_dict = dict()
_path = os.path.split(image_file)[0]
_file_no_ext = FileTool.get_file_name_no_ext(image_file)
_info_file = os.path.join(_path, _file_no_ext + ".info")
if not os.path.exists(_info_file):
_info_file = os.path.join(_path, 'info.json')
if os.path.exists(_info_file):
with open(_info_file, 'rb') as f:
_eval = str(f.read(), encoding='utf-8')
_info = eval(_eval)
_info_dict = copy.deepcopy(key_dict)
for _key in _info_dict:
if _key in _info.keys():
_info_dict[_key] = _info[_key]
return _info_dict
def labelimg_flags_count(cls, input_path: str, mapping: dict):
"""
统计指定目录中的labelimg标记对应标签的数量
@param {str} input_path - 要统计的目录
@param {dict} mapping - mapping.json的字典
@returns {iter_list} - 通过yield返回的处理进度信息清单
[总文件数int, 当前已处理文件数int, 是否成功, 统计结果字典(标签名, 数量)]
"""
try:
# 遍历所有文件夹，获取需要处理的文件数量
_file_list = cls._get_labelimg_annotation_file_list(input_path)
_total = len(_file_list)
_deal_num = 0
_flags_count = dict()
# 先返回进度情况
if _total == 0:
yield [_deal_num, _total, True, _flags_count]
return
# 遍历文件进行处理
for _file in _file_list:
# 当前进展
yield [_deal_num, _total, True, _flags_count]
# 统计当前文件
_tree = ET.parse(_file)
_root = _tree.getroot()
_image_file = os.path.join(
os.path.split(_file)[0],
FileTool.get_file_name_no_ext(_file) + '.jpg')
_info_dict = ExtendLib.get_info_dict(_image_file,
mapping['info_key_dict'])
# 逐个标签处理
for _member in _root.findall('object'):
_member_class = _member[0].text
if _member_class == mapping['set_by_info']['class_name']:
# 需要转换为当前类型
if mapping['set_by_info'][
'info_tag'] in _info_dict.keys():
_member_class = _info_dict[mapping['set_by_info']
['info_tag']]
if _member_class in _flags_count.keys():
_flags_count[_member_class] += 1
else:
_flags_count[_member_class] = 1
_deal_num += 1
# 返回结果
yield [_total, _total, True, _flags_count]
except:
print('labelimg_flags_count error: %s\r\n%s' %
(input_path, traceback.format_exc()))
yield [-1, -1, False]
def labelimg_rename_filename(cls, path: str, fix_len: int = 10):
"""
重名名labelimg对应目录下的文件名（图片文件和标注文件同步修改）
@param {str} path - 要修改文件名的路径
@param {int} fix_len=10 - 文件名长度
"""
_path = os.path.realpath(path)
_files = FileTool.get_filelist(path=_path, is_fullname=False)
_index = 1
for _file in _files:
_file_ext = FileTool.get_file_ext(_file)
if _file_ext == 'xml':
# 标签文件不处理
continue
_file_no_ext = FileTool.get_file_name_no_ext(_file)
# 获取最新的文件名
while True:
_new_name = StringTool.fill_fix_string(str(_index),
fix_len,
'0',
left=True)
_new_file = _new_name + '.' + _file_ext
_index += 1
if os.path.exists(os.path.join(path, _new_file)):
# 文件名已存在
_index += 1
continue
# 文件名不存在，跳出循环
break
# 修改文件名
os.rename(os.path.join(_path, _file),
os.path.join(_path, _new_file))
if os.path.exists(os.path.join(_path, _file_no_ext + '.xml')):
# 需要修改标签文件
_xml_file = _new_name + '.xml'
os.rename(os.path.join(_path, _file_no_ext + '.xml'),
os.path.join(_path, _xml_file))
# 修改标签文件内容
_tree = ET.parse(os.path.join(_path, _xml_file))
_root = _tree.getroot()
_root.find('filename').text = _new_file
_root.find('path').text = os.path.join(_path, _new_file)
_tree.write(os.path.join(_path, _xml_file),
encoding='utf-8',
method="xml",
xml_declaration=None)
def labelimg_del_not_rgb_pic(cls, path: str):
"""
删除位深不为RGB三通道的图片
（解决image_size must contain 3 elements[4]报错）
@param {str} path - 要处理的路径
"""
_path = os.path.realpath(path)
# 遍历所有子目录
_sub_dirs = FileTool.get_dirlist(path=_path, is_fullpath=True)
for _dir in _sub_dirs:
# 递归删除子目录的信息
cls.labelimg_del_not_rgb_pic(_dir)
# 检查自己目录下的图片
_files = FileTool.get_filelist(path=_path, is_fullname=False)
for _file in _files:
_file_ext = FileTool.get_file_ext(_file)
if _file_ext == 'xml':
# 标签文件不处理
continue
# 打开图片判断位深
_fp = open(os.path.join(_path, _file), 'rb')
_img = Image.open(_fp)
if _img.mode != 'RGB':
# 需要删除掉
_fp.close()
_img_file = os.path.join(_path, _file)
_xml_file = os.path.join(
_path,
FileTool.get_file_name_no_ext(_file) + '.xml')
print('delete %s' % _img_file)
FileTool.remove_file(_img_file)
if os.path.exists(_xml_file):
FileTool.remove_file(_xml_file)
else:
_fp.close()
def _graph_cmd_dealfun(self, message='', cmd='', cmd_para='', prompt_obj=None, **kwargs):
"""
生成APP网络关系图
@param {string} message='' - prompt提示信息
@param {string} cmd - 执行的命令key值
@param {string} cmd_para - 传入的命令参数（命令后的字符串，去掉第一个空格）
@param {PromptPlus} prompt_obj=None - 传入调用函数的PromptPlus对象，可以通过该对象的一些方法控制输出显示
@param {kwargs} - 传入的主进程的初始化kwargs对象
@returns {CResult} - 命令执行结果，可通过返回错误码10101通知框架退出命令行, 同时也可以通过CResult对象的
print_str属性要求框架进行打印处理
"""
_ok_result = CResult(code='00000')
try:
# 获取参数及处理参数
_execute_path = self._console_global_para['execute_file_path']
_run_para = {
'file': '',
'formatter': 'col-list',
'comment': 'graph',
'outformat': 'pdf',
'direction': 'forward',
'trace_id': '',
'display_no_trace': 'false',
'down_flow_first': 'true',
'trace_relations': None,
'save': '',
'temp': os.path.join(_execute_path, 'temp'),
'view': 'true',
'cleanup': 'false',
'graph_config': os.path.join(_execute_path, 'conf/graph_config.xml'),
'template': 'default'
}
_in_para = self._cmd_para_to_dict(cmd_para, name_with_sign=False)
_run_para.update(_in_para)
if '{para}1' not in _run_para.keys():
prompt_obj.prompt_print(_('you must give the $1 para!', 'file'))
return CResult(code='20999')
_run_para['file'] = _run_para['{para}1']
if _run_para['save'] == '':
_run_para['save'] = 'graph.pdf'
# 样式模板信息处理
_style_xml = SimpleXml(_run_para['graph_config'], encoding='utf-8')
_style_dict = _style_xml.to_dict()['graphviz']['templates'].get(
_run_para['template'], {
'graph': {},
'node': {},
'edge': {},
'edge_color': {},
'trace': {}
}
)
# 关联关系线颜色
_name_mapping = _style_dict['edge_color'].get('name_mapping', {})
_use_round_color = _style_dict['edge_color'].get('use_round_color', False)
_round_color = _style_dict['edge_color'].get(
'round_color', '').replace(' ', '').split(',')
# 生成关系字典
_graph_json = self._formatter_col_list(_run_para['file'], prompt_obj=prompt_obj)
# 处理生成调用链的情况
_display_no_trace = _run_para['display_no_trace'] == 'true'
if _run_para['trace_id'] != '':
# 参与追踪的关系名
if _run_para['trace_relations'] is None:
# 全部关系
_run_para['trace_relations'] = _graph_json['relation_name']
else:
_run_para['trace_relations'] = _run_para['trace_relations'].split(',')
# 获取追踪信息字典
_trace_info = self._get_trace_info(
_graph_json, _run_para['trace_id'], _run_para['down_flow_first'] == 'true',
_run_para['direction'], _run_para['trace_relations']
)
else:
_trace_info = {
'trace_id': '',
'up_nodes': list(),
'down_nodes': list(),
'up_edges': list(),
'down_edges': list()
}
# 调用链的颜色配置
_attr_center_node = _style_dict.get('trace', {}).get('center_node', {})
_attr_up_node = _style_dict.get('trace', {}).get('up_node', {})
_attr_down_node = _style_dict.get('trace', {}).get('down_node', {})
_attr_up_edge = _style_dict.get('trace', {}).get('up_edge', {})
_attr_down_edge = _style_dict.get('trace', {}).get('down_edge', {})
# 处理关系线条的颜色映射
_relation_color = dict()
_index = 0
for _relation_name in _graph_json['relation_name']:
if _relation_name in _name_mapping:
_relation_color[_relation_name] = _name_mapping[_relation_name]
elif _use_round_color:
_color_index = _index % len(_round_color)
_relation_color[_relation_name] = _round_color[_color_index]
_index += 1
else:
_relation_color[_relation_name] = ''
# 开始处理画图
_dot = Digraph(
comment=_run_para['comment'], format=_run_para['outformat'], encoding='utf-8',
directory=_run_para['temp'],
graph_attr=_style_dict['graph'],
node_attr=_style_dict['node'],
edge_attr=_style_dict['edge']
)
# 开始画节点
_added_rank = list() # 已添加的分组
for _id, _info in _graph_json['list'].items():
if _info['rank'] != '':
# 有分组处理
if _info['rank'] not in _added_rank:
_added_rank.append(_info['rank'])
with _dot.subgraph() as _sub_dot:
_sub_dot.attr(rank='same')
for _sub_id in _graph_json['rank'][_info['rank']]:
# 节点颜色设置
if _sub_id == _trace_info['trace_id']:
# 中心节点
_attr = _attr_center_node
elif _sub_id in _trace_info['up_nodes']:
_attr = _attr_up_node
elif _sub_id in _trace_info['down_nodes']:
_attr = _attr_down_node
elif _display_no_trace and _trace_info['trace_id'] != '':
# 不在链中且不应显示
continue
else:
_attr = {}
_sub_dot.node(
_sub_id, label=_graph_json['list'][_sub_id]['name'], **_attr
)
else:
# 节点颜色设置
if _id == _trace_info['trace_id']:
# 中心节点
_attr = _attr_center_node
elif _id in _trace_info['up_nodes']:
_attr = _attr_up_node
elif _id in _trace_info['down_nodes']:
_attr = _attr_down_node
elif _display_no_trace and _trace_info['trace_id'] != '':
# 不在链中且不应显示
continue
else:
_attr = {}
_dot.node(_id, label=_info['name'], **_attr)
# 开始画关联线
for _id, _info in _graph_json['list'].items():
for _key, _list in _info.items():
if _key in ('name', 'rank'):
continue
# 判断颜色
_edge_color = _relation_color[_key]
for _temp_id in _list:
if _temp_id == '':
continue
if _run_para['direction'] == 'reverse':
_head_id = _temp_id
_tail_id = _id
else:
_head_id = _id
_tail_id = _temp_id
# 处理颜色
_findstr = '%s&&%s' % (_head_id, _tail_id)
if _findstr in _trace_info['up_edges']:
_attr = _attr_up_edge
elif _findstr in _trace_info['down_edges']:
_attr = _attr_down_edge
elif _display_no_trace and _trace_info['trace_id'] != '':
continue
else:
_attr = {'color': _edge_color}
_dot.edge(_head_id, _tail_id, **_attr)
# 保存图片
_dir, _filename = os.path.split(os.path.abspath(_run_para['save']))
_filename = FileTool.get_file_name_no_ext(_filename)
_dot.render(
filename=_filename, directory=_dir,
format=_run_para['outformat'], view=(_run_para['view'] == 'true'),
cleanup=(_run_para['cleanup'] == 'true')
)
except Exception as e:
_prin_str = '%s (%s):\n%s' % (
_('execution exception'), str(e), traceback.format_exc()
)
prompt_obj.prompt_print(_prin_str)
return CResult(code='20999')
# 结束
return _ok_result
def labelimg_pic_deal(cls, path: str):
"""
TFRecord图片兼容处理
1.删除位深不为RGB三通道的图片
（解决image_size must contain 3 elements[4]报错）
2.转换图片格式为jpg
3.检查xml文件的文件名和路径是否正确
@param {str} path - 要处理的路径
"""
_path = os.path.realpath(path)
# 遍历所有子目录
_sub_dirs = FileTool.get_dirlist(path=_path, is_fullpath=True)
for _dir in _sub_dirs:
# 递归删除子目录的信息
cls.labelimg_pic_deal(_dir)
# 检查自己目录下的图片
_files = FileTool.get_filelist(path=_path, is_fullname=False)
for _file in _files:
_file_ext = FileTool.get_file_ext(_file)
if _file_ext == 'xml':
# 标签文件不处理
continue
_img_file = os.path.join(_path, _file)
_file_no_ext = FileTool.get_file_name_no_ext(_file)
if _file_ext in ('png', 'gif'):
# 转换图片格式
_fp = open(_img_file, 'rb')
_img = Image.open(_fp)
_rgb_im = _img.convert('RGB')
_rgb_im.save(os.path.join(_path, _file_no_ext + '.jpg'))
_fp.close()
# 删除原文件，修改xml中的文件名
FileTool.remove_file(_img_file)
_xml_file = os.path.join(_path, _file_no_ext + '.xml')
if os.path.exists(_xml_file):
_tree = ET.parse(os.path.join(_path, _xml_file))
_root = _tree.getroot()
_root.find('filename').text = _file_no_ext + '.jpg'
_root.find('path').text = os.path.join(
_path, _file_no_ext + '.jpg')
_tree.write(os.path.join(_path, _xml_file),
encoding='utf-8',
method="xml",
xml_declaration=None)
# 修改文件名变量
_img_file = os.path.join(_path, _file_no_ext + '.jpg')
# 打开图片判断位深
_fp = open(_img_file, 'rb')
_img = Image.open(_fp)
if _img.mode != 'RGB':
# 需要删除掉
_fp.close()
_xml_file = os.path.join(
_path,
FileTool.get_file_name_no_ext(_file) + '.xml')
print('delete %s' % _img_file)
FileTool.remove_file(_img_file)
if os.path.exists(_xml_file):
FileTool.remove_file(_xml_file)
else:
_fp.close()
# 检查xml文件
_xml_file = os.path.join(_path, _file_no_ext + '.xml')
if os.path.exists(_xml_file):
_tree = ET.parse(os.path.join(_path, _xml_file))
_root = _tree.getroot()
if _root.find('filename'
).text != _file_no_ext + '.jpg' or os.path.split(
_root.find('path').text)[0] != _path:
_root.find('filename').text = _file_no_ext + '.jpg'
_root.find('path').text = os.path.join(
_path, _file_no_ext + '.jpg')
_tree.write(os.path.join(_path, _xml_file),
encoding='utf-8',
method="xml",
xml_declaration=None)
def labelimg_copy_flags_pics(cls,
input_path: str,
output_path: str,
use_mapping: bool = False,
mapping: dict = None):
"""
按类别复制图片和标注文件到指定目录
@param {str} input_path - 图片路径
@param {str} output_path - 输出路径
@param {bool} use_mapping=False - 是否使用mapping处理映射
@param {dict} mapping=None - mapping.josn字典
@returns {iter_list} - 通过yield返回的处理进度信息清单
[总文件数int, 当前已处理文件数int, 是否成功]
"""
try:
# 遍历所有文件夹，获取需要处理的文件数量
_file_list = cls._get_labelimg_annotation_file_list(input_path)
_total = len(_file_list)
_deal_num = 0
# 先返回进度情况
if _total == 0:
yield [_deal_num, _total, True]
return
# 创建复制文件夹
FileTool.create_dir(output_path, exist_ok=True)
# 遍历处理
for _file in _file_list:
# 当前进展
yield [_deal_num, _total, True]
# 逐个标注文件进行处理
_tree = ET.parse(_file)
_root = _tree.getroot()
_annotations = dict()
_annotations['filename'] = _root.find('filename').text
_annotations['file_path'] = os.path.join(
os.path.split(_file)[0], _annotations['filename'])
# 获取信息字典
_info_dict = ExtendLib.get_info_dict(_annotations['file_path'],
mapping['info_key_dict'])
# 逐个标签处理
_save_class_path = '' # 要保存到的分类路径
_is_copy = False # 标注是否已复制文件
_new_xml_name = '' # 新的xml名
for _member in _root.findall('object'):
_member_class = _member[0].text
if use_mapping:
# 使用映射处理
if _member_class == mapping['set_by_info'][
'class_name']:
# 需要获取真实的信息
if mapping['set_by_info'][
'info_tag'] in _info_dict.keys():
_member_class = _info_dict[
mapping['set_by_info']['info_tag']]
# 变更分类名
_member[0].text = _member_class
# 过滤不需要的类别
if _member_class not in mapping['class_int'].keys():
_deal_num += 1
continue
# 保存分类路径
_save_class_path = os.path.join(
output_path, mapping['class'][_member_class])
else:
# 普通分类
_save_class_path = os.path.join(
output_path, _member_class)
# 复制文件
if not _is_copy:
# 处理重复文件名
_file_name = FileTool.get_file_name_no_ext(
_annotations['filename'])
_file_ext = FileTool.get_file_ext(
_annotations['filename'])
_rename_num = 1
_new_file_name = '%s.%s' % (_file_name, _file_ext)
_new_xml_name = '%s.xml' % (_file_name, )
while os.path.exists(
os.path.join(_save_class_path,
_new_file_name)):
_new_file_name = '%s_%d.%s' % (
_file_name, _rename_num, _file_ext)
_new_xml_name = '%s_%d.xml' % (_file_name,
_rename_num)
_rename_num += 1
# 创建文件夹
FileTool.create_dir(_save_class_path, exist_ok=True)
shutil.copyfile(
_annotations['file_path'],
os.path.join(_save_class_path, _new_file_name))
# 修改xml里面的文件名和文件路径
_root.find('filename').text = _new_file_name
_root.find('path').text = os.path.join(
_save_class_path, _new_file_name)
_is_copy = True
if _is_copy:
# 有修改xml内容
_tree.write(os.path.join(_save_class_path, _new_xml_name),
encoding='utf-8',
method="xml",
xml_declaration=None)
# 继续循环处理
_deal_num += 1
# 返回结果
yield [_total, _total, True]
except:
print('labelimg_copy_flags_pics error: %s\r\n%s' %
(input_path, traceback.format_exc()))
yield [-1, -1, False]
def labelimg_copy_flags_pics(cls,
input_path: str,
output_path: str,
is_cc: bool = False):
"""
按类别复制图片和标注文件到指定目录
@param {str} input_path - 图片路径
@param {str} output_path - 输出路径
@param {bool} is_cc=False - 是否CC项目
@returns {iter_list} - 通过yield返回的处理进度信息清单
[总文件数int, 当前已处理文件数int, 是否成功]
"""
try:
# 遍历所有文件夹，获取需要处理的文件数量
_file_list = cls._get_labelimg_annotation_file_list(input_path)
_total = len(_file_list)
_deal_num = 0
# 先返回进度情况
if _total == 0:
yield [_deal_num, _total, True]
return
# 创建复制文件夹
FileTool.create_dir(output_path, exist_ok=True)
# 遍历处理
for _file in _file_list:
# 当前进展
yield [_deal_num, _total, True]
# 逐个标注文件进行处理
_tree = ET.parse(_file)
_root = _tree.getroot()
_annotations = dict()
_annotations['filename'] = _root.find('filename').text
_annotations['file_path'] = os.path.join(
os.path.split(_file)[0], _annotations['filename'])
# 逐个标签处理
_save_class_path = '' # 要保存到的分类路径
_is_copy = False # 标注是否已复制文件
_is_change_class = False # 标注是否有修改分类名
_new_xml_name = '' # 新的xml名
for _member in _root.findall('object'):
_member_class = _member[0].text
if is_cc:
# CC专属的类型转换
if _member_class == '翡翠':
# 需要获取真实的信息
_info_file = os.path.join(
os.path.split(_file)[0], 'info.json')
_info = dict()
with open(_info_file, 'rb') as f:
_eval = str(f.read(), encoding='utf-8')
_info = eval(_eval)
if _info['款式'] == '挂件':
# 挂件，需要二级分类
_member_class = _info['挂件类型']
else:
# 一级分类
_member_class = _info['款式']
# 变更分类名
_member[0].text = _member_class
_is_change_class = True
# 过滤不需要的类别
if _member_class not in USE_CLASS_TEXT_LIST:
_deal_num += 1
continue
# 保存分类路径
_save_class_path = os.path.join(
output_path, cls._cc_get_class_text(_member_class))
else:
# 普通分类
_save_class_path = os.path.join(
output_path, _member_class)
# 复制文件
if not _is_copy:
# 处理重复文件名
_file_name = FileTool.get_file_name_no_ext(
_annotations['filename'])
_file_ext = FileTool.get_file_ext(
_annotations['filename'])
_rename_num = 1
_new_file_name = '%s.%s' % (_file_name, _file_ext)
_new_xml_name = '%s.xml' % (_file_name, )
while os.path.exists(
os.path.join(_save_class_path,
_new_file_name)):
_new_file_name = '%s_%d.%s' % (
_file_name, _rename_num, _file_ext)
_new_xml_name = '%s_%d.xml' % (_file_name,
_rename_num)
_rename_num += 1
# 创建文件夹
FileTool.create_dir(_save_class_path, exist_ok=True)
shutil.copyfile(
_annotations['file_path'],
os.path.join(_save_class_path, _new_file_name))
_is_copy = True
if _is_copy:
# 有复制文件
if _is_change_class:
# 有修改xml内容
_tree.write(os.path.join(_save_class_path,
_new_xml_name),
encoding='utf-8',
method="xml",
xml_declaration=None)
else:
shutil.copyfile(
_file, os.path.join(_save_class_path,
_new_xml_name))
# 继续循环处理
_deal_num += 1
# 返回结果
yield [_total, _total, True]
except:
print('labelimg_copy_flags_pics error: %s\r\n%s' %
(input_path, traceback.format_exc()))
yield [-1, -1, False]
