def create_pbtxt(cls, save_path: str, mapping: dict) -> bool:
"""
创建labelmap.pbtxt文件
@param {str} save_path - 文件保存路径
@param {dict} mapping - mapping.josn文件对应的字典
@return {bool} - 处理结果
"""
try:
_fix_str = "item {\n id: %d\n name: '%s'\n}"
_list = []
# 按int进行排序
_class_int = sorted(mapping['class_int'].items(),
key=lambda kv: (kv[1], kv[0]))
for _item in _class_int:
_id = _item[1]
_name = mapping['class'][_item[0]]
_list.append(_fix_str % (_id, _name))
# 保存到文件中
FileTool.create_dir(save_path, exist_ok=True)
with open(os.path.join(save_path, 'labelmap.pbtxt'), 'wb') as f:
f.write(str.encode('\n\n'.join(_list), encoding='utf-8'))
return True
except:
print('create_pbtxt error: \r\n%s' % (traceback.format_exc(), ))
return False
def __init__(self, desired_caps: dict, **kwargs):
"""
构造函数
@param {dict} desired_caps=None - 设备连接参数字典
deviceName {str} - 使用的手机或模拟器类型, 填入 adb devices -l 显示的设备连接名,
示例: 127.0.0.1:21513
appPackage {str} - 连接时运行的 Android 应用的包名, 例如: com.android.browser
appActivity {str} - 包中所要启动的 Android acticity, 例如: .BrowserActivity
@param {kwargs} - 扩展参数，支持的扩展参数包括
shell_encoding {str} - shell命令的编码方式, 在使用到命令行工具时使用, 默认为 'utf-8'
adb_name {str} - adb命令的启动名称, 安卓adb版专用, 默认为 'adb'
tmp_path {str} - 临时目录, 处理adb资源文件, 安卓adb版专用, 默认为当前工作目录
"""
self._desired_caps = {}
self._desired_caps.update(desired_caps)
# 其他参数
self.shell_encoding = kwargs.get('shell_encoding', 'utf-8')
self.adb_name = kwargs.get('adb_name', 'adb')
self.tmp_path = os.path.abspath(kwargs.get('tmp_path', ''))
FileTool.create_dir(self.tmp_path, exist_ok=True)
# 判断命令行
self.grep_str = 'findstr' if sys.platform == 'win32' else 'grep'
# 缓存字典
self.cache = dict()
# 要安装到设备上的文件路径
self._file_path = os.path.join(
os.path.realpath(os.path.dirname(__file__)), 'adb_apk')
def screenshot(self, filename: str = None) -> Image:
"""
保存屏幕截图
@param {str} filename=None - 要保存的路径
@returns {PIL.Image} - 图片对象
"""
# 截图
_cmd = 'shell uiautomator runtest UiTestTools.jar -c com.snaker.testtools.uiScreenShot'
self.adb_run_inner(_cmd)
# 获取文件
_filename = filename
if filename is None:
_filename = os.path.join(self.tmp_path, 'uiShot.png')
_cmd = 'pull /data/local/tmp/uiShot.png %s' % _filename
self.adb_run_inner(_cmd)
# 加载为对象
with open(_filename, 'rb') as _f:
_image = Image.open(BytesIO(_f.read()))
if filename is None:
# 删除临时文件
FileTool.remove_file(_filename)
return _image
def create_cc_type_pbtxt(cls, save_path: str) -> bool:
"""
创建CC的款式labelmap.pbtxt文件
@param {str} save_path - 文件保存路径
@return {bool} - 处理结果
"""
try:
_fix_str = "item {\n id: %d\n name: '%s'\n}"
_list = []
# 只处理所设定的分类
for _type in USE_CLASS_TEXT_LIST:
_id = CLASS_TEXT_TO_INT[_type]
_name = ''
if _type in PROP_TYPE_TRAN_DICT.keys():
_name = PROP_TYPE_TRAN_DICT[_type]
else:
for _dict_key in PROP_VALUE_TRAN_DICT.keys():
if _type in PROP_VALUE_TRAN_DICT[_dict_key].keys():
_name = PROP_VALUE_TRAN_DICT[_dict_key][_type]
break
_list.append(_fix_str % (_id, _name))
# 保存到文件中
FileTool.create_dir(save_path, exist_ok=True)
with open(os.path.join(save_path, 'labelmap.pbtxt'), 'wb') as f:
f.write(str.encode('\n\n'.join(_list), encoding='utf-8'))
return True
except:
print('create_cc_type_pbtxt error: \r\n%s' %
(traceback.format_exc(), ))
return False
def _write_finished(self):
"""
文件写入结束
@throws {Md5VerifyError} - 当文件校验失败时抛出异常
"""
# 关闭文件句柄
self._info_file_handle.close()
self._info_file_handle = None
self._tmp_file_handle.close()
self._tmp_file_handle = None
os.close(self._lock_file_handle)
self._lock_file_handle = None
# 检查md5
if self._info['md5'] != '':
_file_md5 = NetTool.get_file_md5(self._temp_file)
if self._info['md5'] != _file_md5:
raise Md5VerifyError('md5 verify error')
# 修改临时文件名
os.rename(self._temp_file, self._file)
# 删除临时文件
FileTool.remove_file(self._info_file)
FileTool.remove_file(self._lock_file)
def single_process_get_lockfile(process_name='', base_path=''):
"""
获取进程锁处理锁文件路径
进程锁处理的辅助函数，获取指定进程的锁文件名（含路径）
@param {string} process_name='' - 进程锁的进程名，默认值为''；如果为''代表获取执行程序的模块名
@param {string} base_path='' - 进程锁文件指定的路径，默认值为''；如果为''代表获取执行程序的模块文件目录
@returns {string} - 锁文件名（含路径）
"""
_process_name = process_name
_base_path = base_path
if not len(_process_name) > 0:
# 默认取运行的程序名
_process_name = FileTool.get_exefile_name_no_ext()
if not len(_base_path) > 0:
# 默认取运行程序目录
_base_path = FileTool.get_exefile_path() + os.sep
else:
_base_path = os.path.realpath(_base_path)
if not os.path.isdir(_base_path):
# 如果是文件的情况下，拿文件对应的目录
_base_path = os.path.split(os.path.realpath(_base_path))[0]
_base_path = _base_path + os.sep
return _base_path + _process_name + ".lock" # 要建立锁的文件名
def rename_file_to_num(cls, path: str, start_index: int = 1) -> int:
"""
重命名文件为数字序号
@param {str} path - 要处理的文件夹
@param {int} start_index=1 - 开始序号
@returns {int} - 返回当前序号
"""
# 处理当前目录
_start_index = start_index
_path = os.path.realpath(path)
_file_list = FileTool.get_filelist(_path, is_fullname=False)
for _file in _file_list:
_ext = FileTool.get_file_ext(_file)
os.rename(
os.path.join(_path, _file),
os.path.join(
_path,
StringTool.fill_fix_string(str(_start_index), 10, '0') +
'.' + _ext))
_start_index += 1
# 处理子目录
_sub_dir_list = FileTool.get_dirlist(_path)
for _sub_dir in _sub_dir_list:
_start_index = cls.rename_file_to_num(_sub_dir,
start_index=_start_index)
# 返回当前序号值
return _start_index
def _add_images_doc(self, path: str, import_path: str, url_prefix: str):
"""
为文件夹下的图片添加信息字典文件(id为不含扩展的文件名)
@param {str} path - 要处理的文件目录
"""
_import_path = os.path.realpath(import_path)
# 处理当前文件夹
_file_list = FileTool.get_filelist(path,
regex_str=r'^((?!\.json$).)*$',
is_fullname=True)
for _file in _file_list:
_ext = FileTool.get_file_ext(_file)
_json_file = _file[0:-len(_ext)] + 'json'
if os.path.exists(_json_file):
# 字典文件已存在，无需处理
continue
# 生成并写入字典
_file_name = FileTool.get_file_name_no_ext(_file)
_url = os.path.realpath(_file)[len(_import_path):].replace(
'\\', '/').lstrip('/')
_url = '%s/%s' % (url_prefix, _url)
_image_doc = {'id': _file_name, 'url': _url, 'path': _file}
_json_str = json.dumps(_image_doc, ensure_ascii=False)
with open(_json_file, 'wb') as _fid:
_fid.write(_json_str.encode(encoding='utf-8'))
# 处理子文件夹
_sub_dir_list = FileTool.get_dirlist(path)
for _sub_dir in _sub_dir_list:
self._add_images_doc(_sub_dir, import_path, url_prefix)
def setUp(self):
"""
启动测试执行的初始化
"""
# 初始化日志类
DebugTool.set_debug(False)
try:
# 删除临时日志
FileTool.remove_files(path=os.path.join(
_TEST_DATA_DIR, 'temp/http_service_log/log'),
regex_str='test_case*')
except:
pass
self.logger = simple_log.Logger(
conf_file_name=os.path.join(
_TEST_DATA_DIR, 'tcp_ip_service/test_http_service.json'),
logger_name=simple_log.EnumLoggerName.ConsoleAndFile,
config_type=simple_log.EnumLoggerConfigType.JSON_FILE,
logfile_path=os.path.join(
_TEST_DATA_DIR, 'temp/http_service_log/log/test_case.log'),
is_create_logfile_by_day=True)
self.logger.setLevelWithHandler(simple_log.DEBUG)
# 启动服务
self.server = HttpService(
logger=self.logger,
server_status_info_fun=self.server_status_info_fun,
self_tag='UnitTest',
log_level=simple_log.INFO,
server_http_deal_fun=self.server_http_deal_fun)
_server_opts = HttpService.generate_server_opts()
_server_opts.ip = "127.0.0.1"
_server_opts.port = 9513
self.server.start_server(server_opts=_server_opts)
def labelimg_rename_filename(cls, path: str, fix_len: int = 10):
"""
重名名labelimg对应目录下的文件名（图片文件和标注文件同步修改）
@param {str} path - 要修改文件名的路径
@param {int} fix_len=10 - 文件名长度
"""
_path = os.path.realpath(path)
_files = FileTool.get_filelist(path=_path, is_fullname=False)
_index = 1
for _file in _files:
_file_ext = FileTool.get_file_ext(_file)
if _file_ext == 'xml':
# 标签文件不处理
continue
_file_no_ext = FileTool.get_file_name_no_ext(_file)
# 获取最新的文件名
while True:
_new_name = StringTool.fill_fix_string(str(_index),
fix_len,
'0',
left=True)
_new_file = _new_name + '.' + _file_ext
_index += 1
if os.path.exists(os.path.join(path, _new_file)):
# 文件名已存在
_index += 1
continue
# 文件名不存在，跳出循环
break
# 修改文件名
os.rename(os.path.join(_path, _file),
os.path.join(_path, _new_file))
if os.path.exists(os.path.join(_path, _file_no_ext + '.xml')):
# 需要修改标签文件
_xml_file = _new_name + '.xml'
os.rename(os.path.join(_path, _file_no_ext + '.xml'),
os.path.join(_path, _xml_file))
# 修改标签文件内容
_tree = ET.parse(os.path.join(_path, _xml_file))
_root = _tree.getroot()
_root.find('filename').text = _new_file
_root.find('path').text = os.path.join(_path, _new_file)
_tree.write(os.path.join(_path, _xml_file),
encoding='utf-8',
method="xml",
xml_declaration=None)
def test_case2():
# 测试多个日志类相互影响的情况
try:
FileTool.remove_file(_TEMP_DIR + '/test_case2.json')
except:
pass
try:
FileTool.remove_file(_TEMP_DIR + '/test_case2-1.json')
except:
pass
try:
FileTool.remove_files(path=_TEMP_DIR + '/log/', regex_str='test_case2*')
except:
pass
try:
FileTool.remove_files(path=_TEMP_DIR + '/log/', regex_str='test_case2-1*')
except:
pass
_logger = simple_log.Logger(conf_file_name=_TEMP_DIR + '/test_case2.json',
logger_name=simple_log.EnumLoggerName.ConsoleAndFile,
config_type=simple_log.EnumLoggerConfigType.JSON_FILE,
logfile_path=_TEMP_DIR + '/log/test_case2.log')
# ConsoleAndFile 的配置level为DEBUG，但对应的ConsoleHandler的level为DEBUG，FileHandler的level为INFO
# 日志是否显示会根据logger的level和handler的level，以级别比较高的匹配输出
# 注意默认root的handler应该为空，否则无论如何都会执行root的输出，如果自己又另外指定了输出，那就会有2个相同输出日志
_logger.log(simple_log.DEBUG,
'test_case2:write_log:DEBUG:1:界面应显示本日志，文件不应显示本日志')
_logger.debug('test_case2:write_log:DEBUG:1-1:界面应显示本日志，文件不应显示本日志')
_logger.log(simple_log.INFO,
'test_case2:write_log:INFO:2:界面应显示本日志，文件应显示本日志')
_logger.info('test_case2:write_log:INFO:2-1:界面应显示本日志，文件应显示本日志')
# 新增logger，但与原logger的loggername一样，实际上会互相影响，同时如果handler一样，也会受影响
_logger1 = simple_log.Logger(conf_file_name=_TEMP_DIR + '/test_case2-1.json',
logger_name=simple_log.EnumLoggerName.ConsoleAndFile,
config_type=simple_log.EnumLoggerConfigType.JSON_FILE,
logfile_path=_TEMP_DIR + '/log/test_case2-1.log')
_logger1.setLevel(simple_log.DEBUG)
_logger.log(simple_log.DEBUG,
'test_case2:write_log:DEBUG:3:界面应显示本日志，文件不应显示本日志,但实际受logger1影响，也记录了日志；本应记录在日志1中，但受影响记录在日志2中')
_logger.log(simple_log.INFO,
'test_case2:write_log:INFO:4:界面应显示本日志，文件应显示本日志；本应记录在日志1中，但受影响记录在日志2中')
_logger1.log(simple_log.DEBUG,
'test_case2:write_log:DEBUG:5-1:界面应显示本日志，文件应显示本日志')
_logger1.log(simple_log.INFO,
'test_case2:write_log:INFO:6-1:界面应显示本日志，文件应显示本日志')
del _logger
_logger1.log(simple_log.DEBUG,
'test_case2:write_log:DEBUG:6-1:界面应显示本日志，文件应显示本日志')
_logger1.log(simple_log.INFO,
'test_case2:write_log:INFO:7-1:界面应显示本日志，文件应显示本日志')
del _logger1
def labelimg_flags_count(cls, input_path: str, mapping: dict):
"""
统计指定目录中的labelimg标记对应标签的数量
@param {str} input_path - 要统计的目录
@param {dict} mapping - mapping.json的字典
@returns {iter_list} - 通过yield返回的处理进度信息清单
[总文件数int, 当前已处理文件数int, 是否成功, 统计结果字典(标签名, 数量)]
"""
try:
# 遍历所有文件夹，获取需要处理的文件数量
_file_list = cls._get_labelimg_annotation_file_list(input_path)
_total = len(_file_list)
_deal_num = 0
_flags_count = dict()
# 先返回进度情况
if _total == 0:
yield [_deal_num, _total, True, _flags_count]
return
# 遍历文件进行处理
for _file in _file_list:
# 当前进展
yield [_deal_num, _total, True, _flags_count]
# 统计当前文件
_tree = ET.parse(_file)
_root = _tree.getroot()
_image_file = os.path.join(
os.path.split(_file)[0],
FileTool.get_file_name_no_ext(_file) + '.jpg')
_info_dict = ExtendLib.get_info_dict(_image_file,
mapping['info_key_dict'])
# 逐个标签处理
for _member in _root.findall('object'):
_member_class = _member[0].text
if _member_class == mapping['set_by_info']['class_name']:
# 需要转换为当前类型
if mapping['set_by_info'][
'info_tag'] in _info_dict.keys():
_member_class = _info_dict[mapping['set_by_info']
['info_tag']]
if _member_class in _flags_count.keys():
_flags_count[_member_class] += 1
else:
_flags_count[_member_class] = 1
_deal_num += 1
# 返回结果
yield [_total, _total, True, _flags_count]
except:
print('labelimg_flags_count error: %s\r\n%s' %
(input_path, traceback.format_exc()))
yield [-1, -1, False]
def auto_install(self, is_init: bool):
"""
自动检索目录并进行安装
@param {bool} is_init - 指示是否第一次运行
"""
_deal_plugins = list()
_dirs = FileTool.get_dirlist(self.plugin_path, is_fullpath=False)
for _dir_name in _dirs:
_plugin_config = SimpleXml(
os.path.join(self.plugin_path, _dir_name,
'plugin.xml')).to_dict()['config']
# 登记信息
_deal_plugins.append(_plugin_config['info']['plugin_name'])
if self.is_installed(_plugin_config['info']['plugin_name']):
# 已安装, 判断是否要强制更新
if not (is_init and self.always_update):
# 非强制更新
continue
# 执行安装处理
self.install(_dir_name)
# 卸载插件目录没有的插件
_fetchs = self._exec_sql('select plugin_name from t_installed',
is_fetchall=True)
for _row in _fetchs:
if _row[0] not in _deal_plugins:
# 插件已不存在，卸载处理
self.uninstall(_row[0])
def get_info_dict(cls, image_file: str, key_dict: dict) -> dict:
"""
通过图片文件路径获取信息字典
@param {str} image_file - 图片文件
@param {dict} key_dict - 需要展示信息的模板字典
@returns {dict}
"""
_info_dict = dict()
_path = os.path.split(image_file)[0]
_file_no_ext = FileTool.get_file_name_no_ext(image_file)
_info_file = os.path.join(_path, _file_no_ext + ".info")
if not os.path.exists(_info_file):
_info_file = os.path.join(_path, 'info.json')
if os.path.exists(_info_file):
with open(_info_file, 'rb') as f:
_eval = str(f.read(), encoding='utf-8')
_info = eval(_eval)
_info_dict = copy.deepcopy(key_dict)
for _key in _info_dict:
if _key in _info.keys():
_info_dict[_key] = _info[_key]
return _info_dict
def load_trans_from_dir(self,
trans_file_path,
trans_file_prefix,
encoding='utf-8',
append=True):
"""
从指定路径加载语言信息文件到对象中
@param {string} trans_file_path - 语言信息文件所在路径，如果为None则代表不处理信息文件
@param {string} trans_file_prefix - 语言信息文件前缀，
例如前缀为test，信息文件名为test_en.json、test_zh.json等
@param {string} encoding='utf-8' - 解析文件的编码
@param {bool} append=True - 是否追加模式，如果是则以增量方式更新，否则覆盖原来的配置
"""
file_list = FileTool.get_filelist(path=trans_file_path,
regex_str=r'^' + trans_file_prefix +
r'_\S+\.json$',
is_fullname=True)
for file in file_list:
# 循环加载语言信息文件
file_name = os.path.split(os.path.realpath(file))[1]
file_lang = re.sub(
r'\.json$', '',
re.sub(r'^' + trans_file_prefix + r'_', '', file_name))
self.load_trans_from_file(file_full_path=file,
lang=file_lang,
encoding=encoding,
append=append)
def get_init_objs(cls) -> dict:
"""
获取工具初始化对象
@returns {dict} - 初始化对象
"""
# 获取配置文件信息
_execute_path = os.path.realpath(FileTool.get_file_path(__file__))
_config = os.path.join(_execute_path, '../conf/server.xml')
_config_xml = SimpleXml(_config, encoding='utf-8')
_server_config = _config_xml.to_dict()['server']
# 日志对象
_logger: Logger = None
if 'logger' in _server_config.keys():
_logger = Logger.create_logger_by_dict(_server_config['logger'])
# 连接数据库操作对象
_qa_manager = QAManager(
_server_config['answerdb'],
_server_config['milvus'],
_server_config['bert_client'],
logger=_logger,
excel_batch_num=_server_config['excel_batch_num'],
excel_engine=_server_config['excel_engine'],
load_para=False)
return {'logger': _logger, 'qa_manager': _qa_manager}
def generate_thumbnail(cls, upload_type: str, note: str, file_name: str,
save_path: str, url: str, **kwargs):
"""
生成传入图片的缩略图
@param {str} upload_type - 文件类型，必须在UploadFileConfig表中有配置
@param {str} note - 文件注解
@param {str} file_name - 保存的文件名
@param {str} save_path - 保存的文件路径(含文件名)
@param {str} url - 可访问的url
@returns {str, str, list} - 返回处理结果二元组 status, msg, answers
注：可以通过这个返回改变上传成功的结果，answers是返回的回答字符数组，如果不需要回答则应可返回[]
"""
_path = FileTool.get_file_path(save_path)
_filename = 'thumbnail_' + file_name
_url = os.path.join(url[0:url.rfind('/') + 1], _filename)
# 等比例生成缩略图
_img = Image.open(save_path)
_ori_w, _ori_h = _img.size
_size = (
THUMBNAIL_SIZE[0] if THUMBNAIL_SIZE[0] != 0 else math.ceil(
_ori_w * THUMBNAIL_SIZE[1] / _ori_h),
THUMBNAIL_SIZE[1] if THUMBNAIL_SIZE[1] != 0 else math.ceil(
_ori_h * THUMBNAIL_SIZE[0] / _ori_w),
)
_img.thumbnail(_size, Image.ANTIALIAS)
if _img.mode == "P":
_img = _img.convert('RGB')
_img.save(os.path.join(_path, _filename))
return '00000', 'success', [{'thumbnail': _url}]
def test_JadeTypeDetect():
"""
测试翡翠类型处理器
"""
_execute_path = RunTool.get_global_var('EXECUTE_PATH')
_pipeline = RunTool.get_global_var('EMPTY_PIPELINE')
_processer_class: PipelineProcesser = Pipeline.get_plugin('processer', 'JadeTypeDetect')
_filelist = FileTool.get_filelist(
os.path.join(_execute_path, os.path.pardir, 'test_data/test_pic/'),
is_fullname=True
)
for _file in _filelist:
# 遍历执行
with open(_file, 'rb') as _fid:
_file_bytes = _fid.read()
_input = {
'type': '',
'sub_type': '',
'image': Image.open(_file_bytes),
'score': 0.0
}
_output = _processer_class.execute(_input, {}, _pipeline)
# 输出图片和对应文字
_image = _output['image']
_print_str = 'type: %s\nsub_type: %s\nscore: %s' % (
_output['type'], _output['sub_type'], str(_output['score']))
_draw = ImageDraw.Draw(_image) # PIL图片上打印汉字
# 参数1：字体文件路径，参数2：字体大小；Windows系统“simhei.ttf”默认存储在路径：C:\Windows\Fonts中
_font = ImageFont.truetype("simhei.ttf", 20, encoding="utf-8")
_draw.text((0, 0), _print_str, (255, 0, 0), font=_font)
plt.figure(_file)
plt.imshow(_image)
plt.show()
def change_info_file(cls, image_file: str, prop_name: str,
prop_value: str) -> bool:
"""
修改指定图片的info文件
@param {str} image_file - 传入图片文件
@param {str} prop_name - 属性名
@param {str} prop_value - 属性值
@returns {bool} - 处理结果
"""
_path = os.path.split(image_file)[0]
_file_no_ext = FileTool.get_file_name_no_ext(image_file)
_info_file = os.path.join(_path, _file_no_ext + ".info")
if not os.path.exists(_info_file):
_info_file = os.path.join(_path, 'info.json')
if os.path.exists(_info_file):
# 有信息文件才处理
_info = dict()
with open(_info_file, 'rb') as f:
_eval = str(f.read(), encoding='utf-8')
_info = eval(_eval)
_info[prop_name] = prop_value
# 保存JSON文件
_json = str(_info)
with open(_info_file, 'wb') as f:
f.write(str.encode(_json, encoding='utf-8'))
return True
else:
return False
def _get_dom_files(cls, path: str, files: list):
"""
获取指定目录下的所有dom.html文件
@param {str} path - 路径
@param {list} files - 找到的文件清单
"""
# 先找当前目录下的文件
_temp_list = FileTool.get_filelist(path,
regex_str=r'^dom\.html$',
is_fullname=True)
files.extend(_temp_list)
# 遍历所有子目录获取文件
_dirs = FileTool.get_dirlist(path)
for _dir in _dirs:
cls._get_dom_files(_dir, files)
def remove_data(cls, qa_manager: QAManager, logger: Logger):
"""
清空所有数据
@param {QAManager} qa_manager - 数据管理对象
@param {Logger} logger - 日志对象
"""
AnswerDao.drop_tables(DATA_TABLES)
AnswerDao.create_tables(DATA_TABLES)
# 删除上传文件
_execute_path = os.path.realpath(FileTool.get_file_path(__file__))
FileTool.remove_all_with_path(
path=os.path.join(_execute_path, '../client/static/leave_message'))
if logger is not None:
logger.debug('remove leave message plugin data success!')
def clean_file_path(cls, path: str):
"""
清理文件目录
1、批量删除带括号的图片文件(重复下载)
2、删除宣传图片
2、将文件名修改为"产品编号_main/detail_序号"的格式
3、将文件夹按款式进行分类
@param {str} path - 要清理的文件夹
@return {iter_list} - 通过yield返回的处理进度信息清单
[总文件数int, 当前已处理文件数int, 是否成功]
"""
try:
_path = path
if not (_path.endswith('/') or _path.endswith('\\')):
_path = _path + '/'
# 创建分类目录
_class_path = os.path.join(FileTool.get_parent_dir(_path),
FileTool.get_dir_name(_path) + '_class')
if not os.path.exists(_class_path):
FileTool.create_dir(_class_path, exist_ok=True)
# 获取目录清单
_dir_list = cls._get_child_dir_list(path, with_root=True)
_total = len(_dir_list)
_deal_num = 0
# 先返回进度情况
if _total == 0:
yield [_deal_num, _total, True]
return
# 遍历目录执行处理
for _dir in _dir_list:
yield [_deal_num, _total, True]
cls._clean_file_path(_dir, _class_path)
_deal_num += 1
yield [_deal_num, _total, True]
except:
print('clean_file_path error: %s\r\n%s' %
(path, traceback.format_exc()))
yield [-1, -1, False]
def product_info_to_xls(cls, path: str) -> bool:
"""
将产品信息写入excel文件
@param {str} path - 要获取产品信息的目录
@returns {bool} - 处理是否成功
"""
try:
# 标题行
_title = dict()
_col = 2
for _key in PROP_NAME_TRAN_DICT.keys():
_title[_key] = _col
_col += 1
# 创建excel文件
_xls_file = os.path.join(path, 'product_info_list.xls')
if os.path.exists(_xls_file):
# 删除文件
FileTool.remove_file(_xls_file)
# 创建一个新的Workbook
_book = xlwt.Workbook()
_sheet = _book.add_sheet('product_info') # 在工作簿中新建一个表格
# 写入标题
_sheet.write(0, 0, '网站产品ID')
_sheet.write(0, 1, '产品目录')
for _word in _title.keys():
print()
_sheet.write(0, _title[_word], _word)
_current_row = [1] # 当前行
# 逐个产品进行写入
cls._write_product_info_to_xls(path, _sheet, _title, _current_row)
# 保存excel
_book.save(_xls_file)
return True
except:
print('product_info_to_xls error:\r\n%s' %
(traceback.format_exc(), ))
return False
def _write_product_info_to_xls(cls, path: str, sheet, title: dict,
current_row: list):
"""
按目录逐个将产品信息写入excel文件>
@param {str} path - 要处理的目录
@param {object} sheet - excel的sheet对象
@param {dict} title - 标题清单
@param {list} current_row - 当前行
"""
# 先处理自己
_info_file = os.path.join(path, 'info.json')
if os.path.exists(_info_file):
# 有信息文件才处理
_info = dict()
with open(_info_file, 'rb') as f:
_eval = str(f.read(), encoding='utf-8')
_info = eval(_eval)
# 产品编号和产品目录
_product_num = FileTool.get_dir_name(path)
sheet.write(current_row[0], 0, _product_num)
sheet.write(current_row[0], 1, path)
# 逐个信息项写入
for _key in _info.keys():
if _key in title.keys():
sheet.write(current_row[0], title[_key], _info[_key])
else:
# 要新增列标题
_col = len(title) + 2
title[_key] = _col
sheet.write(0, _col, _key)
# 写入信息值
sheet.write(current_row[0], _col, _info[_key])
# 换到下一行
current_row[0] += 1
# 处理子目录
_dirs = FileTool.get_dirlist(path)
for _dir in _dirs:
cls._write_product_info_to_xls(_dir, sheet, title, current_row)
def load_predef_by_path(self, path: str, encoding: str = 'utf-8'):
"""
通过文件路径装载预定义模板
@param {str} path - 要装载模板的路径
@param {str} encoding='utf-8' - 文件编码
"""
_file_list = FileTool.get_filelist(path, is_fullname=True)
for _file in _file_list:
self.load_predef_by_file(_file, encoding=encoding)
def import_images(self, path: str, pipeline: str, encoding: str = 'utf-8'):
"""
将指定路径的图片导入搜索库
@param {str} path - 图片及信息字典所在路径，包含文件:
图片文件，例如"abc.jpg"
对应的json信息字典文件，例如"abc.json"，文件内容按标准json字符串格式编写
注：json中可以通过添加collection域指定该图片的所属分类集合名
@param {str} pipeline - 处理管道标识
@param {str} encoding='utf-8' - json文件的编码
"""
_pipeline_obj = self._get_pipeline(pipeline)
_file_list = FileTool.get_filelist(path,
regex_str=r'^((?!\.json$).)*$',
is_fullname=True)
for _file in _file_list:
try:
# 获取图片的信息字典
_ext = FileTool.get_file_ext(_file)
_json_file = _file[0:-len(_ext)] + 'json'
if not os.path.exists(_json_file):
self.log_debug(
'Json file not exists, not imported: [%s]!' % _file)
continue
with open(_json_file, 'r', encoding=encoding) as _fid:
_image_doc = json.loads(_fid.read())
# 导入图片
_collection = _image_doc.get('collection', '')
with open(_file, 'rb') as _fid:
self._image_to_search_db(_fid.read(),
_image_doc,
_pipeline_obj,
init_collection=_collection)
# 输出日志
self.log_debug('image [%s] imported success' % _file)
except:
self.log_debug('image [%s] import error: %s' %
(_file, traceback.format_exc()))
def uninstall(self, plugin_name: str):
"""
卸载指定插件
@param {str} plugin_name - 插件名
"""
# 删除主界面入口
self.unload_plugin(plugin_name)
# 删除相应文件
_static_path = os.path.join(self.static_path, 'plugin', plugin_name)
if os.path.exists(_static_path):
FileTool.remove_dir(_static_path)
_templates_path = os.path.join(self.templates_path, 'plugin',
plugin_name)
if os.path.exists(_templates_path):
FileTool.remove_dir(_templates_path)
_config_path = os.path.join(self.config_path, 'plugin', plugin_name)
if os.path.exists(_config_path):
FileTool.remove_dir(_config_path)
# 删除安装信息
self._exec_sql("delete from t_installed where plugin_name=?",
para=(plugin_name, ))
def _get_pic_file_list(cls, input_path: str) -> list:
"""
获取制定目录下的所有图片文件清单
@param {str} input_path - 要处理的目录
@returns {list} - 文件清单列表
"""
_list = []
# 先获取当前目录下的所有xml文件
for _file in FileTool.get_filelist(input_path, is_fullname=True):
_ext = FileTool.get_file_ext(_file)
if _ext.lower() in ('jpg', 'jpeg'):
_list.append(_file)
# 获取子目录
for _dir in FileTool.get_dirlist(input_path):
_temp_list = cls._get_pic_file_list(_dir)
_list.extend(_temp_list)
return _list
def _get_labelimg_annotation_file_list(cls, input_path: str) -> list:
"""
获取要处理的LabelImg标注文件清单
@param {str} input_path - 起始目录
@returns {list} - 返回文件清单
"""
_list = []
# 先获取当前目录下的所有xml文件
for _file in FileTool.get_filelist(input_path, regex_str=r'.*\.xml$'):
_pic_file = _file[0:-3] + 'jpg'
if os.path.exists(_pic_file):
_list.append(_file)
# 获取子目录
for _dir in FileTool.get_dirlist(input_path):
_temp_list = cls._get_labelimg_annotation_file_list(_dir)
_list.extend(_temp_list)
return _list
def load_plugins_by_path(cls, path: str):
"""
装载指定目录下的管道插件(处理器和路由器)
@param {str} path - 要装载的目录
"""
_file_list = FileTool.get_filelist(path=path,
regex_str=r'.*\.py$',
is_fullname=True)
for _file in _file_list:
if _file == '__init__.py':
continue
cls.load_plugins_by_file(_file)
