def _write_finished(self):
"""
文件写入结束
@throws {Md5VerifyError} - 当文件校验失败时抛出异常
"""
# 关闭文件句柄
self._info_file_handle.close()
self._info_file_handle = None
self._tmp_file_handle.close()
self._tmp_file_handle = None
os.close(self._lock_file_handle)
self._lock_file_handle = None
# 检查md5
if self._info['md5'] != '':
_file_md5 = NetTool.get_file_md5(self._temp_file)
if self._info['md5'] != _file_md5:
raise Md5VerifyError('md5 verify error')
# 修改临时文件名
os.rename(self._temp_file, self._file)
# 删除临时文件
FileTool.remove_file(self._info_file)
FileTool.remove_file(self._lock_file)
def screenshot(self, filename: str = None) -> Image:
"""
保存屏幕截图
@param {str} filename=None - 要保存的路径
@returns {PIL.Image} - 图片对象
"""
# 截图
_cmd = 'shell uiautomator runtest UiTestTools.jar -c com.snaker.testtools.uiScreenShot'
self.adb_run_inner(_cmd)
# 获取文件
_filename = filename
if filename is None:
_filename = os.path.join(self.tmp_path, 'uiShot.png')
_cmd = 'pull /data/local/tmp/uiShot.png %s' % _filename
self.adb_run_inner(_cmd)
# 加载为对象
with open(_filename, 'rb') as _f:
_image = Image.open(BytesIO(_f.read()))
if filename is None:
# 删除临时文件
FileTool.remove_file(_filename)
return _image
def test_case2():
# 测试多个日志类相互影响的情况
try:
FileTool.remove_file(_TEMP_DIR + '/test_case2.json')
except:
pass
try:
FileTool.remove_file(_TEMP_DIR + '/test_case2-1.json')
except:
pass
try:
FileTool.remove_files(path=_TEMP_DIR + '/log/', regex_str='test_case2*')
except:
pass
try:
FileTool.remove_files(path=_TEMP_DIR + '/log/', regex_str='test_case2-1*')
except:
pass
_logger = simple_log.Logger(conf_file_name=_TEMP_DIR + '/test_case2.json',
logger_name=simple_log.EnumLoggerName.ConsoleAndFile,
config_type=simple_log.EnumLoggerConfigType.JSON_FILE,
logfile_path=_TEMP_DIR + '/log/test_case2.log')
# ConsoleAndFile 的配置level为DEBUG，但对应的ConsoleHandler的level为DEBUG，FileHandler的level为INFO
# 日志是否显示会根据logger的level和handler的level，以级别比较高的匹配输出
# 注意默认root的handler应该为空，否则无论如何都会执行root的输出，如果自己又另外指定了输出，那就会有2个相同输出日志
_logger.log(simple_log.DEBUG,
'test_case2:write_log:DEBUG:1:界面应显示本日志，文件不应显示本日志')
_logger.debug('test_case2:write_log:DEBUG:1-1:界面应显示本日志，文件不应显示本日志')
_logger.log(simple_log.INFO,
'test_case2:write_log:INFO:2:界面应显示本日志，文件应显示本日志')
_logger.info('test_case2:write_log:INFO:2-1:界面应显示本日志，文件应显示本日志')
# 新增logger，但与原logger的loggername一样，实际上会互相影响，同时如果handler一样，也会受影响
_logger1 = simple_log.Logger(conf_file_name=_TEMP_DIR + '/test_case2-1.json',
logger_name=simple_log.EnumLoggerName.ConsoleAndFile,
config_type=simple_log.EnumLoggerConfigType.JSON_FILE,
logfile_path=_TEMP_DIR + '/log/test_case2-1.log')
_logger1.setLevel(simple_log.DEBUG)
_logger.log(simple_log.DEBUG,
'test_case2:write_log:DEBUG:3:界面应显示本日志，文件不应显示本日志,但实际受logger1影响，也记录了日志；本应记录在日志1中，但受影响记录在日志2中')
_logger.log(simple_log.INFO,
'test_case2:write_log:INFO:4:界面应显示本日志，文件应显示本日志；本应记录在日志1中，但受影响记录在日志2中')
_logger1.log(simple_log.DEBUG,
'test_case2:write_log:DEBUG:5-1:界面应显示本日志，文件应显示本日志')
_logger1.log(simple_log.INFO,
'test_case2:write_log:INFO:6-1:界面应显示本日志，文件应显示本日志')
del _logger
_logger1.log(simple_log.DEBUG,
'test_case2:write_log:DEBUG:6-1:界面应显示本日志，文件应显示本日志')
_logger1.log(simple_log.INFO,
'test_case2:write_log:INFO:7-1:界面应显示本日志，文件应显示本日志')
del _logger1
def product_info_to_xls(cls, path: str) -> bool:
"""
将产品信息写入excel文件
@param {str} path - 要获取产品信息的目录
@returns {bool} - 处理是否成功
"""
try:
# 标题行
_title = dict()
_col = 2
for _key in PROP_NAME_TRAN_DICT.keys():
_title[_key] = _col
_col += 1
# 创建excel文件
_xls_file = os.path.join(path, 'product_info_list.xls')
if os.path.exists(_xls_file):
# 删除文件
FileTool.remove_file(_xls_file)
# 创建一个新的Workbook
_book = xlwt.Workbook()
_sheet = _book.add_sheet('product_info') # 在工作簿中新建一个表格
# 写入标题
_sheet.write(0, 0, '网站产品ID')
_sheet.write(0, 1, '产品目录')
for _word in _title.keys():
print()
_sheet.write(0, _title[_word], _word)
_current_row = [1] # 当前行
# 逐个产品进行写入
cls._write_product_info_to_xls(path, _sheet, _title, _current_row)
# 保存excel
_book.save(_xls_file)
return True
except:
print('product_info_to_xls error:\r\n%s' %
(traceback.format_exc(), ))
return False
def labelimg_del_not_rgb_pic(cls, path: str):
"""
删除位深不为RGB三通道的图片
（解决image_size must contain 3 elements[4]报错）
@param {str} path - 要处理的路径
"""
_path = os.path.realpath(path)
# 遍历所有子目录
_sub_dirs = FileTool.get_dirlist(path=_path, is_fullpath=True)
for _dir in _sub_dirs:
# 递归删除子目录的信息
cls.labelimg_del_not_rgb_pic(_dir)
# 检查自己目录下的图片
_files = FileTool.get_filelist(path=_path, is_fullname=False)
for _file in _files:
_file_ext = FileTool.get_file_ext(_file)
if _file_ext == 'xml':
# 标签文件不处理
continue
# 打开图片判断位深
_fp = open(os.path.join(_path, _file), 'rb')
_img = Image.open(_fp)
if _img.mode != 'RGB':
# 需要删除掉
_fp.close()
_img_file = os.path.join(_path, _file)
_xml_file = os.path.join(
_path,
FileTool.get_file_name_no_ext(_file) + '.xml')
print('delete %s' % _img_file)
FileTool.remove_file(_img_file)
if os.path.exists(_xml_file):
FileTool.remove_file(_xml_file)
else:
_fp.close()
def page_source(self) -> str:
"""
获取当前页面源码
@property {str}
"""
# 生成源码
_cmd = 'shell uiautomator runtest UiTestTools.jar -c com.snaker.testtools.uiDumpXml'
self.adb_run_inner(_cmd)
# 获取文件
_cmd = 'pull /data/local/tmp/local/tmp/uidump.xml %s' % self.tmp_path
self.adb_run_inner(_cmd)
# 打开文件
_page_source = ''
_file = os.path.join(self.tmp_path, 'uidump.xml')
with open(_file, 'r', encoding='utf-8') as _f:
_page_source = _f.read()
# 删除临时文件
FileTool.remove_file(_file)
return _page_source
def UploadFile(cls,
upload_type: str,
note: str,
interface_seq_id: str,
methods=['POST']):
"""
上传文件(单文件上传) (/api/Qa/UploadFiles/<upload_type>/<note>/<interface_seq_id>)
@param {str} upload_type - 文件类型，必须在UploadFileConfig表中有配置
@param {str} note - 文件注解
@param {str} interface_seq_id - 客户端序号，客户端可传入该值来支持异步调用
@return {str} - 返回回答的json字符串
status : 处理状态
00000 - 成功, 返回一条回答
10001 - 没有指定上传文件
2XXXX - 处理失败
msg : 处理状态对应的描述
answer_type: 'text'或'json'，指示返回的答案是文本数组，还是一个json对象
answers : 回答内容
url : 文件上传后的url，含文件名和url路径
"""
_ret_json = {
'interface_seq_id': interface_seq_id,
'status': '00000',
'msg': 'success',
'answer_type': 'text',
'answers': [],
'url': ''
}
_qa_loader = RunTool.get_global_var('QA_LOADER')
try:
if 'file' not in request.files or request.files[
'file'].filename == '':
_ret_json['status'] = '10001'
_ret_json['msg'] = 'No file upload!'
return jsonify(_ret_json)
# 获取上传类型配置
_upload_config = UploadFileConfig.get_or_none(
UploadFileConfig.upload_type == upload_type)
if _upload_config is None:
_ret_json['status'] = '10002'
_ret_json['msg'] = 'upload type not exists!'
return jsonify(_ret_json)
# 检查文件大小
if _upload_config.size > 0:
if request.content_length > _upload_config.size * 1024 * 1024:
_ret_json['status'] = '10003'
_ret_json['msg'] = 'upload file size to large!'
return jsonify(_ret_json)
# 检查文件类型是否支持
_file = request.files['file']
_old_filename = _file.filename
_file_ext = FileTool.get_file_ext(_old_filename)
_allow_ext = eval(_upload_config.exts.upper())
if len(_allow_ext) > 0 and _file_ext.upper() not in _allow_ext:
_ret_json['status'] = '10004'
_ret_json['msg'] = 'Type [%s] not allow upload [.%s] file!' % (
upload_type, _file_ext)
return jsonify(_ret_json)
# 处理新的文件名
def _replace_var_fun(m):
_match_str = m.group(0)
_value = None
if _match_str.startswith('{$datetime='):
# 按格式化字符替换当前的时间
_key = _match_str[11:-2]
_value = datetime.datetime.now().strftime(_key)
elif _match_str.startswith('{$uuid='):
# 指定uuid字符类型
_key = _match_str[7:-2]
str(uuid.uuid1())
_value = eval('str(uuid.uuid%s())' % _key)
elif _match_str.startswith('{$random'):
# 产生指定两个整数之间的随机数，总位数与最大的数一致，左补零
_key = _match_str[8:-2]
_args = eval('(%s)' % _key)
_value = StringTool.fill_fix_string(
str(random.randint(*_args)), len(_args[1]), '0')
elif _match_str.startswith('{$file_ext='):
# 原文件扩展名
_value = _file_ext
elif _match_str.startswith('{$file_name='):
# 原文件指定位置的字符
_key = _match_str[12:-2]
_args = eval('(%s)' % _key)
if len(_args) > 1:
_value = _old_filename[_args[0]:_args[1]]
else:
_value = _old_filename[_args[0]:]
if _value is not None:
return str(_value)
else:
return _match_str
if _upload_config.rename == '':
_new_filename = _old_filename
else:
_new_filename = re.sub(r'\{\$.+?\$\}', _replace_var_fun,
_upload_config.rename, re.M)
# 处理保存文件路径和url路径
if _upload_config.url != '':
_ret_json['url'] = '%s/%s' % (_upload_config.url,
_new_filename)
_save_path = os.path.realpath(
os.path.join(_qa_loader.execute_path, _upload_config.save_path,
_new_filename))
# 创建文件目录
FileTool.create_dir(os.path.split(_save_path)[0], exist_ok=True)
# 保存文件
_file.save(_save_path)
# 上传后处理
_after = eval(_upload_config.after)
if len(_after) > 0:
_after_fun = _qa_loader.plugins['upload_after'][_after[0]][
_after[1]]
_status, _msg, _answers = _after_fun(upload_type, note,
_new_filename, _save_path,
_ret_json['url'],
**_after[2])
_ret_json['status'] = _status
_ret_json['msg'] = _msg
if len(_answers) > 0 and type(_answers[0]) == dict:
_ret_json['answer_type'] = 'json'
_ret_json['answers'] = _answers[0]
else:
_ret_json['answers'] = _answers
if _ret_json['status'] != '00000':
# 后处理失败，删除文件
FileTool.remove_file(_save_path)
if _qa_loader.logger:
_qa_loader.logger.debug(
'remove upload file [dest:%s][source:%s] when after deal error[%s]: %s'
% (_new_filename, _old_filename, _status, _msg))
except:
if _qa_loader.logger:
_qa_loader.logger.error('Exception: %s' %
traceback.format_exc(),
extra={'callFunLevel': 1})
_ret_json = {
'interface_seq_id': interface_seq_id,
'status': '20001',
'msg': '上传文件异常'
}
return jsonify(_ret_json)
def labelimg_pic_deal(cls, path: str):
"""
TFRecord图片兼容处理
1.删除位深不为RGB三通道的图片
（解决image_size must contain 3 elements[4]报错）
2.转换图片格式为jpg
3.检查xml文件的文件名和路径是否正确
@param {str} path - 要处理的路径
"""
_path = os.path.realpath(path)
# 遍历所有子目录
_sub_dirs = FileTool.get_dirlist(path=_path, is_fullpath=True)
for _dir in _sub_dirs:
# 递归删除子目录的信息
cls.labelimg_pic_deal(_dir)
# 检查自己目录下的图片
_files = FileTool.get_filelist(path=_path, is_fullname=False)
for _file in _files:
_file_ext = FileTool.get_file_ext(_file)
if _file_ext == 'xml':
# 标签文件不处理
continue
_img_file = os.path.join(_path, _file)
_file_no_ext = FileTool.get_file_name_no_ext(_file)
if _file_ext in ('png', 'gif'):
# 转换图片格式
_fp = open(_img_file, 'rb')
_img = Image.open(_fp)
_rgb_im = _img.convert('RGB')
_rgb_im.save(os.path.join(_path, _file_no_ext + '.jpg'))
_fp.close()
# 删除原文件，修改xml中的文件名
FileTool.remove_file(_img_file)
_xml_file = os.path.join(_path, _file_no_ext + '.xml')
if os.path.exists(_xml_file):
_tree = ET.parse(os.path.join(_path, _xml_file))
_root = _tree.getroot()
_root.find('filename').text = _file_no_ext + '.jpg'
_root.find('path').text = os.path.join(
_path, _file_no_ext + '.jpg')
_tree.write(os.path.join(_path, _xml_file),
encoding='utf-8',
method="xml",
xml_declaration=None)
# 修改文件名变量
_img_file = os.path.join(_path, _file_no_ext + '.jpg')
# 打开图片判断位深
_fp = open(_img_file, 'rb')
_img = Image.open(_fp)
if _img.mode != 'RGB':
# 需要删除掉
_fp.close()
_xml_file = os.path.join(
_path,
FileTool.get_file_name_no_ext(_file) + '.xml')
print('delete %s' % _img_file)
FileTool.remove_file(_img_file)
if os.path.exists(_xml_file):
FileTool.remove_file(_xml_file)
else:
_fp.close()
# 检查xml文件
_xml_file = os.path.join(_path, _file_no_ext + '.xml')
if os.path.exists(_xml_file):
_tree = ET.parse(os.path.join(_path, _xml_file))
_root = _tree.getroot()
if _root.find('filename'
).text != _file_no_ext + '.jpg' or os.path.split(
_root.find('path').text)[0] != _path:
_root.find('filename').text = _file_no_ext + '.jpg'
_root.find('path').text = os.path.join(
_path, _file_no_ext + '.jpg')
_tree.write(os.path.join(_path, _xml_file),
encoding='utf-8',
method="xml",
xml_declaration=None)
def __init__(self,
file: str,
is_resume: bool = True,
file_size: int = None,
md5: str = None,
is_overwrite: bool = False,
temp_ext: str = 'tmp',
info_ext: str = 'info',
extend_info: dict = None,
thread_num: int = 1,
block_size: int = 4096,
cache_size: int = 1024,
auto_expand: bool = True):
"""
初始化文件保存对象
@param {str} file - 文件保存路径（含文件名）
@param {bool} is_resume=True - 指定是否续传（自动查找已下载的信息）, 如果不指定续传将自动删除原来已下载临时文件
注：如果指定续传，且可以找到原来的临时文件，则以下参数将使用原来的信息，如果有传入则会进行差异值的校验：
file_size、md5
@param {int} file_size=None - 文件大小，单位为byte, 如果为None代表未知文件大小, 此时auto_expand参数固定为True
@param {str} md5=None - 验证文件的md5字符串，如果不传代表不进行验证
@param {bool} is_overwrite=False - 是否覆盖已有文件，如果为否，则目标文件已存在的情况下抛出异常
@param {str} temp_ext='tmp' - 处理过程中临时文件扩展名
@param {str} info_ext='info' - 处理过程中信息文件扩展名
@param {dict} extend_info=None - 处理过程中要保存的信息字典，例如保存文件下载路径，引用页等信息
@param {int} thread_num=1 - 写入处理线程数量
@param {int} block_size=4096 - 每次写入块大小，单位为byte
@param {int} cache_size=1024 - 单线程缓存大小，单位为kb（注意：真实缓存大小还需要乘以处理线程数量）
@param {bool} auto_expand=True - 是否自动扩展文件大小（否则在初始化时会自动创建指定大小的文件）
@throws {FileExistsError} - 如果下载文件已存在且不允许覆盖的情况抛出异常
@throws {FileNotFoundError} - 续传情况下临时文件不存在则抛出异常
@throws {InfoFileLockError} - 如果已打开信息文件进行文件存储处理，抛出该异常
"""
# 检查文件是否存在
self._file = os.path.abspath(file)
self._path, self._filename = os.path.split(self._file)
if os.path.exists(self._file):
# 文件已存在
if is_overwrite:
FileTool.remove_file(self._file)
else:
raise FileExistsError('file exists: %s' % self._file)
else:
# 创建目录
FileTool.create_dir(self._path, exist_ok=True)
# 文件信息字典，该字典登记文件基本信息和写入情况
self._info: dict = None
# 锁文件，控制一个文件不能被多个类处理, 先尝试创建锁文件，如果创建失败会抛出异常
self._lock_file = os.path.join(self._path,
'%s.%s' % (self._filename, 'lock'))
try:
self._lock_file_handle = os.open(
self._lock_file, os.O_CREAT | os.O_EXCL | os.O_RDWR)
except:
raise InfoFileLockError('info file is locked')
try:
# 获取是否debug状态
self._debug_on = DebugTool.is_debug_on()
self._lock_print_timeout = None
if self._debug_on:
self._lock_print_timeout = 5.0 # 打印锁等待超时时间
# 处理信息字典、临时文件、信息文件
self._temp_file = os.path.join(
self._path, '%s.%s' % (self._filename, temp_ext))
self._info_file = os.path.join(
self._path, '%s.%s' % (self._filename, info_ext))
self._auto_expand = auto_expand
self._thread_num = thread_num
self._block_size = block_size
self._cache_size = cache_size * 1024
# 数据处理锁
self._cache_info_lock = threading.RLock() # 缓存信息更新锁
self._tmp_file_lock = threading.RLock() # 缓存文件写入锁
self._is_finished = False # 要控制的完成状态
self._dealed_finished_lock = threading.RLock() # 控制多线程操作结束函数的状态更新锁
self._dealed_finished = False # 控制多线程操作结束函数只执行一次的变量
if is_resume and os.path.exists(self._info_file):
# 自动续传情况
self._info_file_handle = open(self._info_file,
'r+',
encoding='utf-8')
self._info_file_handle.seek(0)
self._info = json.loads(self._info_file_handle.read())
# 检查传入信息是否一致
if file_size is not None and file_size != self._info[
'file_size']:
raise AttributeError(
'resume info [file_size] inconsistency, info file [%s], now [%s]'
% (str(self._info['file_size']), str(file_size)))
if md5 is not None and md5 != self._info['md5']:
raise AttributeError(
'resume info [md5] inconsistency, info file [%s], now [%s]'
% (self._info['md5'], md5))
# 检查临时文件
self._temp_file = os.path.join(self._path,
self._info['tmp_file'])
if not os.path.exists(self._temp_file):
# 临时文件不存在
raise FileNotFoundError('temp file is not found: %s' %
self._temp_file)
self._tmp_file_handle = open(self._temp_file, 'rb+')
self._tmp_file_handle.seek(0)
else:
# 删除已存在的临时文件信息
if os.path.exists(self._temp_file):
FileTool.remove_file(self._temp_file)
if os.path.exists(self._info_file):
FileTool.remove_file(self._info_file)
# 形成信息字典
self._info = {
'tmp_file': '%s.%s' % (self._filename, temp_ext), # 临时文件名称
'file_size':
-1 if file_size is None else file_size, # 文件大小
'write_size': 0, # 已写入数据大小
'md5': '' if md5 is None else md5, # md5校验值
'extend_info':
{} if extend_info is None else extend_info, # 传入的扩展信息
# 存储索引，按位置顺序在数组中登记未写入区间，数组每一项登记未写入数据的开始位置和结束位置
'store_index': [[0, file_size - 1]]
}
# 生成临时文件
self._tmp_file_handle = open(self._temp_file, 'wb')
if not auto_expand and file_size is not None:
# 直接生成指定大小的文件
self._tmp_file_handle.seek(file_size - 1) # 跳到指定位置
self._tmp_file_handle.write(b'\x00') # 一定要写入一个字符，否则无效
self._tmp_file_handle.flush()
# 写入信息字典文件
self._info_file_handle = open(self._info_file,
'w',
encoding='utf-8')
self._write_info_file()
# 合并存储索引，把碎片合并成为大块
self._info['store_index'] = self._f_merge_store_index(
self._info['store_index'])
# 初始化缓存等信息
if self._info['file_size'] == -1:
# 如果没有文件大小的情况，不支持拆分多写入线程和一次性创建指定大小文件的情况
self._thread_num = 1
self._auto_expand = True
# 缓存处理
self._max_cache_pos = [
-1,
] # 当前缓存分配到的区域最大位置
self._cache = dict()
for _i in range(self._thread_num):
self._cache[_i] = {
'start': -1, # 缓存数据对应文件的写入位置, -1代表没有设置
'size': 0, # 缓存数据大小
'buffer': bytes(), # 具体的缓存数据
'end_pos': -1, # 该缓存对应线程要处理的文件块结束位置
'lock': threading.RLock(), # 用于缓存线程处理的锁）
'get_start': -1, # 当前正在获取的数据的开始位置
'get_size': 0, # 当前要获取数据的大小
}
# 分配每个缓存要处理文件区域
for _i in range(self._thread_num):
self._set_cache_area(_i)
except:
# 如果初始化出现异常，清理文件句柄及锁文件
self._clear_file_handle_and_lock()
raise
def test_grpc_to_local(self):
if not TEST_FLAG['test_grpc_to_local']:
return
print('测试获取gRpc远程文件')
# DebugTool.set_debug(set_on=True)
_copy_file = os.path.join(_temp_path, 'grpc_to_local.bin')
# 删除临时文件
if os.path.exists(_copy_file):
FileTool.remove_file(_copy_file)
for _ext in ('.lock', '.tmp', '.info'):
if os.path.exists(_copy_file + _ext):
FileTool.remove_file(_copy_file + _ext)
# 连接参数
_connect_para = SimpleGRpcConnection.generate_connect_para(
conn_str='127.0.0.1:50051')
# 单线程-缓存大于文件大小
_tips = '单线程-缓存大于文件大小'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 单线程-缓存小于文件大小
_tips = '单线程-缓存小于文件大小'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 单线程-暂停重复
_tips = '单线程-暂停重复'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.1)
_status = _reader.start()
time.sleep(1)
_reader.stop()
self.assertTrue(_reader.status == 'stop',
msg="本地gRpc推送-%s(暂停): %s" %
(_tips, _reader.status))
_reader.thread_interval = 0.0
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 单线程-停止后续传
_tips = '单线程-停止后续传'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.1)
_status = _reader.start()
time.sleep(1)
_reader.stop()
self.assertTrue(_reader.status == 'stop',
msg="本地gRpc推送-%s(暂停): %s" % (_tips, _reader.status))
# 续传处理
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 多线程
_tips = '多线程'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
thread_num=230,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 多线程-缓存大于文件大小
_tips = '多线程-缓存大于文件大小'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
thread_num=5,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 多线程-每次传输块大小大于文件
_tips = '多线程-每次传输块大小大于文件'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
thread_num=5,
block_size=900000,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 多线程-暂停重复
_tips = '多线程-暂停重复'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
thread_num=5,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.3)
_status = _reader.start()
time.sleep(1)
_reader.stop()
self.assertTrue(_reader.status == 'stop',
msg="本地gRpc推送-%s(暂停): %s" %
(_tips, _reader.status))
_reader.thread_interval = 0.0
_status = _reader.start(wait_finished=True)
if _status != 'finished':
print(NetTool.get_file_md5(_temp_file))
print(NetTool.get_file_md5(_copy_file + '.tmp'))
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
# 多线程-停止后续传
_tips = '多线程-停止后续传'
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
thread_num=5,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.3)
_status = _reader.start()
time.sleep(1)
_reader.stop()
self.assertTrue(_reader.status == 'stop',
msg="本地gRpc推送-%s(暂停): %s" % (_tips, _reader.status))
# 续传处理
with GRpcPullProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
thread_num=5,
block_size=40960,
connect_para=_connect_para) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
if _status != 'finished':
print(NetTool.get_file_md5(_temp_file))
print(NetTool.get_file_md5(_copy_file + '.tmp'))
self.assertTrue(_status == 'finished',
msg="本地gRpc推送-%s: %s" % (_tips, _status))
def test_local_to_local(self):
if not TEST_FLAG['test_local_to_local']:
return
print('测试本地文件复制')
# DebugTool.set_debug(set_on=True)
_copy_file = os.path.join(_temp_path, 'local_to_local_copy.bin')
# 删除临时文件
if os.path.exists(_copy_file):
FileTool.remove_file(_copy_file)
for _ext in ('.lock', '.tmp', '.info'):
if os.path.exists(_copy_file + _ext):
FileTool.remove_file(_copy_file + _ext)
# 单线程-缓存大于文件大小
_tips = '单线程-缓存大于文件大小'
with LocalProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地文件复制-%s: %s" % (_tips, _status))
# 单线程-每次传输块大小大于文件
_tips = '单线程-每次传输块大小大于文件'
with LocalProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
block_size=600000,
auto_expand=False) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地文件复制-%s: %s" % (_tips, _status))
# 单线程-缓存小于文件大小
_tips = '单线程-缓存小于文件大小'
with LocalProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地文件复制-%s: %s" % (_tips, _status))
# 单线程-暂停重复
_tips = '单线程-暂停重复'
with LocalProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.1)
_status = _reader.start()
time.sleep(1)
_reader.stop()
self.assertTrue(_reader.status == 'stop',
msg="本地文件复制-%s(暂停): %s" % (_tips, _reader.status))
_reader.thread_interval = 0.0
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地文件复制-%s: %s" % (_tips, _status))
# 单线程-停止后续传
_tips = '单线程-停止后续传'
with LocalProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.1)
_status = _reader.start()
time.sleep(1)
_reader.stop()
self.assertTrue(_reader.status == 'stop',
msg="本地文件复制-%s(暂停): %s" % (_tips, _reader.status))
# 续传处理
with LocalProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地文件复制-%s: %s" % (_tips, _status))
# 多线程
_tips = '多线程'
with LocalProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
thread_num=200) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地文件复制-%s: %s" % (_tips, _status))
# 多线程-缓存大于文件大小
_tips = '多线程-缓存大于文件大小'
with LocalProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
thread_num=5) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地文件复制-%s: %s" % (_tips, _status))
# 多线程-每次传输块大小大于文件
_tips = '多线程-每次传输块大小大于文件'
with LocalProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
thread_num=5,
block_size=600000) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
self.assertTrue(_status == 'finished',
msg="本地文件复制-%s: %s" % (_tips, _status))
# 多线程-暂停重复
_tips = '多线程-暂停重复'
with LocalProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
thread_num=5) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.3)
_status = _reader.start()
time.sleep(1)
_reader.stop()
self.assertTrue(_reader.status == 'stop',
msg="本地文件复制-%s(暂停): %s" % (_tips, _reader.status))
_reader.thread_interval = 0.0
_status = _reader.start(wait_finished=True)
if _status != 'finished':
print(NetTool.get_file_md5(_temp_file))
print(NetTool.get_file_md5(_copy_file + '.tmp'))
self.assertTrue(_status == 'finished',
msg="本地文件复制-%s: %s" % (_tips, _status))
# 多线程-停止后续传
_tips = '多线程-停止后续传'
with LocalProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
thread_num=5) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.3)
_status = _reader.start()
time.sleep(1)
_reader.stop()
self.assertTrue(_reader.status == 'stop',
msg="本地文件复制-%s(暂停): %s" % (_tips, _reader.status))
# 续传处理
with LocalProtocol(_temp_file,
_copy_file,
is_resume=True,
is_overwrite=True,
cache_size=2,
thread_num=5) as _protocol:
_reader = Transfer(
_protocol,
show_process_bar_fun=ProgressRate.show_cmd_process_bar,
process_bar_label=_tips,
thread_interval=0.0)
_status = _reader.start(wait_finished=True)
if _status != 'finished':
print(NetTool.get_file_md5(_temp_file))
print(NetTool.get_file_md5(_copy_file + '.tmp'))
print(_status)
self.assertTrue(_status == 'finished',
msg="本地文件复制-%s: %s" % (_tips, _status))
def _clean_file_path(cls, path: str, class_path: str):
"""
清理当前目录文件
@param {str} path - 要处理的目录地址
@param {str} class_path - 类目录
"""
# 处理自身目录，先获取商品信息
_info = dict()
_info_file = os.path.join(path, 'info.json')
if os.path.exists(_info_file):
with open(_info_file, 'rb') as f:
_eval = str(f.read(), encoding='utf-8')
_info = eval(_eval)
# 判断是否不处理
_shop_name = _info['店名']
# if _info['款式'] == '挂件' and _info['挂件类型'] == '':
# return
# 遍历文件进行处理
_product_num = FileTool.get_dir_name(path)
_files = FileTool.get_filelist(path)
_order = 1
for _file in _files:
_file_ext = FileTool.get_file_ext(_file).lower()
if _file_ext not in ['jpg', 'jpeg', 'png', 'bmp']:
# 不是合适的文件类型
continue
# 判断是否有括号
if _file.find('(') >= 0:
FileTool.remove_file(_file)
continue
# 判断是否匹配上要删除的图片大小
if _shop_name in DEL_SHOP_PIC_SIZE.keys() and os.path.getsize(
_file) in DEL_SHOP_PIC_SIZE[_shop_name]:
FileTool.remove_file(_file)
continue
# 修改文件名
if not FileTool.get_file_name(_file).startswith(_product_num):
os.rename(
_file,
os.path.join(
path, '%s_%s_%d.%s' %
(_product_num, 'main' if _file.find('主图') >= 0
or _file.find('main') >= 0 else 'detail', _order,
_file_ext)))
# 下一个文件
_order += 1
# 移动文件夹到指定的分类目录
_class_path = _info['款式']
if _class_path in PROP_TYPE_TRAN_DICT.keys():
_class_path = PROP_TYPE_TRAN_DICT[_info['款式']]
shutil.move(path,
os.path.join(class_path, _class_path, _product_num))
# 处理完成，返回
return
def test_case1():
# 测试单日志最基本功能，日志输出，变更日志级别，修改日志格式
# 删除临时日志
try:
FileTool.remove_file(_TEMP_DIR + '/test_case1.json')
print('del %s%s' % (_TEMP_DIR, '/test_case1.json'))
except:
pass
try:
FileTool.remove_files(path=_TEMP_DIR + '/log/', regex_str='test_case1*')
except:
pass
_logger = simple_log.Logger(conf_file_name=_TEMP_DIR + '/test_case1.json',
logger_name=simple_log.EnumLoggerName.ConsoleAndFile,
config_type=simple_log.EnumLoggerConfigType.JSON_FILE,
logfile_path=_TEMP_DIR + '/log/test_case1.log')
# ConsoleAndFile 的配置level为DEBUG，但对应的ConsoleHandler的level为DEBUG，FileHandler的level为INFO
# 日志是否显示会根据logger的level和handler的level，以级别比较高的匹配输出
# 注意默认root的handler应该为空，否则无论如何都会执行root的输出，如果自己又另外指定了输出，那就会有2个相同输出日志
_logger.log(simple_log.DEBUG,
'test_case1:write_log:DEBUG:1:界面应显示本日志，文件不应显示本日志')
_logger.debug('test_case1:write_log:DEBUG:1-1:界面应显示本日志，文件不应显示本日志')
_logger.log(simple_log.INFO,
'test_case1:write_log:INFO:2:界面应显示本日志，文件应显示本日志')
_logger.info('test_case1:write_log:INFO:2-1:界面应显示本日志，文件应显示本日志')
# 修改ConsoleAndFile的level为INFO，handler仍不变
_logger.setLevel(simple_log.INFO)
_logger.log(simple_log.DEBUG,
'test_case1:write_log:DEBUG:3:界面不应显示本日志，文件不应显示本日志')
_logger.log(simple_log.INFO,
'test_case1:write_log:INFO:4:界面应显示本日志，文件应显示本日志')
# 修改ConsoleAndFile的level为DEBUG, FileHandler的level为WARNING
_logger.setLevel(simple_log.DEBUG)
for _handler in _logger.base_logger.handlers:
if _handler.name == 'FileHandler':
_logger.set_handler_log_level(_handler, simple_log.WARNING)
_logger.log(simple_log.DEBUG,
'test_case1:write_log:DEBUG:5:界面应显示本日志，文件不应显示本日志')
_logger.log(simple_log.WARNING,
'test_case1:write_log:WARNING:6:界面应显示本日志，文件应显示本日志')
# 修改整个日志级别为INFO
_logger.setLevel(simple_log.INFO)
_logger.log(simple_log.DEBUG,
'test_case1:write_log:DEBUG:7:界面不应显示本日志，文件不应显示本日志')
_logger.log(simple_log.INFO,
'test_case1:write_log:INFO:8:界面应显示本日志，文件应显示本日志')
# 修改日志类型为Console，日志级别应根据配置文件恢复原状态（DEBUG）
_logger.change_logger_name(logger_name=simple_log.EnumLoggerName.Console)
_logger.log(simple_log.DEBUG,
'test_case1:write_log:DEBUG:9:界面应显示本日志，文件不应显示本日志')
# 修改日志输出格式
_logger.set_logger_formater(format_str='[%(asctime)s]%(message)s')
_logger.log(simple_log.DEBUG,
'test_case1:write_log:DEBUG:9:格式发生变化，界面应显示本日志，文件不应显示本日志')
del _logger
