def prepare_env_and_get_db_filepath(self) -> str:
"""
逻辑说明
假设key的md5为md5
本地缓存文件路径为.db/md5{0:3}/md5.json
文件内容为val_type的实例的json序列化结果
:return: 是否是首次运行
"""
from util import make_sure_dir_exists
db_dir = ""
db_file = ""
if self.db_filepath != "":
db_dir = os.path.dirname(self.db_filepath)
db_file = self.db_filepath
else:
key_md5 = self.get_db_filename()
db_dir = os.path.join(db_top_dir, key_md5[0:3])
db_file = os.path.join(db_dir, key_md5)
make_sure_dir_exists(db_dir)
return db_file
def upload_to_lanzouyun(self,
filepath,
target_folder,
history_file_prefix="",
also_upload_compressed_version=False,
only_upload_compressed_version=False) -> bool:
if not only_upload_compressed_version:
ok = self._upload_to_lanzouyun(filepath, target_folder,
history_file_prefix)
if not ok:
return False
if also_upload_compressed_version:
make_sure_dir_exists('.cached')
filename = os.path.basename(filepath)
compressed_filepath = os.path.join(
'.cached', self.get_compressed_version_filename(filename))
compressed_history_file_prefix = f"{self.compressed_version_prefix}{history_file_prefix}"
logger.info(
color("bold_green") + f"创建压缩版本并上传 {compressed_filepath}")
# 创建压缩版本
with open(f"{filepath}", "rb") as file_in:
with lzma.open(f"{compressed_filepath}", "wb") as file_out:
file_out.writelines(file_in)
# 上传
return self._upload_to_lanzouyun(compressed_filepath,
target_folder,
compressed_history_file_prefix)
return True
def __init__(self, novel: Novel, ext: str, out_path: str = 'out'):
super(Output, self).__init__()
if not novel.success:
self.log.warning("Novel wasn't parsed successfully.")
self.novel = novel
self.ext = ext
self.out_path = out_path
self.slug_title = slugify(novel.title, lowercase=False)
self.filename = f"{self.slug_title}.{self.ext}"
self.path = os.path.join(out_path, self.novel.url.hostname)
make_sure_dir_exists(self.path)
def download_ide_act_desc_json(actId: str) -> str:
first_two = str(actId[:2])
act_cache_dir = f"{cached_dir}/ideActDesc/{first_two}"
act_cache_file = f"{act_cache_dir}/{actId}.json"
# 然后从服务器获取活动信息
url = f"https://comm.ams.game.qq.com/ide/page/{actId}"
res = requests.get(url, timeout=1)
if res.status_code != 200:
return ""
make_sure_dir_exists(act_cache_dir)
with open(act_cache_file, "w", encoding="utf-8") as f:
f.write(res.text)
return act_cache_file
def test():
import json
from util import make_sure_dir_exists
dir_src = os.getcwd()
test_root_dir = realpath("test/compress")
make_sure_dir_exists(test_root_dir)
os.chdir(test_root_dir)
# 测试文件解压缩
test_file_name = "test_file.json"
test_compressed_file_name = test_file_name + ".7z"
test_decompressed_file_name = test_compressed_file_name + ".json"
with open(test_file_name, "w", encoding="utf-8") as f:
json.dump("test_file_compress", f)
compress_file_with_lzma(test_file_name, test_compressed_file_name)
decompress_file_with_lzma(test_compressed_file_name,
test_decompressed_file_name)
# 测试目录解压缩
test_folder_name = "test_folder"
test_compressed_folder_name = test_folder_name + ".7z"
make_sure_dir_exists(test_folder_name)
with open(os.path.join(test_folder_name, "test.json"),
"w",
encoding="utf-8") as f:
json.dump("test_folder_compress", f)
compress_dir_with_bandizip(test_folder_name, test_compressed_folder_name,
dir_src)
shutil.rmtree(test_folder_name)
decompress_dir_with_bandizip(test_compressed_folder_name, dir_src)
shutil.rmtree(test_root_dir, ignore_errors=True)
# 测试内存内解压缩
test_text = "this is a test 测试内容 压缩前"
compressed_bytes = compress_in_memory_with_lzma(test_text.encode())
decompressed = decompress_in_memory_with_lzma(compressed_bytes).decode()
print(compressed_bytes)
print(decompressed)
print(test_text == decompressed)
def download_file(url: str,
download_dir=".cached/downloads",
filename="",
connect_timeout=10) -> str:
download_dir = os.path.realpath(download_dir)
filename = filename or os.path.basename(url)
start_time = get_now()
target_file_path = os.path.join(download_dir, filename)
logger.info(f"开始下载 {url} 到 {target_file_path}")
response = requests.get(url, stream=True, timeout=connect_timeout)
if response.status_code != 200:
raise Exception(f"下载失败，状态码 {response.status_code}")
make_sure_dir_exists(download_dir)
with open(target_file_path, "wb") as f:
total_length_optional = response.headers.get("content-length")
if total_length_optional is None: # no content length header
f.write(response.content)
else:
dl = 0
total_length = int(total_length_optional)
for data in response.iter_content(chunk_size=4096):
# filter out keep-alive new lines
if not data:
continue
f.write(data)
dl += len(data)
show_progress(filename, total_length, dl)
end_time = get_now()
logger.info(color("bold_yellow") + f"下载完成，耗时 {end_time - start_time}")
return target_file_path
def download_act_desc_js(actId: str) -> str:
last_three = str(actId[-3:])
act_cache_dir = f"{cached_dir}/actDesc/{last_three}/{actId}"
act_cache_file = f"{act_cache_dir}/act.desc.js"
# 然后从服务器获取活动信息
actUrls = [
f"https://dnf.qq.com/comm-htdocs/js/ams/actDesc/{last_three}/{actId}/act.desc.js",
f"https://apps.game.qq.com/comm-htdocs/js/ams/actDesc/{last_three}/{actId}/act.desc.js",
f"https://apps.game.qq.com/comm-htdocs/js/ams/v0.2R02/act/{actId}/act.desc.js",
]
for url in actUrls:
res = requests.get(url, timeout=1)
if res.status_code != 200:
continue
make_sure_dir_exists(act_cache_dir)
with open(act_cache_file, "w", encoding="utf-8") as f:
f.write(res.text)
return act_cache_file
return ""
def upload_to_lanzouyun(self,
filepath: str,
target_folder: Folder,
history_file_prefix="",
also_upload_compressed_version=False,
only_upload_compressed_version=False) -> bool:
if not self.login_ok:
logger.info("未登录，不能上传文件")
return False
if history_file_prefix == "":
# 未设置历史文件前缀，默认为当前文件名
history_file_prefix = os.path.basename(filepath)
if not only_upload_compressed_version:
ok = self._upload_to_lanzouyun(filepath, target_folder,
history_file_prefix)
if not ok:
return False
if also_upload_compressed_version:
make_sure_dir_exists('.cached')
filename = os.path.basename(filepath)
compressed_filepath = os.path.join(
'.cached', self.get_compressed_version_filename(filename))
compressed_history_file_prefix = f"{self.compressed_version_prefix}{history_file_prefix}"
logger.info(
color("bold_green") + f"创建压缩版本并上传 {compressed_filepath}")
# 创建压缩版本
compress_file_with_lzma(filepath, compressed_filepath)
# 上传
return self._upload_to_lanzouyun(compressed_filepath,
target_folder,
compressed_history_file_prefix)
return True
def test():
from util import make_sure_dir_exists
import json
dir_src = os.getcwd()
test_root_dir = realpath("test/compress")
make_sure_dir_exists(test_root_dir)
os.chdir(test_root_dir)
# 测试文件解压缩
test_file_name = "test_file.json"
test_compressed_file_name = test_file_name + ".7z"
test_decompressed_file_name = test_compressed_file_name + ".json"
with open(test_file_name, 'w', encoding='utf-8') as f:
json.dump("test_file_compress", f)
compress_file_with_lzma(test_file_name, test_compressed_file_name)
decompress_file_with_lzma(test_compressed_file_name,
test_decompressed_file_name)
# 测试目录解压缩
test_folder_name = "test_folder"
test_compressed_folder_name = test_folder_name + ".7z"
make_sure_dir_exists(test_folder_name)
with open(os.path.join(test_folder_name, "test.json"),
'w',
encoding='utf-8') as f:
json.dump("test_folder_compress", f)
compress_dir_with_bandizip(test_folder_name, test_compressed_folder_name,
dir_src)
shutil.rmtree(test_folder_name)
decompress_dir_with_bandizip(test_compressed_folder_name, dir_src)
shutil.rmtree(test_root_dir, ignore_errors=True)
os.system(set_title_cmd)
# 先声明一些需要用到的目录的地址
dir_src = os.path.realpath('.')
dir_all_release = os.path.realpath(os.path.join("releases"))
release_dir_name = f"DNF蚊子腿小助手_{version}_by风之凌殇"
release_7z_name = f'{release_dir_name}.7z'
dir_github_action_artifact = "_github_action_artifact"
# ---------------构建
# 调用构建脚本
os.chdir(dir_src)
build()
# ---------------清除一些历史数据
make_sure_dir_exists(dir_all_release)
os.chdir(dir_all_release)
clear_github_artifact(dir_all_release, dir_github_action_artifact)
# ---------------打包
os.chdir(dir_src)
package(dir_src, dir_all_release, release_dir_name, release_7z_name,
dir_github_action_artifact)
# ---------------构建增量补丁
create_patch_for_latest_n_version = 3
# ---------------构建增量包
os.chdir(dir_all_release)
show_head_line(f"开始构建增量包，最多包含过去{create_patch_for_latest_n_version}个版本到最新版本的补丁",
color("bold_yellow"))
def release():
# ---------------准备工作
prompt = f"如需直接使用默认版本号：{now_version} 请直接按回车\n或手动输入版本号后按回车："
version = input(prompt) or now_version
version_reg = r"\d+\.\d+\.\d+"
if re.match(version_reg, version) is None:
logger.info(f"版本号格式有误，正确的格式类似：1.0.0 ，而不是 {version}")
pause_and_exit(-1)
# 最大化窗口
change_console_window_mode_async(disable_min_console=True)
version = "v" + version
run_start_time = datetime.now()
show_head_line(f"开始发布版本 {version}", color("bold_yellow"))
set_title_cmd = f"title 发布 {version}"
os.system(set_title_cmd)
# 先声明一些需要用到的目录的地址
dir_src = os.path.realpath(".")
dir_all_release = os.path.realpath(os.path.join("releases"))
release_dir_name = f"DNF蚊子腿小助手_{version}_by风之凌殇"
release_7z_name = f"{release_dir_name}.7z"
dir_github_action_artifact = "_github_action_artifact"
# ---------------构建
# 调用构建脚本
os.chdir(dir_src)
build()
# ---------------清除一些历史数据
make_sure_dir_exists(dir_all_release)
os.chdir(dir_all_release)
clear_github_artifact(dir_all_release, dir_github_action_artifact)
# ---------------打包
os.chdir(dir_src)
package(dir_src, dir_all_release, release_dir_name, release_7z_name, dir_github_action_artifact)
# ---------------构建增量补丁
create_patch_for_latest_n_version = 3
# ---------------构建增量包
os.chdir(dir_all_release)
show_head_line(f"开始构建增量包，最多包含过去{create_patch_for_latest_n_version}个版本到最新版本的补丁", color("bold_yellow"))
create_patch(dir_src, dir_all_release, create_patch_for_latest_n_version, dir_github_action_artifact)
# ---------------获取补丁地址（分开方便调试）
os.chdir(dir_all_release)
patch_file_name = create_patch(
dir_src,
dir_all_release,
create_patch_for_latest_n_version,
dir_github_action_artifact,
get_final_patch_path_only=True,
)
# ---------------标记新版本
show_head_line("提交版本和版本变更说明，并同步到docs目录，用于生成github pages", color("bold_yellow"))
os.chdir(dir_src)
commit_new_version()
# ---------------上传到蓝奏云
show_head_line("开始上传到蓝奏云", color("bold_yellow"))
os.chdir(dir_src)
with open("upload_cookie.json") as fp:
cookie = json.load(fp)
os.chdir(dir_all_release)
uploader = Uploader()
uploader.login(cookie)
if uploader.login_ok:
logger.info("蓝奏云登录成功，开始上传压缩包")
def path_in_src(filepath_relative_to_src: str) -> str:
return os.path.realpath(os.path.join(dir_src, filepath_relative_to_src))
realpath = os.path.realpath
upload_info_list = [
(
uploader.folder_djc_helper,
[
(realpath(release_7z_name), uploader.history_version_prefix),
(path_in_src("utils/auto_updater.exe"), ""),
(path_in_src("使用教程/使用文档.docx"), ""),
(path_in_src("使用教程/视频教程.txt"), ""),
(path_in_src("付费指引/付费指引.docx"), ""),
(path_in_src("utils/不要下载增量更新文件_这个是给自动更新工具使用的.txt"), ""),
(realpath(patch_file_name), uploader.history_patches_prefix),
],
),
(
uploader.folder_dnf_calc,
[
(realpath(release_7z_name), uploader.history_version_prefix),
],
),
]
logger.info(color("bold_green") + "具体上传列表如下：")
for upload_folder, upload_list in upload_info_list:
logger.info(color("bold_cyan") + f"\t{upload_folder.name}：")
for local_filepath, _history_file_prefix in upload_list:
logger.info(f"\t\t{local_filepath}")
logger.info("\n")
for upload_folder, upload_list in upload_info_list:
for local_filepath, history_file_prefix in reversed(upload_list):
# 逆序遍历，确保同一个网盘目录中，列在前面的最后才上传，从而在网盘显示时显示在最前方
total_try_count = 1
for try_index in range_from_one(total_try_count):
upload_ok = uploader.upload_to_lanzouyun(
local_filepath, upload_folder, history_file_prefix=history_file_prefix
)
if upload_ok:
break
logger.warning(f"第{try_index}/{total_try_count}次尝试上传{local_filepath}失败，等待一会后重试")
if try_index < total_try_count:
count_down("上传到网盘", 5 * try_index)
else:
logger.error("蓝奏云登录失败")
# ---------------推送版本到github
# 打包完成后git添加标签
os.chdir(dir_src)
show_head_line("开始推送到github", color("bold_yellow"))
push_github(version)
# ---------------结束
logger.info("+" * 40)
logger.info(color("bold_yellow") + f"{version} 发布完成，共用时{datetime.now() - run_start_time}，请检查上传至蓝奏云流程是否OK")
logger.info("+" * 40)
os.system("PAUSE")
def download_file(self,
fileinfo: FileInFolder,
download_dir: str,
overwrite=True,
show_log=True,
download_only_if_server_version_is_newer=True) -> str:
"""
下载最新版本压缩包到指定目录，并返回最终压缩包的完整路径
"""
make_sure_dir_exists(download_dir)
download_dir = os.path.realpath(download_dir)
target_path = StrWrapper(os.path.join(download_dir, fileinfo.name))
if download_only_if_server_version_is_newer and os.path.isfile(
target_path.value):
# 仅在服务器版本比本地已有文件要新的时候才重新下载
# 由于蓝奏云时间显示不精确，将其往前一分钟，避免同一文件下次检查时其蓝奏云时间显示为xx分钟前，解析后会有最多一分钟内的误差，而导致不必要的重新下载
# 比如本次是x分y秒检查并更新，下次检查时是x+6分y+10秒，此时解析蓝奏云时间得到上传时间为x分y+10秒，就会产生额外的不必要下载
server_version_upload_time = parse_time(
fileinfo.time) - timedelta(minutes=1)
local_version_last_modify_time = parse_timestamp(
os.stat(target_path.value).st_mtime)
get_log_func(
logger.info, show_log
)(f"{fileinfo.name} 本地修改时间为：{local_version_last_modify_time} 网盘版本上传时间为：{server_version_upload_time}"
)
if server_version_upload_time <= local_version_last_modify_time:
# 暂无最新版本，无需重试
get_log_func(
logger.info, show_log
)(color("bold_cyan") +
f"当前设置了对比修改时间参数，网盘中最新版本 {fileinfo.name} 上传于{server_version_upload_time}左右，在当前版本{local_version_last_modify_time}之前，无需重新下载"
)
return target_path.value
def after_downloaded(file_name):
"""下载完成后的回调函数"""
target_path.value = file_name
get_log_func(logger.info, show_log)(f"最终下载文件路径为 {file_name}")
get_log_func(logger.info, show_log)(f"即将开始下载 {target_path.value}")
callback = None
if show_log: callback = self.show_progress
retCode = self.down_file_by_url(fileinfo.url,
"",
download_dir,
callback=callback,
downloaded_handler=after_downloaded,
overwrite=overwrite)
if retCode != LanZouCloud.SUCCESS:
get_log_func(logger.error, show_log)(f"下载失败，retCode={retCode}")
if retCode == LanZouCloud.NETWORK_ERROR:
get_log_func(
logger.warning, show_log
)(color("bold_yellow") +
("蓝奏云api返回网络错误，这很可能是由于dns的问题导致的\n"
"分别尝试在浏览器中访问下列两个网页，是否一个打的开一个打不开？\n"
"https://fzls.lanzoux.com/s/djc-helper\n"
"https://fzls.lanzous.com/s/djc-helper\n"
"\n"
"如果是这样，请按照下面这个链接，修改本机的dns，使用阿里、腾讯、百度、谷歌dns中的任意一个应该都可以解决。\n"
"https://www.ypojie.com/9830.html\n"
"\n"
"如果两个都打不开，大概率是蓝奏云挂了-。-可选择忽略后面的弹框，继续运行旧版本，或者手动去QQ群或github下载最新版本"
))
raise Exception("下载失败")
return target_path.value
def build(disable_douban=False, enable_proxy=False, use_upx=True):
# 初始化相关路径变量
venv_path = ".venv"
pyinstaller_path = os.path.join(venv_path, "Scripts", "pyinstaller")
# 初始化venv和依赖
init_venv_and_requirements(".venv", "requirements.txt", disable_douban,
enable_proxy)
show_head_line(f"将使用.venv环境进行编译", color("bold_yellow"))
temp_remove_file_dir = os.path.join(".cached", "build_temp_remove_files")
site_packages_path = os.path.join(venv_path, "Lib", "site-packages")
dep_files_to_remove_during_build = {
"PyQt5/Qt5": [
"Translations",
],
"PyQt5/Qt5/bin": [
"opengl32sw.dll",
"libEGL.dll",
"libGLESV2.dll",
"Qt5Svg.dll",
"Qt5Network.dll",
"Qt5Qml.dll",
"Qt5QmlModels.dll",
"Qt5Quick.dll",
"Qt5WebSockets.dll",
"d3dcompiler_47.dll",
],
"PyQt5/Qt5/plugins": [
"iconengines/qsvgicon.dll",
"imageformats/qsvg.dll",
"imageformats/qwebp.dll",
"platforms/qwebgl.dll",
]
}
logger.info(
color("bold_green") +
f"开始编译前先尝试移动这些确定用不到的库文件到临时目录 {temp_remove_file_dir}，从而尽可能减少最终编译的大小")
for parent_directory, file_or_directory_name_list in dep_files_to_remove_during_build.items(
):
for file_or_directory_name in file_or_directory_name_list:
path = os.path.join(site_packages_path, parent_directory,
file_or_directory_name)
backup_path = os.path.join(temp_remove_file_dir, parent_directory,
file_or_directory_name)
if not os.path.exists(path):
logger.warning(f"\t{path} 不存在，将跳过")
continue
# 将文件移动到备份目录
logger.info(f"\t开始移动 {path}")
make_sure_dir_exists(os.path.dirname(backup_path))
shutil.move(path, backup_path)
# 实际编译流程
build_configs = [
("main.py", "DNF蚊子腿小助手.exe", "utils/icons/DNF蚊子腿小助手.ico", ".",
["PyQt5"], []),
("auto_updater.py", "auto_updater.exe", "", "utils", ["PyQt5"], []),
("ark_lottery_special_version.py", "DNF蚊子腿小助手_集卡特别版.exe",
"utils/icons/ark_lottery_special_version.ico", ".", ["PyQt5"], []),
("config_ui.py", "DNF蚊子腿小助手配置工具.exe", "utils/icons/config_ui.ico", ".",
[], ["--noconsole"]),
]
for idx, config in enumerate(build_configs):
prefix = f"{idx + 1}/{len(build_configs)}"
src_path, exe_name, icon_path, target_dir, exclude_modules, extra_args = config
logger.info(color("bold_yellow") + f"{prefix} 开始编译 {exe_name}")
cmd_build = [
pyinstaller_path,
'--name',
exe_name,
'-F',
src_path,
]
if icon_path != "":
cmd_build.extend(['--icon', icon_path])
for module in exclude_modules:
cmd_build.extend(['--exclude-module', module])
if use_upx:
cmd_build.extend(['--upx-dir', "utils"])
cmd_build.extend(extra_args)
logger.info(f"{prefix} 开始编译 {exe_name}，命令为：{' '.join(cmd_build)}")
subprocess.call(cmd_build)
logger.info(f"编译结束，进行善后操作")
# 复制二进制
logger.info(f"复制{exe_name}到目标目录{target_dir}")
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
target_path = os.path.join(target_dir, exe_name)
shutil.copyfile(os.path.join("dist", exe_name), target_path)
# 删除临时文件
logger.info("删除临时文件")
for directory in ["build", "dist", "__pycache__"]:
shutil.rmtree(directory, ignore_errors=True)
os.remove(f"{exe_name}.spec")
filesize = os.path.getsize(target_path)
logger.info(
color("bold_green") +
f"{prefix} 编译{exe_name}结束，最终大小为{human_readable_size(filesize)}")
logger.info(color("bold_green") + f"编译完毕将库文件移动回来 - {site_packages_path}")
for parent_directory, file_or_directory_name_list in dep_files_to_remove_during_build.items(
):
for file_or_directory_name in file_or_directory_name_list:
path = os.path.join(site_packages_path, parent_directory,
file_or_directory_name)
backup_path = os.path.join(temp_remove_file_dir, parent_directory,
file_or_directory_name)
if not os.path.exists(backup_path):
logger.warning(f"\t备份文件 {backup_path} 不存在，将跳过")
continue
# 将文件移动到备份目录
logger.info(f"开始还原备份文件/目录 {backup_path}")
make_sure_dir_exists(os.path.dirname(path))
shutil.move(backup_path, path)
logger.info("done")