def main():
# GUI窗口
gui = tkinter.Tk()
gui.withdraw()
# 原始存档文件所在路径
options = {
"initialdir": os.path.join(os.path.dirname(__file__), "video"),
"initialfile": "save.data",
"filetypes": [("data", ".data"), ("all file", "*")],
"title": "原始存档文件",
}
save_data_file_path = tkinter.filedialog.askopenfilename(**options)
if not save_data_file_path:
return
options["title"] = "临时存档文件"
temp_save_data_file_path = tkinter.filedialog.askopenfilename(**options)
if not save_data_file_path:
return
if save_data_file_path == temp_save_data_file_path:
output.print_msg("存档文件相同，无需合并")
return
# 临时存档文件所在路径
save_data = crawler.read_save_data(save_data_file_path, PRIME_KEY_INDEX)
temp_save_data = crawler.read_save_data(temp_save_data_file_path,
PRIME_KEY_INDEX)
save_data.update(temp_save_data)
temp_list = [save_data[key] for key in sorted(save_data.keys())]
file.write_file(tool.list_to_string(temp_list), save_data_file_path,
file.WRITE_FILE_TYPE_REPLACE)
def read_save_data(save_data_path, key_index, default_value_list):
result_list = {}
if not os.path.exists(path.change_path_encoding(save_data_path)):
return result_list
for single_save_data in tool.read_file(save_data_path, tool.READ_FILE_TYPE_LINE):
single_save_data = single_save_data.replace("\xef\xbb\xbf", "").replace("\n", "").replace("\r", "")
if len(single_save_data) == 0:
continue
single_save_list = single_save_data.split("\t")
if single_save_list[key_index] in result_list:
output.print_msg("存档中存在重复行 %s" % single_save_list[key_index])
tool.process_exit()
# 去除前后空格
single_save_list = map(lambda value: value.strip(), single_save_list)
# 根据default_value_list给没给字段默认值
index = 0
for default_value in default_value_list:
# _开头表示和该数组下标的值一直，如["", "_0"] 表示第1位为空时数值和第0位一致
if default_value != "" and default_value[0] == "_":
default_value = single_save_list[int(default_value.replace("_", ""))]
if len(single_save_list) <= index:
single_save_list.append(default_value)
if single_save_list[index] == "":
single_save_list[index] = default_value
index += 1
result_list[single_save_list[key_index]] = single_save_list
return result_list
def main(account_id, include_type, min_discount_percent, min_discount_price):
# 获取登录状态
try:
login_cookie = steamCommon.get_login_cookie_from_browser()
except crawler.CrawlerException, e:
output.print_msg("登录状态检测失败，原因：%s" % e.message)
raise
def resume_request():
"""
Resume thread
"""
if not thread_event.is_set():
output.print_msg("resume process")
thread_event.set()
def run(self):
header_list = {"Range": f"bytes={self.start_pos}-{self.end_pos}"}
range_size = self.end_pos - self.start_pos + 1
for retry_count in range(0, NET_CONFIG["DOWNLOAD_RETRY_COUNT"]):
response = net.request(self.file_url,
method="GET",
header_list=header_list)
if response.status == 206:
# 下载的文件和请求的文件大小不一致
if len(response.data) != range_size:
output.print_msg(
f"网络文件{self.file_url}：range {self.start_pos} - {self.end_pos}实际下载大小 {len(response.data)} 不一致"
)
time.sleep(net.NET_CONFIG["HTTP_REQUEST_RETRY_WAIT_TIME"])
else:
# 写入本地文件后退出
self.fd_handle.seek(self.start_pos)
self.fd_handle.write(response.data)
self.fd_handle.close()
break
else:
self.error_flag.append(self)
# 唤醒主线程
multi_download_thread_semaphore.release()
def pause_request():
"""
Block thread when use request()
"""
if thread_event.is_set():
output.print_msg("pause process")
thread_event.clear()
def main(account_id):
# 获取库存
try:
inventory_item_list = steamCommon.get_account_inventory(account_id)
except crawler.CrawlerException, e:
output.print_msg("获取库存失败，原因：%s" % e.message)
raise
def main(account_id):
# 获取登录状态
try:
steamCommon.init_cookie_from_browser()
except crawler.CrawlerException, e:
output.print_msg("登录状态检测失败，原因：%s" % e.message)
raise
def get_all_cookie_from_browser(browser_type, file_path):
if not os.path.exists(file_path):
output.print_msg("cookie目录：" + file_path + " 不存在")
return None
all_cookies = {}
if browser_type == 1:
for cookie_name in os.listdir(file_path):
if cookie_name.find(".txt") == -1:
continue
with open(os.path.join(file_path, cookie_name),
"r",
encoding="UTF-8") as cookie_file:
cookie_info = cookie_file.read()
for cookies in cookie_info.split("*"):
cookie_list = cookies.strip("\n").split("\n")
if len(cookie_list) < 8:
continue
cookie_domain = cookie_list[2].split("/")[0]
cookie_key = cookie_list[0]
cookie_value = cookie_list[1]
if cookie_domain not in all_cookies:
all_cookies[cookie_domain] = {}
all_cookies[cookie_domain][cookie_key] = cookie_value
elif browser_type == 2:
con = sqlite3.connect(os.path.join(file_path, "cookies.sqlite"))
cur = con.cursor()
cur.execute("SELECT host, path, name, value FROM moz_cookies")
for cookie_info in cur.fetchall():
cookie_domain = cookie_info[0]
cookie_key = cookie_info[2]
cookie_value = cookie_info[3]
if cookie_domain not in all_cookies:
all_cookies[cookie_domain] = {}
all_cookies[cookie_domain][cookie_key] = cookie_value
con.close()
elif browser_type == 3:
# chrome仅支持windows系统的解密
if platform.system() != "Windows":
return None
con = sqlite3.connect(os.path.join(file_path, "Cookies"))
cur = con.cursor()
cur.execute(
"SELECT host_key, path, name, value, encrypted_value FROM cookies")
for cookie_info in cur.fetchall():
cookie_domain = cookie_info[0]
cookie_key = cookie_info[2]
try:
cookie_value = win32crypt.CryptUnprotectData(
cookie_info[4], None, None, None, 0)[1]
except:
continue
if cookie_domain not in all_cookies:
all_cookies[cookie_domain] = {}
all_cookies[cookie_domain][cookie_key] = cookie_value.decode()
con.close()
else:
output.print_msg("不支持的浏览器类型：" + browser_type)
return None
return all_cookies
def multipart_download(self):
"""
分段下载
"""
# 先创建文件（同时删除之前下载失败，可能生成的临时文件）
with open(self.file_path, "w"):
pass
with open(self.file_path, "rb+") as file_handle:
file_no = file_handle.fileno()
end_pos = -1
while end_pos < self.content_length - 1:
start_pos = end_pos + 1
end_pos = min(
self.content_length - 1, start_pos +
NET_CONFIG["DOWNLOAD_MULTIPART_BLOCK_SIZE"] - 1)
multipart_kwargs = self.kwargs.copy()
# 分段的header信息
if "header_list" in multipart_kwargs:
header_list = multipart_kwargs["header_list"]
del multipart_kwargs["header_list"]
else:
header_list = {}
header_list["Range"] = f"bytes={start_pos}-{end_pos}"
# 创建一个副本
with os.fdopen(os.dup(file_no), "rb+", -1) as fd_handle:
for multipart_retry_count in range(
0, NET_CONFIG["DOWNLOAD_RETRY_COUNT"]):
try:
multipart_response = request(
self.file_url,
method="GET",
header_list=header_list,
connection_timeout=NET_CONFIG[
"DOWNLOAD_CONNECTION_TIMEOUT"],
read_timeout=NET_CONFIG[
"DOWNLOAD_READ_TIMEOUT"],
**multipart_kwargs)
except SystemExit:
return False
if multipart_response.status == 206:
# 下载的文件和请求的文件大小不一致
if len(multipart_response.data) != (end_pos -
start_pos + 1):
output.print_msg(
f"网络文件{self.file_url}：range {start_pos} - {end_pos}实际下载大小 {len(multipart_response.data)} 不一致"
)
time.sleep(
NET_CONFIG["HTTP_REQUEST_RETRY_WAIT_TIME"])
else:
# 写入本地文件后退出
fd_handle.seek(start_pos)
fd_handle.write(multipart_response.data)
break
else:
self.code = self.CODE_RETRY_MAX_COUNT
return False
return True
def start_download(self):
"""
主体下载逻辑
"""
# 默认读取配置
if not isinstance(self.replace_if_exist, bool):
self.replace_if_exist = DOWNLOAD_REPLACE_IF_EXIST
# 同名文件已经存在，直接返回
if not self.replace_if_exist and os.path.exists(
self.file_path) and os.path.getsize(self.file_path) > 0:
output.print_msg(f"文件{self.file_path}（{self.file_url}）已存在，跳过")
self.status = self.DOWNLOAD_SUCCEED
return
# 判断保存目录是否存在
if not path.create_dir(os.path.dirname(self.file_path)):
self.code = self.CODE_FILE_CREATE_FAILED
return
# 是否需要分段下载
self.check_auto_multipart_download()
# 下载
for retry_count in range(0, NET_CONFIG["DOWNLOAD_RETRY_COUNT"]):
if EXIT_FLAG:
self.code = self.CODE_PROCESS_EXIT
break
if not self.is_multipart_download:
# 单线程下载
if not self.single_download():
continue
else:
# 分段下载
if not self.multipart_download():
continue
# 如果没有返回文件的长度，直接下载成功
if self.content_length == 0:
self.status = self.DOWNLOAD_SUCCEED
self.code = 0
return
# 判断文件下载后的大小和response中的Content-Length是否一致
file_size = os.path.getsize(self.file_path)
if self.content_length == file_size:
self.status = self.DOWNLOAD_SUCCEED
self.code = 0
return
else:
self.code = self.CODE_FILE_SIZE_INVALID
output.print_msg(
f"本地文件{self.file_path}：{self.content_length}和网络文件{self.file_url}：{file_size}不一致"
)
time.sleep(NET_CONFIG["HTTP_REQUEST_RETRY_WAIT_TIME"])
# 删除可能出现的临时文件
path.delete_dir_or_file(self.file_path)
def get_cookie_value_from_browser(cookie_key,
file_path,
browser_type,
target_domains=""):
if not os.path.exists(file_path):
output.print_msg("cookie目录：" + file_path + " 不存在")
return None
if browser_type == BROWSER_TYPE_IE:
for cookie_name in os.listdir(file_path):
if cookie_name.find(".txt") == -1:
continue
with open(os.path.join(file_path, cookie_name),
"r") as cookie_file:
cookie_info = cookie_file.read()
for cookies in cookie_info.split("*"):
cookie_list = cookies.strip("\n").split("\n")
if len(cookie_list) < 8:
continue
domain = cookie_list[2].split("/")[0]
if _filter_domain(domain, target_domains):
continue
if cookie_list[0] == cookie_key:
return cookie_list[1]
elif browser_type == BROWSER_TYPE_FIREFOX:
con = sqlite3.connect(os.path.join(file_path, "cookies.sqlite"))
cur = con.cursor()
cur.execute(
"select host, path, isSecure, expiry, name, value from moz_cookies"
)
for cookie_info in cur.fetchall():
domain = cookie_info[0]
if _filter_domain(domain, target_domains):
continue
if cookie_info[4] == cookie_key:
return cookie_info[5]
elif browser_type == BROWSER_TYPE_CHROME:
# chrome仅支持windows系统的解密
if platform.system() != "Windows":
return None
con = sqlite3.connect(os.path.join(file_path, "Cookies"))
cur = con.cursor()
cur.execute(
"select host_key, path, secure, expires_utc, name, value, encrypted_value from cookies"
)
for cookie_info in cur.fetchall():
domain = cookie_info[0]
if _filter_domain(domain, target_domains):
continue
if cookie_info[4] == cookie_key:
try:
value = win32crypt.CryptUnprotectData(
cookie_info[6], None, None, None, 0)[1]
except:
return None
return value
else:
output.print_msg("不支持的浏览器类型：" + browser_type)
return None
return None
def trace(msg):
"""Trace(Debugger) message logger"""
msg = _get_time() + " " + str(msg)
if IS_SHOW_TRACE:
output.print_msg(msg, False)
if TRACE_LOG_PATH != "":
with thread_lock:
tool.write_file(msg, TRACE_LOG_PATH)
def step(msg):
"""Step message logger"""
msg = _get_time() + " " + str(msg)
if IS_SHOW_STEP:
output.print_msg(msg, False)
if STEP_LOG_PATH != "":
with thread_lock:
tool.write_file(msg, STEP_LOG_PATH)
def error(msg):
"""Error message logger"""
msg = _get_time() + " [Error] " + str(msg)
if IS_SHOW_ERROR:
output.print_msg(msg, False)
if ERROR_LOG_PATH != "":
with thread_lock:
tool.write_file(msg, ERROR_LOG_PATH)
def error(msg):
"""Error message logger"""
msg = _get_time() + " [Error] " + str(msg)
if LOG_CONFIG["IS_SHOW_ERROR"]:
output.print_msg(msg, False)
if LOG_CONFIG["IS_LOG_ERROR"]:
with thread_lock:
file.write_file(msg, _replace_path_macro(ERROR_LOG_PATH))
def notice(msg):
"""Debug message logger"""
msg = _get_time() + " " + str(msg)
if LOG_CONFIG["IS_SHOW_NOTICE"]:
output.print_msg(msg, False)
if LOG_CONFIG["IS_LOG_NOTICE"]:
with thread_lock:
file.write_file(msg, _replace_path_macro(NOTICE_LOG_PATH))
def step(msg):
"""Step message logger"""
msg = _get_time() + " " + str(msg)
if LOG_CONFIG["IS_SHOW_STEP"]:
output.print_msg(msg, False)
if LOG_CONFIG["IS_LOG_STEP"]:
with thread_lock:
file.write_file(msg, _replace_path_macro(STEP_LOG_PATH))
def check_is_repeat():
history = []
for line in tool.read_file(SAVE_FILE_PATH, tool.READ_FILE_TYPE_LINE):
temp_list = line.replace("\n", "").split("\t")
if temp_list[NAME_COLUMN] in history:
output.print_msg(temp_list[NAME_COLUMN])
else:
history.append(temp_list[NAME_COLUMN])
return history
def get_save_data_file_count():
if not os.path.exists(SAVE_DATA_FILE_PATH):
output.print_msg("save data %s not exist" % SAVE_DATA_FILE_PATH)
return {}
account_list = {}
for line in tool.read_file(SAVE_DATA_FILE_PATH, tool.READ_FILE_TYPE_LINE):
temp_list = line.replace("\n", "").split("\t")
account_list[temp_list[PRIME_KEY_INDEX].decode("UTF-8")] = int(temp_list[COUNT_INDEX])
return account_list
def set_proxy(ip, port):
"""init urllib3 proxy connection pool"""
if not str(port).isdigit() or int(port) <= 0:
return
match = re.match("((25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))\.){3}(25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))", ip)
if not match or match.group() != ip:
return
global PROXY_HTTP_CONNECTION_POOL
PROXY_HTTP_CONNECTION_POOL = urllib3.ProxyManager("http://%s:%s" % (ip, port), retries=False)
output.print_msg("设置代理成功(%s:%s)" % (ip, port))
def trace(msg: str):
"""
trace日志
"""
msg = _get_time() + " " + str(msg)
if LOG_CONFIG["IS_SHOW_TRACE"]:
output.print_msg(msg, False)
if LOG_CONFIG["IS_LOG_TRACE"]:
with thread_lock:
file.write_file(msg, _replace_path_macro(TRACE_LOG_PATH))
def main():
# 获取登录状态
steam_class = steam.Steam(need_login=False)
steam_class.format_cache_app_info()
# 已删除的游戏
deleted_app_list = steam_class.load_deleted_app_list()
# 已资料受限制的游戏
restricted_app_list = steam_class.load_restricted_app_list()
output.print_msg(deleted_app_list + restricted_app_list)
def run(self):
try:
hook_manager = pyHook.HookManager()
except NameError:
output.print_msg("PyHook安装失败，键盘监听功能失效")
pass
else:
hook_manager.KeyDown = self.on_keyboard_down
hook_manager.KeyUp = self.on_keyboard_up
hook_manager.HookKeyboard()
pythoncom.PumpMessages()
def get_default_browser_cookie_path(browser_type: int) -> Optional[str]:
"""
根据浏览器和操作系统，自动查找默认浏览器cookie路径(只支持windows)
"""
if platform.system() != "Windows":
return None
if browser_type == BROWSER_TYPE_IE:
return os.path.join(os.getenv("APPDATA"),
"Microsoft\\Windows\\Cookies")
elif browser_type == BROWSER_TYPE_FIREFOX:
default_browser_path = os.path.join(os.getenv("APPDATA"),
"Mozilla\\Firefox\\Profiles")
for dir_name in os.listdir(default_browser_path):
sub_path = os.path.join(default_browser_path, dir_name)
if os.path.isdir(sub_path):
if os.path.exists(os.path.join(sub_path, "cookies.sqlite")):
return os.path.abspath(sub_path)
elif browser_type == BROWSER_TYPE_CHROME:
profile_file_path = os.path.abspath(
os.path.join(os.getenv("LOCALAPPDATA"),
"Google\\Chrome\\User Data\\Local State"))
default_profile_name = "Default"
if os.path.exists(profile_file_path):
with open(profile_file_path, "r", encoding="UTF-8") as file_handle:
profile_info = json.load(file_handle)
if "profile" in profile_info:
if "info_cache" in profile_info["profile"] and isinstance(
profile_info["profile"]["info_cache"],
dict) and len(
profile_info["profile"]["info_cache"]) == 1:
default_profile_name = list(
profile_info["profile"]["info_cache"].keys())[0]
elif "last_used" in profile_info["profile"]:
default_profile_name = profile_info["profile"][
"last_used"]
elif "last_active_profiles" in profile_info[
"profile"] and isinstance(
profile_info["profile"]
["last_active_profiles"], dict) and len(
profile_info["profile"]
["last_active_profiles"]) == 1:
default_profile_name = profile_info["profile"][
"last_active_profiles"][0]
return os.path.abspath(
os.path.join(os.getenv("LOCALAPPDATA"),
"Google\\Chrome\\User Data", default_profile_name))
elif browser_type == BROWSER_TYPE_TEXT:
return os.path.abspath(
os.path.join(crawler.PROJECT_APP_PATH, "info/cookies.data"))
else:
output.print_msg("不支持的浏览器类型：" + str(browser_type))
return None
def analysis_config(config, key, default_value, mode=CONFIG_ANALYSIS_MODE_RAW):
"""Analysis config
:param config:
Dictionary of config
:param key:
key of config
:param default_value:
default value
:param mode:
type of analysis mode
None direct assignment
1 conversion to integer
2 conversion to boolean
the value Equivalent to False, or string of "0" and "false" will conversion to False
other string will conversion to True
3 conversion to file path
startup with '\', project root path
startup with '\\', application root path
"""
key = key.lower()
if isinstance(config, dict) and key in config:
value = config[key]
else:
output.print_msg("配置文件config.ini中没有找到key为'" + key + "'的参数，使用程序默认设置")
value = default_value
if mode == CONFIG_ANALYSIS_MODE_INTEGER:
if isinstance(value, int) or isinstance(
value, long) or (isinstance(value, str) and value.isdigit()):
value = int(value)
else:
output.print_msg("配置文件config.ini中key为'" + key +
"'的值必须是一个整数，使用程序默认设置")
value = default_value
elif mode == CONFIG_ANALYSIS_MODE_BOOLEAN:
if not value or value == "0" or (isinstance(value, str)
and value.lower() == "false"):
value = False
else:
value = True
elif mode == CONFIG_ANALYSIS_MODE_PATH:
if value[:2] == "\\\\": # \\ 开头，程序所在目录
value = os.path.join(tool.PROJECT_APP_PATH,
value[2:]) # \\ 仅做标记使用，实际需要去除
elif value[0] == "\\": # \ 开头，项目根目录（common目录上级）
value = os.path.join(tool.PROJECT_ROOT_PATH,
value[1:]) # \ 仅做标记使用，实际需要去除
value = os.path.abspath(value)
return value
def check_count():
account_list_from_storage = get_storage_file_count()
account_list_from_save_data = get_save_data_file_count()
if not account_list_from_storage or not account_list_from_save_data:
return
for account_id in dict(account_list_from_storage):
if account_id not in account_list_from_save_data:
account_list_from_save_data[account_id] = 0
if account_list_from_save_data[account_id] != account_list_from_storage[account_id]:
output.print_msg("%s count in save data: %s, in root path: %s" % (account_id, account_list_from_save_data[account_id], account_list_from_storage[account_id]))
for account_id in dict(account_list_from_storage):
if account_id not in account_list_from_save_data:
output.print_msg("%s not found in save data" % account_id)
def main():
# 获取登录状态
steam_class = steam.Steam(need_login=False)
# 已删除的游戏
deleted_app_list = steam_class.load_deleted_app_list()
try:
banned_game_list = madjoki.get_banned_game_list()
except crawler.CrawlerException as e:
output.print_msg(e.http_error("已下线游戏列表"))
else:
output.print_msg(f"总共获取{len(banned_game_list)}个已删除游戏")
banned_game_id_list = {}
for game_info in banned_game_list:
banned_game_id_list[str(game_info["game_id"])] = 1
for game_id in deleted_app_list:
if game_id not in banned_game_id_list:
# 获取游戏信息
try:
game_data = steam.get_game_store_index(game_id)
except crawler.CrawlerException as e:
output.print_msg(e.http_error(f"游戏{game_id}"))
continue
if game_data["deleted"] is False:
output.print_msg(f"游戏 {game_id} 不在已删除列表中")
def print_list(apps_cache_data, game_dlc_list, print_type=0):
for game_id in apps_cache_data["can_review_lists"]:
# 是DLC
if game_id in game_dlc_list:
if print_type == 1:
continue
# 本体没有评测过
if game_dlc_list[game_id] in apps_cache_data["can_review_lists"]:
if print_type == 3:
continue
else:
if print_type == 2 or print_type == 3:
continue
output.print_msg(f"https://store.steampowered.com/app/{game_id}")
def print_list(print_type=0):
review_data = load_review_list()
for game_id in review_data["can_review_lists"]:
# 是DLC
if game_id in review_data["dlc_in_game"]:
if print_type == 1:
continue
# 本体没有评测过
if review_data["dlc_in_game"][game_id] in review_data["can_review_lists"]:
if print_type == 3:
continue
else:
if print_type == 2 or print_type == 3:
continue
output.print_msg("https://store.steampowered.com/app/%s" % game_id)
