def main(): # GUI窗口 gui = tkinter.Tk() gui.withdraw() # 原始存档文件所在路径 options = { "initialdir": os.path.join(os.path.dirname(__file__), "video"), "initialfile": "save.data", "filetypes": [("data", ".data"), ("all file", "*")], "title": "原始存档文件", } save_data_file_path = tkinter.filedialog.askopenfilename(**options) if not save_data_file_path: return options["title"] = "临时存档文件" temp_save_data_file_path = tkinter.filedialog.askopenfilename(**options) if not save_data_file_path: return if save_data_file_path == temp_save_data_file_path: output.print_msg("存档文件相同,无需合并") return # 临时存档文件所在路径 save_data = crawler.read_save_data(save_data_file_path, PRIME_KEY_INDEX) temp_save_data = crawler.read_save_data(temp_save_data_file_path, PRIME_KEY_INDEX) save_data.update(temp_save_data) temp_list = [save_data[key] for key in sorted(save_data.keys())] file.write_file(tool.list_to_string(temp_list), save_data_file_path, file.WRITE_FILE_TYPE_REPLACE)
def read_save_data(save_data_path, key_index, default_value_list): result_list = {} if not os.path.exists(path.change_path_encoding(save_data_path)): return result_list for single_save_data in tool.read_file(save_data_path, tool.READ_FILE_TYPE_LINE): single_save_data = single_save_data.replace("\xef\xbb\xbf", "").replace("\n", "").replace("\r", "") if len(single_save_data) == 0: continue single_save_list = single_save_data.split("\t") if single_save_list[key_index] in result_list: output.print_msg("存档中存在重复行 %s" % single_save_list[key_index]) tool.process_exit() # 去除前后空格 single_save_list = map(lambda value: value.strip(), single_save_list) # 根据default_value_list给没给字段默认值 index = 0 for default_value in default_value_list: # _开头表示和该数组下标的值一直,如["", "_0"] 表示第1位为空时数值和第0位一致 if default_value != "" and default_value[0] == "_": default_value = single_save_list[int(default_value.replace("_", ""))] if len(single_save_list) <= index: single_save_list.append(default_value) if single_save_list[index] == "": single_save_list[index] = default_value index += 1 result_list[single_save_list[key_index]] = single_save_list return result_list
def main(account_id, include_type, min_discount_percent, min_discount_price): # 获取登录状态 try: login_cookie = steamCommon.get_login_cookie_from_browser() except crawler.CrawlerException, e: output.print_msg("登录状态检测失败,原因:%s" % e.message) raise
def resume_request(): """ Resume thread """ if not thread_event.is_set(): output.print_msg("resume process") thread_event.set()
def run(self): header_list = {"Range": f"bytes={self.start_pos}-{self.end_pos}"} range_size = self.end_pos - self.start_pos + 1 for retry_count in range(0, NET_CONFIG["DOWNLOAD_RETRY_COUNT"]): response = net.request(self.file_url, method="GET", header_list=header_list) if response.status == 206: # 下载的文件和请求的文件大小不一致 if len(response.data) != range_size: output.print_msg( f"网络文件{self.file_url}:range {self.start_pos} - {self.end_pos}实际下载大小 {len(response.data)} 不一致" ) time.sleep(net.NET_CONFIG["HTTP_REQUEST_RETRY_WAIT_TIME"]) else: # 写入本地文件后退出 self.fd_handle.seek(self.start_pos) self.fd_handle.write(response.data) self.fd_handle.close() break else: self.error_flag.append(self) # 唤醒主线程 multi_download_thread_semaphore.release()
def pause_request(): """ Block thread when use request() """ if thread_event.is_set(): output.print_msg("pause process") thread_event.clear()
def main(account_id): # 获取库存 try: inventory_item_list = steamCommon.get_account_inventory(account_id) except crawler.CrawlerException, e: output.print_msg("获取库存失败,原因:%s" % e.message) raise
def main(account_id): # 获取登录状态 try: steamCommon.init_cookie_from_browser() except crawler.CrawlerException, e: output.print_msg("登录状态检测失败,原因:%s" % e.message) raise
def get_all_cookie_from_browser(browser_type, file_path): if not os.path.exists(file_path): output.print_msg("cookie目录:" + file_path + " 不存在") return None all_cookies = {} if browser_type == 1: for cookie_name in os.listdir(file_path): if cookie_name.find(".txt") == -1: continue with open(os.path.join(file_path, cookie_name), "r", encoding="UTF-8") as cookie_file: cookie_info = cookie_file.read() for cookies in cookie_info.split("*"): cookie_list = cookies.strip("\n").split("\n") if len(cookie_list) < 8: continue cookie_domain = cookie_list[2].split("/")[0] cookie_key = cookie_list[0] cookie_value = cookie_list[1] if cookie_domain not in all_cookies: all_cookies[cookie_domain] = {} all_cookies[cookie_domain][cookie_key] = cookie_value elif browser_type == 2: con = sqlite3.connect(os.path.join(file_path, "cookies.sqlite")) cur = con.cursor() cur.execute("SELECT host, path, name, value FROM moz_cookies") for cookie_info in cur.fetchall(): cookie_domain = cookie_info[0] cookie_key = cookie_info[2] cookie_value = cookie_info[3] if cookie_domain not in all_cookies: all_cookies[cookie_domain] = {} all_cookies[cookie_domain][cookie_key] = cookie_value con.close() elif browser_type == 3: # chrome仅支持windows系统的解密 if platform.system() != "Windows": return None con = sqlite3.connect(os.path.join(file_path, "Cookies")) cur = con.cursor() cur.execute( "SELECT host_key, path, name, value, encrypted_value FROM cookies") for cookie_info in cur.fetchall(): cookie_domain = cookie_info[0] cookie_key = cookie_info[2] try: cookie_value = win32crypt.CryptUnprotectData( cookie_info[4], None, None, None, 0)[1] except: continue if cookie_domain not in all_cookies: all_cookies[cookie_domain] = {} all_cookies[cookie_domain][cookie_key] = cookie_value.decode() con.close() else: output.print_msg("不支持的浏览器类型:" + browser_type) return None return all_cookies
def multipart_download(self): """ 分段下载 """ # 先创建文件(同时删除之前下载失败,可能生成的临时文件) with open(self.file_path, "w"): pass with open(self.file_path, "rb+") as file_handle: file_no = file_handle.fileno() end_pos = -1 while end_pos < self.content_length - 1: start_pos = end_pos + 1 end_pos = min( self.content_length - 1, start_pos + NET_CONFIG["DOWNLOAD_MULTIPART_BLOCK_SIZE"] - 1) multipart_kwargs = self.kwargs.copy() # 分段的header信息 if "header_list" in multipart_kwargs: header_list = multipart_kwargs["header_list"] del multipart_kwargs["header_list"] else: header_list = {} header_list["Range"] = f"bytes={start_pos}-{end_pos}" # 创建一个副本 with os.fdopen(os.dup(file_no), "rb+", -1) as fd_handle: for multipart_retry_count in range( 0, NET_CONFIG["DOWNLOAD_RETRY_COUNT"]): try: multipart_response = request( self.file_url, method="GET", header_list=header_list, connection_timeout=NET_CONFIG[ "DOWNLOAD_CONNECTION_TIMEOUT"], read_timeout=NET_CONFIG[ "DOWNLOAD_READ_TIMEOUT"], **multipart_kwargs) except SystemExit: return False if multipart_response.status == 206: # 下载的文件和请求的文件大小不一致 if len(multipart_response.data) != (end_pos - start_pos + 1): output.print_msg( f"网络文件{self.file_url}:range {start_pos} - {end_pos}实际下载大小 {len(multipart_response.data)} 不一致" ) time.sleep( NET_CONFIG["HTTP_REQUEST_RETRY_WAIT_TIME"]) else: # 写入本地文件后退出 fd_handle.seek(start_pos) fd_handle.write(multipart_response.data) break else: self.code = self.CODE_RETRY_MAX_COUNT return False return True
def start_download(self): """ 主体下载逻辑 """ # 默认读取配置 if not isinstance(self.replace_if_exist, bool): self.replace_if_exist = DOWNLOAD_REPLACE_IF_EXIST # 同名文件已经存在,直接返回 if not self.replace_if_exist and os.path.exists( self.file_path) and os.path.getsize(self.file_path) > 0: output.print_msg(f"文件{self.file_path}({self.file_url})已存在,跳过") self.status = self.DOWNLOAD_SUCCEED return # 判断保存目录是否存在 if not path.create_dir(os.path.dirname(self.file_path)): self.code = self.CODE_FILE_CREATE_FAILED return # 是否需要分段下载 self.check_auto_multipart_download() # 下载 for retry_count in range(0, NET_CONFIG["DOWNLOAD_RETRY_COUNT"]): if EXIT_FLAG: self.code = self.CODE_PROCESS_EXIT break if not self.is_multipart_download: # 单线程下载 if not self.single_download(): continue else: # 分段下载 if not self.multipart_download(): continue # 如果没有返回文件的长度,直接下载成功 if self.content_length == 0: self.status = self.DOWNLOAD_SUCCEED self.code = 0 return # 判断文件下载后的大小和response中的Content-Length是否一致 file_size = os.path.getsize(self.file_path) if self.content_length == file_size: self.status = self.DOWNLOAD_SUCCEED self.code = 0 return else: self.code = self.CODE_FILE_SIZE_INVALID output.print_msg( f"本地文件{self.file_path}:{self.content_length}和网络文件{self.file_url}:{file_size}不一致" ) time.sleep(NET_CONFIG["HTTP_REQUEST_RETRY_WAIT_TIME"]) # 删除可能出现的临时文件 path.delete_dir_or_file(self.file_path)
def get_cookie_value_from_browser(cookie_key, file_path, browser_type, target_domains=""): if not os.path.exists(file_path): output.print_msg("cookie目录:" + file_path + " 不存在") return None if browser_type == BROWSER_TYPE_IE: for cookie_name in os.listdir(file_path): if cookie_name.find(".txt") == -1: continue with open(os.path.join(file_path, cookie_name), "r") as cookie_file: cookie_info = cookie_file.read() for cookies in cookie_info.split("*"): cookie_list = cookies.strip("\n").split("\n") if len(cookie_list) < 8: continue domain = cookie_list[2].split("/")[0] if _filter_domain(domain, target_domains): continue if cookie_list[0] == cookie_key: return cookie_list[1] elif browser_type == BROWSER_TYPE_FIREFOX: con = sqlite3.connect(os.path.join(file_path, "cookies.sqlite")) cur = con.cursor() cur.execute( "select host, path, isSecure, expiry, name, value from moz_cookies" ) for cookie_info in cur.fetchall(): domain = cookie_info[0] if _filter_domain(domain, target_domains): continue if cookie_info[4] == cookie_key: return cookie_info[5] elif browser_type == BROWSER_TYPE_CHROME: # chrome仅支持windows系统的解密 if platform.system() != "Windows": return None con = sqlite3.connect(os.path.join(file_path, "Cookies")) cur = con.cursor() cur.execute( "select host_key, path, secure, expires_utc, name, value, encrypted_value from cookies" ) for cookie_info in cur.fetchall(): domain = cookie_info[0] if _filter_domain(domain, target_domains): continue if cookie_info[4] == cookie_key: try: value = win32crypt.CryptUnprotectData( cookie_info[6], None, None, None, 0)[1] except: return None return value else: output.print_msg("不支持的浏览器类型:" + browser_type) return None return None
def trace(msg): """Trace(Debugger) message logger""" msg = _get_time() + " " + str(msg) if IS_SHOW_TRACE: output.print_msg(msg, False) if TRACE_LOG_PATH != "": with thread_lock: tool.write_file(msg, TRACE_LOG_PATH)
def step(msg): """Step message logger""" msg = _get_time() + " " + str(msg) if IS_SHOW_STEP: output.print_msg(msg, False) if STEP_LOG_PATH != "": with thread_lock: tool.write_file(msg, STEP_LOG_PATH)
def error(msg): """Error message logger""" msg = _get_time() + " [Error] " + str(msg) if IS_SHOW_ERROR: output.print_msg(msg, False) if ERROR_LOG_PATH != "": with thread_lock: tool.write_file(msg, ERROR_LOG_PATH)
def error(msg): """Error message logger""" msg = _get_time() + " [Error] " + str(msg) if LOG_CONFIG["IS_SHOW_ERROR"]: output.print_msg(msg, False) if LOG_CONFIG["IS_LOG_ERROR"]: with thread_lock: file.write_file(msg, _replace_path_macro(ERROR_LOG_PATH))
def notice(msg): """Debug message logger""" msg = _get_time() + " " + str(msg) if LOG_CONFIG["IS_SHOW_NOTICE"]: output.print_msg(msg, False) if LOG_CONFIG["IS_LOG_NOTICE"]: with thread_lock: file.write_file(msg, _replace_path_macro(NOTICE_LOG_PATH))
def step(msg): """Step message logger""" msg = _get_time() + " " + str(msg) if LOG_CONFIG["IS_SHOW_STEP"]: output.print_msg(msg, False) if LOG_CONFIG["IS_LOG_STEP"]: with thread_lock: file.write_file(msg, _replace_path_macro(STEP_LOG_PATH))
def check_is_repeat(): history = [] for line in tool.read_file(SAVE_FILE_PATH, tool.READ_FILE_TYPE_LINE): temp_list = line.replace("\n", "").split("\t") if temp_list[NAME_COLUMN] in history: output.print_msg(temp_list[NAME_COLUMN]) else: history.append(temp_list[NAME_COLUMN]) return history
def get_save_data_file_count(): if not os.path.exists(SAVE_DATA_FILE_PATH): output.print_msg("save data %s not exist" % SAVE_DATA_FILE_PATH) return {} account_list = {} for line in tool.read_file(SAVE_DATA_FILE_PATH, tool.READ_FILE_TYPE_LINE): temp_list = line.replace("\n", "").split("\t") account_list[temp_list[PRIME_KEY_INDEX].decode("UTF-8")] = int(temp_list[COUNT_INDEX]) return account_list
def set_proxy(ip, port): """init urllib3 proxy connection pool""" if not str(port).isdigit() or int(port) <= 0: return match = re.match("((25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))\.){3}(25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))", ip) if not match or match.group() != ip: return global PROXY_HTTP_CONNECTION_POOL PROXY_HTTP_CONNECTION_POOL = urllib3.ProxyManager("http://%s:%s" % (ip, port), retries=False) output.print_msg("设置代理成功(%s:%s)" % (ip, port))
def trace(msg: str): """ trace日志 """ msg = _get_time() + " " + str(msg) if LOG_CONFIG["IS_SHOW_TRACE"]: output.print_msg(msg, False) if LOG_CONFIG["IS_LOG_TRACE"]: with thread_lock: file.write_file(msg, _replace_path_macro(TRACE_LOG_PATH))
def main(): # 获取登录状态 steam_class = steam.Steam(need_login=False) steam_class.format_cache_app_info() # 已删除的游戏 deleted_app_list = steam_class.load_deleted_app_list() # 已资料受限制的游戏 restricted_app_list = steam_class.load_restricted_app_list() output.print_msg(deleted_app_list + restricted_app_list)
def run(self): try: hook_manager = pyHook.HookManager() except NameError: output.print_msg("PyHook安装失败,键盘监听功能失效") pass else: hook_manager.KeyDown = self.on_keyboard_down hook_manager.KeyUp = self.on_keyboard_up hook_manager.HookKeyboard() pythoncom.PumpMessages()
def get_default_browser_cookie_path(browser_type: int) -> Optional[str]: """ 根据浏览器和操作系统,自动查找默认浏览器cookie路径(只支持windows) """ if platform.system() != "Windows": return None if browser_type == BROWSER_TYPE_IE: return os.path.join(os.getenv("APPDATA"), "Microsoft\\Windows\\Cookies") elif browser_type == BROWSER_TYPE_FIREFOX: default_browser_path = os.path.join(os.getenv("APPDATA"), "Mozilla\\Firefox\\Profiles") for dir_name in os.listdir(default_browser_path): sub_path = os.path.join(default_browser_path, dir_name) if os.path.isdir(sub_path): if os.path.exists(os.path.join(sub_path, "cookies.sqlite")): return os.path.abspath(sub_path) elif browser_type == BROWSER_TYPE_CHROME: profile_file_path = os.path.abspath( os.path.join(os.getenv("LOCALAPPDATA"), "Google\\Chrome\\User Data\\Local State")) default_profile_name = "Default" if os.path.exists(profile_file_path): with open(profile_file_path, "r", encoding="UTF-8") as file_handle: profile_info = json.load(file_handle) if "profile" in profile_info: if "info_cache" in profile_info["profile"] and isinstance( profile_info["profile"]["info_cache"], dict) and len( profile_info["profile"]["info_cache"]) == 1: default_profile_name = list( profile_info["profile"]["info_cache"].keys())[0] elif "last_used" in profile_info["profile"]: default_profile_name = profile_info["profile"][ "last_used"] elif "last_active_profiles" in profile_info[ "profile"] and isinstance( profile_info["profile"] ["last_active_profiles"], dict) and len( profile_info["profile"] ["last_active_profiles"]) == 1: default_profile_name = profile_info["profile"][ "last_active_profiles"][0] return os.path.abspath( os.path.join(os.getenv("LOCALAPPDATA"), "Google\\Chrome\\User Data", default_profile_name)) elif browser_type == BROWSER_TYPE_TEXT: return os.path.abspath( os.path.join(crawler.PROJECT_APP_PATH, "info/cookies.data")) else: output.print_msg("不支持的浏览器类型:" + str(browser_type)) return None
def analysis_config(config, key, default_value, mode=CONFIG_ANALYSIS_MODE_RAW): """Analysis config :param config: Dictionary of config :param key: key of config :param default_value: default value :param mode: type of analysis mode None direct assignment 1 conversion to integer 2 conversion to boolean the value Equivalent to False, or string of "0" and "false" will conversion to False other string will conversion to True 3 conversion to file path startup with '\', project root path startup with '\\', application root path """ key = key.lower() if isinstance(config, dict) and key in config: value = config[key] else: output.print_msg("配置文件config.ini中没有找到key为'" + key + "'的参数,使用程序默认设置") value = default_value if mode == CONFIG_ANALYSIS_MODE_INTEGER: if isinstance(value, int) or isinstance( value, long) or (isinstance(value, str) and value.isdigit()): value = int(value) else: output.print_msg("配置文件config.ini中key为'" + key + "'的值必须是一个整数,使用程序默认设置") value = default_value elif mode == CONFIG_ANALYSIS_MODE_BOOLEAN: if not value or value == "0" or (isinstance(value, str) and value.lower() == "false"): value = False else: value = True elif mode == CONFIG_ANALYSIS_MODE_PATH: if value[:2] == "\\\\": # \\ 开头,程序所在目录 value = os.path.join(tool.PROJECT_APP_PATH, value[2:]) # \\ 仅做标记使用,实际需要去除 elif value[0] == "\\": # \ 开头,项目根目录(common目录上级) value = os.path.join(tool.PROJECT_ROOT_PATH, value[1:]) # \ 仅做标记使用,实际需要去除 value = os.path.abspath(value) return value
def check_count(): account_list_from_storage = get_storage_file_count() account_list_from_save_data = get_save_data_file_count() if not account_list_from_storage or not account_list_from_save_data: return for account_id in dict(account_list_from_storage): if account_id not in account_list_from_save_data: account_list_from_save_data[account_id] = 0 if account_list_from_save_data[account_id] != account_list_from_storage[account_id]: output.print_msg("%s count in save data: %s, in root path: %s" % (account_id, account_list_from_save_data[account_id], account_list_from_storage[account_id])) for account_id in dict(account_list_from_storage): if account_id not in account_list_from_save_data: output.print_msg("%s not found in save data" % account_id)
def main(): # 获取登录状态 steam_class = steam.Steam(need_login=False) # 已删除的游戏 deleted_app_list = steam_class.load_deleted_app_list() try: banned_game_list = madjoki.get_banned_game_list() except crawler.CrawlerException as e: output.print_msg(e.http_error("已下线游戏列表")) else: output.print_msg(f"总共获取{len(banned_game_list)}个已删除游戏") banned_game_id_list = {} for game_info in banned_game_list: banned_game_id_list[str(game_info["game_id"])] = 1 for game_id in deleted_app_list: if game_id not in banned_game_id_list: # 获取游戏信息 try: game_data = steam.get_game_store_index(game_id) except crawler.CrawlerException as e: output.print_msg(e.http_error(f"游戏{game_id}")) continue if game_data["deleted"] is False: output.print_msg(f"游戏 {game_id} 不在已删除列表中")
def print_list(apps_cache_data, game_dlc_list, print_type=0): for game_id in apps_cache_data["can_review_lists"]: # 是DLC if game_id in game_dlc_list: if print_type == 1: continue # 本体没有评测过 if game_dlc_list[game_id] in apps_cache_data["can_review_lists"]: if print_type == 3: continue else: if print_type == 2 or print_type == 3: continue output.print_msg(f"https://store.steampowered.com/app/{game_id}")
def print_list(print_type=0): review_data = load_review_list() for game_id in review_data["can_review_lists"]: # 是DLC if game_id in review_data["dlc_in_game"]: if print_type == 1: continue # 本体没有评测过 if review_data["dlc_in_game"][game_id] in review_data["can_review_lists"]: if print_type == 3: continue else: if print_type == 2 or print_type == 3: continue output.print_msg("https://store.steampowered.com/app/%s" % game_id)