def update_proxy(start_date, end_date): """获取最新的代理,并且验证代理 """ # itmop itmop_proxys = itmop.get_proxys(start_date, end_date) print "itmop_proxys update:", len(itmop_proxys) # youdaili youdaili_proxys = youdaili.get_proxys(start_date, end_date) print "youdaili_proxys update:", len(youdaili_proxys) # 历史 proxy_dat_file_path = main_config.PROXY_DAT_FILE_PATH history_proxys = util.read_proxys(proxy_dat_file_path) \ if os.path.exists(proxy_dat_file_path) else set() print "read history proxy:", len(history_proxys) # 合并 proxy_list = list(youdaili_proxys.union(history_proxys).union(itmop_proxys)) print "union history, itmop, youdaili proxys:", len(proxy_list) # 验证 print "start to validate proxys" success_pers = validate.validate_proxys( proxy_list, max_clients=main_config.MAX_CLIENTS, interval=main_config.INTERVAL) print "end to validate proxys" # 持久化代理 util.write_proxys([proxy_list[i] for i, success_per in enumerate(success_pers) if success_per >= main_config.PROXY_THRESHOLD], proxy_dat_file_path) print "write proxy into file end"
def validate_proxy(): """验证代理 """ # 获取本地代理 proxy_dat_file_path = main_config.PROXY_DAT_FILE_PATH history_proxys = util.read_proxys(proxy_dat_file_path) \ if os.path.exists(proxy_dat_file_path) else set() print "read history proxy:", len(history_proxys) # 验证代理 proxy_list = list(history_proxys) print "start to validate proxys" success_pers = validate.validate_proxys( proxy_list, max_clients=main_config.MAX_CLIENTS, interval=main_config.INTERVAL) print "end to validate proxys:", len(proxy_list) # 持久化代理 util.write_proxys([proxy_list[i] for i, success_per in enumerate(success_pers) if success_per >= main_config.PROXY_THRESHOLD], proxy_dat_file_path) print "write proxy into file end"
def update_proxy(start_date, end_date): """获取最新的代理,并且验证代理 """ # itmop itmop_proxys = itmop.get_proxys(start_date, end_date) print "itmop_proxys update:", len(itmop_proxys) # youdaili youdaili_proxys = youdaili.get_proxys(start_date, end_date) print "youdaili_proxys update:", len(youdaili_proxys) # 历史 proxy_dat_file_path = main_config.PROXY_DAT_FILE_PATH history_proxys = util.read_proxys(proxy_dat_file_path) \ if os.path.exists(proxy_dat_file_path) else set() print "read history proxy:", len(history_proxys) # 合并 proxy_list = list( youdaili_proxys.union(history_proxys).union(itmop_proxys)) print "union history, itmop, youdaili proxys:", len(proxy_list) # 验证 print "start to validate proxys" success_pers = validate.validate_proxys( proxy_list, max_clients=main_config.MAX_CLIENTS, interval=main_config.INTERVAL) print "end to validate proxys" # 持久化代理 util.write_proxys([ proxy_list[i] for i, success_per in enumerate(success_pers) if success_per >= main_config.PROXY_THRESHOLD ], proxy_dat_file_path) print "write proxy into file end"
def validate_proxy(): """验证代理 """ # 获取本地代理 proxy_dat_file_path = main_config.PROXY_DAT_FILE_PATH history_proxys = util.read_proxys(proxy_dat_file_path) \ if os.path.exists(proxy_dat_file_path) else set() print "read history proxy:", len(history_proxys) # 验证代理 proxy_list = list(history_proxys) print "start to validate proxys" success_pers = validate.validate_proxys( proxy_list, max_clients=main_config.MAX_CLIENTS, interval=main_config.INTERVAL) print "end to validate proxys:", len(proxy_list) # 持久化代理 util.write_proxys([ proxy_list[i] for i, success_per in enumerate(success_pers) if success_per >= main_config.PROXY_THRESHOLD ], proxy_dat_file_path) print "write proxy into file end"