class ProxyRefresh(): def __init__(self, proxy_type='https'): if proxy_type == 'https': self.redis_handler = RedisClient('https_proxy') elif proxy_type == 'http': self.redis_handler = RedisClient('http_proxy') else: raise Exception('type must be https or http') self.proxy_type = proxy_type self.proxy_pool = set([*fuzz_all(), *self.redis_handler.get_all()]) def refresh(self, pool_num=10): pool = ThreadPool(pool_num) pool.map(self.valid_ip, self.proxy_pool) pool.close() pool.join() def refresh_in_async(self): asynctask = AsyncTask() for ip in self.proxy_pool: asynctask.add_task(self.valid_ip, ip) asynctask.run() def valid_ip(self, ip): if proxy_is_useful(ip, self.proxy_type): self.redis_handler.add(ip) print('ok', ip) else: self.redis_handler.delete(ip)
class ProxyManage(Resource): def __init__(self): self.http_proxy = RedisClient('http_proxy') self.https_proxy = RedisClient('https_proxy') self.parser = reqparse.RequestParser() self.parser.add_argument('type', type=str, required=True, help='required args of proxy type: http/https, like ?type=http') @error_handle def get(self): self.parser.add_argument('all', type=str, default='false', required=False, help='') get_all = self.parser.parse_args()['all'] proxy_type = self.parser.parse_args()['type'] if proxy_type == 'http': if get_all == 'true': return self.http_proxy.get_all() else: return self.http_proxy.get_one() elif proxy_type == 'https': if get_all == 'true': return self.https_proxy.get_all() else: return self.https_proxy.get_one() else: raise ParamError(msg='proxy type param error,must be http/https') @error_handle def delete(self): self.parser.add_argument('ip', type=str, required=True) args = self.parser.parse_args() proxy_type = args.get('type') ip = args.get('ip') if proxy_type == 'http': return self.http_proxy.delete(ip) if proxy_type == 'https': return self.https_proxy.delete(ip) else: self.http_proxy.delete(ip) self.https_proxy.delete(ip) return