class ProxyRefresh():
def __init__(self, proxy_type='https'):
if proxy_type == 'https':
self.redis_handler = RedisClient('https_proxy')
elif proxy_type == 'http':
self.redis_handler = RedisClient('http_proxy')
else:
raise Exception('type must be https or http')
self.proxy_type = proxy_type
self.proxy_pool = set([*fuzz_all(), *self.redis_handler.get_all()])
def refresh(self, pool_num=10):
pool = ThreadPool(pool_num)
pool.map(self.valid_ip, self.proxy_pool)
pool.close()
pool.join()
def refresh_in_async(self):
asynctask = AsyncTask()
for ip in self.proxy_pool:
asynctask.add_task(self.valid_ip, ip)
asynctask.run()
def valid_ip(self, ip):
if proxy_is_useful(ip, self.proxy_type):
self.redis_handler.add(ip)
print('ok', ip)
else:
self.redis_handler.delete(ip)
class ProxyManage(Resource):
def __init__(self):
self.http_proxy = RedisClient('http_proxy')
self.https_proxy = RedisClient('https_proxy')
self.parser = reqparse.RequestParser()
self.parser.add_argument('type', type=str, required=True,
help='required args of proxy type: http/https, like ?type=http')
@error_handle
def get(self):
self.parser.add_argument('all', type=str, default='false', required=False, help='')
get_all = self.parser.parse_args()['all']
proxy_type = self.parser.parse_args()['type']
if proxy_type == 'http':
if get_all == 'true':
return self.http_proxy.get_all()
else:
return self.http_proxy.get_one()
elif proxy_type == 'https':
if get_all == 'true':
return self.https_proxy.get_all()
else:
return self.https_proxy.get_one()
else:
raise ParamError(msg='proxy type param error,must be http/https')
@error_handle
def delete(self):
self.parser.add_argument('ip', type=str, required=True)
args = self.parser.parse_args()
proxy_type = args.get('type')
ip = args.get('ip')
if proxy_type == 'http':
return self.http_proxy.delete(ip)
if proxy_type == 'https':
return self.https_proxy.delete(ip)
else:
self.http_proxy.delete(ip)
self.https_proxy.delete(ip)
return
