def __init__(self):
self.db = RedisClient()
# self.db.recheck()
# self.db.clear_all()
self.crawlers = [xiciproxy.xiciproxy()
] #,xiciproxy.xiciproxy()_66proxy._66proxy(),
self.check_task = threading.Thread(target=self.check_useful_task)
self.check_task.daemon = True
self.check_task.start()
self.check_task = threading.Thread(target=self.interface)
self.check_task.daemon = True
self.check_task.start()
self.update()
def __init__(self, bind_ip, bind_port, process_id):
self.bind_ip = bind_ip
self.bind_port = bind_port
self.process_id = process_id
self.nid = get_rand_id()
# nodes 节点是一个双端队列
self.nodes = deque(maxlen=MAX_NODE_QSIZE)
# KRPC 协议是由 bencode 编码组成的一个简单的 RPC 结构，使用 UDP 报文发送。
self.udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM,
socket.IPPROTO_UDP)
# UDP 地址绑定
self.udp.bind((self.bind_ip, self.bind_port))
# redis 客户端
self.rc = RedisClient()
self.logger = get_logger("logger_{}".format(bind_port))
def __init__(self):
self.redis = RedisClient()
#!/usr/bin/env python
# coding=utf-8
from sanic import Sanic
from sanic.response import json, html
import os
import sys
base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, base_dir)
from database import RedisClient
app = Sanic()
redis_conn = RedisClient()
@app.route("/")
async def index(request):
return html('<h2>Welcome to Proxy Pool System</h2>')
@app.route("/pop")
async def pop_proxy(request):
proxy = await redis_conn.pop_proxy()
if proxy:
proxy = proxy.decode('utf-8')
if proxy[:5] == "https":
return json({"https": proxy})
else:
return json({"http": proxy})
