def fetch_recent(request):
m = Manager(database=redis)
target_name = request.GET.get("target", "www.baidu.com")
px_list = m.get_range_from(":".join(["spoon", target_name, "current_proxy"]))
res_list = [px.decode('utf-8') for px in px_list]
return HttpResponse("\r\n".join(res_list))
def __init__(self, redis_):
self.listen_host = None
self.listen_port = None
self.remote_host = None
self.remote_port = None
self.default_remote_host = None
self.default_remote_port = None
self.m = Manager(database=redis_)
def fetch_stale(request):
m = Manager(database=redis)
num = int(request.GET.get("num", 100))
px_kv = m.get_all_kv_from("spoon:proxy_stale")
res_list_pre = [[k.decode('utf-8'), float(v.decode('utf-8'))] for (k, v) in px_kv.items()]
res_list = [k for [k, _] in sorted(res_list_pre, key=lambda item: -item[1])][0:num]
return HttpResponse("\r\n".join(res_list))
def fetch_hundred_recent(request):
m = Manager(database=redis)
target_name = request.GET.get("target", "www.baidu.com")
filter_num = int(request.GET.get("filter", 30))
search_name = ":".join(["spoon", target_name, "hundred_proxy"])
px_kv = m.get_all_kv_from(search_name)
res_list = [k.decode('utf-8') for (k, v) in px_kv.items() if
abs(float(v.decode('utf-8')) - time.time()) < filter_num]
return HttpResponse("\r\n".join(res_list))
def fetchall_from(request):
m = Manager(database=redis)
target_name = request.GET.get("target", "www.baidu.com")
filter_num = int(request.GET.get("filter", 10))
search_name = ":".join(["spoon", target_name, "useful_proxy"])
px_kv = m.get_all_kv_from(search_name)
res_list = [k.decode('utf-8') for (k, v) in px_kv.items() if int(v) > filter_num]
return HttpResponse("\r\n".join(res_list))
def fetchone_from(request):
m = Manager()
target_name = request.GET.get("target", "www.baidu.com")
filter_num = int(request.GET.get("filter", 10))
search_name = ":".join(["spoon", target_name, "useful_proxy"])
px_kv = m.get_all_kv_from(search_name)
res_list = random.sample([
k.decode('utf-8')
for (k, v) in px_kv.items() if int(v.decode('utf-8')) > filter_num
], 1)
return HttpResponse(res_list[0])
def get_keys(request):
m = Manager(database=redis)
return HttpResponse(json.dumps(m.get_keys()))
def get_keys(request):
m = Manager()
return HttpResponse(json.dumps(m.get_keys()))
class ForwardServer(object):
PAGE_SIZE = 4096
def __init__(self, redis_):
self.listen_host = None
self.listen_port = None
self.remote_host = None
self.remote_port = None
self.default_remote_host = None
self.default_remote_port = None
self.m = Manager(database=redis_)
def set_listen(self, host, port):
self.listen_host = host
self.listen_port = port
return self
def set_default_remote(self, host, port):
self.default_remote_host = host
self.default_remote_port = port
return self
def set_remote(self, host, port):
self.remote_host = host
self.remote_port = port
return self
def _listen(self):
sock_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # tcp
sock_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock_server.bind((self.listen_host, self.listen_port))
sock_server.listen(5)
log.info('Listening at %s:%d ...' %
(self.listen_host, self.listen_port))
return sock_server
def serve(self):
sock_server = self._listen()
while not is_exit:
try:
sock, addr = sock_server.accept()
except (KeyboardInterrupt, SystemExit):
log.warn('Closing...')
sock_server.shutdown(socket.SHUT_RDWR)
sock_server.close()
break
except Exception as e:
log.error('Exception exit {0}'.format(e))
sock_server.shutdown(socket.SHUT_RDWR)
sock_server.close()
break
threading.Thread(target=self._forward, args=(sock, )).start()
log.info('New clients from {0}'.format(addr))
log.info('exit server')
def _forward(self, sock_in):
try:
sock_out = ForwardClient()
log.info('get the client socks done')
except Exception as e:
log.error('Get Remote Client error: %s' % str(e))
raise e
threading.Thread(target=self._do_data_forward,
args=(sock_in, sock_out)).start()
threading.Thread(target=self._do_data_forward,
args=(sock_out, sock_in)).start()
def _do_data_forward(self, sock_in, sock_out):
if isinstance(sock_in, ForwardClient):
sock_in = sock_in.get_client(self.default_remote_host,
self.default_remote_port)
addr_in = '%s:%d' % sock_in.getpeername()
while True:
try:
data = sock_in.recv(ForwardServer.PAGE_SIZE)
if isinstance(sock_out, ForwardClient):
print("sock_in", data)
if b'Host' in data:
host_match = re.match(r'.*Host:\s(.*?)\r\n.*',
data.decode("utf-8"), re.S)
if host_match:
hostname = host_match[1]
current_proxy_list = self.m.get_range_from(
":".join(["spoon", hostname, "current_proxy"]))
if current_proxy_list:
ran_num = random.randint(
0,
len(current_proxy_list) // 3)
proxy = current_proxy_list[ran_num].decode(
"utf-8")
sock_out = sock_out.get_client(
proxy.split(":")[0],
int(proxy.split(":")[1]))
log.info(
"Change Remote Proxy: {0}".format(proxy))
else:
log.info(
"Change Remote Proxy: ",
self.default_remote_host + ":" +
self.default_remote_port)
sock_out = sock_out.get_client(
self.default_remote_host,
self.default_remote_port)
sock_out = sock_out.get_client(self.default_remote_host,
self.default_remote_port)
except Exception as e:
if isinstance(sock_out, ForwardClient):
sock_out = sock_out.get_client(self.default_remote_host,
self.default_remote_port)
log.error('Socket read error of %s: %s' % (addr_in, str(e)))
break
if not data:
log.info('Socket closed by ' + addr_in)
break
addr_out = '%s:%d' % sock_out.getpeername()
try:
sock_out.sendall(data)
except Exception as e:
log.error('Socket write error of %s: %s' % (addr_out, str(e)))
break
log.info('%s -> %s (%d B)' % (addr_in, addr_out, len(data)))
sock_in.close()
sock_out.close()