def main_run():
redis = RedisConfig("127.0.0.1", 21009)
p1 = ProxyPipe(url_prefix="https://www.baidu.com",
fetcher=Fetcher(use_default=False),
database=redis,
checker=CheckerBaidu()).set_fetcher(
[KuaiProvider()]).add_fetcher([XiciProvider()])
p1.start()
def main_run():
redis = RedisConfig("127.0.0.1", 21009)
p1 = ProxyPipe(url_prefix="https://www.baidu.com", database=redis)
p2 = ProxyPipe(url_prefix="https://www.google.com", database=redis)
p3 = ProxyPipe(database=redis)
proc_list = [
Process(target=p1.start),
Process(target=p2.start),
Process(target=p3.start)
]
for p in proc_list:
p.start()
time.sleep(1)
for p in proc_list:
p.join()
import json
import random
from django.http import HttpResponse
from spoon_server.database.redis_config import RedisConfig
from spoon_server.main.manager import Manager
redis = RedisConfig("127.0.0.1", 21009)
def get_keys(request):
m = Manager(database=redis)
return HttpResponse(json.dumps(m.get_keys()))
def fetchone_from(request):
m = Manager(database=redis)
target_name = request.GET.get("target", "www.baidu.com")
filter_num = int(request.GET.get("filter", 10))
search_name = ":".join(["spoon", target_name, "useful_proxy"])
px_kv = m.get_all_kv_from(search_name)
res_list = random.sample([
k.decode('utf-8')
for (k, v) in px_kv.items() if int(v.decode('utf-8')) > filter_num
], 1)
return HttpResponse(res_list[0])
def fetchall_from(request):
def main_run():
redis = RedisConfig("127.0.0.1", 21009)
p1 = ProxyPipe(url_prefix="https://www.google.com",
fetcher=Fetcher(use_default=False),
database=redis).set_fetcher([UsProvider()])
p1.start()
import json
import time
import random
from django.http import HttpResponse
from spoon_server.database.redis_config import RedisConfig
from spoon_server.main.manager import Manager
redis = RedisConfig("127.0.0.1", 6379, 0)
def get_keys(request):
m = Manager(database=redis)
return HttpResponse(json.dumps(m.get_keys()))
def fetchone_from(request):
m = Manager(database=redis)
target_name = request.GET.get("target", "www.baidu.com")
filter_num = int(request.GET.get("filter", 10))
search_name = ":".join(["spoon", target_name, "useful_proxy"])
px_kv = m.get_all_kv_from(search_name)
res_list = random.sample([k.decode('utf-8') for (k, v) in px_kv.items() if int(v.decode('utf-8')) > filter_num], 1)
return HttpResponse(res_list[0])
def fetchall_from(request):
m = Manager(database=redis)
target_name = request.GET.get("target", "www.baidu.com")
try:
print('remote,=', (self.remote_host, self.remote_port))
sock_out.connect((self.remote_host, self.remote_port))
except socket.error as e:
sock_out.close()
log.error('Remote connect error: %s' % str(e))
raise Exception('Remote connect error: %s' % str(e))
return sock_out
def handler(signum, frame):
print(signum, frame)
global is_exit
is_exit = True
print("receive a signal %d, is_exit = %d" % (signum, is_exit))
if __name__ == '__main__':
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
listen = ("127.0.0.1", 12001)
default_remote = ("47.93.234.57", 42251)
redis = RedisConfig("10.1.10.10", 6379, 0, 123456)
serv = ForwardServer(redis)
serv.set_listen(listen[0], listen[1])
serv.set_default_remote(default_remote[0], default_remote[1])
serv.serve()