def main_run(): redis = RedisConfig("127.0.0.1", 21009) p1 = ProxyPipe(url_prefix="https://www.baidu.com", fetcher=Fetcher(use_default=False), database=redis, checker=CheckerBaidu()).set_fetcher( [KuaiProvider()]).add_fetcher([XiciProvider()]) p1.start()
def main_run(): redis = RedisConfig("127.0.0.1", 21009) p1 = ProxyPipe(url_prefix="https://www.baidu.com", database=redis) p2 = ProxyPipe(url_prefix="https://www.google.com", database=redis) p3 = ProxyPipe(database=redis) proc_list = [ Process(target=p1.start), Process(target=p2.start), Process(target=p3.start) ] for p in proc_list: p.start() time.sleep(1) for p in proc_list: p.join()
import json import random from django.http import HttpResponse from spoon_server.database.redis_config import RedisConfig from spoon_server.main.manager import Manager redis = RedisConfig("127.0.0.1", 21009) def get_keys(request): m = Manager(database=redis) return HttpResponse(json.dumps(m.get_keys())) def fetchone_from(request): m = Manager(database=redis) target_name = request.GET.get("target", "www.baidu.com") filter_num = int(request.GET.get("filter", 10)) search_name = ":".join(["spoon", target_name, "useful_proxy"]) px_kv = m.get_all_kv_from(search_name) res_list = random.sample([ k.decode('utf-8') for (k, v) in px_kv.items() if int(v.decode('utf-8')) > filter_num ], 1) return HttpResponse(res_list[0]) def fetchall_from(request):
def main_run(): redis = RedisConfig("127.0.0.1", 21009) p1 = ProxyPipe(url_prefix="https://www.google.com", fetcher=Fetcher(use_default=False), database=redis).set_fetcher([UsProvider()]) p1.start()
import json import time import random from django.http import HttpResponse from spoon_server.database.redis_config import RedisConfig from spoon_server.main.manager import Manager redis = RedisConfig("127.0.0.1", 6379, 0) def get_keys(request): m = Manager(database=redis) return HttpResponse(json.dumps(m.get_keys())) def fetchone_from(request): m = Manager(database=redis) target_name = request.GET.get("target", "www.baidu.com") filter_num = int(request.GET.get("filter", 10)) search_name = ":".join(["spoon", target_name, "useful_proxy"]) px_kv = m.get_all_kv_from(search_name) res_list = random.sample([k.decode('utf-8') for (k, v) in px_kv.items() if int(v.decode('utf-8')) > filter_num], 1) return HttpResponse(res_list[0]) def fetchall_from(request): m = Manager(database=redis) target_name = request.GET.get("target", "www.baidu.com")
try: print('remote,=', (self.remote_host, self.remote_port)) sock_out.connect((self.remote_host, self.remote_port)) except socket.error as e: sock_out.close() log.error('Remote connect error: %s' % str(e)) raise Exception('Remote connect error: %s' % str(e)) return sock_out def handler(signum, frame): print(signum, frame) global is_exit is_exit = True print("receive a signal %d, is_exit = %d" % (signum, is_exit)) if __name__ == '__main__': signal.signal(signal.SIGINT, handler) signal.signal(signal.SIGTERM, handler) listen = ("127.0.0.1", 12001) default_remote = ("47.93.234.57", 42251) redis = RedisConfig("10.1.10.10", 6379, 0, 123456) serv = ForwardServer(redis) serv.set_listen(listen[0], listen[1]) serv.set_default_remote(default_remote[0], default_remote[1]) serv.serve()