def validate(target_url, proxy, checker): if target_url == "default": target_url = "https://www.baidu.com" proxies = { "http": "http://{proxy}".format(proxy=proxy), "https": "http://{proxy}".format(proxy=proxy) } try: r = requests.get(target_url, proxies=proxies, timeout=checker.timeout, verify=False, headers=HEADERS_IPHONE) if r.status_code == 200: if checker.checker_func(r.content): log.info('validate success target {0} proxy {1}'.format( target_url, proxy)) return True else: return False else: return False except Exception as e: log.error("validate failed with {0}".format(e)) return False
def __init__(self, database=None, url_prefix=None, fetcher=None, checker=None): if not database: self.database = RedisWrapper("127.0.0.1", 6379, 0) else: self.database = RedisWrapper(database.host, database.port, database.db, database.password) self._origin_prefix = 'origin_proxy' self._useful_prefix = 'useful_proxy' self._hundred_prefix = 'hundred_proxy' self._current_prefix = 'current_proxy' if not url_prefix: self._url_prefix = "default" else: self._url_prefix = url_prefix if not fetcher: # validater self._fetcher = Fetcher() else: # refresher self._fetcher = fetcher self._fetcher.backup_provider() log.info("REFRESH FETCHER BACKUP PROVIDER {0}".format( str(self._fetcher))) if not checker: self._checker = Checker() else: self._checker = checker self.log = log
def _do_data_forward(self, sock_in, sock_out): addr_in = '%s:%d' % sock_in.getpeername() addr_out = '%s:%d' % sock_out.getpeername() while True: try: data = sock_in.recv(ForwardServer.PAGE_SIZE) except Exception as e: log.error('Socket read error of %s: %s' % (addr_in, str(e))) break if not data: log.info('Socket closed by ' + addr_in) break try: sock_out.sendall(data) except Exception as e: log.error('Socket write error of %s: %s' % (addr_out, str(e))) break log.info('%s -> %s (%d B)' % (addr_in, addr_out, len(data))) sock_in.close() sock_out.close()
def validate(target_url, proxy): if target_url == "default": target_url = "https://www.baidu.com" proxies = {"https": "https://{proxy}".format(proxy=proxy)} else: if urlparse(target_url).scheme == "https": proxies = {"https": "https://{proxy}".format(proxy=proxy)} else: proxies = { "http": "http://{proxy}".format(proxy=proxy), "https": "http://{proxy}".format(proxy=proxy) } try: r = requests.get(target_url, proxies=proxies, timeout=5, verify=False, headers=HEADERS_IPHONE) if r.status_code == 200: log.info('validate success target {0} proxy{1}'.format( target_url, proxy)) return True else: return False except Exception as e: log.error("{0}".format(e)) return False
def _listen(self): sock_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # tcp sock_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock_server.bind((self.listen_host, self.listen_port)) sock_server.listen(5) log.info('Listening at %s:%d ...' % (self.listen_host, self.listen_port)) return sock_server
def _forward(self, sock_in): try: print("Remote host and remote port", self.default_remote_host, self.default_remote_port) sock_out = ForwardClient(self.default_remote_host, self.default_remote_port).get_client() log.info('get the client socks done') except Exception as e: log.error('Get Remote Client error: %s' % str(e)) raise e threading.Thread(target=self._do_data_forward, args=(sock_in, sock_out)).start() threading.Thread(target=self._do_data_forward, args=(sock_out, sock_in)).start()
def _forward(self, sock_in): try: sock_out = ForwardClient() log.info('get the client socks done') except Exception as e: log.error('Get Remote Client error: %s' % str(e)) raise e threading.Thread(target=self._do_data_forward, args=(sock_in, sock_out)).start() threading.Thread(target=self._do_data_forward, args=(sock_out, sock_in)).start()
def serve(self): sock_server = self._listen() while not is_exit: try: sock, addr = sock_server.accept() except (KeyboardInterrupt, SystemExit): log.warn('Closing...') sock_server.shutdown(socket.SHUT_RDWR) sock_server.close() break except Exception as e: log.error('Exception exit {0}'.format(e)) sock_server.shutdown(socket.SHUT_RDWR) sock_server.close() break threading.Thread(target=self._forward, args=(sock,)).start() log.info('New clients from {0}'.format(addr)) log.info('exit server')
def refresh(self): log.info("REFRESH START WITH {0} TARGET {1}".format( str(self._fetcher), self.get_netloc())) if not self.refresh_condition(): log.info("REFRESH DID NOT MEET CONDITION. TARGET{0}".format( self.get_netloc())) return if len(self._fetcher) < 6: self._fetcher.restore_provider() log.info( "REFRESH FETCHER FAILED: NO ENOUGH PROVIDER, RESTORE PROVIDERS TO {0} for TARGET {1}" .format(str(self._fetcher), self.get_netloc())) proxy_set = set() provider_to_be_removed_index = [] for index in range(len(self._fetcher)): provider = self._fetcher.get_provider(index) try: for proxy in provider.getter(): if proxy.strip(): self.log.info( "REFRESH FETCHER: TARGET {0} PROVIDER {1} PROXY {2}" .format(self.get_netloc(), provider.__class__.__name__, proxy.strip())) proxy_set.add(proxy.strip()) except Exception as e: provider_to_be_removed_index.append(index) log.error( "REFRESH FETCHER FAILED: PROVIDER {0} WILL BE REMOVED ERROR {1}" .format(provider.__class__.__name__, e)) for proxy in proxy_set: self.database.set_value("spoon:proxy_stale", proxy, time.time()) self.database.put(self.generate_name(self._origin_prefix), proxy) log.info("REFRESH FETCHER DELETE {0}. TARGET {1}".format( provider_to_be_removed_index, self.get_netloc())) self._fetcher.remove_provider(provider_to_be_removed_index)
def start(): try: pid = os.fork() if pid > 0: # exit first parent log.info('parent process exit') sys.exit(0) except OSError as e: sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror)) sys.exit(1) # write pid pid = str(os.getpid()) pidfile = "./proxy_daemon.pid" if os.path.isfile(pidfile): f = open(pidfile, 'r') file_pid = int(f.read()) log.info('read pid file pid=%s' % file_pid) if pid_exists(file_pid): log.info("%s already exists, and pid=%s exists exiting" % (pidfile, file_pid)) sys.exit(1) else: log.info('the pid file pid=%s not exists' % file_pid) f.close() open(pidfile, 'w').write(pid) log.info('write pid to %s' % pidfile) log.info('now is child process do') re_ip_port = r'^(?P<addr>.+:)?(?P<port>[0-9]{1,5})$' listen = "127.0.0.1:12001" remote = "119.39.48.205:9090" local_addr, local_port = None, None remote_addr, remote_port = None, None x = re.match(re_ip_port, listen) if not x: log.info('listen format error!') sys.exit(1) local_addr = x.group('addr') or '0.0.0.0' local_addr = local_addr.rstrip(':') local_port = int(x.group('port')) x = re.match(re_ip_port, remote) if not x: log.info('listen format error!') sys.exit(1) remote_addr = x.group('addr') or '0.0.0.0' remote_addr = remote_addr.rstrip(':') remote_port = int(x.group('port')) threading.Thread( target=run_proxy, args=(local_addr, local_port, remote_addr, remote_port) ).start() log.info('start all proxy done')
def exit(): pidfile = "./proxy_daemon.pid" os.remove(pidfile) log.info('exit')
def _do_data_forward(self, sock_in, sock_out): if isinstance(sock_in, ForwardClient): sock_in = sock_in.get_client(self.default_remote_host, self.default_remote_port) addr_in = '%s:%d' % sock_in.getpeername() while True: try: data = sock_in.recv(ForwardServer.PAGE_SIZE) if isinstance(sock_out, ForwardClient): print("sock_in", data) if b'Host' in data: host_match = re.match(r'.*Host:\s(.*?)\r\n.*', data.decode("utf-8"), re.S) if host_match: hostname = host_match[1] current_proxy_list = self.m.get_range_from( ":".join(["spoon", hostname, "current_proxy"])) if current_proxy_list: ran_num = random.randint( 0, len(current_proxy_list) // 3) proxy = current_proxy_list[ran_num].decode( "utf-8") sock_out = sock_out.get_client( proxy.split(":")[0], int(proxy.split(":")[1])) log.info( "Change Remote Proxy: {0}".format(proxy)) else: log.info( "Change Remote Proxy: ", self.default_remote_host + ":" + self.default_remote_port) sock_out = sock_out.get_client( self.default_remote_host, self.default_remote_port) sock_out = sock_out.get_client(self.default_remote_host, self.default_remote_port) except Exception as e: if isinstance(sock_out, ForwardClient): sock_out = sock_out.get_client(self.default_remote_host, self.default_remote_port) log.error('Socket read error of %s: %s' % (addr_in, str(e))) break if not data: log.info('Socket closed by ' + addr_in) break addr_out = '%s:%d' % sock_out.getpeername() try: sock_out.sendall(data) except Exception as e: log.error('Socket write error of %s: %s' % (addr_out, str(e))) break log.info('%s -> %s (%d B)' % (addr_in, addr_out, len(data))) sock_in.close() sock_out.close()