def validate(target_url, proxy, checker):
if target_url == "default":
target_url = "https://www.baidu.com"
proxies = {
"http": "http://{proxy}".format(proxy=proxy),
"https": "http://{proxy}".format(proxy=proxy)
}
try:
r = requests.get(target_url,
proxies=proxies,
timeout=checker.timeout,
verify=False,
headers=HEADERS_IPHONE)
if r.status_code == 200:
if checker.checker_func(r.content):
log.info('validate success target {0} proxy {1}'.format(
target_url, proxy))
return True
else:
return False
else:
return False
except Exception as e:
log.error("validate failed with {0}".format(e))
return False
def __init__(self,
database=None,
url_prefix=None,
fetcher=None,
checker=None):
if not database:
self.database = RedisWrapper("127.0.0.1", 6379, 0)
else:
self.database = RedisWrapper(database.host, database.port,
database.db, database.password)
self._origin_prefix = 'origin_proxy'
self._useful_prefix = 'useful_proxy'
self._hundred_prefix = 'hundred_proxy'
self._current_prefix = 'current_proxy'
if not url_prefix:
self._url_prefix = "default"
else:
self._url_prefix = url_prefix
if not fetcher: # validater
self._fetcher = Fetcher()
else: # refresher
self._fetcher = fetcher
self._fetcher.backup_provider()
log.info("REFRESH FETCHER BACKUP PROVIDER {0}".format(
str(self._fetcher)))
if not checker:
self._checker = Checker()
else:
self._checker = checker
self.log = log
def _do_data_forward(self, sock_in, sock_out):
addr_in = '%s:%d' % sock_in.getpeername()
addr_out = '%s:%d' % sock_out.getpeername()
while True:
try:
data = sock_in.recv(ForwardServer.PAGE_SIZE)
except Exception as e:
log.error('Socket read error of %s: %s' % (addr_in, str(e)))
break
if not data:
log.info('Socket closed by ' + addr_in)
break
try:
sock_out.sendall(data)
except Exception as e:
log.error('Socket write error of %s: %s' % (addr_out, str(e)))
break
log.info('%s -> %s (%d B)' % (addr_in, addr_out, len(data)))
sock_in.close()
sock_out.close()
def validate(target_url, proxy):
if target_url == "default":
target_url = "https://www.baidu.com"
proxies = {"https": "https://{proxy}".format(proxy=proxy)}
else:
if urlparse(target_url).scheme == "https":
proxies = {"https": "https://{proxy}".format(proxy=proxy)}
else:
proxies = {
"http": "http://{proxy}".format(proxy=proxy),
"https": "http://{proxy}".format(proxy=proxy)
}
try:
r = requests.get(target_url,
proxies=proxies,
timeout=5,
verify=False,
headers=HEADERS_IPHONE)
if r.status_code == 200:
log.info('validate success target {0} proxy{1}'.format(
target_url, proxy))
return True
else:
return False
except Exception as e:
log.error("{0}".format(e))
return False
def _listen(self):
sock_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # tcp
sock_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock_server.bind((self.listen_host, self.listen_port))
sock_server.listen(5)
log.info('Listening at %s:%d ...' % (self.listen_host, self.listen_port))
return sock_server
def _forward(self, sock_in):
try:
print("Remote host and remote port", self.default_remote_host, self.default_remote_port)
sock_out = ForwardClient(self.default_remote_host, self.default_remote_port).get_client()
log.info('get the client socks done')
except Exception as e:
log.error('Get Remote Client error: %s' % str(e))
raise e
threading.Thread(target=self._do_data_forward, args=(sock_in, sock_out)).start()
threading.Thread(target=self._do_data_forward, args=(sock_out, sock_in)).start()
def _forward(self, sock_in):
try:
sock_out = ForwardClient()
log.info('get the client socks done')
except Exception as e:
log.error('Get Remote Client error: %s' % str(e))
raise e
threading.Thread(target=self._do_data_forward,
args=(sock_in, sock_out)).start()
threading.Thread(target=self._do_data_forward,
args=(sock_out, sock_in)).start()
def serve(self):
sock_server = self._listen()
while not is_exit:
try:
sock, addr = sock_server.accept()
except (KeyboardInterrupt, SystemExit):
log.warn('Closing...')
sock_server.shutdown(socket.SHUT_RDWR)
sock_server.close()
break
except Exception as e:
log.error('Exception exit {0}'.format(e))
sock_server.shutdown(socket.SHUT_RDWR)
sock_server.close()
break
threading.Thread(target=self._forward, args=(sock,)).start()
log.info('New clients from {0}'.format(addr))
log.info('exit server')
def refresh(self):
log.info("REFRESH START WITH {0} TARGET {1}".format(
str(self._fetcher), self.get_netloc()))
if not self.refresh_condition():
log.info("REFRESH DID NOT MEET CONDITION. TARGET{0}".format(
self.get_netloc()))
return
if len(self._fetcher) < 6:
self._fetcher.restore_provider()
log.info(
"REFRESH FETCHER FAILED: NO ENOUGH PROVIDER, RESTORE PROVIDERS TO {0} for TARGET {1}"
.format(str(self._fetcher), self.get_netloc()))
proxy_set = set()
provider_to_be_removed_index = []
for index in range(len(self._fetcher)):
provider = self._fetcher.get_provider(index)
try:
for proxy in provider.getter():
if proxy.strip():
self.log.info(
"REFRESH FETCHER: TARGET {0} PROVIDER {1} PROXY {2}"
.format(self.get_netloc(),
provider.__class__.__name__,
proxy.strip()))
proxy_set.add(proxy.strip())
except Exception as e:
provider_to_be_removed_index.append(index)
log.error(
"REFRESH FETCHER FAILED: PROVIDER {0} WILL BE REMOVED ERROR {1}"
.format(provider.__class__.__name__, e))
for proxy in proxy_set:
self.database.set_value("spoon:proxy_stale", proxy,
time.time())
self.database.put(self.generate_name(self._origin_prefix),
proxy)
log.info("REFRESH FETCHER DELETE {0}. TARGET {1}".format(
provider_to_be_removed_index, self.get_netloc()))
self._fetcher.remove_provider(provider_to_be_removed_index)
def start():
try:
pid = os.fork()
if pid > 0:
# exit first parent
log.info('parent process exit')
sys.exit(0)
except OSError as e:
sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
sys.exit(1)
# write pid
pid = str(os.getpid())
pidfile = "./proxy_daemon.pid"
if os.path.isfile(pidfile):
f = open(pidfile, 'r')
file_pid = int(f.read())
log.info('read pid file pid=%s' % file_pid)
if pid_exists(file_pid):
log.info("%s already exists, and pid=%s exists exiting" %
(pidfile, file_pid))
sys.exit(1)
else:
log.info('the pid file pid=%s not exists' % file_pid)
f.close()
open(pidfile, 'w').write(pid)
log.info('write pid to %s' % pidfile)
log.info('now is child process do')
re_ip_port = r'^(?P<addr>.+:)?(?P<port>[0-9]{1,5})$'
listen = "127.0.0.1:12001"
remote = "119.39.48.205:9090"
local_addr, local_port = None, None
remote_addr, remote_port = None, None
x = re.match(re_ip_port, listen)
if not x:
log.info('listen format error!')
sys.exit(1)
local_addr = x.group('addr') or '0.0.0.0'
local_addr = local_addr.rstrip(':')
local_port = int(x.group('port'))
x = re.match(re_ip_port, remote)
if not x:
log.info('listen format error!')
sys.exit(1)
remote_addr = x.group('addr') or '0.0.0.0'
remote_addr = remote_addr.rstrip(':')
remote_port = int(x.group('port'))
threading.Thread(
target=run_proxy,
args=(local_addr, local_port,
remote_addr, remote_port)
).start()
log.info('start all proxy done')
def exit():
pidfile = "./proxy_daemon.pid"
os.remove(pidfile)
log.info('exit')
def _do_data_forward(self, sock_in, sock_out):
if isinstance(sock_in, ForwardClient):
sock_in = sock_in.get_client(self.default_remote_host,
self.default_remote_port)
addr_in = '%s:%d' % sock_in.getpeername()
while True:
try:
data = sock_in.recv(ForwardServer.PAGE_SIZE)
if isinstance(sock_out, ForwardClient):
print("sock_in", data)
if b'Host' in data:
host_match = re.match(r'.*Host:\s(.*?)\r\n.*',
data.decode("utf-8"), re.S)
if host_match:
hostname = host_match[1]
current_proxy_list = self.m.get_range_from(
":".join(["spoon", hostname, "current_proxy"]))
if current_proxy_list:
ran_num = random.randint(
0,
len(current_proxy_list) // 3)
proxy = current_proxy_list[ran_num].decode(
"utf-8")
sock_out = sock_out.get_client(
proxy.split(":")[0],
int(proxy.split(":")[1]))
log.info(
"Change Remote Proxy: {0}".format(proxy))
else:
log.info(
"Change Remote Proxy: ",
self.default_remote_host + ":" +
self.default_remote_port)
sock_out = sock_out.get_client(
self.default_remote_host,
self.default_remote_port)
sock_out = sock_out.get_client(self.default_remote_host,
self.default_remote_port)
except Exception as e:
if isinstance(sock_out, ForwardClient):
sock_out = sock_out.get_client(self.default_remote_host,
self.default_remote_port)
log.error('Socket read error of %s: %s' % (addr_in, str(e)))
break
if not data:
log.info('Socket closed by ' + addr_in)
break
addr_out = '%s:%d' % sock_out.getpeername()
try:
sock_out.sendall(data)
except Exception as e:
log.error('Socket write error of %s: %s' % (addr_out, str(e)))
break
log.info('%s -> %s (%d B)' % (addr_in, addr_out, len(data)))
sock_in.close()
sock_out.close()
