def test_alive(ip_str="180.188.250.54", begin=45, end=60, interval=2):
start_time = time.time()
test_array = {}
for i in range(begin, end, interval):
sslsock, _, _ = connect_ssl(ip_str)
try:
result = test_app_check(sslsock, ip_str)
except:
break
test_array[i] = {}
test_array[i]["ssl"] = sslsock
test_array[i]["start_time"] = time.time()
time.sleep(interval)
time_now = time.time()
time.sleep(end - (time_now - start_time))
for i in test_array:
stat = test_array[i]
sslsock = stat["ssl"]
time_now = time.time()
try:
result = test_app_check(sslsock, ip_str)
xlog.info("time alive:%d", time_now - stat["start_time"])
except:
xlog.info("time alive fail:%d", time_now - stat["start_time"])
continue
def create_ca():
key = OpenSSL.crypto.PKey()
key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
req = OpenSSL.crypto.X509Req()
subj = req.get_subject()
subj.countryName = 'CN'
subj.stateOrProvinceName = 'Internet'
subj.localityName = 'Cernet'
subj.organizationName = CertUtil.ca_vendor
subj.organizationalUnitName = '%s Root' % CertUtil.ca_vendor
subj.commonName = '%s XX-Net' % CertUtil.ca_vendor #TODO: here should be GoAgent
req.set_pubkey(key)
req.sign(key, CertUtil.ca_digest)
ca = OpenSSL.crypto.X509()
ca.set_version(2)
ca.set_serial_number(0)
ca.gmtime_adj_notBefore(0)
ca.gmtime_adj_notAfter(24 * 60 * 60 * 3652)
ca.set_issuer(req.get_subject())
ca.set_subject(req.get_subject())
ca.set_pubkey(req.get_pubkey())
ca.add_extensions([
OpenSSL.crypto.X509Extension('basicConstraints', False, 'CA:TRUE',
ca, ca)
])
ca.sign(key, CertUtil.ca_digest)
#logging.debug("CA key:%s", key)
xlog.info("create ca")
return key, ca
def scan_ip_worker(self):
while self.searching_thread_count <= self.scan_ip_thread_num:
if not connect_control.allow_scan():
time.sleep(10)
continue
try:
time.sleep(1)
ip_int = ip_range.get_ip()
ip_str = ip_utils.ip_num_to_string(ip_int)
if self.is_bad_ip(ip_str):
continue
result = check_ip.test_gws(ip_str)
if not result:
continue
if self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type):
#logging.info("add %s CN:%s type:%s time:%d gws:%d ", ip_str,
# result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list))
xlog.info("scan_ip add ip:%s time:%d", ip_str, result.handshake_time)
scan_ip_log.info("Add %s time:%d CN:%s type:%s", ip_str, result.handshake_time, result.domain, result.server_type)
self.remove_slowest_ip()
self.save_ip_list()
except check_ip.HoneypotError as e:
self.report_bad_ip(ip_str)
connect_control.fall_into_honeypot()
continue
except Exception as e:
xlog.exception("google_ip.runJob fail:%s", e)
self.ncount_lock.acquire()
self.searching_thread_count -= 1
self.ncount_lock.release()
xlog.info("scan_ip_worker exit")
def test2(self):
work_ciphers = ["AES128-SHA"]
for cipher in self.cipher_list:
if cipher in work_ciphers:
continue
else:
work_ciphers.append(cipher)
xlog.debug("%s", cipher)
cipher_suites = (work_ciphers)
openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=cipher_suites)
try:
ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context)
server_type = test_server_type(ssl, self.ip)
xlog.debug("%s", server_type)
if "gws" not in server_type:
work_ciphers.remove(cipher)
except Exception as e:
xlog.warn("err:%s", e)
try:
work_ciphers.remove(cipher)
except:
pass
work_str = ""
for cipher in work_ciphers:
work_str += cipher + ":"
xlog.info("work ciphers:%s", work_str)
def context_builder(ca_certs=None, cipher_suites=('ALL:!RC4-SHA:!ECDHE-RSA-RC4-SHA:!ECDHE-RSA-AES128-GCM-SHA256:!AES128-GCM-SHA256',)):
# 'ALL', '!aNULL', '!eNULL'
global ssl_version
if not ssl_version:
if hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"):
ssl_version = "TLSv1_2"
elif hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"):
ssl_version = "TLSv1_1"
elif hasattr(OpenSSL.SSL, "TLSv1_METHOD"):
ssl_version = "TLSv1"
else:
ssl_version = "SSLv23"
if sys.platform == "darwin":
ssl_version = "TLSv1"
# freenas openssl support fix from twitter user "himanzero"
# https://twitter.com/himanzero/status/645231724318748672
if sys.platform == "freebsd9":
ssl_version = "TLSv1"
xlog.info("SSL use version:%s", ssl_version)
protocol_version = getattr(OpenSSL.SSL, '%s_METHOD' % ssl_version)
ssl_context = OpenSSL.SSL.Context(protocol_version)
if ca_certs:
ssl_context.load_verify_locations(os.path.abspath(ca_certs))
ssl_context.set_verify(OpenSSL.SSL.VERIFY_PEER, lambda c, x, e, d, ok: ok)
else:
ssl_context.set_verify(OpenSSL.SSL.VERIFY_NONE, lambda c, x, e, d, ok: ok)
ssl_context.set_cipher_list(':'.join(cipher_suites))
return ssl_context
def check_win10():
if sys.platform != "win32":
return False
import ctypes
class OSVERSIONINFOEXW(ctypes.Structure):
_fields_ = [
("dwOSVersionInfoSize", ctypes.c_ulong),
("dwMajorVersion", ctypes.c_ulong),
("dwMinorVersion", ctypes.c_ulong),
("dwBuildNumber", ctypes.c_ulong),
("dwPlatformId", ctypes.c_ulong),
("szCSDVersion", ctypes.c_wchar * 128),
("wServicePackMajor", ctypes.c_ushort),
("wServicePackMinor", ctypes.c_ushort),
("wSuiteMask", ctypes.c_ushort),
("wProductType", ctypes.c_byte),
("wReserved", ctypes.c_byte),
]
os_version = OSVERSIONINFOEXW()
os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version)
retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version))
if retcode != 0:
xlog.warn("Failed to get win32 OS version")
return False
if os_version.dwMajorVersion == 10:
xlog.info("detect Win10, enable connect concurent control.")
return True
return False
def load_openssl():
global loaded, libcrypto, buf
from ctypes.util import find_library
for p in ('crypto', 'eay32', 'libeay32'):
libcrypto_path = find_library(p)
if libcrypto_path:
break
else:
raise Exception('libcrypto(OpenSSL) not found')
xlog.info('loading libcrypto from %s', libcrypto_path)
libcrypto = CDLL(libcrypto_path)
libcrypto.EVP_get_cipherbyname.restype = c_void_p
libcrypto.EVP_CIPHER_CTX_new.restype = c_void_p
libcrypto.EVP_CipherInit_ex.argtypes = (c_void_p, c_void_p, c_char_p,
c_char_p, c_char_p, c_int)
libcrypto.EVP_CipherUpdate.argtypes = (c_void_p, c_void_p, c_void_p,
c_char_p, c_int)
libcrypto.EVP_CIPHER_CTX_cleanup.argtypes = (c_void_p,)
libcrypto.EVP_CIPHER_CTX_free.argtypes = (c_void_p,)
if hasattr(libcrypto, 'OpenSSL_add_all_ciphers'):
libcrypto.OpenSSL_add_all_ciphers()
buf = create_string_buffer(buf_size)
loaded = True
def test2(self):
work_ciphers = ["AES128-SHA"]
for cipher in self.cipher_list:
if cipher in work_ciphers:
continue
else:
work_ciphers.append(cipher)
xlog.debug("%s", cipher)
cipher_suites = (work_ciphers)
openssl_context = SSLConnection.context_builder(
ca_certs=g_cacertfile, cipher_suites=cipher_suites)
try:
ssl, _, _ = connect_ssl(self.ip,
openssl_context=openssl_context)
server_type = test_server_type(ssl, self.ip)
xlog.debug("%s", server_type)
if "gws" not in server_type:
work_ciphers.remove(cipher)
except Exception as e:
xlog.warn("err:%s", e)
try:
work_ciphers.remove(cipher)
except:
pass
work_str = ""
for cipher in work_ciphers:
work_str += cipher + ":"
xlog.info("work ciphers:%s", work_str)
def load_ip(self):
if os.path.isfile(self.good_ip_file):
file_path = self.good_ip_file
else:
file_path = self.default_good_ip_file
with open(file_path, "r") as fd:
lines = fd.readlines()
for line in lines:
try:
if line.startswith("#"):
continue
str_l = line.split(" ")
if len(str_l) < 4:
xlog.warning("line err: %s", line)
continue
ip_str = str_l[0]
domain = str_l[1]
server = str_l[2]
handshake_time = int(str_l[3])
if len(str_l) > 4:
fail_times = int(str_l[4])
else:
fail_times = 0
# logging.info("load ip: %s time:%d domain:%s server:%s", ip_str, handshake_time, domain, server)
self.add_ip(ip_str, handshake_time, domain, server, fail_times)
except Exception as e:
xlog.exception("load_ip line:%s err:%s", line, e)
xlog.info("load google ip_list num:%d, gws num:%d", len(self.ip_dict), len(self.gws_ip_list))
self.try_sort_gws_ip(force=True)
def remove_slowest_ip(self):
if len(self.gws_ip_list) <= self.max_good_ip_num:
return
self.try_sort_ip_by_handshake_time(force=True)
self.ip_lock.acquire()
try:
ip_num = len(self.gws_ip_list)
while ip_num > self.max_good_ip_num:
ip_str = self.gws_ip_list[ip_num - 1]
property = self.ip_dict[ip_str]
server = property['server']
handshake_time = property['handshake_time']
xlog.info("remove_slowest_ip:%s handshake_time:%d", ip_str,
handshake_time)
del self.ip_dict[ip_str]
if 'gws' in server and ip_str in self.gws_ip_list:
self.gws_ip_list.remove(ip_str)
ip_num -= 1
except Exception as e:
xlog.exception("remove_slowest_ip err:%s", e)
finally:
self.ip_lock.release()
def check_win10():
if sys.platform != "win32":
return False
import ctypes
class OSVERSIONINFOEXW(ctypes.Structure):
_fields_ = [('dwOSVersionInfoSize', ctypes.c_ulong),
('dwMajorVersion', ctypes.c_ulong),
('dwMinorVersion', ctypes.c_ulong),
('dwBuildNumber', ctypes.c_ulong),
('dwPlatformId', ctypes.c_ulong),
('szCSDVersion', ctypes.c_wchar*128),
('wServicePackMajor', ctypes.c_ushort),
('wServicePackMinor', ctypes.c_ushort),
('wSuiteMask', ctypes.c_ushort),
('wProductType', ctypes.c_byte),
('wReserved', ctypes.c_byte)]
os_version = OSVERSIONINFOEXW()
os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version)
retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version))
if retcode != 0:
xlog.warn("Failed to get win32 OS version")
return False
if os_version.dwMajorVersion == 10:
xlog.info("detect Win10, enable connect concurent control.")
return True
return False
def do_POST(self):
try:
refer = self.headers.getheader('Referer')
netloc = urlparse.urlparse(refer).netloc
if not netloc.startswith("127.0.0.1") and not netloc.startswitch("localhost"):
xlog.warn("web control ref:%s refuse", netloc)
return
except:
pass
xlog.debug ('GAEProxy web_control %s %s %s ', self.address_string(), self.command, self.path)
try:
ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
if ctype == 'multipart/form-data':
self.postvars = cgi.parse_multipart(self.rfile, pdict)
elif ctype == 'application/x-www-form-urlencoded':
length = int(self.headers.getheader('content-length'))
self.postvars = urlparse.parse_qs(self.rfile.read(length), keep_blank_values=1)
else:
self.postvars = {}
except:
self.postvars = {}
path = urlparse.urlparse(self.path).path
if path == '/deploy':
return self.req_deploy_handler()
elif path == "/config":
return self.req_config_handler()
elif path == "/scan_ip":
return self.req_scan_ip_handler()
elif path.startswith("/importip"):
return self.req_importip_handler()
else:
self.wfile.write(b'HTTP/1.1 404\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n404 Not Found')
xlog.info('%s "%s %s HTTP/1.1" 404 -', self.address_string(), self.command, self.path)
def scan_ip_worker(self):
while self.searching_thread_count <= self.scan_ip_thread_num and connect_control.keep_running:
if not connect_control.allow_scan():
time.sleep(10)
continue
try:
time.sleep(1)
ip_int = ip_range.get_ip()
ip_str = ip_utils.ip_num_to_string(ip_int)
if ip_str in self.ip_dict:
continue
connect_control.start_connect_register()
result = check_ip.test_gae(ip_str)
connect_control.end_connect_register()
if not result:
continue
if self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type):
#logging.info("add %s CN:%s type:%s time:%d gws:%d ", ip_str,
# result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list))
xlog.info("scan_ip add ip:%s time:%d", ip_str, result.handshake_time)
scan_ip_log.info("Add %s time:%d CN:%s type:%s", ip_str, result.handshake_time, result.domain, result.server_type)
self.remove_slowest_ip()
self.save_ip_list()
except Exception as e:
xlog.exception("google_ip.runJob fail:%s", e)
self.ncount_lock.acquire()
self.searching_thread_count -= 1
self.ncount_lock.release()
xlog.info("scan_ip_worker exit")
def test_alive(ip_str="74.125.96.107", begin=50, end=60, interval=2):
test_array = {}
for i in range(begin, end, interval):
sslsock, _, _ = connect_ssl(ip_str)
try:
result = test_app_check(sslsock, ip_str)
except:
break
test_array[i] = {}
test_array[i]["ssl"] = sslsock
test_array[i]["start_time"] = time.time()
time.sleep(interval)
time.sleep(begin)
for i in test_array:
stat = test_array[i]
sslsock = stat["ssl"]
time_now = time.time()
try:
result = test_app_check(sslsock, ip_str)
xlog.info("time alive:%d", time_now - stat["start_time"])
except:
xlog.info("time alive fail")
break
def remove_ip_process(self):
try:
while connect_control.keep_running:
try:
ip_str = self.to_remove_ip_list.get_nowait()
except:
break
result = check_ip.test(ip_str)
if result and result.appspot_ok:
self.add_ip(ip_str, result.handshake_time, result.domain,
result.server_type)
xlog.debug("remove ip process, restore ip:%s", ip_str)
continue
if not check_ip.network_is_ok():
self.to_remove_ip_list.put(ip_str)
xlog.warn(
"network is unreachable. check your network connection."
)
return
xlog.info("real remove ip:%s ", ip_str)
self.iplist_need_save = 1
finally:
self.remove_ip_thread_num_lock.acquire()
self.remove_ip_thread_num -= 1
self.remove_ip_thread_num_lock.release()
def check_all_exist_ip():
good_ip_file_name = "good_ip.txt"
good_ip_file = os.path.abspath( os.path.join(config.DATA_PATH, good_ip_file_name))
if not os.path.isfile(good_ip_file):
print "open file ", good_ip_file_name, " fail."
return
with open(good_ip_file, "r") as fd:
lines = fd.readlines()
for line in lines:
try:
str_l = line.split(' ')
if len(str_l) != 4:
xlog.warning("line err: %s", line)
continue
ip_str = str_l[0]
domain = str_l[1]
server = str_l[2]
handshake_time = int(str_l[3])
xlog.info("test ip: %s time:%d domain:%s server:%s", ip_str, handshake_time, domain, server)
#test_with_app(ip_str)
test_gws(ip_str)
#self.add_ip(ip_str, handshake_time, domain, server)
except Exception as e:
xlog.exception("load_ip line:%s err:%s", line, e)
def load_ip(self):
if os.path.isfile(self.good_ip_file):
file_path = self.good_ip_file
else:
file_path = self.default_good_ip_file
with open(file_path, "r") as fd:
lines = fd.readlines()
for line in lines:
try:
str_l = line.split(' ')
if len(str_l) < 4:
xlog.warning("line err: %s", line)
continue
ip_str = str_l[0]
domain = str_l[1]
server = str_l[2]
handshake_time = int(str_l[3])
if len(str_l) > 4:
fail_times = int(str_l[4])
else:
fail_times = 0
#logging.info("load ip: %s time:%d domain:%s server:%s", ip_str, handshake_time, domain, server)
self.add_ip(ip_str, handshake_time, domain, server, fail_times)
except Exception as e:
xlog.exception("load_ip line:%s err:%s", line, e)
xlog.info("load google ip_list num:%d, gws num:%d", len(self.ip_dict),
len(self.gws_ip_list))
self.try_sort_gws_ip(force=True)
def load_tasks():
tasks = []
intervals = []
import imp
tasks_path = config.tasks_path
for fileName in os.listdir(tasks_path):
if fileName.lower().endswith('.py'):
xlog.info('start load %s' % fileName)
try:
with open(os.path.join(tasks_path, fileName), 'rb') as fpy:
task = imp.load_source('Auto-tasks_%d_%s' % (len(tasks), fileName), tasks_path, fpy)
name = task.name
interval = task.run_interval
task.init()
tasks.append(task)
intervals.append(interval)
xlog.info('load task %s success.' % name)
except:
xlog.warn('load %s fail.' % fileName)
del imp
return (tasks, intervals)
def main(data_path="."):
xlog.info("listen http on 8880")
httpd = HTTPServer(('', 8880), TestHttpServer, data_path)
httpd.start()
while True:
time.sleep(10)
def remove_slowest_ip(self):
if len(self.gws_ip_list) <= self.max_good_ip_num:
return
self.try_sort_gws_ip(force=True)
self.ip_lock.acquire()
try:
ip_num = len(self.gws_ip_list)
while ip_num > self.max_good_ip_num:
ip_str = self.gws_ip_list[ip_num - 1]
property = self.ip_dict[ip_str]
server = property["server"]
fails = property["fail_times"]
handshake_time = property["handshake_time"]
xlog.info("remove_slowest_ip:%s handshake_time:%d, fails:%d", ip_str, handshake_time, fails)
del self.ip_dict[ip_str]
if "gws" in server and ip_str in self.gws_ip_list:
self.gws_ip_list.remove(ip_str)
ip_num -= 1
except Exception as e:
xlog.exception("remove_slowest_ip err:%s", e)
finally:
self.ip_lock.release()
def test(ip_str, loop=1):
xlog.info("==>%s", ip_str)
check = Check_frame(ip_str, check_cert=False)
for i in range(loop):
result = check.check(callback=test_app_head)
if not result:
if "gws" in check.result.server_type:
xlog.warn("ip:%s server_type:%s but appengine check fail.",
ip_str, check.result.server_type)
xlog.warn("check fail")
#continue
else:
xlog.debug("=======app check ok: %s", ip_str)
check.result.appspot_ok = result
result = check.check(callback=test_server_type, check_ca=True)
if not result:
xlog.debug("test server type fail")
continue
check.result.server_type = result
xlog.info("========== %s type:%s domain:%s handshake:%d", ip_str,
check.result.server_type, check.result.domain,
check.result.handshake_time)
return check.result
def test_alive(ip_str="180.188.250.54", begin=45, end=60, interval=2):
start_time = time.time()
test_array = {}
for i in range(begin, end, interval):
sslsock, _, _ = connect_ssl(ip_str)
try:
result = test_app_check(sslsock, ip_str)
except:
break
test_array[i] = {}
test_array[i]["ssl"] = sslsock
test_array[i]["start_time"] = time.time()
time.sleep(interval)
time_now = time.time()
time.sleep(end - (time_now - start_time))
for i in test_array:
stat = test_array[i]
sslsock = stat["ssl"]
time_now = time.time()
try:
result = test_app_check(sslsock, ip_str)
xlog.info("time alive:%d", time_now - stat["start_time"])
except:
xlog.info("time alive fail:%d", time_now - stat["start_time"])
continue
def check_all_exist_ip():
good_ip_file_name = "good_ip.txt"
good_ip_file = os.path.abspath(
os.path.join(config.DATA_PATH, good_ip_file_name))
if not os.path.isfile(good_ip_file):
print "open file ", good_ip_file_name, " fail."
return
with open(good_ip_file, "r") as fd:
lines = fd.readlines()
for line in lines:
try:
str_l = line.split(' ')
if len(str_l) != 4:
xlog.warning("line err: %s", line)
continue
ip_str = str_l[0]
domain = str_l[1]
server = str_l[2]
handshake_time = int(str_l[3])
xlog.info("test ip: %s time:%d domain:%s server:%s", ip_str,
handshake_time, domain, server)
#test_with_app(ip_str)
test_gws(ip_str)
#self.add_ip(ip_str, handshake_time, domain, server)
except Exception as e:
xlog.exception("load_ip line:%s err:%s", line, e)
def test(ip_str, loop=1):
xlog.info("==>%s", ip_str)
check = Check_frame(ip_str, check_cert=False)
for i in range(loop):
result = check.check(callback=test_app_head)
if not result:
if "gws" in check.result.server_type:
xlog.warn("ip:%s server_type:%s but appengine check fail.", ip_str, check.result.server_type)
xlog.warn("check fail")
#continue
else:
xlog.debug("=======app check ok: %s", ip_str)
check.result.appspot_ok = result
result = check.check(callback=test_server_type, check_ca=True)
if not result:
xlog.debug("test server type fail")
continue
check.result.server_type = result
xlog.info("========== %s type:%s domain:%s handshake:%d", ip_str, check.result.server_type,
check.result.domain, check.result.handshake_time)
return check.result
def remove_slowest_ip(self):
if len(self.gws_ip_list) <= self.max_good_ip_num:
return
self.try_sort_ip_by_handshake_time(force=True)
self.ip_lock.acquire()
try:
ip_num = len(self.gws_ip_list)
while ip_num > self.max_good_ip_num:
ip_str = self.gws_ip_list[ip_num - 1]
property = self.ip_dict[ip_str]
server = property['server']
handshake_time = property['handshake_time']
xlog.info("remove_slowest_ip:%s handshake_time:%d", ip_str, handshake_time)
del self.ip_dict[ip_str]
if 'gws' in server and ip_str in self.gws_ip_list:
self.gws_ip_list.remove(ip_str)
ip_num -= 1
except Exception as e:
xlog.exception("remove_slowest_ip err:%s", e)
finally:
self.ip_lock.release()
def create_ca():
key = OpenSSL.crypto.PKey()
key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
req = OpenSSL.crypto.X509Req()
subj = req.get_subject()
subj.countryName = 'CN'
subj.stateOrProvinceName = 'Internet'
subj.localityName = 'Cernet'
subj.organizationName = CertUtil.ca_vendor
subj.organizationalUnitName = '%s Root' % CertUtil.ca_vendor
subj.commonName = '%s XX-Net' % CertUtil.ca_vendor #TODO: here should be GoAgent
req.set_pubkey(key)
req.sign(key, CertUtil.ca_digest)
ca = OpenSSL.crypto.X509()
ca.set_version(2)
ca.set_serial_number(0)
ca.gmtime_adj_notBefore(0)
ca.gmtime_adj_notAfter(24 * 60 * 60 * 3652)
ca.set_issuer(req.get_subject())
ca.set_subject(req.get_subject())
ca.set_pubkey(req.get_pubkey())
ca.add_extensions([
OpenSSL.crypto.X509Extension(
'basicConstraints', False, 'CA:TRUE', ca, ca)
])
ca.sign(key, CertUtil.ca_digest)
#logging.debug("CA key:%s", key)
xlog.info("create ca")
return key, ca
def test_alive(ip_str="74.125.96.107", begin=50, end=60, interval=2):
test_array = {}
for i in range(begin, end, interval):
sslsock, _, _ = connect_ssl(ip_str)
try:
result = test_app_check(sslsock, ip_str)
except:
break
test_array[i] = {}
test_array[i]["ssl"] = sslsock
test_array[i]["start_time"] = time.time()
time.sleep(interval)
time.sleep(begin)
for i in test_array:
stat = test_array[i]
sslsock = stat["ssl"]
time_now = time.time()
try:
result = test_app_check(sslsock, ip_str)
xlog.info("time alive:%d", time_now - stat["start_time"])
except:
xlog.info("time alive fail")
break
def remove_ip_process(self):
try:
while True:
try:
ip_str = self.to_remove_ip_list.get_nowait()
except:
break
result = check_ip.test(ip_str)
if result and result.appspot_ok:
self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type)
xlog.debug("remove ip process, restore ip:%s", ip_str)
continue
if not check_ip.network_is_ok():
self.to_remove_ip_list.put(ip_str)
xlog.warn("network is unreachable. check your network connection.")
return
xlog.info("real remove ip:%s ", ip_str)
self.iplist_need_save = 1
finally:
self.remove_ip_thread_num_lock.acquire()
self.remove_ip_thread_num -= 1
self.remove_ip_thread_num_lock.release()
def import_mac_ca(common_name, certfile):
commonname = "GoAgent XX-Net" #TODO: need check again
ca_hash = CertUtil.ca_thumbprint.replace(':', '')
def get_exist_ca_sha1():
args = [
'security', 'find-certificate', '-Z', '-a', '-c', commonname
]
output = subprocess.check_output(args)
for line in output.splitlines(True):
if len(line) == 53 and line.startswith("SHA-1 hash:"):
sha1_hash = line[12:52]
return sha1_hash
exist_ca_sha1 = get_exist_ca_sha1()
if exist_ca_sha1 == ca_hash:
xlog.info("GoAgent CA exist")
return
import_command = 'security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ../../data/gae_proxy/CA.crt' # % certfile.decode('utf-8')
if exist_ca_sha1:
delete_ca_command = 'security delete-certificate -Z %s' % exist_ca_sha1
exec_command = "%s;%s" % (delete_ca_command, import_command)
else:
exec_command = import_command
admin_command = """osascript -e 'do shell script "%s" with administrator privileges' """ % exec_command
cmd = admin_command.encode('utf-8')
xlog.info("try auto import CA command:%s", cmd)
os.system(cmd)
def report_connect_fail(self, ip_str, force_remove=False):
self.ip_lock.acquire()
try:
time_now = time.time()
if not ip_str in self.ip_dict:
return
self.ip_dict[ip_str]["links"] -= 1
# ignore if system network is disconnected.
if not force_remove:
if not check_ip.network_is_ok():
xlog.debug("report_connect_fail network fail")
# connect_control.fall_into_honeypot()
return
fail_time = self.ip_dict[ip_str]["fail_time"]
if not force_remove and time_now - fail_time < 1:
xlog.debug("fail time too near")
return
# increase handshake_time to make it can be used in lower probability
self.ip_dict[ip_str]["handshake_time"] += 300
if self.ip_dict[ip_str]["fail_times"] == 0:
self.good_ip_num -= 1
self.ip_dict[ip_str]["fail_times"] += 1
self.append_ip_history(ip_str, "fail")
self.ip_dict[ip_str]["fail_time"] = time_now
if force_remove or self.ip_dict[ip_str]["fail_times"] >= 50:
property = self.ip_dict[ip_str]
server = property["server"]
del self.ip_dict[ip_str]
if "gws" in server and ip_str in self.gws_ip_list:
self.gws_ip_list.remove(ip_str)
if not force_remove:
self.to_remove_ip_list.put(ip_str)
self.try_remove_thread()
xlog.info(
"remove ip tmp:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)
)
else:
xlog.info(
"remove ip:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)
)
if self.good_ip_num > len(self.ip_dict):
self.good_ip_num = len(self.ip_dict)
self.iplist_need_save = 1
except Exception as e:
xlog.exception("set_ip err:%s", e)
finally:
self.ip_lock.release()
if not self.is_ip_enough():
self.search_more_google_ip()
def import_mac_ca(common_name, certfile):
commonname = "GoAgent XX-Net" #TODO: need check again
ca_hash = CertUtil.ca_thumbprint.replace(':', '')
def get_exist_ca_sha1():
args = ['security', 'find-certificate', '-Z', '-a', '-c', commonname]
output = subprocess.check_output(args)
for line in output.splitlines(True):
if len(line) == 53 and line.startswith("SHA-1 hash:"):
sha1_hash = line[12:52]
return sha1_hash
exist_ca_sha1 = get_exist_ca_sha1()
if exist_ca_sha1 == ca_hash:
xlog.info("GoAgent CA exist")
return
import_command = 'security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ../../data/gae_proxy/CA.crt'# % certfile.decode('utf-8')
if exist_ca_sha1:
delete_ca_command = 'security delete-certificate -Z %s' % exist_ca_sha1
exec_command = "%s;%s" % (delete_ca_command, import_command)
else:
exec_command = import_command
admin_command = """osascript -e 'do shell script "%s" with administrator privileges' """ % exec_command
cmd = admin_command.encode('utf-8')
xlog.info("try auto import CA command:%s", cmd)
os.system(cmd)
def load_openssl():
global loaded, libcrypto, buf
from ctypes.util import find_library
for p in ('crypto', 'eay32', 'libeay32'):
libcrypto_path = find_library(p)
if libcrypto_path:
break
else:
raise Exception('libcrypto(OpenSSL) not found')
xlog.info('loading libcrypto from %s', libcrypto_path)
libcrypto = CDLL(libcrypto_path)
libcrypto.EVP_get_cipherbyname.restype = c_void_p
libcrypto.EVP_CIPHER_CTX_new.restype = c_void_p
libcrypto.EVP_CipherInit_ex.argtypes = (c_void_p, c_void_p, c_char_p,
c_char_p, c_char_p, c_int)
libcrypto.EVP_CipherUpdate.argtypes = (c_void_p, c_void_p, c_void_p,
c_char_p, c_int)
libcrypto.EVP_CIPHER_CTX_cleanup.argtypes = (c_void_p, )
libcrypto.EVP_CIPHER_CTX_free.argtypes = (c_void_p, )
if hasattr(libcrypto, 'OpenSSL_add_all_ciphers'):
libcrypto.OpenSSL_add_all_ciphers()
buf = create_string_buffer(buf_size)
loaded = True
def __init__(self, address, handler, args=()):
self.socket = None
self.running = True
self.server_address = address
self.handler = handler
self.args = args
self.init_socket()
xlog.info("server %s:%d started.", address[0], address[1])
def __init__(self, address, handler, args=()):
self.socket = None
self.running = True
self.server_address = address
self.handler = handler
self.args = args
self.init_socket()
xlog.info("server %s:%d started.", address[0], address[1])
def report_connect_fail(self, ip_str, force_remove=False):
self.ip_lock.acquire()
try:
time_now = time.time()
if not ip_str in self.ip_dict:
return
self.ip_dict[ip_str]['links'] -= 1
# ignore if system network is disconnected.
if not force_remove:
if not check_ip.network_is_ok():
xlog.debug("report_connect_fail network fail")
#connect_control.fall_into_honeypot()
return
fail_time = self.ip_dict[ip_str]["fail_time"]
if not force_remove and time_now - fail_time < 1:
xlog.debug("fail time too near")
return
# increase handshake_time to make it can be used in lower probability
self.ip_dict[ip_str]['handshake_time'] += 300
if self.ip_dict[ip_str]['fail_times'] == 0:
self.good_ip_num -= 1
self.ip_dict[ip_str]['fail_times'] += 1
self.append_ip_history(ip_str, "fail")
self.ip_dict[ip_str]["fail_time"] = time_now
if force_remove or self.ip_dict[ip_str]['fail_times'] >= 50:
property = self.ip_dict[ip_str]
server = property['server']
del self.ip_dict[ip_str]
if 'gws' in server and ip_str in self.gws_ip_list:
self.gws_ip_list.remove(ip_str)
if not force_remove:
self.to_remove_ip_list.put(ip_str)
self.try_remove_thread()
xlog.info("remove ip tmp:%s left amount:%d gws_num:%d",
ip_str, len(self.ip_dict), len(self.gws_ip_list))
else:
xlog.info("remove ip:%s left amount:%d gws_num:%d", ip_str,
len(self.ip_dict), len(self.gws_ip_list))
if self.good_ip_num > len(self.ip_dict):
self.good_ip_num = len(self.ip_dict)
self.iplist_need_save = 1
except Exception as e:
xlog.exception("set_ip err:%s", e)
finally:
self.ip_lock.release()
if not self.is_ip_enough():
self.search_more_google_ip()
def import_debian_ca(common_name, ca_file):
def get_debian_ca_sha1(nss_path):
commonname = "GoAgent XX-Net - GoAgent" #TODO: here should be GoAgent - XX-Net
cmd = [
'certutil', '-L', '-d',
'sql:%s' % nss_path, '-n', commonname
]
lines = get_cmd_out(cmd)
get_sha1_title = False
sha1 = ""
for line in lines:
if line.endswith("Fingerprint (SHA1):\n"):
get_sha1_title = True
continue
if get_sha1_title:
sha1 = line
break
sha1 = sha1.replace(' ', '').replace(':', '').replace('\n', '')
if len(sha1) != 40:
return False
else:
return sha1
home_path = os.path.expanduser("~")
nss_path = os.path.join(home_path, ".pki/nssdb")
if not os.path.isdir(nss_path):
return False
if not any(
os.path.isfile('%s/certutil' % x)
for x in os.environ['PATH'].split(os.pathsep)):
xlog.warning(
'please install *libnss3-tools* package to import GoAgent root ca'
)
return False
sha1 = get_debian_ca_sha1(nss_path)
ca_hash = CertUtil.ca_thumbprint.replace(':', '')
if sha1 == ca_hash:
xlog.info("system cert exist")
return
# shell command to list all cert
# certutil -L -d sql:$HOME/.pki/nssdb
# remove old cert first
cmd_line = 'certutil -L -d sql:$HOME/.pki/nssdb |grep "GoAgent" && certutil -d sql:$HOME/.pki/nssdb -D -n "%s" ' % (
common_name)
os.system(cmd_line)
# install new cert
cmd_line = 'certutil -d sql:$HOME/.pki/nssdb -A -t "C,," -n "%s" -i "%s"' % (
common_name, ca_file)
os.system(cmd_line)
return True
def roll_log(self):
for i in range(1000):
file_name = os.path.join(config.DATA_PATH, "scan_ip.%d.log" % i)
if os.path.isfile(file_name):
continue
xlog.info("scan_ip_log roll %s -> %s", self.log_path, file_name)
shutil.move(self.log_path, file_name)
return
def remove_channels_for_task(task, channels):
xlog.info('remove channels for task %s' % task)
for root, dirnames, filenames in os.walk(task + '/cyberecord'):
# print root, dirnames, filenames
for filename in filenames:
path = os.path.join(root, filename)
if path.endswith('.record'):
xlog.info('remove channels for record %s' % path)
remove_channels_for_record(str(path), channels)
def roll_log(self):
for i in range(1000):
file_name = os.path.join(config.DATA_PATH, "scan_ip.%d.log" % i)
if os.path.isfile(file_name):
continue
xlog.info("scan_ip_log roll %s -> %s", self.log_path, file_name)
shutil.move(self.log_path, file_name)
return
def get_connect_interval():
if sys.platform != "win32":
return 0
win_version = env_info.win32_version()
if win_version == 10:
xlog.info("detect Win10, enable connect concurent control, interval:%d", config.connect_interval)
return config.connect_interval
return 0
def generate_ca_file():
xlog.info("generate CA file:%s", CertUtil.ca_keyfile)
key, ca = CertUtil.create_ca()
with open(CertUtil.ca_keyfile, 'wb') as fp:
fp.write(
OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM,
ca))
fp.write(
OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM,
key))
def __init__(self, address, handler, args=()):
self.running = True
self.server_address = address
self.handler = handler
self.args = args
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind(self.server_address)
self.socket.listen(200)
xlog.info("server %s:%d started.", address[0], address[1])
def __init__(self, address, handler, args=()):
self.running = True
self.server_address = address
self.handler = handler
self.args = args
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind(self.server_address)
self.socket.listen(200)
xlog.info("server %s:%d started.", address[0], address[1])
def main():
global ready
# to profile gae_proxy, run proxy.py, visit some web by proxy, then visit http://127.0.0.1:8084/quit to quit and print result.
do_profile = False
if do_profile:
import cProfile, pstats
pr = cProfile.Profile()
pr.enable()
global __file__
__file__ = os.path.abspath(__file__)
if os.path.islink(__file__):
__file__ = getattr(os, 'readlink', lambda x: x)(__file__)
os.chdir(os.path.dirname(os.path.abspath(__file__)))
xlog.basicConfig(
level=xlog.DEBUG if config.LISTEN_DEBUGINFO else xlog.INFO,
format='%(levelname)s - %(asctime)s %(message)s',
datefmt='[%b %d %H:%M:%S]')
pre_start()
log_info()
CertUtil.init_ca()
proxy_daemon = LocalProxyServer((config.LISTEN_IP, config.LISTEN_PORT),
proxy_handler.GAEProxyHandler)
proxy_thread = threading.Thread(target=proxy_daemon.serve_forever)
proxy_thread.setDaemon(True)
proxy_thread.start()
if config.PAC_ENABLE:
pac_daemon = LocalProxyServer((config.PAC_IP, config.PAC_PORT),
pac_server.PACServerHandler)
pac_thread = threading.Thread(target=pac_daemon.serve_forever)
pac_thread.setDaemon(True)
pac_thread.start()
ready = True #checked by launcher.module_init
while connect_control.keep_running:
time.sleep(1)
xlog.info("Exiting gae_proxy module...")
proxy_daemon.shutdown()
proxy_daemon.server_close()
proxy_thread.join()
if config.PAC_ENABLE:
pac_daemon.shutdown()
pac_daemon.server_close()
pac_thread.join()
ready = False #checked by launcher.module_init
xlog.info("Finished Exiting gae_proxy module...")
if do_profile:
pr.disable()
pr.print_stats()
def main():
global ready
connect_control.keep_running = True
config.load()
connect_manager.https_manager.load_config()
xlog.debug("## GAEProxy set keep_running: %s", connect_control.keep_running)
# to profile gae_proxy, run proxy.py, visit some web by proxy, then visit http://127.0.0.1:8084/quit to quit and print result.
do_profile = False
if do_profile:
import cProfile, pstats
pr = cProfile.Profile()
pr.enable()
global __file__
__file__ = os.path.abspath(__file__)
if os.path.islink(__file__):
__file__ = getattr(os, 'readlink', lambda x: x)(__file__)
os.chdir(os.path.dirname(os.path.abspath(__file__)))
xlog.basicConfig(level=xlog.DEBUG if config.LISTEN_DEBUGINFO else xlog.INFO, format='%(levelname)s - %(asctime)s %(message)s', datefmt='[%b %d %H:%M:%S]')
pre_start()
log_info()
CertUtil.init_ca()
proxy_daemon = simple_http_server.HTTPServer((config.LISTEN_IP, config.LISTEN_PORT), proxy_handler.GAEProxyHandler)
proxy_thread = threading.Thread(target=proxy_daemon.serve_forever)
proxy_thread.setDaemon(True)
proxy_thread.start()
if config.PAC_ENABLE:
pac_daemon = simple_http_server.HTTPServer((config.PAC_IP, config.PAC_PORT), pac_server.PACServerHandler)
pac_thread = threading.Thread(target=pac_daemon.serve_forever)
pac_thread.setDaemon(True)
pac_thread.start()
ready = True # checked by launcher.module_init
while connect_control.keep_running:
time.sleep(1)
xlog.info("Exiting gae_proxy module...")
proxy_daemon.shutdown()
proxy_daemon.server_close()
proxy_thread.join()
if config.PAC_ENABLE:
pac_daemon.shutdown()
pac_daemon.server_close()
pac_thread.join()
ready = False # checked by launcher.module_init
xlog.debug("## GAEProxy set keep_running: %s", connect_control.keep_running)
if do_profile:
pr.disable()
pr.print_stats()
def req_deploy_handler(self):
global deploy_proc
req = urlparse.urlparse(self.path).query
reqs = urlparse.parse_qs(req, keep_blank_values=True)
data = ""
log_path = os.path.abspath(os.path.join(current_path, os.pardir, "server", "upload.log"))
time_now = datetime.datetime.today().strftime("%H:%M:%S-%a/%d/%b/%Y")
if reqs["cmd"] == ["deploy"]:
appid = self.postvars["appid"][0]
if deploy_proc and deploy_proc.poll() == None:
xlog.warn("deploy is running, request denied.")
data = '{"res":"deploy is running", "time":"%s"}' % (time_now)
else:
try:
if os.path.isfile(log_path):
os.remove(log_path)
script_path = os.path.abspath(os.path.join(current_path, os.pardir, "server", "uploader.py"))
email = self.postvars["email"][0]
passwd = self.postvars["passwd"][0]
rc4_passwd = self.postvars["rc4_passwd"][0]
deploy_proc = subprocess.Popen([sys.executable, script_path, appid, email, passwd, rc4_passwd])
xlog.info("deploy begin.")
data = '{"res":"success", "time":"%s"}' % time_now
except Exception as e:
data = '{"res":"%s", "time":"%s"}' % (e, time_now)
elif reqs["cmd"] == ["cancel"]:
if deploy_proc and deploy_proc.poll() == None:
deploy_proc.kill()
data = '{"res":"deploy is killed", "time":"%s"}' % (time_now)
else:
data = '{"res":"deploy is not running", "time":"%s"}' % (time_now)
elif reqs["cmd"] == ["get_log"]:
if deploy_proc and os.path.isfile(log_path):
with open(log_path, "r") as f:
content = f.read()
else:
content = ""
status = "init"
if deploy_proc:
if deploy_proc.poll() == None:
status = "running"
else:
status = "finished"
data = json.dumps({"status": status, "log": content, "time": time_now})
self.send_response("text/html", data)
def req_deploy_handler(self):
global deploy_proc
req = urlparse.urlparse(self.path).query
reqs = urlparse.parse_qs(req, keep_blank_values=True)
data = ''
log_path = os.path.abspath(os.path.join(current_path, os.pardir, "server", 'upload.log'))
time_now = datetime.datetime.today().strftime('%H:%M:%S-%a/%d/%b/%Y')
if reqs['cmd'] == ['deploy']:
appid = self.postvars['appid'][0]
if deploy_proc and deploy_proc.poll() == None:
xlog.warn("deploy is running, request denied.")
data = '{"res":"deploy is running", "time":"%s"}' % (time_now)
else:
try:
if os.path.isfile(log_path):
os.remove(log_path)
script_path = os.path.abspath(os.path.join(current_path, os.pardir, "server", 'uploader.py'))
email = self.postvars['email'][0]
passwd = self.postvars['passwd'][0]
rc4_passwd = self.postvars['rc4_passwd'][0]
deploy_proc = subprocess.Popen([sys.executable, script_path, appid, email, passwd, rc4_passwd])
xlog.info("deploy begin.")
data = '{"res":"success", "time":"%s"}' % time_now
except Exception as e:
data = '{"res":"%s", "time":"%s"}' % (e, time_now)
elif reqs['cmd'] == ['cancel']:
if deploy_proc and deploy_proc.poll() == None:
deploy_proc.kill()
data = '{"res":"deploy is killed", "time":"%s"}' % (time_now)
else:
data = '{"res":"deploy is not running", "time":"%s"}' % (time_now)
elif reqs['cmd'] == ['get_log']:
if deploy_proc and os.path.isfile(log_path):
with open(log_path, "r") as f:
content = f.read()
else:
content = ""
status = 'init'
if deploy_proc:
if deploy_proc.poll() == None:
status = 'running'
else:
status = 'finished'
data = json.dumps({'status':status,'log':content, 'time':time_now})
self.send_response('text/html', data)
def test_gae(ip_str):
xlog.info("==>%s", ip_str)
check = Check_frame(ip_str)
result = check.check(callback=test_app_head, check_ca=True)
if not result:
return False
check.result.server_type = result
return check.result
def test_gws(ip_str):
xlog.info("==>%s", ip_str)
check = Check_frame(ip_str)
result = check.check(callback=test_server_type, check_ca=True)
if not result or not "gws" in result:
return False
check.result.server_type = result
return check.result
def check_all_domain(check_ip):
with open(os.path.join(current_path, "front_domains.json"), "r") as fd:
content = fd.read()
cs = json.loads(content)
for host in cs:
host = "scan1." + host
res = check_ip.check_ip(ip, host=host, wait_time=wait_time)
if not res or not res.ok:
xlog.warn("host:%s fail", host)
else:
xlog.info("host:%s ok", host)
def check_ssl_cert(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#raise HoneypotError(' certficate is none')
raise SSLError("no cert")
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
ssl_cert = cert_util.SSLCert(cert)
xlog.info("%s CN:%s", self.ip, ssl_cert.cn)
self.result.domain = ssl_cert.cn
def test_gae(ip_str):
xlog.info("==>%s", ip_str)
check = Check_frame(ip_str)
result = check.check(callback=test_app_head, check_ca=True)
if not result:
return False
check.result.server_type = result
return check.result
def check_all_domain(check_ip):
with open(os.path.join(current_path, "front_domains.json"), "r") as fd:
content = fd.read()
cs = json.loads(content)
for host in cs:
host = "scan1." + host
res = check_ip.check_ip(ip, host=host, wait_time=wait_time)
if not res or not res.ok:
xlog.warn("host:%s fail", host)
else:
xlog.info("host:%s ok", host)
def check_ssl_cert(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#raise HoneypotError(' certficate is none')
raise SSLError("no cert")
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
ssl_cert = cert_util.SSLCert(cert)
xlog.info("%s CN:%s", self.ip, ssl_cert.cn)
self.result.domain = ssl_cert.cn
def import_debian_ca(common_name, ca_file):
def get_debian_ca_sha1(nss_path):
commonname = "GoAgent XX-Net - GoAgent" #TODO: here should be GoAgent - XX-Net
cmd = ['certutil', '-L','-d', 'sql:%s' % nss_path, '-n', commonname]
lines = get_cmd_out(cmd)
get_sha1_title = False
sha1 = ""
for line in lines:
if line.endswith("Fingerprint (SHA1):\n"):
get_sha1_title = True
continue
if get_sha1_title:
sha1 = line
break
sha1 = sha1.replace(' ', '').replace(':', '').replace('\n', '')
if len(sha1) != 40:
return False
else:
return sha1
home_path = os.path.expanduser("~")
nss_path = os.path.join(home_path, ".pki/nssdb")
if not os.path.isdir(nss_path):
return False
if not any(os.path.isfile('%s/certutil' % x) for x in os.environ['PATH'].split(os.pathsep)):
xlog.warning('please install *libnss3-tools* package to import GoAgent root ca')
return False
sha1 = get_debian_ca_sha1(nss_path)
ca_hash = CertUtil.ca_thumbprint.replace(':', '')
if sha1 == ca_hash:
xlog.info("system cert exist")
return
# shell command to list all cert
# certutil -L -d sql:$HOME/.pki/nssdb
# remove old cert first
cmd_line = 'certutil -L -d sql:$HOME/.pki/nssdb |grep "GoAgent" && certutil -d sql:$HOME/.pki/nssdb -D -n "%s" ' % ( common_name)
os.system(cmd_line)
# install new cert
cmd_line = 'certutil -d sql:$HOME/.pki/nssdb -A -t "C,," -n "%s" -i "%s"' % (common_name, ca_file)
os.system(cmd_line)
return True
def do_GET(self):
xlog.info('PAC from:%s %s %s ', self.address_string(), self.command,
self.path)
path = urlparse.urlparse(self.path).path # '/proxy.pac'
filename = os.path.normpath('./' + path) # proxy.pac
if self.path.startswith(('http://', 'https://')):
data = b'HTTP/1.1 200\r\nCache-Control: max-age=86400\r\nExpires:Oct, 01 Aug 2100 00:00:00 GMT\r\nConnection: close\r\n'
if filename.endswith(('.jpg', '.gif', '.jpeg', '.bmp')):
data += b'Content-Type: image/gif\r\n\r\n' + self.onepixel
else:
data += b'\r\n This is the Pac server, not proxy port, use 8087 as proxy port.'
self.wfile.write(data)
xlog.info('%s "%s %s HTTP/1.1" 200 -', self.address_string(),
self.command, self.path)
return
# check for '..', which will leak file
if re.search(r'(\.{2})', self.path) is not None:
self.wfile.write(b'HTTP/1.1 404\r\n\r\n')
xlog.warn('%s %s %s haking', self.address_string(), self.command,
self.path)
return
if filename != 'proxy.pac':
xlog.warn("pac_server GET %s fail", filename)
self.wfile.write(b'HTTP/1.1 404\r\n\r\n')
return
mimetype = 'text/plain'
if self.path.endswith('.pac?flush') or time.time() - os.path.getmtime(
get_serving_pacfile()) > config.PAC_EXPIRED:
thread.start_new_thread(PacUtil.update_pacfile, (user_pacfile, ))
pac_filename = get_serving_pacfile()
with open(pac_filename, 'rb') as fp:
data = fp.read()
host = self.headers.getheader('Host')
host, _, port = host.rpartition(":")
gae_proxy_proxy = host + ":" + str(config.LISTEN_PORT)
pac_proxy = host + ":" + str(config.PAC_PORT)
data = data.replace(gae_proxy_listen, gae_proxy_proxy)
data = data.replace(pac_listen, pac_proxy)
self.wfile.write(
('HTTP/1.1 200\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n' %
(mimetype, len(data))).encode())
self.wfile.write(data)
def load_range_content(self):
if os.path.isfile(user_range_file):
self.range_file = user_range_file
else:
self.range_file = default_range_file
xlog.info("load ip range file:%s", self.range_file)
fd = open(self.range_file, "r")
if not fd:
xlog.error("load ip range %s fail", self.range_file)
return
content = fd.read()
fd.close()
return content
