def remove_slowest_ip(self):
if len(self.gws_ip_list) <= self.max_good_ip_num:
return
self.try_sort_gws_ip(force=True)
self.ip_lock.acquire()
try:
ip_num = len(self.gws_ip_list)
while ip_num > self.max_good_ip_num:
ip_str = self.gws_ip_list[ip_num - 1]
property = self.ip_dict[ip_str]
server = property['server']
fails = property['fail_times']
handshake_time = property['handshake_time']
xlog.info("remove_slowest_ip:%s handshake_time:%d, fails:%d",
ip_str, handshake_time, fails)
del self.ip_dict[ip_str]
if 'gws' in server and ip_str in self.gws_ip_list:
self.gws_ip_list.remove(ip_str)
ip_num -= 1
except Exception as e:
xlog.exception("remove_slowest_ip err:%s", e)
finally:
self.ip_lock.release()
def req_config_handler(self):
req = urlparse.urlparse(self.path).query
reqs = urlparse.parse_qs(req, keep_blank_values=True)
data = ''
appid_updated = False
try:
if reqs['cmd'] == ['get_config']:
data = json.dumps(user_config.user_special, default=lambda o: o.__dict__)
elif reqs['cmd'] == ['set_config']:
appids = self.postvars['appid'][0]
if appids != user_config.user_special.appid:
if appids:
fail_appid_list = test_appid.test_appids(appids)
if len(fail_appid_list):
fail_appid = "|".join(fail_appid_list)
return self.send_response('text/html', '{"res":"fail", "reason":"appid fail:%s"}' % fail_appid)
appid_updated = True
user_config.user_special.appid = appids
user_config.user_special.password = self.postvars['password'][0]
user_config.user_special.proxy_enable = self.postvars['proxy_enable'][0]
user_config.user_special.proxy_type = self.postvars['proxy_type'][0]
user_config.user_special.proxy_host = self.postvars['proxy_host'][0]
user_config.user_special.proxy_port = self.postvars['proxy_port'][0]
if not user_config.user_special.proxy_port:
user_config.user_special.proxy_port = 0
user_config.user_special.proxy_user = self.postvars['proxy_user'][0]
user_config.user_special.proxy_passwd = self.postvars['proxy_passwd'][0]
user_config.user_special.host_appengine_mode = self.postvars['host_appengine_mode'][0]
use_ipv6 = int(self.postvars['use_ipv6'][0])
if user_config.user_special.use_ipv6 != use_ipv6:
if use_ipv6:
if not check_ip.check_ipv6():
xlog.warn("Enable Ipv6 but check failed.")
return self.send_response('text/html', '{"res":"fail", "reason":"IPv6 fail"}')
user_config.user_special.use_ipv6 = use_ipv6
user_config.save()
config.load()
appid_manager.reset_appid()
import connect_manager
connect_manager.load_proxy_config()
connect_manager.https_manager.load_config()
if appid_updated:
connect_manager.https_manager.clean_old_connection()
google_ip.reset()
check_ip.load_proxy_config()
data = '{"res":"success"}'
self.send_response('text/html', data)
#http_request("http://127.0.0.1:8085/init_module?module=gae_proxy&cmd=restart")
return
except Exception as e:
xlog.exception("req_config_handler except:%s", e)
data = '{"res":"fail", "except":"%s"}' % e
self.send_response('text/html', data)
def load_ip_range(self):
self.ip_range_map = {}
self.ip_range_list = []
self.ip_range_index = []
self.candidate_amount_ip = 0
content = self.load_range_content()
lines = content.splitlines()
for line in lines:
if len(line) == 0 or line[0] == '#':
continue
try:
begin, end = ip_utils.split_ip(line)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
if not nbegin or not nend or nend < nbegin:
xlog.warn("load ip range:%s fail", line)
continue
except Exception as e:
xlog.exception("load ip range:%s fail:%r", line, e)
continue
self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
self.ip_range_list.append( [nbegin, nend] )
self.ip_range_index.append(self.candidate_amount_ip)
num = nend - nbegin
self.candidate_amount_ip += num
# print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num
self.ip_range_index.sort()
def report_connect_fail(self, ip_str, force_remove=False):
self.ip_lock.acquire()
try:
time_now = time.time()
if not ip_str in self.ip_dict:
return
self.ip_dict[ip_str]["links"] -= 1
# ignore if system network is disconnected.
if not force_remove:
if not check_ip.network_is_ok():
xlog.debug("report_connect_fail network fail")
# connect_control.fall_into_honeypot()
return
fail_time = self.ip_dict[ip_str]["fail_time"]
if not force_remove and time_now - fail_time < 1:
xlog.debug("fail time too near")
return
# increase handshake_time to make it can be used in lower probability
self.ip_dict[ip_str]["handshake_time"] += 300
if self.ip_dict[ip_str]["fail_times"] == 0:
self.good_ip_num -= 1
self.ip_dict[ip_str]["fail_times"] += 1
self.append_ip_history(ip_str, "fail")
self.ip_dict[ip_str]["fail_time"] = time_now
if force_remove or self.ip_dict[ip_str]["fail_times"] >= 50:
property = self.ip_dict[ip_str]
server = property["server"]
del self.ip_dict[ip_str]
if "gws" in server and ip_str in self.gws_ip_list:
self.gws_ip_list.remove(ip_str)
if not force_remove:
self.to_remove_ip_list.put(ip_str)
self.try_remove_thread()
xlog.info(
"remove ip tmp:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)
)
else:
xlog.info(
"remove ip:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)
)
if self.good_ip_num > len(self.ip_dict):
self.good_ip_num = len(self.ip_dict)
self.iplist_need_save = 1
except Exception as e:
xlog.exception("set_ip err:%s", e)
finally:
self.ip_lock.release()
if not self.is_ip_enough():
self.search_more_google_ip()
def load_ip(self):
if os.path.isfile(self.good_ip_file):
file_path = self.good_ip_file
else:
file_path = self.default_good_ip_file
with open(file_path, "r") as fd:
lines = fd.readlines()
for line in lines:
try:
if line.startswith("#"):
continue
str_l = line.split(' ')
if len(str_l) < 4:
xlog.warning("line err: %s", line)
continue
ip_str = str_l[0]
domain = str_l[1]
server = str_l[2]
handshake_time = int(str_l[3])
if len(str_l) > 4:
fail_times = int(str_l[4])
else:
fail_times = 0
#logging.info("load ip: %s time:%d domain:%s server:%s", ip_str, handshake_time, domain, server)
self.add_ip(ip_str, handshake_time, domain, server, fail_times)
except Exception as e:
xlog.exception("load_ip line:%s err:%s", line, e)
xlog.info("load google ip_list num:%d, gws num:%d", len(self.ip_dict), len(self.gws_ip_list))
self.try_sort_gws_ip(force=True)
def load_ip(self):
if os.path.isfile(self.good_ip_file):
file_path = self.good_ip_file
else:
file_path = self.default_good_ip_file
with open(file_path, "r") as fd:
lines = fd.readlines()
for line in lines:
try:
if line.startswith("#"):
continue
str_l = line.split(' ')
if len(str_l) < 4:
xlog.warning("line err: %s", line)
continue
ip = str_l[0]
domain = str_l[1]
server = str_l[2]
handshake_time = int(str_l[3])
if len(str_l) > 4:
fail_times = int(str_l[4])
else:
fail_times = 0
#logging.info("load ip: %s time:%d domain:%s server:%s", ip, handshake_time, domain, server)
self.add_ip(ip, handshake_time, domain, server, fail_times)
except Exception as e:
xlog.exception("load_ip line:%s err:%s", line, e)
xlog.info("load google ip_list num:%d, gws num:%d", len(self.ip_dict), len(self.gws_ip_list))
self.try_sort_gws_ip(force=True)
def request(headers={}, payload=None):
max_retry = 3
for i in range(max_retry):
ssl_sock = None
try:
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
xlog.debug('create_ssl_connection fail')
continue
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
google_ip.report_connect_closed(ssl_sock.ip, "no appid")
raise GAE_Exception(1, "no appid can use")
headers['Host'] = ssl_sock.appid + ".appspot.com"
ssl_sock.host = headers['Host']
else:
headers['Host'] = ssl_sock.host
response = _request(ssl_sock, headers, payload)
if not response:
google_ip.report_connect_closed(ssl_sock.ip, "request_fail")
ssl_sock.close()
continue
response.ssl_sock = ssl_sock
return response
except Exception as e:
xlog.exception('request failed:%s', e)
if ssl_sock:
google_ip.report_connect_closed(ssl_sock.ip, "request_except")
ssl_sock.close()
raise GAE_Exception(2, "try max times")
def scan_ip_worker(self):
while self.scan_thread_count <= self.scan_ip_thread_num and connect_control.keep_running:
if not connect_control.allow_scan():
time.sleep(10)
continue
try:
time.sleep(1)
ip = self.ip_range.get_ip()
if ip in self.ip_dict:
continue
connect_control.start_connect_register()
result = check_ip.test_gae_ip(ip)
connect_control.end_connect_register()
if not result:
continue
if self.add_ip(ip, result.handshake_time, result.domain, "gws"):
#logging.info("add %s CN:%s type:%s time:%d gws:%d ", ip,
# result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list))
xlog.info("scan_ip add ip:%s time:%d", ip, result.handshake_time)
scan_ip_log.info("Add %s time:%d CN:%s ", ip, result.handshake_time, result.domain)
self.remove_slowest_ip()
self.save_ip_list()
except Exception as e:
xlog.exception("google_ip.runJob fail:%r", e)
self.scan_thread_lock.acquire()
self.scan_thread_count -= 1
self.scan_thread_lock.release()
xlog.info("scan_ip_worker exit")
def remove_slowest_ip(self):
if len(self.gws_ip_list) <= self.max_good_ip_num:
return
self.try_sort_gws_ip(force=True)
self.ip_lock.acquire()
try:
ip_num = len(self.gws_ip_list)
while ip_num > self.max_good_ip_num:
ip_str = self.gws_ip_list[ip_num - 1]
property = self.ip_dict[ip_str]
server = property['server']
fails = property['fail_times']
handshake_time = property['handshake_time']
xlog.info("remove_slowest_ip:%s handshake_time:%d, fails:%d", ip_str, handshake_time, fails)
del self.ip_dict[ip_str]
if 'gws' in server and ip_str in self.gws_ip_list:
self.gws_ip_list.remove(ip_str)
ip_num -= 1
except Exception as e:
xlog.exception("remove_slowest_ip err:%s", e)
finally:
self.ip_lock.release()
def load_ip_range(self):
self.ip_range_map = {}
self.ip_range_list = []
self.ip_range_index = []
self.candidate_amount_ip = 0
content = self.load_range_content()
lines = content.splitlines()
for line in lines:
if len(line) == 0 or line[0] == '#':
continue
try:
begin, end = ip_utils.split_ip(line)
nbegin = ip_utils.ip_string_to_num(begin)
nend = ip_utils.ip_string_to_num(end)
if not nbegin or not nend or nend < nbegin:
xlog.warn("load ip range:%s fail", line)
continue
except Exception as e:
xlog.exception("load ip range:%s fail:%r", line, e)
continue
self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
self.ip_range_list.append( [nbegin, nend] )
self.ip_range_index.append(self.candidate_amount_ip)
num = nend - nbegin
self.candidate_amount_ip += num
# print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num
self.ip_range_index.sort()
def scan_ip_worker(self):
while self.searching_thread_count <= self.scan_ip_thread_num and connect_control.keep_running:
if not connect_control.allow_scan():
time.sleep(10)
continue
try:
time.sleep(1)
ip_int = ip_range.get_ip()
ip_str = ip_utils.ip_num_to_string(ip_int)
if ip_str in self.ip_dict:
continue
connect_control.start_connect_register()
result = check_ip.test_gae_ip(ip_str)
connect_control.end_connect_register()
if not result:
continue
if self.add_ip(ip_str, result.handshake_time, result.domain, "gws"):
#logging.info("add %s CN:%s type:%s time:%d gws:%d ", ip_str,
# result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list))
xlog.info("scan_ip add ip:%s time:%d", ip_str, result.handshake_time)
scan_ip_log.info("Add %s time:%d CN:%s ", ip_str, result.handshake_time, result.domain)
self.remove_slowest_ip()
self.save_ip_list()
except Exception as e:
xlog.exception("google_ip.runJob fail:%r", e)
self.ncount_lock.acquire()
self.searching_thread_count -= 1
self.ncount_lock.release()
xlog.info("scan_ip_worker exit")
def request(headers={}, payload=None):
max_retry = 3
for i in range(max_retry):
ssl_sock = None
try:
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
xlog.debug('create_ssl_connection fail')
continue
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
google_ip.report_connect_closed(ssl_sock.ip, "no appid")
raise GAE_Exception(1, "no appid can use")
headers['Host'] = ssl_sock.appid + ".appspot.com"
ssl_sock.host = headers['Host']
else:
headers['Host'] = ssl_sock.host
response = _request(ssl_sock, headers, payload)
if not response:
google_ip.report_connect_closed(ssl_sock.ip, "request_fail")
ssl_sock.close()
continue
response.ssl_sock = ssl_sock
return response
except Exception as e:
xlog.exception('request failed:%s', e)
if ssl_sock:
google_ip.report_connect_closed(ssl_sock.ip, "request_except")
ssl_sock.close()
raise GAE_Exception(2, "try max times")
def http_request(url, method="GET"):
proxy_handler = urllib2.ProxyHandler({})
opener = urllib2.build_opener(proxy_handler)
try:
req = opener.open(url)
except Exception as e:
xlog.exception("web_control http_request:%s fail:%s", url, e)
return
def http_request(url, method="GET"):
proxy_handler = urllib2.ProxyHandler({})
opener = urllib2.build_opener(proxy_handler)
try:
req = opener.open(url)
except Exception as e:
xlog.exception("web_control http_request:%s fail:%s", url, e)
return
def report_connect_fail(self, ip, force_remove=False):
self.ip_lock.acquire()
try:
time_now = time.time()
if not ip in self.ip_dict:
xlog.debug("report_connect_fail %s not exist", ip)
return
if force_remove:
if self.ip_dict[ip]['fail_times'] == 0:
self.good_ip_num -= 1
del self.ip_dict[ip]
if ip in self.gws_ip_list:
self.gws_ip_list.remove(ip)
xlog.info("remove ip:%s left amount:%d gws_num:%d", ip,
len(self.ip_dict), len(self.gws_ip_list))
return
self.ip_dict[ip]['links'] -= 1
# ignore if system network is disconnected.
if check_ip.network_stat == "Fail":
xlog.debug("report_connect_fail network fail")
return
check_ip.continue_fail_count += 1
if check_ip.continue_fail_count > 10:
check_ip.network_stat = "unknown"
xlog.debug("report_connect_fail continue_fail_count:%d",
check_ip.continue_fail_count)
check_ip.triger_check_network()
return
fail_time = self.ip_dict[ip]["fail_time"]
if time_now - fail_time < 1:
xlog.debug("fail time too near %s", ip)
return
if self.ip_dict[ip]['fail_times'] == 0:
self.good_ip_num -= 1
self.ip_dict[ip]['fail_times'] += 1
self.append_ip_history(ip, "fail")
self.ip_dict[ip]["fail_time"] = time_now
check_ip.triger_check_network()
self.to_check_ip_queue.put((ip, time_now + 10))
xlog.info("report_connect_fail:%s", ip)
except Exception as e:
xlog.exception("report_connect_fail err:%s", e)
finally:
self.iplist_need_save = 1
self.ip_lock.release()
if not self.is_ip_enough():
self.search_more_google_ip()
def report_connect_fail(self, ip_str, force_remove=False):
self.ip_lock.acquire()
try:
time_now = time.time()
if not ip_str in self.ip_dict:
return
self.ip_dict[ip_str]['links'] -= 1
# ignore if system network is disconnected.
if not force_remove:
if not check_ip.network_is_ok():
xlog.debug("report_connect_fail network fail")
#connect_control.fall_into_honeypot()
return
fail_time = self.ip_dict[ip_str]["fail_time"]
if not force_remove and time_now - fail_time < 1:
xlog.debug("fail time too near")
return
# increase handshake_time to make it can be used in lower probability
#self.ip_dict[ip_str]['handshake_time'] += 300
if self.ip_dict[ip_str]['fail_times'] == 0:
self.good_ip_num -= 1
self.ip_dict[ip_str]['fail_times'] += 1
self.append_ip_history(ip_str, "fail")
self.ip_dict[ip_str]["fail_time"] = time_now
if force_remove or self.ip_dict[ip_str]['fail_times'] >= 50:
property = self.ip_dict[ip_str]
server = property['server']
del self.ip_dict[ip_str]
if 'gws' in server and ip_str in self.gws_ip_list:
self.gws_ip_list.remove(ip_str)
if not force_remove:
self.to_remove_ip_list.put(ip_str)
self.try_remove_thread()
xlog.info("remove ip tmp:%s left amount:%d gws_num:%d",
ip_str, len(self.ip_dict), len(self.gws_ip_list))
else:
xlog.info("remove ip:%s left amount:%d gws_num:%d", ip_str,
len(self.ip_dict), len(self.gws_ip_list))
if self.good_ip_num > len(self.ip_dict):
self.good_ip_num = len(self.ip_dict)
self.iplist_need_save = 1
except Exception as e:
xlog.exception("set_ip err:%s", e)
finally:
self.ip_lock.release()
if not self.is_ip_enough():
self.search_more_google_ip()
def report_connect_fail(self, ip, force_remove=False):
self.ip_lock.acquire()
try:
time_now = time.time()
if not ip in self.ip_dict:
xlog.debug("report_connect_fail %s not exist", ip)
return
if force_remove:
if self.ip_dict[ip]['fail_times'] == 0:
self.good_ip_num -= 1
del self.ip_dict[ip]
if ip in self.gws_ip_list:
self.gws_ip_list.remove(ip)
xlog.info("remove ip:%s left amount:%d gws_num:%d", ip, len(self.ip_dict), len(self.gws_ip_list))
return
self.ip_dict[ip]['links'] -= 1
# ignore if system network is disconnected.
if check_ip.network_stat == "Fail":
xlog.debug("report_connect_fail network fail")
return
check_ip.continue_fail_count += 1
if check_ip.continue_fail_count > 10:
check_ip.network_stat = "unknown"
xlog.debug("report_connect_fail continue_fail_count:%d", check_ip.continue_fail_count)
check_ip.triger_check_network()
return
fail_time = self.ip_dict[ip]["fail_time"]
if time_now - fail_time < 1:
xlog.debug("fail time too near %s", ip)
return
if self.ip_dict[ip]['fail_times'] == 0:
self.good_ip_num -= 1
self.ip_dict[ip]['fail_times'] += 1
self.append_ip_history(ip, "fail")
self.ip_dict[ip]["fail_time"] = time_now
check_ip.triger_check_network()
self.to_check_ip_queue.put((ip, time_now + 10))
xlog.info("report_connect_fail:%s", ip)
except Exception as e:
xlog.exception("report_connect_fail err:%s", e)
finally:
self.iplist_need_save = 1
self.ip_lock.release()
if not self.is_ip_enough():
self.search_more_google_ip()
def get_gws_ip(self):
self.try_sort_gws_ip()
self.ip_lock.acquire()
try:
ip_num = len(self.gws_ip_list)
if ip_num == 0:
#logging.warning("no gws ip")
#time.sleep(10)
return None
for i in range(ip_num):
time_now = time.time()
if self.gws_ip_pointer >= ip_num:
if time_now - self.gws_ip_pointer_reset_time < 1:
time.sleep(1)
continue
else:
self.gws_ip_pointer = 0
self.gws_ip_pointer_reset_time = time_now
elif self.gws_ip_pointer > 0 and time_now - self.gws_ip_pointer_reset_time > 3:
self.gws_ip_pointer = 0
self.gws_ip_pointer_reset_time = time_now
ip = self.gws_ip_list[self.gws_ip_pointer]
get_time = self.ip_dict[ip]["get_time"]
if time_now - get_time < self.ip_connect_interval:
self.gws_ip_pointer += 1
continue
if time_now - self.ip_dict[ip]['success_time'] > 300: # 5 min
fail_connect_interval = 1800 # 30 min
else:
fail_connect_interval = 120 # 2 min
fail_time = self.ip_dict[ip]["fail_time"]
if time_now - fail_time < fail_connect_interval:
self.gws_ip_pointer += 1
continue
if self.ip_dict[ip]['links'] >= config.max_links_per_ip:
self.gws_ip_pointer += 1
continue
handshake_time = self.ip_dict[ip]["handshake_time"]
xlog.debug("get ip:%s t:%d", ip, handshake_time)
self.append_ip_history(ip, "get")
self.ip_dict[ip]['get_time'] = time_now
self.ip_dict[ip]['links'] += 1
self.gws_ip_pointer += 1
return ip
except Exception as e:
xlog.exception("get_gws_ip fail:%r", e)
finally:
self.ip_lock.release()
def get_gws_ip(self):
self.try_sort_gws_ip()
self.ip_lock.acquire()
try:
ip_num = len(self.gws_ip_list)
if ip_num == 0:
#logging.warning("no gws ip")
#time.sleep(10)
return None
for i in range(ip_num):
time_now = time.time()
if self.gws_ip_pointer >= ip_num:
if time_now - self.gws_ip_pointer_reset_time < 1:
time.sleep(1)
continue
else:
self.gws_ip_pointer = 0
self.gws_ip_pointer_reset_time = time_now
elif self.gws_ip_pointer > 0 and time_now - self.gws_ip_pointer_reset_time > 3:
self.gws_ip_pointer = 0
self.gws_ip_pointer_reset_time = time_now
ip = self.gws_ip_list[self.gws_ip_pointer]
get_time = self.ip_dict[ip]["get_time"]
if time_now - get_time < self.ip_connect_interval:
self.gws_ip_pointer += 1
continue
if time_now - self.ip_dict[ip]['success_time'] > 300: # 5 min
fail_connect_interval = 1800 # 30 min
else:
fail_connect_interval = 120 # 2 min
fail_time = self.ip_dict[ip]["fail_time"]
if time_now - fail_time < fail_connect_interval:
self.gws_ip_pointer += 1
continue
if self.ip_dict[ip]['links'] >= config.max_links_per_ip:
self.gws_ip_pointer += 1
continue
handshake_time = self.ip_dict[ip]["handshake_time"]
xlog.debug("get ip:%s t:%d", ip, handshake_time)
self.append_ip_history(ip, "get")
self.ip_dict[ip]['get_time'] = time_now
self.ip_dict[ip]['links'] += 1
self.gws_ip_pointer += 1
return ip
except Exception as e:
xlog.exception("get_gws_ip fail:%r", e)
finally:
self.ip_lock.release()
def add_ip(self,
ip_str,
handshake_time,
domain=None,
server='',
fail_times=0):
if not isinstance(ip_str, basestring):
xlog.error("add_ip input")
return
if config.USE_IPV6 and ":" not in ip_str:
xlog.warn("add %s but ipv6", ip_str)
return
handshake_time = int(handshake_time)
self.ip_lock.acquire()
try:
if ip_str in self.ip_dict:
self.ip_dict[ip_str]['handshake_time'] = handshake_time
self.ip_dict[ip_str]['fail_times'] = fail_times
if self.ip_dict[ip_str]['fail_time'] > 0:
self.ip_dict[ip_str]['fail_time'] = 0
self.good_ip_num += 1
self.append_ip_history(ip_str, handshake_time)
return False
self.iplist_need_save = 1
self.good_ip_num += 1
self.ip_dict[ip_str] = {
'handshake_time': handshake_time,
"fail_times": fail_times,
"transfered_data": 0,
'data_active': 0,
'domain': domain,
'server': server,
"history": [[time.time(), handshake_time]],
"fail_time": 0,
"success_time": 0,
"get_time": 0,
"links": 0
}
if 'gws' in server:
self.gws_ip_list.append(ip_str)
return True
except Exception as e:
xlog.exception("add_ip err:%s", e)
finally:
self.ip_lock.release()
return False
def test_appid(appid):
for i in range(0, 3):
ssl_sock = https_manager.get_new_ssl()
if not ssl_sock:
return False
try:
return test_appid_exist(ssl_sock, appid)
except Exception as e:
xlog.exception("check_appid %s %r", appid, e)
continue
return False
def test_appid(appid):
for i in range(0, 3):
ssl_sock = https_manager.get_new_ssl()
if not ssl_sock:
return True
try:
return test_appid_exist(ssl_sock, appid)
except Exception as e:
xlog.exception("check_appid %s %r", appid, e)
continue
return False
def test_appid(self, appid):
for ip in self.ip_dict:
try:
ssl_sock = check_ip.connect_ssl(ip)
except:
continue
try:
return check_ip.check_appid(ssl_sock, appid)
except Exception as e:
xlog.exception("check_appid %s %r", appid, e)
continue
return False
def req_config_handler(self):
req = urlparse.urlparse(self.path).query
reqs = urlparse.parse_qs(req, keep_blank_values=True)
data = ''
try:
if reqs['cmd'] == ['get_config']:
data = json.dumps(user_config.user_special,
default=lambda o: o.__dict__)
elif reqs['cmd'] == ['set_config']:
user_config.user_special.appid = self.postvars['appid'][0]
user_config.user_special.password = self.postvars['password'][
0]
user_config.user_special.proxy_enable = self.postvars[
'proxy_enable'][0]
user_config.user_special.proxy_type = self.postvars[
'proxy_type'][0]
user_config.user_special.proxy_host = self.postvars[
'proxy_host'][0]
user_config.user_special.proxy_port = int(
self.postvars['proxy_port'][0])
user_config.user_special.proxy_user = self.postvars[
'proxy_user'][0]
user_config.user_special.proxy_passwd = self.postvars[
'proxy_passwd'][0]
user_config.user_special.host_appengine_mode = self.postvars[
'host_appengine_mode'][0]
user_config.user_special.use_ipv6 = int(
self.postvars['use_ipv6'][0])
user_config.save()
config.load()
appid_manager.reset_appid()
import connect_manager
connect_manager.load_proxy_config()
connect_manager.https_manager.load_config()
connect_manager.forwork_manager.load_config()
google_ip.reset()
check_ip.load_proxy_config()
data = '{"res":"success"}'
self.send_response('text/html', data)
#http_request("http://127.0.0.1:8085/init_module?module=gae_proxy&cmd=restart")
return
except Exception as e:
xlog.exception("req_config_handler except:%s", e)
data = '{"res":"fail", "except":"%s"}' % e
self.send_response('text/html', data)
def xxnet_version():
readme_file = os.path.join(root_path, "README.md")
try:
fd = open(readme_file, "r")
lines = fd.readlines()
import re
p = re.compile(r'https://codeload.github.com/XX-net/XX-Net/zip/([0-9]+)\.([0-9]+)\.([0-9]+)') #zip/([0-9]+).([0-9]+).([0-9]+)
#m = p.match(content)
for line in lines:
m = p.match(line)
if m:
version = m.group(1) + "." + m.group(2) + "." + m.group(3)
return version
except Exception as e:
xlog.exception("xxnet_version fail")
return "get_version_fail"
def do_CONNECT_FWD(self):
"""socket forward for http CONNECT command"""
host, _, port = self.path.rpartition(':')
port = int(port)
xlog.info('FWD %s %s:%d ', self.command, host, port)
if host == "appengine.google.com" or host == "www.google.com":
connected_in_s = 5 # gae_proxy upload to appengine is slow, it need more 'fresh' connection.
else:
connected_in_s = 10 # gws connect can be used after tcp connection created 15 s
try:
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
data = self.connection.recv(1024)
except Exception as e:
xlog.exception('do_CONNECT_FWD (%r, %r) Exception:%s', host, port,
e)
self.connection.close()
return
remote = forwork_manager.create_connection(host=host,
port=port,
sock_life=connected_in_s)
if remote is None:
self.connection.close()
xlog.warn('FWD %s %s:%d create_connection fail', self.command,
host, port)
return
try:
if data:
remote.send(data)
except Exception as e:
xlog.exception('do_CONNECT_FWD (%r, %r) Exception:%s', host, port,
e)
self.connection.close()
remote.close()
return
# reset timeout default to avoid long http upload failure, but it will delay timeout retry :(
remote.settimeout(None)
forwork_manager.forward_socket(self.connection,
remote,
bufsize=self.bufsize)
xlog.debug('FWD %s %s:%d with closed', self.command, host, port)
def add_ip(self, ip_str, handshake_time, domain=None, server="", fail_times=0):
if not isinstance(ip_str, basestring):
xlog.error("add_ip input")
return
if config.USE_IPV6 and ":" not in ip_str:
xlog.warn("add %s but ipv6", ip_str)
return
handshake_time = int(handshake_time)
self.ip_lock.acquire()
try:
if ip_str in self.ip_dict:
self.ip_dict[ip_str]["handshake_time"] = handshake_time
self.ip_dict[ip_str]["fail_times"] = fail_times
self.ip_dict[ip_str]["fail_time"] = 0
self.append_ip_history(ip_str, handshake_time)
return False
self.iplist_need_save = 1
self.good_ip_num += 1
self.ip_dict[ip_str] = {
"handshake_time": handshake_time,
"fail_times": fail_times,
"transfered_data": 0,
"data_active": 0,
"domain": domain,
"server": server,
"history": [[time.time(), handshake_time]],
"fail_time": 0,
"success_time": 0,
"get_time": 0,
"links": 0,
}
if "gws" in server:
self.gws_ip_list.append(ip_str)
return True
except Exception as e:
xlog.exception("add_ip err:%s", e)
finally:
self.ip_lock.release()
return False
def xxnet_version():
readme_file = os.path.join(root_path, "README.md")
try:
fd = open(readme_file, "r")
lines = fd.readlines()
import re
p = re.compile(
r'https://codeload.github.com/XX-net/XX-Net/zip/([0-9]+)\.([0-9]+)\.([0-9]+)'
) #zip/([0-9]+).([0-9]+).([0-9]+)
#m = p.match(content)
for line in lines:
m = p.match(line)
if m:
version = m.group(1) + "." + m.group(2) + "." + m.group(3)
return version
except Exception as e:
xlog.exception("xxnet_version fail")
return "get_version_fail"
def get_ip(self):
#return self.get_real_random_ip()
while True:
index = random.randint(0, len(self.ip_range_list) - 1)
ip_range = self.ip_range_list[index]
#logging.debug("random.randint %d - %d", ip_range[0], ip_range[1])
if ip_range[1] == ip_range[0]:
return ip_range[1]
try:
id_2 = random.randint(0, ip_range[1] - ip_range[0])
except Exception as e:
xlog.exception("random.randint:%r %d - %d, %d", e, ip_range[0], ip_range[1], ip_range[1] - ip_range[0])
ip = ip_range[0] + id_2
add_last_byte = ip % 255
if add_last_byte == 0 or add_last_byte == 255:
continue
return ip
def req_config_handler(self):
req = urlparse.urlparse(self.path).query
reqs = urlparse.parse_qs(req, keep_blank_values=True)
data = ''
try:
if reqs['cmd'] == ['get_config']:
data = json.dumps(user_config.user_special, default=lambda o: o.__dict__)
elif reqs['cmd'] == ['set_config']:
user_config.user_special.appid = self.postvars['appid'][0]
user_config.user_special.password = self.postvars['password'][0]
user_config.user_special.proxy_enable = self.postvars['proxy_enable'][0]
user_config.user_special.proxy_type = self.postvars['proxy_type'][0]
user_config.user_special.proxy_host = self.postvars['proxy_host'][0]
user_config.user_special.proxy_port = self.postvars['proxy_port'][0]
if not user_config.user_special.proxy_port:
user_config.user_special.proxy_port = 0
user_config.user_special.proxy_user = self.postvars['proxy_user'][0]
user_config.user_special.proxy_passwd = self.postvars['proxy_passwd'][0]
user_config.user_special.host_appengine_mode = self.postvars['host_appengine_mode'][0]
user_config.user_special.use_ipv6 = int(self.postvars['use_ipv6'][0])
user_config.save()
config.load()
appid_manager.reset_appid()
import connect_manager
connect_manager.load_proxy_config()
connect_manager.https_manager.load_config()
connect_manager.forwork_manager.load_config()
google_ip.reset()
check_ip.load_proxy_config()
data = '{"res":"success"}'
self.send_response('text/html', data)
#http_request("http://127.0.0.1:8085/init_module?module=gae_proxy&cmd=restart")
return
except Exception as e:
xlog.exception("req_config_handler except:%s", e)
data = '{"res":"fail", "except":"%s"}' % e
self.send_response('text/html', data)
def add_ip(self, ip, handshake_time, domain=None, server='', fail_times=0):
if not isinstance(ip, basestring):
xlog.error("add_ip input")
return
if config.USE_IPV6 and ":" not in ip:
xlog.warn("add %s but ipv6", ip)
return
handshake_time = int(handshake_time)
self.ip_lock.acquire()
try:
if ip in self.ip_dict:
self.ip_dict[ip]['handshake_time'] = handshake_time
self.ip_dict[ip]['fail_times'] = fail_times
if self.ip_dict[ip]['fail_time'] > 0:
self.ip_dict[ip]['fail_time'] = 0
self.good_ip_num += 1
self.append_ip_history(ip, handshake_time)
return False
self.iplist_need_save = 1
self.good_ip_num += 1
self.ip_dict[ip] = {'handshake_time':handshake_time, "fail_times":fail_times,
"transfered_data":0, 'data_active':0,
'domain':domain, 'server':server,
"history":[[time.time(), handshake_time]], "fail_time":0,
"success_time":0, "get_time":0, "links":0}
if 'gws' in server:
self.gws_ip_list.append(ip)
return True
except Exception as e:
xlog.exception("add_ip err:%s", e)
finally:
self.ip_lock.release()
return False
def is_traffic_quota_allow(self, ip_str):
if self.trafic_control == 0:
return True
self.ip_lock.acquire()
try:
if ip_str in self.ip_dict:
transfered_data = self.ip_dict[ip_str]['transfered_data']
if transfered_data == 0:
return True
active_time = self.ip_dict[ip_str]['data_active']
transfered_data = transfered_data - ((time.time() - active_time) * config.ip_traffic_quota)
if transfered_data <= 0:
self.ip_dict[ip_str]['transfered_data'] = 0
if transfered_data < config.ip_traffic_quota_base:
return True
except Exception as e:
xlog.exception("is_traffic_quota_exceed err:%s", e)
finally:
self.ip_lock.release()
return False
def forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192):
try:
timecount = timeout
while 1:
timecount -= tick
if timecount <= 0:
break
(ins, _, errors) = select.select([local, remote], [],
[local, remote], tick)
if errors:
break
if not ins:
continue
for sock in ins:
data = sock.recv(bufsize)
if not data:
if sock is remote:
xlog.debug("forward remote disconnected.")
else:
xlog.debug("forward local disconnected.")
return
if sock is remote:
local.sendall(data)
timecount = timeout
else:
remote.sendall(data)
timecount = timeout
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET,
errno.ENOTCONN, errno.EPIPE):
xlog.exception("forward except:%s.", e)
finally:
if local:
local.close()
if remote:
remote.close()
def do_CONNECT_FWD(self):
"""socket forward for http CONNECT command"""
host, _, port = self.path.rpartition(':')
port = int(port)
xlog.info('FWD %s %s:%d ', self.command, host, port)
if host == "appengine.google.com" or host == "www.google.com":
connected_in_s = 5 # gae_proxy upload to appengine is slow, it need more 'fresh' connection.
else:
connected_in_s = 10 # gws connect can be used after tcp connection created 15 s
try:
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
data = self.connection.recv(1024)
except Exception as e:
xlog.exception('do_CONNECT_FWD (%r, %r) Exception:%s', host, port, e)
self.connection.close()
return
remote = forwork_manager.create_connection(host=host, port=port, sock_life=connected_in_s)
if remote is None:
self.connection.close()
xlog.warn('FWD %s %s:%d create_connection fail', self.command, host, port)
return
try:
if data:
remote.send(data)
except Exception as e:
xlog.exception('do_CONNECT_FWD (%r, %r) Exception:%s', host, port, e)
self.connection.close()
remote.close()
return
# reset timeout default to avoid long http upload failure, but it will delay timeout retry :(
remote.settimeout(None)
forwork_manager.forward_socket(self.connection, remote, bufsize=self.bufsize)
xlog.debug('FWD %s %s:%d with closed', self.command, host, port)
def is_traffic_quota_allow(self, ip_str):
if self.trafic_control == 0:
return True
self.ip_lock.acquire()
try:
if ip_str in self.ip_dict:
transfered_data = self.ip_dict[ip_str]['transfered_data']
if transfered_data == 0:
return True
active_time = self.ip_dict[ip_str]['data_active']
transfered_data = transfered_data - (
(time.time() - active_time) * config.ip_traffic_quota)
if transfered_data <= 0:
self.ip_dict[ip_str]['transfered_data'] = 0
if transfered_data < config.ip_traffic_quota_base:
return True
except Exception as e:
xlog.exception("is_traffic_quota_exceed err:%s", e)
finally:
self.ip_lock.release()
return False
def forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192):
try:
timecount = timeout
while 1:
timecount -= tick
if timecount <= 0:
break
(ins, _, errors) = select.select([local, remote], [], [local, remote], tick)
if errors:
break
if not ins:
continue
for sock in ins:
data = sock.recv(bufsize)
if not data:
if sock is remote:
xlog.debug("forward remote disconnected.")
else:
xlog.debug("forward local disconnected.")
return
if sock is remote:
local.sendall(data)
timecount = timeout
else:
remote.sendall(data)
timecount = timeout
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.ENOTCONN, errno.EPIPE):
xlog.exception("forward except:%s.", e)
finally:
if local:
local.close()
if remote:
remote.close()
def handler(method, host, url, headers, body, wfile):
time_request = time.time()
if "Connection" in headers and headers["Connection"] == "close":
del headers["Connection"]
errors = []
response = None
while True:
if time.time() - time_request > 30:
return return_fail_message(wfile)
try:
response = fetch(method, host, url, headers, body)
if response:
if response.status > 400:
server_type = response.getheader('Server', "")
if "gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type:
xlog.warn("IP:%s not support GAE, server type:%s status:%d", response.ssl_sock.ip, server_type, response.status)
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
break
except OpenSSL.SSL.SysCallError as e:
errors.append(e)
xlog.warn("direct_handler.handler err:%r %s/%s", e, host, url)
except Exception as e:
errors.append(e)
xlog.exception('direct_handler.handler %r %s %s , retry...', e, host, url)
try:
send_to_browser = True
try:
response_headers = dict((k.title(), v) for k, v in response.getheaders())
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
for key, value in response.getheaders():
send_header(wfile, key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
wait_time = time.time()-time_request
xlog.warn("direct_handler.handler send response fail. t:%d e:%r %s%s", wait_time, e, host, url)
if method == 'HEAD' or response.status in (204, 304):
xlog.info("DIRECT t:%d %d %s %s", (time.time()-time_request)*1000, response.status, host, url)
https_manager.save_ssl_connection_for_reuse(response.ssl_sock, host)
response.close()
return
if 'Transfer-Encoding' in response_headers:
length = 0
while True:
try:
data = response.read(8192)
except httplib.IncompleteRead, e:
data = e.partial
except Exception as e:
google_ip.report_connect_closed(response.ssl_sock.ip, "receive fail")
xlog.warn("direct_handler.handler send Transfer-Encoding t:%d e:%r %s/%s", time.time()-time_request, e, host, url)
response.close()
return
if send_to_browser:
try:
if not data:
wfile.write('0\r\n\r\n')
break
length += len(data)
wfile.write('%x\r\n' % len(data))
wfile.write(data)
wfile.write('\r\n')
except Exception as e:
send_to_browser = False
xlog.warn("direct_handler.handler send Transfer-Encoding t:%d e:%r %s/%s", time.time()-time_request, e, host, url)
else:
if not data:
break
continue
time_last_read = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b):
xlog.warn('direct_handler send to browser return %r %s %r', e_b, host, url)
else:
xlog.warn('direct_handler send to browser return %r %s %r', e_b, host, url)
send_to_browser = False
except NetWorkIOError as e:
google_ip.report_connect_closed(response.ssl_sock.ip, "receive fail")
time_except = time.time()
time_cost = time_except - time_request
if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e):
xlog.exception("direct_handler err:%r %s %s time:%d", e, host, url, time_cost)
else:
xlog.exception("direct_handler except:%r %s %s", e, host, url)
except Exception as e:
google_ip.report_connect_closed(response.ssl_sock.ip, "receive fail")
xlog.exception("direct_handler except:%r %s %s", e, host, url)
def __fetchlet(self, range_queue, data_queue, range_delay_size):
headers = dict((k.title(), v) for k, v in self.headers.items())
headers['Connection'] = 'close'
while not self._stopped:
try:
try:
start, end, response = range_queue.get(timeout=1)
if self.expect_begin < start and data_queue.qsize(
) * self.bufsize + range_delay_size > 30 * 1024 * 1024:
range_queue.put((start, end, response))
time.sleep(10)
continue
headers['Range'] = 'bytes=%d-%d' % (start, end)
if not response:
response = fetch(self.method, self.url, headers,
self.body)
except Queue.Empty:
continue
except Exception as e:
xlog.warning("RangeFetch fetch response %r in __fetchlet",
e)
range_queue.put((start, end, None))
continue
if not response:
xlog.warning('RangeFetch %s return %r', headers['Range'],
response)
range_queue.put((start, end, None))
continue
if response.app_status != 200:
xlog.warning('Range Fetch return %s "%s %s" %s ',
response.app_status, self.method, self.url,
headers['Range'])
if response.app_status == 404:
xlog.warning('APPID %r not exists, remove it.',
response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
xlog.error("no appid left")
self._stopped = True
response.close()
return
if response.app_status == 503:
xlog.warning(
'APPID %r out of Quota, remove it temporary.',
response.ssl_sock.appid)
appid_manager.report_out_of_quota(
response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
xlog.error("no appid left")
self._stopped = True
response.close()
return
google_ip.report_connect_closed(response.ssl_sock.ip,
"app err")
response.close()
range_queue.put((start, end, None))
continue
if response.getheader('Location'):
self.url = urlparse.urljoin(self.url,
response.getheader('Location'))
xlog.info('RangeFetch Redirect(%r)', self.url)
google_ip.report_connect_closed(response.ssl_sock.ip,
"reLocation")
response.close()
range_queue.put((start, end, None))
continue
if 200 <= response.status < 300:
content_range = response.getheader('Content-Range')
if not content_range:
xlog.warning(
'RangeFetch "%s %s" return Content-Range=%r: response headers=%r, retry %s-%s',
self.method, self.url, content_range,
response.getheaders(), start, end)
google_ip.report_connect_closed(
response.ssl_sock.ip, "no range")
response.close()
range_queue.put((start, end, None))
continue
content_length = int(
response.getheader('Content-Length', 0))
xlog.info('>>>>>>>>>>>>>>> [thread %s] %s %s',
threading.currentThread().ident, content_length,
content_range)
time_last_read = time.time()
while start < end + 1:
try:
data = response.read(self.bufsize)
if not data:
if time.time() - time_last_read > 20:
break
else:
time.sleep(0.1)
continue
time_last_read = time.time()
data_len = len(data)
data_queue.put((start, data))
start += data_len
except Exception as e:
xlog.warning('RangeFetch "%s %s" %s failed: %s',
self.method, self.url,
headers['Range'], e)
break
if start < end + 1:
xlog.warning('RangeFetch "%s %s" retry %s-%s',
self.method, self.url, start, end)
google_ip.report_connect_closed(
response.ssl_sock.ip, "down err")
response.close()
range_queue.put((start, end, None))
continue
https_manager.save_ssl_connection_for_reuse(
response.ssl_sock)
xlog.info('>>>>>>>>>>>>>>> Successfully reached %d bytes.',
start - 1)
else:
xlog.error('RangeFetch %r return %s', self.url,
response.status)
google_ip.report_connect_closed(response.ssl_sock.ip,
"status err")
response.close()
range_queue.put((start, end, None))
continue
except StandardError as e:
xlog.exception('RangeFetch._fetchlet error:%s', e)
raise
def handler(method, url, headers, body, wfile):
time_request = time.time()
headers = clean_empty_header(headers)
errors = []
response = None
while True:
if time.time() - time_request > 30: #time out
return return_fail_message(wfile)
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
xlog.warn("fetch gae status:%s url:%s", response.app_status,
url)
try:
server_type = response.getheader('Server', "")
if "gws" not in server_type and "Google Frontend" not in server_type:
xlog.warn("IP:%s not support GAE, server type:%s",
response.ssl_sock.ip, server_type)
google_ip.report_connect_fail(response.ssl_sock.ip,
force_remove=True)
response.close()
continue
except Exception as e:
errors.append(e)
xlog.warn('gae_handler.handler %r %s , retry...', e, url)
continue
if response.app_status == 404:
xlog.warning('APPID %r not exists, remove it.',
response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
google_ip.report_connect_closed(response.ssl_sock.ip,
"appid not exist")
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists',
u'没有可用appid了，请配置可用的appid')
send_response(wfile, 404, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status == 403 or response.app_status == 405: #Method not allowed
# google have changed from gws to gvs, need to remove.
xlog.warning('405 Method not allowed. remove %s ',
response.ssl_sock.ip)
# some ip can connect, and server type is gws
# but can't use as GAE server
# so we need remove it immediately
google_ip.report_connect_fail(response.ssl_sock.ip,
force_remove=True)
response.close()
continue
if response.app_status == 503:
xlog.warning('APPID %r out of Quota, remove it.',
response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
google_ip.report_connect_closed(response.ssl_sock.ip,
"get_timeout")
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists',
u'appid流量不足，请增加appid')
send_response(wfile, 503, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
xlog.warn("gae_exception:%r %s", e, url)
except Exception as e:
errors.append(e)
xlog.exception('gae_handler.handler %r %s , retry...', e, url)
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == 'Transfer-Encoding':
#http://en.wikipedia.org/wiki/Chunked_transfer_encoding
continue
if key in skip_headers:
continue
response_headers[key] = value
if 'X-Head-Content-Length' in response_headers:
if method == "HEAD":
response_headers['Content-Length'] = response_headers[
'X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
send_to_browser = True
try:
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
#logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
xlog.warn("gae_handler.handler send response fail. t:%d e:%r %s",
time.time() - time_request, e, url)
if len(response.app_msg):
xlog.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
google_ip.report_connect_closed(response.ssl_sock.ip, "app err")
response.close()
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(
int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)',
content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length - 1, content_length
body_length = end - start + 1
last_read_time = time.time()
time_response = time.time()
while True:
if start > end:
time_finished = time.time()
if body_length > 1024 and time_finished - time_response > 0:
speed = body_length / (time_finished - time_response)
xlog.info("GAE %d|%s|%d t:%d s:%d hs:%d Spd:%d %d %s",
response.ssl_sock.fd, response.ssl_sock.ip,
response.ssl_sock.received_size,
(time_finished - time_request) * 1000,
length, response.ssl_sock.handshake_time,
int(speed), response.status, url)
else:
xlog.info("GAE %d|%s|%d t:%d s:%d hs:%d %d %s",
response.ssl_sock.fd, response.ssl_sock.ip,
response.ssl_sock.received_size,
(time_finished - time_request) * 1000, length,
response.ssl_sock.handshake_time,
response.status, url)
response.ssl_sock.received_size += body_length
google_ip.report_ip_traffic(response.ssl_sock.ip, body_length)
https_manager.save_ssl_connection_for_reuse(
response.ssl_sock, call_time=time_request)
return
data = response.read(config.AUTORANGE_BUFSIZE)
if not data:
if time.time() - last_read_time > 20:
google_ip.report_connect_closed(response.ssl_sock.ip,
"down fail")
response.close()
xlog.warn("read timeout t:%d len:%d left:%d %s",
(time.time() - time_request) * 1000, length,
(end - start), url)
return
else:
time.sleep(0.1)
continue
last_read_time = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE,
errno.ECONNRESET
) or 'bad write retry' in repr(e_b):
xlog.warn('gae_handler send to browser return %r %r',
e_b, url)
else:
xlog.warn('gae_handler send to browser return %r %r',
e_b, url)
send_to_browser = False
except NetWorkIOError as e:
time_except = time.time()
time_cost = time_except - time_request
if e[0] in (errno.ECONNABORTED,
errno.EPIPE) or 'bad write retry' in repr(e):
xlog.warn("gae_handler err:%r time:%d %s ", e, time_cost, url)
google_ip.report_connect_closed(response.ssl_sock.ip, "Net")
else:
xlog.exception("gae_handler except:%r %s", e, url)
except Exception as e:
xlog.exception("gae_handler except:%r %s", e, url)
def update_pacfile(filename):
opener = get_opener()
listen_ip = config.LISTEN_IP
autoproxy = gae_proxy_listen
blackhole = pac_listen
default = 'DIRECT'
if config.PAC_ADBLOCK:
try:
xlog.info('try download %r to update_pacfile(%r)',
config.PAC_ADBLOCK, filename)
adblock_content = opener.open(config.PAC_ADBLOCK).read()
except Exception as e:
xlog.warn("pac_update download adblock fail:%r", e)
return
try:
xlog.info('try download %r to update_pacfile(%r)',
config.PAC_GFWLIST, filename)
pac_content = opener.open(config.PAC_GFWLIST).read()
except Exception as e:
xlog.warn("pac_update download gfwlist fail:%r", e)
return
content = ''
need_update = True
with open(get_serving_pacfile(), 'rb') as fp:
content = fp.read()
try:
placeholder = '// AUTO-GENERATED RULES, DO NOT MODIFY!'
content = content[:content.index(placeholder) + len(placeholder)]
content = re.sub(r'''blackhole\s*=\s*['"]PROXY [\.\w:]+['"]''',
'blackhole = \'PROXY %s\'' % blackhole, content)
content = re.sub(r'''autoproxy\s*=\s*['"]PROXY [\.\w:]+['"]''',
'autoproxy = \'PROXY %s\'' % autoproxy, content)
if content.startswith('//'):
line = '// Proxy Auto-Config file generated by autoproxy2pac, %s\r\n' % time.strftime(
'%Y-%m-%d %H:%M:%S')
content = line + '\r\n'.join(content.splitlines()[1:])
except ValueError:
need_update = False
try:
if config.PAC_ADBLOCK:
xlog.info('%r downloaded, try convert it with adblock2pac',
config.PAC_ADBLOCK)
jsrule = PacUtil.adblock2pac(adblock_content,
'FindProxyForURLByAdblock',
blackhole, default)
content += '\r\n' + jsrule + '\r\n'
xlog.info('%r downloaded and parsed', config.PAC_ADBLOCK)
else:
content += '\r\nfunction FindProxyForURLByAdblock(url, host) {return "DIRECT";}\r\n'
except Exception as e:
need_update = False
xlog.exception('update_pacfile failed: %r', e)
return
try:
autoproxy_content = base64.b64decode(pac_content)
xlog.info('%r downloaded, try convert it with autoproxy2pac',
config.PAC_GFWLIST)
jsrule = PacUtil.autoproxy2pac(autoproxy_content,
'FindProxyForURLByAutoProxy',
autoproxy, default)
content += '\r\n' + jsrule + '\r\n'
xlog.info('%r downloaded and parsed', config.PAC_GFWLIST)
except Exception as e:
need_update = False
xlog.exception('update_pacfile failed: %r', e)
return
if need_update:
with open(user_pacfile, 'wb') as fp:
fp.write(content)
xlog.info('%r successfully updated', user_pacfile)
serving_pacfile = user_pacfile
def __fetchlet(self, range_queue, data_queue, range_delay_size):
headers = dict((k.title(), v) for k, v in self.headers.items())
headers['Connection'] = 'close'
while not self._stopped:
try:
try:
start, end, response = range_queue.get(timeout=1)
if self.expect_begin < start and data_queue.qsize() * self.bufsize + range_delay_size > 30*1024*1024:
range_queue.put((start, end, response))
time.sleep(10)
continue
headers['Range'] = 'bytes=%d-%d' % (start, end)
if not response:
response = fetch(self.method, self.url, headers, self.body)
except Queue.Empty:
continue
except Exception as e:
xlog.warning("RangeFetch fetch response %r in __fetchlet", e)
range_queue.put((start, end, None))
continue
if not response:
xlog.warning('RangeFetch %s return %r', headers['Range'], response)
range_queue.put((start, end, None))
continue
if response.app_status != 200:
xlog.warning('Range Fetch return %s "%s %s" %s ', response.app_status, self.method, self.url, headers['Range'])
if response.app_status == 404:
xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip)
appid = appid_manager.get_appid()
if not appid:
xlog.error("no appid left")
self._stopped = True
response.close()
return
if response.app_status == 503:
xlog.warning('APPID %r out of Quota, remove it temporary.', response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
xlog.error("no appid left")
self._stopped = True
response.close()
return
google_ip.report_connect_closed(response.ssl_sock.ip, "app err")
response.close()
range_queue.put((start, end, None))
continue
if response.getheader('Location'):
self.url = urlparse.urljoin(self.url, response.getheader('Location'))
xlog.info('RangeFetch Redirect(%r)', self.url)
google_ip.report_connect_closed(response.ssl_sock.ip, "reLocation")
response.close()
range_queue.put((start, end, None))
continue
if 200 <= response.status < 300:
content_range = response.getheader('Content-Range')
if not content_range:
xlog.warning('RangeFetch "%s %s" return Content-Range=%r: response headers=%r, retry %s-%s',
self.method, self.url, content_range, response.getheaders(), start, end)
google_ip.report_connect_closed(response.ssl_sock.ip, "no range")
response.close()
range_queue.put((start, end, None))
continue
content_length = int(response.getheader('Content-Length', 0))
xlog.info('>>>>>>>>>>>>>>> [thread %s] %s %s', threading.currentThread().ident, content_length, content_range)
time_last_read = time.time()
while start < end + 1:
try:
data = response.read(self.bufsize)
if not data:
if time.time() - time_last_read > 20:
break
else:
time.sleep(0.1)
continue
time_last_read = time.time()
data_len = len(data)
data_queue.put((start, data))
start += data_len
except Exception as e:
xlog.warning('RangeFetch "%s %s" %s failed: %s', self.method, self.url, headers['Range'], e)
break
if start < end + 1:
xlog.warning('RangeFetch "%s %s" retry %s-%s', self.method, self.url, start, end)
google_ip.report_connect_closed(response.ssl_sock.ip, "down err")
response.close()
range_queue.put((start, end, None))
continue
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
xlog.info('>>>>>>>>>>>>>>> Successfully reached %d bytes.', start - 1)
else:
xlog.error('RangeFetch %r return %s', self.url, response.status)
google_ip.report_connect_closed(response.ssl_sock.ip, "status err")
response.close()
range_queue.put((start, end, None))
continue
except StandardError as e:
xlog.exception('RangeFetch._fetchlet error:%s', e)
raise
def handler(method, url, headers, body, wfile):
time_request = time.time()
headers = clean_empty_header(headers)
errors = []
response = None
while True:
if time.time() - time_request > 30: #time out
return return_fail_message(wfile)
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
xlog.warn("fetch gae status:%s url:%s", response.app_status, url)
try:
server_type = response.getheader('Server', "")
if "gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type:
xlog.warn("IP:%s not support GAE, server type:%s", response.ssl_sock.ip, server_type)
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
except Exception as e:
errors.append(e)
xlog.warn('gae_handler.handler %r %s , retry...', e, url)
continue
if response.app_status == 404:
#xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip)
google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了，请配置可用的appid')
send_response(wfile, 404, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status == 403 or response.app_status == 405: #Method not allowed
# google have changed from gws to gvs, need to remove.
xlog.warning('405 Method not allowed. remove %s ', response.ssl_sock.ip)
# some ip can connect, and server type is gws
# but can't use as GAE server
# so we need remove it immediately
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
if response.app_status == 503:
xlog.warning('APPID %r out of Quota, remove it.', response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
google_ip.report_connect_closed(response.ssl_sock.ip, "get_timeout")
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists', u'appid流量不足，请增加appid')
send_response(wfile, 503, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
xlog.warn("gae_exception:%r %s", e, url)
except Exception as e:
errors.append(e)
xlog.exception('gae_handler.handler %r %s , retry...', e, url)
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == 'Transfer-Encoding':
#http://en.wikipedia.org/wiki/Chunked_transfer_encoding
continue
if key in skip_headers:
continue
response_headers[key] = value
if 'X-Head-Content-Length' in response_headers:
if method == "HEAD":
response_headers['Content-Length'] = response_headers['X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
send_to_browser = True
try:
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
#logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
xlog.warn("gae_handler.handler send response fail. t:%d e:%r %s", time.time()-time_request, e, url)
if len(response.app_msg):
xlog.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
google_ip.report_connect_closed(response.ssl_sock.ip, "app err")
response.close()
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length-1, content_length
body_length = end - start + 1
last_read_time = time.time()
time_response = time.time()
while True:
if start > end:
time_finished = time.time()
if body_length > 1024 and time_finished - time_response > 0:
speed = body_length / (time_finished - time_response)
xlog.info("GAE %d|%s|%d t:%d s:%d hs:%d Spd:%d %d %s",
response.ssl_sock.fd, response.ssl_sock.ip, response.ssl_sock.received_size, (time_finished-time_request)*1000,
length, response.ssl_sock.handshake_time, int(speed), response.status, url)
else:
xlog.info("GAE %d|%s|%d t:%d s:%d hs:%d %d %s",
response.ssl_sock.fd, response.ssl_sock.ip, response.ssl_sock.received_size, (time_finished-time_request)*1000,
length, response.ssl_sock.handshake_time, response.status, url)
response.ssl_sock.received_size += body_length
https_manager.save_ssl_connection_for_reuse(response.ssl_sock, call_time=time_request)
return
data = response.read(config.AUTORANGE_BUFSIZE)
if not data:
if time.time() - last_read_time > 20:
google_ip.report_connect_closed(response.ssl_sock.ip, "down fail")
response.close()
xlog.warn("read timeout t:%d len:%d left:%d %s", (time.time()-time_request)*1000, length, (end-start), url)
return
else:
time.sleep(0.1)
continue
last_read_time = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b):
xlog.warn('gae_handler send to browser return %r %r', e_b, url)
else:
xlog.warn('gae_handler send to browser return %r %r', e_b, url)
send_to_browser = False
except NetWorkIOError as e:
time_except = time.time()
time_cost = time_except - time_request
if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e):
xlog.warn("gae_handler err:%r time:%d %s ", e, time_cost, url)
google_ip.report_connect_closed(response.ssl_sock.ip, "Net")
else:
xlog.exception("gae_handler except:%r %s", e, url)
except Exception as e:
xlog.exception("gae_handler except:%r %s", e, url)
def load(self):
"""load config from proxy.ini"""
current_path = os.path.dirname(os.path.abspath(__file__))
ConfigParser.RawConfigParser.OPTCRE = re.compile(r'(?P<option>[^=\s][^=]*)\s*(?P<vi>[=])\s*(?P<value>.*)$')
self.CONFIG = ConfigParser.ConfigParser()
self.CONFIG_FILENAME = os.path.abspath( os.path.join(current_path, 'proxy.ini'))
self.DATA_PATH = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir, 'data', 'gae_proxy'))
if not os.path.isdir(self.DATA_PATH):
self.DATA_PATH = current_path
# load ../../../data/gae_proxy/config.ini, set by web_ui
self.CONFIG_USER_FILENAME = os.path.abspath( os.path.join(self.DATA_PATH, 'config.ini'))
self.CONFIG.read(self.CONFIG_FILENAME)
if os.path.isfile(self.CONFIG_USER_FILENAME):
with open(self.CONFIG_USER_FILENAME, 'rb') as fp:
content = fp.read()
self.CONFIG.readfp(io.BytesIO(content))
# load ../../../data/gae_proxy/manual.ini, set by manual
self.CONFIG_MANUAL_FILENAME = os.path.abspath( os.path.join(self.DATA_PATH, 'manual.ini'))
if os.path.isfile(self.CONFIG_MANUAL_FILENAME):
with open(self.CONFIG_MANUAL_FILENAME, 'rb') as fp:
content = fp.read()
try:
self.CONFIG.readfp(io.BytesIO(content))
xlog.info("load manual.ini success")
except Exception as e:
xlog.exception("data/gae_proxy/manual.ini load error:%s", e)
self.LISTEN_IP = self.CONFIG.get('listen', 'ip')
self.LISTEN_PORT = self.CONFIG.getint('listen', 'port')
self.LISTEN_VISIBLE = self.CONFIG.getint('listen', 'visible')
self.LISTEN_DEBUGINFO = self.CONFIG.getint('listen', 'debuginfo')
self.GAE_APPIDS = re.findall(r'[\w\-\.]+', self.CONFIG.get('gae', 'appid').replace('.appspot.com', ''))
self.GAE_PASSWORD = self.CONFIG.get('gae', 'password').strip()
fwd_endswith = []
fwd_hosts = []
direct_endswith = []
direct_hosts = []
gae_endswith = []
gae_hosts = []
for k, v in self.CONFIG.items('hosts'):
if v == "fwd":
if k.startswith('.'):
fwd_endswith.append(k)
else:
fwd_hosts.append(k)
elif v == "direct":
if k.startswith('.'):
direct_endswith.append(k)
else:
direct_hosts.append(k)
elif v == "gae":
if k.startswith('.'):
gae_endswith.append(k)
else:
gae_hosts.append(k)
self.HOSTS_FWD_ENDSWITH = tuple(fwd_endswith)
self.HOSTS_FWD = tuple(fwd_hosts)
self.HOSTS_GAE_ENDSWITH = tuple(gae_endswith)
self.HOSTS_GAE = tuple(gae_hosts)
self.HOSTS_DIRECT_ENDSWITH = tuple(direct_endswith)
self.HOSTS_DIRECT = tuple(direct_hosts)
self.AUTORANGE_MAXSIZE = self.CONFIG.getint('autorange', 'maxsize')
self.AUTORANGE_WAITSIZE = self.CONFIG.getint('autorange', 'waitsize')
self.AUTORANGE_BUFSIZE = self.CONFIG.getint('autorange', 'bufsize')
self.AUTORANGE_THREADS = self.CONFIG.getint('autorange', 'threads')
self.PAC_ENABLE = self.CONFIG.getint('pac', 'enable')
self.PAC_IP = self.CONFIG.get('pac', 'ip')
self.PAC_PORT = self.CONFIG.getint('pac', 'port')
self.PAC_FILE = self.CONFIG.get('pac', 'file').lstrip('/')
self.PAC_GFWLIST = self.CONFIG.get('pac', 'gfwlist')
self.PAC_ADBLOCK = self.CONFIG.get('pac', 'adblock') if self.CONFIG.has_option('pac', 'adblock') else ''
self.PAC_EXPIRED = self.CONFIG.getint('pac', 'expired')
self.pac_url = 'http://%s:%d/%s\n' % (self.PAC_IP, self.PAC_PORT, self.PAC_FILE)
self.CONTROL_ENABLE = self.CONFIG.getint('control', 'enable')
self.CONTROL_IP = self.CONFIG.get('control', 'ip')
self.CONTROL_PORT = self.CONFIG.getint('control', 'port')
self.PROXY_ENABLE = self.CONFIG.getint('proxy', 'enable')
self.PROXY_TYPE = self.CONFIG.get('proxy', 'type')
self.PROXY_HOST = self.CONFIG.get('proxy', 'host')
self.PROXY_PORT = self.CONFIG.get('proxy', 'port')
if self.PROXY_PORT == "":
self.PROXY_PORT = 0
else:
self.PROXY_PORT = int(self.PROXY_PORT)
self.PROXY_USER = self.CONFIG.get('proxy', 'user')
self.PROXY_PASSWD = self.CONFIG.get('proxy', 'passwd')
self.LOVE_ENABLE = self.CONFIG.getint('love', 'enable')
self.LOVE_TIP = self.CONFIG.get('love', 'tip').encode('utf8').decode('unicode-escape').split('|')
self.USE_IPV6 = self.CONFIG.getint('google_ip', 'use_ipv6')
self.ip_traffic_quota = self.CONFIG.getint('google_ip', 'ip_traffic_quota')
self.ip_traffic_quota_base = self.CONFIG.getint('google_ip', 'ip_traffic_quota_base')
self.max_links_per_ip = self.CONFIG.getint('google_ip', 'max_links_per_ip')
self.record_ip_history = self.CONFIG.getint('google_ip', 'record_ip_history')
self.https_max_connect_thread = config.CONFIG.getint("connect_manager", "https_max_connect_thread")
self.connect_interval = config.CONFIG.getint("connect_manager", "connect_interval")
# change to True when finished import CA cert to browser
# launcher will wait import ready then open browser to show status, check update etc
self.cert_import_ready = False
def load(self):
"""load config from proxy.ini"""
current_path = os.path.dirname(os.path.abspath(__file__))
ConfigParser.RawConfigParser.OPTCRE = re.compile(
r'(?P<option>[^=\s][^=]*)\s*(?P<vi>[=])\s*(?P<value>.*)$')
self.CONFIG = ConfigParser.ConfigParser()
self.CONFIG_FILENAME = os.path.abspath(
os.path.join(current_path, 'proxy.ini'))
self.DATA_PATH = os.path.abspath(
os.path.join(current_path, os.pardir, os.pardir, 'data',
'gae_proxy'))
if not os.path.isdir(self.DATA_PATH):
self.DATA_PATH = current_path
# load ../../../data/gae_proxy/config.ini, set by web_ui
self.CONFIG_USER_FILENAME = os.path.abspath(
os.path.join(self.DATA_PATH, 'config.ini'))
self.CONFIG.read(self.CONFIG_FILENAME)
if os.path.isfile(self.CONFIG_USER_FILENAME):
with open(self.CONFIG_USER_FILENAME, 'rb') as fp:
content = fp.read()
self.CONFIG.readfp(io.BytesIO(content))
# load ../../../data/gae_proxy/manual.ini, set by manual
self.CONFIG_MANUAL_FILENAME = os.path.abspath(
os.path.join(self.DATA_PATH, 'manual.ini'))
if os.path.isfile(self.CONFIG_MANUAL_FILENAME):
with open(self.CONFIG_MANUAL_FILENAME, 'rb') as fp:
content = fp.read()
try:
self.CONFIG.readfp(io.BytesIO(content))
xlog.info("load manual.ini success")
except Exception as e:
xlog.exception("data/gae_proxy/manual.ini load error:%s",
e)
self.LISTEN_IP = self.CONFIG.get('listen', 'ip')
self.LISTEN_PORT = self.CONFIG.getint('listen', 'port')
self.LISTEN_VISIBLE = self.CONFIG.getint('listen', 'visible')
self.LISTEN_DEBUGINFO = self.CONFIG.getint('listen', 'debuginfo')
self.GAE_APPIDS = re.findall(
r'[\w\-\.]+',
self.CONFIG.get('gae', 'appid').replace('.appspot.com', ''))
self.GAE_PASSWORD = self.CONFIG.get('gae', 'password').strip()
fwd_endswith = []
fwd_hosts = []
direct_endswith = []
direct_hosts = []
gae_endswith = []
gae_hosts = []
for k, v in self.CONFIG.items('hosts'):
if v == "fwd":
if k.startswith('.'):
fwd_endswith.append(k)
else:
fwd_hosts.append(k)
elif v == "direct":
if k.startswith('.'):
direct_endswith.append(k)
else:
direct_hosts.append(k)
elif v == "gae":
if k.startswith('.'):
gae_endswith.append(k)
else:
gae_hosts.append(k)
self.HOSTS_FWD_ENDSWITH = tuple(fwd_endswith)
self.HOSTS_FWD = tuple(fwd_hosts)
self.HOSTS_GAE_ENDSWITH = tuple(gae_endswith)
self.HOSTS_GAE = tuple(gae_hosts)
self.HOSTS_DIRECT_ENDSWITH = tuple(direct_endswith)
self.HOSTS_DIRECT = tuple(direct_hosts)
self.AUTORANGE_MAXSIZE = self.CONFIG.getint('autorange', 'maxsize')
self.AUTORANGE_WAITSIZE = self.CONFIG.getint('autorange', 'waitsize')
self.AUTORANGE_BUFSIZE = self.CONFIG.getint('autorange', 'bufsize')
self.AUTORANGE_THREADS = self.CONFIG.getint('autorange', 'threads')
self.PAC_ENABLE = self.CONFIG.getint('pac', 'enable')
self.PAC_IP = self.CONFIG.get('pac', 'ip')
self.PAC_PORT = self.CONFIG.getint('pac', 'port')
self.PAC_FILE = self.CONFIG.get('pac', 'file').lstrip('/')
self.PAC_GFWLIST = self.CONFIG.get('pac', 'gfwlist')
self.PAC_ADBLOCK = self.CONFIG.get(
'pac', 'adblock') if self.CONFIG.has_option('pac',
'adblock') else ''
self.PAC_EXPIRED = self.CONFIG.getint('pac', 'expired')
self.pac_url = 'http://%s:%d/%s\n' % (self.PAC_IP, self.PAC_PORT,
self.PAC_FILE)
self.CONTROL_ENABLE = self.CONFIG.getint('control', 'enable')
self.CONTROL_IP = self.CONFIG.get('control', 'ip')
self.CONTROL_PORT = self.CONFIG.getint('control', 'port')
self.PROXY_ENABLE = self.CONFIG.getint('proxy', 'enable')
self.PROXY_TYPE = self.CONFIG.get('proxy', 'type')
self.PROXY_HOST = self.CONFIG.get('proxy', 'host')
self.PROXY_PORT = self.CONFIG.get('proxy', 'port')
if self.PROXY_PORT == "":
self.PROXY_PORT = 0
else:
self.PROXY_PORT = int(self.PROXY_PORT)
self.PROXY_USER = self.CONFIG.get('proxy', 'user')
self.PROXY_PASSWD = self.CONFIG.get('proxy', 'passwd')
self.LOVE_ENABLE = self.CONFIG.getint('love', 'enable')
self.LOVE_TIP = self.CONFIG.get(
'love', 'tip').encode('utf8').decode('unicode-escape').split('|')
self.USE_IPV6 = self.CONFIG.getint('google_ip', 'use_ipv6')
self.ip_traffic_quota = self.CONFIG.getint('google_ip',
'ip_traffic_quota')
self.ip_traffic_quota_base = self.CONFIG.getint(
'google_ip', 'ip_traffic_quota_base')
self.max_links_per_ip = self.CONFIG.getint('google_ip',
'max_links_per_ip')
self.record_ip_history = self.CONFIG.getint('google_ip',
'record_ip_history')
self.https_max_connect_thread = config.CONFIG.getint(
"connect_manager", "https_max_connect_thread")
self.connect_interval = config.CONFIG.getint("connect_manager",
"connect_interval")
# change to True when finished import CA cert to browser
# launcher will wait import ready then open browser to show status, check update etc
self.cert_import_ready = False
def handler(method, host, url, headers, body, wfile):
time_request = time.time()
if "Connection" in headers and headers["Connection"] == "close":
del headers["Connection"]
errors = []
response = None
while True:
if time.time() - time_request > 30:
return return_fail_message(wfile)
try:
response = fetch(method, host, url, headers, body)
if response:
if response.status > 400:
server_type = response.getheader('Server', "")
if "gws" not in server_type and "Google Frontend" not in server_type:
xlog.warn("IP:%s not support GAE, server type:%s status:%d", response.ssl_sock.ip, server_type, response.status)
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
break
except OpenSSL.SSL.SysCallError as e:
errors.append(e)
xlog.warn("direct_handler.handler err:%r %s/%s", e, host, url)
except Exception as e:
errors.append(e)
xlog.exception('direct_handler.handler %r %s %s , retry...', e, host, url)
try:
send_to_browser = True
try:
response_headers = dict((k.title(), v) for k, v in response.getheaders())
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
for key, value in response.getheaders():
send_header(wfile, key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
wait_time = time.time()-time_request
xlog.warn("direct_handler.handler send response fail. t:%d e:%r %s%s", wait_time, e, host, url)
if method == 'HEAD' or response.status in (204, 304):
xlog.info("DIRECT t:%d %d %s %s", (time.time()-time_request)*1000, response.status, host, url)
https_manager.save_ssl_connection_for_reuse(response.ssl_sock, host)
response.close()
return
if 'Transfer-Encoding' in response_headers:
length = 0
while True:
try:
data = response.read(8192)
except httplib.IncompleteRead, e:
data = e.partial
except Exception as e:
google_ip.report_connect_closed(response.ssl_sock.ip, "receive fail")
xlog.warn("direct_handler.handler send Transfer-Encoding t:%d e:%r %s/%s", time.time()-time_request, e, host, url)
response.close()
return
if send_to_browser:
try:
if not data:
wfile.write('0\r\n\r\n')
break
length += len(data)
wfile.write('%x\r\n' % len(data))
wfile.write(data)
wfile.write('\r\n')
except Exception as e:
send_to_browser = False
xlog.warn("direct_handler.handler send Transfer-Encoding t:%d e:%r %s/%s", time.time()-time_request, e, host, url)
else:
if not data:
break
def do_CONNECT_AGENT(self):
"""deploy fake cert to client"""
# GAE supports the following HTTP methods: GET, POST, HEAD, PUT, DELETE, and PATCH
host, _, port = self.path.rpartition(':')
port = int(port)
certfile = CertUtil.get_cert(host)
xlog.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection,
keyfile=certfile,
certfile=certfile,
server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e,
host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception(
'ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
xlog.warn("read request line len:%d",
len(self.raw_requestline))
return
if not self.raw_requestline:
xlog.warn("read request line empty")
return
if not self.parse_request():
xlog.warn("parse request fail:%s", self.raw_requestline)
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET,
errno.EPIPE):
xlog.exception(
'ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
if self.path == "https://www.twitter.com/xxnet":
# for web_ui status page
# auto detect browser proxy setting is work
xlog.debug("CONNECT %s %s", self.command, self.path)
return self.wfile.write(self.self_check_response_data)
xlog.debug('GAE CONNECT %s %s', self.command, self.path)
if self.command not in self.gae_support_methods:
if host.endswith(".google.com") or host.endswith(
config.HOSTS_DIRECT_ENDSWITH) or host.endswith(
config.HOSTS_GAE_ENDSWITH):
if host in config.HOSTS_GAE:
gae_set = [s for s in config.HOSTS_GAE]
gae_set.remove(host)
config.HOSTS_GAE = tuple(gae_set)
if host not in config.HOSTS_DIRECT:
fwd_set = [s for s in config.HOSTS_DIRECT]
fwd_set.append(host)
config.HOSTS_DIRECT = tuple(fwd_set)
xlog.warn(
"Method %s not support in GAE, Redirect to DIRECT for %s",
self.command, self.path)
return self.wfile.write(
('HTTP/1.1 301\r\nLocation: %s\r\n\r\n' %
self.path).encode())
else:
xlog.warn("Method %s not support in GAEProxy for %s",
self.command, self.path)
return self.wfile.write(
('HTTP/1.1 404 Not Found\r\n\r\n').encode())
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
return self.do_AGENT()
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT,
errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def do_CONNECT_DIRECT(self):
"""deploy fake cert to client"""
host, _, port = self.path.rpartition(':')
port = int(port)
if port != 443:
xlog.warn("CONNECT %s port:%d not support", host, port)
return
certfile = CertUtil.get_cert(host)
xlog.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection,
keyfile=certfile,
certfile=certfile,
server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e,
host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception(
'ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
return
if not self.raw_requestline:
self.close_connection = 1
return
if not self.parse_request():
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET,
errno.EPIPE):
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
xlog.debug('GAE CONNECT Direct %s %s', self.command, self.path)
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
if len(self.parsed_url[4]):
path = '?'.join([self.parsed_url[2], self.parsed_url[4]])
else:
path = self.parsed_url[2]
request_headers = dict(
(k.title(), v) for k, v in self.headers.items())
payload = b''
if 'Content-Length' in request_headers:
try:
payload_len = int(request_headers.get('Content-Length', 0))
#logging.debug("payload_len:%d %s %s", payload_len, self.command, self.path)
payload = self.rfile.read(payload_len)
except NetWorkIOError as e:
xlog.error('handle_method_urlfetch read payload failed:%s',
e)
return
direct_handler.handler(self.command, host, path, request_headers,
payload, self.wfile)
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT,
errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def update_pacfile(filename):
opener = get_opener()
listen_ip = config.LISTEN_IP
autoproxy = gae_proxy_listen
blackhole = pac_listen
default = 'DIRECT'
if config.PAC_ADBLOCK:
try:
xlog.info('try download %r to update_pacfile(%r)', config.PAC_ADBLOCK, filename)
adblock_content = opener.open(config.PAC_ADBLOCK).read()
except Exception as e:
xlog.warn("pac_update download adblock fail:%r", e)
return
try:
xlog.info('try download %r to update_pacfile(%r)', config.PAC_GFWLIST, filename)
pac_content = opener.open(config.PAC_GFWLIST).read()
except Exception as e:
xlog.warn("pac_update download gfwlist fail:%r", e)
return
content = ''
need_update = True
with open(get_serving_pacfile(), 'rb') as fp:
content = fp.read()
try:
placeholder = '// AUTO-GENERATED RULES, DO NOT MODIFY!'
content = content[:content.index(placeholder)+len(placeholder)]
content = re.sub(r'''blackhole\s*=\s*['"]PROXY [\.\w:]+['"]''', 'blackhole = \'PROXY %s\'' % blackhole, content)
content = re.sub(r'''autoproxy\s*=\s*['"]PROXY [\.\w:]+['"]''', 'autoproxy = \'PROXY %s\'' % autoproxy, content)
if content.startswith('//'):
line = '// Proxy Auto-Config file generated by autoproxy2pac, %s\r\n' % time.strftime('%Y-%m-%d %H:%M:%S')
content = line + '\r\n'.join(content.splitlines()[1:])
except ValueError:
need_update = False
try:
if config.PAC_ADBLOCK:
xlog.info('%r downloaded, try convert it with adblock2pac', config.PAC_ADBLOCK)
jsrule = PacUtil.adblock2pac(adblock_content, 'FindProxyForURLByAdblock', blackhole, default)
content += '\r\n' + jsrule + '\r\n'
xlog.info('%r downloaded and parsed', config.PAC_ADBLOCK)
else:
content += '\r\nfunction FindProxyForURLByAdblock(url, host) {return "DIRECT";}\r\n'
except Exception as e:
need_update = False
xlog.exception('update_pacfile failed: %r', e)
return
try:
autoproxy_content = base64.b64decode(pac_content)
xlog.info('%r downloaded, try convert it with autoproxy2pac', config.PAC_GFWLIST)
jsrule = PacUtil.autoproxy2pac(autoproxy_content, 'FindProxyForURLByAutoProxy', autoproxy, default)
content += '\r\n' + jsrule + '\r\n'
xlog.info('%r downloaded and parsed', config.PAC_GFWLIST)
except Exception as e:
need_update = False
xlog.exception('update_pacfile failed: %r', e)
return
if need_update:
with open(user_pacfile, 'wb') as fp:
fp.write(content)
xlog.info('%r successfully updated', user_pacfile)
serving_pacfile = user_pacfile
def do_CONNECT_DIRECT(self):
"""deploy fake cert to client"""
host, _, port = self.path.rpartition(':')
port = int(port)
if port != 443:
xlog.warn("CONNECT %s port:%d not support", host, port)
return
certfile = CertUtil.get_cert(host)
xlog.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection, keyfile=certfile, certfile=certfile, server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e, host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s', self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
return
if not self.raw_requestline:
self.close_connection = 1
return
if not self.parse_request():
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.EPIPE):
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
xlog.debug('GAE CONNECT %s %s', self.command, self.path)
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
if len(self.parsed_url[4]):
path = '?'.join([self.parsed_url[2], self.parsed_url[4]])
else:
path = self.parsed_url[2]
request_headers = dict((k.title(), v) for k, v in self.headers.items())
payload = b''
if 'Content-Length' in request_headers:
try:
payload_len = int(request_headers.get('Content-Length', 0))
#logging.debug("payload_len:%d %s %s", payload_len, self.command, self.path)
payload = self.rfile.read(payload_len)
except NetWorkIOError as e:
xlog.error('handle_method_urlfetch read payload failed:%s', e)
return
direct_handler.handler(self.command, host, path, request_headers, payload, self.wfile)
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT, errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def do_CONNECT_AGENT(self):
"""deploy fake cert to client"""
# GAE supports the following HTTP methods: GET, POST, HEAD, PUT, DELETE, and PATCH
host, _, port = self.path.rpartition(':')
port = int(port)
certfile = CertUtil.get_cert(host)
xlog.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection, keyfile=certfile, certfile=certfile, server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e, host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s', self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
xlog.warn("read request line len:%d", len(self.raw_requestline))
return
if not self.raw_requestline:
xlog.warn("read request line empty")
return
if not self.parse_request():
xlog.warn("parse request fail:%s", self.raw_requestline)
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.EPIPE):
xlog.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s', self.connection, e, self.path, e.args[0])
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
xlog.debug('GAE CONNECT %s %s', self.command, self.path)
if self.command not in self.gae_support_methods:
if host.endswith(".google.com") or host.endswith(config.HOSTS_DIRECT_ENDSWITH) or host.endswith(config.HOSTS_GAE_ENDSWITH):
if host in config.HOSTS_GAE:
gae_set = [s for s in config.HOSTS_GAE]
gae_set.remove(host)
config.HOSTS_GAE = tuple(gae_set)
if host not in config.HOSTS_DIRECT:
fwd_set = [s for s in config.HOSTS_DIRECT]
fwd_set.append(host)
config.HOSTS_DIRECT = tuple(fwd_set)
xlog.warn("Method %s not support in GAE, Redirect to DIRECT for %s", self.command, self.path)
return self.wfile.write(('HTTP/1.1 301\r\nLocation: %s\r\n\r\n' % self.path).encode())
else:
xlog.warn("Method %s not support in GAEProxy for %s", self.command, self.path)
return self.wfile.write(('HTTP/1.1 404 Not Found\r\n\r\n').encode())
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
return self.do_AGENT()
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT, errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
continue
time_last_read = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b):
xlog.warn('direct_handler send to browser return %r %s %r', e_b, host, url)
else:
xlog.warn('direct_handler send to browser return %r %s %r', e_b, host, url)
send_to_browser = False
except NetWorkIOError as e:
google_ip.report_connect_closed(response.ssl_sock.ip, "receive fail")
time_except = time.time()
time_cost = time_except - time_request
if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e):
xlog.exception("direct_handler err:%r %s %s time:%d", e, host, url, time_cost)
else:
xlog.exception("direct_handler except:%r %s %s", e, host, url)
except Exception as e:
google_ip.report_connect_closed(response.ssl_sock.ip, "receive fail")
xlog.exception("direct_handler except:%r %s %s", e, host, url)
def req_config_handler(self):
req = urlparse.urlparse(self.path).query
reqs = urlparse.parse_qs(req, keep_blank_values=True)
data = ''
appid_updated = False
try:
if reqs['cmd'] == ['get_config']:
data = json.dumps(user_config.user_special,
default=lambda o: o.__dict__)
elif reqs['cmd'] == ['set_config']:
appids = self.postvars['appid'][0]
if appids != user_config.user_special.appid:
if appids:
fail_appid_list = test_appid.test_appids(appids)
if len(fail_appid_list):
fail_appid = "|".join(fail_appid_list)
return self.send_response(
'text/html',
'{"res":"fail", "reason":"appid fail:%s"}' %
fail_appid)
appid_updated = True
user_config.user_special.appid = appids
user_config.user_special.password = self.postvars['password'][
0]
user_config.user_special.proxy_enable = self.postvars[
'proxy_enable'][0]
user_config.user_special.proxy_type = self.postvars[
'proxy_type'][0]
user_config.user_special.proxy_host = self.postvars[
'proxy_host'][0]
user_config.user_special.proxy_port = self.postvars[
'proxy_port'][0]
if not user_config.user_special.proxy_port:
user_config.user_special.proxy_port = 0
user_config.user_special.proxy_user = self.postvars[
'proxy_user'][0]
user_config.user_special.proxy_passwd = self.postvars[
'proxy_passwd'][0]
user_config.user_special.host_appengine_mode = self.postvars[
'host_appengine_mode'][0]
use_ipv6 = int(self.postvars['use_ipv6'][0])
if user_config.user_special.use_ipv6 != use_ipv6:
if use_ipv6:
if not check_ip.check_ipv6():
xlog.warn("Enable Ipv6 but check failed.")
return self.send_response(
'text/html',
'{"res":"fail", "reason":"IPv6 fail"}')
user_config.user_special.use_ipv6 = use_ipv6
user_config.save()
config.load()
appid_manager.reset_appid()
import connect_manager
connect_manager.load_proxy_config()
connect_manager.https_manager.load_config()
if appid_updated:
connect_manager.https_manager.clean_old_connection()
google_ip.reset()
check_ip.load_proxy_config()
data = '{"res":"success"}'
self.send_response('text/html', data)
#http_request("http://127.0.0.1:8085/init_module?module=gae_proxy&cmd=restart")
return
except Exception as e:
xlog.exception("req_config_handler except:%s", e)
data = '{"res":"fail", "except":"%s"}' % e
self.send_response('text/html', data)
