def send_response(self):
if self.task.responsed:
xlog.error("http2_stream send_response but responsed.%s", self.task.url)
self.close("h2 stream send_response but sended.")
return
self.task.responsed = True
status = int(self.response_headers[b':status'][0])
strip_headers(self.response_headers)
response = simple_http_client.BaseResponse(status=status, headers=self.response_headers)
response.ssl_sock = self.connection.ssl_sock
response.worker = self.connection
response.task = self.task
self.task.queue.put(response)
check_local_network.report_ok(self.connection.ssl_sock.ip)
def send_response(self):
if self.task.responsed:
xlog.error("http2_stream send_response but responsed.%s",
self.task.url)
self.close("h2 stream send_response but sended.")
return
self.task.responsed = True
status = int(self.response_headers[b':status'][0])
strip_headers(self.response_headers)
response = simple_http_client.BaseResponse(
status=status, headers=self.response_headers)
response.ssl_sock = self.connection.ssl_sock
response.worker = self.connection
response.task = self.task
self.task.queue.put(response)
check_local_network.report_ok(self.connection.ssl_sock.ip)
def update_ip(self, ip, handshake_time):
if not isinstance(ip, basestring):
xlog.error("set_ip input")
return
handshake_time = int(handshake_time)
if handshake_time < 5: # that's impossible
xlog.warn("%s handshake:%d impossible", ip, 1000 * handshake_time)
return
time_now = time.time()
check_local_network.report_ok(ip)
check_ip.last_check_time = time_now
check_ip.continue_fail_count = 0
self.ip_lock.acquire()
try:
if ip in self.ip_dict:
# Case: some good ip, average handshake time is 300ms
# some times ip package lost cause handshake time become 2000ms
# this ip will not return back to good ip front until all become bad
# There for, prevent handshake time increase too quickly.
org_time = self.ip_dict[ip]['handshake_time']
if handshake_time - org_time > 500:
self.ip_dict[ip]['handshake_time'] = org_time + 500
else:
self.ip_dict[ip]['handshake_time'] = handshake_time
self.ip_dict[ip]['success_time'] = time_now
if self.ip_dict[ip]['fail_times'] > 0:
self._add_ip_num(ip, 1)
self.ip_dict[ip]['fail_times'] = 0
self.append_ip_history(ip, handshake_time)
self.ip_dict[ip]["fail_time"] = 0
self.iplist_need_save = True
#xlog.debug("update ip:%s not exist", ip)
except Exception as e:
xlog.error("update_ip err:%s", e)
finally:
self.ip_lock.release()
self.save_ip_list()
def update_ip(self, ip, handshake_time):
if not isinstance(ip, basestring):
xlog.error("set_ip input")
return
handshake_time = int(handshake_time)
if handshake_time < 5: # that's impossible
xlog.warn("%s handshake:%d impossible", ip, 1000 * handshake_time)
return
time_now = time.time()
check_local_network.report_ok(ip)
check_ip.last_check_time = time_now
check_ip.continue_fail_count = 0
self.ip_lock.acquire()
try:
if ip in self.ip_dict:
# Case: some good ip, average handshake time is 300ms
# some times ip package lost cause handshake time become 2000ms
# this ip will not return back to good ip front until all become bad
# There for, prevent handshake time increase too quickly.
org_time = self.ip_dict[ip]['handshake_time']
if handshake_time - org_time > 500:
self.ip_dict[ip]['handshake_time'] = org_time + 500
else:
self.ip_dict[ip]['handshake_time'] = handshake_time
self.ip_dict[ip]['success_time'] = time_now
if self.ip_dict[ip]['fail_times'] > 0:
self._add_ip_num(ip, 1)
self.ip_dict[ip]['fail_times'] = 0
self.append_ip_history(ip, handshake_time)
self.ip_dict[ip]["fail_time"] = 0
self.iplist_need_save = True
#xlog.debug("update ip:%s not exist", ip)
except Exception as e:
xlog.error("update_ip err:%s", e)
finally:
self.ip_lock.release()
self.save_ip_list()
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None):
if not check_local_network.is_ok(ip):
with network_fail_lock:
time.sleep(0.1)
ip_port = (ip, port)
sni = sni_generater.get()
if int(config.PROXY_ENABLE):
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb)
ssl_sock.set_connect_state()
if hasattr(ssl_sock, 'set_tlsext_host_name'):
try:
ssl_sock.set_tlsext_host_name(sni)
except:
pass
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
else:
ssl_sock.h2 = False
xlog.debug("%s alpn h2:%s", ip, h2)
except Exception as e:
#xlog.exception("alpn:%r", e)
if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
# xlog.debug("ip:%s http/1.1", ip)
time_handshaked = time.time()
check_local_network.report_ok(ip)
def verify_SSL_certificate_issuer(ssl_sock):
# cert = ssl_sock.get_peer_certificate()
# if not cert:
# #google_ip.report_bad_ip(ssl_sock.ip)
# #connect_control.fall_into_honeypot()
# raise socket.error(' certficate is none')
# issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
# if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
# raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
certs = ssl_sock.get_peer_cert_chain()
if not certs:
# google_ip.report_bad_ip(ssl_sock.ip)
# connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
if len(certs) < 3:
# google_ip.report_connect_fail(ip, force_remove=True)
raise Cert_Exception('No intermediate CA was found.')
if hasattr(OpenSSL.crypto, "dump_publickey"):
# old OpenSSL not support this function.
if OpenSSL.crypto.dump_publickey(OpenSSL.crypto.FILETYPE_PEM, certs[1].get_pubkey()) not in GoogleG23PKP:
# google_ip.report_connect_fail(ip, force_remove=True)
raise Cert_Exception('The intermediate CA is mismatching.')
issuer_commonname = next((v for k, v in certs[0].get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
raise Cert_Exception(' certficate is issued by %r, not Google' % (issuer_commonname))
if check_cert:
verify_SSL_certificate_issuer(ssl_sock)
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
#xlog.debug("conn: %d handshake:%d", connct_time, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock._sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.sni = sni
ssl_sock.host = ""
return ssl_sock
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None):
if not check_local_network.is_ok(ip):
with network_fail_lock:
time.sleep(0.1)
ip_port = (ip, port)
sni = sni_generater.get()
if int(config.PROXY_ENABLE):
sock = socks.socksocket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb)
ssl_sock.set_connect_state()
if hasattr(ssl_sock, 'set_tlsext_host_name'):
try:
ssl_sock.set_tlsext_host_name(sni)
except:
pass
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
else:
ssl_sock.h2 = False
xlog.debug("%s alpn h2:%s", ip, h2)
except Exception as e:
#xlog.exception("alpn:%r", e)
if hasattr(ssl_sock._connection,
"protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
# xlog.debug("ip:%s http/1.1", ip)
time_handshaked = time.time()
check_local_network.report_ok(ip)
def verify_SSL_certificate_issuer(ssl_sock):
# cert = ssl_sock.get_peer_certificate()
# if not cert:
# #google_ip.report_bad_ip(ssl_sock.ip)
# #connect_control.fall_into_honeypot()
# raise socket.error(' certficate is none')
# issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
# if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
# raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
certs = ssl_sock.get_peer_cert_chain()
if not certs:
# google_ip.report_bad_ip(ssl_sock.ip)
# connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
if len(certs) < 3:
# google_ip.report_connect_fail(ip, force_remove=True)
raise Cert_Exception('No intermediate CA was found.')
if hasattr(OpenSSL.crypto, "dump_publickey"):
# old OpenSSL not support this function.
if OpenSSL.crypto.dump_publickey(
OpenSSL.crypto.FILETYPE_PEM,
certs[1].get_pubkey()) not in GoogleG23PKP:
# google_ip.report_connect_fail(ip, force_remove=True)
raise Cert_Exception('The intermediate CA is mismatching.')
issuer_commonname = next(
(v
for k, v in certs[0].get_issuer().get_components() if k == 'CN'),
'')
if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
raise Cert_Exception(' certficate is issued by %r, not Google' %
(issuer_commonname))
if check_cert:
verify_SSL_certificate_issuer(ssl_sock)
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
#xlog.debug("conn: %d handshake:%d", connct_time, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock._sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.sni = sni
ssl_sock.host = ""
return ssl_sock
def request_task(self, task):
start_time = time.time()
task.set_state("h1_req")
self.ssl_sock.last_use_time = time.time()
task.headers['Host'] = self.ssl_sock.host
request_data = 'POST /_gh/ HTTP/1.1\r\n'
request_data += ''.join('%s: %s\r\n' % (k, v)
for k, v in task.headers.items())
request_data += '\r\n'
try:
self.ssl_sock.send(request_data.encode())
payload_len = len(task.body)
start = 0
while start < payload_len:
send_size = min(payload_len - start, 65535)
sended = self.ssl_sock.send(task.body[start:start + send_size])
start += sended
response = simple_http_client.Response(self.ssl_sock)
response.begin(timeout=task.timeout)
except Exception as e:
xlog.warn("%s h1_request:%s %r time_cost:%d inactive:%d", self.ip,
task.url, e, (time.time() - start_time) * 1000,
(time.time() - self.last_active_time) * 1000)
google_ip.report_connect_closed(self.ssl_sock.ip, "request_fail")
self.task = task
self.close("request fail")
return
task.set_state("h1_get_head")
body_length = int(response.getheader("Content-Length", "0"))
task.content_length = body_length
task.responsed = True
response.worker = self
response.task = task
response.ssl_sock = self.ssl_sock
task.queue.put(response)
if body_length == 0:
self.accept_task = True
self.processed_tasks += 1
return
# read response body,
try:
start = 0
end = body_length
time_response = last_read_time = time.time()
while True:
if start >= end:
self.ssl_sock.received_size += body_length
time_cost = (time.time() - time_response)
if time_cost != 0:
speed = body_length / time_cost
task.set_state("h1_finish[SP:%d]" % speed)
self.report_speed(speed, body_length)
self.transfered_size += body_length
task.finish()
self.task = None
self.accept_task = True
self.idle_cb()
self.processed_tasks += 1
self.last_active_time = time.time()
check_local_network.report_ok(self.ssl_sock.ip)
return
data = response.read()
# task.set_state("read body:%d" % len(data))
if not data:
if time.time() - last_read_time > 20:
xlog.warn("%s read timeout t:%d expect:%d left:%d ",
self.ip,
(time.time() - time_response) * 1000,
body_length, (end - start))
break
else:
time.sleep(0.1)
continue
last_read_time = time.time()
data_len = len(data)
start += data_len
task.put_data(data)
except Exception as e:
xlog.warn("%s h1 get data:%r", self.ip, e)
task.finish()
google_ip.report_connect_closed(self.ssl_sock.ip, "down fail")
self.close("request body fail")
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None):
if check_local_network.is_ok(ip):
with network_fail_lock:
time.sleep(0.1)
ip_port = (ip, port)
sni = sni_generater.get()
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb)
ssl_sock.set_connect_state()
ssl_sock.set_tlsext_host_name(sni)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
else:
ssl_sock.h2 = False
except Exception as e:
if hasattr(ssl_sock._connection,
"protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
else:
ssl_sock.h2 = False
time_handshaked = time.time()
# report network ok
check_local_network.report_ok(ip)
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
if check_cert:
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components() if k == 'CN'),
'')
if not issuer_commonname.startswith('Google'):
raise socket.error(' certficate is issued by %r, not Google' %
(issuer_commonname))
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock._sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.sni = sni
ssl_sock.host = ""
return ssl_sock
def request_task(self, task):
start_time = time.time()
task.set_state("h1_req")
self.ssl_sock.last_use_time = time.time()
task.headers['Host'] = self.ssl_sock.host
request_data = 'POST /_gh/ HTTP/1.1\r\n'
request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in task.headers.items())
request_data += '\r\n'
try:
self.ssl_sock.send(request_data.encode())
payload_len = len(task.body)
start = 0
while start < payload_len:
send_size = min(payload_len - start, 65535)
sended = self.ssl_sock.send(task.body[start:start+send_size])
start += sended
response = simple_http_client.Response(self.ssl_sock)
response.begin(timeout=task.timeout)
except Exception as e:
xlog.warn("%s h1_request:%s %r time_cost:%d inactive:%d", self.ip, task.url, e,
(time.time()-start_time)*1000, (time.time() - self.last_active_time)*1000)
google_ip.report_connect_closed(self.ssl_sock.ip, "request_fail")
self.task = task
self.close("request fail")
return
task.set_state("h1_get_head")
body_length = int(response.getheader("Content-Length", "0"))
task.content_length = body_length
task.responsed = True
response.worker = self
response.task = task
response.ssl_sock = self.ssl_sock
task.queue.put(response)
if body_length == 0:
self.accept_task = True
self.processed_tasks += 1
return
# read response body,
try:
start = 0
end = body_length
time_response = last_read_time = time.time()
while True:
if start >= end:
self.ssl_sock.received_size += body_length
time_cost = (time.time() - time_response)
if time_cost != 0:
speed = body_length / time_cost
task.set_state("h1_finish[SP:%d]" % speed)
self.report_speed(speed, body_length)
self.transfered_size += body_length
task.finish()
self.task = None
self.accept_task = True
self.idle_cb()
self.processed_tasks += 1
self.last_active_time = time.time()
check_local_network.report_ok(self.ssl_sock.ip)
return
data = response.read()
# task.set_state("read body:%d" % len(data))
if not data:
if time.time() - last_read_time > 20:
xlog.warn("%s read timeout t:%d expect:%d left:%d ",
self.ip, (time.time()-time_response)*1000, body_length, (end-start))
break
else:
time.sleep(0.1)
continue
last_read_time = time.time()
data_len = len(data)
start += data_len
task.put_data(data)
except Exception as e:
xlog.warn("%s h1 get data:%r", self.ip, e)
task.finish()
google_ip.report_connect_closed(self.ssl_sock.ip, "down fail")
self.close("request body fail")
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None):
if check_local_network.is_ok(ip):
with network_fail_lock:
time.sleep(0.1)
ip_port = (ip, port)
sni = sni_generater.get()
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb)
ssl_sock.set_connect_state()
ssl_sock.set_tlsext_host_name(sni)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
else:
ssl_sock.h2 = False
except Exception as e:
if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
else:
ssl_sock.h2 = False
time_handshaked = time.time()
# report network ok
check_local_network.report_ok(ip)
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
if check_cert:
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock._sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.sni = sni
ssl_sock.host = ""
return ssl_sock