def send_response(self): if self.task.responsed: xlog.error("http2_stream send_response but responsed.%s", self.task.url) self.close("h2 stream send_response but sended.") return self.task.responsed = True status = int(self.response_headers[b':status'][0]) strip_headers(self.response_headers) response = simple_http_client.BaseResponse(status=status, headers=self.response_headers) response.ssl_sock = self.connection.ssl_sock response.worker = self.connection response.task = self.task self.task.queue.put(response) check_local_network.report_ok(self.connection.ssl_sock.ip)
def send_response(self): if self.task.responsed: xlog.error("http2_stream send_response but responsed.%s", self.task.url) self.close("h2 stream send_response but sended.") return self.task.responsed = True status = int(self.response_headers[b':status'][0]) strip_headers(self.response_headers) response = simple_http_client.BaseResponse( status=status, headers=self.response_headers) response.ssl_sock = self.connection.ssl_sock response.worker = self.connection response.task = self.task self.task.queue.put(response) check_local_network.report_ok(self.connection.ssl_sock.ip)
def update_ip(self, ip, handshake_time): if not isinstance(ip, basestring): xlog.error("set_ip input") return handshake_time = int(handshake_time) if handshake_time < 5: # that's impossible xlog.warn("%s handshake:%d impossible", ip, 1000 * handshake_time) return time_now = time.time() check_local_network.report_ok(ip) check_ip.last_check_time = time_now check_ip.continue_fail_count = 0 self.ip_lock.acquire() try: if ip in self.ip_dict: # Case: some good ip, average handshake time is 300ms # some times ip package lost cause handshake time become 2000ms # this ip will not return back to good ip front until all become bad # There for, prevent handshake time increase too quickly. org_time = self.ip_dict[ip]['handshake_time'] if handshake_time - org_time > 500: self.ip_dict[ip]['handshake_time'] = org_time + 500 else: self.ip_dict[ip]['handshake_time'] = handshake_time self.ip_dict[ip]['success_time'] = time_now if self.ip_dict[ip]['fail_times'] > 0: self._add_ip_num(ip, 1) self.ip_dict[ip]['fail_times'] = 0 self.append_ip_history(ip, handshake_time) self.ip_dict[ip]["fail_time"] = 0 self.iplist_need_save = True #xlog.debug("update ip:%s not exist", ip) except Exception as e: xlog.error("update_ip err:%s", e) finally: self.ip_lock.release() self.save_ip_list()
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None): if not check_local_network.is_ok(ip): with network_fail_lock: time.sleep(0.1) ip_port = (ip, port) sni = sni_generater.get() if int(config.PROXY_ENABLE): sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) sock.settimeout(timeout) ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb) ssl_sock.set_connect_state() if hasattr(ssl_sock, 'set_tlsext_host_name'): try: ssl_sock.set_tlsext_host_name(sni) except: pass time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() try: h2 = ssl_sock.get_alpn_proto_negotiated() if h2 == "h2": ssl_sock.h2 = True else: ssl_sock.h2 = False xlog.debug("%s alpn h2:%s", ip, h2) except Exception as e: #xlog.exception("alpn:%r", e) if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2": ssl_sock.h2 = True # xlog.debug("ip:%s http/2", ip) else: ssl_sock.h2 = False # xlog.debug("ip:%s http/1.1", ip) time_handshaked = time.time() check_local_network.report_ok(ip) def verify_SSL_certificate_issuer(ssl_sock): # cert = ssl_sock.get_peer_certificate() # if not cert: # #google_ip.report_bad_ip(ssl_sock.ip) # #connect_control.fall_into_honeypot() # raise socket.error(' certficate is none') # issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') # if not issuer_commonname.startswith('Google'): # google_ip.report_connect_fail(ip, force_remove=True) # raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) certs = ssl_sock.get_peer_cert_chain() if not certs: # google_ip.report_bad_ip(ssl_sock.ip) # connect_control.fall_into_honeypot() raise socket.error(' certficate is none') if len(certs) < 3: # google_ip.report_connect_fail(ip, force_remove=True) raise Cert_Exception('No intermediate CA was found.') if hasattr(OpenSSL.crypto, "dump_publickey"): # old OpenSSL not support this function. if OpenSSL.crypto.dump_publickey(OpenSSL.crypto.FILETYPE_PEM, certs[1].get_pubkey()) not in GoogleG23PKP: # google_ip.report_connect_fail(ip, force_remove=True) raise Cert_Exception('The intermediate CA is mismatching.') issuer_commonname = next((v for k, v in certs[0].get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): # google_ip.report_connect_fail(ip, force_remove=True) raise Cert_Exception(' certficate is issued by %r, not Google' % (issuer_commonname)) if check_cert: verify_SSL_certificate_issuer(ssl_sock) connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) #xlog.debug("conn: %d handshake:%d", connct_time, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock._sock = sock ssl_sock.connct_time = connct_time ssl_sock.handshake_time = handshake_time ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.last_use_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.sni = sni ssl_sock.host = "" return ssl_sock
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None): if not check_local_network.is_ok(ip): with network_fail_lock: time.sleep(0.1) ip_port = (ip, port) sni = sni_generater.get() if int(config.PROXY_ENABLE): sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) sock.settimeout(timeout) ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb) ssl_sock.set_connect_state() if hasattr(ssl_sock, 'set_tlsext_host_name'): try: ssl_sock.set_tlsext_host_name(sni) except: pass time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() try: h2 = ssl_sock.get_alpn_proto_negotiated() if h2 == "h2": ssl_sock.h2 = True else: ssl_sock.h2 = False xlog.debug("%s alpn h2:%s", ip, h2) except Exception as e: #xlog.exception("alpn:%r", e) if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2": ssl_sock.h2 = True # xlog.debug("ip:%s http/2", ip) else: ssl_sock.h2 = False # xlog.debug("ip:%s http/1.1", ip) time_handshaked = time.time() check_local_network.report_ok(ip) def verify_SSL_certificate_issuer(ssl_sock): # cert = ssl_sock.get_peer_certificate() # if not cert: # #google_ip.report_bad_ip(ssl_sock.ip) # #connect_control.fall_into_honeypot() # raise socket.error(' certficate is none') # issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') # if not issuer_commonname.startswith('Google'): # google_ip.report_connect_fail(ip, force_remove=True) # raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) certs = ssl_sock.get_peer_cert_chain() if not certs: # google_ip.report_bad_ip(ssl_sock.ip) # connect_control.fall_into_honeypot() raise socket.error(' certficate is none') if len(certs) < 3: # google_ip.report_connect_fail(ip, force_remove=True) raise Cert_Exception('No intermediate CA was found.') if hasattr(OpenSSL.crypto, "dump_publickey"): # old OpenSSL not support this function. if OpenSSL.crypto.dump_publickey( OpenSSL.crypto.FILETYPE_PEM, certs[1].get_pubkey()) not in GoogleG23PKP: # google_ip.report_connect_fail(ip, force_remove=True) raise Cert_Exception('The intermediate CA is mismatching.') issuer_commonname = next( (v for k, v in certs[0].get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): # google_ip.report_connect_fail(ip, force_remove=True) raise Cert_Exception(' certficate is issued by %r, not Google' % (issuer_commonname)) if check_cert: verify_SSL_certificate_issuer(ssl_sock) connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) #xlog.debug("conn: %d handshake:%d", connct_time, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock._sock = sock ssl_sock.connct_time = connct_time ssl_sock.handshake_time = handshake_time ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.last_use_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.sni = sni ssl_sock.host = "" return ssl_sock
def request_task(self, task): start_time = time.time() task.set_state("h1_req") self.ssl_sock.last_use_time = time.time() task.headers['Host'] = self.ssl_sock.host request_data = 'POST /_gh/ HTTP/1.1\r\n' request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in task.headers.items()) request_data += '\r\n' try: self.ssl_sock.send(request_data.encode()) payload_len = len(task.body) start = 0 while start < payload_len: send_size = min(payload_len - start, 65535) sended = self.ssl_sock.send(task.body[start:start + send_size]) start += sended response = simple_http_client.Response(self.ssl_sock) response.begin(timeout=task.timeout) except Exception as e: xlog.warn("%s h1_request:%s %r time_cost:%d inactive:%d", self.ip, task.url, e, (time.time() - start_time) * 1000, (time.time() - self.last_active_time) * 1000) google_ip.report_connect_closed(self.ssl_sock.ip, "request_fail") self.task = task self.close("request fail") return task.set_state("h1_get_head") body_length = int(response.getheader("Content-Length", "0")) task.content_length = body_length task.responsed = True response.worker = self response.task = task response.ssl_sock = self.ssl_sock task.queue.put(response) if body_length == 0: self.accept_task = True self.processed_tasks += 1 return # read response body, try: start = 0 end = body_length time_response = last_read_time = time.time() while True: if start >= end: self.ssl_sock.received_size += body_length time_cost = (time.time() - time_response) if time_cost != 0: speed = body_length / time_cost task.set_state("h1_finish[SP:%d]" % speed) self.report_speed(speed, body_length) self.transfered_size += body_length task.finish() self.task = None self.accept_task = True self.idle_cb() self.processed_tasks += 1 self.last_active_time = time.time() check_local_network.report_ok(self.ssl_sock.ip) return data = response.read() # task.set_state("read body:%d" % len(data)) if not data: if time.time() - last_read_time > 20: xlog.warn("%s read timeout t:%d expect:%d left:%d ", self.ip, (time.time() - time_response) * 1000, body_length, (end - start)) break else: time.sleep(0.1) continue last_read_time = time.time() data_len = len(data) start += data_len task.put_data(data) except Exception as e: xlog.warn("%s h1 get data:%r", self.ip, e) task.finish() google_ip.report_connect_closed(self.ssl_sock.ip, "down fail") self.close("request body fail")
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None): if check_local_network.is_ok(ip): with network_fail_lock: time.sleep(0.1) ip_port = (ip, port) sni = sni_generater.get() if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) sock.settimeout(timeout) ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb) ssl_sock.set_connect_state() ssl_sock.set_tlsext_host_name(sni) time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() try: h2 = ssl_sock.get_alpn_proto_negotiated() if h2 == "h2": ssl_sock.h2 = True else: ssl_sock.h2 = False except Exception as e: if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2": ssl_sock.h2 = True else: ssl_sock.h2 = False time_handshaked = time.time() # report network ok check_local_network.report_ok(ip) cert = ssl_sock.get_peer_certificate() if not cert: raise socket.error(' certficate is none') if check_cert: issuer_commonname = next( (v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): raise socket.error(' certficate is issued by %r, not Google' % (issuer_commonname)) connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock._sock = sock ssl_sock.connct_time = connct_time ssl_sock.handshake_time = handshake_time ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.last_use_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.sni = sni ssl_sock.host = "" return ssl_sock
def request_task(self, task): start_time = time.time() task.set_state("h1_req") self.ssl_sock.last_use_time = time.time() task.headers['Host'] = self.ssl_sock.host request_data = 'POST /_gh/ HTTP/1.1\r\n' request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in task.headers.items()) request_data += '\r\n' try: self.ssl_sock.send(request_data.encode()) payload_len = len(task.body) start = 0 while start < payload_len: send_size = min(payload_len - start, 65535) sended = self.ssl_sock.send(task.body[start:start+send_size]) start += sended response = simple_http_client.Response(self.ssl_sock) response.begin(timeout=task.timeout) except Exception as e: xlog.warn("%s h1_request:%s %r time_cost:%d inactive:%d", self.ip, task.url, e, (time.time()-start_time)*1000, (time.time() - self.last_active_time)*1000) google_ip.report_connect_closed(self.ssl_sock.ip, "request_fail") self.task = task self.close("request fail") return task.set_state("h1_get_head") body_length = int(response.getheader("Content-Length", "0")) task.content_length = body_length task.responsed = True response.worker = self response.task = task response.ssl_sock = self.ssl_sock task.queue.put(response) if body_length == 0: self.accept_task = True self.processed_tasks += 1 return # read response body, try: start = 0 end = body_length time_response = last_read_time = time.time() while True: if start >= end: self.ssl_sock.received_size += body_length time_cost = (time.time() - time_response) if time_cost != 0: speed = body_length / time_cost task.set_state("h1_finish[SP:%d]" % speed) self.report_speed(speed, body_length) self.transfered_size += body_length task.finish() self.task = None self.accept_task = True self.idle_cb() self.processed_tasks += 1 self.last_active_time = time.time() check_local_network.report_ok(self.ssl_sock.ip) return data = response.read() # task.set_state("read body:%d" % len(data)) if not data: if time.time() - last_read_time > 20: xlog.warn("%s read timeout t:%d expect:%d left:%d ", self.ip, (time.time()-time_response)*1000, body_length, (end-start)) break else: time.sleep(0.1) continue last_read_time = time.time() data_len = len(data) start += data_len task.put_data(data) except Exception as e: xlog.warn("%s h1 get data:%r", self.ip, e) task.finish() google_ip.report_connect_closed(self.ssl_sock.ip, "down fail") self.close("request body fail")
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None): if check_local_network.is_ok(ip): with network_fail_lock: time.sleep(0.1) ip_port = (ip, port) sni = sni_generater.get() if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) sock.settimeout(timeout) ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb) ssl_sock.set_connect_state() ssl_sock.set_tlsext_host_name(sni) time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() try: h2 = ssl_sock.get_alpn_proto_negotiated() if h2 == "h2": ssl_sock.h2 = True else: ssl_sock.h2 = False except Exception as e: if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2": ssl_sock.h2 = True else: ssl_sock.h2 = False time_handshaked = time.time() # report network ok check_local_network.report_ok(ip) cert = ssl_sock.get_peer_certificate() if not cert: raise socket.error(' certficate is none') if check_cert: issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock._sock = sock ssl_sock.connct_time = connct_time ssl_sock.handshake_time = handshake_time ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.last_use_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.sni = sni ssl_sock.host = "" return ssl_sock