def onClose(self):
self.server.cancelCallback(self.sendPingSpeedResponse)
self.server.cancelCallback(self.setStatusSlow)
self.server.cancelCallback(self.requestSlowClose)
for _, connect in self.symmetryConnectList.items():
connect.onSymmetryConnectServerClose()
log.log(2, self, "onClose")
def analysisThread(self,server): try: domainData = self.domainAnalysisCache.pop() if domainData: if domainData.connect: domainConfig.config.domainConnectTimes(domainData.domain,domainData.connect) domain = getDomainName(domainData.domain) data = self.domainAnalysis[domainData.timeMark][domainData.fromIp][domain if domain else domainData.domain] if not "connect" in data: data["connect"] = 0 data["incoming"] = 0 data["outgoing"] = 0 data["connect"] += domainData.connect data["incoming"] += domainData.incoming data["outgoing"] += domainData.outgoing dataExpireTime = time.time()-86400*7 #删除7天之前的数据 for (k,d) in self.domainAnalysis.items(): if(k < dataExpireTime): del self.domainAnalysis[k] break domainAnalysisJson = json.dumps(self.domainAnalysis) open(domainAnalysisConfig, "wt").write(domainAnalysisJson) except: log.log(3,"analysis error!") server.addDelay(10, analysis.analysisThread,server)
def makeReseponse(self, data, ContentType="text/html", code=200, connection="close", header={}):
def httpdate():
dt = datetime.now();
weekday = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][dt.weekday()]
month = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep",
"Oct", "Nov", "Dec"][dt.month - 1]
return "%s, %02d %s %04d %02d:%02d:%02d GMT" % (weekday, dt.day, month,
dt.year, dt.hour, dt.minute, dt.second)
if type(data) is unicode:
data = data.encode("utf-8")
elif not type(data) is str:
try:
data = json.dumps(data)
except:
log.log(3, data)
data = []
ContentType = "application/json"
httpMessage = ""
httpMessage += "HTTP/1.1 " + str(code) + " " + (httplib.responses[code]) + "\r\n"
httpMessage += "Server: DDDProxy/2.0\r\n"
httpMessage += "Date: " + httpdate() + "\r\n"
httpMessage += "Content-Length: " + str(len(data)) + "\r\n"
httpMessage += "Content-Type: " + ContentType + "\r\n"
httpMessage += "Connection: " + connection + "\r\n"
for k, v in header.items():
httpMessage += k + ": " + v + "\r\n"
httpMessage += "\r\n"
httpMessage += data
return httpMessage
def _doConnectSock(self, address, useSsl=False, cb=None, setThreadName=None): self._connecting = True ok = True try: sock = None threadName = "connect %s:%s" % (address[0], address[1]) log.log(1,threadName) setThreadName(threadName) addr = (socket.gethostbyname(address[0]), address[1]) if useSsl: if self.fetchRemoteCert(address[0], address[1]): sock = ssl.wrap_socket( sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM), ca_certs = self.SSLLocalCertPath(address[0], addr[1]), cert_reqs = ssl.CERT_REQUIRED) else: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if sock: sock.connect(addr) else: ok = False except: log.log(3, address) ok = False if ok: self.server.addCallback(self._setConnect, sock, address) else: self._connecting = False self.server.addCallback(self.onClose) if cb: self.server.addCallback(cb, self if ok else None)
def onData(self,sock): data = None try: data = sock.recv(socketBufferMaxLenght) except ssl.SSLError as e: if e.errno == 2: return log.log(3) except: log.log(3) if data: if isinstance(sock, ssl.SSLSocket): while 1: data_left = sock.pending() if data_left: data += sock.recv(data_left) else: break if sock in self._socketConnectList: handler = self._socketConnectList[sock] handler.onRecv(data) else: self.onExcept(sock)
def sendOptToSymmetryConnect(self,opt): if self._requestRemove: return optData = symmetryConnectServerHandler.optChunk(self.symmetryConnectId, opt) if type(optData) != str: log.log(3,"data not is str") self._symmetryConnectSendPendingCache.append(optData)
def analysisThread(self,server): try: domainData = self.domainAnalysisCache.pop() if domainData: if domainData.connect: domainConfig.config.domainConnectTimes(domainData.domain,domainData.connect) domain = getDomainName(domainData.domain) data = self.domainAnalysis[domainData.timeMark][domainData.fromIp][domain if domain else domainData.domain] if not "connect" in data: data["connect"] = 0 data["incoming"] = 0 data["outgoing"] = 0 data["connect"] += domainData.connect data["incoming"] += domainData.incoming data["outgoing"] += domainData.outgoing dataExpireTime = time.time()-86400*7 #删除7天之前的数据 for (k,d) in self.domainAnalysis.items(): if(k < dataExpireTime): del self.domainAnalysis[k] break domainAnalysisJson = json.dumps(self.domainAnalysis) open(domainAnalysisConfig, "wt").write(domainAnalysisJson) except: log.log(3,"analysis error!") server.addDelay(5, analysis.analysisThread,server)
def onSocketEvent(self, event): if event == sockConnect.socketEventCanRecv: self._onReadyRecv() elif event == sockConnect.socketEventCanSend: self._onReadySend() elif event == sockConnect.socketEventExcept: self.shutdown() log.log(2, self, "<<< socketEventExcept, close")
def wrapToSll(self,sock, address,setThreadName): try: createSSLCert() sock = ssl.wrap_socket(sock, certfile=SSLCertPath,keyfile=SSLKeyPath, server_side=True) self.server.addCallback(symmetryConnectServerHandler._setConnect,self, sock, address) except: log.log(3) self.server.addCallback(self.onClose)
def wrapToSll(self, sock, address, setThreadName): try: createSSLCert() sock = ssl.wrap_socket(sock, certfile=SSLCertPath, keyfile=SSLKeyPath, server_side=True) self.server.addCallback(symmetryConnectServerHandler._setConnect, self, sock, address) except: log.log(3) self.server.addCallback(self.close)
def wrapToSll(self,setThreadName=None): try: setThreadName(str(self)+"wrapToSll") createSSLCert() self.sock = ssl.wrap_socket(self.sock, certfile=SSLCertPath,keyfile=SSLKeyPath, server_side=True) self.server.addCallback(remoteServerConnect.onConnected,self) except: log.log(3) self.server.addCallback(self.onClose)
def onSocketEvent(self, sockfileno, event): if sockfileno in self._socketConnectList: connect = self._socketConnectList[sockfileno] connect.onSocketEvent(event) else: log.log(2, "sock not in self._socketConnectList:", sockfileno); return False return True
def addListen(self, handler, port, host=""): # self.server = bind_sockets(port=self.port, address=self.host) log.log(1, "run in ", host, ":", port) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) print "start server on: " + host + ":" + str(port) sock.bind((host, port)) sock.listen(socketBufferMaxLenght) self.addSockListen(sock, handler)
def addListen(self, port, host=""): # self.server = bind_sockets(port=self.port, address=self.host) log.log(1, "run in ", host, ":", port) server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) print "start server on: "+host+":"+str(port) server.bind((host, port)) server.listen(1024) self.addSockListen(server)
def onHTTP(self, method):
try:
if method == "POST":
self.sendDataToSymmetryConnect(self.makeReseponse(self.server.dumpConnects(),
connection=self.messageParse.connection(),
header={"Access-Control-Allow-Origin":self.messageParse.getHeader("origin")}))
return
except:
log.log(3)
self.sendDataToSymmetryConnect(self.makeReseponse("1", code=405))
def getDomainName(host): try: match = hostMatch.match(host) if match: hostGroup = match.groups() if len(hostGroup) > 2: return "%s%s" % (hostGroup[1], hostGroup[2]) except: log(3, host) return None
def onSocketEvent(self, event): if event == sockConnect.socketEventCanRecv: self._onReadyRecv() elif event == sockConnect.socketEventCanSend: self._onReadySend() elif event == sockConnect.socketEventExcept: self.shutdown() log.log(2,self,"<<< socketEventExcept, close") self.makeAlive()
def onServerToServerMessage(self, serverMessage):
opt = serverMessage["opt"]
if opt == "auth":
timenum = serverMessage["time"]
if time.time()-1800 < timenum and time.time()+1800 > timenum and self.authMake(remoteAuth, timenum)["password"] == serverMessage["password"]:
self.authPass = True
self.sendData(symmetryConnectServerHandler.serverToServerJsonMessageConnectId, json.dumps({"opt":"auth","status":"ok"}))
else:
log.log(2,"auth failed",serverMessage,self.authMake(remoteAuth, timenum))
self.close()
def onHTTP(self, method):
try:
if method == "POST":
self.sendDataToSymmetryConnect(self.makeReseponse(self.server.dumpConnects(),
connection=self.messageParse.connection(),
header={"Access-Control-Allow-Origin":self.messageParse.getHeader("origin")}))
return
except:
log.log(3)
self.sendDataToSymmetryConnect(self.makeReseponse("1", code=405))
def onSymmetryConnectOpt(self,opt): if opt == symmetryConnect.optCloseSymmetryConnect: self.close() elif opt == symmetryConnect.optSymmetryPing: self.sendOptToSymmetryConnect(symmetryConnect.optSymmetryPingResponse) elif opt == symmetryConnect.optSymmetryPingResponse: self.waitSymmetryPingResponse = False elif opt == symmetryConnect.optCloseForceSymmetryConnect: self.shutdown() self.onClose() log.log(2,self,"<<< optCloseForceSymmetryConnect, close")
def sendDataToSymmetryConnect(self,data): if self._requestRemove: return if type(data) != str: log.log(3,"data not is str") for part in symmetryConnectServerHandler.dataChunk(self.symmetryConnectId, data): if type(part) != str: log.log(3,"part not is str") self._symmetryConnectSendPendingCache.append(part)
def onServerToServerMessage(self, serverMessage):
opt = serverMessage["opt"]
if opt == "auth":
timenum = serverMessage["time"]
if time.time() - 1800 < timenum and time.time() + 1800 > timenum and symmetryConnectServerHandler.authMake(remoteAuth, timenum)["password"] == serverMessage["password"]:
self.authPass = True
self.sendData(symmetryConnectServerHandler.serverToServerJsonMessageConnectId, json.dumps({"opt":"auth", "status":"ok"}))
else:
log.log(2, "auth failed", serverMessage, symmetryConnectServerHandler.authMake(remoteAuth, timenum))
self.close()
else:
symmetryConnectServerHandler.onServerToServerMessage(self, serverMessage)
def _onReadySend(self): data = self.getSendData(socketBufferMaxLenght) if data: self.info["send"] += len(data) try: self._sock.send(data) self.onSend(data) return except: log.log(3) log.log(1, self, "<<< request close") self.shutdown()
def onSend(self,sock): if sock in self._socketConnectList: connect = self._socketConnectList[sock] data = connect.requestSendData() if data: try: connect.onSend(data) return except: log.log(3) sock.close() self.onExcept(sock)
def onHTTP(self, method):
try:
if method == "POST":
# postJson = json.loads(self.messageParse.getBody())
self.requestSendToLocalDataCache.append(self.makeReseponse(self.server.dumpConnects(),
connection=self.messageParse.connection(),
header={"Access-Control-Allow-Origin":self.messageParse.getHeader("origin")}))
return
except:
log.log(3)
self.requestSendToLocalDataCache.append(self.makeReseponse("1", code=405))
def _onReadySend(self): data = self.getSendData(socketBufferMaxLenght) if data: self.info["send"] += len(data) try: self._sock.send(data) self.onSend(data) return except: log.log(3) log.log(2,self,"<<< request close") self.shutdown()
def fetchRemoteCert(self, remoteServerHost, remoteServerPort): ok = False sockConnect._createCertLock.acquire() try: if not os.path.exists(self.SSLLocalCertPath(remoteServerHost, remoteServerPort)): cert = ssl.get_server_certificate(addr=(remoteServerHost, remoteServerPort)) open(self.SSLLocalCertPath(remoteServerHost, remoteServerPort), "wt").write(cert) ok = True except: log.log(3, remoteServerHost, remoteServerPort) sockConnect._createCertLock.release() return ok
def start(self):
while True:
rlist = []+self.serverList
wlist = []
currentTime = time.time()
for connect in self._socketConnectList.values():
if connect.pauseSendAndRecv():
continue
rlist.append(connect.sock)
if connect.requestSend():
wlist.append(connect.sock)
elif connect.info["lastAlive"] < currentTime-3600:
connect.close()
try:
s_readable,s_writable,s_exceptional = select.select(rlist, wlist, rlist,1)
except:
log.log(3)
continue;
timeCheck = []
timeCheck.append(("start",time.time()))
for sock in s_readable:
if sock in self.serverList:
self.onConnect(sock)
else:
self.onData(sock)
timeCheck.append(("read",time.time(),sock))
for sock in s_writable:
self.onSend(sock)
timeCheck.append(("write",time.time(),sock))
for sock in s_exceptional:
self.onExcept(sock)
timeCheck.append(("except",time.time(),sock))
cblist = self.callbackList
self.callbackList = []
currentTime = time.time()
for cbobj in cblist:
if cbobj[1] <= currentTime:
cbobj[0](*cbobj[2],**cbobj[3])
else:
self.callbackList.append(cbobj)
timeCheck.append(("callback",time.time(),cbobj))
lastCheck = None
for check in timeCheck:
if lastCheck:
usetime = check[1] - lastCheck[1]
if usetime >1:
log.log(3,check[0],"usetime > 1.0s",usetime,check[2])
lastCheck = check
def onSymmetryConnectOpt(self, opt): if opt == symmetryConnect.optCloseSymmetryConnect: self.close() elif opt == symmetryConnect.optSymmetryPing: self.sendOptToSymmetryConnect(symmetryConnect.optSymmetryPingResponse) elif opt == symmetryConnect.optSymmetryPingResponse: flags = sockConnect.socketIOEventFlagsRead if self.getSendPending(): flags |= sockConnect.socketIOEventFlagsWrite self.setIOEventFlags(flags) elif opt == symmetryConnect.optCloseForceSymmetryConnect: self.shutdown() log.log(2, self, "<<< optCloseForceSymmetryConnect, close")
def parserUrlAddrPort(path):
url = urlparse.urlparse(path)
hostname = url.netloc
port = "443" if url.scheme == "https" else "80"
if hostname.find(':') > 0:
try:
addr, port = hostname.split(':')
except:
log(3)
addr = hostname
else:
addr = hostname
port = int(port)
return (addr, port)
def start(self): while True: eventList = self.epollor.poll(1, 1000) for fd, event in eventList: if select.EPOLLIN & event: self.onSocketEvent(fd, sockConnect.socketEventCanRecv) elif select.EPOLLOUT & event: self.onSocketEvent(fd, sockConnect.socketEventCanSend) elif select.EPOLLERR & event or select.EPOLLHUP & event: self.onSocketEvent(fd, sockConnect.socketEventExcept) else: log.log(3, "unknow event", event) self._handlerCallback()
def run(self):
while True:
try:
self.setName("waitingLoop")
self.runing = False
func, args, kargs = self.tasks.get()
except:
pass
try:
self.startTime = int(time.time())
self.runing = True
kargs["setThreadName"]=self.setName
func( *args, **kargs)
except:
log.log(3)
def fetchRemoteCert(self, remoteServerHost, remoteServerPort): ok = False sockConnect._createCertLock.acquire() certPath = self.SSLLocalCertPath(remoteServerHost, remoteServerPort) try: if not os.path.exists(certPath): cert = ssl.get_server_certificate(addr=(remoteServerHost, remoteServerPort)) f = open(certPath, "wt") f.write(cert) f.close() ok = True except: log.log(3, remoteServerHost, remoteServerPort) sockConnect._createCertLock.release() return ok
def start(self): while True: rlist = [] + self.serverList wlist = [] allList = [] + self.serverList currentTime = time.time() for _,connect in self._socketConnectList.items(): allList.append(connect._sock) if connect.info["lastAlive"] < currentTime - 3600: connect.shutdown() continue if connect.pauseSendAndRecv(): continue rlist.append(connect._sock) if connect.getSendPending(): wlist.append(connect._sock) try: s_readable, s_writable, s_exceptional = select.select(rlist, wlist, rlist, 1) except KeyboardInterrupt: break except: time.sleep(1) log.log(3) continue; for sock in s_readable: if sock in self.serverList: self.onConnect(sock) else: self.onSocketEvent(sock, sockConnect.socketEventCanRecv) for sock in s_writable: self.onSocketEvent(sock, sockConnect.socketEventCanSend) for sock in s_exceptional: self.onSocketEvent(sock, sockConnect.socketEventExcept) cblist = self.callbackList self.callbackList = [] currentTime = time.time() for cbobj in cblist: if cbobj[1] <= currentTime: try: cbobj[0](*cbobj[2], **cbobj[3]) except: log.log(3,cbobj) else: self.callbackList.append(cbobj)
def start(self): while True: try: s_readable, s_writable, s_exceptional = select.select(self.rlist, self.wlist, self.allList, 0.5) except KeyboardInterrupt: break except: time.sleep(1) log.log(3) continue; for sock in s_readable: self.onSocketEvent(sock.fileno(), sockConnect.socketEventCanRecv) for sock in s_writable: self.onSocketEvent(sock.fileno(), sockConnect.socketEventCanSend) for sock in s_exceptional: self.onSocketEvent(sock.fileno(), sockConnect.socketEventExcept) self._handlerCallback()
def onMessage(self,connectId,data):
if connectId>=0:
if self.authPass:
self.getRealConnect(connectId).onlocalRecv(data)
elif connectId==remoteServerHandler.serverToServerAuthConnectId:
size = struct.calcsize("i")
timenum = struct.unpack("i",data[:size])[0]
if time.time()-1800 < timenum and time.time()+1800 > timenum:
if self.authMake(remoteAuth, timenum)==data:
self.authPass = True
self.sendOpt(-1, remoteServerConnect.optAuthOK)
else:
log.log(2,self,"auth not Math:",self.authMake(remoteAuth, timenum),repr(data))
else:
log.log(2,self,"timenum is Expired")
if not self.authPass:
self.close()
else:
remoteServerConnect.onMessage(self,connectId,data)
def _onReadyRecv(self): data = None try: data = self._sock.recv(socketBufferMaxLenght) except ssl.SSLError as e: if e.errno == 2: return log.log(3, self) except: log.log(3, self) if data: if isinstance(self._sock, ssl.SSLSocket): while 1: data_left = self._sock.pending() if data_left: data += self._sock.recv(data_left) else: break self.info["recv"] += len(data) self.onRecv(data) else: log.log(1, self, "<<< data is pool,close") self.shutdown()
def _doConnectSock(address, useSsl=False, cb=None, setThreadName=None):
self._connecting = True
ok = True
try:
sock = None
addr = (random.choice(socket.gethostbyname_ex(address[0])[2]), address[1])
threadName = "connect %s:%s" % (address[0], address[1])
log.log(1, threadName)
setThreadName(threadName)
if addr[0] != address[0]:
self.addressIp = addr[0]
if useSsl:
if self.fetchRemoteCert(addr[0], addr[1]):
sock = ssl.wrap_socket(
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM),
ca_certs = self.SSLLocalCertPath(addr[0], addr[1]),
cert_reqs = ssl.CERT_REQUIRED)
else:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if sock:
sock.connect(addr)
else:
ok = False
except Exception as e:
if str(e).find("handshake"):
self.deleteRemoteCert(addr[0], addr[1])
log.log(3, address)
ok = False
if ok:
self.server.addCallback(self._setConnect, sock, address)
else:
self._connecting = False
self.server.addCallback(self.onClose)
if cb:
self.server.addCallback(cb, self if ok else None)
def _onReadyRecv(self): data = None try: data = self._sock.recv(socketBufferMaxLenght) except ssl.SSLError as e: if e.errno == 2: return log.log(3) except: log.log(3) if data: if isinstance(self._sock, ssl.SSLSocket): while 1: data_left = self._sock.pending() if data_left: data += self._sock.recv(data_left) else: break self.info["recv"] += len(data) self.onRecv(data) else: log.log(2,self,"<<< data is pool,close") self.shutdown()
def _handlerCallback(self): currentTime = time.time() cblist = [] cbDelaylist = [] for cbobj in self.callbackList: if cbobj[1] <= currentTime: cblist.append(cbobj) else: cbDelaylist.append(cbobj) self.callbackList = cbDelaylist for cbobj in cblist: try: cbobj[0](*cbobj[2], **cbobj[3]) except: log.log(3, cbobj) currentTime = time.time() for _, connect in self._socketConnectList.items(): if isinstance(connect, sockServerConnect): continue if connect.lastAlive() < currentTime - 1800: connect.shutdown()
def getSendData(self, length):
while sockConnect.getSendPending(self) <= socketBufferMaxLenght * 2:
found = False
for symmetryConnectId, v in self.symmetryConnectList.items():
if v.symmetryConnectSendPending():
data = v.getSymmetryConnectSendData()
found = True
try:
self.send(data)
except:
raise Exception(data, "is generator??")
elif v.requestRemove():
del self.symmetryConnectList[symmetryConnectId]
if len(self.symmetryConnectList) == 0:
self.server.addDelay(30, self.requestIdleClose)
if not found:
break
data = sockConnect.getSendData(self, length)
if not data and not self._requsetClose:
log.log(2, "not data")
return data
def onConnected(self):
sockConnect.onConnected(self)
self.sendPingSpeedResponse()
log.log(2, self, "onConnected")
def requestSlowClose(self):
log.log(2, self, "very slow , close")
self.close()
def onSocketEvent(self, event): if event == sockConnect.socketEventCanRecv: sock, address = self._sock.accept() connect = self.handler(server=self.server) connect._setConnect(sock, address) log.log(1, connect, "* connect")
def onConnect(self, sock): sock, address = sock.accept() connect = self.handler(server=self) connect._setConnect(sock, address) log.log(2, connect, "* connect")
def onExcept(self,sock): if sock in self._socketConnectList: handler = self._socketConnectList[sock] log.log(2,handler,"< close") del self._socketConnectList[sock] handler.onClose()