def Connect(self, tgtHost): # test the dispatcher # tgtHost should also have port if self.appType == BacCfg.APP_TYPE_PROXY: if (tgtHost is None) or len(tgtHost) == 0: bacutil.DbgTrace( "Connect, not a forwarding URL for PROXY. Ignored.") else: if self.connection is None: try: bacutil.DbgTrace( "Connect, trying to connect to '%s' ..." % tgtHost) self.connection = http.client.HTTPConnection(tgtHost) self.connection.connect() except: bacutil.DbgTrace( "Connect, cannot connect to server '%s'." % tgtHost, bacutil.MSG_ERR) self.connection = None self.state = self.SESSION_STATE_ERROR self.errorCode = self.SESSION_ERR_INVALID_SERVER else: bacutil.DbgTrace( "Connect, connection to host '%s' already exists." % self.tgtHost)
def ProcessReqBuf(self): bacutil.DbgTrace("Session::ProcessHttpReq") while True: httpReq = self.httpReqBuf.Get() if httpReq is None: bacutil.DbgTrace("Session::ProcessHttpReq httpReq invalid.") continue self.ProcessHttpReq(httpReq)
def PostPutCommon(self, method): bacutil.DbgTrace("PostPutCommon enter ...") # Get file if self.headers['Content-Length'] is not None: length = int(self.headers['Content-Length']) bacutil.DbgTrace("PostPutCommon, Content-length: %d." % length) #dataObj = urllib.parse.parse_qs(self.rfile.read(length).decode('utf-8')) #for key, value in dataObj.items(): # print("%s: %s" % (key, value)) dataObj = self.rfile.read(length) #.decode('utf-8') httpReq = bacsessionmgr.sessionMgr.CreateHttpReq( self, method, self.path, self.headers) # PC: dataObj = dataObj.decode("utf-8") dataObj = dataObj.split(";base64,")[1] dataObj = base64.b64decode(dataObj) httpReq.SetDataObj(dataObj) bacsessionmgr.sessionMgr.ProcessHttpReq(httpReq) else: bacutil.DbgTrace("PostPutCommon, chunked.") if self.headers['Transfer-Encoding'] == "chunked": httpReq = None status = self.CHUNK_DATA_NONE while status != self.CHUNK_DATA_LAST: status, dataObj = self.GetReqChunkedData(self.rfile) if httpReq is None: httpReq = bacsessionmgr.sessionMgr.CreateHttpReq( self, method, self.path, self.headers) # PC: dataObj = dataObj.decode("utf-8") dataObj = dataObj.split(";base64,")[1] dataObj = base64.b64decode(dataObj) httpReq.SetDataObj(dataObj) #print("@@@@ dataObj:\n", dataObj) bacsessionmgr.sessionMgr.ProcessHttpReq(httpReq) else: bacutil.DbgTrace( "REQ Content-Length not found, and not chunked.") self._set_headers() message = "Return from POST!\n" self.wfile.write(bytes(message, "utf8")) bacutil.DbgTrace("PostPutCommon leave ...")
def SendResponse(self, httpReq, forwardedRsp): # headers is from http.client.HTTPConnection.getresponse().getheaders() # headers is a list of (key, value) pair httpReq.incomingReqHdlr.send_response(200) for field in forwardedRsp.getheaders(): httpReq.incomingReqHdlr.send_header(field[0], field[1]) httpReq.incomingReqHdlr.end_headers() contentLength = forwardedRsp.getheader('Content-Length') if contentLength is None: bacutil.DbgTrace("Forward chunked response ...") encoding = forwardedRsp.getheader('Transfer-Encoding') if encoding == 'chunked': # pass the chunks to the client status = self.CHUNK_DATA_NONE while True: chunkSize = forwardedRsp._get_chunk_left() #chunkData = forwardedRsp.read1(chunkSize) chunkData = self._readall_chunked(forwardedRsp) bacutil.DbgTrace("chunk size: %d, chunkData length: %d" % (chunkSize, len(chunkData))) bacutil.DbgTrace("chunkData: type %s\n%s" % (type(chunkData), chunkData)) if chunkData == '': status = self.CHUNK_DATA_LAST if status != self.CHUNK_DATA_LAST: numStr = "%x\r\n" % len(chunkData) # "utf-8" encoding should not change the bytes in string httpReq.incomingReqHdlr.wfile.write( bytes(numStr, "utf-8")) httpReq.incomingReqHdlr.wfile.write(bytes(chunkData)) httpReq.incomingReqHdlr.wfile.write(b'\r\n') if chunkSize == len(chunkData): status = self.CHUNK_DATA_LAST break httpReq.incomingReqHdlr.wfile.write(b'0\r\n\r\n') else: rspMsg = forwardedRsp.read(int(contentLength)) bacutil.DbgTrace("Content-Length: %s" % contentLength) bacutil.DbgTrace("rspMsg of type %s:\n" % type(rspMsg), rspMsg) httpReq.incomingReqHdlr.wfile.write(rspMsg)
def MonitorBuffer(self): bacutil.DbgTrace("Session::MonitorBuffer") while True: # check buffer pendingReqs = self.httpReqBuf.NumPendingReqs() mediaObjs = self.httpReqBuf.NumMediaObjs() mediaDataBytes = self.httpReqBuf.NumMediaBytes() mediaBufferedTime = mediaObjs * self.segDuration if pendingReqs > 0: bacutil.DbgTrace( "Buffered requests %2d, media objects %2d, data: %5d Kbytes, %5.1f Seconds." % (pendingReqs, mediaObjs, mediaDataBytes, mediaBufferedTime)) sleep(self.BO_SAMPLING_INTERVAL)
def SetHeaders(self, headers): self.headers = headers if self.headers['Transfer-Encoding'] == "chunked": self.chunkedEncoding = True self.chunkIdx = -1 self.lastChunk = False bacutil.DbgTrace("SetHeaders, chunkedEncoding %r" % self.chunkedEncoding)
def ProcessHttpReq(self, httpReq): bacutil.DbgTrace("BacSessionMgr::ProcessHttpReq") session = self.FindSession(httpReq) if session is not None: return session.ProcessHttpReq(httpReq) return None
def FindSession(self, httpReq): for session in self.sessionList: bacutil.DbgTrace("FindSession, session to host: %s, request to host %s" % (session.tgtHost, httpReq.tgtHost)) if session.MatchHttpReq(httpReq): return session bacutil.DbgTrace("FindSession, create a new session") session = BacSession(self.appType, self.localFolder) # configure a new session and start thread inside session.Configure(httpReq, self.sessionCount) self.sessionCount = self.sessionCount + 1 self.sessionList.append(session) return session
def ProcessGetReq(self, httpReq): localFullPath = self.GenerateLocalFullPath(httpReq.tgtPath) # check whether the file exists if not os.path.exists(localFullPath): self.send_response(404) else: if httpReq.tgtPath.endswith(".txt"): mimetype = 'text/plain' elif httpReq.tgtPath.endswith(".html") or httpReq.tgtPath.endswith( ".htm"): mimetype = 'text/html' elif httpReq.tgtPath.endswith(".jpg"): mimetype = 'image/jpg' elif httpReq.tgtPath.endswith(".gif"): mimetype = 'image/gif' elif httpReq.tgtPath.endswith(".js"): mimetype = 'application/javascript' elif httpReq.tgtPath.endswith(".css"): mimetype = 'text/css' elif httpReq.tgtPath.endswith(".ts"): mimetype = 'video/MP2T' elif httpReq.tgtPath.endswith(".m3u8"): mimetype = 'application/vnd.apple.mpegurl' else: mimetype = 'application/octet-stream' try: f = open(localFullPath, mode='rb') fileSize = os.path.getsize(localFullPath) bacutil.DbgTrace("ProcessGetReq, get file size %d." % fileSize) httpReq.incomingReqHdlr.send_response(200) httpReq.incomingReqHdlr.send_header('Content-type', mimetype) httpReq.incomingReqHdlr.send_header('Content-length', fileSize) httpReq.incomingReqHdlr.send_header( 'Access-Control-Allow-Origin', '*') httpReq.incomingReqHdlr.end_headers() fileContent = f.read() f.close() httpReq.incomingReqHdlr.wfile.write(fileContent) except OSError as err: bacutil.DbgTrace("SendResponse, OS error: {0}".format(err))
def PutHttpReq(self, httpReq): if httpReq.segDuration != 0 and self.segDuration == 0: bacutil.DbgTrace("Set segment target duration %d" % self.segDuration) self.segDuration = httpReq.segDuration # collect statistics self.stat.CheckReq(httpReq) self.httpReqBuf.Put(httpReq)
def ProcessLocalReq(self, httpReq): # local host bacutil.DbgTrace("ProcessLocalReq, local command '%s %s'" % (httpReq.method, httpReq.tgtPath)) if httpReq.method == BacHttpReq.HTTP_METHOD_GET: self.ProcessGetReq(httpReq) httpReq.Log() elif (httpReq.method == BacHttpReq.HTTP_METHOD_PUT) or \ (httpReq.method == BacHttpReq.HTTP_METHOD_POST): self.ProcessPutReq(httpReq) httpReq.Log()
def ProcessHttpReq(self, httpReq): # proxy request bacutil.DbgTrace("ProcessHttpReq, method %s, chunkedEncoding %r" % (httpReq.method, httpReq.chunkedEncoding)) response = None if httpReq.tgtHost == '': self.ProcessLocalReq(httpReq) else: response = self.ForwardHttpReq(httpReq) self.SendResponse(httpReq, response) return response
def main(argv): bacutil.InitTrace() bacCfg = BacCfg() bacCfg.ParseArguments(argv) bacsessionmgr.sessionMgr = bacsessionmgr.BacSessionMgr( bacCfg.appType, bacCfg.localFolder, bacCfg.proxyMode, bacCfg.nextServer) bacutil.DbgTrace("Starting server...") # Server settings server_address = ('', bacCfg.listeningPort) httpd = BacHttpServer(server_address, BacReqHandler) if bacCfg.listeningPort == 443 or bacCfg.listeningPort == 8443: httpd.socket = ssl.wrap_socket(httpd.socket, keyfile=bacCfg.keyFile, certfile=bacCfg.certFile, server_side=True) bacutil.DbgTrace("Running server...") httpd.serve_forever()
def SetUrl(self, method, url): # parse URL # URL is formed as "http://proxyIP:port/targetUrl" # 1. Extract the target URL self.method = method parsedTgtUrl = urlparse('http:/' + url) validHost = self.CheckTgtHost(parsedTgtUrl.netloc) if validHost: self.tgtUrl = 'http:/' + url self.tgtHost = parsedTgtUrl.netloc self.tgtPath = parsedTgtUrl.path else: # a local file self.tgtUrl = url self.tgtHost = '' self.tgtPath = self.tgtUrl bacutil.DbgTrace("SetUrl, host '%s', path '%s'" % (self.tgtHost, self.tgtPath))
def CreateHttpReq(self, incomingReqHdlr, method, path, headers): bacutil.DbgTrace("CreateHttpReq, from host %s, '%s %s" % (incomingReqHdlr.client_address[0], method, path)) # Get URL httpReq = BacHttpReq(incomingReqHdlr) # assign a global index to each request for debugging purpose self.lock.acquire() httpReq.SetIdx(self.reqCount) self.reqCount += 1 self.lock.release() httpReq.SetUrl(method, path) httpReq.SetHeaders(headers) return httpReq
def GetReqChunkedData(self, inputStream): # read the size field, also remove "\r\n" from input stream # inputStream.read returns "bytes" object sizeStr = inputStream.read(2) while sizeStr[-2:] != b"\r\n": sizeStr += inputStream.read(1) chunkSize = int(sizeStr[:-2], 16) bacutil.DbgTrace("GetReqChunkedData, chunk size %d." % chunkSize) status = self.CHUNK_DATA_NONE if chunkSize == 0: # return an empty object dataObj = bytes() status = self.CHUNK_DATA_LAST else: dataObj = inputStream.read(chunkSize) status = self.CHUNK_DATA_NORMAL # remove "\r\n", which is in addition to data, from input stream inputStream.read(2) return status, dataObj
def SendRespMsg(self): if self.socket is None: bacutil.DbgTrace("handler does not have valid socket open for the .", type) return 0 self.socket.sendall(self.respHdrStr)
def SetDataObj(self, dataObj): # check input variable if dataObj is None: self.dataObj = None bacutil.DbgTrace("SetDataObj, Data object is invalid.") return False if self.tgtUrl is None: bacutil.DbgTrace("SetDataObj, URL not configured.") return False self.dataObj = dataObj if self.chunkedEncoding: self.chunkIdx += 1 bacutil.DbgTrace( "SetDataObj, req %d, size %d, chunked %r, chunkIdx %d" % (self.globalHttpReqIdx, len(dataObj), self.chunkedEncoding, self.chunkIdx)) if len(dataObj) == 0: if self.chunkedEncoding: self.lastChunk = True return True if (not self.chunkedEncoding) or (self.chunkIdx == 0): # check object type by looking at the first several bytes self.reqType = self.REQ_NONE if self.tgtPath.endswith('.m3u8'): if dataObj[0:7] != b'#EXTM3U': bacutil.DbgTrace("SetDataObj, invalid m3u8 file format.") else: # convert to a string object playlistStr = dataObj.decode("utf-8") # check whether this is master (variant) playlist if "EXT-X-STREAM-INF" in playlistStr: bacutil.DbgTrace("SetDataObj, master playlist") self.reqType = self.REQ_MASTER_MANIFEST else: bacutil.DbgTrace("SetDataObj, bitrate playlist") self.reqType = self.REQ_BITRATE_MANIFEST playlistStrLines = playlistStr.split('\n') for line in playlistStrLines: if "EXT-X-TARGETDURATION" in playlistStr: # extract target segment duration from playlist tokens = line.split(':') if len(tokens) == 2: self.segDuration = int(tokens[1]) else: bacutil.DbgTrace( "SetDataObj, invalid EXT-X-TARGETDURATION line in playlist file." ) break elif self.tgtPath.endswith('.ts'): # do not know yet. will check if it belongs to any session bacutil.DbgTrace("SetDataObj, segment") self.reqType = self.REQ_SEGMENT else: bacutil.DbgTrace("SetDataObj, '%s' file type not recognized." % self.tgtPath)
def ForwardHttpReq(self, httpReq): response = None self.Connect(httpReq.tgtHost) if self.connection is None: bacutil.DbgTrace( "ForwardHttpReq, Req not processed: '%s http://%s/%s'" % (httpReq.method, httpReq.tgtHost, httpReq.tgtPath), bacutil.MSG_ERR) return None if httpReq.segDuration != 0 and self.segDuration == 0: bacutil.DbgTrace("ForwardHttpReq, set segment target duration %d" % self.segDuration) self.segDuration = httpReq.segDuration # send request headers if (not httpReq.chunkedEncoding) or (httpReq.chunkIdx == 0): bacutil.DbgTrace( "ForwardHttpReq, request '%s http://%s%s'" % (httpReq.method, httpReq.tgtHost, httpReq.tgtPath)) self.connection.putrequest(httpReq.method, httpReq.tgtPath) # send all the keys in the incoming headers, except 'Host' keys = httpReq.headers.keys() for key in keys: # get the value for each key if key == 'Host': # replace 'Host' with the host of the next hop self.connection.putheader('Host', httpReq.tgtHost) else: self.connection.putheader(key, httpReq.headers.get(key)) self.connection.endheaders() # send data if httpReq.dataObj: bacutil.DbgTrace("ForwardHttpReq, %d bytes" % (len(httpReq.dataObj))) if httpReq.chunkedEncoding: bacutil.DbgTrace("ForwardHttpReq, chunk %d" % httpReq.chunkIdx) numStr = "%x\r\n" % len(httpReq.dataObj) sendData = bytes(numStr, "utf-8") # try to call connection.send only once if len(httpReq.dataObj) > 0: # "utf-8" encoding should not change the bytes in string sendData = bytes(numStr, "utf-8") + httpReq.dataObj + b'\r\n' self.connection.send(sendData) else: self.connection.send(httpReq.dataObj) # a response is expected only if not chunk encoding, or the end of chunk if (not httpReq.chunkedEncoding) or (httpReq.lastChunk): response = self.connection.getresponse() bacutil.DbgTrace("ForwardHttpReq, exit") return response
def ProcessPutReq(self, httpReq): localFullPath = self.GenerateLocalFullPath(httpReq.tgtPath) bacutil.DbgTrace("Post object of length %d" % len(httpReq.dataObj)) if (httpReq.dataObj is not None) and len(httpReq.dataObj) > 0: if httpReq.postedFile is None: self.CheckPath(localFullPath) httpReq.postedFile = open(localFullPath, mode='wb') httpReq.postedFile.write(httpReq.dataObj) # finish the writing of the file, and send response if (httpReq.postedFile is not None): if (not httpReq.chunkedEncoding) or httpReq.lastChunk: httpReq.postedFile.close() resultPath = self.ProcessCompletedFile(localFullPath) httpReq.postedFile = None # send response fileSize = 0 fileContent = None if resultPath is not None: f = open(resultPath, mode='r') fileSize = os.path.getsize(resultPath) fileContent = f.read() f.close() httpReq.incomingReqHdlr.send_response(200) httpReq.incomingReqHdlr.send_header( 'Access-Control-Allow-Origin', '*') # PC: httpReq.incomingReqHdlr.send_header('Content-length', 0) #if fileSize > 0: # httpReq.incomingReqHdlr.send_header('Content-type', 'text/plain') # httpReq.incomingReqHdlr.send_header('Content-length', fileSize) if fileContent is not None: returnStatus = "None" print(type(fileContent)) """ if "CN" in fileContent: returnStatus = "CNN" if "BREAKING" in fileContent: returnStatus = "CNN-Breaking news" """ # PC: more fine tuning of the AI image recognition logic to_search = [ "CN", "CNN", "BREAKING NEWS", "COMING UP", "ONLY ON", "DEVELOPING" ] for s in to_search: if s in fileContent: returnStatus = "CNN" httpReq.incomingReqHdlr.send_header( 'Access-Control-Expose-Headers', 'Akamai-Program-Detection') httpReq.incomingReqHdlr.send_header( 'Akamai-Program-Detection', returnStatus) bacutil.DbgTrace('Akamai-Program-Detection %s' % returnStatus) httpReq.incomingReqHdlr.end_headers()