def getConn(self): # Connect to the database try: self.conn = pymysql.connect(host=self.__host, user=self.__user, password=self.__password, db=self.__db, charset=self.__charset, cursorclass=pymysql.cursors.DictCursor) except pymysql.Error as e: logging.error('conn', e)
def processer(self, args): try: # {"addr": clientAddr, "connections": clientConn, "acceptTime":acceptTime } requestParam = args[0] fd = args[1] epollFd = args[2] startLinkTime = requestParam['acceptTime'] # 在超时时间内判断是否请求应该成功响应,如果已经响应中断循环 while time.time() - startLinkTime <= TIMEOUT: # 如果 key generateResponseDataTime存在,说明请求已经成功响应,不需要再计算 if requestParam.has_key('responseTime'): # print('TimeoutTasksThread %d 请求已经成功响应,不执行后续操作' % fd) break else: # print('TimeoutTasksThread - %d已经超时' % fd) # 如果超时之后时间还没变化 if requestParam['acceptTime'] == startLinkTime: WebApi._epollFd = epollFd WebApi._fd = fd WebApi().fastResponse(504) # 超时之后更改直接注册epoll out 事件,触发Epoll()下__epollOut引发异常断开socket # epollFd.modify(fd, select.EPOLLET | select.EPOLLOUT) except Exception as message: logging.error('TimeoutTasksThread - %s' % message)
def createEpoll(self): # 开始EPOLL处理 try: # 创建epoll并且注册事件 self.epollFd = select.epoll() self.epollFd.register(self.serverFdFileNo, select.EPOLLIN) except select.error as message: logging.error('createEpoll错误 %s' % message)
def createEpoll(self): # 开始EPOLL处理 try: # 创建epoll并且注册事件 self.epollFd = select.epoll() self.epollFd.register(self.serverFdFileNo, select.EPOLLIN) except select.error as message: logging.error('createEpoll错误 %s' % message)
def run(self): while 1: try: args, kwargs = self.queue.get() self.processer(args) except Queue.Empty: continue except: logging.error("timeout thread error")
def __modifyEpollEvent(self): try: self.__epollFd.modify(self.__clientFd, select.EPOLLET | select.EPOLLOUT) except socket.error as e: # 过滤掉errno 2 和errno 9 两者出现基本都是因为 服务端准备发送数据的时候客户端已经把连接关闭掉的情况下 if e.errno not in (2, 9): logging.error('WebApi - __modifyEpollEvent : %s %d' % (e, e.errno)) finally: pass
def __parseMultipartPost(self, requestBody): try: # logging.debug('requestBody',requestBody) # multipart/form-data; boundary=----WebKitFormBoundaryMmTyCOdF17gBJbHF # 获取boundary=后面的字符串 boundary = self.requestHeaders['Content-Type'].split('=')[-1] # -- 标记结束,[数据内容以两条横线结尾,并同样以一个换行结束] postList = requestBody.split('--' + boundary) # logging.debug('postList',postList) for i in postList: # 排除掉空格等 if 'name' not in i: continue if '\r\n\r\n' in i: splitStr = '\r\n\r\n' else: splitStr = '\n\n' splitItem = i.split(splitStr, maxsplit=1) # logging.debug('itItem',splitItem) describeNameStr = splitItem[0] describeNameDic = {} # 判断有上传文件的情况 if 'Content-Type' in describeNameStr and 'filename=' in describeNameStr: desItems = describeNameStr.split('\r\n')[1].split(';') for desitem in desItems: # logging.debug('----------------',desItems) if '=' in desitem: # logging.debug(desitem) desSplitItem = desitem.split("=") itemName = desSplitItem[0].strip() # logging.debug(desSplitItem[1]) # 转换编码,防止中文乱码 itemValue = desSplitItem[1].strip().strip( '"').encode('ISO-8859-1').decode('UTF-8') describeNameDic[itemName] = itemValue # logging.debug(describeNameDic) # logging.debug(itemName,itemValue) # logging.debug(describeNameDic) postName = describeNameDic.get('name') # logging.debug(splitItem) postValue = [ describeNameDic.get('filename'), splitItem[-1].strip() ] self.postFileDic[postName] = postValue # logging.debug(self.postFileDic) else: postName = describeNameStr.split("name=")[-1].strip('"') postValue = splitItem[-1].strip() # logging.debug('postName=',postName,' postValue=',postValue) self.postParamsDic[postName] = postValue # logging.debug('+++++++++++++++++++++',self.postParamsDic) except Exception as e: logging.error(e) pass
def __handledynamicFile(self): try: ctlFilePath = os.path.join(_ctlPath, self.controller + '.py') # 获取文件的最近修改时间 mtime = os.path.getmtime(ctlFilePath) controller = _ctlImportDic.get(self.controller) # 判断突然增加controller文件的情况 if controller == None: controller = __import__(self.controller) # 更新文件信息 _ctlModifyMTimeDic[self.controller] = mtime _ctlImportDic[self.controller] = controller else: # 获取文件记录的修改时间 lastMTime = _ctlModifyMTimeDic[self.controller] # 如果文件有更新 if mtime > lastMTime: controller = reload(sys.modules[self.controller]) _ctlModifyMTimeDic[self.controller] = mtime _ctlImportDic[self.controller] = controller # 从导入的controller模块中获取要调用的action方法 action = getattr(controller, self.action) # 获取返回值 try: actionReturn = action(self) except Exception as e: self.fastResponse(500, e) else: if actionReturn: self.responseText = str(actionReturn) # 如果配置了gzip,且大小超过设定值 if GZIP and len(self.responseText) > GZIP_MIN_LENGTH: buf = StringIO() f = gzip.GzipFile(mode='wb', fileobj=buf, compresslevel=GZIP_LEVEL) f.write(self.responseText) f.close() # self.responseText = gzip.GzipFile(data=self.responseText, compresslevel=8) self.__startResponse() except ImportError as e: self.fastResponse(404) logging.error(e) except Exception as e: self.fastResponse(500) logging.error(e) finally: pass
def processer(self, args): try: #self.epollFd, fd, connParam epollFd, fd, connParam = args # 处理请求,并生成response数据 WebApi(epollFd, fd, connParam) except Exception as e: logging.error("WorkTasksThread error: %s " % (e)) finally: pass
def clearFd(self, fd): try: self.epollFd.unregister(fd) param = self.__connParams[fd] param.get('connections').shutdown(socket.SHUT_RDWR) param.get('connections').close() del self.__connParams[fd] except socket.error as message: logging.error('clearFd错误 %s' % (message)) pass
def clearFd(self, fd): try: self.epollFd.unregister(fd) param = self.__connParams[fd] param.get('connections').shutdown(socket.SHUT_RDWR) param.get('connections').close() del self.__connParams[fd] except socket.error as message: logging.error('clearFd错误 %s' % (message)) pass
def __handledynamicFile(self): try: ctlFilePath = os.path.join(_ctlPath, self.controller + '.py') # 获取文件的最近修改时间 mtime = os.path.getmtime(ctlFilePath) controller = _ctlImportDic.get(self.controller) # 判断突然增加controller文件的情况 if controller == None: controller = __import__(self.controller) # 更新文件信息 _ctlModifyMTimeDic[self.controller] = mtime _ctlImportDic[self.controller] = controller else: # 获取文件记录的修改时间 lastMTime = _ctlModifyMTimeDic[self.controller] # 如果文件有更新 if mtime > lastMTime: controller = reload(sys.modules[self.controller]) _ctlModifyMTimeDic[self.controller] = mtime _ctlImportDic[self.controller] = controller # 从导入的controller模块中获取要调用的action方法 action = getattr(controller, self.action) # 获取返回值 try: actionReturn = action(self) except Exception as e: self.fastResponse(500, e) else: if actionReturn: self.responseText = str(actionReturn) # 如果配置了gzip,且大小超过设定值 if GZIP and len(self.responseText) > GZIP_MIN_LENGTH: buf = StringIO() f = gzip.GzipFile(mode='wb', fileobj=buf, compresslevel=GZIP_LEVEL) f.write(self.responseText) f.close() # self.responseText = gzip.GzipFile(data=self.responseText, compresslevel=8) self.__startResponse() except ImportError as e: self.fastResponse(404) logging.error(e) except Exception as e: self.fastResponse(500) logging.error(e) finally: pass
def __modifyEpollEvent(self): try: self.__epollFd.modify(self.__clientFd, select.EPOLLET | select.EPOLLOUT) except socket.error as e: # 过滤掉errno 2 和errno 9 两者出现基本都是因为 服务端准备发送数据的时候客户端已经把连接关闭掉的情况下 if e.errno not in (2, 9): logging.error('WebApi - __modifyEpollEvent : %s %d' % (e, e.errno)) finally: pass
def __parseMultipartPost(self,requestBody): try: # logging.debug('requestBody',requestBody) # multipart/form-data; boundary=----WebKitFormBoundaryMmTyCOdF17gBJbHF # 获取boundary=后面的字符串 boundary = self.requestHeaders['Content-Type'].split('=')[-1] # -- 标记结束,[数据内容以两条横线结尾,并同样以一个换行结束] postList = requestBody.split('--'+boundary) # logging.debug('postList',postList) for i in postList: # 排除掉空格等 if 'name' not in i: continue if '\r\n\r\n' in i: splitStr = '\r\n\r\n' else: splitStr = '\n\n' splitItem = i.split(splitStr,maxsplit=1) # logging.debug('itItem',splitItem) describeNameStr = splitItem[0] describeNameDic = {} # 判断有上传文件的情况 if 'Content-Type' in describeNameStr and 'filename=' in describeNameStr: desItems = describeNameStr.split('\r\n')[1].split(';') for desitem in desItems: # logging.debug('----------------',desItems) if '=' in desitem: # logging.debug(desitem) desSplitItem = desitem.split("=") itemName = desSplitItem[0].strip() # logging.debug(desSplitItem[1]) # 转换编码,防止中文乱码 itemValue = desSplitItem[1].strip().strip('"').encode('ISO-8859-1').decode('UTF-8') describeNameDic[itemName] = itemValue # logging.debug(describeNameDic) # logging.debug(itemName,itemValue) # logging.debug(describeNameDic) postName = describeNameDic.get('name') # logging.debug(splitItem) postValue = [describeNameDic.get('filename'),splitItem[-1].strip()] self.postFileDic[postName] = postValue # logging.debug(self.postFileDic) else: postName = describeNameStr.split("name=")[-1].strip('"') postValue = splitItem[-1].strip() # logging.debug('postName=',postName,' postValue=',postValue) self.postParamsDic[postName] = postValue # logging.debug('+++++++++++++++++++++',self.postParamsDic) except Exception as e: logging.error(e) pass
def epollEventHandle(self): eventsDict = {select.EPOLLIN: self.__epollIn, select.EPOLLOUT: self.__epollOut} try: while 1: events = self.epollFd.poll() # 死循环 实时获取epoll中的事件 for fileNo, event in events: eventsDict.get(event, self.clearFd)(fileNo) except KeyError as message: # 变量dict以外的事件不处理 logging.error( '进程ID:%d - epollEventHandle() - 事件超出范围 - 错误代码: %s ' % (self.pid, message)) finally: pass
def __startResponse(self): try: # 例如 403 Forbidden httpStatus = "%d %s" % (self.responseStatus, HTTPStatus(self.responseStatus).phrase) headStr = '' # 获取响应内容长度 responseTextLen = len(self.responseText.encode()) logging.debug('=========================', self.responseText, len(self.responseText)) # Transfer-Encoding 和 Content-Length 不可共存 if 'Transfer-Encoding' not in self.responseHeaders: self.responseHeaders["Content-Length"] = responseTextLen self.connParam['responseTextLen'] = responseTextLen # 拼接Header for key in self.responseHeaders: headStr += "%s: %s\r\n" % (key, self.responseHeaders[key]) # 拼接完整response self.connParam['responseData'] = "%s %s\r\n%s\n%s" % ( self.httpVersion, httpStatus, headStr, self.responseText) # 设置响应数据准备完成的时间 self.connParam['waitTime'] = time.time() # 如果请求头不包含close,即代表开启keepalived if self.responseHeaders.get("Connection", "") != "close": self.connParam["keepalive"] = True # # 删掉request请求数据 # if self.connParam.has_key('requestData'): # del self.connParam['requestData'] self.__modifyEpollEvent() except Exception as e: logging.error('WebApi - startResponse : %s' % e) finally: pass
def __startResponse(self): try: # 例如 403 Forbidden httpStatus = "%d %s" % (self.responseStatus, HTTPStatus(self.responseStatus).phrase) headStr = '' # 获取响应内容长度 responseTextLen = len(self.responseText.encode()) logging.debug('=========================',self.responseText,len(self.responseText)) # Transfer-Encoding 和 Content-Length 不可共存 if 'Transfer-Encoding' not in self.responseHeaders: self.responseHeaders["Content-Length"] = responseTextLen self.connParam['responseTextLen'] = responseTextLen # 拼接Header for key in self.responseHeaders: headStr += "%s: %s\r\n" % (key, self.responseHeaders[key]) # 拼接完整response self.connParam['responseData'] = "%s %s\r\n%s\n%s" % ( self.httpVersion, httpStatus, headStr, self.responseText) # 设置响应数据准备完成的时间 self.connParam['waitTime'] = time.time() # 如果请求头不包含close,即代表开启keepalived if self.responseHeaders.get("Connection", "") != "close": self.connParam["keepalive"] = True # # 删掉request请求数据 # if self.connParam.has_key('requestData'): # del self.connParam['requestData'] self.__modifyEpollEvent() except Exception as e: logging.error('WebApi - startResponse : %s' % e) finally: pass
def trim(self, file): fileFd = open(file) try: data = fileFd.read() # 设置 匹配注释 正则表达式 annotationReg = re.compile('(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)') # 设置 匹配空格,换行符,制表符 正则表达式 spaceReg = re.compile('\s+') # 去掉注释 annotationTrimData = re.sub(annotationReg, '', data) # 再去掉 空格,换行符,制表符 spaceTrimData = re.sub(spaceReg, '', annotationTrimData) return spaceTrimData except Exception as e: logging.error(e) finally: fileFd.close()
def epollEventHandle(self): eventsDict = { select.EPOLLIN: self.__epollIn, select.EPOLLOUT: self.__epollOut } try: while 1: events = self.epollFd.poll() # 死循环 实时获取epoll中的事件 for fileNo, event in events: eventsDict.get(event, self.clearFd)(fileNo) except KeyError as message: # 变量dict以外的事件不处理 logging.error('进程ID:%d - epollEventHandle() - 事件超出范围 - 错误代码: %s ' % (self.pid, message)) finally: pass
def render(self, templateName, context=None): if not context: context = {} renderPath = 'View/' try: engine = tenjin.Engine() # 文件名允许省略后缀 if '.html' not in templateName: templateHtml = engine.render(renderPath + templateName + '.html', context) else: templateHtml = engine.render(renderPath + templateName, context) return templateHtml except tenjin.TemplateNotFoundError as e: self.fastResponse(404, 'Template Not Found') logging.error(e) except tenjin.ParseError as e: self.fastResponse(500, 'Template Parse Error') logging.error(e) except tenjin.TemplateSyntaxError as e: self.fastResponse(500, 'Template Syntax Error') logging.error(e)
def __epollOut(self, fd): logging.debug('%d - %d - 开始send数据 -----------' % (fd, self.pid)) # print self.__connParams,self.__connParams.get(fd) # 获取连接数据 responseParam = self.__connParams.get(fd) # 获取响应数据 responseData = responseParam.get('responseData') logging.debug('=====', responseData) if not responseData: logging.error('responseData为空 %d将执行clearFd() ' % fd) self.clearFd(fd) return currentClientConn = responseParam['connections'] try: currentClientConn.sendall(responseData.encode('utf-8')) # while sendLen <= responseDataLen: # sendLen = currentClientConn.sendto(responseData,addr) except socket.error as message: logging.error( '进程ID:%d - 文件描述符:%d - __epollOut()发送响应数据错误,原因:%s 下面将执行pass' % (self.pid, fd, message)) else: responseParam['responseTime'] = time.time() self.clearFd(fd) logging.debug('%d - 发送完毕\n' % fd) # # 如果没有指定keepalived,执行clearFd()断开连接,否则不断开,保持住连接以继续接受数据 # # if not responseParam.get('keepalive', None): # # # else: # # self.__connParams[fd] = {"addr": responseParam['addr'], # # "connections": currentClientConn} # # # 更新 epoll 句柄中连接 fd 注册事件为 可读 # # self.epollFd.modify(fd, select.EPOLLET | select.EPOLLIN) # # logging.info('进程ID:%d - 文件描述符:%d -数据成功响应 %s %s %d ' % ( # # self.pid, fd, str(responseParam['addr']), responseParam['responseData'][:15], # # responseParam['responseTextLen'])) finally: pass
def render(self, templateName, context=None): if not context: context = {} renderPath = 'View/' try: engine = tenjin.Engine() # 文件名允许省略后缀 if '.html' not in templateName: templateHtml = engine.render( renderPath + templateName + '.html', context) else: templateHtml = engine.render(renderPath + templateName, context) return templateHtml except tenjin.TemplateNotFoundError as e: self.fastResponse(404, 'Template Not Found') logging.error(e) except tenjin.ParseError as e: self.fastResponse(500, 'Template Parse Error') logging.error(e) except tenjin.TemplateSyntaxError as e: self.fastResponse(500, 'Template Syntax Error') logging.error(e)
def createSocketServer(self): try: # 创建socket套接字 # socket.SOCK_STREAM 流式socket # socket.AF_INET 服务器之间网络通信 self.serverFd = serverFd = socket.socket(socket.AF_INET, socket.SOCK_STREAM) serverFd.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # 将套接字绑定到地址, 在AF_INET下,以元组(host,port)的形式表示地址. serverFd.bind((HOST, PORT)) # 开始监听TCP传入连接。backlog指定在拒绝连接之前,操作系统可以挂起的最大连接数量。该值至少为1,大部分应用程序设为5就可以了。 serverFd.listen(BACKLOG) # 如果flag为0,则将套接字设为非阻塞模式,否则将套接字设为阻塞模式(默认值)。非阻塞模式下,如果调用recv()没有发现任何数据,或send()调用无法立即发送数据,那么将引起socket.error异常。 serverFd.setblocking(0) # serverFd.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) except socket.error as message: # print('监听服务失败,请检查端口冲突,或者查看是否重复运行 %s' % message) logging.error('监听服务失败,请检查端口冲突,或者查看是否重复运行 %s' % message) sys.exit(0)
def trim(self, file): fileFd = open(file) try: data = fileFd.read() # 设置 匹配注释 正则表达式 annotationReg = re.compile( '(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)') # 设置 匹配空格,换行符,制表符 正则表达式 spaceReg = re.compile('\s+') # 去掉注释 annotationTrimData = re.sub(annotationReg, '', data) # 再去掉 空格,换行符,制表符 spaceTrimData = re.sub(spaceReg, '', annotationTrimData) return spaceTrimData except Exception as e: logging.error(e) finally: fileFd.close()
def __epollOut(self, fd): logging.debug('%d - %d - 开始send数据 -----------' % (fd, self.pid)) # print self.__connParams,self.__connParams.get(fd) # 获取连接数据 responseParam = self.__connParams.get(fd) # 获取响应数据 responseData = responseParam.get('responseData') logging.debug('=====', responseData) if not responseData: logging.error('responseData为空 %d将执行clearFd() ' % fd) self.clearFd(fd) return currentClientConn = responseParam['connections'] try: currentClientConn.sendall(responseData.encode('utf-8')) # while sendLen <= responseDataLen: # sendLen = currentClientConn.sendto(responseData,addr) except socket.error as message: logging.error('进程ID:%d - 文件描述符:%d - __epollOut()发送响应数据错误,原因:%s 下面将执行pass' % (self.pid, fd, message)) else: responseParam['responseTime'] = time.time() self.clearFd(fd) logging.debug('%d - 发送完毕\n' % fd) # # 如果没有指定keepalived,执行clearFd()断开连接,否则不断开,保持住连接以继续接受数据 # # if not responseParam.get('keepalive', None): # # # else: # # self.__connParams[fd] = {"addr": responseParam['addr'], # # "connections": currentClientConn} # # # 更新 epoll 句柄中连接 fd 注册事件为 可读 # # self.epollFd.modify(fd, select.EPOLLET | select.EPOLLIN) # # logging.info('进程ID:%d - 文件描述符:%d -数据成功响应 %s %s %d ' % ( # # self.pid, fd, str(responseParam['addr']), responseParam['responseData'][:15], # # responseParam['responseTextLen'])) finally: pass
def insert(self, table, data): keyList = [] valueList = [] # 给表名添加斜引号 table = self.__convertTableName(table) if type(data) == dict: for k, v in data.items(): keyList.append("`%s`" % k) # 将值转化为供解释器读取的形式,自带引号 valueList.append("%s" % repr(v)) elif type(data) == list: pass joinKey = ','.join(keyList) joinValue = ','.join(valueList) sql = 'insert into %s (%s) values (%s)' % (table, joinKey, joinValue) # logging.debug(sql) try: with self.conn.cursor() as cursor: cursor.execute(sql) self.conn.commit() except pymysql.Error as e: logging.error(sql, e)
def createSocketServer(self): try: # 创建socket套接字 # socket.SOCK_STREAM 流式socket # socket.AF_INET 服务器之间网络通信 self.serverFd = serverFd = socket.socket(socket.AF_INET, socket.SOCK_STREAM) serverFd.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # 将套接字绑定到地址, 在AF_INET下,以元组(host,port)的形式表示地址. serverFd.bind((HOST, PORT)) # 开始监听TCP传入连接。backlog指定在拒绝连接之前,操作系统可以挂起的最大连接数量。该值至少为1,大部分应用程序设为5就可以了。 serverFd.listen(BACKLOG) # 如果flag为0,则将套接字设为非阻塞模式,否则将套接字设为阻塞模式(默认值)。非阻塞模式下,如果调用recv()没有发现任何数据,或send()调用无法立即发送数据,那么将引起socket.error异常。 serverFd.setblocking(0) # serverFd.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) except socket.error as message: # print('监听服务失败,请检查端口冲突,或者查看是否重复运行 %s' % message) logging.error('监听服务失败,请检查端口冲突,或者查看是否重复运行 %s' % message) sys.exit(0)
def __parse(self, requestData): # 将header解析成dic存起来 def saveResquestHeader(headerList): for item in headerList: # 过滤空行 if item.strip() == "": continue # 获取key的索引值 segindex = item.find(":") if segindex < 0: continue key = item[0:segindex].strip() value = item[segindex + 1:].strip() self.requestHeaders[key] = value # 获取控制器和动作 def getControllerAndAction(splitRequestUri): # 删除所有空元素 while '' in splitRequestUri: splitRequestUri.remove('') splitRequestUriLen = len(splitRequestUri) # 有controller和action或者GET param的情况 if splitRequestUriLen >= 2: # 获取controller和action,并删除,剩下的就是GET参数 self.controller = splitRequestUri.pop(0) # logging.debug('splitRequestUriLen %s ' % str(splitRequestUri)) self.action = splitRequestUri.pop(0) # logging.debug('splitRequestUriLen %s ' % str(splitRequestUri)) i = 1 for param in splitRequestUri: if param.strip() is not '': self.getParamsDic[i] = param i = i + 1 elif splitRequestUriLen == 1: self.controller = splitRequestUri[0] self.action = 'index' else: self.controller = 'index' self.action = 'index' ''' GET /f**k HTTP/1.1 Host: 172.16.190.132:8888 User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0 Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 Accept-Language: zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3 Accept-Encoding: gzip, deflate Connection: keep-alive Cache-Control: max-age=0 ''' try: # 获取 header 和 body header, requestBody = requestData.split("\r\n\r\n", 1) headerList = header.split("\r\n") # 将header解析成dic存起来 saveResquestHeader(headerList) # logging.debug('**********原始头部********',self.requestHeaders) except ValueError as e: self.fastResponse(500) logging.error('@@@@@@@@@@@@@@', requestData) return # 获取request 请求信息 headerFirstLine = headerList.pop(0) # _logger.debug('第一行 %s'%headerFirstLine) # 按照空格切割 ‘GET /xxx HTTP/1.1’ self.requestMethod, self.requestUri, self.httpVersion = re.split( '\s+', headerFirstLine) # 将请求方法类型转为小写 methodToLower = self.requestMethod.lower() # 根据?切割,有get参数的情况下 /controller/action?xxx=xxx splitRequestUri = self.requestUri.split('?') # 如果大于1,说明?后面有get参数 if len(splitRequestUri) > 1: self.baseUri, self.paramStr = splitRequestUri else: self.paramStr = None # /controller/action self.baseUri = splitRequestUri[0] self.requestParamStr = None splitRequestUri = self.baseUri.split('/') # 获取 请求资源的文件名和 后缀(如果有的话) self.__fileName = splitRequestUri[-1] # self.__filePrefixName, self.__fileExtName = os.path.splitext(splitRequestUri[-1]) # 截取controller和action getControllerAndAction(splitRequestUri) # 判断请求类型,保存参数 if methodToLower == "get" and self.paramStr: if "?" in self.requestUri: # 获取?后面的的参数和值,如果有/xxx/xx/xx/xxx/xxx类型的参数,将?后面的参数合并到self.getParamsDic if self.getParamsDic: self.getParamsDic = dict(parse_qs(self.paramStr), **self.getParamsDic) else: self.getParamsDic = parse_qs(self.paramStr) elif methodToLower == "post": # Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryZhSJu2it4yyuMdlM if self.requestHeaders.get('Content-Type', "").find("boundary") > 0: # logging.debug('++++++++++f**k 11111 you+++++++++++++++') self.__parseMultipartPost(requestBody) else: self.postParamsDic = parse_qs(requestBody)
def __handleStaticFile(self): # 判断静态文件是否存在 def isStaticFileExist(): if not os.path.exists(staticFileFullPath): self.fastResponse(404) # 判断缓存文件是否存在 def isCacheDirExist(): dir, filename = os.path.split(cacheFileFullPath) if not os.path.exists(dir): os.makedirs(dir) # 过去时间和文件修改时间做对比,相等就是文件没有修改过,返回304 def isFileCacheExpiry(): if lastTimeStr == fileModifyTimeStr and "Range" not in self.requestHeaders: self.responseStatus = 304 self.responseHeaders['Last-Modified'] = fileModifyTimeStr self.responseHeaders['ETag'] = fileModifyTime self.responseHeaders['Date'] = currentTimeStr # 初始化部分Header def initHeader(): self.responseHeaders['Last-Modified'] = fileModifyTimeStr self.responseHeaders['ETag'] = fileModifyTime self.responseHeaders['Date'] = currentTimeStr # 设置Content-Type,如果是列表中不存在的类型,按照未知获取 contentType = mimetypes.guess_type(self.__fileName)[0] if contentType: self.responseHeaders['Content-Type'] = contentType else: self.responseHeaders['Content-Type'] = 'application/octet-stream' # 根据配置设置缓存时间 if CACHE_TIME == 0: self.responseHeaders['Cache-Control'] = 'no-cache' else: self.responseHeaders['Cache-Control'] = 'max-age=%d' % CACHE_TIME # 判断是否需要删除空格和换行符 def isNeedTrim(): # 如果配置了TRIM,自动去除html,css,js中的注释以及重复的空白符(\n,\r,\t,' '),否则直接读取静态文件 if TRIM and self.responseHeaders.get('Content-Type') in TRIM_TYPES: if not os.path.exists(self.__trimFileFullPath) or os.path.getmtime( self.__trimFileFullPath) < fileModifyTime: trimFileData = self.trim(staticFileFullPath) trimFileFd = open(self.__trimFileFullPath, 'w') trimFileFd.write(trimFileData) trimFileFd.close() return True else: return False # 判断是否需要gzip压缩 def isNeedGzip(): # 判断配置中gzip是否已开启,且请求文件在允许gzip压缩的列表中 if self.responseHeaders.get('Content-Type') in GZIP_TYPES and GZIP: # 如果gzip缓存文件不存在或者已经过期,重新生成 if not os.path.exists(self.__gzipFileFullPath) or os.path.getmtime( self.__gzipFileFullPath) < fileModifyTime: if TRIM: file = self.__trimFileFullPath else: file = staticFileFullPath staticFileFd = open(file) staticFileData = staticFileFd.read() staticFileFd.close() # gzip格式创建缓存文件,读取静态资源文件写入缓存 gZipFileFd = gzip.open(self.__gzipFileFullPath, 'wb') gZipFileFd.write(staticFileData) gZipFileFd.close() # 添加一个response头显示资源已经gzip压缩 self.responseHeaders['Content-Encoding'] = 'gzip' return True else: return False # 处理分段传输,断点续传 def rangeResponse(): rangeValue = self.requestHeaders["Range"].strip(' \r\n') rangeValue = rangeValue.replace("bytes=", "") # 获取请求数据的范围 rangeStart, rangeEnd = rangeValue.split('-') # 如果范围rangeEnd为空,则rangeEnd修改为文件大小(因为初始是0,所以大小-1) if rangeEnd == '': rangeEnd = fileSize - 1 rangeStart = int(rangeStart) rangeEnd = int(rangeEnd) self.responseHeaders['Accept-Ranges'] = 'bytes' self.responseStatus = 206 self.responseHeaders['Content-Range'] = 'bytes %s-%s/%s' % (rangeStart, rangeEnd, fileSize) # 偏移量 offset = rangeStart # 请求的分段数据总长度 partLen = rangeEnd - rangeStart + 1 # logging.debug('长度 %d' % partLen) if partLen < 0: partLen = 0 self.responseHeaders['Content-Length'] = partLen with open(staticFileFullPath) as staticFile: # 设置文件的当前位置偏移,从偏移量开始读取数据 staticFile.seek(offset) readlen = 10240 if readlen > partLen: readlen = partLen firstdata = staticFile.read(readlen) self.responseText += firstdata partLen -= len(firstdata) def generateResponseText(): staticFileData = staticFile.read() bufferSize = fileSize / 4 beginOffset = 0 endOffset = bufferSize if chunked: while 1: offsetData = staticFileData[beginOffset:endOffset] # 取data长度的16进制 offsetDataLen = hex(bufferSize)[2:] if self.responseText: self.responseText = '%s%s\r\n%s\n' % (self.responseText, offsetDataLen, offsetData) else: self.responseText = '%s\n%s\r\n' % (offsetDataLen, offsetData) beginOffset = endOffset endOffset = beginOffset + bufferSize if endOffset > fileSize: endOffset = fileSize bufferSize = endOffset - beginOffset elif endOffset == fileSize: break else: self.responseText = staticFileData ''' 逻辑由此开始 ''' try: staticFile = None chunked = None # 获取静态文件路径 staticFileFullPath = self.baseUri[self.baseUri.find(_staticDirPath) + 0:] # 获取缓存文件路径 cacheFileFullPath = _memoryCacheDirPath + staticFileFullPath[7:] isCacheDirExist() self.__trimFileFullPath = cacheFileFullPath + '-trim' self.__gzipFileFullPath = cacheFileFullPath + '-gzip' # 判断文件是否存在,不存在报404 isStaticFileExist() # 过期时间 lastTimeStr = self.requestHeaders.get("If-Modified-Since", None) # 获取文件修改时间 fileModifyTime = os.path.getmtime(staticFileFullPath) # 获取格林威治(GMT)格式的文件修改时间 fileModifyTimeStr = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(fileModifyTime)) # 获取格林威治(GMT)格式的当前时间 currentTimeStr = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(time.time())) # 检查缓存是否到期 isFileCacheExpiry() # # 后缀去掉.符号 # extName = self.__fileExtName.replace('.', '') # 初始化部分Header initHeader() # 判断是否缓存trim文件 isTrim = isNeedTrim() # 判断是否缓存gzip文件 isGzip = isNeedGzip() # logging.debug('Header %s'%str(self.responseHeaders)) if isGzip: staticFileFullPath = self.__gzipFileFullPath elif isTrim: staticFileFullPath = self.__trimFileFullPath # 获取文件大小 fileSize = os.path.getsize(staticFileFullPath) ''' 随着SNDBUF增大,send()返回已发送字节越大。接收窗口大小对结果影响不是线性的。实际已接收的只有窗口大小。 在no blocking的情况下,发送的数据大于缓存区会导致客户端只能接受到部分数据 如果文件大小大于1M,设置当前socket的发送缓冲区为fileSize的一半 ''' if fileSize > CHUNKED_MIN_LENGTH and CHUNKED_MIN_LENGTH > 0: self.responseHeaders['Transfer-Encoding'] = 'chunked' chunked = True self.connParam['connections'].setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, fileSize * 2) self.connParam['connections'].setsockopt(socket.SOL_SOCKET, socket.TCP_NODELAY, 1) # 打开文件 with open(staticFileFullPath) as staticFile: # Range 只请求实体的一部分,指定范围 ---- 即断点续传 if "Range" in self.requestHeaders: rangeResponse() else: generateResponseText() except OSError as e: self.fastResponse(404) logging.error(e) return except Exception as e: self.fastResponse(500) logging.error(e) return else: self.__startResponse()
def __parse(self, requestData): # 将header解析成dic存起来 def saveResquestHeader(headerList): for item in headerList: # 过滤空行 if item.strip() == "": continue # 获取key的索引值 segindex = item.find(":") if segindex < 0: continue key = item[0:segindex].strip() value = item[segindex + 1:].strip() self.requestHeaders[key] = value # 获取控制器和动作 def getControllerAndAction(splitRequestUri): # 删除所有空元素 while '' in splitRequestUri: splitRequestUri.remove('') splitRequestUriLen = len(splitRequestUri) # 有controller和action或者GET param的情况 if splitRequestUriLen >= 2: # 获取controller和action,并删除,剩下的就是GET参数 self.controller = splitRequestUri.pop(0) # logging.debug('splitRequestUriLen %s ' % str(splitRequestUri)) self.action = splitRequestUri.pop(0) # logging.debug('splitRequestUriLen %s ' % str(splitRequestUri)) i = 1 for param in splitRequestUri: if param.strip() is not '': self.getParamsDic[i] = param i = i + 1 elif splitRequestUriLen == 1: self.controller = splitRequestUri[0] self.action = 'index' else: self.controller = 'index' self.action = 'index' ''' GET /f**k HTTP/1.1 Host: 172.16.190.132:8888 User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0 Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 Accept-Language: zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3 Accept-Encoding: gzip, deflate Connection: keep-alive Cache-Control: max-age=0 ''' try: # 获取 header 和 body header, requestBody = requestData.split("\r\n\r\n", 1) headerList = header.split("\r\n") # 将header解析成dic存起来 saveResquestHeader(headerList) # logging.debug('**********原始头部********',self.requestHeaders) except ValueError as e: self.fastResponse(500) logging.error('@@@@@@@@@@@@@@', requestData) return # 获取request 请求信息 headerFirstLine = headerList.pop(0) # _logger.debug('第一行 %s'%headerFirstLine) # 按照空格切割 ‘GET /xxx HTTP/1.1’ self.requestMethod, self.requestUri, self.httpVersion = re.split('\s+', headerFirstLine) # 将请求方法类型转为小写 methodToLower = self.requestMethod.lower() # 根据?切割,有get参数的情况下 /controller/action?xxx=xxx splitRequestUri = self.requestUri.split('?') # 如果大于1,说明?后面有get参数 if len(splitRequestUri) > 1: self.baseUri, self.paramStr = splitRequestUri else: self.paramStr = None # /controller/action self.baseUri = splitRequestUri[0] self.requestParamStr = None splitRequestUri = self.baseUri.split('/') # 获取 请求资源的文件名和 后缀(如果有的话) self.__fileName = splitRequestUri[-1] # self.__filePrefixName, self.__fileExtName = os.path.splitext(splitRequestUri[-1]) # 截取controller和action getControllerAndAction(splitRequestUri) # 判断请求类型,保存参数 if methodToLower == "get" and self.paramStr: if "?" in self.requestUri: # 获取?后面的的参数和值,如果有/xxx/xx/xx/xxx/xxx类型的参数,将?后面的参数合并到self.getParamsDic if self.getParamsDic: self.getParamsDic = dict(parse_qs(self.paramStr), **self.getParamsDic) else: self.getParamsDic = parse_qs(self.paramStr) elif methodToLower == "post": # Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryZhSJu2it4yyuMdlM if self.requestHeaders.get('Content-Type', "").find("boundary") > 0: # logging.debug('++++++++++f**k 11111 you+++++++++++++++') self.__parseMultipartPost(requestBody) else: self.postParamsDic = parse_qs(requestBody)
def __handleStaticFile(self): # 判断静态文件是否存在 def isStaticFileExist(): if not os.path.exists(staticFileFullPath): self.fastResponse(404) # 判断缓存文件是否存在 def isCacheDirExist(): dir, filename = os.path.split(cacheFileFullPath) if not os.path.exists(dir): os.makedirs(dir) # 过去时间和文件修改时间做对比,相等就是文件没有修改过,返回304 def isFileCacheExpiry(): if lastTimeStr == fileModifyTimeStr and "Range" not in self.requestHeaders: self.responseStatus = 304 self.responseHeaders['Last-Modified'] = fileModifyTimeStr self.responseHeaders['ETag'] = fileModifyTime self.responseHeaders['Date'] = currentTimeStr # 初始化部分Header def initHeader(): self.responseHeaders['Last-Modified'] = fileModifyTimeStr self.responseHeaders['ETag'] = fileModifyTime self.responseHeaders['Date'] = currentTimeStr # 设置Content-Type,如果是列表中不存在的类型,按照未知获取 contentType = mimetypes.guess_type(self.__fileName)[0] if contentType: self.responseHeaders['Content-Type'] = contentType else: self.responseHeaders[ 'Content-Type'] = 'application/octet-stream' # 根据配置设置缓存时间 if CACHE_TIME == 0: self.responseHeaders['Cache-Control'] = 'no-cache' else: self.responseHeaders[ 'Cache-Control'] = 'max-age=%d' % CACHE_TIME # 判断是否需要删除空格和换行符 def isNeedTrim(): # 如果配置了TRIM,自动去除html,css,js中的注释以及重复的空白符(\n,\r,\t,' '),否则直接读取静态文件 if TRIM and self.responseHeaders.get('Content-Type') in TRIM_TYPES: if not os.path.exists( self.__trimFileFullPath) or os.path.getmtime( self.__trimFileFullPath) < fileModifyTime: trimFileData = self.trim(staticFileFullPath) trimFileFd = open(self.__trimFileFullPath, 'w') trimFileFd.write(trimFileData) trimFileFd.close() return True else: return False # 判断是否需要gzip压缩 def isNeedGzip(): # 判断配置中gzip是否已开启,且请求文件在允许gzip压缩的列表中 if self.responseHeaders.get('Content-Type') in GZIP_TYPES and GZIP: # 如果gzip缓存文件不存在或者已经过期,重新生成 if not os.path.exists( self.__gzipFileFullPath) or os.path.getmtime( self.__gzipFileFullPath) < fileModifyTime: if TRIM: file = self.__trimFileFullPath else: file = staticFileFullPath staticFileFd = open(file) staticFileData = staticFileFd.read() staticFileFd.close() # gzip格式创建缓存文件,读取静态资源文件写入缓存 gZipFileFd = gzip.open(self.__gzipFileFullPath, 'wb') gZipFileFd.write(staticFileData) gZipFileFd.close() # 添加一个response头显示资源已经gzip压缩 self.responseHeaders['Content-Encoding'] = 'gzip' return True else: return False # 处理分段传输,断点续传 def rangeResponse(): rangeValue = self.requestHeaders["Range"].strip(' \r\n') rangeValue = rangeValue.replace("bytes=", "") # 获取请求数据的范围 rangeStart, rangeEnd = rangeValue.split('-') # 如果范围rangeEnd为空,则rangeEnd修改为文件大小(因为初始是0,所以大小-1) if rangeEnd == '': rangeEnd = fileSize - 1 rangeStart = int(rangeStart) rangeEnd = int(rangeEnd) self.responseHeaders['Accept-Ranges'] = 'bytes' self.responseStatus = 206 self.responseHeaders['Content-Range'] = 'bytes %s-%s/%s' % ( rangeStart, rangeEnd, fileSize) # 偏移量 offset = rangeStart # 请求的分段数据总长度 partLen = rangeEnd - rangeStart + 1 # logging.debug('长度 %d' % partLen) if partLen < 0: partLen = 0 self.responseHeaders['Content-Length'] = partLen with open(staticFileFullPath) as staticFile: # 设置文件的当前位置偏移,从偏移量开始读取数据 staticFile.seek(offset) readlen = 10240 if readlen > partLen: readlen = partLen firstdata = staticFile.read(readlen) self.responseText += firstdata partLen -= len(firstdata) def generateResponseText(): staticFileData = staticFile.read() bufferSize = fileSize / 4 beginOffset = 0 endOffset = bufferSize if chunked: while 1: offsetData = staticFileData[beginOffset:endOffset] # 取data长度的16进制 offsetDataLen = hex(bufferSize)[2:] if self.responseText: self.responseText = '%s%s\r\n%s\n' % ( self.responseText, offsetDataLen, offsetData) else: self.responseText = '%s\n%s\r\n' % (offsetDataLen, offsetData) beginOffset = endOffset endOffset = beginOffset + bufferSize if endOffset > fileSize: endOffset = fileSize bufferSize = endOffset - beginOffset elif endOffset == fileSize: break else: self.responseText = staticFileData ''' 逻辑由此开始 ''' try: staticFile = None chunked = None # 获取静态文件路径 staticFileFullPath = self.baseUri[self.baseUri.find(_staticDirPath ) + 0:] # 获取缓存文件路径 cacheFileFullPath = _memoryCacheDirPath + staticFileFullPath[7:] isCacheDirExist() self.__trimFileFullPath = cacheFileFullPath + '-trim' self.__gzipFileFullPath = cacheFileFullPath + '-gzip' # 判断文件是否存在,不存在报404 isStaticFileExist() # 过期时间 lastTimeStr = self.requestHeaders.get("If-Modified-Since", None) # 获取文件修改时间 fileModifyTime = os.path.getmtime(staticFileFullPath) # 获取格林威治(GMT)格式的文件修改时间 fileModifyTimeStr = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(fileModifyTime)) # 获取格林威治(GMT)格式的当前时间 currentTimeStr = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(time.time())) # 检查缓存是否到期 isFileCacheExpiry() # # 后缀去掉.符号 # extName = self.__fileExtName.replace('.', '') # 初始化部分Header initHeader() # 判断是否缓存trim文件 isTrim = isNeedTrim() # 判断是否缓存gzip文件 isGzip = isNeedGzip() # logging.debug('Header %s'%str(self.responseHeaders)) if isGzip: staticFileFullPath = self.__gzipFileFullPath elif isTrim: staticFileFullPath = self.__trimFileFullPath # 获取文件大小 fileSize = os.path.getsize(staticFileFullPath) ''' 随着SNDBUF增大,send()返回已发送字节越大。接收窗口大小对结果影响不是线性的。实际已接收的只有窗口大小。 在no blocking的情况下,发送的数据大于缓存区会导致客户端只能接受到部分数据 如果文件大小大于1M,设置当前socket的发送缓冲区为fileSize的一半 ''' if fileSize > CHUNKED_MIN_LENGTH and CHUNKED_MIN_LENGTH > 0: self.responseHeaders['Transfer-Encoding'] = 'chunked' chunked = True self.connParam['connections'].setsockopt( socket.SOL_SOCKET, socket.SO_SNDBUF, fileSize * 2) self.connParam['connections'].setsockopt( socket.SOL_SOCKET, socket.TCP_NODELAY, 1) # 打开文件 with open(staticFileFullPath) as staticFile: # Range 只请求实体的一部分,指定范围 ---- 即断点续传 if "Range" in self.requestHeaders: rangeResponse() else: generateResponseText() except OSError as e: self.fastResponse(404) logging.error(e) return except Exception as e: self.fastResponse(500) logging.error(e) return else: self.__startResponse()