def thread_work():
head = nshead()
pack = mcpack.dumps({"cmd":"test"})
for i in range(COUNT):
head.log_id = 1234560000 + i
head.body_len = len(pack)
resp = cli.invite(head, pack)
if not resp:
log.warning("ub_client invite error")
time.sleep(1)
(h, body) = resp
print mcpack.loads(body)
def recv(self):
"""
description: recv cmd
"""
try:
res_nshead = nshead.NsHead.from_str(self.sock.recv(36))
recved = 0
body = ''
res_body = ''
start_time = time.time()
while recved < res_nshead.body_len:
temp = self.sock.recv(1024)
recved += len(temp)
body = body + temp
# Receive timeout, so break the loop
if time.time()-start_time > self.timeout:
raise Exception('Recv timeout')
if body != '':
res_body = mcpack.loads(body)
return (res_nshead, res_body)
except:
print traceback.format_exc()
sys.exit(2)
def recv_client(cls, clisock):
# read nshead
header=cls.recv(clisock, 36)
if not header:
#logging.info("recv header timeout")
clisock.close()
return None
nshead = PyNshead.unpack(header)
body_len = nshead._body_len
# read body
if body_len <=0:
#logging.info("recv empty body")
clisock.close()
return {}
dat=cls.recv(clisock, body_len)
if not dat:
#logging.info("recv data timeout,body_len %d" % body_len )
clisock.close()
return None
#pack to json
bodydict=mcpack.loads(dat)
#logging.info("recv_body:%s", bodydict)
return bodydict
def handle(self, conn):
sock, addr = conn
timer = ub_timer()
# recv
timer.settask("rev")
sock.settimeout(self.server.rdtmout)
head = self.__recv(sock, self.nshead.size)
if len(head) == 0:
raise socket.error("connection reset by peer")
self.nshead.frombin(head)
log.setlogid(self.nshead.log_id)
log.setreqip("%s", addr[0])
#print 'ready to receive body'
body = self.__recv(sock, self.nshead.body_len)
if len(body) != self.nshead.body_len:
raise socket.error("read body failed")
req = mcpack.loads(body)
# callback
timer.settask("proc")
if self.nshead.provider != "__MONITOR__":
#print 'ready to callback'
#print 'req: %s' % str(req)
resp = self.server.callback(req)
#print 'response: %s' % str(resp)
rpack = mcpack.dumps(resp)
rhead = self.nshead
rhead.provider = self.server.name
rhead.body_len = len(rpack)
rbuf = rhead.tobin() + rpack
else:
#print 'enter in !!!!'
log.trace("receive a monitor request with nshead provider '__MONITOR__'.");
rhead = self.nshead
rhead.provider = self.server.name
rhead.body_len = 0
rbuf = rhead.tobin()
# send
timer.settask("write")
sock.settimeout(self.server.wrtmout)
sock.sendall(rbuf)
timer.endtask()
log.setproctime("total:%u(ms) rev:%u+proc:%u+write:%u",
timer.gettotal(), timer.gettask("rev"),
timer.gettask("proc"), timer.gettask("write"))
log.notice("")
def handle(req_type, text):
#ret infomation
ret = 0
resp_code = 0
resp_content = "xxx"
#video_class server
ip = "127.0.0.1"
port = 9875
server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
#1.build request pack
##@content
body = {
'request_type': mcpack.INT32(req_type),
'text': mcpack.STR(text)
}
mcpack_body = mcpack.dumps(body)
##@head
head = nshead.NsHead()
head.body_len = len(mcpack_body)
#2. connect to server
#server_sock.create_connection((ip, port))
server_sock.connect((ip, port))
server_sock.settimeout(60)
#3. send request
request = head.pack() + mcpack_body
server_sock.sendall(request)
#4. recv response
res_nshead = nshead.NsHead.from_str(server_sock.recv(36))
if res_nshead is False:
return -3, -1, "text is invalid"
recved = 0
body = ''
while recved < res_nshead.body_len:
temp = server_sock.recv(1024)
recved += len(temp)
body = body + temp
res_body = mcpack.loads(body)
#parse response
resp_code = res_body['response_code']
resp_content = res_body['tags']
except Exception as ex:
ret = -1
resp_code = -1
resp_content = "maybe error in param!"
logging.error(traceback.format_exc())
finally:
#5. disconnect with server
server_sock.close()
return ret, resp_code, resp_content
def handle(self):
#1. 获取请求信息
self.clientIp, self.clientPort = self.client_address
thd = threading.currentThread()
self.thread = thd.getName()
self.reqHead = {}
self.reqBody = {}
self.resHead = PyNshead(id=0, version=0, log_id=0, provider=UBServer.SERVER_NAME, reserved=0, body_len=0)
self.resBody = {}
log = "REQUEST [IP:"+self.clientIp+"] "
log += "[thread:"+self.thread+"] "
timeout = UBServer.READ_TIMEOUT + UBServer.WRITE_TIMEOUT
self.request.settimeout(timeout)
try:
#2. 读取请求包head
header = self.recv_pack(self.request, 36)
if not header:
raise Exception("read nshead fail")
nshead = PyNshead.unpack(header)
self.reqHead = nshead.todict()
self.reqHead["provider"] = self.reqHead["provider"].replace('\x00', '')
log += "[req_head:"+str(nshead)+"] "
#3. 读取请求包body
body_len = self.reqHead["body_len"]
if body_len <=0:
raise Exception("read body fail")
dat = self.recv_pack(self.request, body_len)
if not dat:
raise Exception("read body fail")
self.reqBody = mcpack.loads(dat)
log += "[req_body:"+str(self.reqBody)+"] "
#4. 回调函数处理请求
self.callback()
#5. 发送返回包
self.send_pack(self.request)
log += "[res_head:"+str(self.resHead)+"] "
log += "[res_body:"+str(self.resBody)+"] "
#9. 关闭连接
self.request.close()
Env.getlog().notice(log)
except Exception as e:
self.request.close()
log += "[error:"+str(e)+"]"
Env.getlog().warning(log)
def recv_all(self):
head = self.sock.recv(36, socket.MSG_WAITALL)
#parse head
(id, verison, log_id, provider, magic_num, reserved,
body_len) = struct.unpack("2HI16s3I", head)
#recv body
dat = self.sock.recv(body_len)
rev_len = len(dat)
while rev_len < body_len:
dat += self.sock.recv(body_len - rev_len)
rev_len = len(dat)
mcpack.set_default_version(mcpack.mcpackv2)
bodydict = mcpack.loads(dat)
return bodydict
def recv_all(cls,clisock,reply=None):
#header
header=cls.recv(clisock, 36)
if not header:
#logging.info("recv header timeout")
clisock.close()
return None
nshead=PyNshead.unpack(header)
body_len=nshead._body_len
#body
if body_len <=0:
#logging.info("recv empty body")
clisock.close()
return {}
dat=cls.recv(clisock,body_len)
if not dat:
#logging.info("recv data timeout,body_len %d" % body_len )
clisock.close()
return None
#send reply
if reply==None:
clisock.close()
else:
pack=mcpack.dumps_version(mcpack.mcpackv2,reply)
nshead._body_len=len(pack)
header=nshead.pack()
clisock.send(header)
clisock.send(pack)
clisock.close()
#pack to json
bodydict=mcpack.loads(dat)
return bodydict
def getparser(title):#,content,aid,pubtime):
pok=0
termstr=""
tryTimes=0;
__terms=""
while(tryTimes < 3):
tryTimes=tryTimes+1
#try:
if(1):
con=None
if(con==None):
dbtype = "socket"
dbfile = ""
dbwto = 1000
dbrto = 1000
logger = Logger.initlog("./con_sock_log")
con = UbClient.Connector(dbtype, dbhost, dbport, dbfile ,dbwto, dbrto, logger)
con.sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
reqdata = {}
reqdata["title"]=title
reqdata["content"]=title
reqdata["aid"]=0
reqdata["pubtime"]=0
req_pack = mcpack.dumps_version(mcpack.mcpackv1, reqdata, 4*1024*1024)
con.write_pack(req_pack)
(ret ,res_pack) = con.read_pack()
pok=1
if ret == 0:
resdict = mcpack.loads(res_pack)
__terms=resdict["terms"].replace("'"," ")
__terms = __terms.decode("gbk").encode('utf-8','ignore')
__termCount=resdict["termCount"]
#print title
#print __terms
# if(__terms!=""):
# #print '-sock-ok',aid,__dupaids
# dup_old_id=0
# arr = __terms.split(';')
# termcount=len(arr)
#
# reali=0;
# _iterator_lock.acquire()
# _iterator_count+=1
# reali=_iterator_count
# lock.release()
#
# if(reali % 10001 ==0):
# print reali
#
#
# if((termcount-1) !=int(__termCount)):
# print "count not match",aid,termcount,__termCount
#
# termstr=__terms
break;
#except:
if(0):
print 'except-sock-',sys.exc_info()[0]
con.sock.close()
sleep(1)#con.sock.close()
con=None
if(con!=None):
con.sock.close()
con=None
return __terms,__termCount
def decodeBody(self, data):
self.res["body"] = mcpack.loads(data)
ssee = str(self.res["body"])
if len(ssee) < 500:
print "Response body: " + str(self.res["body"])
def deal_mcgi(listen, response_type):
if response_type == 'normal':
req_body = "Hello, Nginx, just do it"
req_nshead = nshead.NsHead()
header = [
mcpack.STR('Cache-Control : max-age=60'),
mcpack.STR('Connection : close'),
mcpack.STR('Content-Type : text/html; charset=UTF-8'),
mcpack.STR('Server : nginx/1.4.4'),
mcpack.STR('Status : 200'),
mcpack.STR('Version : HTTP/1.1'),
mcpack.STR('X-Powered-By : HPHP'),
]
ppack = {
'Status': mcpack.STR('200'),
'Headers': mcpack.ARR(header),
'Content-Length': mcpack.STR(len(req_body)),
}
res_mcpack = mcpack.dumps(ppack)
req_nshead.body_len = len(res_mcpack) + len(req_body)
response = req_nshead.pack() + res_mcpack + req_body
elif response_type == '4xx':
req_nshead = nshead.NsHead()
header = [
mcpack.STR('Cache-Control : max-age=60'),
mcpack.STR('Connection : close'),
mcpack.STR('Content-Type : text/html; charset=UTF-8'),
mcpack.STR('Server : nginx/1.4.4'),
mcpack.STR('Status : 404'),
mcpack.STR('Version : HTTP/1.1'),
mcpack.STR('X-Powered-By : HPHP'),
]
ppack = {
'Status': mcpack.STR('404'),
'Headers': mcpack.ARR(header),
'Content-Length': mcpack.STR('0'),
}
res_mcpack = mcpack.dumps(ppack)
req_nshead.body_len = len(res_mcpack)
response = req_nshead.pack() + res_mcpack
elif response_type == '5xx':
req_nshead = nshead.NsHead()
header = [
mcpack.STR('Cache-Control : max-age=60'),
mcpack.STR('Connection : close'),
mcpack.STR('Content-Type : text/html; charset=UTF-8'),
mcpack.STR('Server : nginx/1.4.4'),
mcpack.STR('Status : 503'),
mcpack.STR('Version : HTTP/1.1'),
mcpack.STR('X-Powered-By : HPHP'),
]
ppack = {
'Status': mcpack.STR('503'),
'Headers': mcpack.ARR(header),
'Content-Length': mcpack.STR('0'),
}
res_mcpack = mcpack.dumps(ppack)
req_nshead.body_len = len(res_mcpack)
response = req_nshead.pack() + res_mcpack
elif response_type == 'retry':
req_nshead = nshead.NsHead()
header = [
mcpack.STR('Cache-Control : max-age=60'),
mcpack.STR('Connection : close'),
mcpack.STR('Content-Type : text/html; charset=UTF-8'),
mcpack.STR('Server : nginx/1.4.4'),
mcpack.STR('Status : 503'),
mcpack.STR('X-MCGI-ONERROR : RETRY'),
mcpack.STR('Version : HTTP/1.1'),
mcpack.STR('X-Powered-By : HPHP'),
]
ppack = {
'Status': mcpack.STR('503'),
'Headers': mcpack.ARR(header),
'Content-Length': mcpack.STR('0'),
}
res_mcpack = mcpack.dumps(ppack)
req_nshead.body_len = len(res_mcpack)
response = req_nshead.pack() + res_mcpack
elif response_type == '302':
req_nshead = nshead.NsHead()
header = [
mcpack.STR('Cache-Control : max-age=60'),
mcpack.STR('Connection : close'),
mcpack.STR('Content-Type : text/html; charset=UTF-8'),
mcpack.STR('Server : nginx/1.4.4'),
mcpack.STR('Status : 302'),
mcpack.STR('Location : http://www.baidu.com'),
mcpack.STR('Version : HTTP/1.1'),
mcpack.STR('X-Powered-By : HPHP'),
]
ppack = {
'Status': mcpack.STR('302'),
'Headers': mcpack.ARR(header),
'Content-Length': mcpack.STR('0'),
}
res_mcpack = mcpack.dumps(ppack)
req_nshead.body_len = len(res_mcpack)
response = req_nshead.pack() + res_mcpack
else:
print 'response type not support'
usage()
serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listen = int(listen)
serverSocket.bind(('0.0.0.0', listen))
serverSocket.listen(200)
serverSocket.setblocking(0)
epoll = select.epoll()
epoll.register(serverSocket.fileno(), select.EPOLLIN)
try:
connections = {}
responses = {}
arrivedTimes = {}
while True:
events = epoll.poll(1)
for fileno, event in events:
if fileno == serverSocket.fileno():
try:
connection, address = serverSocket.accept()
connection.setblocking(0)
epoll.register(connection.fileno(), select.EPOLLIN)
connections[connection.fileno()] = connection
responses[connection.fileno()] = response
except:
print "epoll accept error"
elif event & select.EPOLLIN:
try:
res_nshead = nshead.NsHead.from_str(
connection.recv(36))
received = 0
body = ''
while received < res_nshead.body_len:
tmp = connection.recv(1024)
received += len(tmp)
body = body + tmp
res_mcpack = mcpack.loads(body)
ext_data = ''
if 'HTTP_CONTENT_LENGTH' in res_mcpack and res_mcpack[
'HTTP_CONTENT_LENGTH'] != 0:
ext_data = body[res_nshead.body_len -
res_mcpack['data_len']:]
print "nshead:"
print res_nshead
print ""
print "mcpack:"
for (k, v) in res_mcpack.items():
print " " + k + " : " + v
if ext_data != '':
print "ext_data : " + ext_data
arrivedtime = int(time.time() * 1000)
arrivedTimes[fileno] = arrivedtime
epoll.modify(fileno, select.EPOLLOUT)
except:
print traceback.format_exc()
print "epoll read error"
elif event & select.EPOLLOUT:
try:
currenttime = int(time.time() * 1000)
if (currenttime - arrivedTimes[fileno] > 50):
byteswritten = connections[fileno].send(
responses[fileno])
responses[fileno] = responses[fileno][
byteswritten:]
if len(responses[fileno]) == 0:
epoll.modify(fileno, 0)
connections[fileno].shutdown(socket.SHUT_WR)
except:
print traceback.format_exc()
print "epoll write error"
elif event & select.EPOLLHUP:
try:
epoll.unregister(fileno)
connections[fileno].close()
del connections[fileno]
except:
print traceback.format_exc()
print "epoll close error"
finally:
print 'error'
epoll.unregister(serverSocket.fileno())
epoll.close()
serverSocket.close()
def decodeBody(self, data):
self.res["body"] = mcpack.loads(data)
print "Response body: " + str(self.res["body"])