def db_reader(name, status, req_queue, config_file):
LogUtil.get_instance(config_file, "db_reader")
LogUtil.info("db_reader:"+name+" begin")
config=configparser.ConfigParser()
config.read(config_file)
factory=msg.MessageProcessorFactory()
db_file=config.get("message_config", "db_file")
factory.load_from_db(db_file, [])
read_interval=config.getfloat("reqresp","read_interval")
if read_interval==None:
read_interval=0.5
host=config.get("reqresp", "host")
database=config.get("reqresp", "database")
user=config.get("reqresp","user")
password=config.get("reqresp", "password")
last_req_num=0
conn=pgdb.connect(database=database, host=host, user=user, password=password)
query_curs=conn.cursor()
update_curs=conn.cursor()
query_unreported="""SELECT req_num,message_type,appid,oms_order_id,rept_status,req_text
FROM req_resp
WHERE rept_status='0'
AND req_num>%(req_num)s
ORDER BY req_num
"""
query_dict={'req_num':0}
update_reported="""UPDATE req_resp
SET rept_status=%(rept_status)s,report_time=localtimestamp
WHERE req_num in (%(req_num)s)
"""
update_dict={'rept_status':'0', 'req_num':0}
last_read_cnt=1
while status.value==0:
if last_read_cnt==0:
time.sleep(read_interval)
last_read_cnt=0
query_dict['req_num']=last_req_num
query_curs.execute(query_unreported,query_dict)
for (req_num,message_type,appid,message_id,rept_status,req_text) in query_curs.fetchall():
message_processor=factory.build_message_processor(message_type)
send_buff=message_processor.pack(req_text)
req_queue.put(send_buff)
last_req_num=req_num
update_dict['rept_status']='2'
update_dict['req_num']=last_req_num
update_curs.execute(update_reported,update_dict)
last_read_cnt=last_read_cnt+1
LogUtil.debug("db_reader putQ:"+binascii.hexlify(send_buff).decode())
conn.commit()
LogUtil.info("db_reader:"+name+" end")
def md_requestor(name, status, req_queue, config_file):
LogUtil.get_instance(config_file, "db_reader")
LogUtil.info("db_reader:"+name+" begin")
config = configparser.ConfigParser()
config.read(config_file)
factory = msg.MessageProcessorFactory()
db_file = config.get("message_config", "db_file")
factory.load_from_db(db_file, [])
trade_db_file = config.get("reqresp", "db_file")
read_interval = config.getfloat("reqresp", "read_interval")
if read_interval is None:
read_interval = 0.5
last_req_num = 0
conn = sqlite3.connect(trade_db_file)
query_curs = conn.cursor()
update_curs = conn.cursor()
query_unreported = """SELECT reqnum,message_type,appid,oms_order_id,order_status,req_text
FROM req_resp
WHERE order_status='0'
AND reqnum>?
ORDER BY reqnum
"""
update_reported = """UPDATE req_resp
SET order_status=?,report_time=strftime('%Y-%m-%d %H:%M:%f','now')
WHERE reqnum in(?)
"""
last_read_cnt = 1
while status.value == 0:
if last_read_cnt == 0:
time.sleep(read_interval)
last_read_cnt = 0
query_curs.execute(query_unreported, [last_req_num])
for (reqnum, message_type, appid, message_id, order_status, req_text) in query_curs.fetchall():
message_processor = factory.build_message_processor(message_type)
send_buff = message_processor.pack(req_text)
req_queue.put(send_buff)
last_req_num = reqnum
update_curs.execute(update_reported, ['2', reqnum])
last_read_cnt = last_read_cnt+1
if send_buff:
LogUtil.debug("db_reader putQ:"+binascii.hexlify(send_buff).decode())
else:
LogUtil.debug("db_reader putQ: send_buff NULL")
conn.commit()
LogUtil.info("db_reader:"+name+" end")
def tgw_recv(name, status, sock, resp_queue, config_file):
LogUtil.get_instance(config_file, "tgw_recv")
LogUtil.info("tgw_recv:"+name+" begin")
while status.value==0:
try:
recv_data = sock.recv(1024)
if not recv_data:
LogUtil.error('Recv message error!')
else:
LogUtil.debug('tgw recv:'+binascii.hexlify(recv_data).decode())
#to make the recv faster, do NOT process more, just put the message to the queue
resp_queue.put(recv_data)
finally:
pass
#LogUtil.debug("")
LogUtil.info("tgw_recv:"+name+" end")
def mdgw_recv(name, status, sock, resp_queue, config_file):
LogUtil.get_instance(config_file, "mdgw_recv")
LogUtil.info("mdgw_recv:"+name+" begin")
buf_id = 0
while status.value == 0:
try:
recv_data = sock.recv(1024)
if not recv_data:
LogUtil.error('Recv message error!')
else:
buf_id = buf_id+1
src_time = datetime.datetime.now()
# to make the recv faster, do NOT process more, just put the message to the queue
resp_queue.put((buf_id, src_time, recv_data))
LogUtil.debug('mdgw recv:'+binascii.hexlify(recv_data).decode())
finally:
pass
# LogUtil.debug("")
LogUtil.info("mdgw_recv:"+name+" end")
def tgw_send(name, status, sock, req_queue, config_file):
LogUtil.get_instance(config_file, "tgw_send")
LogUtil.info("tgw_send:"+name+" begin")
config=configparser.ConfigParser()
config.read(config_file)
heartbeat_interval=config.getint("tgw", "heartbeat_interval")
LogUtil.info("heartbeat_interval:"+str(heartbeat_interval))
if heartbeat_interval==None or heartbeat_interval<=0 or heartbeat_interval>=1800:
heartbeat_interval=60
LogUtil.info("heartbeat_interval changed to:"+str(heartbeat_interval))
send_heartbeat_interval=config.getint("tgw_send", "send_heartbeat_interval")
LogUtil.info("send_heartbeat_interval:"+str(send_heartbeat_interval))
if send_heartbeat_interval<=0 or send_heartbeat_interval>=heartbeat_interval:
send_heartbeat_interval=heartbeat_interval
LogUtil.info("send_heartbeat_interval changed to:"+str(send_heartbeat_interval))
read_timeout=config.getint("tgw_send", "req_queue_timeout")
LogUtil.info("read_timeout:"+str(read_timeout))
if read_timeout<=0 or read_timeout>send_heartbeat_interval/2:
read_timeout=send_heartbeat_interval/2
LogUtil.info("read_timeout changed to:"+str(read_timeout))
last_send_time=0
heartbeat=msg.packHeartbeatMessage()
while status.value==0:
try:
message=req_queue.get(block=True, timeout=read_timeout)
except queue.Empty:
LogUtil.debug("req_queue no data")
this_time=time.time()
if this_time-last_send_time>=send_heartbeat_interval:
req_queue.put(heartbeat)
last_send_time=this_time
else:
sock.sendall(message)
LogUtil.info("tgw_send:"+binascii.hexlify(message).decode())
LogUtil.info("tgw_send:"+name+" end")
def db_writer(name, status, resp_queue, config_file):
LogUtil.get_instance(config_file, "db_writer")
LogUtil.info("db_writer:"+name+" begin")
config=configparser.ConfigParser()
config.read(config_file)
factory=msg.MessageProcessorFactory()
db_file=config.get("message_config", "db_file")
factory.load_from_db(db_file, [])
read_timeout=config.getint("db_writer", "resp_queue_timeout")
LogUtil.info("read_timeout:"+str(read_timeout))
if read_timeout<=0 or read_timeout>60:
read_timeout=60
LogUtil.info("read_timeout changed to:"+str(read_timeout))
host=config.get("reqresp", "host")
database=config.get("reqresp", "database")
user=config.get("reqresp","user")
password=config.get("reqresp", "password")
conn=pgdb.connect(database=database, host=host, user=user, password=password)
update_curs=conn.cursor()
update_resp="""UPDATE req_resp
SET rept_status=%(rept_status)s,ex_order_status=%(ex_order_status)s,
err_code=%(err_code)s, resp_text=%(resp_text)s, resp_time=localtimestamp
WHERE oms_order_id=%(oms_order_id)s
"""
update_dict={'rept_status':'', 'ex_order_status':'', 'err_code':'', 'resp_text':'', 'oms_order_id':''}
left_buff=b''
while status.value==0:
#TODO:refactor,try recv; and then process the buff
#TODO:when processing buff, abstract the condition to next_message_ready()
try:
recv_buff=resp_queue.get(block=True, timeout=read_timeout)
left_buff=left_buff+recv_buff
if len(left_buff)<Message.header_len+Message.header_len:
continue
(message_type, body_len)=msg.get_message_header(left_buff)
next_message_len=body_len+ Message.header_len+Message.footer_len
while next_message_len<=len(left_buff):
try:
message_processor=factory.build_message_processor(message_type)
message=message_processor.unpack(left_buff)
LogUtil.debug("message:"+message.toString())
if True:#TODO placeholder, check if was order execution report
update_dict['rept_status']='4'
update_dict['ex_order_status']=message.order_status
update_dict['err_code']=message.order_reject_reason
update_dict['resp_text']=message.message_str
update_dict['oms_order_id']=message.client_order_id
update_curs.execute(update_resp, update_dict)
if update_curs.rowcount!=1:
#TODO error handle, rollback?
LogUtil.error("no data update"+message.toString())
else:
conn.commit()
except KeyError:
LogUtil.error("unkown message type:"+str(message_type))
left_buff=left_buff[next_message_len:]
if len(left_buff)<Message.header_len+Message.footer_len:
break
else:
(message_type, body_len)=msg.get_message_header(left_buff)
next_message_len=body_len+ Message.header_len+Message.footer_len
except queue.Empty:
LogUtil.debug("resp_queue no data")
# except KeyError:
# LogUtil.error("unkown message type:"+str(message_type))
else:
LogUtil.info("db_writer finished processing:"+message.toString())
LogUtil.info("db_writer:"+name+" end")
def main():
if len(sys.argv)<2:
print("Usage: tgw.py config_file")
sys.exit(0)
#read mdgw connection config
config_file=sys.argv[1]
run_status=multiprocessing.Value('i', 0)#0:运行;1:退出
message_header_struct = struct.Struct('!II')
logon_struct = struct.Struct('!20s20sI16s32s')
message_footer_struct = struct.Struct('!I')
send_buff = ctypes.create_string_buffer(message_header_struct.size+logon_struct.size+message_footer_struct.size)
bodyLength = logon_struct.size
message_header =(1, bodyLength)
message_header_struct.pack_into(send_buff, 0, *message_header)
LogUtil.get_instance(config_file, "log")
LogUtil.info("Begin")
config=configparser.ConfigParser()
config.read(config_file)
sender_comp=config.get("tgw","sender_comp")
target_comp=config.get("tgw","target_comp")
password=config.get("tgw","password")
app_ver_id=config.get("tgw","app_ver_id")
sender_comp = str.encode(sender_comp.ljust(20))
target_comp = str.encode(target_comp.ljust(20))
password = str.encode(password.ljust(16))
app_ver_id = str.encode(app_ver_id.ljust(32))
logon_body = (sender_comp, target_comp, 30, password, app_ver_id)
logon_struct.pack_into(send_buff, message_header_struct.size, *logon_body)
check_sum = msg.calculate_check_sum(send_buff, message_header_struct.size+logon_struct.size)
message_footer_struct.pack_into(send_buff, message_header_struct.size+logon_struct.size, check_sum)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_ip=config.get("tgw", "ip")
server_port=config.getint("tgw", "port")
#logger initialize
server_address = (server_ip, server_port)
sock.connect(server_address)
sock.settimeout(5)
sock.setblocking(True)
try:
LogUtil.debug(binascii.hexlify(send_buff))
sock.sendall(send_buff)
recv_data = sock.recv(1024)
if not recv_data:
LogUtil.error('Recv error')
else:
LogUtil.info('Recv OK')
LogUtil.info(binascii.hexlify(recv_data))
unpack_recv_data = message_header_struct.unpack_from(recv_data)
LogUtil.info(unpack_recv_data)
#print(binascii.hexlify(recv_data))
if unpack_recv_data[0]==1:
LogUtil.info('Receive Login Confirm!')
#TODO:send report sync
factory=msg.MessageProcessorFactory()
db_file=config.get("message_config", "db_file")
factory.load_from_db(db_file, [])
message_processor=factory.build_message_processor(5)
buff=message_processor.pack("ReportIndex=1")
sock.sendall(buff)
req_queue=multiprocessing.Queue()
resp_queue=multiprocessing.Queue()
dbreader_proc=multiprocessing.Process(target=db_reader, args=('DBReader', run_status, req_queue, config_file))
dbwriter_proc=multiprocessing.Process(target=db_writer, args=('DBWriter', run_status, resp_queue, config_file))
send_proc=multiprocessing.Process(target=tgw_send, args=('TGW sender', run_status, sock, req_queue, config_file))
recv_proc=multiprocessing.Process(target=tgw_recv, args=('TGW receiver', run_status, sock, resp_queue, config_file))
dbreader_proc.start()
dbwriter_proc.start()
send_proc.start()
recv_proc.start()
time.sleep(10)
cmd=input("enter command:")
while cmd!='q':
time.sleep(2)
cmd=input("enter command:")
LogUtil.warning("sending exit cmd")
run_status.value=1
dbreader_proc.join()
dbwriter_proc.join()
send_proc.join()
recv_proc.join()
#发送退出消息并处理应答
logout_message=msg.packLogoutMessage()
sock.sendall(logout_message)
recv_data = sock.recv(1024)
if not recv_data:
LogUtil.error('Recv logout_message error!')
else:
LogUtil.info('Recv logout_message OK')
LogUtil.debug(binascii.hexlify(recv_data))
finally:
sock.close()
LogUtil.info ('End')
def md_responsor(name, status, resp_queue, config_file):
LogUtil.get_instance(config_file, "md_responsor")
LogUtil.info("md_responsor:"+name+" begin")
config = configparser.ConfigParser()
config.read(config_file)
factory = msg.MessageProcessorFactory()
db_file = config.get("message_config", "db_file")
factory.load_from_db(db_file, [])
read_timeout = config.getint("md_responsor", "resp_queue_timeout")
LogUtil.info("read_timeout:"+str(read_timeout))
if read_timeout <= 0 or read_timeout > 60:
read_timeout = 60
LogUtil.info("read_timeout changed to:"+str(read_timeout))
pub_buf = config.get("md_responsor", "pub_buf")
pub_msg = config.get("md_responsor", "pub_msg")
pub_buf_addr = config.get("md_responsor", "pub_buf_addr")
pub_msg_addr = config.get("md_responsor", "pub_msg_addr")
LogUtil.debug("pub_buf:"+pub_buf+",pub_buf_addr:"+pub_buf_addr)
LogUtil.debug("pub_msg:"+pub_msg+",pub_msg_addr:"+pub_msg_addr)
if pub_buf:
buf_ctx = zmq.Context()
buf_sock = buf_ctx.socket(zmq.PUB)
buf_sock.bind(pub_buf_addr)
if pub_msg:
msg_ctx = zmq.Context()
msg_sock = msg_ctx.socket(zmq.PUB)
msg_sock.bind(pub_msg_addr)
left_buff = b''
message_id = 0
while status.value == 0:
# TODO:refactor,try recv; and then process the buff
# TODO:when processing buff, abstract the condition to next_message_ready()
try:
(buf_id, src_time, recv_buff) = resp_queue.get(block=True, timeout=read_timeout)
if pub_buf: # TODO:topic?
buf_sock.send_pyobj((buf_id, src_time, recv_buff))
left_buff = left_buff+recv_buff
if len(left_buff) < Message.header_len+Message.header_len:
continue
(message_type, body_len) = msg.get_message_header(left_buff)
next_message_len = body_len + Message.header_len+Message.footer_len
while next_message_len <= len(left_buff):
try:
message_processor = factory.build_message_processor(message_type)
message = message_processor.unpack(left_buff)
message_id = message_id+1
LogUtil.debug("message:"+message.toString())
if pub_msg: # TODO:topic?
src_time = datetime.datetime.now()
msg_sock.send_pyobj((message_type, message_id, src_time, message.message_str))
except KeyError:
LogUtil.error("unkown message type:"+str(message_type))
except Exception as e:
LogUtil.error(e)
LogUtil.error("other error:"+traceback.print_exc())
left_buff = left_buff[next_message_len:]
if len(left_buff) < Message.header_len+Message.footer_len:
break
else:
(message_type, body_len) = msg.get_message_header(left_buff)
next_message_len = body_len + Message.header_len+Message.footer_len
except queue.Empty:
LogUtil.debug("resp_queue no data")
# except KeyError:
# LogUtil.error("unkown message type:"+str(message_type))
else:
pass
LogUtil.info("md_responsor:"+name+" end")