def db_reader(name, status, req_queue, config_file): LogUtil.get_instance(config_file, "db_reader") LogUtil.info("db_reader:"+name+" begin") config=configparser.ConfigParser() config.read(config_file) factory=msg.MessageProcessorFactory() db_file=config.get("message_config", "db_file") factory.load_from_db(db_file, []) read_interval=config.getfloat("reqresp","read_interval") if read_interval==None: read_interval=0.5 host=config.get("reqresp", "host") database=config.get("reqresp", "database") user=config.get("reqresp","user") password=config.get("reqresp", "password") last_req_num=0 conn=pgdb.connect(database=database, host=host, user=user, password=password) query_curs=conn.cursor() update_curs=conn.cursor() query_unreported="""SELECT req_num,message_type,appid,oms_order_id,rept_status,req_text FROM req_resp WHERE rept_status='0' AND req_num>%(req_num)s ORDER BY req_num """ query_dict={'req_num':0} update_reported="""UPDATE req_resp SET rept_status=%(rept_status)s,report_time=localtimestamp WHERE req_num in (%(req_num)s) """ update_dict={'rept_status':'0', 'req_num':0} last_read_cnt=1 while status.value==0: if last_read_cnt==0: time.sleep(read_interval) last_read_cnt=0 query_dict['req_num']=last_req_num query_curs.execute(query_unreported,query_dict) for (req_num,message_type,appid,message_id,rept_status,req_text) in query_curs.fetchall(): message_processor=factory.build_message_processor(message_type) send_buff=message_processor.pack(req_text) req_queue.put(send_buff) last_req_num=req_num update_dict['rept_status']='2' update_dict['req_num']=last_req_num update_curs.execute(update_reported,update_dict) last_read_cnt=last_read_cnt+1 LogUtil.debug("db_reader putQ:"+binascii.hexlify(send_buff).decode()) conn.commit() LogUtil.info("db_reader:"+name+" end")
def md_requestor(name, status, req_queue, config_file): LogUtil.get_instance(config_file, "db_reader") LogUtil.info("db_reader:"+name+" begin") config = configparser.ConfigParser() config.read(config_file) factory = msg.MessageProcessorFactory() db_file = config.get("message_config", "db_file") factory.load_from_db(db_file, []) trade_db_file = config.get("reqresp", "db_file") read_interval = config.getfloat("reqresp", "read_interval") if read_interval is None: read_interval = 0.5 last_req_num = 0 conn = sqlite3.connect(trade_db_file) query_curs = conn.cursor() update_curs = conn.cursor() query_unreported = """SELECT reqnum,message_type,appid,oms_order_id,order_status,req_text FROM req_resp WHERE order_status='0' AND reqnum>? ORDER BY reqnum """ update_reported = """UPDATE req_resp SET order_status=?,report_time=strftime('%Y-%m-%d %H:%M:%f','now') WHERE reqnum in(?) """ last_read_cnt = 1 while status.value == 0: if last_read_cnt == 0: time.sleep(read_interval) last_read_cnt = 0 query_curs.execute(query_unreported, [last_req_num]) for (reqnum, message_type, appid, message_id, order_status, req_text) in query_curs.fetchall(): message_processor = factory.build_message_processor(message_type) send_buff = message_processor.pack(req_text) req_queue.put(send_buff) last_req_num = reqnum update_curs.execute(update_reported, ['2', reqnum]) last_read_cnt = last_read_cnt+1 if send_buff: LogUtil.debug("db_reader putQ:"+binascii.hexlify(send_buff).decode()) else: LogUtil.debug("db_reader putQ: send_buff NULL") conn.commit() LogUtil.info("db_reader:"+name+" end")
def tgw_recv(name, status, sock, resp_queue, config_file): LogUtil.get_instance(config_file, "tgw_recv") LogUtil.info("tgw_recv:"+name+" begin") while status.value==0: try: recv_data = sock.recv(1024) if not recv_data: LogUtil.error('Recv message error!') else: LogUtil.debug('tgw recv:'+binascii.hexlify(recv_data).decode()) #to make the recv faster, do NOT process more, just put the message to the queue resp_queue.put(recv_data) finally: pass #LogUtil.debug("") LogUtil.info("tgw_recv:"+name+" end")
def mdgw_recv(name, status, sock, resp_queue, config_file): LogUtil.get_instance(config_file, "mdgw_recv") LogUtil.info("mdgw_recv:"+name+" begin") buf_id = 0 while status.value == 0: try: recv_data = sock.recv(1024) if not recv_data: LogUtil.error('Recv message error!') else: buf_id = buf_id+1 src_time = datetime.datetime.now() # to make the recv faster, do NOT process more, just put the message to the queue resp_queue.put((buf_id, src_time, recv_data)) LogUtil.debug('mdgw recv:'+binascii.hexlify(recv_data).decode()) finally: pass # LogUtil.debug("") LogUtil.info("mdgw_recv:"+name+" end")
def tgw_send(name, status, sock, req_queue, config_file): LogUtil.get_instance(config_file, "tgw_send") LogUtil.info("tgw_send:"+name+" begin") config=configparser.ConfigParser() config.read(config_file) heartbeat_interval=config.getint("tgw", "heartbeat_interval") LogUtil.info("heartbeat_interval:"+str(heartbeat_interval)) if heartbeat_interval==None or heartbeat_interval<=0 or heartbeat_interval>=1800: heartbeat_interval=60 LogUtil.info("heartbeat_interval changed to:"+str(heartbeat_interval)) send_heartbeat_interval=config.getint("tgw_send", "send_heartbeat_interval") LogUtil.info("send_heartbeat_interval:"+str(send_heartbeat_interval)) if send_heartbeat_interval<=0 or send_heartbeat_interval>=heartbeat_interval: send_heartbeat_interval=heartbeat_interval LogUtil.info("send_heartbeat_interval changed to:"+str(send_heartbeat_interval)) read_timeout=config.getint("tgw_send", "req_queue_timeout") LogUtil.info("read_timeout:"+str(read_timeout)) if read_timeout<=0 or read_timeout>send_heartbeat_interval/2: read_timeout=send_heartbeat_interval/2 LogUtil.info("read_timeout changed to:"+str(read_timeout)) last_send_time=0 heartbeat=msg.packHeartbeatMessage() while status.value==0: try: message=req_queue.get(block=True, timeout=read_timeout) except queue.Empty: LogUtil.debug("req_queue no data") this_time=time.time() if this_time-last_send_time>=send_heartbeat_interval: req_queue.put(heartbeat) last_send_time=this_time else: sock.sendall(message) LogUtil.info("tgw_send:"+binascii.hexlify(message).decode()) LogUtil.info("tgw_send:"+name+" end")
def db_writer(name, status, resp_queue, config_file): LogUtil.get_instance(config_file, "db_writer") LogUtil.info("db_writer:"+name+" begin") config=configparser.ConfigParser() config.read(config_file) factory=msg.MessageProcessorFactory() db_file=config.get("message_config", "db_file") factory.load_from_db(db_file, []) read_timeout=config.getint("db_writer", "resp_queue_timeout") LogUtil.info("read_timeout:"+str(read_timeout)) if read_timeout<=0 or read_timeout>60: read_timeout=60 LogUtil.info("read_timeout changed to:"+str(read_timeout)) host=config.get("reqresp", "host") database=config.get("reqresp", "database") user=config.get("reqresp","user") password=config.get("reqresp", "password") conn=pgdb.connect(database=database, host=host, user=user, password=password) update_curs=conn.cursor() update_resp="""UPDATE req_resp SET rept_status=%(rept_status)s,ex_order_status=%(ex_order_status)s, err_code=%(err_code)s, resp_text=%(resp_text)s, resp_time=localtimestamp WHERE oms_order_id=%(oms_order_id)s """ update_dict={'rept_status':'', 'ex_order_status':'', 'err_code':'', 'resp_text':'', 'oms_order_id':''} left_buff=b'' while status.value==0: #TODO:refactor,try recv; and then process the buff #TODO:when processing buff, abstract the condition to next_message_ready() try: recv_buff=resp_queue.get(block=True, timeout=read_timeout) left_buff=left_buff+recv_buff if len(left_buff)<Message.header_len+Message.header_len: continue (message_type, body_len)=msg.get_message_header(left_buff) next_message_len=body_len+ Message.header_len+Message.footer_len while next_message_len<=len(left_buff): try: message_processor=factory.build_message_processor(message_type) message=message_processor.unpack(left_buff) LogUtil.debug("message:"+message.toString()) if True:#TODO placeholder, check if was order execution report update_dict['rept_status']='4' update_dict['ex_order_status']=message.order_status update_dict['err_code']=message.order_reject_reason update_dict['resp_text']=message.message_str update_dict['oms_order_id']=message.client_order_id update_curs.execute(update_resp, update_dict) if update_curs.rowcount!=1: #TODO error handle, rollback? LogUtil.error("no data update"+message.toString()) else: conn.commit() except KeyError: LogUtil.error("unkown message type:"+str(message_type)) left_buff=left_buff[next_message_len:] if len(left_buff)<Message.header_len+Message.footer_len: break else: (message_type, body_len)=msg.get_message_header(left_buff) next_message_len=body_len+ Message.header_len+Message.footer_len except queue.Empty: LogUtil.debug("resp_queue no data") # except KeyError: # LogUtil.error("unkown message type:"+str(message_type)) else: LogUtil.info("db_writer finished processing:"+message.toString()) LogUtil.info("db_writer:"+name+" end")
def main(): if len(sys.argv)<2: print("Usage: tgw.py config_file") sys.exit(0) #read mdgw connection config config_file=sys.argv[1] run_status=multiprocessing.Value('i', 0)#0:运行;1:退出 message_header_struct = struct.Struct('!II') logon_struct = struct.Struct('!20s20sI16s32s') message_footer_struct = struct.Struct('!I') send_buff = ctypes.create_string_buffer(message_header_struct.size+logon_struct.size+message_footer_struct.size) bodyLength = logon_struct.size message_header =(1, bodyLength) message_header_struct.pack_into(send_buff, 0, *message_header) LogUtil.get_instance(config_file, "log") LogUtil.info("Begin") config=configparser.ConfigParser() config.read(config_file) sender_comp=config.get("tgw","sender_comp") target_comp=config.get("tgw","target_comp") password=config.get("tgw","password") app_ver_id=config.get("tgw","app_ver_id") sender_comp = str.encode(sender_comp.ljust(20)) target_comp = str.encode(target_comp.ljust(20)) password = str.encode(password.ljust(16)) app_ver_id = str.encode(app_ver_id.ljust(32)) logon_body = (sender_comp, target_comp, 30, password, app_ver_id) logon_struct.pack_into(send_buff, message_header_struct.size, *logon_body) check_sum = msg.calculate_check_sum(send_buff, message_header_struct.size+logon_struct.size) message_footer_struct.pack_into(send_buff, message_header_struct.size+logon_struct.size, check_sum) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_ip=config.get("tgw", "ip") server_port=config.getint("tgw", "port") #logger initialize server_address = (server_ip, server_port) sock.connect(server_address) sock.settimeout(5) sock.setblocking(True) try: LogUtil.debug(binascii.hexlify(send_buff)) sock.sendall(send_buff) recv_data = sock.recv(1024) if not recv_data: LogUtil.error('Recv error') else: LogUtil.info('Recv OK') LogUtil.info(binascii.hexlify(recv_data)) unpack_recv_data = message_header_struct.unpack_from(recv_data) LogUtil.info(unpack_recv_data) #print(binascii.hexlify(recv_data)) if unpack_recv_data[0]==1: LogUtil.info('Receive Login Confirm!') #TODO:send report sync factory=msg.MessageProcessorFactory() db_file=config.get("message_config", "db_file") factory.load_from_db(db_file, []) message_processor=factory.build_message_processor(5) buff=message_processor.pack("ReportIndex=1") sock.sendall(buff) req_queue=multiprocessing.Queue() resp_queue=multiprocessing.Queue() dbreader_proc=multiprocessing.Process(target=db_reader, args=('DBReader', run_status, req_queue, config_file)) dbwriter_proc=multiprocessing.Process(target=db_writer, args=('DBWriter', run_status, resp_queue, config_file)) send_proc=multiprocessing.Process(target=tgw_send, args=('TGW sender', run_status, sock, req_queue, config_file)) recv_proc=multiprocessing.Process(target=tgw_recv, args=('TGW receiver', run_status, sock, resp_queue, config_file)) dbreader_proc.start() dbwriter_proc.start() send_proc.start() recv_proc.start() time.sleep(10) cmd=input("enter command:") while cmd!='q': time.sleep(2) cmd=input("enter command:") LogUtil.warning("sending exit cmd") run_status.value=1 dbreader_proc.join() dbwriter_proc.join() send_proc.join() recv_proc.join() #发送退出消息并处理应答 logout_message=msg.packLogoutMessage() sock.sendall(logout_message) recv_data = sock.recv(1024) if not recv_data: LogUtil.error('Recv logout_message error!') else: LogUtil.info('Recv logout_message OK') LogUtil.debug(binascii.hexlify(recv_data)) finally: sock.close() LogUtil.info ('End')
def md_responsor(name, status, resp_queue, config_file): LogUtil.get_instance(config_file, "md_responsor") LogUtil.info("md_responsor:"+name+" begin") config = configparser.ConfigParser() config.read(config_file) factory = msg.MessageProcessorFactory() db_file = config.get("message_config", "db_file") factory.load_from_db(db_file, []) read_timeout = config.getint("md_responsor", "resp_queue_timeout") LogUtil.info("read_timeout:"+str(read_timeout)) if read_timeout <= 0 or read_timeout > 60: read_timeout = 60 LogUtil.info("read_timeout changed to:"+str(read_timeout)) pub_buf = config.get("md_responsor", "pub_buf") pub_msg = config.get("md_responsor", "pub_msg") pub_buf_addr = config.get("md_responsor", "pub_buf_addr") pub_msg_addr = config.get("md_responsor", "pub_msg_addr") LogUtil.debug("pub_buf:"+pub_buf+",pub_buf_addr:"+pub_buf_addr) LogUtil.debug("pub_msg:"+pub_msg+",pub_msg_addr:"+pub_msg_addr) if pub_buf: buf_ctx = zmq.Context() buf_sock = buf_ctx.socket(zmq.PUB) buf_sock.bind(pub_buf_addr) if pub_msg: msg_ctx = zmq.Context() msg_sock = msg_ctx.socket(zmq.PUB) msg_sock.bind(pub_msg_addr) left_buff = b'' message_id = 0 while status.value == 0: # TODO:refactor,try recv; and then process the buff # TODO:when processing buff, abstract the condition to next_message_ready() try: (buf_id, src_time, recv_buff) = resp_queue.get(block=True, timeout=read_timeout) if pub_buf: # TODO:topic? buf_sock.send_pyobj((buf_id, src_time, recv_buff)) left_buff = left_buff+recv_buff if len(left_buff) < Message.header_len+Message.header_len: continue (message_type, body_len) = msg.get_message_header(left_buff) next_message_len = body_len + Message.header_len+Message.footer_len while next_message_len <= len(left_buff): try: message_processor = factory.build_message_processor(message_type) message = message_processor.unpack(left_buff) message_id = message_id+1 LogUtil.debug("message:"+message.toString()) if pub_msg: # TODO:topic? src_time = datetime.datetime.now() msg_sock.send_pyobj((message_type, message_id, src_time, message.message_str)) except KeyError: LogUtil.error("unkown message type:"+str(message_type)) except Exception as e: LogUtil.error(e) LogUtil.error("other error:"+traceback.print_exc()) left_buff = left_buff[next_message_len:] if len(left_buff) < Message.header_len+Message.footer_len: break else: (message_type, body_len) = msg.get_message_header(left_buff) next_message_len = body_len + Message.header_len+Message.footer_len except queue.Empty: LogUtil.debug("resp_queue no data") # except KeyError: # LogUtil.error("unkown message type:"+str(message_type)) else: pass LogUtil.info("md_responsor:"+name+" end")