def db_reader(name, status, req_queue, config_file):
LogUtil.get_instance(config_file, "db_reader")
LogUtil.info("db_reader:"+name+" begin")
config=configparser.ConfigParser()
config.read(config_file)
factory=msg.MessageProcessorFactory()
db_file=config.get("message_config", "db_file")
factory.load_from_db(db_file, [])
read_interval=config.getfloat("reqresp","read_interval")
if read_interval==None:
read_interval=0.5
host=config.get("reqresp", "host")
database=config.get("reqresp", "database")
user=config.get("reqresp","user")
password=config.get("reqresp", "password")
last_req_num=0
conn=pgdb.connect(database=database, host=host, user=user, password=password)
query_curs=conn.cursor()
update_curs=conn.cursor()
query_unreported="""SELECT req_num,message_type,appid,oms_order_id,rept_status,req_text
FROM req_resp
WHERE rept_status='0'
AND req_num>%(req_num)s
ORDER BY req_num
"""
query_dict={'req_num':0}
update_reported="""UPDATE req_resp
SET rept_status=%(rept_status)s,report_time=localtimestamp
WHERE req_num in (%(req_num)s)
"""
update_dict={'rept_status':'0', 'req_num':0}
last_read_cnt=1
while status.value==0:
if last_read_cnt==0:
time.sleep(read_interval)
last_read_cnt=0
query_dict['req_num']=last_req_num
query_curs.execute(query_unreported,query_dict)
for (req_num,message_type,appid,message_id,rept_status,req_text) in query_curs.fetchall():
message_processor=factory.build_message_processor(message_type)
send_buff=message_processor.pack(req_text)
req_queue.put(send_buff)
last_req_num=req_num
update_dict['rept_status']='2'
update_dict['req_num']=last_req_num
update_curs.execute(update_reported,update_dict)
last_read_cnt=last_read_cnt+1
LogUtil.debug("db_reader putQ:"+binascii.hexlify(send_buff).decode())
conn.commit()
LogUtil.info("db_reader:"+name+" end")
def md_requestor(name, status, req_queue, config_file):
LogUtil.get_instance(config_file, "db_reader")
LogUtil.info("db_reader:"+name+" begin")
config = configparser.ConfigParser()
config.read(config_file)
factory = msg.MessageProcessorFactory()
db_file = config.get("message_config", "db_file")
factory.load_from_db(db_file, [])
trade_db_file = config.get("reqresp", "db_file")
read_interval = config.getfloat("reqresp", "read_interval")
if read_interval is None:
read_interval = 0.5
last_req_num = 0
conn = sqlite3.connect(trade_db_file)
query_curs = conn.cursor()
update_curs = conn.cursor()
query_unreported = """SELECT reqnum,message_type,appid,oms_order_id,order_status,req_text
FROM req_resp
WHERE order_status='0'
AND reqnum>?
ORDER BY reqnum
"""
update_reported = """UPDATE req_resp
SET order_status=?,report_time=strftime('%Y-%m-%d %H:%M:%f','now')
WHERE reqnum in(?)
"""
last_read_cnt = 1
while status.value == 0:
if last_read_cnt == 0:
time.sleep(read_interval)
last_read_cnt = 0
query_curs.execute(query_unreported, [last_req_num])
for (reqnum, message_type, appid, message_id, order_status, req_text) in query_curs.fetchall():
message_processor = factory.build_message_processor(message_type)
send_buff = message_processor.pack(req_text)
req_queue.put(send_buff)
last_req_num = reqnum
update_curs.execute(update_reported, ['2', reqnum])
last_read_cnt = last_read_cnt+1
if send_buff:
LogUtil.debug("db_reader putQ:"+binascii.hexlify(send_buff).decode())
else:
LogUtil.debug("db_reader putQ: send_buff NULL")
conn.commit()
LogUtil.info("db_reader:"+name+" end")
def tgw_recv(name, status, sock, resp_queue, config_file):
LogUtil.get_instance(config_file, "tgw_recv")
LogUtil.info("tgw_recv:"+name+" begin")
while status.value==0:
try:
recv_data = sock.recv(1024)
if not recv_data:
LogUtil.error('Recv message error!')
else:
LogUtil.debug('tgw recv:'+binascii.hexlify(recv_data).decode())
#to make the recv faster, do NOT process more, just put the message to the queue
resp_queue.put(recv_data)
finally:
pass
#LogUtil.debug("")
LogUtil.info("tgw_recv:"+name+" end")
def mdgw_recv(name, status, sock, resp_queue, config_file):
LogUtil.get_instance(config_file, "mdgw_recv")
LogUtil.info("mdgw_recv:"+name+" begin")
buf_id = 0
while status.value == 0:
try:
recv_data = sock.recv(1024)
if not recv_data:
LogUtil.error('Recv message error!')
else:
buf_id = buf_id+1
src_time = datetime.datetime.now()
# to make the recv faster, do NOT process more, just put the message to the queue
resp_queue.put((buf_id, src_time, recv_data))
LogUtil.debug('mdgw recv:'+binascii.hexlify(recv_data).decode())
finally:
pass
# LogUtil.debug("")
LogUtil.info("mdgw_recv:"+name+" end")
def tgw_send(name, status, sock, req_queue, config_file):
LogUtil.get_instance(config_file, "tgw_send")
LogUtil.info("tgw_send:"+name+" begin")
config=configparser.ConfigParser()
config.read(config_file)
heartbeat_interval=config.getint("tgw", "heartbeat_interval")
LogUtil.info("heartbeat_interval:"+str(heartbeat_interval))
if heartbeat_interval==None or heartbeat_interval<=0 or heartbeat_interval>=1800:
heartbeat_interval=60
LogUtil.info("heartbeat_interval changed to:"+str(heartbeat_interval))
send_heartbeat_interval=config.getint("tgw_send", "send_heartbeat_interval")
LogUtil.info("send_heartbeat_interval:"+str(send_heartbeat_interval))
if send_heartbeat_interval<=0 or send_heartbeat_interval>=heartbeat_interval:
send_heartbeat_interval=heartbeat_interval
LogUtil.info("send_heartbeat_interval changed to:"+str(send_heartbeat_interval))
read_timeout=config.getint("tgw_send", "req_queue_timeout")
LogUtil.info("read_timeout:"+str(read_timeout))
if read_timeout<=0 or read_timeout>send_heartbeat_interval/2:
read_timeout=send_heartbeat_interval/2
LogUtil.info("read_timeout changed to:"+str(read_timeout))
last_send_time=0
heartbeat=msg.packHeartbeatMessage()
while status.value==0:
try:
message=req_queue.get(block=True, timeout=read_timeout)
except queue.Empty:
LogUtil.debug("req_queue no data")
this_time=time.time()
if this_time-last_send_time>=send_heartbeat_interval:
req_queue.put(heartbeat)
last_send_time=this_time
else:
sock.sendall(message)
LogUtil.info("tgw_send:"+binascii.hexlify(message).decode())
LogUtil.info("tgw_send:"+name+" end")
def test_unpack(buff_str):
factory=msg.MessageProcessorFactory()
factory.load_from_db('message_config.s3db', [])
left_buff=binascii.a2b_hex(buff_str)
if len(left_buff)<Message.header_len+Message.header_len:
LogUtil.info("next message not ready1")
(message_type, body_len)=msg.get_message_header(left_buff)
next_message_len=body_len+ Message.header_len+Message.footer_len
while next_message_len<=len(left_buff):
message_processor=factory.build_message_processor(message_type)
message=message_processor.unpack(left_buff)
LogUtil.debug("message:"+message.toString())
left_buff=left_buff[next_message_len:]
if len(left_buff)<Message.header_len+Message.footer_len:
LogUtil.debug("break @left size:"+str(len(left_buff)))
break
else:
(message_type, body_len)=msg.get_message_header(left_buff)
next_message_len=body_len+ Message.header_len+Message.footer_len
LogUtil.debug("MsgType="+str(message_type)+" body_len="+str(body_len))
LogUtil.debug("left buff:"+binascii.hexlify(left_buff).decode())
class LabelUtil:
_log = None
# dataPath
def __init__(self):
self._log = LogUtil().getlogger()
self._log.debug("LabelUtil init")
def load_unicode_set(self, unicodeFilePath):
self.byChar = {}
self.byIndex = {}
self.unicodeFilePath = unicodeFilePath
with open(unicodeFilePath) as data_file:
data_file = csv.reader(data_file, delimiter=',')
self.count = 0
for r in data_file:
self.byChar[r[0]] = int(r[1])
self.byIndex[int(r[1])] = r[0]
self.count += 1
def to_unicode(self, src, index):
# 1 byte
code1 = int(ord(src[index + 0]))
index += 1
result = code1
return result, index
def convert_word_to_grapheme(self, label):
result = []
index = 0
while index < len(label):
(code, nextIndex) = self.to_unicode(label, index)
result.append(label[index])
index = nextIndex
return result, "".join(result)
def convert_word_to_num(self, word):
try:
label_list, _ = self.convert_word_to_grapheme(word)
label_num = []
for char in label_list:
# skip word
if char == "":
pass
else:
label_num.append(int(self.byChar[char]))
# tuple typecast: read only, faster
return tuple(label_num)
except AttributeError:
self._log.error("unicodeSet is not loaded")
exit(-1)
except KeyError as err:
self._log.error("unicodeSet Key not found: %s" % err)
exit(-1)
def convert_bi_graphemes_to_num(self, word):
label_num = []
for char in word:
# skip word
if char == "":
pass
else:
label_num.append(int(self.byChar[char]))
# tuple typecast: read only, faster
return tuple(label_num)
def convert_num_to_word(self, num_list):
try:
label_list = []
for num in num_list:
label_list.append(self.byIndex[num])
return ''.join(label_list)
except AttributeError:
self._log.error("unicodeSet is not loaded")
exit(-1)
except KeyError as err:
self._log.error("unicodeSet Key not found: %s" % err)
exit(-1)
def get_count(self):
try:
return self.count
except AttributeError:
self._log.error("unicodeSet is not loaded")
exit(-1)
def get_unicode_file_path(self):
try:
return self.unicodeFilePath
except AttributeError:
self._log.error("unicodeSet is not loaded")
exit(-1)
def get_blank_index(self):
return self.byChar["-"]
def get_space_index(self):
return self.byChar["$"]
def unpack_nested(self, buff, separator=',', write_tag_name=True):
message=Message()
message.separator=separator
message.has_tag_name=write_tag_name
message.message_type=self.message_type
(message_type, body_len)=self.header_struct.unpack_from(buff)
if message_type!=self.message_type:
#TODO:error format check
LogUtil.error("Error message_type,expected="+str(self.message_type)+",act_type="+str(message_type))
return
#TODO:check buff size
#TODO:check checksum
body_left_buff=buff[self.header_struct.size:]
bytes_processed=0
#body_tuple=self.body_struct.unpack_from(body_buff)
str_value_list=[]
rtn_str=''
#field_list:
#0:field_name;
#1:format_str:
#2:type,eg N,C;
#3:type_len;
#4:ref_field
#5:struct_tag
field_index=0
#support we enounter a body begin tag
(loop_cnt_field, loop_cnt, exec_cnt)=(0, 1, 1)
(var_len_indictor_field, var_len)=(0, 0)
loop_list=[]
while field_index<len(self.field_list) and self.field_list[field_index][5]!='BODY_END':
if self.field_list[field_index][5] in('DATA', 'LOOP_CNT', 'NEXT_VAR_LEN', 'VAR_LEN'):
if field_index>0:
rtn_str=rtn_str+separator
if write_tag_name:
rtn_str=rtn_str+self.field_list[field_index][0]+'='
if self.field_list[field_index][5]=='VAR_LEN':
#self.field_list[field_index][4]-1 is the index of field which holding the length
if self.field_list[field_index][4]!=var_len_indictor_field:
LogUtil.error("Error getting VAR_LEN:field_index="+str(field_index)+" indict field="+str(var_len_indictor_field))
format_str="!"+str(var_len)+"s"
bytes_len=var_len
LogUtil.debug("field:"+self.field_list[field_index][0]+" format:"+format_str)
else:
format_str="!"+self.field_list[field_index][1]
bytes_len=self.field_list[field_index][3]
LogUtil.debug("field:"+self.field_list[field_index][0]+" format:"+format_str)
field_value=struct.unpack_from(format_str, body_left_buff)
if self.field_list[field_index][5]=='NEXT_VAR_LEN':
var_len=field_value[0]
var_len_indictor_field=field_index
if self.field_list[field_index][2]=='C':
field_str_value=bytes.decode(field_value[0])
elif self.field_list[field_index][2]=='N':
field_str_value=str(field_value[0])
else:
field_str_value=bytes.decode(field_value[0])
LogUtil.error("unknown type category:"+self.field_list[field_index][2])
str_value_list.append(field_str_value)
rtn_str=rtn_str+field_str_value
#TODO add a column to message_body_def to indicate whether the column is client Order ID
if self.field_list[field_index][0]=='ClOrdID':
message.client_order_id=field_str_value
elif self.field_list[field_index][0]=='OrdStatus':
message.order_status=field_str_value
elif self.field_list[field_index][0]=='OrdRejReason':
message.order_reject_reason=field_value[0]
bytes_processed=bytes_processed+bytes_len
body_left_buff=body_left_buff[bytes_len:]
if self.field_list[field_index][5]=='LOOP_CNT':
loop_list.append((field_index, loop_cnt, exec_cnt))
LogUtil.debug("push:field_index="+str(field_index)+ ",loop="+str(loop_cnt)+ ",exec="+str(exec_cnt))
loop_cnt=field_value[0]
exec_cnt=0
field_index=field_index+1
elif self.field_list[field_index][5] in('LOOP_BEGIN'):
if exec_cnt<loop_cnt:
exec_cnt=exec_cnt+1
field_index=field_index+1
else: #end loop, move index to LOOP_END+1
(loop_cnt_field, loop_cnt, exec_cnt)=loop_list.pop()
LogUtil.debug("pop:field_index="+str(loop_cnt_field)+",loop="+str(loop_cnt)+ ",exec="+str(exec_cnt))
field_index=self.field_list[field_index][4]
elif self.field_list[field_index][5] in('LOOP_END'):
#goto LOOP BEGIN(which is saved in ref_field)
field_index=self.field_list[field_index][4]
if exec_cnt>loop_cnt:
LogUtil.critical("LOOP_END error: exec_cnt="+str(exec_cnt)+",loop_cnt="+str(loop_cnt))
else:
#TODO:raiseExceptions
LogUtil.error("unsupported struct tag:"+self.field_list[field_index][5])
break
message.message_str=rtn_str
if bytes_processed!=body_len:
LogUtil.error("bytes_process!=body_len,mesType="+str(message_type)+",body_len="+str(body_len)+",bytes_processed="+str(bytes_processed))
return message
def unpack_varlen(self, buff, separator=',', write_tag_name=True):
message=Message()
message.separator=separator
message.has_tag_name=write_tag_name
message.message_type=self.message_type
(message_type, body_len)=self.header_struct.unpack_from(buff)
if message_type!=self.message_type:
#TODO:error format check
LogUtil.error("Error message_type,expected="+str(self.message_type)+",act_type="+str(message_type))
return
#TODO:check buff size
#TODO:check checksum
bytes_processed=0
body_left_buff=buff[self.header_struct.size:]
#body_tuple=self.body_struct.unpack_from(body_buff)
str_value_list=[]
rtn_str=''
#field_list:
#0:field_name;
#1:format_str:
#2:type,eg N,C;
#3:type_len;
#4:ref_var_len_field
for field_index in range(len(self.field_list)):
if field_index>0:
rtn_str=rtn_str+separator
if write_tag_name:
rtn_str=rtn_str+self.field_list[field_index][0]+'='
if self.field_list[field_index][4]>0:
#self.field_list[field_index][4] is the index of field which holding the length
format_str="!"+str_value_list[self.field_list[field_index][4]]+"s"
bytes_len=int(str_value_list[self.field_list[field_index][4]])
LogUtil.debug("field:"+self.field_list[field_index][0]+" format:"+format_str)
else:
format_str="!"+self.field_list[field_index][1]
bytes_len=self.field_list[field_index][3]
LogUtil.debug("field:"+self.field_list[field_index][0]+" format:"+format_str)
field_value=struct.unpack_from(format_str, body_left_buff)
if self.field_list[field_index][2]=='C':
field_str_value=bytes.decode(field_value[0])
elif self.field_list[field_index][2]=='N':
field_str_value=str(field_value[0])
else:
field_str_value=bytes.decode(field_value[0])
str_value_list.append(field_str_value)
rtn_str=rtn_str+field_str_value
#TODO add a column to message_body_def to indicate whether the column is client Order ID
if self.field_list[field_index][0]=='ClOrdID':
message.client_order_id=field_str_value
elif self.field_list[field_index][0]=='OrdStatus':
message.order_status=field_str_value
elif self.field_list[field_index][0]=='OrdRejReason':
message.order_reject_reason=field_value
bytes_processed=bytes_processed+bytes_len
body_left_buff=body_left_buff[bytes_len:]
message.message_str=rtn_str
if bytes_processed!=body_len:
LogUtil.error("bytes_process!=body_len,mesType="+str(message_type)+",body_len="+str(body_len)+",bytes_processed="+str(bytes_processed))
return message
def run(self):
self.dialog.pushButtonBufRecvStart.setEnabled(False)
self.dialog.pushButtonBufRecvStop.setEnabled(True)
LogUtil.getLoggerInstance(self.config, "buf_recv")
LogUtil.info("buf_recv:"+" begin")
config=configparser.ConfigParser()
config.read(self.config)
write_buff=config.get("buf_recv", "write_buff")
if write_buff:
md_host=config.get("buf_recv", "md_host")
md_database=config.get("buf_recv", "md_database")
md_user=config.get("buf_recv", "md_user")
md_password=config.get("buf_recv", "md_password")
md_conn=pgdb.connect(database=md_database, host=md_host, user=md_user, password=md_password)
md_insert_buf_cursor=md_conn.cursor()
#md_insert_buf_sql='''insert into market_data_buff(data_date,insert_time,buff)
#values(%(data_date)s,localtimestamp,%(buff)s)
#'''
md_insert_buf_sql='''insert into market_data_buff(data_date,buff_id,insert_time,buff,src_time)
values(%(data_date)s,%(buf_id)s,%(insert_time)s,%(buff)s,%(src_time)s)
'''
md_insert_buf_dict={'data_date':0, 'buf_id':0, 'insert_time':None, 'buff':'', 'src_time':None}
buf_sub_addr=config.get("buf_recv", "buf_sub_addr")
buf_sub_topic=config.get("buf_recv", "buf_sub_topic")
ctx=zmq.Context()
sock=ctx.socket(zmq.SUB)
sock.connect(buf_sub_addr)
sock.setsockopt_string(zmq.SUBSCRIBE, buf_sub_topic)
bufRecvCnt=0
bufWriteCnt=0
bufUpdateTime=None
bufErrCnt=0
bufStatus='Running'
recvBufStatus=RecvBufStatus()
while not self.toStop:
try:
(buf_id, src_time, recv_buff)=sock.recv_pyobj()
bufRecvCnt=bufRecvCnt+1
bufUpdateTime=datetime.datetime.now().strftime('%H:%M:%S.%f')
if write_buff:
md_insert_buf_dict['data_date']=0
md_insert_buf_dict['insert_time']=datetime.datetime.now()
md_insert_buf_dict['buf_id']=buf_id
md_insert_buf_dict['buff']=binascii.hexlify(recv_buff).decode()
#msg=recv_buff.decode()
md_insert_buf_dict['src_time']=src_time
md_insert_buf_cursor.execute(md_insert_buf_sql, md_insert_buf_dict)
#TODO
md_conn.commit()
bufWriteCnt=bufWriteCnt+1
#self.dialog.bufWriteCnt.setText(str(bufWriteCnt))
except Exception as e:
bufErrCnt=bufErrCnt+1
#self.dialog.bufErrCnt.setText(str(bufErrCnt))
LogUtil.error(e)
LogUtil.error("BufRecvErr:bufErrCnt="+str(bufErrCnt)+",bufRecvCnt="+str(bufRecvCnt)+",bufWriteCnt="+str(bufWriteCnt))
finally:
pass
#TODO
recvBufStatus.bufRecvCnt=str(bufRecvCnt)
recvBufStatus.bufWriteCnt=str(bufWriteCnt)
recvBufStatus.bufErrCnt=str(bufErrCnt)
recvBufStatus.bufUpdateTime=bufUpdateTime
recvBufStatus.bufStatus=bufStatus
self.bufStatusUpdated.emit(recvBufStatus)
LogUtil.debug("BufRecvErr:bufErrCnt="+str(bufErrCnt)+",bufRecvCnt="+str(bufRecvCnt)+",bufWriteCnt="+str(bufWriteCnt))
self.dialog.pushButtonBufRecvStart.setEnabled(True)
self.dialog.pushButtonBufRecvStop.setEnabled(False)
def run(self):
self.dialog.pushButtonMsgRecvStart.setEnabled(False)
self.dialog.pushButtonMsgRecvStop.setEnabled(True)
LogUtil.getLoggerInstance(self.config, "msg_recv")
LogUtil.info("msg_recv:"+" begin")
config=configparser.ConfigParser()
config.read(self.config)
write_msg=config.get("msg_recv", "write_msg")
if write_msg:
md_host=config.get("msg_recv", "md_host")
md_database=config.get("msg_recv", "md_database")
md_user=config.get("msg_recv", "md_user")
md_password=config.get("msg_recv", "md_password")
md_conn=pgdb.connect(database=md_database, host=md_host, user=md_user, password=md_password)
md_insert_msg_cursor=md_conn.cursor()
md_insert_msg_sql='''insert into market_data_message(data_date,insert_time,message_type,message_id,message_content,src_time)
values(%(data_date)s,%(insert_time)s,%(message_type)s,%(message_id)s,%(message_content)s,%(src_time)s)
'''
md_insert_msg_dict={'data_date':0, 'message_type':0, 'message_content':''}
msg_sub_addr=config.get("msg_recv", "msg_sub_addr")
msg_sub_topic=config.get("msg_recv", "msg_sub_topic")
ctx=zmq.Context()
sock=ctx.socket(zmq.SUB)
sock.connect(msg_sub_addr)
sock.setsockopt_string(zmq.SUBSCRIBE, msg_sub_topic)
msgRecvCnt=0
msgWriteCnt=0
msgUpdateTime=None
msgErrCnt=0
recvMsgStatus=RecvMsgStatus()
msgStatus='Running'
while not self.toStop:
try:
(message_type, message_id, src_time, recv_msg)=sock.recv_pyobj()
msgRecvCnt=msgRecvCnt+1
msgUpdateTime=datetime.datetime.now().strftime('%H:%M:%S.%f')
if write_msg:
md_insert_msg_dict['data_date']=0
md_insert_msg_dict['insert_time']=datetime.datetime.now()
md_insert_msg_dict['message_type']=message_type
md_insert_msg_dict['message_id']=message_id
md_insert_msg_dict['message_content']=recv_msg
md_insert_msg_dict['src_time']=src_time
md_insert_msg_cursor.execute(md_insert_msg_sql, md_insert_msg_dict)
#TODO
md_conn.commit()
msgWriteCnt=msgWriteCnt+1
except Exception as e:
msgErrCnt=msgErrCnt+1
LogUtil.error(e)
LogUtil.error("MsgRecvErr:msgErrCnt="+str(msgErrCnt)+",msgRecvCnt="+str(msgRecvCnt)+",msgWriteCnt="+str(msgWriteCnt))
finally:
pass
recvMsgStatus.msgRecvCnt=str(msgRecvCnt)
recvMsgStatus.msgWriteCnt=str(msgWriteCnt)
recvMsgStatus.msgErrCnt=str(msgErrCnt)
recvMsgStatus.msgUpdateTime=msgUpdateTime
recvMsgStatus.msgStatus=msgStatus
self.msgStatusUpdated.emit(recvMsgStatus)
LogUtil.debug("MsgRecvErr:msgErrCnt="+str(msgErrCnt)+",msgRecvCnt="+str(msgRecvCnt)+",msgWriteCnt="+str(msgWriteCnt))
self.dialog.pushButtonMsgRecvStart.setEnabled(True)
self.dialog.pushButtonMsgRecvStop.setEnabled(False)
class LabelUtil:
_log = None
# dataPath
def __init__(self):
self._log = LogUtil().getlogger()
self._log.debug("LabelUtil init")
def load_unicode_set(self, unicodeFilePath):
self.byChar = {}
self.byIndex = {}
self.unicodeFilePath = unicodeFilePath
with open(unicodeFilePath) as data_file:
data_file = csv.reader(data_file, delimiter=',')
self.count = 0
for r in data_file:
self.byChar[r[0]] = int(r[1])
self.byIndex[int(r[1])] = r[0]
self.count += 1
def to_unicode(self, src, index):
# 1 byte
code1 = int(ord(src[index + 0]))
index += 1
result = code1
return result, index
def convert_word_to_grapheme(self, label):
result = []
index = 0
while index < len(label):
(code, nextIndex) = self.to_unicode(label, index)
result.append(label[index])
index = nextIndex
return result, "".join(result)
def convert_word_to_num(self, word):
try:
label_list, _ = self.convert_word_to_grapheme(word)
label_num = []
for char in label_list:
# skip word
if char == "":
pass
else:
label_num.append(int(self.byChar[char]))
# tuple typecast: read only, faster
return tuple(label_num)
except AttributeError:
self._log.error("unicodeSet is not loaded")
exit(-1)
except KeyError as err:
self._log.error("unicodeSet Key not found: %s" % err)
exit(-1)
def convert_bi_graphemes_to_num(self, word):
label_num = []
for char in word:
# skip word
if char == "":
pass
else:
label_num.append(int(self.byChar[char]))
# tuple typecast: read only, faster
return tuple(label_num)
def convert_num_to_word(self, num_list):
try:
label_list = []
for num in num_list:
label_list.append(self.byIndex[num])
return ''.join(label_list)
except AttributeError:
self._log.error("unicodeSet is not loaded")
exit(-1)
except KeyError as err:
self._log.error("unicodeSet Key not found: %s" % err)
exit(-1)
def get_count(self):
try:
return self.count
except AttributeError:
self._log.error("unicodeSet is not loaded")
exit(-1)
def get_unicode_file_path(self):
try:
return self.unicodeFilePath
except AttributeError:
self._log.error("unicodeSet is not loaded")
exit(-1)
def get_blank_index(self):
return self.byChar["-"]
def get_space_index(self):
return self.byChar["$"]
def db_writer(name, status, resp_queue, config_file):
LogUtil.get_instance(config_file, "db_writer")
LogUtil.info("db_writer:"+name+" begin")
config=configparser.ConfigParser()
config.read(config_file)
factory=msg.MessageProcessorFactory()
db_file=config.get("message_config", "db_file")
factory.load_from_db(db_file, [])
read_timeout=config.getint("db_writer", "resp_queue_timeout")
LogUtil.info("read_timeout:"+str(read_timeout))
if read_timeout<=0 or read_timeout>60:
read_timeout=60
LogUtil.info("read_timeout changed to:"+str(read_timeout))
host=config.get("reqresp", "host")
database=config.get("reqresp", "database")
user=config.get("reqresp","user")
password=config.get("reqresp", "password")
conn=pgdb.connect(database=database, host=host, user=user, password=password)
update_curs=conn.cursor()
update_resp="""UPDATE req_resp
SET rept_status=%(rept_status)s,ex_order_status=%(ex_order_status)s,
err_code=%(err_code)s, resp_text=%(resp_text)s, resp_time=localtimestamp
WHERE oms_order_id=%(oms_order_id)s
"""
update_dict={'rept_status':'', 'ex_order_status':'', 'err_code':'', 'resp_text':'', 'oms_order_id':''}
left_buff=b''
while status.value==0:
#TODO:refactor,try recv; and then process the buff
#TODO:when processing buff, abstract the condition to next_message_ready()
try:
recv_buff=resp_queue.get(block=True, timeout=read_timeout)
left_buff=left_buff+recv_buff
if len(left_buff)<Message.header_len+Message.header_len:
continue
(message_type, body_len)=msg.get_message_header(left_buff)
next_message_len=body_len+ Message.header_len+Message.footer_len
while next_message_len<=len(left_buff):
try:
message_processor=factory.build_message_processor(message_type)
message=message_processor.unpack(left_buff)
LogUtil.debug("message:"+message.toString())
if True:#TODO placeholder, check if was order execution report
update_dict['rept_status']='4'
update_dict['ex_order_status']=message.order_status
update_dict['err_code']=message.order_reject_reason
update_dict['resp_text']=message.message_str
update_dict['oms_order_id']=message.client_order_id
update_curs.execute(update_resp, update_dict)
if update_curs.rowcount!=1:
#TODO error handle, rollback?
LogUtil.error("no data update"+message.toString())
else:
conn.commit()
except KeyError:
LogUtil.error("unkown message type:"+str(message_type))
left_buff=left_buff[next_message_len:]
if len(left_buff)<Message.header_len+Message.footer_len:
break
else:
(message_type, body_len)=msg.get_message_header(left_buff)
next_message_len=body_len+ Message.header_len+Message.footer_len
except queue.Empty:
LogUtil.debug("resp_queue no data")
# except KeyError:
# LogUtil.error("unkown message type:"+str(message_type))
else:
LogUtil.info("db_writer finished processing:"+message.toString())
LogUtil.info("db_writer:"+name+" end")
def main():
if len(sys.argv)<2:
print("Usage: tgw.py config_file")
sys.exit(0)
#read mdgw connection config
config_file=sys.argv[1]
run_status=multiprocessing.Value('i', 0)#0:运行;1:退出
message_header_struct = struct.Struct('!II')
logon_struct = struct.Struct('!20s20sI16s32s')
message_footer_struct = struct.Struct('!I')
send_buff = ctypes.create_string_buffer(message_header_struct.size+logon_struct.size+message_footer_struct.size)
bodyLength = logon_struct.size
message_header =(1, bodyLength)
message_header_struct.pack_into(send_buff, 0, *message_header)
LogUtil.get_instance(config_file, "log")
LogUtil.info("Begin")
config=configparser.ConfigParser()
config.read(config_file)
sender_comp=config.get("tgw","sender_comp")
target_comp=config.get("tgw","target_comp")
password=config.get("tgw","password")
app_ver_id=config.get("tgw","app_ver_id")
sender_comp = str.encode(sender_comp.ljust(20))
target_comp = str.encode(target_comp.ljust(20))
password = str.encode(password.ljust(16))
app_ver_id = str.encode(app_ver_id.ljust(32))
logon_body = (sender_comp, target_comp, 30, password, app_ver_id)
logon_struct.pack_into(send_buff, message_header_struct.size, *logon_body)
check_sum = msg.calculate_check_sum(send_buff, message_header_struct.size+logon_struct.size)
message_footer_struct.pack_into(send_buff, message_header_struct.size+logon_struct.size, check_sum)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_ip=config.get("tgw", "ip")
server_port=config.getint("tgw", "port")
#logger initialize
server_address = (server_ip, server_port)
sock.connect(server_address)
sock.settimeout(5)
sock.setblocking(True)
try:
LogUtil.debug(binascii.hexlify(send_buff))
sock.sendall(send_buff)
recv_data = sock.recv(1024)
if not recv_data:
LogUtil.error('Recv error')
else:
LogUtil.info('Recv OK')
LogUtil.info(binascii.hexlify(recv_data))
unpack_recv_data = message_header_struct.unpack_from(recv_data)
LogUtil.info(unpack_recv_data)
#print(binascii.hexlify(recv_data))
if unpack_recv_data[0]==1:
LogUtil.info('Receive Login Confirm!')
#TODO:send report sync
factory=msg.MessageProcessorFactory()
db_file=config.get("message_config", "db_file")
factory.load_from_db(db_file, [])
message_processor=factory.build_message_processor(5)
buff=message_processor.pack("ReportIndex=1")
sock.sendall(buff)
req_queue=multiprocessing.Queue()
resp_queue=multiprocessing.Queue()
dbreader_proc=multiprocessing.Process(target=db_reader, args=('DBReader', run_status, req_queue, config_file))
dbwriter_proc=multiprocessing.Process(target=db_writer, args=('DBWriter', run_status, resp_queue, config_file))
send_proc=multiprocessing.Process(target=tgw_send, args=('TGW sender', run_status, sock, req_queue, config_file))
recv_proc=multiprocessing.Process(target=tgw_recv, args=('TGW receiver', run_status, sock, resp_queue, config_file))
dbreader_proc.start()
dbwriter_proc.start()
send_proc.start()
recv_proc.start()
time.sleep(10)
cmd=input("enter command:")
while cmd!='q':
time.sleep(2)
cmd=input("enter command:")
LogUtil.warning("sending exit cmd")
run_status.value=1
dbreader_proc.join()
dbwriter_proc.join()
send_proc.join()
recv_proc.join()
#发送退出消息并处理应答
logout_message=msg.packLogoutMessage()
sock.sendall(logout_message)
recv_data = sock.recv(1024)
if not recv_data:
LogUtil.error('Recv logout_message error!')
else:
LogUtil.info('Recv logout_message OK')
LogUtil.debug(binascii.hexlify(recv_data))
finally:
sock.close()
LogUtil.info ('End')
def md_responsor(name, status, resp_queue, config_file):
LogUtil.get_instance(config_file, "md_responsor")
LogUtil.info("md_responsor:"+name+" begin")
config = configparser.ConfigParser()
config.read(config_file)
factory = msg.MessageProcessorFactory()
db_file = config.get("message_config", "db_file")
factory.load_from_db(db_file, [])
read_timeout = config.getint("md_responsor", "resp_queue_timeout")
LogUtil.info("read_timeout:"+str(read_timeout))
if read_timeout <= 0 or read_timeout > 60:
read_timeout = 60
LogUtil.info("read_timeout changed to:"+str(read_timeout))
pub_buf = config.get("md_responsor", "pub_buf")
pub_msg = config.get("md_responsor", "pub_msg")
pub_buf_addr = config.get("md_responsor", "pub_buf_addr")
pub_msg_addr = config.get("md_responsor", "pub_msg_addr")
LogUtil.debug("pub_buf:"+pub_buf+",pub_buf_addr:"+pub_buf_addr)
LogUtil.debug("pub_msg:"+pub_msg+",pub_msg_addr:"+pub_msg_addr)
if pub_buf:
buf_ctx = zmq.Context()
buf_sock = buf_ctx.socket(zmq.PUB)
buf_sock.bind(pub_buf_addr)
if pub_msg:
msg_ctx = zmq.Context()
msg_sock = msg_ctx.socket(zmq.PUB)
msg_sock.bind(pub_msg_addr)
left_buff = b''
message_id = 0
while status.value == 0:
# TODO:refactor,try recv; and then process the buff
# TODO:when processing buff, abstract the condition to next_message_ready()
try:
(buf_id, src_time, recv_buff) = resp_queue.get(block=True, timeout=read_timeout)
if pub_buf: # TODO:topic?
buf_sock.send_pyobj((buf_id, src_time, recv_buff))
left_buff = left_buff+recv_buff
if len(left_buff) < Message.header_len+Message.header_len:
continue
(message_type, body_len) = msg.get_message_header(left_buff)
next_message_len = body_len + Message.header_len+Message.footer_len
while next_message_len <= len(left_buff):
try:
message_processor = factory.build_message_processor(message_type)
message = message_processor.unpack(left_buff)
message_id = message_id+1
LogUtil.debug("message:"+message.toString())
if pub_msg: # TODO:topic?
src_time = datetime.datetime.now()
msg_sock.send_pyobj((message_type, message_id, src_time, message.message_str))
except KeyError:
LogUtil.error("unkown message type:"+str(message_type))
except Exception as e:
LogUtil.error(e)
LogUtil.error("other error:"+traceback.print_exc())
left_buff = left_buff[next_message_len:]
if len(left_buff) < Message.header_len+Message.footer_len:
break
else:
(message_type, body_len) = msg.get_message_header(left_buff)
next_message_len = body_len + Message.header_len+Message.footer_len
except queue.Empty:
LogUtil.debug("resp_queue no data")
# except KeyError:
# LogUtil.error("unkown message type:"+str(message_type))
else:
pass
LogUtil.info("md_responsor:"+name+" end")
class LabelUtil(Singleton):
_log = None
# dataPath
def __init__(self):
self._log = LogUtil().getlogger()
self._log.debug("LabelUtil init")
def load_unicode_set(self, unicodeFilePath):
self.byChar = {}
self.byIndex = {}
self.byList = []
self.unicodeFilePath = unicodeFilePath
with open(unicodeFilePath, 'rt', encoding='UTF-8') as data_file:
self.count = 0
for i, r in enumerate(data_file):
ch, inx = r.rsplit(",", 1)
self.byChar[ch] = int(inx)
self.byIndex[int(inx)] = ch
self.byList.append(ch)
self.count += 1
def to_unicode(self, src, index):
# 1 byte
code1 = int(ord(src[index + 0]))
index += 1
result = code1
return result, index
def convert_word_to_grapheme(self, label):
result = []
index = 0
while index < len(label):
(code, nextIndex) = self.to_unicode(label, index)
result.append(label[index])
index = nextIndex
return result, "".join(result)
def convert_word_to_num(self, word):
try:
label_list, _ = self.convert_word_to_grapheme(word)
label_num = []
for char in label_list:
# skip word
if char == "":
pass
else:
label_num.append(int(self.byChar[strQ2B(char)]))
# tuple typecast: read only, faster
return tuple(label_num)
except AttributeError:
self._log.error("unicodeSet is not loaded")
exit(-1)
except KeyError as err:
self._log.error("unicodeSet Key not found: %s" % err)
exit(-1)
def convert_bi_graphemes_to_num(self, word):
label_num = []
for char in word:
# skip word
if char == "":
pass
else:
label_num.append(int(self.byChar[strQ2B(
char.decode("utf-8"))]))
# tuple typecast: read only, faster
return tuple(label_num)
def convert_num_to_word(self, num_list):
try:
label_list = []
for num in num_list:
label_list.append(self.byIndex[num])
return ' '.join(label_list)
except AttributeError:
self._log.error("unicodeSet is not loaded")
exit(-1)
except KeyError as err:
self._log.error("unicodeSet Key not found: %s" % err)
exit(-1)
def get_count(self):
try:
return self.count
except AttributeError:
self._log.error("unicodeSet is not loaded")
exit(-1)
def get_unicode_file_path(self):
try:
return self.unicodeFilePath
except AttributeError:
self._log.error("unicodeSet is not loaded")
exit(-1)
def get_blank_index(self):
return self.byChar["-"]
def get_space_index(self):
return self.byChar["$"]
