class Register(object):
def __init__(self, eventEngine, handler):
self._eventEngine = eventEngine
self._handler = handler
self._logger = Logger()
# 事件
# listen event
self.LISTEN_ACCOUNT_BALANCE_EVENT_TYPE = json.loads(
LISTEN_ACCOUNT_BALANCE_EVENT.substitute())["type"]
self.LISTEN_ACCOUNT_WITHDRAW_EVENT_TYPE = json.loads(
LISTEN_ACCOUNT_WITHDRAW_EVENT.substitute())["type"]
self.LISTEN_MARKET_KLINE_EVENT_TYPE = json.loads(
LISTEN_MARKET_KLINE_EVENT.substitute())["type"]
self.LISTEN_MARKET_TICKER_EVENT_TYPE = json.loads(
LISTEN_MARKET_TICKER_EVENT.substitute())["type"]
self.LISTEN_MARKET_DEPTH_EVENT_TYPE = json.loads(
LISTEN_MARKET_DEPTH_EVENT.substitute())["type"]
# judge event
self.JUDGE_MARKET_DEPTH_EVENT_TYPE = json.loads(
JUDGE_MARKET_DEPTH_EVENT.substitute())["type"]
self.JUDGE_MARKET_KLINE_EVENT_TYPE = json.loads(
JUDGE_MARKET_KLINE_EVENT.substitute())["type"]
self.JUDGE_MARKET_TICKER_EVENT_TYPE = json.loads(
JUDGE_MARKET_TICKER_EVENT.substitute())["type"]
# backtest event
self.BACKTEST_HISTORY_CREAT_EVENT_TYPE = json.loads(
BACKTEST_HISTORY_CREAT_EVENT.substitute())["type"]
# order event
self.ORDER_HISTORY_SYNC_EVENT_TYPE = json.loads(
ORDER_HISTORY_SYNC_EVENT.substitute())["type"]
self.ORDER_HISTORY_CREAT_EVENT_TYPE = json.loads(
ORDER_HISTORY_CREAT_EVENT.substitute())["type"]
# statistic event
self.STATISTIC_JUDGE_EVENT_TYPE = json.loads(
STATISTIC_JUDGE_EVENT.substitute())["type"]
self.STATISTIC_BACKTEST_EVENT_TYPE = json.loads(
STATISTIC_BACKTEST_EVENT.substitute())["type"]
self.STATISTIC_ORDER_EVENT_TYPE = json.loads(
STATISTIC_ORDER_EVENT.substitute())["type"]
# handler
# listen handler
self.LISTEN_ACCOUNT_BALANCE_EVENT_HANDLER = self._handler.handleListenAccountBalanceEvent
self.LISTEN_ACCOUNT_WITHDRAW_EVENT_HANDLER = self._handler.handleListenAccountWithdrawEvent
self.LISTEN_MARKET_KLINE_EVENT_HANDLER = self._handler.handleListenMarketKlineEvent
self.LISTEN_MARKET_TICKER_EVENT_HANDLER = self._handler.handleListenMarketTickerEvent
self.LISTEN_MARKET_DEPTH_EVENT_HANDLER = self._handler.handleListenMarketDepthEvent
# judge handler
self.JUDGE_MARKET_DEPTH_EVENT_HANDLER = self._handler.handleJudgeMarketDepthEvent
self.JUDGE_MARKET_KLINE_EVENT_HANDLER = self._handler.handleJudgeMarketKlineEvent
self.JUDGE_MARKET_TICKER_EVENT_HANDLER = self._handler.handleJudgeMarketTickerEvent
# backtest handler
self.BACKTEST_HISTORY_CREAT_EVENT_HANDLER = self._handler.handleBacktestHistoryCreatEvent
# order handler
self.ORDER_HISTORY_SYNC_EVENT_HANDLER = self._handler.handleOrderHistorySyncEvent
self.ORDER_HISTORY_CREAT_EVENT_HANDLER = self._handler.handleOrderHistoryCreatEvent
# statistic handler
self.STATISTIC_JUDGE_EVENT_HANDLER = self._handler.handleStatisticJudgeEvent
self.STATISTIC_BACKTEST_EVENT_HANDLER = self._handler.handleStatisticBacktestEvent
self.STATISTIC_ORDER_EVENT_HANDLER = self._handler.handleStatisticOrderEvent
def register(self):
self._logger.debug("src.core.engine.register.Register.register")
try:
# 注册事件
self._eventEngine.register(
self.LISTEN_ACCOUNT_BALANCE_EVENT_TYPE,
self.LISTEN_ACCOUNT_BALANCE_EVENT_HANDLER)
self._eventEngine.register(
self.LISTEN_ACCOUNT_WITHDRAW_EVENT_TYPE,
self.LISTEN_ACCOUNT_WITHDRAW_EVENT_HANDLER)
self._eventEngine.register(self.LISTEN_MARKET_KLINE_EVENT_TYPE,
self.LISTEN_MARKET_KLINE_EVENT_HANDLER)
self._eventEngine.register(self.LISTEN_MARKET_TICKER_EVENT_TYPE,
self.LISTEN_MARKET_TICKER_EVENT_HANDLER)
self._eventEngine.register(self.LISTEN_MARKET_DEPTH_EVENT_TYPE,
self.LISTEN_MARKET_DEPTH_EVENT_HANDLER)
self._eventEngine.register(self.JUDGE_MARKET_DEPTH_EVENT_TYPE,
self.JUDGE_MARKET_DEPTH_EVENT_HANDLER)
self._eventEngine.register(self.JUDGE_MARKET_KLINE_EVENT_TYPE,
self.JUDGE_MARKET_KLINE_EVENT_HANDLER)
self._eventEngine.register(self.JUDGE_MARKET_TICKER_EVENT_TYPE,
self.JUDGE_MARKET_TICKER_EVENT_HANDLER)
self._eventEngine.register(
self.BACKTEST_HISTORY_CREAT_EVENT_TYPE,
self.BACKTEST_HISTORY_CREAT_EVENT_HANDLER)
self._eventEngine.register(self.ORDER_HISTORY_SYNC_EVENT_TYPE,
self.ORDER_HISTORY_SYNC_EVENT_HANDLER)
self._eventEngine.register(self.ORDER_HISTORY_CREAT_EVENT_TYPE,
self.ORDER_HISTORY_CREAT_EVENT_HANDLER)
self._eventEngine.register(self.STATISTIC_JUDGE_EVENT_TYPE,
self.STATISTIC_JUDGE_EVENT_HANDLER)
self._eventEngine.register(self.STATISTIC_BACKTEST_EVENT_TYPE,
self.STATISTIC_BACKTEST_EVENT_HANDLER)
self._eventEngine.register(self.STATISTIC_ORDER_EVENT_TYPE,
self.STATISTIC_ORDER_EVENT_HANDLER)
except Exception as err:
errStr = "src.core.engine.register.Register.register: %s" % EngineException(
err)
self._logger.error(errStr)
raise EngineException(err)
def unregister(self):
self._logger.debug("src.core.engine.register.Register.unregister")
try:
# 注销事件
self._eventEngine.unregister(
self.LISTEN_ACCOUNT_BALANCE_EVENT_TYPE,
self.LISTEN_ACCOUNT_BALANCE_EVENT_HANDLER)
self._eventEngine.unregister(
self.LISTEN_ACCOUNT_WITHDRAW_EVENT_TYPE,
self.LISTEN_ACCOUNT_WITHDRAW_EVENT_HANDLER)
self._eventEngine.unregister(
self.LISTEN_MARKET_KLINE_EVENT_TYPE,
self.LISTEN_MARKET_KLINE_EVENT_HANDLER)
self._eventEngine.unregister(
self.LISTEN_MARKET_TICKER_EVENT_TYPE,
self.LISTEN_MARKET_TICKER_EVENT_HANDLER)
self._eventEngine.unregister(
self.LISTEN_MARKET_DEPTH_EVENT_TYPE,
self.LISTEN_MARKET_DEPTH_EVENT_HANDLER)
self._eventEngine.unregister(self.JUDGE_MARKET_KLINE_EVENT_TYPE,
self.JUDGE_MARKET_KLINE_EVENT_HANDLER)
self._eventEngine.unregister(
self.BACKTEST_HISTORY_CREAT_EVENT_TYPE,
self.BACKTEST_HISTORY_CREAT_EVENT_HANDLER)
self._eventEngine.unregister(self.ORDER_HISTORY_SYNC_EVENT_TYPE,
self.ORDER_HISTORY_SYNC_EVENT_HANDLER)
self._eventEngine.unregister(
self.ORDER_HISTORY_CREAT_EVENT_TYPE,
self.ORDER_HISTORY_CREAT_EVENT_HANDLER)
self._eventEngine.unregister(self.STATISTIC_JUDGE_EVENT_TYPE,
self.STATISTIC_JUDGE_EVENT_HANDLER)
self._eventEngine.unregister(self.STATISTIC_BACKTEST_EVENT_TYPE,
self.STATISTIC_BACKTEST_EVENT_HANDLER)
self._eventEngine.unregister(self.STATISTIC_ORDER_EVENT_TYPE,
self.STATISTIC_ORDER_EVENT_HANDLER)
except Exception as err:
errStr = "src.core.engine.register.Register.unregister: %s" % EngineException(
err)
self._logger.error(errStr)
raise EngineException(err)
class EventEngine(object):
# 初始化事件事件驱动引擎
def __init__(self):
# Config init
self.__epoch = Config()._Engine_epoch
self.__maxProcess = Config()._Engine_maxProcess
# 保存事件列表 按优先级不同分别保存
self.__lowEnventQueue = Queue()
self.__mediumEventQueue = Queue()
self.__highEventQueue = Queue()
# 引擎开关
self.__active = Value('b', False)
# 事件处理字典{'event1': [handler1,handler2] , 'event2':[handler3, ...,handler4]}
self.__handlers = {}
# 保存事件处理进程池 控制最大进程数量 以及关闭引擎时处理已启动进程
self.__processPool = Manager().list()
# 保存已执行事件处理状态
self.__status = Status()
# 事件引擎主进程
self.__mainProcess = Process(target=self.__run)
# logger
self.__logger = Logger()
# 执行事件循环
def __run(self):
self.__logger.debug(
"src.core.engine.engine.EventEngine.__mainProcess.__run")
try:
while self.__active.value:
# 执行 Epoch
time.sleep(self.__epoch)
# 控制最大进程数量
if self.getActiveEventNum() > int(self.__maxProcess):
self.__logger.warn(
"src.core.engine.engine.EventEngine.__mainProcess.__run.__eventQueue: Too Many"
)
else:
# 按优先级 获取队列中的事件 超时1秒
event = None
if not self.__highEventQueue.empty():
self.__logger.debug(
"src.core.engine.engine.EventEngine.__mainProcess.__run.__highEventQueue"
)
event = self.__highEventQueue.get(block=False)
while utcnow_timestamp(
) - event.timeStamp > HIGH_PRIORITY_EVENT_TIMEOUT:
self.__status.delEventStatus(event)
if not self.__highEventQueue.empty():
self.__logger.error(
"src.core.engine.engine.EventEngine.__mainProcess.__run.__highEventQueue TIMEOUT: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority,
event.timeStamp, event.args))
event = self.__highEventQueue.get(block=False)
else:
event = None
break
if not self.__mediumEventQueue.empty() and event == None:
self.__logger.debug(
"src.core.engine.engine.EventEngine.__mainProcess.__run.__mediumEventQueue"
)
event = self.__mediumEventQueue.get(block=False)
while utcnow_timestamp(
) - event.timeStamp > MEDIUM_PRIORITY_EVENT_TIMEOUT:
self.__status.delEventStatus(event)
if not self.__mediumEventQueue.empty():
self.__logger.error(
"src.core.engine.engine.EventEngine.__mainProcess.__run.__mediumEventQueue TIMEOUT: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority,
event.timeStamp, event.args))
event = self.__mediumEventQueue.get(
block=False)
else:
event = None
break
if not self.__lowEnventQueue.empty() and event == None:
self.__logger.debug(
"src.core.engine.engine.EventEngine.__mainProcess.__run.__lowEnventQueue"
)
event = self.__lowEnventQueue.get(block=False)
while utcnow_timestamp(
) - event.timeStamp > LOW_PRIORITY_EVENT_TIMEOUT:
self.__status.delEventStatus(event)
if not self.__lowEnventQueue.empty():
self.__logger.error(
"src.core.engine.engine.EventEngine.__mainProcess.__run.__lowEnventQueue TIMEOUT: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority,
event.timeStamp, event.args))
event = self.__lowEnventQueue.get(block=False)
else:
event = None
break
# 事件队列非空
if not event == None:
# 执行事件
self.__logger.debug(
"src.core.engine.engine.EventEngine.__mainProcess.__run.__eventQueue: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority,
event.timeStamp, event.args))
self.__process(event)
else:
self.__logger.debug(
"src.core.engine.engine.EventEngine.__mainProcess.__run.__eventQueue: empty"
)
# 定期清理进程池
if len(self.__processPool) > self.__maxProcess:
for (_id, _pid) in self.__processPool:
if not psutil.pid_exists(_pid):
self.__processPool.remove((_id, _pid))
# break out while
# 终止所有事件处理进程
for (_id, _pid) in self.__processPool:
if psutil.pid_exists(_pid):
_p = psutil.Process(_pid)
_p.terminate()
self.__processPool.remove((_id, _pid))
except Exception as err:
errStr = "src.core.engine.engine.EventEngine.__mainProcess.__run: exception err=%s" % EngineException(
err)
raise EngineException(errStr)
# 执行事件
def __process(self, event):
self.__logger.debug(
"src.core.engine.engine.EventEngine.__mainProcess.__run.__process: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
try:
if event.type in self.__handlers:
for handler in self.__handlers[event.type]:
# 开一个进程去异步处理
p = Process(
target=handler, args=(event, self.__status.delEventStatus))
# 运行进程
p.start()
# 同步抄送至事件运行状态表格里
self.__status.addEventStatus(event.id)
# 保存到进程池
self.__processPool.append((event.id, p.pid))
except Exception as err:
errStr = "src.core.engine.engine.EventEngine.__mainProcess.__run.__process: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}, exception err=%s" % (
event.id, event.type, event.priority, event.timeStamp, event.args, EngineException(err))
raise EngineException(errStr)
# 开启事件引擎
def start(self):
self.__logger.debug("src.core.engine.engine.EventEngine.start")
try:
if not self.__active.value:
self.__active.value = True
# 开启事件引擎主进程
self.__mainProcess.start()
except Exception as err:
errStr = "src.core.engine.engine.EventEngine.start: exception err=%s" % EngineException(
err)
raise EngineException(errStr)
# 暂停事件引擎
def stop(self):
self.__logger.debug("src.core.engine.engine.EventEngine.stop")
try:
if self.__active.value:
# 将事件管理器设为停止
self.__active.value = False
# 等待事件引擎主进程退出
self.__mainProcess.join()
except Exception as err:
errStr = "src.core.engine.engine.EventEngine.stop: exception err=%s" % EngineException(
err)
raise EngineException(errStr)
# 终止事件引擎
def terminate(self):
self.__logger.debug("src.core.engine.engine.EventEngine.terminate")
try:
# 将事件管理器设为停止
self.__active.value = False
# 引擎主进程直接退出
self.__mainProcess.terminate()
except Exception as err:
errStr = "src.core.engine.engine.EventEngine.terminate: exception err=%s" % EngineException(
err)
raise EngineException(errStr)
# 注册事件
def register(self, type, handler):
self.__logger.debug(
"src.core.engine.engine.EventEngine.register: {type:%s, handler:%s}"
% (type, handler))
# 尝试获取该事件类型对应的处理函数列表，若无则创建
try:
try:
handlerList = self.__handlers[type]
except KeyError:
handlerList = []
# 若要注册的处理器不在该事件的处理器列表中，则注册该事件
if handler not in handlerList:
handlerList.append(handler)
self.__handlers[type] = handlerList
except (KeyError, Exception) as err:
errStr = "src.core.engine.engine.EventEngine.register: {type:%s, handler:%s}, exception err=%s" % (type, handler, EngineException(
err))
raise EngineException(errStr)
# 注销事件
def unregister(self, type, handler):
self.__logger.debug(
"src.core.engine.engine.EventEngine.unregister: {type:%s, handler:%s}"
% (type, handler))
# 尝试获取该事件类型对应的处理函数列表，若无则忽略该次注销请求
try:
handlerList = self.__handlers[type]
# 如果该函数存在于列表中，则移除
if handler in handlerList:
handlerList.remove(handler)
# 如果函数列表为空，则从引擎中移除该事件类型
if not handlerList:
del self.__handlers[type]
except (KeyError, Exception) as err:
errStr = "src.core.engine.engine.EventEngine.unregister: {type:%s, handler:%s}, exception err=%s" % (
type, handler, EngineException(err))
raise EngineException(errStr)
# 发送事件
def sendEvent(self, event):
self.__logger.debug(
"src.core.engine.engine.EventEngine.sendEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}" % (
event.id, event.type, event.priority, event.timeStamp, event.args))
try:
# 发送事件 像队列里存入事件
if event.priority == LOW_PRIORITY_EVENT:
self.__lowEnventQueue.put(event)
if event.priority == MEDIUM_PRIORITY_EVENT:
self.__mediumEventQueue.put(event)
if event.priority == HIGH_PRIORITY_EVENT:
self.__highEventQueue.put(event)
except Exception as err:
errStr = "src.core.engine.engine.EventEngine.sendEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}, exception err=%s" % (
event.id, event.type, event.priority, event.timeStamp, event.args, EngineException(err))
raise EngineException(errStr)
# kill 事件
def killEvent(self, id):
self.__logger.debug(
"src.core.engine.engine.EventEngine.killEvent: {id=%s}" % id)
try:
# 查询事件状态
status = self.getEventStatus(id)
# 删除事件进程
if not status == DONE_STATUS_EVENT:
for (_id, _pid) in self.__processPool:
if _id == id:
if psutil.pid_exists(_pid):
_p = psutil.Process(_pid)
_p.terminate()
# 更新事件状态
self.__processPool.remove((_id, _pid))
self.__status.delEventStatus(id)
# 确认事件状态
status = self.getEventStatus(id)
if not status == DONE_STATUS_EVENT:
return False
return True
except Exception as err:
errStr = "src.core.engine.engine.EventEngine.killEvent: {id=%s}, exception err=%s" % (
id, EngineException(err))
raise EngineException(errStr)
# 获取事件ID
def getEventID(self):
try:
id = self.__status.calcEventID()
self.__logger.debug(
"src.core.engine.engine.EventEngine.getEventID: {id=%s}" % id)
return id
except Exception as err:
errStr = "src.core.engine.engine.EventEngine.getEventID: exception err=%s" % EngineException(
err)
raise EngineException(errStr)
def getEventStatus(self, id):
try:
status = QUEUE_STATUS_EVENT
res = self.__status.getActiveStatusTable()
if not res == []:
if id in res:
status = ACTIVE_STATUS_EVENT
res = self.__status.getDoneStatusTable()
if not res == []:
if id in res:
status = DONE_STATUS_EVENT
# 进程池二次确认
if status == ACTIVE_STATUS_EVENT:
for (_id, _pid) in self.__processPool:
if _id == id:
if not psutil.pid_exists(_pid):
status = DONE_STATUS_EVENT
self.__status.delEventStatus(_id)
self.__processPool.remove((_id, _pid))
self.__logger.debug(
"src.core.engine.engine.EventEngine.getEventStatus: {id=%s, status=%s}"
% (id, status))
return status
except Exception as err:
errStr = "src.core.engine.engine.EventEngine.getEventStatus: exception err=%s" % EngineException(
err)
raise EngineException(errStr)
def getActiveEventNum(self):
try:
res = self.__status.calcActiveEventNum()
self.__logger.debug(
"src.core.engine.engine.EventEngine.getActiveEventNum: {res=%s}" %
res)
return res
except Exception as err:
errStr = "src.core.engine.engine.EventEngine.getActiveEventNum: exception err=%s" % EngineException(
err)
raise EngineException(errStr)
def getActiveEventTable(self):
try:
res = self.__status.getActiveStatusTable()
self.__logger.debug(
"src.core.engine.engine.EventEngine.getActiveEventTable: {res=%s}" %
res)
return res
except Exception as err:
errStr = "src.core.engine.engine.EventEngine.getActiveEventTable: exception err=%s" % EngineException(
err)
raise EngineException(errStr)
def getDoneEventTable(self):
try:
res = self.__status.getDoneStatusTable()
self.__logger.debug(
"src.core.engine.engine.EventEngine.getDoneEventTable: {res=%s}" %
res)
return res
except Exception as err:
errStr = "src.core.engine.engine.EventEngine.getDoneEventTable: exception err=%s" % EngineException(
err)
raise EngineException(errStr)
class Handler(object):
def __init__(self, eventEngine):
self._engine = eventEngine
self._logger = Logger()
# Account Balance 事件
def handleListenAccountBalanceEvent(self, event, callback):
# 接收事件
self._logger.debug(
"src.core.engine.handler.Handler.handleListenAccountBalanceEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
[exchange] = event.args
exchange = str_to_list(exchange)
try:
db = DB()
db.insertAccountBalanceHistory(exchange)
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleListenAccountBalanceEvent: { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)
# Account Withdraw 事件
def handleListenAccountWithdrawEvent(self, event, callback):
# 接收事件
self._logger.debug(
"src.core.engine.handler.Handler.handleListenAccountWithdrawEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
[exchange, asset] = event.args
exchange = str_to_list(exchange)
try:
db = DB()
db.insertAccountWithdrawHistoryAsset(exchange, asset)
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleListenAccountWithdrawEvent: { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)
# Market Depth 事件
def handleListenMarketDepthEvent(self, event, callback):
# 接收事件
self._logger.debug(
"src.core.engine.handler.Handler.handleListenMarketDepthEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
[exchange, fSymbol, tSymbol, limit] = event.args
exchange = str_to_list(exchange)
try:
db = DB()
db.insertMarketDepth(exchange, fSymbol, tSymbol, limit)
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleListenDepthEvent { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)
# Market Kline 事件
def handleListenMarketKlineEvent(self, event, callback):
# 接收事件
self._logger.debug(
"src.core.engine.handler.Handler.handleListenMarketKlineEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
[exchange, fSymbol, tSymbol, interval, start, end] = event.args
exchange = str_to_list(exchange)
try:
db = DB()
db.insertMarketKline(exchange, fSymbol, tSymbol, interval, start,
end)
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleListenKlineEvent: { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)
# Market ticker 事件
def handleListenMarketTickerEvent(self, event, callback):
self._logger.debug(
"src.core.engine.handler.Handler.handleListenMarketTickerEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
# 接收事件
[exchange, fSymbol, tSymbol, aggDepth] = event.args
exchange = str_to_list(exchange)
try:
db = DB()
db.insertMarketTicker(exchange, fSymbol, tSymbol, aggDepth)
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleListenTickerEvent: { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)
# Judge 事件
def handleJudgeMarketDepthEvent(self, event, callback):
self._logger.debug(
"src.core.engine.handler.Handler.handleJudgeMarketDepthEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
# 接收事件
[args] = event.args
try:
pass
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleJudgeMarketDepthEvent: { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)
def handleJudgeMarketKlineEvent(self, event, callback):
self._logger.debug(
"src.core.engine.handler.Handler.handleJudgeMarketKlineEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
# 接收事件
[args] = event.args
try:
pass
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleJudgeMarketKlineEvent: { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)
def handleJudgeMarketTickerEvent(self, event, callback):
self._logger.debug(
"src.core.engine.handler.Handler.handleJudgeMarketTickerEvent: " +
event.type)
# 接收事件
[exchange, types] = event.args
exchange = str_to_list(exchange)
types = str_to_list(types)
try:
db = DB()
calc = Calc()
resInfoSymbol = pd.DataFrame(db.getViewMarketSymbolPairs(exchange))
prs = []
# calc dis type
if TYPE_DIS in types:
signalDis = calc.calcJudgeMarketTickerDis(
exchange, TYPE_DIS_THRESHOLD, resInfoSymbol)
if not signalDis == []:
db.insertJudgeMarketTickerDis(signalDis)
# calc tra type
if TYPE_TRA in types:
signalTra = calc.calcJudgeMarketTickerTra(
exchange, TYPE_TRA_THRESHOLD, resInfoSymbol)
if not signalTra == []:
db.insertJudgeMarketTickerTra(signalTra)
# calc pair type
if TYPE_PAIR in types:
signalPair = calc.calcJudgeMarketTickerPair(
exchange, TYPE_PAIR_THRESHOLD, resInfoSymbol)
if not signalPair == []:
db.insertJudgeMarketTickerPair(signalPair)
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleJudgeMarketTickerEvent: { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)
# Backtest 事件
def rollbackHandleBacktestHistoryCreatEvent(self, sgn, pdOrders, exchange,
resInfoSymbol):
self._logger.debug(
"src.core.engine.handler.Handler.rollbackHandleBacktestHistoryCreatEvent"
)
try:
db = DB()
# update orders sgn status
infoOrders = []
if not pdOrders.empty:
for server in exchange:
res = []
orderIDs = pdOrders[(
pdOrders['server'] == server)]['order_id'].tolist()
res = db.getTradeBacktestHistoryServerOrder([server],
orderIDs)
if not res == []:
infoOrders.extend(res)
if not infoOrders == []:
infoOrders = pd.DataFrame(infoOrders)
isError = SIGNAL_MAX_NUM
while isError > 0:
try:
isError = isError - 1
sgn.backtestUpdateSignalStatusByOrders(
infoOrders, resInfoSymbol)
isError = 0
except Exception as err:
self._logger.warn('rollback failed, will try again...')
if isError > 0:
raise Exception(
'rollback error, start_base assets may loss control.')
# insert db signals
db.insertSignalTradeDis(sgn.signals(exchange, [TYPE_DIS]),
SIGNAL_BACKTEST)
db.insertSignalTradeTra(sgn.signals(exchange, [TYPE_TRA]),
SIGNAL_BACKTEST)
db.insertSignalTradePair(sgn.signals(exchange, [TYPE_PAIR]),
SIGNAL_BACKTEST)
# rollback assets
isError = SIGNAL_MAX_NUM
while isError > 0:
isError = isError - 1
# calc after orders
afterOrders = []
isSubError = SIGNAL_MAX_NUM
while isSubError > 0:
try:
isSubError = isSubError - 1
afterOrders = sgn.backtestSignalsAfterTrade(
resInfoSymbol)
isSubError = 0
except Exception as err:
self._logger.warn('rollback failed, will try again...')
if isSubError > 0:
raise Exception(
'rollback error, start_base assets may loss control.')
# calc afterExecOrders
afterExecOrders = []
if not afterOrders == []:
for order in afterOrders:
identify = identify + 1
isSubError = SIGNAL_MAX_NUM
res = []
while isSubError > 0:
try:
isSubError = isSubError - 1
res = db.insertCreatTradeBacktestHistory(
order['server'], order['fSymbol'],
order['tSymbol'], order['ask_or_bid'],
order['price'], order['quantity'],
order['ratio'], order['type'],
order['signal_id'], order['group_id'],
identify)
isSubError = 0
except Exception as err:
self._logger.warn(
'rollback failed, will try again...')
if not res == []:
afterExecOrders.extend(res)
if isSubError > 0:
raise Exception(
'rollback error, start_base assets may loss control.'
)
# calc afterInfoOrders
afterInfoOrders = []
if not afterExecOrders == []:
afterExecOrders = pd.DataFrame(afterExecOrders)
for server in exchange:
res = []
orderIDs = preExecOrders[(
preExecOrders['server'] == server
)]['order_id'].tolist()
res = db.getTradeBacktestHistoryServerOrder([server],
orderIDs)
if not res == []:
afterInfoOrders.extend(res)
# update signals status
if not afterInfoOrders == []:
afterInfoOrders = pd.DataFrame(afterInfoOrders)
isSubError = SIGNAL_MAX_NUM
while isSubError > 0:
try:
isSubError = isSubError - 1
sgn.backtestUpdateSignalStatusByOrders(
afterInfoOrders, resInfoSymbol)
isSubError = 0
except Exception as err:
self._logger.warn(
'rollback failed, will try again...')
if isSubError > 0:
raise Exception(
'rollback error, start_base assets may loss control.'
)
# insert db signals
db.insertSignalTradeDis(sgn.signals(exchange, [TYPE_DIS]),
SIGNAL_BACKTEST)
db.insertSignalTradeTra(sgn.signals(exchange, [TYPE_TRA]),
SIGNAL_BACKTEST)
db.insertSignalTradePair(
sgn.signals(exchange, [TYPE_PAIR]), SIGNAL_BACKTEST)
# update isError
isMore = sgn.backtestSignalsIsRunMore(resInfoSymbol)
if not isMore:
isError = 0
except Exception as err:
errStr = "src.core.engine.handler.Handler.rollbackHandleBacktestHistoryCreatEvent, err=%s" % err
self._logger.critical(errStr)
def handleBacktestHistoryCreatEvent(self, event, callback):
self._logger.debug(
"src.core.engine.handler.Handler.handleBacktestHistoryCreatEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
# 接收事件
[exchange, signals, timeout] = event.args
exchange = str_to_list(exchange)
signals = str_to_list(signals)
timeout = float(timeout)
# 处理事件
try:
db = DB()
sgn = Signal(signals)
resInfoSymbol = pd.DataFrame(db.getViewMarketSymbolPairs(exchange))
# 0. start
identify = 0
startTime = time.time()
str = "src.core.engine.handler.Handler.handleBacktestHistoryCreatEvent: { type=%s, priority=%s, args=%s }" % (
event.type, event.priority, event.args)
warnStr = Template(
", err=$err, backtest trade filed at $here, will try again...")
errStr = Template(
"BACKTEST TRADE ERROR. pre trade failed at $here.")
########################################
# 1. pre trade
# 1.1 calc pre orders
preOrders = []
isError = SIGNAL_MAX_NUM
while isError > 0:
try:
isError = isError - 1
preOrders = sgn.backtestSignalsPreTrade(resInfoSymbol)
isError = 0
except Exception as err:
self._logger.warn(str + warnStr.substitute(
err=err, here='1.1 calc pre orders'))
if isError > 0:
raise Exception(errStr.substitute(here='1.1 calc pre orders'))
# print('1. pre signals preOrders:\n%s' % preOrders)
# 1.2 calc preExecOrders
preExecOrders = []
if not preOrders == []:
for order in preOrders:
identify = identify + 1
isError = SIGNAL_MAX_NUM
res = []
while isError > 0:
try:
isError = isError - 1
res = db.insertCreatTradeBacktestHistory(
order['server'], order['fSymbol'],
order['tSymbol'], order['ask_or_bid'],
order['price'], order['quantity'],
order['ratio'], order['type'],
order['signal_id'], order['group_id'],
identify)
isError = 0
except Exception as err:
self._logger.warn(str + warnStr.substitute(
err=err, here='1.2 excute preOrders'))
if not res == []:
preExecOrders.extend(res)
if isError > 0:
# rollback:
pdOrders = pd.DataFrame(preExecOrders)
self.rollbackHandleBacktestHistoryCreatEvent(
sgn, pdOrders, exchange, resInfoSymbol)
raise Exception(
errStr.substitute(here='1.2 excute preOrders'))
# print('1. pre signals preExecOrders:\n%s' % preExecOrders)
# 1.3 calc preInfoOrders
preInfoOrders = []
if not preExecOrders == []:
preExecOrders = pd.DataFrame(preExecOrders)
for server in exchange:
res = []
orderIDs = preExecOrders[(preExecOrders['server'] == server
)]['order_id'].tolist()
res = db.getTradeBacktestHistoryServerOrder([server],
orderIDs)
if not res == []:
preInfoOrders.extend(res)
# print('1. pre signals preInfoOrders:\n%s' % preInfoOrders)
# 1.4 update signals status
if not preInfoOrders == []:
preInfoOrders = pd.DataFrame(preInfoOrders)
isError = SIGNAL_MAX_NUM
while isError > 0:
try:
isError = isError - 1
sgn.backtestUpdateSignalStatusByOrders(
preInfoOrders, resInfoSymbol)
isError = 0
except Exception as err:
self._logger.warn(str + warnStr.substitute(
err=err, here='1.4 update signals status'))
if isError > 0:
# rollback:
pdOrders = preExecOrders
self.rollbackHandleBacktestHistoryCreatEvent(
sgn, pdOrders, exchange, resInfoSymbol)
raise Exception(
errStr.substitute(here='1.4 update signals status'))
# insert db signals
db.insertSignalTradeDis(sgn.signals(exchange, [TYPE_DIS]),
SIGNAL_BACKTEST)
db.insertSignalTradeTra(sgn.signals(exchange, [TYPE_TRA]),
SIGNAL_BACKTEST)
db.insertSignalTradePair(sgn.signals(exchange, [TYPE_PAIR]),
SIGNAL_BACKTEST)
# print('1. pre signals after update:\n%s' % sgn.signals())
########################################
# 2. run trade
isError = True
while isError and (time.time() - startTime < timeout
or timeout == 0):
# 2.1 calc run orders
runOrders = []
isSubError = SIGNAL_MAX_NUM
while isSubError > 0 and (time.time() - startTime < timeout
or timeout == 0):
try:
isSubError = isSubError - 1
runOrders = sgn.backtestSignalsRunTrade(resInfoSymbol)
isSubError = 0
except Exception as err:
self._logger.warn(str + warnStr.substitute(
err=err, here='2.1 calc run orders'))
if isSubError > 0:
# rollback:
pdOrders = []
self.rollbackHandleBacktestHistoryCreatEvent(
sgn, pdOrders, exchange, resInfoSymbol)
raise Exception(
errStr.substitute(here='2.1 calc run orders'))
# print('2. run signals runOrders:\n%s' % runOrders)
# 2.2 calc runExecOrders
runExecOrders = []
if not runOrders == []:
for order in runOrders:
identify = identify + 1
isSubError = SIGNAL_MAX_NUM
res = []
while isSubError > 0 and (time.time() - startTime <
timeout or timeout == 0):
try:
isSubError = isSubError - 1
res = db.insertCreatTradeBacktestHistory(
order['server'], order['fSymbol'],
order['tSymbol'], order['ask_or_bid'],
order['price'], order['quantity'],
order['ratio'], order['type'],
order['signal_id'], order['group_id'],
identify)
isSubError = 0
except Exception as err:
self._logger.warn(str + warnStr.substitute(
err=err, here='2.2 execute runOrders'))
if not res == []:
runExecOrders.extend(res)
if isSubError > 0:
# rollback:
pdOrders = pd.DataFrame(runExecOrders)
self.rollbackHandleBacktestHistoryCreatEvent(
sgn, pdOrders, exchange, resInfoSymbol)
raise Exception(
errStr.substitute(here='2.2 execute runOrders'))
# print('2. run signals runExecOrders:\n%s' % runExecOrders)
# 2.3 calc runInfoOrders
runInfoOrders = []
if not runExecOrders == []:
runExecOrders = pd.DataFrame(runExecOrders)
for server in exchange:
res = []
orderIDs = runExecOrders[(
runExecOrders['server'] == server
)]['order_id'].tolist()
res = db.getTradeBacktestHistoryServerOrder([server],
orderIDs)
if not res == []:
runInfoOrders.extend(res)
# print('2. run signals runInfoOrders:\n%s' % runInfoOrders)
# 2.4 update signals status
if not runInfoOrders == []:
runInfoOrders = pd.DataFrame(runInfoOrders)
isSubError = SIGNAL_MAX_NUM
while isSubError > 0 and (time.time() - startTime < timeout
or timeout == 0):
try:
isSubError = isSubError - 1
sgn.backtestUpdateSignalStatusByOrders(
runInfoOrders, resInfoSymbol)
isSubError = 0
except Exception as err:
self._logger.warn(str + warnStr.substitute(
err=err, here='2.4 update signals status'))
if isSubError > 0:
# rollback:
pdOrders = runExecOrders
self.rollbackHandleBacktestHistoryCreatEvent(
sgn, pdOrders, exchange, resInfoSymbol)
raise Exception(
errStr.substitute(
here='2.4 update signals status'))
# insert db signals
db.insertSignalTradeDis(sgn.signals(exchange, [TYPE_DIS]),
SIGNAL_BACKTEST)
db.insertSignalTradeTra(sgn.signals(exchange, [TYPE_TRA]),
SIGNAL_BACKTEST)
db.insertSignalTradePair(
sgn.signals(exchange, [TYPE_PAIR]), SIGNAL_BACKTEST)
# print('2. run signals after update:\n%s' % sgn.signals())
# 2.5 update isError
isMore = sgn.backtestSignalsIsRunMore(resInfoSymbol)
if not isMore:
isError = False
########################################
# 3. after trade
isError = SIGNAL_MAX_NUM
while isError > 0:
isError = isError - 1
# 3.1 calc after orders
afterOrders = []
isSubError = SIGNAL_MAX_NUM
while isSubError > 0:
try:
isSubError = isSubError - 1
afterOrders = sgn.backtestSignalsAfterTrade(
resInfoSymbol)
isSubError = 0
except Exception as err:
self._logger.warn(str + warnStr.substitute(
err=err, here='3.1 calc after orders'))
if isSubError > 0:
# rollback
pdOrders = []
self.rollbackHandleBacktestHistoryCreatEvent(
sgn, pdOrders, exchange, resInfoSymbol)
raise Exception(
errStr.substitute(here='3.1 calc after orders'))
# print('3. after signals afterOrders:\n%s' % afterOrders)
# 3.2 calc afterExecOrders
afterExecOrders = []
if not afterOrders == []:
for order in afterOrders:
identify = identify + 1
isSubError = SIGNAL_MAX_NUM
res = []
while isSubError > 0:
try:
isSubError = isSubError - 1
res = db.insertCreatTradeBacktestHistory(
order['server'], order['fSymbol'],
order['tSymbol'], order['ask_or_bid'],
order['price'], order['quantity'],
order['ratio'], order['type'],
order['signal_id'], order['group_id'],
identify)
isSubError = 0
except Exception as err:
self._logger.warn(str + warnStr.substitute(
err=err, here='3.2 excute preOrders'))
if not res == []:
afterExecOrders.extend(res)
if isSubError > 0:
# rollback:
pdOrders = pd.DataFrame(afterExecOrders)
self.rollbackHandleBacktestHistoryCreatEvent(
sgn, pdOrders, exchange, resInfoSymbol)
raise Exception(
errStr.substitute(here='3.2 excute preOrders'))
# print('3. after signals afterExecOrders:\n%s' % afterExecOrders)
# 3.3 calc afterInfoOrders
afterInfoOrders = []
if not afterExecOrders == []:
afterExecOrders = pd.DataFrame(afterExecOrders)
for server in exchange:
res = []
orderIDs = preExecOrders[(
preExecOrders['server'] == server
)]['order_id'].tolist()
res = db.getTradeBacktestHistoryServerOrder([server],
orderIDs)
if not res == []:
afterInfoOrders.extend(res)
# print('3. after signals afterInfoOrders:\n%s' % afterInfoOrders)
# 3.4 update signals status
if not afterInfoOrders == []:
afterInfoOrders = pd.DataFrame(afterInfoOrders)
isSubError = SIGNAL_MAX_NUM
while isSubError > 0:
try:
isSubError = isSubError - 1
sgn.backtestUpdateSignalStatusByOrders(
afterInfoOrders, resInfoSymbol)
isSubError = 0
except Exception as err:
self._logger.warn(str + warnStr.substitute(
err=err, here='3.4 update signals status'))
if isSubError > 0:
# rollback:
pdOrders = afterExecOrders
self.rollbackHandleBacktestHistoryCreatEvent(
sgn, pdOrders, exchange, resInfoSymbol)
raise Exception(
errStr.substitute(
here='3.4 update signals status'))
# insert db signals
db.insertSignalTradeDis(sgn.signals(exchange, [TYPE_DIS]),
SIGNAL_BACKTEST)
db.insertSignalTradeTra(sgn.signals(exchange, [TYPE_TRA]),
SIGNAL_BACKTEST)
db.insertSignalTradePair(
sgn.signals(exchange, [TYPE_PAIR]), SIGNAL_BACKTEST)
# print('3. after signals after update:\n%s' % sgn.signals())
# 3.5 update isError
isMore = sgn.backtestSignalsIsRunMore(resInfoSymbol)
if not isMore:
isError = 0
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleBacktestHistoryCreatEvent: { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)
# Order 事件
def handleOrderHistorySyncEvent(self, event, callback):
self._logger.debug(
"src.core.engine.handler.Handler.handleOrderHistorySyncEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
# 接收事件
[exchange, fSymbol, tSymbol, limit, ratio] = event.args
exchange = str_to_list(exchange)
try:
db = DB()
db.insertSyncTradeOrderHistory(exchange, fSymbol, tSymbol, limit,
ratio)
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleOrderHistorySyncEvent: { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)
def handleOrderHistoryCreatEvent(self, event, callback):
self._logger.debug(
"src.core.engine.handler.Handler.handleOrderHistoryCreatEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
# 接收事件
pass
# Statistic 事件
def handleStatisticJudgeEvent(self, event, callback):
self._logger.debug(
"src.core.engine.handler.Handler.handleStatisticJudgeEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
# 接收事件
[exchange, types] = event.args
exchange = str_to_list(exchange)
types = str_to_list(types)
try:
db = DB()
calc = Calc()
# calc dis type
if TYPE_DIS in types:
statisticDis = calc.calcStatisticJudgeMarketTickerDis(
exchange, TYPE_DIS_TIMEWINDOW)
if not statisticDis == []:
db.insertStatisticJudgeMarketTickerDis(statisticDis)
# calc tra type
if TYPE_TRA in types:
statisticTra = calc.calcStatisticJudgeMarketTickerTra(
exchange, TYPE_TRA_TIMEWINDOW)
if not statisticTra == []:
db.insertStatisticJudgeMarketTickerTra(statisticTra)
# calc pair type
if TYPE_PAIR in types:
statisticPair = calc.calcStatisticJudgeMarketTickerPair(
exchange, TYPE_PAIR_TIMEWINDOW)
if not statisticPair == []:
db.insertStatisticJudgeMarketTickerPair(statisticPair)
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleStatisticJudgeEvent: { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)
def handleStatisticBacktestEvent(self, event, callback):
self._logger.debug(
"src.core.engine.handler.Handler.handleStatisticBacktestEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
# 接收事件
[signals] = event.args
signals = str_to_list(signals)
for signal in signals:
signal['status_assets'] = json.loads(
json_reverse(signal['status_assets']))
try:
db = DB()
calc = Calc()
statistic = calc.calcStatisticTradeBacktestHistory(signals)
db.insertStatisticTradeBacktestHistory(statistic)
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleStatisticBacktestEvent: { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)
def handleStatisticOrderEvent(self, event, callback):
self._logger.debug(
"src.core.engine.handler.Handler.handleStatisticOrderEvent: {id=%s, type=%s, priority=%s, timeStamp=%s, args=%s}"
% (event.id, event.type, event.priority, event.timeStamp,
event.args))
# 接收事件
[signals] = event.args
signals = str_to_list(signals)
for signal in signals:
signal['status_assets'] = json.loads(
json_reverse(signal['status_assets']))
try:
db = DB()
calc = Calc()
statistic = calc.calcStatisticTradeOrderHistory(signals)
db.insertStatisticTradeOrderHistory(statistic)
except (DBException, CalcException, EngineException, Exception) as err:
errStr = "src.core.engine.handler.Handler.handleStatisticOrderEvent: { type=%s, priority=%s, args=%s }, err=%s" % (
event.type, event.priority, event.args, err)
self._logger.error(errStr)
callback(event.id)