def _revertOpenTrades(self, strategyExecId):
trades = TradeManager.getAllTrades(strategyExecId = strategyExecId, tradeState = REF_TRADE_STATE.OPENED)
self.log.debug('')
self.log.debug('Open trades to be reversed: ' + str(len(trades)))
for trade in trades:
self.log.debug('Reverting open trade ' + str(trade.trade_id))
orders = TradeManager.getAllOrders(tradeId = trade.trade_id, orderState = [REF_ORDER_STATE.FILLED, REF_ORDER_STATE.PARTIALLY_FILLED])
for dbOrder in orders:
order = Order()
order.setFromEntity(dbOrder)
self.log.debug('Reverting order ' + str(order))
basePosition = self.portfolioManager.getPosition(trade.base_asset)
quotePosition = self.portfolioManager.getPosition(trade.quote_asset)
if order.getOrderSide() == OrderSide.BUY:
basePosition.setFree(basePosition.getFree() - float(order.getQty()))
quotePosition.setFree(quotePosition.getFree() + float(order.getQty() * order.getPrice()))
else:
basePosition.setFree(basePosition.getFree() + float(order.getQty()))
quotePosition.setFree(quotePosition.getFree() - float(order.getQty() * order.getPrice()))
# transaction nesting is required to ensure that session invalidation in below method does not
# detach objects (trades and orders) loaded and iterated within this method
Db.Session().begin_nested()
self.exchangeDataListener.processPortfolioUpdate(basePosition)
Db.Session().begin_nested()
self.exchangeDataListener.processPortfolioUpdate(quotePosition)
def createCretenExecDetl(cretenExecId, inputConf):
ced = CretenExecDetl()
ced.creten_exec_id = cretenExecId
ced.exec_state = str(
REF_EXEC_STATE.STARTED) # conversion to string due to SQLite
ced.conf = json.dumps(inputConf)
ced.interval = inputConf['interval']
try:
ced.start_tmstmp = datetime.strptime(inputConf['start_tmstmp'],
BACKTEST_TMSTMP_FORMAT)
except KeyError:
pass
try:
ced.end_tmstmp = datetime.strptime(inputConf['end_tmstmp'],
BACKTEST_TMSTMP_FORMAT)
except KeyError:
pass
try:
ced.dscp = inputConf['dscp']
except KeyError:
pass
Db.Session().add(ced)
Db.Session().flush()
return ced
def storeOrder(self, order): Db.Session().add(order) # flush in order to generate db maintained sequence Db.Session().flush() # update internal reference only when we got new order_id if not order.int_order_ref: order.int_order_ref = Settings.ORDER_REFERENCE_PREFIX + str(order.order_id) # track order updates in the cache. o = Order() o.setFromEntity(order) self.liveOrderCache[order.order_id] = o
def __init__(self, cretenExecDetlId, pair, strategyConf, exchangeClient,
marketDataManager, marketRulesManager, portfolioManager,
orderManager):
self.cretenExecDetlId = cretenExecDetlId
self.pair = pair
self.strategyConf = strategyConf
self.exchangeClient = exchangeClient
self.marketDataManager = marketDataManager
self.marketRulesManager = marketRulesManager
self.portfolioManager = portfolioManager
self.orderManager = orderManager
try:
self.params = strategyConf['params']
except KeyError:
self.params = None
try:
self.tradeCloseType = strategyConf['trade_close_type']
except KeyError:
self.tradeCloseType = Settings.client['TRADES'][
'DEFAULT_TRADE_CLOSE_TYPE']
self.positionAllocator = PositionAllocator(portfolioManager, None)
self.log = Logger()
with Db.session_scope():
se = ExecManager.createStrategyExec(cretenExecDetlId, strategyConf,
self.tradeCloseType, pair)
self.strategyExecId = se.strategy_exec_id
def getAllOrders(cretenExecDetlId=None,
tradeId=None,
orderSide=None,
orderType=None,
orderState=None):
q = Db.Session().query(Orders)
if cretenExecDetlId:
q = q.filter(
StrategyExec.creten_exec_detl_id == cretenExecDetlId,
StrategyExec.strategy_exec_id == Trade.strategy_exec_id,
Trade.trade_id == Orders.trade_id)
if tradeId:
q = q.filter(Orders.trade_id == tradeId)
if orderSide:
orderSide = makeList(orderSide)
q = q.filter(Orders.order_side.in_(orderSide))
if orderType:
orderType = makeList(orderType)
q = q.filter(Orders.order_type.in_(orderType))
if orderState:
orderState = makeList(orderState)
q = q.filter(Orders.order_state.in_(orderState))
return q.all()
def createStrategyExec(cretenExecDetlId, strategyConf, tradeCloseType,
pair):
se = StrategyExec()
se.creten_exec_detl_id = cretenExecDetlId
se.base_asset = pair.getBaseAsset()
se.quote_asset = pair.getQuoteAsset()
se.conf = json.dumps(strategyConf)
se.trade_close_type = tradeCloseType
try:
se.dscp = strategyConf['dscp']
except KeyError:
pass
Db.Session().add(se)
Db.Session().flush()
return se
def processOrderUpdate(self, msg):
with self.lock:
try:
orderResponse = self.parseOrderUpdate(msg)
# TODO verify that the order is incoming from this instance of creten
# process only orders originating from creten
if orderResponse.getClientOrderId(
)[:len(Settings.ORDER_REFERENCE_PREFIX
)] == Settings.ORDER_REFERENCE_PREFIX:
self.orderManager.processOrderUpdate(orderResponse)
if self.callbackOrders:
for callback in self.callbackOrders:
callback(orderResponse)
else:
self.log.info(
'Order message received for an order not originating from Creten. Msg ['
+ str(msg) + ']')
Db.Session().commit()
except:
self.log.error('Order update processing failed! Msg [' +
str(msg) + ']')
Db.Session().rollback()
raise
finally:
Db.Session.remove()
try:
# create outbound orders
self.orderManager.sendOrders(self.cretenExecDetlId)
Db.Session().commit()
except:
self.log.error(
'Order update processing failed while generating outbound orders! Msg ['
+ str(msg) + ']')
Db.Session().rollback()
raise
finally:
Db.Session.remove()
def getPendOrders(tradeId):
return Db.Session().query(Orders).filter(
Orders.trade_id == tradeId,
Orders.order_state.in_([
REF_ORDER_STATE.OPENED, REF_ORDER_STATE.OPEN_PENDING_INT,
REF_ORDER_STATE.OPEN_PENDING_EXT,
REF_ORDER_STATE.PARTIALLY_FILLED,
REF_ORDER_STATE.CANCEL_PENDING_INT,
REF_ORDER_STATE.CANCEL_PENDING_EXT
])).all()
def processCandleUpdate(self, msg):
with self.lock:
self.log.debug('Candle message received: ' + str(msg))
try:
# 1. parse incoming message
candle = self.parseCandleUpdate(msg)
# 2. update market data manager
self.marketDataManager.processCandle(candle)
# 3. invoke custom callbacks
for callback in self.candleListenerCallbackMap[
CandleSubscriptionKey(
Pair(candle.getBaseAsset(),
candle.getQuoteAsset()),
candle.getInterval())]:
callback(candle)
Db.Session().commit()
except:
self.log.error('Candle processing failed! Msg [' + str(msg) +
']')
Db.Session().rollback()
raise
finally:
Db.Session.remove()
try:
# 4. create outbound orders
self.orderManager.sendOrders(self.cretenExecDetlId)
Db.Session().commit()
except:
self.log.error(
'Candle processing failed while generating outbound orders! Msg ['
+ str(msg) + ']')
Db.Session().rollback()
raise
finally:
Db.Session.remove()
def processPortfolioUpdate(self, msg):
with self.lock:
try:
positions = self.parsePortfolioUpdate(msg)
for position in positions:
self.portfolioManager.addPosition(position)
if self.callbackPortfolio:
for callback in self.callbackPortfolio:
callback(position)
Db.Session().commit()
except:
self.log.error('Portfolio update processing failed! Msg [' +
str(msg) + ']')
Db.Session().rollback()
raise
finally:
Db.Session.remove()
def createCretenExec(mode, inputConf):
ce = CretenExec()
ce.conf = inputConf
inputJson = json.loads(inputConf)
try:
ce.dscp = inputJson['dscp']
except KeyError:
pass
if mode == PARAM_MODE_BACKTEST:
ce.exec_type = REF_EXEC_TYPE.BACKTEST
elif mode == PARAM_MODE_REALTIMETEST:
ce.exec_type = REF_EXEC_TYPE.REALTIMETEST
Db.Session().add(ce)
Db.Session().flush()
return ce
def __init__(self):
with SequenceManager.__singleton_lock:
self.seqValMap = {}
self.seqUsedMap = {}
seq = Db.Session().query(Sequence).filter().first()
self.seq = seq
for key, val in SequenceManager.SEQ_COLUMN_MAP.items():
self.seqValMap[key] = getattr(seq, val)
self.seqUsedMap[key] = SequenceManager.SEQ_INCREMENT
def getTrade(tradeId):
trades = Db.Session().query(Trade).filter(
Trade.trade_id == tradeId).all()
if len(trades) == 0:
raise Exception()
if len(trades) > 1:
raise Exception()
return trades[0]
def getAllTrades(strategyExecId=None, tradeState=None):
q = Db.Session().query(Trade)
if strategyExecId:
q = q.filter(Trade.strategy_exec_id == strategyExecId)
if tradeState:
tradeState = makeList(tradeState)
q = q.filter(Trade.trade_state.in_(tradeState))
return q.all()
def getActiveTrades(self, pair=None):
if not pair:
pair = self.pair
return Db.Session().query(Trade).filter(
Trade.trade_state.in_([
REF_TRADE_STATE.OPEN_PENDING, REF_TRADE_STATE.OPENED,
REF_TRADE_STATE.CLOSE_PENDING
]), Trade.base_asset == pair.getBaseAsset(),
Trade.quote_asset == pair.getQuoteAsset(),
Trade.strategy_exec_id == self.strategyExecId).all()
def _storeTrade(self, trade): Db.Session().add(trade) # flush in order to generate db maintained sequence Db.Session().flush() # track trade updates in the cache. if trade reached a final state, remove it from the cache t = orders.Trade.Trade() t.setFromEntity(trade) if not trade.trade_id in self.liveTradeCache: self.liveTradeCache[trade.trade_id] = t else: if trade.trade_state in TradeStateMapper.getDbValue([TradeState.CLOSED, TradeState.CLOSE_FAILED, TradeState.OPEN_FAILED]): self.liveTradeCache.pop(trade.trade_id, None) # once trade is removed, clean also cache of corresponding trade orders for orderKey in list(self.liveOrderCache.keys()): if self.liveOrderCache[orderKey].getTradeId() == t.getTradeId(): self.liveOrderCache.pop(orderKey) else: self.liveTradeCache[trade.trade_id] = t
def getSequence(cls, sequenceName):
with SequenceManager.__singleton_lock:
i = cls.instance()
# check if we exhausted cached values
if i.seqUsedMap[sequenceName] == SequenceManager.SEQ_INCREMENT:
# update value in the db
currVal = getattr(i.seq,
SequenceManager.SEQ_COLUMN_MAP[sequenceName])
setattr(i.seq, SequenceManager.SEQ_COLUMN_MAP[sequenceName],
currVal + SequenceManager.SEQ_INCREMENT)
try:
Db.Session().commit()
except:
Db.Session().rollback()
raise
# reset cache counter
i.seqUsedMap[sequenceName] = 0
i.seqValMap[sequenceName] += 1
i.seqUsedMap[sequenceName] += 1
return i.seqValMap[sequenceName]
def getOrder(orderId=None, intOrderRef=None):
orders = Db.Session().query(Orders)
if orderId:
orders = orders.filter(Orders.order_id == orderId)
if intOrderRef:
orders = orders.filter(Orders.int_order_ref == intOrderRef)
orders = orders.all()
if len(orders) == 0:
raise Exception()
if len(orders) > 1:
raise Exception()
return orders[0]
def run(self):
self.log.info('')
self.log.info('Starting real time simulation')
inputConf = json.loads(self.inputConfRaw)
jsonschema.validate(inputConf, RealtimetestSchema.schema)
with Db.session_scope():
ced = ExecManager.createCretenExecDetl(self.cretenExecId, inputConf['realtimetest'])
cretenExecDetlId = ced.creten_exec_detl_id
self.exchangeDataListener.setCretenExecDetlId(cretenExecDetlId)
self.exchangeEventSimulator.setCretenExecDetlId(cretenExecDetlId)
cretenInterval = getattr(CretenInterval, "INTERVAL_" + inputConf['realtimetest']['interval'])
for pairConf in inputConf['realtimetest']['pairs']:
pair = Pair(baseAsset = pairConf[0], quoteAsset = pairConf[1])
self.marketDataManager.init(pair, cretenInterval)
self.marketRulesManager.init(pair.getSymbol())
strategyExecutor = StrategyExecutor()
self.strategyManager.addStrategyExecutor(strategyExecutor)
for strategyConf in inputConf['realtimetest']['strategies']:
strategy = StrategyFactory.getStrategy(strategyConf, pair, cretenExecDetlId,
self.exchangeClient, self.marketDataManager,
self.marketRulesManager, self.portfolioManager,
self.orderManager)
strategyExecutor.addStrategy(strategy)
self.exchangeDataListener.registerCandleListener(pair, cretenInterval, [self.exchangeEventSimulator.simulateEvent, strategyExecutor.execute])
self.portfolioManager.init()
self.exchangeDataListener.registerUserDataListener()
self.exchangeDataListener.start()
while True:
time.sleep(1)
args = parseArgs()
initLogger(args['loglevel'])
if not os.path.isfile(args['clientconfig']):
raise Exception('Client config file could not be loaded. ' + os.path.realpath(args['clientconfig']) + " does not correspond to a valid file.")
Settings.initClientConfig(clientConfigPath = args['clientconfig'])
printLogo()
printInfo(args)
Db.init(Settings.client['DB_CONNECTION']['ENGINE'],
Settings.client['DB_CONNECTION']['USER'],
Settings.client['DB_CONNECTION']['PASSWORD'],
Settings.client['DB_CONNECTION']['HOST'],
Settings.client['DB_CONNECTION']['PORT'],
Settings.client['DB_CONNECTION']['DB'],
True if args['loglevel'] == 'debugalll' else False)
# must be imported after db session has been initialised
from db_entities.CretenExec import CretenExec
from db_entities.CretenExecDetl import CretenExecDetl
from db_entities.StrategyExec import StrategyExec
from strategy.StrategyPerformance import StrategyPerformance
log = Logger()
logging.info('')
with Db.session_scope():
cretenExecs = Db.Session().query(CretenExec.creten_exec_id).order_by(desc(CretenExec.creten_exec_id)).limit(args['count']).all()
def getCandles(self,
pair,
interval,
limit=None,
startTime=None,
endTime=None):
if pair.getSymbol(
) != self.exchangeConf['baseAsset'] + self.exchangeConf['quoteAsset']:
raise Exception('Requested symbol ' + pair.getSymbol() +
' does not match loaded symbol ' +
self.exchangeConf['baseAsset'] +
self.exchangeConf['quoteAsset'] + '.')
if interval != DbExchangeIntervalMapper.getCretenValue(
self.exchangeConf['data']['interval']):
raise Exception(
'Requested interval ' +
str(DbExchangeIntervalMapper.getDbValue(interval)) +
' does not match loaded interval ' +
self.exchangeConf['data']['interval'] + '.')
fm = self.exchangeConf['data']['fieldMap']
# if order of the fields is changed, do not forget to updated indices in sqlCandle below
sqlQuery = 'SELECT ' + \
fm['openTmstmp'] + ', ' + \
fm['open'] + ', ' + \
fm['high'] + ', ' + \
fm['low'] + ', ' + \
fm['close'] + ', ' + \
fm['volume'] + \
' FROM ' + self.exchangeConf['data']['db']['tableName']
self.log.debug('Query to load input candles: ' + sqlQuery)
sqlCandles = Db.Session().execute(sqlQuery).fetchall()
sqlCandles.sort(key=lambda x: datetime.strptime(
x[0], self.exchangeConf['data']['timeFormat']))
candles = []
for sqlCandle in sqlCandles:
openTime = datetime.strptime(
sqlCandle[0], self.exchangeConf['data']['timeFormat'])
closeTime = self._calcClosingTmstmp(
datetime.strptime(sqlCandle[0],
self.exchangeConf['data']['timeFormat']),
interval)
if openTime >= startTime and closeTime <= endTime:
candles.append(
Candle(baseAsset=pair.getBaseAsset(),
quoteAsset=pair.getQuoteAsset(),
interval=interval,
openTime=openTime,
open=Decimal(sqlCandle[1]),
high=Decimal(sqlCandle[2]),
low=Decimal(sqlCandle[3]),
close=Decimal(sqlCandle[4]),
volume=sqlCandle[5],
closeTime=closeTime,
quoteAssetVolume=-1,
tradesNb=-1,
takerBuyBaseAssetVol=-1,
takerBuyQuoteAssetVol=-1,
isClosing=True))
self.log.debug(candles[-1])
return candles
Settings.initClientConfig(clientConfigPath = args['clientconfig'])
printLogo()
args = parseArgs()
initLogger(args['loglevel'], args['logfile'])
initClientConfig(args)
printInfo(args)
Db.init(Settings.client['DB_CONNECTION']['ENGINE'],
Settings.client['DB_CONNECTION']['USER'],
Settings.client['DB_CONNECTION']['PASSWORD'],
Settings.client['DB_CONNECTION']['HOST'],
Settings.client['DB_CONNECTION']['PORT'],
Settings.client['DB_CONNECTION']['DB'],
True if args['loglevel'] == 'debugalll' else False)
# following packages must be imported only after database initilization
from engines.BackTester import BackTester
from engines.RealTimeSimulator import RealTimeSimulator
from db_managers.ExecManager import ExecManager
if not os.path.isfile(args['inputconfig']):
raise Exception('Input configuration file could not be loaded. ' + os.path.realpath(args['inputconfig']) + " does not correspond to a valid file.")
with open(os.path.realpath(args['inputconfig'])) as myfile:
inputConf = myfile.read()
def run(self):
with Timer("BackTester::run", endDebugLevel = 'INFO'):
inputConf = json.loads(self.inputConfRaw)
jsonschema.validate(inputConf, BacktestSchema.schema)
# generate backtest configuration based on the input file
if 'backtestConf' in inputConf:
for conf in inputConf['backtestConf']:
backtestConf = BacktestGenerator.getBacktest(conf)
self.log.debug("Generated backtest config: " + str(backtestConf))
self.log.debug('')
if 'backtest' not in inputConf:
inputConf['backtest'] = []
inputConf['backtest'] += backtestConf
for conf in inputConf['backtest']:
with Db.session_scope():
ced = ExecManager.createCretenExecDetl(self.cretenExecId, conf)
cretenExecDetlId = ced.creten_exec_detl_id
self.exchangeDataListener.setCretenExecDetlId(cretenExecDetlId)
self.exchangeEventSimulator.setCretenExecDetlId(cretenExecDetlId)
for pairConf in conf['pairs']:
pair = Pair(baseAsset = pairConf[0], quoteAsset = pairConf[1])
self.log.info('')
self.log.info('Loading market rules')
self.marketRulesManager.init([pair.getSymbol()])
self.log.info('Loading market rules completed')
self.log.info('')
for strategyConf in conf['strategies']:
strategy = StrategyFactory.getStrategy(strategyConf, pair, cretenExecDetlId,
self.exchangeClient, self.marketDataManager,
self.marketRulesManager, self.portfolioManager, self.orderManager)
self.strategyManager.reset()
strategyExecutor = StrategyExecutor()
strategyExecutor.addStrategy(strategy)
self.strategyManager.addStrategyExecutor(strategyExecutor)
startTimestamp = datetime.strptime(conf['start_tmstmp'], BACKTEST_TMSTMP_FORMAT)
endTimestamp = datetime.strptime(conf['end_tmstmp'], BACKTEST_TMSTMP_FORMAT)
self.log.info('Backtesting period: ' + str(startTimestamp) + ' - ' + str(endTimestamp))
self.log.info('')
self.exchangeDataListener.resetCandleListener()
self.exchangeDataListener.resetUserDataListener()
self.marketDataManager.removeAllCandles()
self.orderManager.reset()
self.log.info('Initialize portfolio:')
for position in conf['portfolio']:
self.log.info('\t' + position['asset'] + ': ' + str(position['quantity']))
self.log.info('')
self.initPortfolio(conf['portfolio'])
interval = getattr(CretenInterval, "INTERVAL_" + conf['interval'])
self.exchangeDataListener.init(startTimestamp, endTimestamp)
self.exchangeDataListener.registerCandleListener(pair, interval, [self.exchangeEventSimulator.simulateEvent, strategyExecutor.execute])
self.exchangeDataListener.start()
# revert open trades in order not to interfere with overall statistics
self._revertOpenTrades(strategy.getStrategyExecId())
# evaluate results
pos = self.portfolioManager.getPosition(pair.getQuoteAsset())
self.log.info('')
self.log.info('Final balance for ' + pair.getQuoteAsset() + ': ' + str(pos.getFree()))
# calculate metrics
self._showPerformance(strategy.getStrategyExecId())
ced.exec_state = REF_EXEC_STATE.FINISHED
with Db.session_scope():
Db.Session().add(ced)