def run_backtester():
while True:
backtests = []
num_tries = 0
while num_tries < 3:
try:
logging.debug("Querying backtest_request for new requests")
with sqlite3.connect(get_backtest_db()) as db:
cursor = db.execute(
"select id, algoname, start_ts, end_ts, "
"tradepair, candlesize, strategyname, parameters "
"from backtest_request where status = ?",
[Backtest.STATUS_NEW])
for row in cursor:
backtests += [
Backtest(row[0], row[1], row[2], row[3], row[4],
row[5], row[6], json.loads(row[7]))
]
break
except:
num_tries += 1
else:
time.sleep(1)
continue
logging.info("Found %s new backtest_request" % len(backtests))
sdex_history = SdexHistory(sdex_db=get_backtest_db())
sdex_history.init()
try:
for backtest in backtests:
if not update_backtest_status(backtest.bid,
Backtest.STATUS_RUNNING):
continue
r, err = sdex_history.run(backtest)
if r:
if not update_backtest_status(backtest.bid,
Backtest.STATUS_FINISHED):
logging.error(
'Cannot updated db for bid = %s with status = %s',
backtest.bid, Backtest.STATUS_FINISHED)
elif not update_backtest_status(backtest.bid,
Backtest.STATUS_ERROR):
logging.error(
'Cannot updated db for bid = %s with status = %s',
backtest.bid, Backtest.STATUS_ERROR)
finally:
sdex_history.close()
if len(backtests) == 0:
time.sleep(1)
async def delete(request):
userid = request.user
algoname = request.match_info['algoname']
try:
algo = await get_existing_algo(userid, algoname)
except:
return json_response(STATUS_ERR % ERRORS[ERR_INTERNAL_ERROR], status=500)
if algo:
try:
async with aiosqlite.connect(get_main_db()) as db:
await db.execute("delete from algos where userid = ? and algoname = ?", [userid, algoname])
await db.execute("delete from deployed_algos where userid = ? and algoname = ?", [userid, algoname])
await db.commit()
try:
async with aiosqlite.connect(get_backtest_db()) as db:
await db.execute("delete from backtest_request where userid = ? and algoname = ?",
[userid, algoname])
await db.commit()
except:
pass
except:
return json_response(STATUS_ERR % ERRORS[ERR_INTERNAL_ERROR], status=500)
else:
return json_response(STATUS_ERR % ERRORS[ERR_RESOURCE_NOT_FOUND], status=400)
return web.Response(text="OK")
async def backtest_list(request):
userid = request.user
reqlist = []
num_tries = 0
while num_tries < 3:
try:
async with aiosqlite.connect(get_backtest_db()) as db:
async with db.execute(
"select id, algoname, start_ts, end_ts, status from backtest_request where userid = ?",
[userid]) as cursor:
async for row in cursor:
reqlist += [
{
'id': row[0],
'algo_name': row[1],
'start_ts': row[2],
'end_ts': row[3],
'status': row[4],
}
]
break
except:
logging.exception('Exception occurred while updating backtest_request')
num_tries += 1
else:
return json_response(STATUS_ERR % ERRORS[ERR_INTERNAL_ERROR], status=500)
return json_response(json.dumps(reqlist))
async def backtest_trades(request):
userid = request.user
backtest_id = request.match_info['backtest_id']
num_tries = 0
exist = False
while num_tries < 3:
try:
async with aiosqlite.connect(get_backtest_db()) as db:
async with db.execute("select id, algoname, start_ts, end_ts, "
"tradepair, candlesize, strategyname, parameters, status "
"from backtest_request where userid = ? and id = ?",
[userid, backtest_id]) as cursor:
async for row in cursor:
exist = True
break
except:
logging.exception('Exception occurred while updating backtest_request')
num_tries += 1
else:
return json_response(STATUS_ERR % ERRORS[ERR_INTERNAL_ERROR], status=500)
if not exist:
return json_response(STATUS_ERR % ERRORS[ERR_RESOURCE_NOT_FOUND], status=500)
trades = []
try:
async with aiosqlite.connect(get_backtest_db()) as db:
async with db.execute("select ts, advice, sold_asset, sold_amount, bought_asset, bought_amount "
"from backtest_trades where backtest_id = ?",
[backtest_id]) as cursor:
async for row in cursor:
trades += [
{
'ts': row[0],
'advice': row[1],
'sold_asset': row[2],
'sold_amount': row[3],
'bought_asset': row[4],
'bought_amount': row[5],
}
]
except:
return json_response(STATUS_ERR % ERRORS[ERR_INTERNAL_ERROR], status=500)
return json_response(json.dumps(trades))
def perform_recovery():
start_trade = None
last_unprocessed_candles = None
db_conn = sqlite3.connect(get_backtest_db())
cursor = db_conn.execute('SELECT key, value FROM state')
for row in cursor:
if row[0] == 'LAST_HANDLED_TRADE':
start_trade = row[1]
elif row[0] == 'UNPROCESSED_CANDLES':
last_unprocessed_candles = json.loads(row[1])
else:
logging.error('Unhandled state variables = %s %s ', row[0], row[1])
db_conn.close()
return start_trade, last_unprocessed_candles
def update_backtest_status(bid, status):
num_tries = 0
while num_tries < 3:
try:
with sqlite3.connect(get_backtest_db()) as db:
db.execute(
"update backtest_request set status = ? where id = ?",
[status, bid])
db.commit()
break
except:
num_tries += 1
else:
return False
return True
async def backtest_run(request):
userid = request.user
reqparams = await request.json()
if type(reqparams) != dict:
return json_response(STATUS_ERR % ERRORS[ERR_INCORRECT_REQUEST], status=400)
algoname = reqparams['algo_name'] if 'algo_name' in reqparams else ''
start_ts = reqparams['start_ts'] if 'start_ts' in reqparams else ''
end_ts = reqparams['end_ts'] if 'end_ts' in reqparams else ''
if not algoname:
return json_response(STATUS_ERR % ERRORS[ERR_INCORRECT_REQUEST], status=400)
try:
algo = await get_existing_algo(userid, algoname)
except:
return json_response(STATUS_ERR % ERRORS[ERR_INTERNAL_ERROR], status=500)
if algo:
num_tries = 0
while num_tries < 3:
try:
async with aiosqlite.connect(get_backtest_db()) as db:
cursor = await db.execute("insert into backtest_request(userid, algoname, start_ts, end_ts, "
"tradepair, candlesize, strategyname, parameters, status) "
"values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
[userid, algoname, start_ts, end_ts,
algo['trade_pair'], algo['candle_size'], algo['strategy_name'],
json.dumps(algo['strategy_parameters']), Backtest.STATUS_NEW])
await db.commit()
breq = {'req_id': cursor.lastrowid}
break
except:
logging.exception('Exception occurred while updating backtest_request')
num_tries += 1
else:
return json_response(STATUS_ERR % ERRORS[ERR_INTERNAL_ERROR], status=400)
else:
return json_response(STATUS_ERR % ERRORS[ERR_RESOURCE_NOT_FOUND], status=400)
return json_response(json.dumps(breq))
async def backtest_status(request):
userid = request.user
breq_id = request.match_info['req_id']
bstatus = None
num_tries = 0
while num_tries < 3:
try:
async with aiosqlite.connect(get_backtest_db()) as db:
async with db.execute("select id, algoname, start_ts, end_ts, "
"tradepair, candlesize, strategyname, parameters, status "
"from backtest_request where userid = ? and id = ?",
[userid, breq_id]) as cursor:
async for row in cursor:
bstatus = {
'id': row[0],
'algo_name': row[1],
'start_ts': row[2],
'end_ts': row[3],
'trade_pair': row[4],
'candle_size': row[5],
'strategy_name': row[6],
'strategy_parameters': row[7],
'status': row[8],
}
break
except:
logging.exception('Exception occurred while updating backtest_request')
num_tries += 1
else:
return json_response(STATUS_ERR % ERRORS[ERR_INTERNAL_ERROR], status=500)
if bstatus:
return json_response(json.dumps(bstatus))
else:
return json_response(STATUS_ERR % ERRORS[ERR_RESOURCE_NOT_FOUND], status=400)
key = asset_format(base) + '_' + asset_format(counter)
if key not in asset_candles:
asset_candles[key] = Candle(key)
if not asset_candles[key].process_row(e):
data += [asset_candles[key].db_values()]
asset_candles[key] = Candle(key) # create new candle
asset_candles[key].process_row(e) # process new candle
unprocessed_candles = {}
for key, candle in asset_candles.items():
unprocessed_candles[key] = candle.to_dict()
unprocessed_candles = json.dumps(unprocessed_candles)
if len(data) > 0:
conn = sqlite3.connect(get_backtest_db())
conn.isolation_level = None
c = conn.cursor()
try:
c.execute('BEGIN')
c.executemany(
'INSERT INTO SDEX_OHLCV(TRADE_PAIR, TS, YEAR, MONTH, WEEK, DAY, HOUR4, HOUR, MINUTE15, MINUTE5, MINUTE,\
OPEN, HIGH, LOW, CLOSE, BASE_VOLUME, COUNTER_VOLUME) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
data)
c.execute('UPDATE state SET value = ? WHERE key = ?',
(e.paging_token, 'LAST_HANDLED_TRADE'))
if c.rowcount == 0:
c.execute('INSERT INTO state(value, key) values(?, ?)',
def run(self, backtest_req: Backtest):
bid, algoname, tradepair, start_ts, end_ts, candlesize, strategyname, parameters = \
backtest_req.bid, backtest_req.algoname, backtest_req.tradepair, backtest_req.start_ts, \
backtest_req.end_ts, backtest_req.candlesize, backtest_req.strategyname, backtest_req.parameters
logging.debug('Starting backtest for bid = %s' % bid)
try:
strategy = strategy_factory[strategyname](bid, parameters)
except Exception as e:
logging.exception('Error occurred while instantiating strategy')
return False, str(e)
page_size = 100
last_advice = None
last_bought = 0
count, candle = self.get_candles(tradepair, start_ts, end_ts,
candlesize, page_size)
logging.debug('Got %s candles to process' % count)
while True:
i = 0
while i < count:
current_candle = Candle(tradepair)
current_candle.c_open = candle.open[i]
current_candle.c_high = candle.high[i]
current_candle.c_low = candle.low[i]
current_candle.c_close = candle.close[i]
current_candle.c_base_volume = candle.volume[i]
current_candle.c_counter_volume = candle.counter_volume[i]
current_candle.c_date = candle.time[i]
strategy.ohlcv['open'] += [candle.open[i]]
strategy.ohlcv['high'] += [candle.high[i]]
strategy.ohlcv['low'] += [candle.low[i]]
strategy.ohlcv['close'] += [candle.close[i]]
strategy.ohlcv['volume'] += [candle.volume[i]]
result = {}
for k, v in strategy.indicator_values.items():
# compute indicator
ohlcv = {}
ohlcv['open'] = np.array(strategy.ohlcv['open'])
ohlcv['high'] = np.array(strategy.ohlcv['high'])
ohlcv['low'] = np.array(strategy.ohlcv['low'])
ohlcv['close'] = np.array(strategy.ohlcv['close'])
ohlcv['volume'] = np.array(strategy.ohlcv['volume'])
indicator = strategy.indicators[strategy.indicator_type[k]]
result[k] = indicator(ohlcv, strategy.indicator_params[k])
for k, vals in result.items():
for param_name, param_val in vals.items():
# get the last value of the indicator
lastval = param_val[len(param_val) - 1]
strategy.indicator_values[k][
param_name] = None if lastval == np.nan else lastval
try:
logging.debug('Starting strategy execution for bid = %s' %
bid)
strategy.process_candle(current_candle)
strategy.current_candle = candle
strategy.execute(strategy.indicator_values)
except Exception as e:
logging.exception('Strategy generated error')
return False, e
advice = strategy.current_advice
logging.debug('Done executing. generated advice = %s' % advice)
if advice:
if last_advice and last_advice == advice:
logging.info(
'Got sequential %s order from bid=%s. Ignoring recent advice.'
% (last_advice, bid))
continue
if not last_advice and advice == TradeAdvice.SELL:
logging.info(
'Sell order without first buy order from bid=%s. Ignoring advice'
% bid)
continue
logging.debug(
'Saving %s from strategy %s of backtest_request %s' %
(strategy.current_advice, strategyname, bid))
asset_pairs = tradepair.split('_')
base_asset = get_asset(asset_pairs[0], asset_pairs[1])
counter_asset = get_asset(asset_pairs[2], asset_pairs[3])
if advice == TradeAdvice.BUY:
sell_asset, buy_asset = base_asset, counter_asset
total_sold = 1
last_bought = total_bought = current_candle.c_close * total_sold
elif advice == TradeAdvice.SELL:
sell_asset, buy_asset = counter_asset, base_asset
total_sold = last_bought
total_bought = total_sold / current_candle.c_close
else:
logging.error('Algo generated incorrect advice %s' %
advice)
continue
ts = (datetime.datetime.utcnow() - EPOCH).total_seconds()
num_tries = 0
while num_tries < 3:
try:
with sqlite3.connect(get_backtest_db()) as db:
db.execute(
"insert into backtest_trades"
"(ts, backtest_id, advice, sold_asset, sold_amount, bought_asset, bought_amount)"
" values (?, ?, ?, ?, ?, ?, ?)", [
ts, bid, strategy.current_advice,
format_asset(sell_asset),
float(total_sold),
format_asset(buy_asset),
float(total_bought)
])
db.commit()
break
except:
num_tries += 1
else:
logging.fatal('Cannot update db after retries')
return False, 'Cannot update db after retries'
logging.debug(
'Trade executed for did=%s, sold_asset=%s, sold_amount=%s, '
'bought_asset=%s, bought_amount=%s' %
(bid, sell_asset, total_sold, buy_asset, total_bought))
last_advice = advice
i += 1
if count < page_size:
break
else:
count, candle = self.get_candles(tradepair, start_ts, end_ts,
candlesize, page_size,
candle.page_token)
return True, None