def barDictToBar(b):
tstamp = int(b['open_time'] if 'open_time' in b.keys() else b['start'])
bar = Bar(tstamp=tstamp, open=float(b['open']), high=float(b['high']),
low=float(b['low']), close=float(b['close']), volume=float(b['volume']))
if 'timestamp' in b:
bar.last_tick_tstamp = b['timestamp'] / 1000000.0
return bar
def recalcBar(bar: Bar):
if "trades" not in bar.bot_data or len(bar.bot_data['trades']) == 0:
return
lastTstamp = 0
firstTstamp = bar.last_tick_tstamp
bar.volume = 0
bar.buyVolume = 0
bar.sellVolume = 0
bar.high = bar.low = list(bar.bot_data['trades'].values())[0][3]
for data in bar.bot_data['trades'].values():
tstamp = int(data[1]) / 1000
price = data[3]
volume = abs(data[2])
isBuy = data[2] > 0
if tstamp > lastTstamp:
bar.close = price
lastTstamp = tstamp
if tstamp <= firstTstamp:
bar.open = price
firstTstamp = tstamp
bar.low = min(bar.low, price)
bar.high = max(bar.high, price)
bar.volume += volume
bar.last_tick_tstamp = tstamp
if isBuy:
bar.buyVolume += volume
else:
bar.sellVolume += volume
def barArrayToBar(b):
return Bar(tstamp=b[0] / 1000,
open=float(b[1]),
high=float(b[2]),
low=float(b[3]),
close=float(b[4]),
volume=float(b[5]))
def convertBarevent(apiBar: binance_f.model.candlestickevent.Candlestick):
return Bar(tstamp=apiBar.startTime / 1000,
open=float(apiBar.open),
high=float(apiBar.high),
low=float(apiBar.low),
close=float(apiBar.close),
volume=float(apiBar.volume))
def barArrayToBar(kline, priceScale):
return Bar(tstamp=kline[0],
open=kline[3] / priceScale,
high=kline[4] / priceScale,
low=kline[5] / priceScale,
close=kline[6] / priceScale,
volume=kline[7])
def convertBar(self, apiBar: binance_f.model.candlestick.Candlestick):
return Bar(tstamp=apiBar.openTime / 1000,
open=apiBar.open,
high=apiBar.high,
low=apiBar.low,
close=apiBar.close,
volume=apiBar.volume)
def barDictToBar(b):
tstamp = int(b['open_time'] if 'open_time' in b.keys() else b['start'])
return Bar(tstamp=tstamp,
open=float(b['open']),
high=float(b['high']),
low=float(b['low']),
close=float(b['close']),
volume=float(b['volume']))
def _websocket_callback(self, table):
if self.bitmex is None:
#not started yet
return
if table == 'trade':
trades = self.bitmex.recent_trades_and_clear()
if len(self.h1Bars) == 0:
return
for trade in trades:
tstamp = parse_utc_timestamp(trade['timestamp'])
bar = self.h1Bars[0]
if bar is not None and bar.last_tick_tstamp > tstamp:
continue #trade already counted
barstamp = int(tstamp / (60 * 60)) * 60 * 60
price = float(trade['price'])
size = float(trade['size'])
if bar is not None and barstamp == bar.tstamp:
bar.high = max(bar.high, price)
bar.low = min(bar.low, price)
bar.close = price
bar.volume = bar.volume + size
bar.last_tick_tstamp = tstamp
elif len(self.h1Bars) == 0 or barstamp > bar.tstamp:
self.h1Bars.insert(
0,
Bar(tstamp=barstamp,
open=price,
high=price,
low=price,
close=price,
volume=size))
elif table == 'tradeBin1h':
dicts = self.bitmex.recent_H1_bars()
bars = []
for d in dicts:
bars.append(self.barDictToBar(d, 60))
bars.sort(key=lambda b: b.tstamp,
reverse=False) # order with latest bar first
# merge into self.h1Bars
for b in bars:
if b.tstamp > self.h1Bars[0].tstamp:
#newer than newest in history
self.h1Bars.insert(0, b)
else:
for i in range(len(self.h1Bars)):
if b.tstamp == self.h1Bars[i].tstamp:
self.h1Bars[i] = b
break
if self.on_tick_callback is not None and table in [
"tradeBin1h", "order", "execution"
]:
self.on_tick_callback(
fromAccountAction=table in ["order", "execution"])
def barDictToBar(b, barLengthMinutes):
if 'tstamp' not in b.keys():
b['tstamp'] = parse_utc_timestamp(
b['timestamp']
) - barLengthMinutes * 60 # bitmex uses endtime for bar timestamp
return Bar(tstamp=b['tstamp'],
open=b['open'],
high=b['high'],
low=b['low'],
close=b['close'],
volume=b['volume'])
def read_data_file(self, filename):
try:
with open(filename, 'r') as file:
data = json.load(file)
for entry in data:
d = VolubaData(entry['tstamp'])
for exchange, bar in entry['barsByExchange'].items():
bar = dotdict(bar)
b = Bar(tstamp=bar.tstamp,
open=bar.open,
high=bar.high,
low=bar.low,
close=bar.close,
volume=bar.volume)
b.buyVolume = bar.buyVolume
b.sellVolume = bar.sellVolume
d.barsByExchange[exchange] = b
self.m1Data[entry['tstamp']] = d
except Exception as e:
self.logger.error("Error reading data " + str(e))
def socket_callback(self, topic):
try:
data = self.ws.get_data(topic)
gotTick = False
while len(data) > 0:
if topic == 'trade':
tstamp = int(data['ts'])/1000
bar_time = math.floor(tstamp / 60) * 60
price = data['price']
volume = data['amount']
isBuy= data['direction'] == "buy"
if len(self.m1_bars) > 0 and self.m1_bars[-1].tstamp == bar_time:
last_bar = self.m1_bars[-1]
else:
last_bar = Bar(tstamp=bar_time, open=price, high=price, low=price, close=price, volume=0)
self.m1_bars.append(last_bar)
gotTick = True
last_bar.close = price
last_bar.low = min(last_bar.low, price)
last_bar.high = max(last_bar.high, price)
last_bar.volume += volume
last_bar.last_tick_tstamp = tstamp
if isBuy:
last_bar.buyVolume += volume
else:
last_bar.sellVolume += volume
data = self.ws.get_data(topic)
# new bars is handling directly in the messagecause we get a new one on each tick
if gotTick and self.on_tick_callback is not None:
self.on_tick_callback(fromAccountAction=False)
except Exception as e:
self.logger.error("error in socket data(%s): %s " % (topic, str(e)))
def update_bars(self):
"""get data from exchange"""
if len(self.bars) < 10:
self.bars = self.exchange.get_bars(self.settings.MINUTES_PER_BAR,
0)
else:
new_bars = self.exchange.recent_bars(self.settings.MINUTES_PER_BAR,
0)
for b in reversed(new_bars):
if b.tstamp < self.bars[0].tstamp:
continue
elif b.tstamp == self.bars[0].tstamp:
# merge?
if b.subbars[-1].tstamp == self.bars[0].subbars[-1].tstamp:
b.bot_data = self.bars[
0].bot_data # merge bot data to not loose it
self.bars[0] = b
else:
# merge!
first = self.bars[0].subbars[-1]
newBar = Bar(tstamp=b.tstamp,
open=first.open,
high=first.high,
low=first.low,
close=first.close,
volume=first.volume,
subbars=[first])
for sub in reversed(self.bars[0].subbars[:-1]):
if sub.tstamp < b.subbars[-1].tstamp:
newBar.add_subbar(sub)
else:
break
for sub in reversed(b.subbars):
if sub.tstamp > newBar.subbars[0].tstamp:
newBar.add_subbar(sub)
else:
continue
newBar.bot_data = self.bars[
0].bot_data # merge bot data to not loose it
self.bars[0] = newBar
else: # b.tstamp > self.bars[0].tstamp
self.bars.insert(0, b)
del self.bars[400:]
for bar in self.bars[3:]:
# remove minute data from older bars
bar.subbars = []
def run(self):
self.reset()
self.logger.info("starting backtest with " + str(len(self.bars)) +
" bars and " + str(self.account.equity) + " equity")
for i in range(self.bot.min_bars_needed(), len(self.bars)):
if i == len(self.bars) - 1 or i < self.bot.min_bars_needed():
continue # ignore last bar and first x
# slice bars. TODO: also slice intrabar to simulate tick
self.current_bars = self.bars[-(i + 1):]
# add one bar with 1 tick on open to show to bot that the old one is closed
next_bar = self.bars[-i - 2]
forming_bar = Bar(tstamp=next_bar.tstamp,
open=next_bar.open,
high=next_bar.open,
low=next_bar.open,
close=next_bar.open,
volume=0,
subbars=[])
self.current_bars.insert(0, forming_bar)
self.current_bars[0].did_change = True
self.current_bars[1].did_change = True
self.do_funding()
# self.bot.on_tick(self.current_bars, self.account)
for subbar in reversed(next_bar.subbars):
# check open orders & update account
self.handle_open_orders(subbar)
open = len(self.account.open_orders)
forming_bar.add_subbar(subbar)
self.bot.on_tick(self.current_bars, self.account)
if open != len(self.account.open_orders):
self.handle_open_orders(subbar) # got new ones
self.current_bars[1].did_change = False
next_bar.bot_data = forming_bar.bot_data
for b in self.current_bars:
if b.did_change:
b.did_change = False
else:
break # no need to go further
if self.account.open_position.quantity != 0:
self.send_order(
Order(orderId="endOfTest",
amount=-self.account.open_position.quantity))
self.handle_open_orders(self.bars[0].subbars[-1])
if len(self.bot.position_history) > 0:
daysInPos = 0
maxDays = 0
minDays = self.bot.position_history[0].daysInPos() if len(
self.bot.position_history) > 0 else 0
for pos in self.bot.position_history:
if pos.status != PositionStatus.CLOSED:
continue
if pos.exit_tstamp is None:
pos.exit_tstamp = self.bars[0].tstamp
daysInPos += pos.daysInPos()
maxDays = max(maxDays, pos.daysInPos())
minDays = min(minDays, pos.daysInPos())
daysInPos /= len(self.bot.position_history)
profit = self.account.equity - self.initialEquity
uw_updates_per_day = 1440 # every minute
total_days = (self.bars[0].tstamp -
self.bars[-1].tstamp) / (60 * 60 * 24)
rel = profit / (self.maxDD if self.maxDD > 0 else 1)
rel_per_year = rel / (total_days / 365)
self.logger.info(
"finished | closed pos: " +
str(len(self.bot.position_history)) + " | open pos: " +
str(len(self.bot.open_positions)) + " | profit: " +
("%.2f" % (100 * profit / self.initialEquity)) + " | HH: " +
("%.2f" % (100 * (self.hh / self.initialEquity - 1))) +
" | maxDD: " + ("%.2f" %
(100 * self.maxDD / self.initialEquity)) +
" | maxExp: " + ("%.2f" %
(self.maxExposure / self.initialEquity)) +
" | rel: " + ("%.2f" % (rel_per_year)) + " | UW days: " +
("%.1f" % (self.max_underwater / uw_updates_per_day)) +
" | pos days: " + ("%.1f/%.1f/%.1f" %
(minDays, daysInPos, maxDays)))
else:
self.logger.info("finished with no trades")
#self.write_results_to_files()
return self
def set_data_from_json(bar: Bar, jsonData):
if "modules" not in bar.bot_data.keys():
bar.bot_data['modules'] = {}
for key in jsonData.keys():
if len(jsonData[key].keys()) > 0:
bar.bot_data['modules'][key] = dotdict(jsonData[key])
def write_data(self, bar: Bar, dataId, data):
if "modules" not in bar.bot_data.keys():
bar.bot_data['modules'] = {}
bar.bot_data["modules"][dataId] = data
def socket_callback(self, topic):
try:
data = self.ws.get_data(topic)
gotTick = False
while len(data) > 0:
if topic == 'trade':
tstamp = int(data[1]) / 1000
bar_time = math.floor(tstamp / 60) * 60
price = data[3]
volume = abs(data[2])
isBuy = data[2] > 0
if len(self.m1_bars
) > 0 and self.m1_bars[-1].tstamp == bar_time:
last_bar = self.m1_bars[-1]
else:
if len(self.m1_bars) > 0:
self.recalcBar(self.m1_bars[-1])
for bar in self.m1_bars[-5:-2]:
if "trades" in bar.bot_data:
del bar.bot_data['trades']
last_bar = Bar(tstamp=bar_time,
open=price,
high=price,
low=price,
close=price,
volume=0)
last_bar.bot_data['trades'] = {}
self.m1_bars.append(last_bar)
gotTick = True
last_bar.close = price
last_bar.low = min(last_bar.low, price)
last_bar.high = max(last_bar.high, price)
last_bar.volume += volume
last_bar.last_tick_tstamp = tstamp
last_bar.bot_data['trades'][data[0]] = data
if isBuy:
last_bar.buyVolume += volume
else:
last_bar.sellVolume += volume
if topic == 'tradeupdate':
tstamp = int(data[1]) / 1000
bar_time = math.floor(tstamp / 60) * 60
found = False
for i in range(len(self.m1_bars)):
bar = self.m1_bars[-i - 1]
if bar_time == bar.tstamp:
found = True
if "trades" in bar.bot_data:
if data[0] not in bar.bot_data['trades']:
self.logger.warn(
"got trade update before trade entry")
bar.bot_data['trades'][data[0]] = data
else:
self.logger.error(
"wanted to update trade but no trades in bar at index -"
+ str(i + 1))
if i > 0:
# need to recalc, cause wasn't last bar that changed
self.recalcBar(bar)
break
if not found:
self.logger.error(
"didn't find bar for trade to update! " +
str(data))
data = self.ws.get_data(topic)
# new bars is handling directly in the messagecause we get a new one on each tick
if gotTick and self.on_tick_callback is not None:
self.on_tick_callback(fromAccountAction=False)
except Exception as e:
self.logger.error("error in socket data(%s): %s " %
(topic, str(e)))
def write_data_static(bar: Bar, data, indiId: str):
if "indicators" not in bar.bot_data.keys():
bar.bot_data['indicators'] = {}
bar.bot_data["indicators"][indiId] = data