def __init__(
self,
size,
micro_trend_hl,
micro_accel_hl,
trend_hl,
accel_hl,
edge_trend_hl,
edge_accel_hl,
warmup_data,
):
self.size = size
warmup_prices = warmup_data.xs("price", axis=1, level=1)
latest_prices = warmup_prices.iloc[-1]
self.micro_trend_estimator = TrendEstimator(
HoltEma(micro_trend_hl, micro_accel_hl), latest_prices)
self.trend_estimator = TrendEstimator(HoltEma(trend_hl, accel_hl),
latest_prices)
for _, prices in warmup_prices.iloc[-4 * accel_hl:].iterrows():
self.micro_trend_estimator.step(prices)
self.trend_estimator.step(prices)
self.edge_trend_estimator = TrendEstimator(
HoltEma(edge_trend_hl, edge_accel_hl))
if not self.trend_estimator.ready:
Log.warn(
"Execution strategy initialized but had insufficient warmup data. Will \
warm up slowly in real time.")
else:
Log.info("Execution strategy initialized and warm.")
def add_order(self, order):
net_size = order.size * (1 if order.side == Side.BUY else -1)
self.__positions[order.pair.base] += net_size
self.__positions[order.pair.quote] -= (
net_size + order.size * self.__fees["taker"]) * order.price
Log.info("dummy order", order)
Log.info("dummy positions", self.__positions)
return OpenOrder(order, order.id)
def step_time(self):
"""Returns False if all data has been exhausted."""
self.time += 1
if self.time >= len(self.__data.index):
return False
frame = self.__data.iloc[self.time]
for ep, price in frame.xs("price", level=1).iteritems():
if ep.exchange_id != self.id:
continue
ep = ExchangePair(self.id, ep.pair)
dummy_book = OrderBook(ep, [BookLevel(price, 1)],
[BookLevel(price, 1)])
self.__book_queues[ep.pair].put(dummy_book)
self.__latest_books[ep.pair] = dummy_book
Log.info("dummy-step", self.time)
return True
def __init__(
self, window_size, movement_hl, trend_hl, cointegration_period, warmup_signals, warmup_data
):
self.window_size = window_size
self.moving_prices = HoltEma(movement_hl, trend_hl, trend_hl)
self.moving_err_from_prev_fair = Emse(trend_hl)
self.cointegration_period = cointegration_period
self.sample_counter = 0
self.r = None
self.r2 = None
# TODO: do some checks for length/pairs of warmup signals/outputs
prices = pd.concat(
[warmup_signals.xs("price", axis=1, level=1), warmup_data.xs("price", axis=1, level=1)],
axis=1,
sort=False,
)
volumes = pd.concat(
[
warmup_signals.xs("volume", axis=1, level=1),
warmup_data.xs("volume", axis=1, level=1),
],
axis=1,
sort=False,
)
self.price_history = RingBuffer(self.window_size, dtype=(np.float64, len(prices.columns)))
self.price_history.extend(prices.values)
for _, p in prices.iloc[-trend_hl * 4 :].iterrows():
self.moving_prices.step(p)
self.moving_volumes = Ema(movement_hl, volumes.mean())
self.moving_variances = TrendEstimator(
Emse(window_size / 2, (prices.diff()[1:] ** 2).mean()), prices.iloc[-1]
)
self.prev_fair = Gaussian(self.moving_prices.value, [1e100 for _ in prices.columns])
self.coint_f = pd.DataFrame(
1, index=warmup_signals.columns.unique(0), columns=prices.columns
)
if len(self.price_history) < self.window_size or not self.moving_prices.ready:
Log.warn("Insufficient warmup data. Price model will warm up (slowly) in real time.")
else:
Log.info("Price model initialized and warm.")
def add_order(self, order):
assert order.exchange_id == self.id
payload = {
"request": "/v1/order/new",
"nonce": self.__bfxv1._nonce(),
# Bitfinex v1 API expects "BTCUSD", v2 API expects "tBTCUSD":
"symbol": Bitfinex.encode_trading_pair(order.pair)[1:],
"amount": str(order.size),
"price": str(order.price),
"exchange": "bitfinex",
"side": order.side.name.lower(),
"type": self.__order_types[order.order_type],
"is_postonly": order.maker_only,
}
try:
response = self.__bfxv1._post("/order/new",
payload=payload,
verify=True)
except TypeError as err:
Log.warn("Bitfinex _post type error: {}".format(err))
except Exception as err:
Log.warn("Swallowing unexpected error: {}".format(err))
return None
Log.debug("Bitfinex-order-response", response)
if "id" in response:
order = OpenOrder(order, response["id"])
if not response["is_live"]:
order.update_status(Order.Status.REJECTED)
elif not response["is_cancelled"]:
order.update_status(Order.Status.CANCELLED)
return order
return None
def dummy_main():
pairs = [BTC_USDT, ETH_USDT, XRP_USDT, LTC_USDT, NEO_USDT, EOS_USDT]
Log.info("Loading dummy data.")
data = pd.read_hdf("research/data/1min.h5")
data = data.resample("15Min").first()
window_size = 500
converter = UsdConverter()
aggregator = SignalAggregator(window_size, {"total_market": [p.base for p in pairs]})
Log.info("Processing warmup data.")
warmup_data = data.iloc[:window_size]
data = data.iloc[window_size:]
warmup_data = warmup_data.apply(converter.step, axis=1)
warmup_signals = warmup_data.apply(aggregator.step, axis=1)
Log.info("Initializing components.")
kalman_strategy = strategy.Kalman(
window_size=window_size,
movement_hl=288,
trend_hl=256,
cointegration_period=96,
warmup_signals=warmup_signals,
warmup_data=warmup_data,
)
# use same params for trend and micro trend because 15min is too wide for micro trends to have
# effect
execution_strategy = ExecutionStrategy(10, 3, 9, 3, 9, 4, 12, warmup_data)
dummy_exchange = DummyExchange(
THREAD_MANAGER, BINANCE, data, {"maker": 0.00075, "taker": 0.00075}
)
executor = Executor(THREAD_MANAGER, {dummy_exchange: pairs}, execution_strategy)
while True:
if not dummy_exchange.step_time():
break
frame = dummy_exchange.frame(pairs)
frame_usd = converter.step(frame)
signals = aggregator.step(frame_usd)
kalman_fairs = kalman_strategy.tick(frame_usd, signals)
fairs = kalman_fairs & Gaussian(
frame_usd.xs("price", level=1), [1e100 for _ in frame_usd.xs("price", level=1).index]
)
Log.info("fairs", fairs)
executor.tick_fairs(fairs)
def on_close(ws):
Log.warn("WS closed unexpectedly for exchange {}".format(self.id))
Log.info("restarting WS for exchange {}".format(self.id))
time.sleep(3)
self.__track_positions()
def on_error(ws, error):
Log.warn(
"WS error within __track_positions for exchange {}: {}".format(
self.id, error))
def __trade(self, wait_for_other_trade=False):
"""
If `wait_for_other_trade` is false, doesn't try to trade if there is another thread
attempting to trade. If true, it will wait for the other thread to finish and try
immediately after.
TODO: requires that __latest_fairs and self.__exchange_pairs have the same indexing. Make
this explicit or don't require it.
"""
if self.__latest_fairs is None:
Log.warn(
"Attempted to trade but executor has not received any fairs.")
return
# component warmup may not be synchronized
for ep in self.__latest_fairs.mean.index:
if not ep in self.__latest_books:
Log.warn(
"Attempted to trade but executor has no book data for exchange pair:",
ep)
return
if wait_for_other_trade:
self.__trade_lock.acquire()
elif not self.__trade_lock.acquire(blocking=False):
Log.debug("Other thread trading.")
return
bids = pd.Series(index=self.__latest_fairs.mean.index)
asks = pd.Series(index=self.__latest_fairs.mean.index)
fees = pd.Series(index=self.__latest_fairs.mean.index)
positions = {}
# self.__books_lock.acquire()
for exchange_pair in self.__latest_fairs.mean.index:
exchange = self.__exchanges[exchange_pair.exchange_id]
book = self.__latest_books[exchange_pair]
bids[exchange_pair] = book.bids[0].price
asks[exchange_pair] = book.asks[0].price
fees[exchange_pair] = exchange.fees["taker"]
positions[exchange.id, exchange_pair.
base] = exchange.positions[exchange_pair.base] or 0
# self.__books_lock.release()
positions = pd.Series(positions)
Log.info("Positions", positions)
order_sizes = self.execution_strategy.tick(positions, bids, asks,
self.__latest_fairs,
fees).fillna(0.0)
Log.debug("Order size", order_sizes)
for exchange_pair, order_size in order_sizes.items():
if order_size == 0:
continue
exchange = self.__exchanges[exchange_pair.exchange_id]
side = Side.BUY if order_size > 0 else Side.SELL
price = (asks if order_size > 0 else bids)[exchange_pair]
order = Order(self.__next_order_id(), exchange_pair, side,
Order.Type.IOC, price, abs(order_size))
exchange.add_order(order) # TODO: require this to be async?
Log.info("sent order", order)
self.__trade_lock.release()
def main():
pairs = [BTC_USD, ETH_USD, XRP_USD, LTC_USD, EOS_USD, BCH_USD, BSV_USD]
window_size = 9999 # biggest window that will fit in 2 api calls to candles
Log.info("Connecting to exchanges.")
with open("keys/bitfinex.json") as bitfinex_key_file:
bitfinex_keys = json.load(bitfinex_key_file)
bitfinex = Bitfinex(THREAD_MANAGER, bitfinex_keys, pairs)
Log.info("Fetching warmup data.")
warmup_data = bitfinex.get_warmup_data(pairs, window_size, "1m")
Log.info("Prepping warmup data.")
converter = UsdConverter()
aggregator = SignalAggregator(window_size, {"total_market": [p.base for p in pairs]})
warmup_data = warmup_data.apply(converter.step, axis=1)
warmup_signals = warmup_data.apply(aggregator.step, axis=1)
Log.info("Initializing components.")
kalman_strategy = strategy.Kalman(
window_size=window_size,
movement_hl=1440 * 3,
trend_hl=window_size,
cointegration_period=720,
warmup_signals=warmup_signals,
warmup_data=warmup_data,
)
execution_strategy = ExecutionStrategy(500, 1, 3, 45, 135, 60, 180, warmup_data)
executor = Executor(THREAD_MANAGER, {bitfinex: pairs}, execution_strategy)
beat = Beat(60000)
while beat.loop():
Log.info("Beat")
bfx_frame = bitfinex.frame(pairs)
frame_usd = converter.step(bfx_frame)
signals = aggregator.step(frame_usd)
kalman_fairs = kalman_strategy.tick(frame_usd, signals)
fairs = kalman_fairs & Gaussian(
frame_usd.xs("price", level=1), [1e100 for _ in frame_usd.xs("price", level=1).index]
)
Log.info("fairs", fairs)
executor.tick_fairs(fairs)