def _handle_market_snapshot(selection, instrument, timestamp_ns): updates = [] # Check we only have one of [best bets / depth bets / all bets] bid_keys = [k for k in B_BID_KINDS if k in selection] or ["atb"] ask_keys = [k for k in B_ASK_KINDS if k in selection] or ["atl"] assert len(bid_keys) <= 1 assert len(ask_keys) <= 1 # OrderBook Snapshot # TODO Clean this crap up if bid_keys[0] == "atb": bids = selection.get("atb", []) else: bids = [(p, v) for _, p, v in selection.get(bid_keys[0], [])] if ask_keys[0] == "atl": asks = selection.get("atl", []) else: asks = [(p, v) for _, p, v in selection.get(ask_keys[0], [])] snapshot = OrderBookSnapshot( level=OrderBookLevel.L2, instrument_id=instrument.id, bids=[(price_to_probability(p, OrderSide.BUY), v) for p, v in asks], asks=[(price_to_probability(p, OrderSide.SELL), v) for p, v in bids], timestamp_ns=timestamp_ns, ) updates.append(snapshot) if "trd" in selection: updates.extend( _handle_market_trades(runner=selection, instrument=instrument, timestamp_ns=timestamp_ns)) return updates
def test_price_to_probability(): # Exact match assert price_to_probability(1.69, side=OrderSide.BUY) == Price("0.59172") # Rounding match assert price_to_probability(2.01, side=OrderSide.BUY) == Price("0.49505") assert price_to_probability(2.01, side=OrderSide.SELL) == Price("0.50000") # Force for TradeTicks which can have non-tick prices assert price_to_probability(10.4, force=True) == Price("0.09615")
def _handle_market_snapshot(selection, instrument, ts_event, ts_init): updates = [] # Check we only have one of [best bets / depth bets / all bets] bid_keys = [k for k in B_BID_KINDS if k in selection] or ["atb"] ask_keys = [k for k in B_ASK_KINDS if k in selection] or ["atl"] if set(bid_keys) == {"batb", "atb"}: bid_keys = ["atb"] if set(ask_keys) == {"batl", "atl"}: ask_keys = ["atl"] assert len(bid_keys) <= 1 assert len(ask_keys) <= 1 # OrderBook Snapshot # TODO(bm): Clean this up if bid_keys[0] == "atb": bids = selection.get("atb", []) else: bids = [(p, v) for _, p, v in selection.get(bid_keys[0], [])] if ask_keys[0] == "atl": asks = selection.get("atl", []) else: asks = [(p, v) for _, p, v in selection.get(ask_keys[0], [])] if bids or asks: snapshot = OrderBookSnapshot( book_type=BookType.L2_MBP, instrument_id=instrument.id, bids=[(price_to_probability(str(p)), v) for p, v in asks if p], asks=[(price_to_probability(str(p)), v) for p, v in bids if p], ts_event=ts_event, ts_init=ts_init, ) updates.append(snapshot) if "trd" in selection: updates.extend( _handle_market_trades( runner=selection, instrument=instrument, ts_event=ts_event, ts_init=ts_init, ) ) if "spb" in selection or "spl" in selection: updates.extend( _handle_bsp_updates( runner=selection, instrument=instrument, ts_event=ts_event, ts_init=ts_init, ) ) return updates
def build_market_snapshot_messages( raw, instrument_provider: BetfairInstrumentProvider ) -> List[OrderBookSnapshot]: updates = [] for market in raw.get("mc", []): # Market status events # market_definition = market.get("marketDefinition", {}) # TODO - Need to handle instrument status = CLOSED here # Orderbook snapshots if market.get("img") is True: market_id = market["id"] for (selection_id, handicap), selections in itertools.groupby( market.get("rc", []), lambda x: (x["id"], x.get("hc"))): for selection in list(selections): kw = dict( market_id=market_id, selection_id=str(selection_id), handicap=str(handicap or "0.0"), ) instrument = instrument_provider.get_betting_instrument( **kw) # Check we only have one of [best bets / depth bets / all bets] bid_keys = [k for k in B_BID_KINDS if k in selection ] or ["atb"] ask_keys = [k for k in B_ASK_KINDS if k in selection ] or ["atl"] assert len(bid_keys) <= 1 assert len(ask_keys) <= 1 # TODO Clean this crap up if bid_keys[0] == "atb": bids = selection.get("atb", []) else: bids = [(p, v) for _, p, v in selection.get(bid_keys[0], [])] if ask_keys[0] == "atl": asks = selection.get("atl", []) else: asks = [(p, v) for _, p, v in selection.get(ask_keys[0], [])] snapshot = OrderBookSnapshot( level=OrderBookLevel.L2, instrument_id=instrument.id, bids=[(price_to_probability(p, OrderSide.BUY), v) for p, v in asks], asks=[(price_to_probability(p, OrderSide.SELL), v) for p, v in bids], timestamp_ns=millis_to_nanos(raw["pt"]), ) updates.append(snapshot) return updates
def _handle_market_trades(runner, instrument, timestamp_ns): trade_ticks = [] for price, volume in runner.get("trd", []): if volume == 0: continue # Betfair doesn't publish trade ids, so we make our own # TODO - should we use clk here for ID instead of the hash? trade_id = hash_json(data=( timestamp_ns, instrument.market_id, str(runner["id"]), str(runner.get("hc", "0.0")), price, volume, )) tick = TradeTick( instrument_id=instrument.id, price=Price(price_to_probability(price, force=True)), size=Quantity(volume, precision=4), side=OrderSide.BUY, match_id=TradeMatchId(trade_id), timestamp_ns=timestamp_ns, ) trade_ticks.append(tick) return trade_ticks
def _determine_fill_price(self, update: Dict, order: Order): if "avp" not in update: # We don't have any specifics about the fill, assume it was filled at our price return update["p"] if order.filled_qty == 0: # New fill, simply return average price return update["avp"] else: new_price = price_to_probability(str(update["avp"])) prev_price = order.avg_px if prev_price == new_price: # Matched at same price return update["avp"] else: prev_price = probability_to_price(order.avg_px) prev_size = order.filled_qty new_price = Price.from_str(str(update["avp"])) new_size = update["sm"] - prev_size total_size = prev_size + new_size price = (new_price - ((prev_price * (prev_size / total_size)))) / ( new_size / total_size ) self._log.debug( f"Calculating fill price {prev_price=} {prev_size=} {new_price=} {new_size=} == {price=}" ) return price
def _handle_book_updates(runner, instrument, ts_event_ns, ts_recv_ns): deltas = [] for side in B_SIDE_KINDS: for upd in runner.get(side, []): # TODO(bm): Clean this up if len(upd) == 3: _, price, volume = upd else: price, volume = upd deltas.append( OrderBookDelta( instrument_id=instrument.id, level=BookLevel.L2, delta_type=DeltaType.DELETE if volume == 0 else DeltaType.UPDATE, order=Order( price=price_to_probability( price, side=B2N_MARKET_STREAM_SIDE[side]), size=Quantity(volume, precision=8), side=B2N_MARKET_STREAM_SIDE[side], ), ts_event_ns=ts_event_ns, ts_recv_ns=ts_recv_ns, )) if deltas: ob_update = OrderBookDeltas( level=BookLevel.L2, instrument_id=instrument.id, deltas=deltas, ts_event_ns=ts_event_ns, ts_recv_ns=ts_recv_ns, ) return [ob_update] else: return []
def _handle_market_trades( runner, instrument, ts_event_ns, ts_recv_ns, ): trade_ticks = [] for price, volume in runner.get("trd", []): if volume == 0: continue # Betfair doesn't publish trade ids, so we make our own # TODO - should we use clk here for ID instead of the hash? trade_id = hash_json(data=(ts_event_ns, price, volume)) tick = TradeTick( instrument_id=instrument.id, price=price_to_probability( price, force=True), # Already wrapping in Price size=Quantity(volume, precision=4), aggressor_side=AggressorSide.UNKNOWN, match_id=trade_id, ts_event_ns=ts_event_ns, ts_recv_ns=ts_recv_ns, ) trade_ticks.append(tick) return trade_ticks
def _handle_book_updates(runner, instrument, timestamp_ns): deltas = [] for side in B_SIDE_KINDS: for upd in runner.get(side, []): # TODO - Fix this crap if len(upd) == 3: _, price, volume = upd else: price, volume = upd deltas.append( OrderBookDelta( delta_type=OrderBookDeltaType.DELETE if volume == 0 else OrderBookDeltaType.UPDATE, order=Order( price=price_to_probability( price, side=B2N_MARKET_STREAM_SIDE[side]), volume=volume, side=B2N_MARKET_STREAM_SIDE[side], ), instrument_id=instrument.id, timestamp_ns=timestamp_ns, )) if deltas: ob_update = OrderBookDeltas( level=OrderBookLevel.L2, instrument_id=instrument.id, deltas=deltas, timestamp_ns=timestamp_ns, ) return [ob_update] else: return []
def _handle_market_snapshot(selection, instrument, timestamp_ns): updates = [] # Check we only have one of [best bets / depth bets / all bets] bid_keys = [k for k in B_BID_KINDS if k in selection] or ["atb"] ask_keys = [k for k in B_ASK_KINDS if k in selection] or ["atl"] assert len(bid_keys) <= 1 assert len(ask_keys) <= 1 # OrderBook Snapshot # TODO Clean this crap up if bid_keys[0] == "atb": bids = selection.get("atb", []) else: bids = [(p, v) for _, p, v in selection.get(bid_keys[0], [])] if ask_keys[0] == "atl": asks = selection.get("atl", []) else: asks = [(p, v) for _, p, v in selection.get(ask_keys[0], [])] snapshot = OrderBookSnapshot( level=OrderBookLevel.L2, instrument_id=instrument.id, bids=[(price_to_probability(p, OrderSide.BUY), v) for p, v in asks], asks=[(price_to_probability(p, OrderSide.SELL), v) for p, v in bids], timestamp_ns=timestamp_ns, ) updates.append(snapshot) # Trade Ticks for price, volume in selection.get("trd", []): trade_id = hash_json((timestamp_ns, price, volume)) tick = TradeTick( instrument_id=instrument.id, price=Price(price_to_probability(price, force=True)), size=Quantity(volume, precision=4), side=OrderSide.BUY, match_id=TradeMatchId(trade_id), timestamp_ns=timestamp_ns, ) updates.append(tick) return updates
def _handle_ticker(runner: dict, instrument: BettingInstrument, ts_event, ts_init): last_traded_price, traded_volume = None, None if "ltp" in runner: last_traded_price = price_to_probability(str(runner["ltp"])) if "tv" in runner: traded_volume = Quantity(value=runner.get("tv"), precision=instrument.size_precision) return BetfairTicker( instrument_id=instrument.id, last_traded_price=last_traded_price, traded_volume=traded_volume, ts_init=ts_init, ts_event=ts_event, )
def _handle_stream_executable_order_update(self, update: Dict) -> None: """ Handle update containing "E" (executable) order update """ venue_order_id = VenueOrderId(update["id"]) client_order_id = self.venue_order_id_to_client_order_id[ venue_order_id] order = self._cache.order(client_order_id) instrument = self._cache.instrument(order.instrument_id) # Check if this is the first time seeing this order (backtest or replay) if venue_order_id in self.venue_order_id_to_client_order_id: # We've already sent an accept for this order in self._submit_order self._log.debug( f"Skipping order_accept as order exists: venue_order_id={update['id']}" ) else: raise RuntimeError() # self.generate_order_accepted( # strategy_id=order.strategy_id, # instrument_id=instrument.id, # client_order_id=client_order_id, # venue_order_id=venue_order_id, # ts_event=millis_to_nanos(order_update["pd"]), # ) # Check for any portion executed if update["sm"] > 0 and update["sm"] > order.filled_qty: trade_id = create_trade_id(update) if trade_id not in self.published_executions[client_order_id]: fill_qty = update["sm"] - order.filled_qty fill_price = self._determine_fill_price(update=update, order=order) self.generate_order_filled( strategy_id=order.strategy_id, instrument_id=order.instrument_id, client_order_id=client_order_id, venue_order_id=venue_order_id, venue_position_id=None, # Can be None trade_id=trade_id, order_side=B2N_ORDER_STREAM_SIDE[update["side"]], order_type=OrderType.LIMIT, last_qty=Quantity(fill_qty, instrument.size_precision), last_px=price_to_probability(str(fill_price)), # avg_px=Decimal(order['avp']), quote_currency=instrument.quote_currency, commission=Money(0, self.base_currency), liquidity_side=LiquiditySide.NONE, ts_event=millis_to_nanos(update["md"]), ) self.published_executions[client_order_id].append(trade_id)
async def test_order_stream_filled_multiple_prices(self): # Arrange await self._setup_account() update1 = BetfairStreaming.generate_order_update( price="1.50", size=20, side="B", status="E", sm=10, avp="1.60", ) self._setup_exec_client_and_cache(update1) await self.client._handle_order_stream_update(update=update1) await asyncio.sleep(0) order = self.cache.order(client_order_id=ClientOrderId("0")) event = self.messages[-1] order.apply(event) # Act update2 = BetfairStreaming.generate_order_update( price="1.50", size=20, side="B", status="EC", sm=20, avp="1.55", ) self._setup_exec_client_and_cache(update2) await self.client._handle_order_stream_update(update=update2) await asyncio.sleep(0) # Assert assert len(self.messages) == 3 assert isinstance(self.messages[1], OrderFilled) assert isinstance(self.messages[2], OrderFilled) assert self.messages[1].last_px == price_to_probability("1.60") assert self.messages[2].last_px == price_to_probability("1.50")
def _handle_bsp_updates(runner, instrument, ts_event, ts_init): updates = [] for side in ("spb", "spl"): for upd in runner.get(side, []): price, volume = upd delta = BSPOrderBookDelta( instrument_id=instrument.id, book_type=BookType.L2_MBP, action=BookAction.DELETE if volume == 0 else BookAction.UPDATE, order=Order( price=price_to_probability(str(price)), size=Quantity(volume, precision=8), side=B2N_MARKET_STREAM_SIDE[side], ), ts_event=ts_event, ts_init=ts_init, ) updates.append(delta) return updates
def _handle_book_updates(runner, instrument, ts_event, ts_init): deltas = [] for side in B_SIDE_KINDS: for upd in runner.get(side, []): # TODO(bm): Clean this up if len(upd) == 3: _, price, volume = upd else: price, volume = upd if price == 0.0: continue deltas.append( OrderBookDelta( instrument_id=instrument.id, book_type=BookType.L2_MBP, action=BookAction.DELETE if volume == 0 else BookAction.UPDATE, order=Order( price=price_to_probability(str(price)), size=Quantity(volume, precision=8), side=B2N_MARKET_STREAM_SIDE[side], ), ts_event=ts_event, ts_init=ts_init, )) if deltas: ob_update = OrderBookDeltas( book_type=BookType.L2_MBP, instrument_id=instrument.id, deltas=deltas, ts_event=ts_event, ts_init=ts_init, ) return [ob_update] else: return []
def test_price_to_probability(self, price, prob): result = price_to_probability(price) expected = Price.from_str(prob) assert result == expected
def _handle_stream_execution_complete_order_update(self, update: Dict) -> None: """ Handle "EC" (execution complete) order updates """ venue_order_id = VenueOrderId(str(update["id"])) client_order_id = self._cache.client_order_id(venue_order_id=venue_order_id) order = self._cache.order(client_order_id=client_order_id) instrument = self._cache.instrument(order.instrument_id) if update["sm"] > 0 and update["sm"] > order.filled_qty: self._log.debug("") execution_id = create_execution_id(update) if execution_id not in self.published_executions[client_order_id]: fill_qty = update["sm"] - order.filled_qty fill_price = self._determine_fill_price(update=update, order=order) # At least some part of this order has been filled self.generate_order_filled( strategy_id=order.strategy_id, instrument_id=instrument.id, client_order_id=client_order_id, venue_order_id=venue_order_id, venue_position_id=None, # Can be None execution_id=execution_id, order_side=B2N_ORDER_STREAM_SIDE[update["side"]], order_type=OrderType.LIMIT, last_qty=Quantity(fill_qty, instrument.size_precision), last_px=price_to_probability(str(fill_price)), quote_currency=instrument.quote_currency, # avg_px=order['avp'], commission=Money(0, self.base_currency), liquidity_side=LiquiditySide.TAKER, # TODO - Fix this? ts_event=millis_to_nanos(update["md"]), ) self.published_executions[client_order_id].append(execution_id) cancel_qty = update["sc"] + update["sl"] + update["sv"] if cancel_qty > 0 and not order.is_completed: assert ( update["sm"] + cancel_qty == update["s"] ), f"Size matched + canceled != total: {update}" # If this is the result of a ModifyOrder, we don't want to emit a cancel key = (client_order_id, venue_order_id) self._log.debug( f"cancel key: {key}, pending_update_order_client_ids: {self.pending_update_order_client_ids}" ) if key not in self.pending_update_order_client_ids: # The remainder of this order has been canceled cancelled_ts = update.get("cd") or update.get("ld") or update.get("md") if cancelled_ts is not None: cancelled_ts = millis_to_nanos(cancelled_ts) else: cancelled_ts = self._clock.timestamp_ns() self.generate_order_canceled( strategy_id=order.strategy_id, instrument_id=instrument.id, client_order_id=client_order_id, venue_order_id=venue_order_id, ts_event=cancelled_ts, ) if venue_order_id in self.venue_order_id_to_client_order_id: del self.venue_order_id_to_client_order_id[venue_order_id] # Market order will not be in self.published_executions if client_order_id in self.published_executions: # This execution is complete - no need to track this anymore del self.published_executions[client_order_id]
def test_price_to_probability(): # Exact match assert price_to_probability(1.69, side=OrderSide.BUY) == Price("0.59172") # Rounding match assert price_to_probability(2.01, side=OrderSide.BUY) == Price("0.49505") assert price_to_probability(2.01, side=OrderSide.SELL) == Price("0.50000")
async def _modify_order(self, command: ModifyOrder) -> None: self._log.debug(f"Received modify_order {command}") client_order_id: ClientOrderId = command.client_order_id instrument = self._cache.instrument(command.instrument_id) PyCondition.not_none(instrument, "instrument") existing_order = self._cache.order(client_order_id) # type: Order self.generate_order_pending_update( strategy_id=command.strategy_id, instrument_id=command.instrument_id, client_order_id=command.client_order_id, venue_order_id=command.venue_order_id, ts_event=self._clock.timestamp_ns(), ) if existing_order is None: self._log.warning( f"Attempting to update order that does not exist in the cache: {command}" ) self.generate_order_modify_rejected( strategy_id=command.strategy_id, instrument_id=command.instrument_id, client_order_id=client_order_id, venue_order_id=command.venue_order_id, reason="ORDER NOT IN CACHE", ts_event=self._clock.timestamp_ns(), ) return if existing_order.venue_order_id is None: self._log.warning(f"Order found does not have `id` set: {existing_order}") PyCondition.not_none(command.strategy_id, "command.strategy_id") PyCondition.not_none(command.instrument_id, "command.instrument_id") PyCondition.not_none(client_order_id, "client_order_id") self.generate_order_modify_rejected( strategy_id=command.strategy_id, instrument_id=command.instrument_id, client_order_id=client_order_id, venue_order_id=VenueOrderId("-1"), reason="ORDER MISSING VENUE_ORDER_ID", ts_event=self._clock.timestamp_ns(), ) return # Send order to client kw = order_update_to_betfair( command=command, venue_order_id=existing_order.venue_order_id, side=existing_order.side, instrument=instrument, ) self.pending_update_order_client_ids.add( (command.client_order_id, existing_order.venue_order_id) ) try: result = await self._client.replace_orders(**kw) except Exception as exc: if isinstance(exc, BetfairAPIError): await self.on_api_exception(exc=exc) self._log.warning(f"Modify failed: {exc}") self.generate_order_modify_rejected( strategy_id=command.strategy_id, instrument_id=command.instrument_id, client_order_id=command.client_order_id, venue_order_id=existing_order.venue_order_id, reason="client error", ts_event=self._clock.timestamp_ns(), ) return self._log.debug(f"result={result}") for report in result["instructionReports"]: if report["status"] == "FAILURE": reason = f"{result['errorCode']}: {report['errorCode']}" self._log.warning(f"replace failed - {reason}") self.generate_order_rejected( strategy_id=command.strategy_id, instrument_id=command.instrument_id, client_order_id=command.client_order_id, reason=reason, ts_event=self._clock.timestamp_ns(), ) return # Check the venue_order_id that has been deleted currently exists on our order deleted_bet_id = report["cancelInstructionReport"]["instruction"]["betId"] self._log.debug(f"{existing_order}, {deleted_bet_id}") assert existing_order.venue_order_id == VenueOrderId(deleted_bet_id) update_instruction = report["placeInstructionReport"] venue_order_id = VenueOrderId(update_instruction["betId"]) self.venue_order_id_to_client_order_id[venue_order_id] = client_order_id self.generate_order_updated( strategy_id=command.strategy_id, instrument_id=command.instrument_id, client_order_id=client_order_id, venue_order_id=VenueOrderId(update_instruction["betId"]), quantity=Quantity( update_instruction["instruction"]["limitOrder"]["size"], precision=4 ), price=price_to_probability( str(update_instruction["instruction"]["limitOrder"]["price"]) ), trigger=None, # Not applicable for Betfair ts_event=self._clock.timestamp_ns(), venue_order_id_modified=True, )
def build_market_update_messages( # noqa TODO: cyclomatic complexity 14 raw, instrument_provider: BetfairInstrumentProvider ) -> List[Union[OrderBookOperation, TradeTick]]: updates = [] for market in raw.get("mc", []): market_id = market["id"] for runner in market.get("rc", []): kw = dict( market_id=market_id, selection_id=str(runner["id"]), handicap=str(runner.get("hc") or "0.0"), ) instrument = instrument_provider.get_betting_instrument(**kw) if not instrument: continue operations = [] for side in B_SIDE_KINDS: for upd in runner.get(side, []): # TODO - Fix this crap if len(upd) == 3: _, price, volume = upd else: price, volume = upd operations.append( OrderBookOperation( op_type=OrderBookOperationType.DELETE if volume == 0 else OrderBookOperationType.UPDATE, order=Order( price=price_to_probability( price, side=B2N_MARKET_STREAM_SIDE[side]), volume=volume, side=B2N_MARKET_STREAM_SIDE[side], ), timestamp_ns=millis_to_nanos(raw["pt"]), )) ob_update = OrderBookOperations( level=OrderBookLevel.L2, instrument_id=instrument.id, ops=operations, timestamp_ns=millis_to_nanos(raw["pt"]), ) updates.append(ob_update) for price, volume in runner.get("trd", []): # Betfair doesn't publish trade ids, so we make our own # TODO - should we use clk here? trade_id = hash_json(data=( raw["pt"], market_id, str(runner["id"]), str(runner.get("hc", "0.0")), price, volume, )) trade_tick = TradeTick( instrument_id=instrument.id, price=Price(price_to_probability(price)), size=Quantity(volume, precision=4), side=OrderSide.BUY, match_id=TradeMatchId(trade_id), timestamp_ns=millis_to_nanos(raw["pt"]), ) updates.append(trade_tick) if market.get("marketDefinition", {}).get("status") == "CLOSED": for runner in market["marketDefinition"]["runners"]: kw = dict( market_id=market_id, selection_id=str(runner["id"]), handicap=str(runner.get("hc") or "0.0"), ) instrument = instrument_provider.get_betting_instrument(**kw) assert instrument # TODO - handle market closed # on_market_status() if runner["status"] == "LOSER": # TODO - handle closing valuation = 0 pass elif runner["status"] == "WINNER": # TODO handle closing valuation = 1 pass if (market.get("marketDefinition", {}).get("inPlay") and not market.get("marketDefinition", {}).get("status") == "CLOSED"): for selection in market["marketDefinition"]["runners"]: kw = dict( market_id=market_id, selection_id=str(selection["id"]), handicap=str( selection.get("hc", selection.get("handicap")) or "0.0"), ) instrument = instrument_provider.get_betting_instrument(**kw) assert instrument # TODO - handle instrument status IN_PLAY return updates