def parse_diff_depth_stream_ws( instrument_id: InstrumentId, data: BinanceOrderBookData, ts_init: int, ) -> OrderBookDeltas: ts_event: int = millis_to_nanos( data.T) if data.T is not None else millis_to_nanos(data.E) bid_deltas: List[OrderBookDelta] = [ parse_book_delta_ws(instrument_id, OrderSide.BUY, d, ts_event, ts_init, data.u) for d in data.b ] ask_deltas: List[OrderBookDelta] = [ parse_book_delta_ws(instrument_id, OrderSide.SELL, d, ts_event, ts_init, data.u) for d in data.a ] return OrderBookDeltas( instrument_id=instrument_id, book_type=BookType.L2_MBP, deltas=bid_deltas + ask_deltas, ts_event=ts_event, ts_init=ts_init, update_id=data.u, )
def parse_line(d): if "status" in d: return {} elif "close_price" in d: # return {'timestamp': d['remote_timestamp'], "close_price": d['close_price']} return {} if "trade" in d: return { "timestamp": d["remote_timestamp"], "op": "trade", "trade": TradeTick( instrument_id=InstrumentId(Symbol("TEST"), Venue("BETFAIR")), price=Price(d["trade"]["price"], 4), size=Quantity(d["trade"]["volume"], 4), aggressor_side=d["trade"]["side"], match_id=TradeMatchId(d["trade"]["trade_id"]), timestamp_origin_ns=millis_to_nanos( pd.Timestamp(d["remote_timestamp"]).timestamp()), timestamp_ns=millis_to_nanos( pd.Timestamp(d["remote_timestamp"]).timestamp( ) # TODO(cs): Hardcoded identical for now ), ), } elif "level" in d and d["level"]["orders"][0]["volume"] == 0: op = "delete" else: op = "update" order_like = d["level"]["orders"][0] if op != "trade" else d[ "trade"] return { "timestamp": d["remote_timestamp"], "op": op, "order": Order( price=Price(order_like["price"], precision=6), volume=Quantity(abs(order_like["volume"]), precision=4), # Betting sides are reversed side={ 2: OrderSide.BUY, 1: OrderSide.SELL }[order_like["side"]], id=str(order_like["order_id"]), ), }
def build_market_update_messages( instrument_provider, raw ) -> List[ Union[OrderBookDelta, TradeTick, InstrumentStatusEvent, InstrumentClosePrice] ]: updates = [] book_updates = [] timestamp_origin_ns = millis_to_nanos(raw["pt"]) timestamp_ns = millis_to_nanos( raw["pt"] ) # TODO(bm): Could call self._clock.timestamp_ns() for market in raw.get("mc", []): updates.extend( _handle_market_runners_status( instrument_provider=instrument_provider, market=market, timestamp_ns=timestamp_ns, ) ) for runner in market.get("rc", []): kw = dict( market_id=market["id"], selection_id=str(runner["id"]), handicap=str(runner.get("hc") or "0.0"), ) instrument = instrument_provider.get_betting_instrument(**kw) if instrument is None: continue # Delay appending book updates until we can merge at the end book_updates.extend( _handle_book_updates( runner=runner, instrument=instrument, timestamp_origin_ns=timestamp_origin_ns, timestamp_ns=timestamp_ns, ) ) if "trd" in runner: updates.extend( _handle_market_trades( runner=runner, instrument=instrument, timestamp_origin_ns=timestamp_origin_ns, timestamp_ns=timestamp_ns, ) ) if book_updates: updates.extend(_merge_order_book_deltas(book_updates)) return updates
def parse_trade_report_http( account_id: AccountId, instrument_id: InstrumentId, data: BinanceFuturesAccountTrade, report_id: UUID4, ts_init: int, ) -> TradeReport: return TradeReport( account_id=account_id, instrument_id=instrument_id, venue_order_id=VenueOrderId(str(data.orderId)), venue_position_id=PositionId( f"{instrument_id}-{data.positionSide.value}"), trade_id=TradeId(str(data.id)), order_side=OrderSide[data.side.value], last_qty=Quantity.from_str(data.qty), last_px=Price.from_str(data.price), commission=Money(data.commission, Currency.from_str(data.commissionAsset)), liquidity_side=LiquiditySide.MAKER if data.maker else LiquiditySide.TAKER, report_id=report_id, ts_event=millis_to_nanos(data.time), ts_init=ts_init, )
def _handle_account_update(self, msg: BinanceFuturesAccountUpdateMsg): self.generate_account_state( balances=parse_account_balances_ws(raw_balances=msg.a.B), margins=[], reported=True, ts_event=millis_to_nanos(msg.T), )
async def generate_trades_list( self, venue_order_id: VenueOrderId, symbol: Symbol, since: datetime = None) -> List[ExecutionReport]: filled = self.client().betting.list_cleared_orders( bet_ids=[venue_order_id], ) if not filled["clearedOrders"]: self._log.warn(f"Found no existing order for {venue_order_id}") return [] fill = filled["clearedOrders"][0] timestamp_ns = millis_to_nanos( pd.Timestamp(fill["lastMatchedDate"]).timestamp()) return [ ExecutionReport( client_order_id=self. venue_order_id_to_client_order_id[venue_order_id], venue_order_id=VenueOrderId(fill["betId"]), execution_id=ExecutionId(fill["lastMatchedDate"]), last_qty=Decimal(fill["sizeSettled"]), last_px=Decimal(fill["priceMatched"]), commission_amount=None, # Can be None commission_currency=None, # Can be None liquidity_side=LiquiditySide.NONE, execution_ns=timestamp_ns, timestamp_ns=timestamp_ns, ) ]
def test_millis_to_nanos(self, value, expected): # Arrange # Act result = millis_to_nanos(value) # Assert assert result == expected
def parse_ticker_24hr_ws( instrument_id: InstrumentId, data: BinanceTickerData, ts_init: int, ) -> BinanceTicker: return BinanceTicker( instrument_id=instrument_id, price_change=Decimal(data.p), price_change_percent=Decimal(data.P), weighted_avg_price=Decimal(data.w), prev_close_price=Decimal(data.x) if data.x is not None else None, last_price=Decimal(data.c), last_qty=Decimal(data.Q), bid_price=Decimal(data.b), bid_qty=Decimal(data.B), ask_price=Decimal(data.a), ask_qty=Decimal(data.A), open_price=Decimal(data.o), high_price=Decimal(data.h), low_price=Decimal(data.l), volume=Decimal(data.v), quote_volume=Decimal(data.q), open_time_ms=data.O, close_time_ms=data.C, first_id=data.F, last_id=data.L, count=data.n, ts_event=millis_to_nanos(data.E), ts_init=ts_init, )
def parse_ticker_ws(instrument_id: InstrumentId, msg: Dict, ts_init: int) -> BinanceTicker: return BinanceTicker( instrument_id=instrument_id, price_change=Decimal(msg["p"]), price_change_percent=Decimal(msg["P"]), weighted_avg_price=Decimal(msg["w"]), prev_close_price=Decimal(msg["x"]), last_price=Decimal(msg["c"]), last_qty=Decimal(msg["Q"]), bid_price=Decimal(msg["b"]), ask_price=Decimal(msg["a"]), open_price=Decimal(msg["o"]), high_price=Decimal(msg["h"]), low_price=Decimal(msg["l"]), volume=Decimal(msg["v"]), quote_volume=Decimal(msg["q"]), open_time_ms=msg["O"], close_time_ms=msg["C"], first_id=msg["F"], last_id=msg["L"], count=msg["n"], ts_event=millis_to_nanos(msg["E"]), ts_init=ts_init, )
def _handle_account_update(self, data: Dict[str, Any]): self.generate_account_state( balances=parse_account_balances_ws(raw_balances=data["B"]), margins=[], reported=True, ts_event=millis_to_nanos(data["u"]), )
async def load_all_async(self, filters: Optional[Dict] = None) -> None: """ Load the latest instruments into the provider asynchronously, optionally applying the given filters. Parameters ---------- filters : Dict, optional The venue specific instrument loading filters to apply. """ filters_str = "..." if not filters else f" with filters {filters}..." self._log.info(f"Loading all instruments{filters_str}") # Get current commission rates try: fee_res: List[BinanceSpotTradeFees] = await self._wallet.trade_fees() fees: Dict[str, BinanceSpotTradeFees] = {s.symbol: s for s in fee_res} except BinanceClientError: self._log.error( "Cannot load instruments: API key authentication failed " "(this is needed to fetch the applicable account fee tier).", ) return # Get exchange info for all assets exchange_info: BinanceSpotExchangeInfo = await self._market.exchange_info() for symbol_info in exchange_info.symbols: self._parse_instrument( symbol_info=symbol_info, fees=fees[symbol_info.symbol], ts_event=millis_to_nanos(exchange_info.serverTime), )
def build_market_snapshot_messages( instrument_provider, raw) -> List[Union[OrderBookSnapshot, InstrumentStatusEvent]]: updates = [] timestamp_ns = millis_to_nanos(raw["pt"]) for market in raw.get("mc", []): # Instrument Status updates.extend( _handle_market_runners_status( instrument_provider=instrument_provider, market=market, timestamp_ns=timestamp_ns, )) # OrderBook snapshots if market.get("img") is True: market_id = market["id"] for (selection_id, handicap), selections in itertools.groupby( market.get("rc", []), lambda x: (x["id"], x.get("hc"))): for selection in list(selections): kw = dict( market_id=market_id, selection_id=str(selection_id), handicap=str(handicap or "0.0"), ) instrument = instrument_provider.get_betting_instrument( **kw) updates.extend( _handle_market_snapshot( selection=selection, instrument=instrument, timestamp_ns=timestamp_ns, )) return updates
def test_set_two_repeating_timers(self): # Arrange start_time = self.clock.utc_now() interval = timedelta(milliseconds=100) # Act self.clock.set_timer( name="TEST_TIMER1", interval=interval, start_time=self.clock.utc_now(), stop_time=None, ) self.clock.set_timer( name="TEST_TIMER2", interval=interval, start_time=self.clock.utc_now(), stop_time=None, ) events = self.clock.advance_time(to_time_ns=millis_to_nanos(500)) # Assert assert len(events) == 10 assert self.clock.utc_now() == start_time + timedelta(milliseconds=500) assert self.clock.timestamp_ns() == 500_000_000
async def load_ids_async( self, instrument_ids: List[InstrumentId], filters: Optional[Dict] = None, ) -> None: """ Load the instruments for the given IDs into the provider, optionally applying the given filters. Parameters ---------- instrument_ids: List[InstrumentId] The instrument IDs to load. filters : Dict, optional The venue specific instrument loading filters to apply. Raises ------ ValueError If any `instrument_id.venue` is not equal to `self.venue`. """ if not instrument_ids: self._log.info("No instrument IDs given for loading.") return # Check all instrument IDs for instrument_id in instrument_ids: PyCondition.equal(instrument_id.venue, self.venue, "instrument_id.venue", "self.venue") filters_str = "..." if not filters else f" with filters {filters}..." self._log.info(f"Loading instruments {instrument_ids}{filters_str}.") # # Get current commission rates # try: # fees: Optional[Dict[str, Dict[str, str]]] = None # except BinanceClientError: # self._log.error( # "Cannot load instruments: API key authentication failed " # "(this is needed to fetch the applicable account fee tier).", # ) # return # Extract all symbol strings symbols: List[str] = [ instrument_id.symbol.value for instrument_id in instrument_ids ] # Get exchange info for all assets exchange_info: BinanceFuturesExchangeInfo = await self._market.exchange_info( symbols=symbols) for symbol_info in exchange_info.symbols: self._parse_instrument( symbol_info=symbol_info, fees=None, ts_event=millis_to_nanos(exchange_info.serverTime), )
def make_news_event(df, state=None): for _, row in df.iterrows(): yield NewsEvent( name=row["Name"], impact=row["Impact"], currency=row["Currency"], ts_event_ns=millis_to_nanos( pd.Timestamp(row["Start"]).timestamp()), )
def parse_trade_tick_ws(instrument_id: InstrumentId, msg: Dict, ts_init: int) -> TradeTick: return TradeTick( instrument_id=instrument_id, price=Price.from_str(msg["p"]), size=Quantity.from_str(msg["q"]), aggressor_side=AggressorSide.SELL if msg["m"] else AggressorSide.BUY, trade_id=str(msg["t"]), ts_event=millis_to_nanos(msg["T"]), ts_init=ts_init, )
async def generate_order_status_report(self, order) -> Optional[OrderStatusReport]: return [ OrderStatusReport( client_order_id=ClientOrderId(), venue_order_id=VenueOrderId(), order_state=OrderState(), filled_qty=Quantity.zero(), timestamp_ns=millis_to_nanos(), ) for order in self.client().betting.list_current_orders()["currentOrders"] ]
def parse_trade_tick_http(instrument_id: InstrumentId, msg: Dict, ts_init: int) -> TradeTick: return TradeTick( instrument_id=instrument_id, price=Price.from_str(msg["price"]), size=Quantity.from_str(msg["qty"]), aggressor_side=AggressorSide.SELL if msg["isBuyerMaker"] else AggressorSide.BUY, trade_id=TradeId(str(msg["id"])), ts_event=millis_to_nanos(msg["time"]), ts_init=ts_init, )
def test_set_timer_with_immediate_start_time(self): # Arrange name = "TEST_TIMER" # Act self.clock.set_timer( name=name, interval=timedelta(milliseconds=100), start_time=None, stop_time=None, ) events = self.clock.advance_time(to_time_ns=millis_to_nanos(400)) # Assert assert self.clock.timer_names() == [name] assert len(events) == 4 assert events[0].event.event_timestamp_ns == 100_000_000 assert events[1].event.event_timestamp_ns == 200_000_000 assert events[2].event.event_timestamp_ns == 300_000_000 assert events[3].event.event_timestamp_ns == 400_000_000 assert events[0].event.event_timestamp == datetime(1970, 1, 1, 0, 0, 0, 100000, tzinfo=pytz.utc) assert events[1].event.event_timestamp == datetime(1970, 1, 1, 0, 0, 0, 200000, tzinfo=pytz.utc) assert events[2].event.event_timestamp == datetime(1970, 1, 1, 0, 0, 0, 300000, tzinfo=pytz.utc) assert events[3].event.event_timestamp == datetime(1970, 1, 1, 0, 0, 0, 400000, tzinfo=pytz.utc)
def test_set_multiple_time_alerts(self): # Arrange alert_time1 = self.clock.utc_now() + timedelta(milliseconds=200) alert_time2 = self.clock.utc_now() + timedelta(milliseconds=300) # Act self.clock.set_time_alert("TEST_ALERT1", alert_time1) self.clock.set_time_alert("TEST_ALERT2", alert_time2) events = self.clock.advance_time(to_time_ns=millis_to_nanos(300)) # Assert assert self.clock.timer_names() == [] assert len(events) == 2
def test_set_time_alert(self): # Arrange name = "TEST_ALERT" alert_time = self.clock.utc_now() + timedelta(milliseconds=100) # Act self.clock.set_time_alert(name, alert_time) events = self.clock.advance_time(to_time_ns=millis_to_nanos(200)) # Assert assert self.clock.timer_names() == [] assert len(events) == 1 assert type(events[0]) == TimeEventHandler
def parse_csv_tick(df, instrument_id, state=None): for _, r in df.iterrows(): ts = millis_to_nanos(pd.Timestamp(r["timestamp"]).timestamp()) tick = QuoteTick( instrument_id=instrument_id, bid=Price.from_str(str(r["bid"])), ask=Price.from_str(str(r["ask"])), bid_size=Quantity.from_int(1_000_000), ask_size=Quantity.from_int(1_000_000), ts_event_ns=ts, ts_recv_ns=ts, ) yield tick
def parse_order_report_http( account_id: AccountId, instrument_id: InstrumentId, data: BinanceFuturesOrder, report_id: UUID4, ts_init: int, ) -> OrderStatusReport: price = Decimal(data.price) trigger_price = Decimal(data.stopPrice) avg_px = Decimal(data.avgPrice) time_in_force = BinanceFuturesTimeInForce(data.timeInForce.upper()) return OrderStatusReport( account_id=account_id, instrument_id=instrument_id, client_order_id=ClientOrderId(data.clientOrderId) if data.clientOrderId != "" else None, venue_order_id=VenueOrderId(str(data.orderId)), order_side=OrderSide[data.side.upper()], order_type=parse_order_type(data.type), time_in_force=parse_time_in_force(time_in_force), order_status=parse_order_status(data.status), price=Price.from_str(data.price) if price is not None else None, quantity=Quantity.from_str(data.origQty), filled_qty=Quantity.from_str(data.executedQty), avg_px=avg_px if avg_px > 0 else None, post_only=time_in_force == BinanceFuturesTimeInForce.GTX, reduce_only=data.reduceOnly, report_id=report_id, ts_accepted=millis_to_nanos(data.time), ts_last=millis_to_nanos(data.updateTime), ts_init=ts_init, trigger_price=Price.from_str(str(trigger_price)) if trigger_price > 0 else None, trigger_type=parse_trigger_type(data.workingType), trailing_offset=Decimal(data.priceRate) * 100 if data.priceRate is not None else None, offset_type=TrailingOffsetType.BASIS_POINTS if data.priceRate is not None else TrailingOffsetType.NONE, )
def _handle_stream_executable_order_update(self, update: Dict) -> None: """ Handle update containing "E" (executable) order update """ venue_order_id = VenueOrderId(update["id"]) client_order_id = self.venue_order_id_to_client_order_id[ venue_order_id] order = self._cache.order(client_order_id) instrument = self._cache.instrument(order.instrument_id) # Check if this is the first time seeing this order (backtest or replay) if venue_order_id in self.venue_order_id_to_client_order_id: # We've already sent an accept for this order in self._submit_order self._log.debug( f"Skipping order_accept as order exists: venue_order_id={update['id']}" ) else: raise RuntimeError() # self.generate_order_accepted( # strategy_id=order.strategy_id, # instrument_id=instrument.id, # client_order_id=client_order_id, # venue_order_id=venue_order_id, # ts_event=millis_to_nanos(order_update["pd"]), # ) # Check for any portion executed if update["sm"] > 0 and update["sm"] > order.filled_qty: trade_id = create_trade_id(update) if trade_id not in self.published_executions[client_order_id]: fill_qty = update["sm"] - order.filled_qty fill_price = self._determine_fill_price(update=update, order=order) self.generate_order_filled( strategy_id=order.strategy_id, instrument_id=order.instrument_id, client_order_id=client_order_id, venue_order_id=venue_order_id, venue_position_id=None, # Can be None trade_id=trade_id, order_side=B2N_ORDER_STREAM_SIDE[update["side"]], order_type=OrderType.LIMIT, last_qty=Quantity(fill_qty, instrument.size_precision), last_px=price_to_probability(str(fill_price)), # avg_px=Decimal(order['avp']), quote_currency=instrument.quote_currency, commission=Money(0, self.base_currency), liquidity_side=LiquiditySide.NONE, ts_event=millis_to_nanos(update["md"]), ) self.published_executions[client_order_id].append(trade_id)
def build_market_snapshot_messages( raw, instrument_provider: BetfairInstrumentProvider ) -> List[OrderBookSnapshot]: updates = [] for market in raw.get("mc", []): # Market status events # market_definition = market.get("marketDefinition", {}) # TODO - Need to handle instrument status = CLOSED here # Orderbook snapshots if market.get("img") is True: market_id = market["id"] for (selection_id, handicap), selections in itertools.groupby( market.get("rc", []), lambda x: (x["id"], x.get("hc"))): for selection in list(selections): kw = dict( market_id=market_id, selection_id=str(selection_id), handicap=str(handicap or "0.0"), ) instrument = instrument_provider.get_betting_instrument( **kw) # Check we only have one of [best bets / depth bets / all bets] bid_keys = [k for k in B_BID_KINDS if k in selection ] or ["atb"] ask_keys = [k for k in B_ASK_KINDS if k in selection ] or ["atl"] assert len(bid_keys) <= 1 assert len(ask_keys) <= 1 # TODO Clean this crap up if bid_keys[0] == "atb": bids = selection.get("atb", []) else: bids = [(p, v) for _, p, v in selection.get(bid_keys[0], [])] if ask_keys[0] == "atl": asks = selection.get("atl", []) else: asks = [(p, v) for _, p, v in selection.get(ask_keys[0], [])] snapshot = OrderBookSnapshot( level=OrderBookLevel.L2, instrument_id=instrument.id, bids=[(price_to_probability(p, OrderSide.BUY), v) for p, v in asks], asks=[(price_to_probability(p, OrderSide.SELL), v) for p, v in bids], timestamp_ns=millis_to_nanos(raw["pt"]), ) updates.append(snapshot) return updates
def parse_book_snapshot( instrument_id: InstrumentId, data: BinanceOrderBookData, ts_init: int, ) -> OrderBookSnapshot: return OrderBookSnapshot( instrument_id=instrument_id, book_type=BookType.L2_MBP, bids=[[float(o[0]), float(o[1])] for o in data.bids], asks=[[float(o[0]), float(o[1])] for o in data.asks], ts_event=millis_to_nanos(data.T), ts_init=ts_init, update_id=data.u, )
def parse_trade_tick_ws( instrument_id: InstrumentId, data: BinanceTradeData, ts_init: int, ) -> TradeTick: return TradeTick( instrument_id=instrument_id, price=Price.from_str(data.p), size=Quantity.from_str(data.q), aggressor_side=AggressorSide.SELL if data.m else AggressorSide.BUY, trade_id=TradeId(str(data.t)), ts_event=millis_to_nanos(data.T), ts_init=ts_init, )
def parse_order_report_http( account_id: AccountId, instrument_id: InstrumentId, data: Dict[str, Any], report_id: UUID4, ts_init: int, ) -> OrderStatusReport: client_id_str = data.get("clientOrderId") order_type = data["type"].upper() price = data.get("price") trigger_price = Decimal(data["stopPrice"]) avg_px = Decimal(data["price"]) return OrderStatusReport( account_id=account_id, instrument_id=instrument_id, client_order_id=ClientOrderId(client_id_str) if client_id_str is not None else None, venue_order_id=VenueOrderId(str(data["orderId"])), order_side=OrderSide[data["side"].upper()], order_type=parse_order_type(order_type), time_in_force=parse_time_in_force(data["timeInForce"].upper()), order_status=TimeInForce(data["status"].upper()), price=Price.from_str(price) if price is not None else None, quantity=Quantity.from_str(data["origQty"]), filled_qty=Quantity.from_str(data["executedQty"]), avg_px=avg_px if avg_px > 0 else None, post_only=order_type == "LIMIT_MAKER", reduce_only=False, report_id=report_id, ts_accepted=millis_to_nanos(data["time"]), ts_last=millis_to_nanos(data["updateTime"]), ts_init=ts_init, trigger_price=Price.from_str(str(trigger_price)) if trigger_price > 0 else None, trigger_type=TriggerType.LAST if trigger_price > 0 else TriggerType.NONE, )
def _handle_kline(self, data: Dict[str, Any]): kline = data["k"] if data["E"] < kline["T"]: return # Bar has not closed yet instrument_id = InstrumentId( symbol=Symbol(kline["s"]), venue=BINANCE_VENUE, ) bar: BinanceBar = parse_bar_ws( instrument_id=instrument_id, kline=kline, ts_event=millis_to_nanos(data["E"]), ts_init=self._clock.timestamp_ns(), ) self._handle_data(bar)
def parse_bar(bar_type: BarType, values: List, ts_init: int) -> BinanceBar: return BinanceBar( bar_type=bar_type, open=Price.from_str(values[1]), high=Price.from_str(values[2]), low=Price.from_str(values[3]), close=Price.from_str(values[4]), volume=Quantity.from_str(values[5]), quote_volume=Quantity.from_str(values[7]), count=values[8], taker_buy_base_volume=Quantity.from_str(values[9]), taker_buy_quote_volume=Quantity.from_str(values[10]), ts_event=millis_to_nanos(values[0]), ts_init=ts_init, )