def get_marks(self, instrument: ExchangeInstrument) -> Signal:
if self.mark_type == MarkType.LAST:
trades = self.mds.get_trades(instrument)
return Map(self.network, trades, lambda x: Mark(instrument.get_instrument(), x.get_price()))
else:
books = self.mds.get_order_books(instrument)
return Map(self.network, books, lambda x: Mark(instrument.get_instrument(), x.get_mid()))
def get_order_books(self, instrument: ExchangeInstrument) -> Signal:
symbol = instrument.get_exchange_instrument_code()
exchange_code_upper = instrument.get_exchange().get_exchange_code().upper()
order_books = self.book_signal_by_symbol.get(symbol, None)
if order_books is None:
self._maybe_notify_subscription(instrument)
order_books = MutableSignal()
self.scheduler.network.attach(order_books)
self.book_signal_by_symbol[symbol] = order_books
class OrderBookGenerator(SignalGenerator):
def __init__(self, mds):
self.mds = mds
def generate(self, scheduler: NetworkScheduler):
tickstore = AzureBlobTickstore(self.mds.credential, f'{exchange_code_upper}_BOOKS',
timestamp_column='time')
books_df = tickstore.select(symbol, self.mds.start_time, self.mds.end_time)
for row in books_df.itertuples():
at_time = row[0].asm8.astype('int64') / 10**6
best_bid = BookLevel(row[2], row[1])
best_ask = BookLevel(row[4], row[3])
book = OrderBook(bids=[best_bid], asks=[best_ask])
scheduler.schedule_update_at(order_books, book, at_time)
tickstore.close()
self.scheduler.add_generator(OrderBookGenerator(self))
# workaround: force scheduling of all historic trade events
self.get_trades(instrument)
return order_books
def __get_feed_uri(self, instrument: ExchangeInstrument) -> str:
if instrument not in self.notified_instruments:
self.notified_instruments.add(instrument)
self.scheduler.schedule_update(self.subscribed_instruments,
instrument)
symbol = instrument.get_exchange_instrument_code()
return f'{instrument.get_exchange().get_exchange_code().lower()}:{self.instance_id}:{symbol}'
def get_trades(self, instrument: ExchangeInstrument) -> Signal:
trade_symbol = instrument.get_exchange_instrument_code()
exchange_code_upper = instrument.get_exchange().get_exchange_code(
).upper()
self._maybe_notify_subscription(instrument)
trades = self.trade_signal_by_symbol.get(trade_symbol, None)
if trades is None:
trades = MutableSignal()
self.scheduler.network.attach(trades)
self.trade_signal_by_symbol[trade_symbol] = trades
class TradeGenerator(SignalGenerator):
def __init__(self, mds):
self.mds = mds
def generate(self, scheduler: NetworkScheduler):
tickstore = AzureBlobTickstore(
self.mds.azure_connect_str,
f'{exchange_code_upper}_TRADES',
timestamp_column='time')
trades_df = tickstore.select(trade_symbol,
self.mds.start_time,
self.mds.end_time)
for row in trades_df.itertuples():
at_time = row[0].asm8.astype('int64') / 10**6
sequence = row[1]
trade_id = row[2]
side = Side.SELL if row[4] == 'sell' else Side.BUY
qty = row[5]
price = row[6]
trade = Trade(instrument, sequence, trade_id, side,
qty, price)
scheduler.schedule_update_at(trades, trade, at_time)
tickstore.close()
self.scheduler.add_generator(TradeGenerator(self))
# workaround: force scheduling of all historic order book events
self.get_order_books(instrument)
return trades
def capture_daily_position(self, account: str,
instrument: ExchangeInstrument, qty: float):
self.capture(
'DailyPositions', {
'date': pd.to_datetime(self.scheduler.get_time(),
unit='ms').date(),
'account': account,
'instrument': instrument.get_exchange_instrument_code(),
'qty': qty
})
def get_or_create_exchange_instrument(self, exchange_symbol, instrument, exchange_code) -> ExchangeInstrument:
ak = (exchange_code, exchange_symbol)
if ak in self.entity_by_ak[ExchangeInstrument]:
return self.entity_by_ak[ExchangeInstrument][ak]
else:
exchange = self.type_code_cache.get_by_code(Exchange, exchange_code)
self.cur.execute("INSERT INTO serenity.exchange_instrument (exchange_instrument_code, instrument_id, "
"exchange_id) VALUES (%s, %s, %s) RETURNING exchange_instrument_id",
(exchange_symbol, instrument.get_instrument_id(), exchange.get_type_id()))
exchange_instrument_id = self.cur.fetchone()[0]
exchange_instrument = ExchangeInstrument(exchange_instrument_id, exchange, instrument, exchange_symbol)
self.entity_by_id[ExchangeInstrument][exchange_instrument_id] = exchange_instrument
self.entity_by_ak[ExchangeInstrument][ak] = exchange_instrument
return exchange_instrument
def test_routing_marketdata_service(mocker: MockFixture):
scheduler = HistoricNetworkScheduler(0, 1000)
mds1 = mocker.MagicMock(MarketdataService)
mds2 = mocker.MagicMock(MarketdataService)
rules = CompositeRoutingRule([
ExchangeRoutingRule('POLYGON', mds1),
ExchangeRoutingRule('PHEMEX', mds2)
])
router = RoutingMarketdataService(scheduler.get_network(), rules)
# noinspection DuplicatedCode
venue_type = VenueType(-1, 'DataAggregator')
exch1 = Exchange(-1, venue_type, 'POLYGON', 'Polygon.io')
instr_type1 = InstrumentType(-1, 'ETF')
instr1 = Instrument(-1, instr_type1, 'SPY')
xinstr1 = ExchangeInstrument(-1, exch1, instr1, 'SPY')
venue_type = VenueType(-1, 'CryptoExchange')
exch2 = Exchange(-1, venue_type, 'PHEMEX', 'Phemex')
instr_type2 = InstrumentType(-1, 'CurrencyPair')
instr2 = Instrument(-1, instr_type2, 'BTCUSD')
xinstr2 = ExchangeInstrument(-1, exch2, instr2, 'BTCUSD')
assert router.get_subscribed_instruments() is not None
assert router.get_order_book_events(
xinstr1) is mds1.get_order_book_events()
assert router.get_order_book_events(
xinstr2) is mds2.get_order_book_events()
assert router.get_order_books(xinstr1) is mds1.get_order_books()
assert router.get_order_books(xinstr2) is mds2.get_order_books()
assert router.get_trades(xinstr1) is mds1.get_trades()
assert router.get_trades(xinstr2) is mds2.get_trades()
scheduler.run()
def get_marks(self, instrument: ExchangeInstrument) -> Signal:
economics = instrument.get_instrument().get_economics()
if isinstance(economics, FutureContract):
economics = economics.get_underlier().get_economics()
if isinstance(economics, CurrencyPair):
ccy_code = economics.get_base_currency().get_currency_code()
symbol = f'.M{ccy_code}'
cash_instrument = self.instrument_cache.get_or_create_cash_instrument(
ccy_code)
else:
raise ValueError(
'Unable to get marks: expected CurrencyPair or FutureContract on CurrencyPair'
)
network = self.scheduler.get_network()
marks = self.instrument_marks.get(symbol)
if marks is None:
marks = MutableSignal()
network.attach(marks)
self.instrument_marks[symbol] = marks
subscribe_msg = {
'id': 1,
'method': 'tick.subscribe',
'params': [symbol]
}
messages = MutableSignal()
json_messages = Map(network, messages, lambda x: json.loads(x))
tick_messages = Filter(network, json_messages,
lambda x: 'tick' in x)
ticks = Map(
network, tick_messages, lambda x: self.__extract_tick(
x, cash_instrument.get_instrument()))
Pipe(self.scheduler, ticks, marks)
asyncio.ensure_future(
websocket_subscribe_with_retry(self.ws_uri, self.timeout,
self.logger, subscribe_msg,
self.scheduler, messages,
symbol, 'ticks'))
return marks
def _create_feed(self, instrument: ExchangeInstrument):
symbol = instrument.get_exchange_instrument_code()
return Feed(instrument, self.instrument_trades[symbol], self.instrument_order_book_events[symbol],
self.instrument_order_books[symbol])
def __init__(self, cur, type_code_cache: TypeCodeCache):
self.cur = cur
self.type_code_cache = type_code_cache
self.entity_by_id = {
Currency: {},
Instrument: {},
CashInstrument: {},
CurrencyPair: {},
ExchangeInstrument: {}
}
self.entity_by_ak = {
Currency: {},
Instrument: {},
CashInstrument: {},
CurrencyPair: {},
ExchangeInstrument: {}
}
self.cur.execute("SELECT currency_id, currency_code FROM serenity.currency")
for row in self.cur.fetchall():
currency = Currency(row[0], row[1])
self.entity_by_id[Currency][row[0]] = currency
self.entity_by_ak[Currency][row[1]] = currency
self.cur.execute("SELECT instrument_id, instrument_type_id, instrument_code FROM serenity.instrument")
for row in self.cur.fetchall():
instrument_type = self.type_code_cache.get_by_id(InstrumentType, row[1])
instrument = Instrument(row[0], instrument_type, row[2])
self.entity_by_id[Instrument][row[0]] = instrument
self.entity_by_ak[Instrument][row[2]] = instrument
self.cur.execute("SELECT cash_instrument_id, instrument_id, currency_id FROM serenity.cash_instrument")
for row in self.cur.fetchall():
cash_instrument_id = row[0]
instrument = self.entity_by_id[Instrument][row[1]]
ccy = self.entity_by_id[Currency][row[2]]
cash_instrument = CashInstrument(cash_instrument_id, instrument, ccy)
self.entity_by_id[CashInstrument][row[0]] = cash_instrument
self.entity_by_ak[CashInstrument][ccy.get_currency_code()] = cash_instrument
self.cur.execute("SELECT currency_pair_id, instrument_id, base_currency_id, quote_currency_id "
"FROM serenity.currency_pair")
for row in self.cur.fetchall():
currency_pair_id = row[0]
instrument = self.entity_by_id[Instrument][row[1]]
base_ccy = self.entity_by_id[Currency][row[2]]
quote_ccy = self.entity_by_id[Currency][row[3]]
ccy_pair = CurrencyPair(currency_pair_id, instrument, base_ccy, quote_ccy)
self.entity_by_id[CurrencyPair][currency_pair_id] = ccy_pair
self.entity_by_ak[CurrencyPair][(base_ccy, quote_ccy)] = ccy_pair
self.cur.execute("SELECT exchange_instrument_id, instrument_id, exchange_id, exchange_instrument_code "
"FROM serenity.exchange_instrument")
for row in self.cur.fetchall():
exchange_instrument_id = row[0]
instrument = self.entity_by_id[Instrument][row[1]]
exchange = self.type_code_cache.get_by_id(Exchange, row[2])
exchange_instrument_code = row[3]
exch_instr = ExchangeInstrument(exchange_instrument_id, exchange, instrument, exchange_instrument_code)
self.entity_by_id[ExchangeInstrument][exchange_instrument_id] = exch_instr
self.entity_by_ak[ExchangeInstrument][(exchange.get_type_code(), exchange_instrument_code)] = exch_instr
def route(self, instrument: ExchangeInstrument) -> Optional[MarketdataService]:
if instrument.get_exchange().get_exchange_code() == self.exchange_code:
return self.mds
else:
return None
def __get_feed_uri(self, instrument: ExchangeInstrument) -> str:
symbol = instrument.get_exchange_instrument_code()
return f'{instrument.get_exchange().get_type_code().lower()}:{self.instance_id}:{symbol}'
def __init__(self, cur, type_code_cache: TypeCodeCache):
self.cur = cur
self.type_code_cache = type_code_cache
self.entity_by_id = {
Exchange: {},
Currency: {},
Instrument: {},
CashInstrument: {},
CurrencyPair: {},
FutureContract: {},
ExchangeInstrument: {}
}
self.entity_by_ak = {
Exchange: {},
Currency: {},
Instrument: {},
CashInstrument: {},
CurrencyPair: {},
FutureContract: {},
ExchangeInstrument: {}
}
self.cur.execute(
"SELECT exchange_id, venue_type_id, exchange_code, exchange_name, "
"exchange_calendar, exchange_tz FROM serenity.exchange")
for row in self.cur.fetchall():
venue_type = type_code_cache.get_by_id(VenueType, row[1])
exchange = Exchange(row[0], venue_type, row[2], row[3], row[4],
row[5])
self.entity_by_id[Exchange][row[0]] = exchange
self.entity_by_ak[Exchange][(row[2], venue_type)] = exchange
self.cur.execute(
"SELECT currency_id, currency_code FROM serenity.currency")
for row in self.cur.fetchall():
currency = Currency(row[0], row[1])
self.entity_by_id[Currency][row[0]] = currency
self.entity_by_ak[Currency][row[1]] = currency
self.cur.execute(
"SELECT instrument_id, instrument_type_id, instrument_code FROM serenity.instrument"
)
for row in self.cur.fetchall():
instrument_type = self.type_code_cache.get_by_id(
InstrumentType, row[1])
instrument = Instrument(row[0], instrument_type, row[2])
self.entity_by_id[Instrument][row[0]] = instrument
self.entity_by_ak[Instrument][row[2]] = instrument
self.cur.execute(
"SELECT cash_instrument_id, instrument_id, currency_id FROM serenity.cash_instrument"
)
for row in self.cur.fetchall():
cash_instrument_id = row[0]
instrument = self.entity_by_id[Instrument][row[1]]
ccy = self.entity_by_id[Currency][row[2]]
cash_instrument = CashInstrument(cash_instrument_id, instrument,
ccy)
instrument.set_economics(cash_instrument)
self.entity_by_id[CashInstrument][row[0]] = cash_instrument
self.entity_by_ak[CashInstrument][
ccy.get_currency_code()] = cash_instrument
self.cur.execute(
"SELECT currency_pair_id, instrument_id, base_currency_id, quote_currency_id "
"FROM serenity.currency_pair")
for row in self.cur.fetchall():
currency_pair_id = row[0]
instrument = self.entity_by_id[Instrument][row[1]]
base_ccy = self.entity_by_id[Currency][row[2]]
quote_ccy = self.entity_by_id[Currency][row[3]]
ccy_pair = CurrencyPair(currency_pair_id, instrument, base_ccy,
quote_ccy)
instrument.set_economics(ccy_pair)
self.entity_by_id[CurrencyPair][currency_pair_id] = ccy_pair
self.entity_by_ak[CurrencyPair][(base_ccy, quote_ccy)] = ccy_pair
self.cur.execute(
"SELECT future_contract_id, instrument_id, underlier_instrument_id, contract_size, expiry "
"FROM serenity.future_contract")
for row in self.cur.fetchall():
future_contract_id = row[0]
instrument = self.entity_by_id[Instrument][row[1]]
underlier = self.entity_by_id[Instrument][row[2]]
contract_size = row[3]
expiry = row[4]
future = FutureContract(future_contract_id, instrument, underlier,
contract_size, expiry)
instrument.set_economics(future)
self.entity_by_id[FutureContract][future_contract_id] = future
self.entity_by_ak[FutureContract][(underlier.get_instrument_code(),
contract_size, expiry)] = future
self.cur.execute(
"SELECT exchange_instrument_id, instrument_id, exchange_id, exchange_instrument_code "
"FROM serenity.exchange_instrument")
for row in self.cur.fetchall():
exchange_instrument_id = row[0]
instrument = self.entity_by_id[Instrument][row[1]]
exchange = self.entity_by_id[Exchange][row[2]]
exchange_instrument_code = row[3]
exch_instr = ExchangeInstrument(exchange_instrument_id, exchange,
instrument,
exchange_instrument_code)
self.entity_by_id[ExchangeInstrument][
exchange_instrument_id] = exch_instr
self.entity_by_ak[ExchangeInstrument][(
exchange, exchange_instrument_code)] = exch_instr