def test_none_default_1_deep():
d = NDeepDict(1)
d[['a']] = 16
assert d.get(['a']) == 16
assert d.get(['c']) is None
with pytest.raises(KeyError):
x = d[['c']]
def test_1_deep_with_default_list_access():
d = NDeepDict(1, int)
assert len(d) == 0
# setting something should work
d[['a']] = 1
assert len(d) == 1
# getting 'a' should work
assert d[['a']] == 1
assert d.get(['a']) == 1
# getting 'b' should return the default of 0
assert d[['b']] == 0
assert d.get(['b']) == 0
# length should now be two because b was created on the get
assert len(d) == 2
assert 'b' in d
assert 'a' in d
# modifying an unknown int should work too
assert not 'c' in d
d[['c']] += 1
assert d[['c']] == 1
def __init__(self, logger):
OrderLevelBookListener.__init__(self, logger)
self._market_to_subchain_id_to_time = NDeepDict(
depth=2, default_factory=list
) # this a list of tuples (start_time, end_time)
self._market_to_side_to_prev_tob_subchain_ids = NDeepDict(
depth=2, default_factory=set)
def __init__(self, logger, handle_market_orders=False):
OrderLevelBookListener.__init__(self, logger)
OrderEventListener.__init__(self, logger)
self._market_to_order_book = {}
self._market_to_event_to_priority = NDeepDict(depth=2)
self._market_to_event_to_priority_before = NDeepDict(depth=2)
self._handle_market_orders = handle_market_orders
def test_one_level_contains_none_default():
d = NDeepDict(1)
d[['a']] = 1
assert ['a'] in d
assert 'a' in d
assert ['b'] not in d
assert 'b' not in d
def test_2_deep_with_default_list_access():
d = NDeepDict(2, int)
assert len(d) == 0, "Nothing is in new dict so length is 0"
assert len(d[['a']]) == 0, "Nothing is in the second level dict so should be 0"
assert len(d) == 1, "Looking for length of 'a' above should have added it to the first level dict, so now first level length is 0"
assert d['b'] == {}
assert 'b' in d
assert len(d[['b']]) == 0
d[['b','level2-a']], "Getting a key for first time in second level dict, so should create an item in b"
assert len(d[['b']]) == 1
assert d[['b','level2-a']] == 0
d[['b', 'level2-b']] += 50
assert d[['b','level2-b']] == 50
assert d.get(['b',"level2-b"]) == 50
def test_4_deep_create_at_once_standard_access():
d = NDeepDict(4)
assert len(d) == 0, "Starts completely empty"
d["level1"]["level2"]["level3"]["level4"] = 5
assert "level1" in d
assert "level2" in d["level1"]
assert "level3" in d["level1"]["level2"]
assert "level4" in d["level1"]["level2"]["level3"]
assert d["level1"]["level2"]["level3"]["level4"] == 5
def test_4_deep_create_at_once_list_access():
d = NDeepDict(4)
assert len(d) == 0, "Starts completely empty"
d[["level1", "level2", "level3", "level4"]] = 5
assert len(d) == 1
assert "level1" in d
assert "level2" in d[["level1"]]
assert "level3" in d[["level1", "level2"]]
assert "level4" in d[["level1", "level2", "level3"]]
assert len(d) == 1
assert d[["level1", "level2", "level3", "level4"]] == 5
def test_1_deep_with_no_default_list_access():
d = NDeepDict(1, None)
assert len(d) == 0
# setting something should work
d[['a']] = 1
assert len(d) == 1
# getting 'a' should work
assert d[['a']] == 1
assert d.get(['a']) == 1
# getting 'b' should return the default of None
with pytest.raises(KeyError):
d[['b']]
d.get(['b'])
# length should still be 1 because getting 'b' shouldn't have added anything
assert len(d) == 1
assert 'b' not in d
assert 'a' in d
def __init__(self, logger):
OrderLevelBookListener.__init__(self, logger)
OrderEventListener.__init__(self, logger)
self._market_to_side_prev_price = NDeepDict(depth=2)
self._market_side_price_time = NDeepDict(depth=3) # market to side to price to last time it was top of book
self._market_side_best_price = NDeepDict(depth=2)
self._event_id_to_last_time_crossed = {}
self._event_id_to_last_time_tob = {}
def test_1_deep_del_standard_access():
d = NDeepDict(4)
d[['a1','a1b1','a1b1c1']] = "test value"
d[['a2','a2b1','a2b1c1']] = "test value 2"
d[['a1','a1b2','a1b2c1']] = "test value 3"
d[['a1','a1b1','a1b1c2']] = "test value 4"
d[['a1','a1b1','a1b1c3']] = "test value 5"
assert 'a1b1c2' in d['a1']['a1b1']
del d[['a1', 'a1b1', 'a1b1c2']]
assert 'a1b1c2' not in d['a1']['a1b1'], "was deleted so should be gone"
assert 'a1b1c1' in d['a1']['a1b1'], "was not deleted so should be there"
assert 'a1b1c3' in d['a1']['a1b1'], "was not deleted so should be there"
assert 'a2b1c1' in d['a2']['a2b1'], "was not deleted so should be there"
assert 'a1b2c1' in d['a1']['a1b2'], "was not deleted so should be there"
def __init__(self, logger):
OrderLevelBookListener.__init__(self, logger)
OrderEventListener.__init__(self, logger)
self._market_event_id_aggressive_act = NDeepDict(depth=2)
self._market_to_orderbook = defaultdict(lambda: None)
self._market_to_agg_acts_to_close = defaultdict(set)
class EventPriorityListener(OrderLevelBookListener, OrderEventListener):
"""
priority from event on commands can be considered intended priority
priority from event on execution reports can be considered achieved priority
priority at command is what the priority was before the event .
Designed to work with the order books of multiple markets.
"""
# TODO UNIT TEST!
def __init__(self, logger, handle_market_orders=False):
OrderLevelBookListener.__init__(self, logger)
OrderEventListener.__init__(self, logger)
self._market_to_order_book = {}
self._market_to_event_to_priority = NDeepDict(depth=2)
self._market_to_event_to_priority_before = NDeepDict(depth=2)
self._handle_market_orders = handle_market_orders
def _calculate_priority_not_in_book(self,
price,
side,
market,
ignore_order_ids=set()):
"""
Calculate the priority when not reflected in the book yet.
:param price: MarketObjects.Price.Price
:param side: MarketObjects.Side.Side
:param market: MarketObjects.Market.Market
:return: PriorityListeners.Priority
"""
# if the order book is None then the orderbook hasn't been established
# and we can assume that ack is best priority for its side and there is
# other side.
order_book = self._market_to_order_book.get(market)
if order_book is None:
return Priority(0, None, 0)
else: # if order book is not None then we need to figure out what priority would be
# calculate distance from opposite side since needed in both
opposite_best_price = order_book.best_price(side.other_side())
ticks_from_opposite_tob = None
if opposite_best_price is not None: # if it is None then ticks from opposite is None
ticks_from_opposite_tob = price.ticks_behind(
opposite_best_price, side, market)
#best_price = order_book.best_price(side)
best_price = self._best_price(order_book, side, ignore_order_ids)
# if the best price is None then it is best priority and only need distance from opposite side of book
if best_price is None:
return Priority(0, ticks_from_opposite_tob, 0)
else: # if best price is not None then we need to calculate
ticks_from_tob = price.ticks_behind(best_price, side, market)
# assuming visible qty gets priority over hidden so only want visible qty for qty ahead
if len(ignore_order_ids) == 0:
qty_ahead = order_book.visible_qty_at_price(side, price)
else:
qty_ahead = modified_qty_at_price(
order_book,
side,
price,
ignore_order_ids=ignore_order_ids,
ignore_hidden=True)
return Priority(ticks_from_tob, ticks_from_opposite_tob,
qty_ahead)
def _best_price(self, order_book, side, order_chain_ids):
prices = order_book.prices(side)
best_price = None
for price in prices:
for order_chain in order_book.iter_order_chains_at_price(
side, price):
if order_chain.chain_id() not in order_chain_ids:
best_price = price
break
if best_price is not None:
break
return best_price
def _calculate_priority_in_book(self, order_chain):
"""
Calculate the priority when is in the book already.
:param order_chain: MarketObjects.Events.EventChains.OrderEventChain
:return: PriorityListeners.Priority
"""
market = order_chain.market()
side = order_chain.side()
price = order_chain.current_price() if order_chain.is_open(
) else order_chain.price_at_close()
order_book = self._market_to_order_book.get(market)
if order_book is None:
return Priority(0, None, 0)
else: # if order book is not None then we need to figure out what priority would be
# calculate distance from opposite side since needed in both
opposite_best_price = order_book.best_price(side.other_side())
ticks_from_opposite_tob = None
if opposite_best_price is not None: # if it is None then ticks from opposite is None
ticks_from_opposite_tob = price.ticks_behind(
opposite_best_price, side, market)
# in getting best price, ignore the same order chain so it doesn't factor in with cancel-replace back off of top of book.
best_price = order_book.best_price(side)
# if the best price is None then it is best priority and only need distance from opposite side of book
if best_price is None:
return Priority(0, ticks_from_opposite_tob, 0)
else: # if best price is not None then we need to calculate
ticks_from_tob = price.ticks_behind(best_price, side, market)
# assuming visible qty gets priority over hidden so only want visible qty for qty ahead
qty_ahead = 0
for chain in order_book.iter_order_chains_at_price(
side, price):
if chain.chain_id() == order_chain.chain_id():
break
qty_ahead += chain.visible_qty()
return Priority(ticks_from_tob, ticks_from_opposite_tob,
qty_ahead)
def handle_new_order_command(self, new_order_command,
resulting_order_chain):
if new_order_command.is_market_order(
) and not self._handle_market_orders:
self._logger.debug(
"%s set to ignore market orders. New Order %s is a market order. Ignoring"
% (self.__class__.__name__, str(new_order_command.event_id())))
return
event_id = new_order_command.event_id()
price = new_order_command.price()
side = resulting_order_chain.side()
market = new_order_command.market()
if market not in self._market_to_order_book:
return
# new orders are not in the book so calculate what priority *would be*
priority = self._calculate_priority_not_in_book(price, side, market)
self._market_to_event_to_priority[[market, event_id]] = priority
# No need to set priority before event on new orders
def handle_cancel_replace_command(self, cancel_replace_command,
resulting_order_chain):
event_id = cancel_replace_command.event_id()
price = cancel_replace_command.price()
side = resulting_order_chain.side()
market = resulting_order_chain.market()
if market not in self._market_to_order_book:
return
# need the most recently requested exposure, if none, get the ack'd exposure
exposure = resulting_order_chain.most_recent_requested_exposure()
if exposure is None:
exposure = resulting_order_chain.current_exposure()
before_event_priority = self._calculate_priority_in_book(
resulting_order_chain)
self._market_to_event_to_priority_before[[market, event_id
]] = before_event_priority
# cancel_replace_same_priority is True if cancel replace down and same price
cancel_replace_same_priority = (
price == exposure.price()
and exposure.qty() > cancel_replace_command.qty())
# cancel replaces have priority at time of event, because order already in book
if cancel_replace_same_priority:
priority = before_event_priority
else: # otherwise, have to calculate what priority *would be*
# ignore itself so that we do include it as in front of itself on cancel replace up in size
priority = self._calculate_priority_not_in_book(
price,
side,
market,
ignore_order_ids={cancel_replace_command.chain_id()})
self._market_to_event_to_priority[[market, event_id]] = priority
def handle_cancel_command(self, cancel_command, resulting_order_chain):
"""
at time fo a cancel, the order should be in the book so just go ahead and calculate priority in book.
"""
event_id = cancel_command.event_id()
market = resulting_order_chain.market()
if market not in self._market_to_order_book:
return
priority = self._calculate_priority_in_book(resulting_order_chain)
self._market_to_event_to_priority[[market, event_id]] = priority
# priority before the event is the same calculated aftet event for cancel command
self._market_to_event_to_priority_before[[market, event_id]] = priority
def handle_acknowledgement_report(self, acknowledgement_report,
resulting_order_chain):
event_id = acknowledgement_report.event_id()
market = resulting_order_chain.market()
# don't do anything! orderbook not known
if market not in self._market_to_order_book:
return
# acks are in the book already and no need to do anything fancy with ignoring orders
priority = self._calculate_priority_in_book(resulting_order_chain)
self._market_to_event_to_priority[[market, event_id]] = priority
# if acking a new order no priority before, so use default of None.
# Calculate for cancel replace with current priority since hasn't been applied to book yet
if isinstance(acknowledgement_report.causing_command(),
CancelReplaceCommand):
self._market_to_event_to_priority_before[[market,
event_id]] = priority
def _priority_at_fill(self, fill_event, resulting_order_chain):
"""
A fill we can assume priority at fill was best.
Still need to calculate the ticks from opposite side though.
"""
market = fill_event.market()
event_id = fill_event.event_id()
order_book = self._market_to_order_book.get(market)
# don't do anything! orderbook not known
if order_book is None:
return
# do closes
opposite_best_price = order_book.best_price(
resulting_order_chain.side().other_side())
ticks_from_opposite_tob = None
if opposite_best_price is not None: # if it is None then ticks from opposite is None
ticks_from_opposite_tob = abs(
(opposite_best_price - fill_event.fill_price()) / market.mpi())
priority = Priority(0, ticks_from_opposite_tob, 0)
self._market_to_event_to_priority[[market, event_id]] = priority
# for a fill, priority before fill is always 0
self._market_to_event_to_priority_before[[market, event_id]] = priority
def handle_partial_fill_report(self, partial_fill_report,
resulting_order_chain):
self._priority_at_fill(partial_fill_report, resulting_order_chain)
def handle_full_fill_report(self, full_fill_report, resulting_order_chain):
self._priority_at_fill(full_fill_report, resulting_order_chain)
def handle_cancel_report(self, cancel_report, resulting_order_chain):
"""
When an order is cancelled we want to calculate what its priority was at
time of cancel. Since the cancel hasn't impacted the order book yet,
can find the order chain in the order book and calculate the priority.
"""
# we really only care if a limit and FAR order because otherwise, it wouldn't have any priority since never in book.
# and if it has no ack then it was cancelled for self trade purposes (or some such thing) so no impact on book
if not (resulting_order_chain.is_far()
and resulting_order_chain.is_limit_order()
and resulting_order_chain.has_acknowledgement()):
return
market = resulting_order_chain.market()
if market not in self._market_to_order_book:
return
event_id = cancel_report.event_id()
priority = self._calculate_priority_in_book(resulting_order_chain)
self._market_to_event_to_priority[[market, event_id]] = priority
self._market_to_event_to_priority_before[[market, event_id]] = priority
def notify_book_update(self, order_book, causing_order_chain, tob_updated):
"""
All that needs to be done here is save off the most order_book to be used
when an event listener call back needs it.
"""
self._market_to_order_book[order_book.market()] = order_book
def event_priority(self, market, event_id):
"""
Get's the priority of the event. If the event was not tracked (/doesn't exist) then will return None.
:param market: MarketObjects.Market.Market
:param event_id: unique identifier of event
:return: MarketMetrics.OrderLevelBookListeners.PriorityListeners.Priority
"""
return self._market_to_event_to_priority.get([market, event_id])
def priority_before_event(self, market, event_id):
"""
Gets the priority of the event's order chain right before the event occurred. If the event was not
tracked (/doesn't exist) then will return None.
:param market: MarketObjects.Market.Market
:param event_id: unique identifier of event
:return: MarketMetrics.OrderLevelBookListeners.PriorityListeners.Priority
"""
return self._market_to_event_to_priority_before.get([market, event_id])
def clean_up(self, order_chain):
"""
If clean_up is called with an order_chain than this will go through the
order chain's events and remove them from tracking
WARNING: once this is called, the event_priority no longer return the values that are
meaningful; rather, you'll get None.
:param order_chain: MarketObjects.Events.EventChains.OrderEventChain
"""
market = order_chain.market()
events = order_chain.events()
for event in events:
del self._market_to_event_to_priority[[market, event.event_id()]]
if event.event_id(
) in self._market_to_event_to_priority_before.get([market]):
del self._market_to_event_to_priority_before[[
market, event.event_id()
]]
def __init__(self, logger):
OrderLevelBookListener.__init__(self, logger)
self._market_to_subchain_id_to_time = NDeepDict(depth=2,
default_factory=list)
self._market_to_side_to_prev_tob_subchain_id = NDeepDict(depth=2)
class SubchainTimeAtTopPriorityListener(OrderLevelBookListener):
def __init__(self, logger):
OrderLevelBookListener.__init__(self, logger)
self._market_to_subchain_id_to_time = NDeepDict(depth=2,
default_factory=list)
self._market_to_side_to_prev_tob_subchain_id = NDeepDict(depth=2)
def notify_book_update(self, order_book, causing_order_chain, tob_updated):
"""
every time an order book updates, for the side of the causing order chain check the top priority subchain and
track the amount of time that subchain is at top priority.
Top priority means it is the next to be filled.
"""
side = causing_order_chain.side()
top_priority_subchain_id = None
chains = order_book.order_chains_at_price(side,
order_book.best_price(side))
if len(chains) > 0:
top_priority_subchain_id = chains[0].most_recent_subchain(
).subchain_id()
market = order_book.market()
use_time = order_book.last_update_time()
prev_top_priority_subchain_id = self._market_to_side_to_prev_tob_subchain_id.get(
[market, side])
if prev_top_priority_subchain_id is not None:
# if top priority subchain is none and previous top subchain is not None, then close out open time range
# of prev
if top_priority_subchain_id is None:
l = self._market_to_subchain_id_to_time.get(
[market, prev_top_priority_subchain_id])
l[-1] = (l[-1][0], use_time)
# if top priority id is different than the previous then close out previous and open new one
elif top_priority_subchain_id != prev_top_priority_subchain_id:
l_old = self._market_to_subchain_id_to_time.get(
[market, prev_top_priority_subchain_id])
l_old[-1] = (l_old[-1][0], use_time)
l_new = self._market_to_subchain_id_to_time.get(
[market, top_priority_subchain_id])
l_new.append((use_time, None))
# if the same then just keep going, no need to update anything
else: # if prev is None then we are just creating a new one
l = self._market_to_subchain_id_to_time.get(
[market, top_priority_subchain_id])
l.append((use_time, None))
# and set prev to current
self._market_to_side_to_prev_tob_subchain_id[[
market, side
]] = top_priority_subchain_id
def time_at_top_priority(self, market, subchain_id, query_time=None):
"""
Gets the time a subchain spent at top priority. If subchain was never
at top priority (or didn't even exit) returns 0.
if query_time is not None (defaults to None) then that time will be used as the ceiling for the query.
This means if no end time on the last tuple this will be used as the end time. It also means that if any tuple
overlaps the query time the query time will be used as a cut off.
:param market: MarketObjects.Market.Market
:param subchain_id: subchain identifier
:param query_time: float. The time since epoch of the query (seconds.milli/microseconds)
:return: float
"""
total_time = 0
tob_time_ranges = self._market_to_subchain_id_to_time.get(
[market, subchain_id])
if len(tob_time_ranges) == 0:
return 0
for tob_time_range in tob_time_ranges[:-1]:
if query_time is not None and tob_time_range[0] >= query_time:
return total_time # no need to go through the rest of list, found the forced end here
if tob_time_range[1] is None:
self._logger.warning(
"%s %s: Cannot correctly calculate top priority time because a non last range end time is None: %s"
% (str(market), str(subchain_id), str(tob_time_ranges)))
continue
if query_time is not None and tob_time_range[1] > query_time:
total_time += query_time - tob_time_range[0]
return total_time # no need to go through the rest of list, found the forced end here
total_time += tob_time_range[1] - tob_time_range[0]
tob_time_range = tob_time_ranges[-1]
if query_time is None:
if tob_time_range[1] is None:
self._logger.warning(
"%s %s: Cannot correctly calculate top priority time because last range end time and query time are None: %s"
% (str(market), str(subchain_id), str(tob_time_ranges)))
else:
total_time += tob_time_range[1] - tob_time_range[0]
else:
if tob_time_range[0] < query_time:
if query_time >= tob_time_range[1]:
total_time += tob_time_range[1] - tob_time_range[0]
else:
total_time += query_time - tob_time_range[0]
return total_time
def clean_up_order_chain(self, order_chain):
market = order_chain.market()
for subchain in order_chain.subchains():
del self._market_to_subchain_id_to_time[[
market, subchain.subchain_id()
]]
class OrderEventCountListener(OrderEventListener):
# TODO unit test
# TODO document listener
# COUNT TYPE
NEW_ORDER = 1
ACK = 2
ACK_NEW_ORDERS = 3
ACK_CANCEL_REPLACE = 4
CANCEL_REPLACE = 5
CANCEL_REQUEST = 6
CANCEL_CONFIRM = 7
NEW_FAK = 8
NEW_FAR = 9
NEW_FOK = 10
PARTIAL_FILL = 11
FULL_FILL = 12
REJECT = 13
REJECT_NEW = 14
REJECT_CANCEL_REPLACE = 15
REJECT_CANCEL = 16
NEW_LIMIT = 17
NEW_MARKET = 18
FAKS_FULLY_FILLED = 19
FAKS_PARTIALLY_FILLED = 20
FOKS_FULLY_FILLED = 21
FARS_FULLY_FILLED_ON_PLACEMENT = 22
FARS_PARTIALLY_FILLED_ON_PLACEMENT = 23
def __init__(self, logger):
OrderEventListener.__init__(self, logger)
# storing this market -> user -> count type -> count
self._event_counts = NDeepDict(3, int)
def get_count(
self, market, user_id, count_type
): # get_count(two_year, "user_a", OrderEventCountListener.NEW_FAK)
return self._event_counts.get([market, user_id, count_type])
# REQUESTS / COMMANDS IN ######################################
def handle_new_order_command(self, new_order_command,
resulting_order_chain):
# to be optionally implemented by child class
self._event_counts[[
new_order_command.market(),
new_order_command.user_id(), self.NEW_ORDER
]] += 1
# Time In Force Counts
if new_order_command.time_in_force() == OrderEventConstants.FAR:
self._event_counts[[
new_order_command.market(),
new_order_command.user_id(), self.NEW_FAR
]] += 1
elif new_order_command.time_in_force() == OrderEventConstants.FAK:
self._event_counts[[
new_order_command.market(),
new_order_command.user_id(), self.NEW_FAK
]] += 1
elif new_order_command.time_in_force() == OrderEventConstants.FOK:
self._event_counts[[
new_order_command.market(),
new_order_command.user_id(), self.NEW_FOK
]] += 1
# Market and limit
if new_order_command.is_market_order():
self._event_counts[[
new_order_command.market(),
new_order_command.user_id(), self.NEW_MARKET
]] += 1
elif new_order_command.is_limit_order():
self._event_counts[[
new_order_command.market(),
new_order_command.user_id(), self.NEW_LIMIT
]] += 1
def handle_cancel_replace_command(self, cancel_replace_command,
resulting_order_chain):
self._event_counts[[
cancel_replace_command.market(),
cancel_replace_command.user_id(), self.CANCEL_REPLACE
]] += 1
def handle_cancel_command(self, cancel_command, resulting_order_chain):
self._event_counts[[
cancel_command.market(),
cancel_command.user_id(), self.CANCEL_REQUEST
]] += 1
# RESPONSES / MESSAGES OUT #####################################
def handle_acknowledgement_report(self, acknowledgement_report,
resulting_order_chain):
self._event_counts[[
acknowledgement_report.market(),
acknowledgement_report.user_id(), self.ACK
]] += 1
if isinstance(acknowledgement_report.acknowledged_command(),
NewOrderCommand):
self._event_counts[[
acknowledgement_report.market(),
acknowledgement_report.user_id(), self.ACK_NEW_ORDERS
]] += 1
# if ack comes back for a FAR for a new order, and there is a partial fill in teh orderchain then partially filled on placement
if resulting_order_chain.time_in_force(
) == OrderEventConstants.FAR and resulting_order_chain.has_partial_fill(
):
self._event_counts[[
acknowledgement_report.market(),
acknowledgement_report.user_id(),
self.FARS_PARTIALLY_FILLED_ON_PLACEMENT
]] += 1
elif isinstance(acknowledgement_report.acknowledged_command(),
CancelReplaceCommand):
self._event_counts[[
acknowledgement_report.market(),
acknowledgement_report.user_id(), self.ACK_CANCEL_REPLACE
]] += 1
def handle_partial_fill_report(self, partial_fill_report,
resulting_order_chain):
self._event_counts[[
partial_fill_report.market(),
partial_fill_report.user_id(), self.PARTIAL_FILL
]] += 1
def handle_full_fill_report(self, full_fill_report, resulting_order_chain):
self._event_countS[[
full_fill_report.market(),
full_fill_report.user_id(), self.FULL_FILL
]] += 1
def handle_cancel_report(self, cancel_report, resulting_order_chain):
self._event_counts[[
cancel_report.market(),
cancel_report.user_id(), self.CANCEL_CONFIRM
]] += 1
def handle_reject_report(self, reject_report, resulting_order_chain):
self._event_counts[[
reject_report.market(),
reject_report.user_id(), self.REJECT
]] += 1
if isinstance(reject_report.rejected_command(), NewOrderCommand):
self._event_counts[[
reject_report.market(),
reject_report.user_id(), self.REJECT_NEW
]] += 1
elif isinstance(reject_report.rejected_command(),
CancelReplaceCommand):
self._event_counts[[
reject_report.market(),
reject_report.user_id(), self.REJECT_CANCEL_REPLACE
]] += 1
elif isinstance(reject_report.rejected_command(), CancelCommand):
self._event_counts[[
reject_report.market(),
reject_report.user_id(), self.REJECT_CANCEL
]] += 1
# CLOSE OUT THE CHAIN ##########################################
def handle_chain_close(self, closed_order_chain):
if closed_order_chain.has_full_fill():
if closed_order_chain.time_in_force() == OrderEventConstants.FAK:
self._event_counts[[
closed_order_chain.market(),
closed_order_chain.user_id(), self.FAKS_FULLY_FILLED
]] += 1
elif closed_order_chain.time_in_force() == OrderEventConstants.FOK:
self._event_counts[[
closed_order_chain.market(),
closed_order_chain.user_id(), self.FOKS_FULLY_FILLED
]] += 1
elif closed_order_chain.time_in_force() == OrderEventConstants.FAR:
# if a FAR has no acknowledgement when fully filled, then it was fully filled on placement
if not closed_order_chain.has_acknowledgement():
self._event_counts[[
closed_order_chain.market(),
closed_order_chain.user_id(),
self.FARS_FULLY_FILLED_ON_PLACEMENT
]] += 1
elif closed_order_chain.has_partial_fill():
if closed_order_chain.time_in_force() == OrderEventConstants.FAK:
self._event_counts[[
closed_order_chain.market(),
closed_order_chain.user_id(), self.FAKS_PARTIALLY_FILLED
]] += 1
class SubchainTimeAtTOBListener(OrderLevelBookListener):
"""
Tracks how long a subchain is at the top of book.
This is designed so it can work with mulitiple order books at once.
"""
# TODO UNIT TEST
def __init__(self, logger):
OrderLevelBookListener.__init__(self, logger)
self._market_to_subchain_id_to_time = NDeepDict(
depth=2, default_factory=list
) # this a list of tuples (start_time, end_time)
self._market_to_side_to_prev_tob_subchain_ids = NDeepDict(
depth=2, default_factory=set)
def notify_book_update(self, order_book, causing_order_chain, tob_updated):
"""
Every time an orderbook comes in, look at the top of book subchains. Only need to do it for the side that is
updating.
If the same as the previous top of book price then we just update that
price's last TOB time.
If not in previous then create a new time (and make sure previous time, if there is one, is valid)
If in previous then move on with no change.
If in open list and not in top of book, then close it with the time of update
"""
# TODO only run the below code if the top of book for the side in question has changed
side = causing_order_chain.side()
use_time = order_book.last_update_time()
if use_time != causing_order_chain.last_update_time():
self._logger.warning(
"Order book update time (%.6f) and causing order chain update time (%.6f) do not match!"
% (use_time, causing_order_chain.last_update_time()))
order_chains = order_book.iter_order_chains_at_price(
side, order_book.best_price(side))
market = order_book.market()
prev_subchain_ids = self._market_to_side_to_prev_tob_subchain_ids.get(
[market, side])
found_subchain_ids = set()
for order_chain in order_chains:
subchain_id = order_chain.most_recent_subchain().subchain_id()
found_subchain_ids.add(subchain_id)
# if in the previous grouping then do nothing
# if not in the previous grouping then create a new tuple with None for end time
if subchain_id not in prev_subchain_ids:
l = self._market_to_subchain_id_to_time.get(
[market, subchain_id])
l.append((use_time, None))
for prev_subchain_id in prev_subchain_ids:
# if something previously found wasn't found this time around then we need to close it out
if prev_subchain_id not in found_subchain_ids:
l = self._market_to_subchain_id_to_time.get(
[market, prev_subchain_id])
l[-1] = (l[-1][0], use_time)
# set previously found to be the new found
self._market_to_side_to_prev_tob_subchain_ids[[market, side
]] = found_subchain_ids
def time_at_top_of_book(self, market, subchain_id, query_time=None):
"""
Gets the time a subchain spent at top of book. If subchain was never
at top of book (or didn't even exit) returns 0.
if query_time is not None (defaults to None) then that time will be used as the ceiling for the query.
This means if no end time on the last tuple this will be used as the end time. It also means that if any tuple
overlaps the query time the query time will be used as a cut off.
:param market: MarketObjects.Market.Market
:param subchain_id: subchain identifier
:param query_time: float. The time since epoch of the query (seconds.milli/microseconds)
:return: float
"""
total_time = 0
tob_time_ranges = self._market_to_subchain_id_to_time.get(
[market, subchain_id])
if len(tob_time_ranges) == 0:
return 0
for tob_time_range in tob_time_ranges[:-1]:
if query_time is not None and tob_time_range[0] >= query_time:
return total_time # no need to go through the rest of list, found the forced end here
if tob_time_range[1] is None:
self._logger.warning(
"%s %s: Cannot correctly calculate TOB time because a non last range end time is None: %s"
% (str(market), str(subchain_id), str(tob_time_ranges)))
continue
if query_time is not None and tob_time_range[1] > query_time:
total_time += query_time - tob_time_range[0]
return total_time # no need to go through the rest of list, found the forced end here
total_time += tob_time_range[1] - tob_time_range[0]
tob_time_range = tob_time_ranges[-1]
if query_time is None:
if tob_time_range[1] is None:
self._logger.warning(
"%s %s: Cannot correctly calculate TOB time because last range end time and query time are None: %s"
% (str(market), str(subchain_id), str(tob_time_ranges)))
else:
total_time += tob_time_range[1] - tob_time_range[0]
else:
if tob_time_range[0] < query_time:
if query_time >= tob_time_range[1]:
total_time += tob_time_range[1] - tob_time_range[0]
else:
total_time += query_time - tob_time_range[0]
return total_time
def clean_up_order_chain(self, order_chain):
market = order_chain.market()
for subchain in order_chain.subchains():
del self._market_to_subchain_id_to_time[[
market, subchain.subchain_id()
]]
def __init__(self, logger):
OrderEventListener.__init__(self, logger)
# storing this market -> user -> count type -> count
self._event_counts = NDeepDict(3, int)
class LastTimeTOBListener(OrderLevelBookListener, OrderEventListener):
"""
Tracks when prices were top of book so that at any given time we can query
with a price and side and determine when the last time that price was top
of book.
If you use the same price but the other side, you get when the last time the
price would have crossed the book.
This is designed so it can work with multiple order books at once.
"""
# TODO UNIT TEST
def __init__(self, logger):
OrderLevelBookListener.__init__(self, logger)
OrderEventListener.__init__(self, logger)
self._market_to_side_prev_price = NDeepDict(depth=2)
self._market_side_price_time = NDeepDict(depth=3) # market to side to price to last time it was top of book
self._market_side_best_price = NDeepDict(depth=2)
self._event_id_to_last_time_crossed = {}
self._event_id_to_last_time_tob = {}
def _update_based_on_new_event(self, market, time):
for side in [BID_SIDE, ASK_SIDE]:
# TODO could also loop over every market if we needed to do so here but that impacts performance for what I think is minimal gain
prev_best_price = self._market_to_side_prev_price.get([market, side])
if prev_best_price is not None:
self._market_side_price_time[market, side, prev_best_price] = time
def handle_new_order_command(self, new_order_command, resulting_order_chain):
assert isinstance(new_order_command, NewOrderCommand)
event_id = new_order_command.event_id()
market = new_order_command.market()
time = new_order_command.timestamp()
self._update_based_on_new_event(market, time)
price = new_order_command.price()
side = new_order_command.side()
if event_id not in self._event_id_to_last_time_crossed:
self._event_id_to_last_time_crossed[event_id] = self._last_time_crossed(market, side, price)
if event_id not in self._event_id_to_last_time_tob:
self._event_id_to_last_time_tob[event_id] = self._last_time_was_tob(market, side, price)
def handle_cancel_replace_command(self, cancel_replace_command, resulting_order_chain):
assert isinstance(cancel_replace_command, CancelReplaceCommand)
event_id = cancel_replace_command.event_id()
market = cancel_replace_command.market()
time = cancel_replace_command.timestamp()
self._update_based_on_new_event(market, time)
price = cancel_replace_command.price()
side = cancel_replace_command.side()
if event_id not in self._event_id_to_last_time_crossed:
self._event_id_to_last_time_crossed[event_id] = self._last_time_crossed(market, side, price)
if event_id not in self._event_id_to_last_time_tob:
self._event_id_to_last_time_tob[event_id] = self._last_time_was_tob(market, side, price)
def last_time_was_tob(self, event_id):
last_time_tob = self._event_id_to_last_time_tob[event_id]
return last_time_tob
def last_time_crossed(self, event_id):
last_time_crossed = self._event_id_to_last_time_crossed[event_id]
if last_time_crossed is not None:
del self._event_id_to_last_time_crossed[event_id]
return last_time_crossed
def notify_book_update(self, order_book, causing_order_chain, tob_updated):
"""
Every time an orderbook comes in, look at the top of book price.
If TOB didn't change at all then just
If the same as the previous top of book price then we just update that
price's last TOB time.
If a new top of book, we need to:
1) update the previous price to the new time (since it was TOB up to and including this new time)
2) update the new price to the new time
"""
time = order_book.last_update_time()
market = order_book.market()
# only need to do the side that was updated by the causing order chain as that is only side that changed
side = causing_order_chain.side()
best_price = order_book.best_price(side)
# if current book best price is not none then need to set its time
if best_price is not None:
self._market_side_price_time[market, side, best_price] = time
prev_best_price = self._market_side_best_price.get([market, side])
if prev_best_price is None or best_price.better_than(prev_best_price, side):
self._market_side_best_price[market, side] = best_price
# if previous best price is not none and is different than new best price then need to set its time to
# update it to include previous price period since it was best price until the change
prev_best_price = self._market_to_side_prev_price.get([market, side])
if prev_best_price is not None and prev_best_price != best_price:
self._market_side_price_time[market, side, prev_best_price] = time
# set previous best price to be the current best price
self._market_to_side_prev_price[market, side] = best_price
def _last_time_was_tob(self, market, side, price):
"""
Returns the last time the price was the top of book
Will return None if the passed in price has never been top of book for
market and side during the scope of this listener
:param market: MarketObjects.Market.Market
:param side: MarketObjects.Side.Side
:param price: MarketObjects.Price.Price
:return: float. (Could be None)
"""
return self._market_side_price_time.get([market, side, price])
def _last_time_crossed(self, market, side, price):
"""
Returns the last time the price for the given side would have crossed the book.
None if it never would have crossed the book or if there was never a book to cross.
:param market: MarketObjects.Market.Market
:param side: MarketObjects.Side.Side (side of the order we are checking)
:param price: MarketObjects.Price.Price (price of the order we are checking)
:return: float. (Could be None)
"""
resting_side = side.other_side()
# if book hasn't been established yet, so return None
# there are a lot of messages that come in that never would have crossed. so just tracking a "best bid and best
# offer seen all day" would keep us from doing the below sorting and iterating for these and save a ton of time
best_resting_price = self._market_side_best_price.get([market, resting_side])
if best_resting_price is None or price.worse_than(best_resting_price, side):
return None
else:
resting_prices_to_time = self._market_side_price_time.get([market, resting_side])
# sort list based on time, where most recent time (highest number) is first
sorted_x = sorted(resting_prices_to_time.items(), key=operator.itemgetter(1), reverse=True)
for x in sorted_x:
if price.better_or_same_as(x[0], side):
return x[1]
self._logger.error("last_time_crossed has a price that would have crossed today but still has no cross time value. Something is broken!")
# failsafe return?
return None
def clean_up(self, order_chain):
# if order chain events in map, clean up
assert isinstance(order_chain, OrderEventChain)
events = order_chain.events()
for event in events:
try:
del self._event_id_to_last_time_crossed[event.event_id()]
except:
pass # silent fail is okay here. All I'm doing is deleting if it exists in dict. If it doesn't exist no need to delete so failure state is acceptable and faster than checking for existence first.
try:
del self._event_id_to_last_time_tob[event.event_id()]
except:
pass # silent fail is okay here. All I'm doing is deleting if it exists in dict. If it doesn't exist no need to delete so failure state is acceptable and faster than checking for existence first.
def test_first_none_default_2_deep():
d = NDeepDict(2)
d[['a', 'b']] = 16
assert d.get(['d', 'e']) is None
class AggressiveImpactListener(OrderLevelBookListener, OrderEventListener):
"""
Tracks the Aggressive Impact of each subchain.
The aggressive impact of a subchain is how many levels of the order book
that the subchain took out upon open. The left of the decimal is how many
entire price levels the subchain took out and the right of the decimal is
the percentage of the next level.
Aggressive impact value examples:
* 2 and a half levels --> 2.5
* 80% of top of book only --> 0.8
* 4 levels exactly --> 4.0
* doesn't aggress --> 0.0
"""
# TODO UNIT TEST
def __init__(self, logger):
OrderLevelBookListener.__init__(self, logger)
OrderEventListener.__init__(self, logger)
self._market_event_id_aggressive_act = NDeepDict(depth=2)
self._market_to_orderbook = defaultdict(lambda: None)
self._market_to_agg_acts_to_close = defaultdict(set)
def handle_acknowledgement_report(self, acknowledgement_report,
resulting_order_chain):
event_id = acknowledgement_report.causing_command().event_id()
market = acknowledgement_report.market()
if market not in self._market_to_orderbook:
return
agg_event = self._market_event_id_aggressive_act.get(
[market, event_id])
#if aggressor is acked then all fills on both sides shoud be done and we can calculate
if agg_event:
agg_event.calculate(self._market_to_orderbook[market])
def _handle_fill(self, fill, resulting_order_chain):
match_id = fill.match_id()
market = fill.market()
if market not in self._market_to_orderbook:
return
event_id = fill.causing_command().event_id()
agg_act = self._market_event_id_aggressive_act.get([market, event_id])
# if the event_id of the aggressor does not already exist, we create it
if not agg_act:
agg_act = AggressiveAct(match_id)
self._market_event_id_aggressive_act[[market, event_id]] = agg_act
agg_act.add_fill(fill)
def handle_partial_fill_report(self, partial_fill_report,
resulting_order_chain):
self._handle_fill(partial_fill_report, resulting_order_chain)
def handle_full_fill_report(self, full_fill_report, resulting_order_chain):
market = full_fill_report.market()
if market not in self._market_to_orderbook:
return
self._handle_fill(full_fill_report, resulting_order_chain)
# on an aggressive full fill we close out the open order chain
if full_fill_report.is_aggressor():
event_id = full_fill_report.causing_command().event_id()
agg_act = self._market_event_id_aggressive_act[[market, event_id]]
if agg_act is not None:
# if full fill and qty balanced we should be able to do the impact calculation right now because book up to date
if agg_act.balanced_match_qty():
agg_act.calculate(self._market_to_orderbook[market])
else: # we need to get the order level books after all the updates are done.
self._market_to_agg_acts_to_close[market].add(
self._market_event_id_aggressive_act.get(
[market, event_id]))
else:
raise Exception(
"Got an aggressive full fill but not tracking aggressive acts for event: %s"
% str(event_id))
def handle_cancel_report(self, cancel_report, resulting_order_chain):
# on a cancel we close out the open order chain because of FAKs
market = cancel_report.market()
if market not in self._market_to_orderbook:
return
event_id = cancel_report.causing_command().event_id()
agg_event = self._market_event_id_aggressive_act.get(
[market, event_id])
# if aggressor is cancelled then all fills on both sides should be done and we can calculate
if agg_event:
agg_event.calculate(self._market_to_orderbook[market])
def clean_up(self, order_chain):
"""
If clean_up is called with an order_chain than this will go through the
order chain's subchain IDs and delete each one from the maps that store
the aggressive acts for a subchain.
WARNING: once this is called, the get_aggressive_impact and
get_aggressive_qty will no longer return the values that are
meaningful; rather, you'll get the 0 values
:param order_chain: MarketObjects.Events.EventChains.OrderEventChain
"""
market = order_chain.market()
for event in order_chain.events():
del self._market_event_id_aggressive_act[[
market, event.event_id()
]]
def notify_book_update(self, order_book, causing_order_chain, tob_updated):
market = order_book.market()
self._market_to_orderbook[market] = order_book
remove_set = set()
agg_acts_to_close = self._market_to_agg_acts_to_close[market]
for agg_act in agg_acts_to_close:
if agg_act.balanced_match_qty():
agg_act.calculate(order_book)
self._market_to_agg_acts_to_close[
market] = agg_acts_to_close.difference(remove_set)
def get_aggressive_impact(self, market, event_id):
if self._market_event_id_aggressive_act.get([market,
event_id]) is not None:
impact = self._market_event_id_aggressive_act.get(
(market, event_id)).impact()
return impact
return 0.0
def get_aggressive_qty(self, market, event_id):
if self._market_event_id_aggressive_act.get([market,
event_id]) is not None:
return self._market_event_id_aggressive_act.get([market, event_id
]).match_qty()
return 0
class MarketOrderTicksFromCrossing(OrderLevelBookListener, OrderEventListener):
"""
Tracks how far away a MarketOrder was from crossing the opposite TOB.
* Greater than 0 is how many ticks away from crossing (but did not cross)
* 0 means it crossed TOB
* less than 0 is how many ticks through the opposite TOB the order went
* None means there was no opposite side to cross (or the order chain id
being asked about wasn't a market order)
Tracks only for order chain id because at the time of the new order, the
subchain hasn't been created yet (doesn't get created until the execution
report) so does not have a subchain id to use.
This is designed so it can work with mulitiple order books at once.
"""
# TODO UNIT TEST
def __init__(self, logger):
OrderLevelBookListener.__init__(self, logger)
self._market_chain_id_ticks = NDeepDict(2)
self._market_side_tob = {}
self._market_side_tob[BID_SIDE] = defaultdict()
self._market_side_tob[ASK_SIDE] = defaultdict()
def handle_new_order_command(self, new_order_command,
resulting_order_chain):
# only applies to new order commands
if new_order_command.is_market_order():
side = new_order_command.side()
price = new_order_command.price()
market = new_order_command.market()
mpi = market.mpi()
opp_price = self._market_side_tob[market][side.other_side()]
ticks_away = ((opp_price - price) if side.is_bid() else
(price - opp_price)) / mpi
self._market_chain_id_ticks[
market, new_order_command.chain_id()] = ticks_away
def notify_book_update(self, order_book, causing_order_chain, tob_updated):
"""
Every time an orderbook comes in and tob updated need to save the bid price and the ask
price. And only need to update for the side that is being updated.
"""
if tob_updated:
market = order_book.market()
side = causing_order_chain.side()
self._market_side_tob[market][side] = order_book.best_price(side)
def ticks_from_crossing(self, market, chain_id):
"""
Gets how many ticks away from crossing the chain id was.
* Greater than 0 is how many ticks away from crossing (but did not cross)
* 0 means it crossed TOB
* less than 0 is how many ticks through the opposite TOB the order went
* None means there was no opposite side to cross (or the order chain id
being asked about wasn't a market order)
:param market: MarketObjects.Market.Market
:param chain_id: order chain's unique identifier
:return: int
"""
return self._market_chain_id_ticks.get([market, chain_id])
def __init__(self, logger):
OrderLevelBookListener.__init__(self, logger)
self._market_chain_id_ticks = NDeepDict(2)
self._market_side_tob = {}
self._market_side_tob[BID_SIDE] = defaultdict()
self._market_side_tob[ASK_SIDE] = defaultdict()