def __init__(self, **kwargs):
self.path = os.path.dirname(__file__)
ticker = kwargs.get('ticker')
date = kwargs.get('date')
year = date.year
month = date.month
day = date.day
start_h = kwargs.get('start_h', 9)
end_h = kwargs.get('end_h', 17.5)
start_secs = int(start_h * 3600)
end_secs = int(end_h * 3600)
start_time = datetime(year, month, day) + timedelta(0, start_secs)
end_time = datetime(year, month, day) + timedelta(0, end_secs)
self.ob_time = start_time
self.latency = kwargs.get('latency', 20000)
self.my_queue = deque()
self.ob_idx = 0
self.ob = Orderbook(ticker=ticker)
date = f'{year}-{month}-{day}'
self.ob.date = ticker, date
self.OrdTuple = namedtuple('Order',
'ordtype uid is_buy qty price timestamp')
self.my_last_uid = 0
# load historical orders from csv file
session = f'{self.path}/../data/historic_orders/orders-{ticker}-{date}.csv'
csv = pd.read_csv(session, sep=';', float_precision='round_trip')
csv['timestamp'] = pd.to_datetime(csv['timestamp'])
# We will be working with ndarrays instead of DataFrames for speed
self.hist_orders = csv.values
self.ob_nord = csv.shape[0]
# we store index positions of columns for array indexing
columns = csv.columns
self.col_idx = {}
for col_name in csv.columns:
self.col_idx.update({col_name: np.argmax(columns == col_name)})
last_ord_time = self.hist_orders[-1][self.col_idx['timestamp']]
self.end_time = min(last_ord_time, end_time)
self.stop_time = self.end_time
# book positions (bid+ask) available in historical data
book_pos = 20
# send first 20 orders that will compose first orderbook snapshot
# this is the real orderbook that was present when the orderbook opened
# right after the opening auction
for ord_idx in range(book_pos):
oborder = self.hist_orders[self.ob_idx]
self._send_historical_order(oborder)
self.move_historic_until(start_time)
self.ob.reset_ob(reset_all=False)
self.in_queue = dict()
self.vol_in_queue = 0
def test_send_bid_to_empty_book(self, bid1):
orderbook = Orderbook('band6stock')
orderbook.send(*bid1)
assert orderbook._bids.best.price == bid1.price
assert orderbook._bids.best.head.uid == bid1.uid
assert orderbook._bids.best.tail.uid == bid1.uid
assert orderbook._orders[bid1.uid].uid == bid1.uid
assert orderbook.get(bid1.uid)['active']
def test_send_ask_to_empty_book(self, ask1):
orderbook = Orderbook('band6stock')
orderbook.send(*ask1)
assert orderbook._asks.best.price == ask1.price
assert orderbook._asks.best.head.uid == ask1.uid
assert orderbook._asks.best.tail.uid == ask1.uid
assert orderbook._orders[ask1.uid].uid == ask1.uid
assert orderbook.get(ask1.uid)['active']
def test_band6_stock_get_new_price(self):
# we test in the price boundary of tick size change
orderbook = Orderbook('band6stock')
assert orderbook.get_new_price(5, n_moves=-1) == 4.9995
assert orderbook.get_new_price(5, n_moves=1) == 5.001
assert orderbook.get_new_price(10, n_moves=-1) == 9.999
assert orderbook.get_new_price(10, n_moves=1) == 10.002
assert orderbook.get_new_price(20, n_moves=-1) == 19.998
assert orderbook.get_new_price(20, n_moves=1) == 20.005
assert orderbook.get_new_price(100, n_moves=-1) == 99.99
assert orderbook.get_new_price(100, n_moves=1) == 100.02
class Gateway():
""" Creates an empty Python Matching Engine (orderbook simulator) and injects
real historical orders to it creating the real orderbooks and trades
that happened in that orderbook session. It also allows you to send
your own orders to make them interact (i.e. cross) with the historical
orderbooks that were present that day. The orders you send to the orderbook
through this Gateway will also experience latency as they
would in real life.
The Gateway allows us to run a synchronous simulation of the interaction
of your algorithm with a Python Matching Engine (orderbook simulator)
that will be injected with real life historical orders of a
past orderbook session while taking into account the effect
of this latency.
For example, when your algorithm receives a new orderbook best bid price,
actually this price happened "md_latency" microseconds in the past,
the time it took to reach your algorithm. Your algo will take "algo_latency"
microseconds to make a decission and send a message (new/cancel/modif),
and finally, this message will take "ob_latency" microseconds to reach
the orderbook because of the physical distance and the different systems
it needs to cross through before reaching the orderbook.
The total latency will be:
latency = md_latency + algo_latency + ob_latency
When you send messages to a orderbook through this Gateway,
your messages will reach the orderbook "latency" microseconds
after the time of the last historical order that reached the orderbook
and that produced the last orderbook data update upon which your
algo made its last decission.
Args:
ticker (str): symbol of the shares
year (int): year
month (int): month
day (int): day
latency (int): mean latency in microseconds that we expect
our orders to have in real life when sent
to the orderbook (orderbook data one way
+ algo decission time
+ orderbook access one way)
"""
def __init__(self, **kwargs):
ticker = kwargs.get('ticker')
date = kwargs.get('date')
datetimedate = datetime.strptime(date, '%Y-%m-%d')
year = datetimedate.year
month = datetimedate.month
day = datetimedate.day
start_h = kwargs.get('start_h', 9)
end_h = kwargs.get('end_h', 17.5)
start_secs = int(start_h * 3600)
end_secs = int(end_h * 3600)
start_time = datetimedate + timedelta(0, start_secs)
end_time = datetimedate + timedelta(0, end_secs)
self.latency = kwargs.get('latency', 20_000)
self.my_queue = deque()
self.ob_idx = 0
self.ob = Orderbook(ticker=ticker)
self.ob.date = ticker, date
self.OrdTuple = namedtuple('Order',
'ordtype uid is_buy qty price timestamp')
self.my_last_uid = 0
# load historical orders from csv file
session = f'./data/historic_orders/orders-{ticker}-{date}.csv'
csv = pd.read_csv(session, sep=';', float_precision='round_trip')
csv['timestamp'] = pd.to_datetime(csv['timestamp'])
# We will be working with ndarrays instead of DataFrames for speed
self.hist_orders = csv.values
self.ob_nord = csv.shape[0]
# we store index positions of columns for array indexing
columns = csv.columns
self.col_idx = {}
for col_name in csv.columns:
self.col_idx.update({col_name: np.argmax(columns == col_name)})
last_ord_time = self.hist_orders[-1][self.col_idx['timestamp']]
self.end_time = min(last_ord_time, end_time)
self.stop_time = self.end_time
# book positions (bid+ask) available in historical data
BOOK_POS = 20
# send first 20 orders that will compose first orderbook snapshot
# this is the real orderbook that was present when the orderbook opened
# right after the opening auction
for ord_idx in range(BOOK_POS):
oborder = self.hist_orders[self.ob_idx]
self._send_historical_order(oborder)
self.move_until(start_time)
self.ob.reset_ob(reset_all=False)
def _send_to_orderbook(self, order, is_mine):
""" Send an order/modif/cancel to the orderbook
order (ndarray): order to be sent
is_mine (bool): False if historical, True if user sent
"""
ord_type = order[self.col_idx['ordtype']]
timestamp = order[self.col_idx['timestamp']]
# ob_open = self.check_ob_open(timestamp)
if self.check_ord_in_time(timestamp):
self.update_ob_time(timestamp)
if ord_type == "new":
self.ob.send(is_buy=order[self.col_idx['is_buy']],
qty=order[self.col_idx['qty']],
price=order[self.col_idx['price']],
uid=order[self.col_idx['uid']],
is_mine=is_mine,
timestamp=timestamp)
elif ord_type == "cancel":
self.ob.cancel(uid=order[self.col_idx['uid']])
elif ord_type == "modif":
self.ob.modif(uid=order[self.col_idx['uid']],
qty_down=order[self.col_idx['qty']])
else:
raise ValueError(f'Unexpected ordtype: {ord_type}')
return
else:
self.update_ob_time(self.stop_time)
if not is_mine:
self.ob_idx -= 1
return
def update_ob_time(self, new_ob_time):
self.ob_time = new_ob_time
def move_n_seconds(self, n_seconds):
"""
"""
self.stop_time = min(self.ob_time + timedelta(0, n_seconds),
self.end_time)
while (self.ob_time < self.stop_time):
self.tick()
self.stop_time = self.end_time
def check_ord_in_time(self, ord_timestamp):
"""
"""
return (ord_timestamp < self.stop_time)
def _send_historical_order(self, oborder):
self.ob_idx += 1
self._send_to_orderbook(oborder, is_mine=False)
def move_until(self, stop_time):
"""
Params:
stop_time (datetime):
"""
while (self.ob_time <= stop_time):
oborder = self.hist_orders[self.ob_idx]
self._send_historical_order(oborder)
def tick(self):
""" Move the orderbook forward one tick (process next order)
If the user has messages (new/cancel/modif) queued, it will
decide whether to send a user or historical order based on
their theoretical arrival time (timestamp)
"""
# next historical order to be sent
oborder = self.hist_orders[self.ob_idx]
# if I have queued orders
if self.my_queue:
# if my order reaches the orderbook before the next historical order
if self.my_queue[0].timestamp < oborder[self.col_idx['timestamp']]:
my_order = self.my_queue.popleft()
self._send_to_orderbook(my_order, is_mine=True)
return
# otherwise sent next historical order
self._send_historical_order(oborder)
def queue_my_new(self, is_buy, qty, price):
""" Queue a user new order to be sent to the orderbook when time is due
Args:
is_buy (bool): True for buy orders
qty (int): quantity or volume
price (float): limit price of the order
Reuturns:
An int indicating the uid that the orderbook will assign to
it when it is introudced.
NOTES: as the order is queued by this function, its uid does
not exist yet in the orderbook. It will not exist until
the time is due and the order reaches the orderbook. Requesting
the status of this uid will therefore raise a KeyError
meanwhile.
Uids of user orders will be negative, this
way we ensure no collisions with historical positive uids and
have an easy way to know if an order is ours
"""
self.my_last_uid -= 1
message = self.OrdTuple(ordtype="new",
uid=self.my_last_uid,
is_buy=is_buy,
qty=qty,
price=price,
timestamp=self._arrival_time())
self.my_queue.append(message)
return self.my_last_uid
def queue_my_modif(self, uid, qty_down):
""" Modify an order identified by its uid without loosing priority.
Modifications can only downsize the volume.
If you attempt to increase the volume, the
modification message will do nothing. Downsizing volume will
mantain your price-time priority in the orderbook. If you want to
increase volume or change price, you need to cancel your previous
order and send a new one.
Args:
uid (int): uid of our order to be modified
qty_down(int): Downsizing quantity. Only downsizing allowed.
"""
message = self.OrdTuple(ordtype="modif",
uid=uid,
is_buy=np.nan,
qty=qty_down,
price=np.nan,
timestamp=self._arrival_time())
self.my_queue.append(message)
def queue_my_cancel(self, uid):
""" Cancel an order by its uid
"""
message = self.OrdTuple(ordtype="cancel",
uid=uid,
is_buy=np.nan,
qty=np.nan,
price=np.nan,
timestamp=self._arrival_time())
self.my_queue.append(message)
def ord_status(self, uid):
""" Returns the current ob status of an order identified by its uid.
Args:
uid (int): unique order identifier
NOTE: when an order is queued, its uid does not exist yet in the
orderbook since it did not arrive there yet. Calling this function
on a uid that is queued by not yet in the orderbook will raise a
KeyError exception that will have to be handled.
"""
# TODO: use ticker to select orderbook
return self.ob.get(uid)
def _arrival_time(self):
""" Returns the estimated time of arrival of an order
"""
return self.ob_time + timedelta(0, 0, self.latency)
def plot(self):
trades = pd.DataFrame(self.ob.trades)
def test_cancel_order(self):
""" Cancels active orders and checks PriceLevel deletion,
price-time priority of left resting orders, and doubly linked
list of orders inside Price Level
"""
orderbook = Orderbook('san')
o1uid = 1
o2uid = 2
o3uid = 3
o4uid = 4
o5uid = 5
order = namedtuple('Order', 'is_buy, qty, price, uid')
o1 = order(is_buy=True, qty=1000, price=0.2, uid=o1uid)
o2 = order(is_buy=True, qty=500, price=0.2, uid=o2uid)
o3 = order(is_buy=True, qty=600, price=0.2, uid=o3uid)
o4 = order(is_buy=True, qty=200, price=0.2, uid=o4uid)
o5 = order(is_buy=True, qty=77, price=0.19, uid=o5uid)
orderbook.send(*o1)
orderbook.send(*o2)
orderbook.send(*o3)
orderbook.send(*o4)
orderbook.send(*o5)
# CANCEL MIDDLE ORDER IN QUEUE
orderbook.cancel(o2uid)
# Check order is not active & leavesqty is 0
self.assertEqual(orderbook.get(o2uid)['active'], False)
self.assertEqual(orderbook.get(o2uid)['leavesqty'], 0)
# Check Doubly Linked List
self.assertIs(orderbook._orders[o1uid].next.uid, o3uid)
self.assertIs(orderbook._orders[o3uid].prev.uid, o1uid)
self.assertIs(orderbook._orders[o3uid].next.uid, o4uid)
self.assertIs(orderbook._orders[o4uid].prev.uid, o3uid)
self.assertIs(orderbook.bbid[0], o1.price)
self.assertIs(orderbook._bids.best.tail.uid, o4uid)
# CANCEL TAIL
orderbook.cancel(o4uid)
# Check order is removed
self.assertEqual(orderbook.get(o4uid)['leavesqty'], 0)
# Check Doubly Linked List
self.assertIs(orderbook._orders[o1uid].next.uid, o3uid)
self.assertIs(orderbook._orders[o3uid].prev.uid, o1uid)
self.assertIsNone(orderbook._orders[o3uid].next)
self.assertIs(orderbook._bids.best.head.uid, o1uid)
self.assertIs(orderbook._bids.best.tail.uid, o3uid)
# CANCEL HEAD
# alternate oders in PriceLevels
orderbook.cancel(o1uid)
# Check order is removed
self.assertEqual(orderbook.get(o1uid)['leavesqty'], 0)
# Check Doubly Linked List
self.assertIsNone(orderbook._orders[o3uid].prev)
self.assertIsNone(orderbook._orders[o3uid].next)
self.assertIs(orderbook._bids.best.head.uid, o3uid)
self.assertIs(orderbook._bids.best.tail.uid, o3uid)
# CANCEL HEAD&TAIL ORDER => REMOVE PRICE_LEVEL
orderbook.cancel(o3uid)
# Check order is removed
self.assertEqual(orderbook.get(o3uid)['leavesqty'], 0)
# Check PriceLevel removed
self.assertNotIn(0.2, orderbook._bids.book)
# Check new best bid
self.assertIs(orderbook._bids.best.head.uid, o5uid)
orderbook.cancel(o5uid)
self.assertIs(len(orderbook._bids.book), 0)
########################
#### SAME FOR ASKS #####
########################
orderbook = Orderbook('san')
o1uid = 1
o2uid = 2
o3uid = 3
o4uid = 4
o5uid = 5
order = namedtuple('Order', 'is_buy, qty, price, uid')
o1 = order(is_buy=False, qty=1000, price=0.2, uid=o1uid)
o2 = order(is_buy=False, qty=500, price=0.2, uid=o2uid)
o3 = order(is_buy=False, qty=600, price=0.2, uid=o3uid)
o4 = order(is_buy=False, qty=200, price=0.2, uid=o4uid)
o5 = order(is_buy=False, qty=77, price=0.21, uid=o5uid)
orderbook.send(*o1)
orderbook.send(*o2)
orderbook.send(*o3)
orderbook.send(*o4)
orderbook.send(*o5)
# CANCEL MIDDLE ORDER IN QUEUE
orderbook.cancel(o2uid)
# Check order is not active & leavesqty is 0
self.assertEqual(orderbook.get(o2uid)['active'], False)
self.assertEqual(orderbook.get(o2uid)['leavesqty'], 0)
# Check Doubly Linked List
self.assertIs(orderbook._orders[o1uid].next.uid, o3uid)
self.assertIs(orderbook._orders[o3uid].prev.uid, o1uid)
self.assertIs(orderbook._orders[o3uid].next.uid, o4uid)
self.assertIs(orderbook._orders[o4uid].prev.uid, o3uid)
self.assertIs(orderbook.bask[0], o1.price)
self.assertIs(orderbook._asks.best.tail.uid, o4uid)
# CANCEL TAIL ORDER
orderbook.cancel(o4uid)
# Check order is removed
self.assertEqual(orderbook.get(o4uid)['leavesqty'], 0)
# Check Doubly Linked List
self.assertIs(orderbook._orders[o1uid].next.uid, o3uid)
self.assertIs(orderbook._orders[o3uid].prev.uid, o1uid)
self.assertIsNone(orderbook._orders[o3uid].next)
self.assertIs(orderbook._asks.best.head.uid, o1uid)
self.assertIs(orderbook._asks.best.tail.uid, o3uid)
# CANCEL HEAD
# alternate oders in PriceLevels
orderbook.cancel(o1uid)
# Check order is removed
self.assertEqual(orderbook.get(o1uid)['leavesqty'], 0)
# Check Doubly Linked List
self.assertIsNone(orderbook._orders[o3uid].prev)
self.assertIsNone(orderbook._orders[o3uid].next)
self.assertIs(orderbook._asks.best.head.uid, o3uid)
self.assertIs(orderbook._asks.best.tail.uid, o3uid)
# CANCEL HEAD&TAIL ORDER => REMOVE PRICE_LEVEL
orderbook.cancel(o3uid)
# Check order is removed
self.assertEqual(orderbook.get(o3uid)['leavesqty'], 0)
# Check PriceLevel removed
self.assertNotIn(0.2, orderbook._asks.book)
# Check new best ask
self.assertIs(orderbook._asks.best.head.uid, o5uid)
orderbook.cancel(o5uid)
self.assertIs(len(orderbook._asks.book), 0)
def test_aggressive_orders(self):
orderbook = Orderbook('san')
# alternate oders in PriceLevels
order = namedtuple('Order', 'is_buy, qty, price, uid')
o1uid = 1
o2uid = 2
o3uid = 3
o4uid = 4
o5uid = 5
o1 = order(is_buy=True, qty=100, price=10.001, uid=o1uid)
o2 = order(is_buy=True, qty=100, price=10.002, uid=o2uid)
o3 = order(is_buy=True, qty=100, price=10.001, uid=o3uid)
o4 = order(is_buy=True, qty=100, price=10.002, uid=o4uid)
o5 = order(is_buy=True, qty=100, price=10.003, uid=o5uid)
orderbook.send(*o1)
orderbook.send(*o2)
orderbook.send(*o3)
orderbook.send(*o4)
orderbook.send(*o5)
# aggressive order to sweep all positions and place rest
o6uid = 6
o6 = order(is_buy=False, qty=100, price=9.999, uid=o6uid)
orderbook.send(*o6)
# check new top of book
self.assertIs(orderbook.bbid[0], o2.price)
# check best price
self.assertEqual(orderbook.trades_px[0], 10.003)
# send copy of o6 order
o7uid = 7
o7 = order(is_buy=False, qty=100, price=9.999, uid=o7uid)
orderbook.send(*o7)
self.assertIs(orderbook._bids.best.head.uid, o4uid)
self.assertIs(orderbook.get(o5uid)['leavesqty'], 0)
self.assertEqual(orderbook.trades_px[1], 10.002)
# send order and sweep 4 positions and leave rest in book as ask
o8uid = 8
o8 = order(is_buy=False, qty=500, price=9.999, uid=o8uid)
orderbook.send(*o8)
# check worst price
self.assertIs(len(orderbook.trades), 5)
self.assertEqual(orderbook.trades_px[4], 10.001)
# check empty bids book
self.assertIs(len(orderbook._bids.book), 0)
# check new pricelevel at o8 price
self.assertIs(orderbook._asks.best.price, o8.price)
def test_new_pricelevel(self):
""" Add new bid and ask orders and checks price-time priority
"""
orderbook = Orderbook('san')
bidprice = np.random.uniform(0.0001, 100000)
o1uid = 1
o2uid = 2
o3uid = 3
order = namedtuple('Order', 'is_buy, qty, price, uid')
o1 = order(is_buy=True, qty=10, price=bidprice, uid=o1uid)
o2 = order(is_buy=True, qty=5, price=bidprice, uid=o2uid)
o3 = order(is_buy=True, qty=7, price=bidprice, uid=o3uid)
# Check price level creation, heads and tails, uid & order active
orderbook.send(*o1)
self.assertIn(o1.price, orderbook._bids.book.keys())
self.assertEqual(orderbook._bids.best.price, o1.price)
self.assertEqual(orderbook._bids.best.head.uid, o1uid)
self.assertEqual(orderbook._bids.best.tail.uid, o1uid)
self.assertEqual(orderbook._orders[o1uid].uid, o1uid)
self.assertEqual(orderbook.get(o1uid)['active'], True)
orderbook.send(*o2)
self.assertEqual(orderbook._bids.best.price, bidprice)
# Check time priority inside PriceLevel
self.assertEqual(orderbook._bids.best.head.uid, o1uid)
orderbook.send(*o3)
self.assertEqual(orderbook._bids.best.head.uid, o1uid)
self.assertEqual(orderbook._bids.best.head.next.uid, o2uid)
self.assertEqual(orderbook._bids.best.tail.uid, o3uid)
# Check list of orders
### SAME FOR ASKS
askprice = bidprice + 0.0001
o4uid = 4
o5uid = 5
o6uid = 6
o4 = order(is_buy=False, qty=10, price=askprice, uid=o4uid)
o5 = order(is_buy=False, qty=5, price=askprice, uid=o5uid)
o6 = order(is_buy=False, qty=7, price=askprice, uid=o6uid)
# Check price level creation, heads and tails
orderbook.send(*o4)
self.assertIn(askprice, orderbook._asks.book.keys())
self.assertEqual(orderbook._asks.best.price, o4.price)
self.assertEqual(orderbook._asks.best.head.uid, o4uid)
self.assertEqual(orderbook._asks.best.tail.uid, o4uid)
self.assertEqual(orderbook._orders[o4uid].uid, o4uid)
orderbook.send(*o5)
# Check time priority inside PriceLevel
self.assertIs(orderbook._asks.best.head.uid, o4uid)
orderbook.send(*o6)
self.assertEqual(orderbook._asks.best.head.uid, o4uid)
self.assertEqual(orderbook._asks.best.head.next.uid, o5uid)
self.assertEqual(orderbook._asks.best.tail.uid, o6uid)
def ask_orderbook(ask_lmt_orders):
orderbook = Orderbook('san')
for order in ask_lmt_orders:
orderbook.send(*order)
return orderbook
def bid_orderbook(bid_lmt_orders):
orderbook = Orderbook('san')
for order in bid_lmt_orders:
orderbook.send(*order)
return orderbook