class RangeList(object):
"""List of ranges.
Ranges are automatically merged and split when a new range is appended
or removed.
"""
def __init__(self, sequence=None):
self.tree = RBTree(sequence)
def __iter__(self):
return iter(self.tree.items())
def __repr__(self):
return `list(self.tree.items())`
def append(self, start, stop):
"""Append range (start, stop) to the list."""
if start >= stop:
raise ValueError('stop has to be greater than start')
delete = []
for nextstart, nextstop in self.tree[start:].items():
if nextstart <= stop:
stop = max(nextstop, stop)
delete.append(nextstart)
else: break
for key in delete: del self.tree[key]
left = self.tree[:start]
try:
prevstart = max(left)
prevstop = self.tree[prevstart]
if prevstop >= start:
start = prevstart
stop = max(prevstop, stop)
del self.tree[prevstart]
except ValueError: pass
self.tree[start] = stop
def remove(self, start, stop):
"""Remove range (start, stop) from the list."""
delete = []
for nextstart, nextstop in self.tree[start:].items():
if nextstart < stop:
if nextstop > stop: self.tree[stop] = nextstop
delete.append(nextstart)
else: break
for key in delete: del self.tree[key]
left = self.tree[:start]
try:
prevstart = max(left)
prevstop = self.tree[prevstart]
if prevstop > stop: self.tree[stop] = prevstop
if prevstop >= start: self.tree[prevstart] = start
except ValueError: pass
class Scope(object):
# Static constants uses for indicating the result of the inserting
# a symbol into the Scope.
INSERT_SUCCESS = 0
INSERT_SHADOWED = 1
INSERT_REDECL = 2
# Initializes the Scope with a balanced Red-Black tree.
def __init__(self):
self.map = RBTree()
# Inserts a symbol into the Scope.
#
# @return INSERT_SUCCESS or INSERT_REDECL.
def insert(self, symbol):
if not isinstance(symbol, Symbol):
raise TypeError("'symbol' is not an instance of Symbol.")
if symbol.identifier not in self.map:
self.map[symbol.identifier] = symbol
return Scope.INSERT_SUCCESS
else:
return Scope.INSERT_REDECL
# Finds a symbol in the Scope.
#
# @param name String identifier for the Symbol to find.
#
# @return Symbol if found or None if not found.
def find(self, name):
return self.map.get(name, None)
# Size of the scope.
#
# @return Number of Symbols in the Scope.
def size(self):
return len(self.map)
# Clones the current Scope and returns a deep copy.
#
# @return Copy of the Scope.
def clone(self):
result = Scope()
for identifier, value in self.map.items():
result.map[identifier] = value.clone()
return result
# @return String representation of the current Scope.
def __repr__(self):
symbols = []
for key, value in self.map.items():
symbols.append(' ' + key + ' : ' + repr(value))
return '\n'.join(symbols)
return total_traded
def get_last_price(last_price, trades):
return trades[len(trades) -
1]['price'] if len(trades) > 0 else last_price
def get_traded_volume(trades):
return sum([trade['qty'] for trade in trades] + [0])
# Loop
for t in range(max_iterations):
traded_volume = 0
next_orders = []
# Limit order cancellations
for idNum, order in limit_orders.items():
if t >= order['time_limit']:
lob.cancelOrder(order['side'], idNum)
limit_orders.remove(idNum)
# Limit order arrivals
df['orders'] = df['lambda'].apply(lambda x: np.random.poisson(x))
for idx, row in df.iterrows():
for i in range(int(row['orders'])):
if row['price'] < ask:
side = 'bid'
order_id = order_id + 1
order = {
'type': 'limit',
'side': side,
tree[16.47] = 2694
tree[16.5] = 2079
tree[16.48] = 2599
tree[16.46] = 2564
tree[16.44] = 2709
# tree[16.45] = 2644
# tree[16.43] = 2621
# tree[16.49] = 3959
# tree[16.47] = 3494
# tree[16.48] = 3399
# tree[16.46] = 3364
# tree[16.44] = 3509
# tree[16.45] = 3444
# tree[16.43] = 3421
# tree[16.46] = 3735
# del tree[15.84]
# tree[16.43] = 4921
# tree[16.48] = 4099
# tree[16.5] = 1279
# tree[16.49] = 1959
# tree[16.39] = tree.get(16.39,0) + 2000
rbt = RBTree()
frbt = FastRBTree()
populate(rbt)
populate(frbt)
print('RBT len: {0} FRBT len: {1}'.format(len(rbt), len(frbt)))
for key, value in rbt.items():
print("RBTree[{key}] = {value} <-> FastRBTree[{key}] = {value2}".format(key=key, value=value, value2=frbt[key]))
tree[16.46] = 2564
tree[16.44] = 2709
# tree[16.45] = 2644
# tree[16.43] = 2621
# tree[16.49] = 3959
# tree[16.47] = 3494
# tree[16.48] = 3399
# tree[16.46] = 3364
# tree[16.44] = 3509
# tree[16.45] = 3444
# tree[16.43] = 3421
# tree[16.46] = 3735
# del tree[15.84]
# tree[16.43] = 4921
# tree[16.48] = 4099
# tree[16.5] = 1279
# tree[16.49] = 1959
# tree[16.39] = tree.get(16.39,0) + 2000
rbt = RBTree()
frbt = FastRBTree()
populate(rbt)
populate(frbt)
print('RBT len: {0} FRBT len: {1}'.format(len(rbt), len(frbt)))
for key, value in rbt.items():
print("RBTree[{key}] = {value} <-> FastRBTree[{key}] = {value2}".format(
key=key, value=value, value2=frbt[key]))
class OrderBook(object):
def __init__(self, product_id='BTC-USD', feed=None, log_to=None):
self._product_id = product_id
self._asks = RBTree()
self._bids = RBTree()
self._sequence = -1
self._current_ticker = None
self._feed = feed
@property
def product_id(self):
""" Currently OrderBook only supports a single product even though it is stored as a list of products. """
return self.product_id
def reset_book(self, snapshot):
self._asks = RBTree()
self._bids = RBTree()
for bid in snapshot['bids']:
self._add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in snapshot['asks']:
self._add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = snapshot['sequence']
def on_message(self, message):
sequence = message['sequence']
if self._sequence == -1:
logger.error("Expected snapshot before any message")
sys.exit()
# self.reset_book()
return
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
self.on_sequence_gap(self._sequence, sequence)
# return
msg_type = message['type']
if msg_type == 'open':
self._add(message)
elif msg_type == 'done' and 'price' in message:
self._remove(message)
elif msg_type == 'match':
self._match(message)
self._current_ticker = message
elif msg_type == 'change':
self._change(message)
self._sequence = sequence
def on_sequence_gap(self, gap_start, gap_end):
# self.reset_book()
logger.error(
'Error: messages missing ({} - {}). ignoring the gap.'.format(
gap_start, gap_end, self._sequence))
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
@staticmethod
def meet_min_diff_price(low_price, high_price, min_diff_price):
if low_price is None or high_price is None or min_diff_price is None:
return False
return high_price - low_price > min_diff_price
@staticmethod
def meet_max_size(total_size, max_size):
if total_size is None or max_size is None:
return False
return total_size > max_size
def get_aggr_bids(self, min_diff_price=None, max_size=None):
max_bid = self.get_bid()
total_size = 0
aggr_bids = list()
for price, bid_orders in reversed(list(self._bids.items())):
if (OrderBook.meet_min_diff_price(price, max_bid, min_diff_price)
and OrderBook.meet_max_size(total_size, max_size)):
break
size = sum(bid_order['size'] for bid_order in bid_orders)
num_orders = len(bid_orders)
aggr_bids.append({
'price': price,
'size': size,
'num_orders': num_orders
})
total_size += size
return aggr_bids
def get_aggr_asks(self, min_diff_price=None, max_size=None):
min_ask = self.get_ask()
total_size = 0
aggr_asks = list()
for price, ask_orders in self._asks.items():
if (OrderBook.meet_min_diff_price(min_ask, price, min_diff_price)
and OrderBook.meet_max_size(total_size, max_size)):
break
size = sum(ask_order['size'] for ask_order in ask_orders)
num_orders = len(ask_orders)
aggr_asks.append({
'price': price,
'size': size,
'num_orders': num_orders
})
total_size += size
return aggr_asks
# Internal operations
def _add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def _remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def _match(self, order):
if self._feed is not None:
self._feed._match(order)
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def _change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
try:
price = Decimal(order['price'])
except KeyError:
return
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
class MarinerOrderBook(gdax.OrderBook):
current_milli_time = lambda self: int(time.time() * 1000)
def __init__(self, ticker, threshold):
gdax.OrderBook.__init__(self, product_id=ticker)
self._threshold = threshold
self.bookChanged = None
self.whaleEnteredMarket = None
self.whaleExitedMarket = None
def registerHandlers(self, bookChangedHandler, whaleEnteredMarketHandler,
whaleExitedMarketHandler, whaleChangedHandler):
Logging.logger.info("registering callbacks...")
self.bookChanged = bookChangedHandler
self.whaleEnteredMarket = whaleEnteredMarketHandler
self.whaleExitedMarket = whaleExitedMarketHandler
self.whaleChanged = whaleChangedHandler
def onMessage(self, message):
#Logging.logger.info(message)
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.getProductOrderBook(level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
Logging.logger.error(
'Error: messages missing ({} - {}). Re-initializing websocket.'
.format(sequence, self._sequence))
self.close()
self.start()
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
if not self.bookChanged is None:
self.bookChanged()
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
if not self.whaleEnteredMarket == None and self.isWhale(order['size']):
self.whaleEnteredMarket(order)
def remove(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
if not self.whaleExitedMarket == None and self.isWhale(order['size']):
self.whaleExitedMarket(order)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
price = Decimal(order['price'])
new_volume = Decimal(order['new_size'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = map(itemgetter('id'), bids).index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = map(itemgetter('id'), asks).index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
if not self.whaleChanged == None and self.isWhale(order['size']):
self.whaleChanged(order)
def get_current_book(self,
num_levels=10
): #fetch only 100 levels off book either way
book = []
for index, bid_entry in enumerate(self._bids.items([True])):
if index == num_levels:
break
else:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
thisBid = bid_entry[1]
except KeyError:
continue
for order in thisBid:
order['price'] = Decimal128(order['price'])
order['size'] = Decimal128(order['size'])
book.append(order)
for index, ask_entry in enumerate(self._asks.items()):
if index == num_levels:
break
else:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
thisAsk = ask_entry[1]
except KeyError:
continue
for order in thisAsk:
order['price'] = Decimal128(order['price'])
order['size'] = Decimal128(order['size'])
book.append(order)
book_frame = pd.DataFrame(book)
return book_frame
def isWhale(self, aVolume):
return aVolume >= self._threshold
#For general book management purposes
def get_bid(self):
if not self._bids.is_empty():
return self._bids.max_key()
else:
return None
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
def get_ask(self):
if not self._asks.is_empty():
return self._asks.min_key()
else:
return None
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
class WhaleTracker():
current_milli_time = lambda self: int(time.time() * 1000)
def __init__(self, ticker):
self._ticker = ticker
self._bid_whales = RBTree()
self._ask_whales = RBTree()
def whaleEnteredMarketHandler(self, order):
if order['side'] == 'buy':
self.addBidWhale(order)
else:
self.addAskWhale(order)
def whaleExitedMarketHandler(self, order):
if order['side'] == 'buy':
self.removeBidWhale(order)
else:
self.removeAskWhale(order)
def whaleChangedHandler(self, order):
if order['side'] == 'buy':
self.changeBidWhale(order)
else:
self.changeAskWhale(order)
def addBidWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
bid_whale_order = self.get_bid_whale(price)
if bid_whale_order is None or volume > bid_whale_order.get_volume():
#Logging.logger.info("NEW WHALE ENTERED (BID): price=" + str(price) + " volume=" + str(volume))
bid_whale_order = WhaleOrder(ID, price, volume)
self.set_bid_whale(price, bid_whale_order)
def addAskWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
ask_whale_order = self.get_ask_whale(price)
if ask_whale_order is None or volume > ask_whale_order.get_volume():
#Logging.logger.info("NEW WHALE ENTERED (ASK): price=" + str(price) + " volume=" + str(volume))
ask_whale_order = WhaleOrder(ID, price, volume)
self.set_ask_whale(price, ask_whale_order)
def removeBidWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
bid_whale_order = self.get_bid_whale(price)
if bid_whale_order is not None and bid_whale_order.get_id() == ID:
#Logging.logger.info("WHALE LEFT (BID): price=" + str(price) + " volume=" + str(volume))
self.remove_bid_whale(price)
def removeAskWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
ask_whale_order = self.get_ask_whale(price)
if ask_whale_order is not None and ask_whale_order.get_id() == ID:
#Logging.logger.info("WHALE LEFT (ASK): price=" + str(price) + " volume=" + str(volume))
self.remove_ask_whale(price)
def changeBidWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
new_volume = order['new_size']
bid_whale_order = self.get_bid_whale(price)
if not bid_whale_order == None:
#Logger.logging.info("WHALE CHANGED (BID): price=" + price + " new_volume=" + new_volume)
if self.isWhale(new_volume):
bid_whale_order.setVolume(new_volume)
self.set_bid_whale(price, bid_whale_order)
else:
self.removeBidWhale(order)
def changeAskWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
new_volume = order['new_size']
ask_whale_order = self.get_ask_whale(price)
if not ask_whale_order == None:
#Logger.logging.info("WHALE CHANGED (ASK): price=" + price + " new_volume=" + new_volume)
if self.isWhale(new_volume):
ask_whale_order.setVolume(new_volume)
self.set_ask_whale(price, ask_whale_order)
else:
self.removeAskWhale(order)
def get_current_whales(
self,
num_whales=30
): #by default, fetch only 30 whales off the book either way
whales = []
for index, bid_whale in self._bid_whales.items(True):
if index == num_whales:
break
else:
whales.append({
'price': Decimal128(bid_whale.get_price()),
'volume': Decimal128(bid_whale.get_volume()),
'id': bid_whale.get_id(),
})
for index, ask_whale in self._ask_whales.items():
if index == num_whales:
break
else:
whales.append({
'price': Decimal128(ask_whale.get_price()),
'volume': Decimal128(ask_whale.get_volume()),
'id': ask_whale.get_id(),
})
whales_frame = pd.DataFrame(whales)
return whales_frame
def get_top_bid_whale(self):
if not self._bid_whales.is_empty():
top_bid = self._bid_whales.max_key()
return self.get_bid_whale(top_bid)
else:
return None
def get_bid_whale(self, price):
return self._bid_whales.get(price)
def remove_bid_whale(self, price):
self._bid_whales.remove(price)
def set_bid_whale(self, price, whale):
self._bid_whales.insert(price, whale)
def get_top_ask_whale(self):
if not self._ask_whales.is_empty():
top_ask = self._ask_whales.min_key()
return self.get_ask_whale(top_ask)
else:
return None
def get_ask_whale(self, price):
return self._ask_whales.get(price)
def remove_ask_whale(self, price):
self._ask_whales.remove(price)
def set_ask_whale(self, price, whale):
self._ask_whales.insert(price, whale)