def solve(n, k):
tree = FastRBTree()
tree.insert(n, 1)
ls = rs = n
for i in range(k):
key, val = tree.max_item()
tree.remove(key)
if val > 1:
tree.insert(key, val - 1)
if key % 2 == 1:
key //= 2
ls = rs = key
update_tree(tree, key)
update_tree(tree, key)
else:
key //= 2
ls = key
rs = key - 1
update_tree(tree, ls)
update_tree(tree, rs)
return str(ls) + " " + str(rs)
class Node:
def __init__(self, name):
self.name = name
self.children = {}
self.visited = False
self.ancestors = FastRBTree()
def updateChildren(self, child, nameOfSequence):
if child in self.children:
self.children[child].add(nameOfSequence)
else:
self.children[child] = set()
self.children[child].add(nameOfSequence)
def getChild(self, nameOfChild):
return filter(lambda x: x == nameOfChild, self.children.keys())
def updateAncestors(self, ancestors, nameOfSequence):
for ancestor in ancestors:
if self.name == ancestor.name:
continue
elementFromTree = self.ancestors.get(ancestor.name, None)
if elementFromTree != None:
elementFromTree.add(nameOfSequence)
self.ancestors.__setitem__(ancestor.name, elementFromTree)
else: # NIE JE V SEK
tmpValue = set()
tmpValue.add(nameOfSequence)
self.ancestors.insert(ancestor.name, tmpValue)
class Tree(object):
def __init__(self):
self.price_tree = FastRBTree()
self.price_map = {}
self.order_map = {}
self.received_orders = {}
def receive(self, order_id, size):
self.received_orders[order_id] = size
def create_price(self, price):
new_list = []
self.price_tree.insert(price, new_list)
self.price_map[price] = new_list
def remove_price(self, price):
self.price_tree.remove(price)
del self.price_map[price]
def insert_order(self, order_id, size, price, initial=False):
if not initial:
del self.received_orders[order_id]
if price not in self.price_map:
self.create_price(price)
order = {
'order_id': order_id,
'size': size,
'price': price,
'price_map': self.price_map[price]
}
self.price_map[price].append(order)
self.order_map[order_id] = order
def match(self, maker_order_id, match_size):
order = self.order_map[maker_order_id]
original_size = order['size']
new_size = original_size - match_size
order['size'] = new_size
def change(self, order_id, new_size):
order = self.order_map[order_id]
order['size'] = new_size
def remove_order(self, order_id):
if order_id in self.order_map:
order = self.order_map[order_id]
self.price_map[order['price']] = [
o for o in self.price_map[order['price']]
if o['order_id'] != order_id
]
if not self.price_map[order['price']]:
self.remove_price(order['price'])
del self.order_map[order_id]
else:
del self.received_orders[order_id]
class Tree(object):
def __init__(self):
self.priceTree = FastRBTree()
self.volume = 0
self.priceMap = {} # Map from price -> orderList object
self.orderMap = {} # Order ID to Order object
def __len__(self):
return len(self.orderMap)
def getPrice(self, price):
return self.priceMap[price]
def getOrder(self, idNum):
return self.orderMap[idNum]
def createPrice(self, price):
newList = OrderList()
self.priceTree.insert(price, newList)
self.priceMap[price] = newList
def removePrice(self, price):
self.priceTree.remove(price)
del self.priceMap[price]
def priceExists(self, price):
return price in self.priceMap
def orderExists(self, idNum):
return idNum in self.orderMap
def insertTick(self, tick):
if tick.price not in self.priceMap:
self.createPrice(tick.price)
order = Order(tick, self.priceMap[tick.price])
self.priceMap[order.price].appendOrder(order)
self.orderMap[order.idNum] = order
self.volume += order.qty
def updateOrder(self, tick):
order = self.orderMap[tick.idNum]
originalVolume = order.qty
if tick.price != order.price:
# Price changed
orderList = self.priceMap[order.price]
orderList.removeOrder(order)
if len(orderList) == 0:
removePrice(order.price)
self.insertTick(tick)
else:
# Quantity changed
order.updateQty(tick.qty,tick.price)
self.volume += order.qty - originalVolume
def removeOrderById(self, idNum):
order = self.orderMap[idNum]
self.volume -= order.qty
order.orderList.removeOrder(order)
if len(order.orderList) == 0:
self.removePrice(order.price)
del self.orderMap[idNum]
def max(self):
return min(self.priceTree)
def min(self):
return max(self.priceTree)
def _update(self, p, seeds: FastRBTree):
# since eps is INF, the neighbours are all the points in the dataset.
for o in range(self.N):
if self.processed[o]:
continue
new_reach_dist = max(self.core_dist[p], self.dist_mat[o][p])
if np.isinf(self.reachability_dist[o]):
self.reachability_dist[o] = new_reach_dist
seeds.insert((new_reach_dist, o), o)
self.predecessor[o] = p
elif new_reach_dist < self.reachability_dist[o]:
seeds.remove((self.reachability_dist[o], o))
self.reachability_dist[o] = new_reach_dist
seeds.insert((new_reach_dist, o), o)
self.predecessor[o] = p
def GetLeastNumbers2(l,k):
count = 0
tree = FastRBTree()
for i in l:
if count <k:
tree.insert(i,i)
count += 1
else:
maxVal = max(tree)
if i < maxVal:
tree.remove(maxVal)
tree.insert(i,i)
datas = [item for item in tree]
return datas
class Tree(object):
def __init__(self):
self.price_tree = FastRBTree()
self.price_map = {} # Map from price -> order_list object
self.order_map = {} # Order ID to Order object
self.received_orders = {}
def receive(self, order_id, size):
self.received_orders[order_id] = size
def create_price(self, price):
new_list = OrderList()
self.price_tree.insert(price, new_list)
self.price_map[price] = new_list
def remove_price(self, price):
self.price_tree.remove(price)
del self.price_map[price]
def insert_order(self, order_id, size, price, initial=False):
if not initial:
del self.received_orders[order_id]
if price not in self.price_map:
self.create_price(price)
order = Order(order_id, size, price, self.price_map[price])
self.price_map[order.price].append_order(order)
self.order_map[order.order_id] = order
def match(self, maker_order_id, size):
order = self.order_map[maker_order_id]
original_size = order.size
new_size = original_size - size
order.update_size(new_size)
def change(self, order_id, new_size):
order = self.order_map[order_id]
order.update_size(new_size)
def remove_order(self, order_id):
if order_id in self.order_map:
order = self.order_map[order_id]
order.order_list.remove_order(order)
if len(order.order_list) == 0:
self.remove_price(order.price)
del self.order_map[order_id]
else:
del self.received_orders[order_id]
class Tree(object):
def __init__(self):
self.price_tree = FastRBTree()
self.price_map = {}
self.order_map = {}
self.received_orders = {}
def receive(self, order_id, size):
self.received_orders[order_id] = size
def create_price(self, price):
new_list = []
self.price_tree.insert(price, new_list)
self.price_map[price] = new_list
def remove_price(self, price):
self.price_tree.remove(price)
del self.price_map[price]
def insert_order(self, order_id, size, price, initial=False):
if not initial:
del self.received_orders[order_id]
if price not in self.price_map:
self.create_price(price)
order = {'order_id': order_id, 'size': size, 'price': price, 'price_map': self.price_map[price]}
self.price_map[price].append(order)
self.order_map[order_id] = order
def match(self, maker_order_id, match_size):
order = self.order_map[maker_order_id]
original_size = order['size']
new_size = original_size - match_size
order['size'] = new_size
def change(self, order_id, new_size):
order = self.order_map[order_id]
order['size'] = new_size
def remove_order(self, order_id):
if order_id in self.order_map:
order = self.order_map[order_id]
self.price_map[order['price']] = [o for o in self.price_map[order['price']] if o['order_id'] != order_id]
if not self.price_map[order['price']]:
self.remove_price(order['price'])
del self.order_map[order_id]
else:
del self.received_orders[order_id]
class OrderTree(object):
def __init__(self):
self.price_tree = FastRBTree()
self.price_map = {}
self.min_price = None
self.max_price = None
def insert_price(self, price, amount):
self.price_tree.insert(price, amount)
self.price_map[price] = amount
if self.max_price == None or price > self.max_price:
self.max_price = price
if self.min_price == None or price < self.min_price:
self.min_price = price
def update_price(self, price, amount):
self.price_tree.insert(price, amount) #updates if key exists
self.price_map[price] = amount
def remove_price(self, price):
self.price_tree.remove(price)
del self.price_map[price]
if self.max_price == price:
try:
self.max_price = max(self.price_tree)
except ValueError:
self.max_price = None
if self.min_price == price:
try:
self.min_price = min(self.price_tree)
except ValueError:
self.min_price = None
def price_exists(self, price):
return price in self.price_map
def max(self):
return self.max_price
def min(self):
return self.min_price
class Hyperedge:
def __init__(self, hyperkey, col, hlabel):
self.hyperkey = hyperkey
self.col = col
self._alerts = Tree()
self.insert_alert(hlabel, 1)
self.nalerts = 1
def get_alert(self, key):
return self._alerts.get(key)
def insert_alert(self, alert_key, count):
self._alerts.insert(alert_key, count)
def foreach_alert(self, func):
self._alerts.foreach(func)
def pop_alert(self, key):
return self._alerts.pop(key)
class Hyperedge:
def __init__(self, hyperkey, col, hlabel):
self.hyperkey = hyperkey
self.col = col
self._alerts = Tree()
self.insert_alert(hlabel, 1)
self.nalerts = 1
def get_alert(self, key):
return self._alerts.get(key)
def insert_alert(self, alert_key, count):
self._alerts.insert(alert_key, count)
def foreach_alert(self, func):
self._alerts.foreach(func)
def pop_alert(self, key):
return self._alerts.pop(key)
def test_add_symbol():
ST = SymbolTable("")
T = FastRBTree()
Content = {'Type': "int" , 'Attribute': None , 'TokenLocation': (10,2) }
ST.InsertSymbol("age", Content)
T.insert("age", Content)
Content = {'Type': "float" , 'Attribute': 'static' , 'TokenLocation': (11,2) }
ST.InsertSymbol("temperature", Content)
T.insert("temperature", Content)
Content = {'Type': "char" , 'Attribute': 'const' , 'TokenLocation': (12,2) }
ST.InsertSymbol("letter", Content)
T.insert("letter", Content)
keys = ST.TopScope.keys()
for key in keys:
assert(T.__contains__(key))
assert(T.get(key) == ST.TopScope.get(key))
assert(T.get(key) is not ST.TopScope.get(key))
#write test to prove that the returned item is a pointer
return
alert = []
for y in range(0, 6):
alert.append(repr(randint(0, 8 - 1)) + "test")
for c in hcombinations:
label = []
key = alert[:]
for x in range(2):
key[c[x]] = '*'
label.append(alert[c[x]])
hyperkey = tuple(key)
hlabel = tuple(label)
if hyperkey in hyper_dict:
hyper_dict[hyperkey].nalerts += 1
result = hyper_dict[hyperkey].get_alert(hlabel)
if result is not None:
hyper_dict[hyperkey].insert_alert(hlabel, result + 1)
else:
hyper_dict[hyperkey].insert_alert(hlabel, 1)
else:
hyper_dict[hyperkey] = Hyperedge(key, c, hlabel)
for hyperedge in hyper_dict.values():
hypersize_list.insert((hyperedge.nalerts, hyperedge.hyperkey),
hyperedge)
while not hypersize_list.is_empty():
(key, hyperedge) = hypersize_list.pop_max()
hyperedge.foreach_alert(treeloop)
print("Completed iteration %d of 10" % z)
class OrderQueue(object):
"""
"""
def __init__(self):
self.Orders = FastRBTree()
self.totalAmount = 0.0
def append(self,order):
k = order.timestamp
self.Orders.insert(k,order)
self.totalAmount += self.Orders[k].amount
def count(self):
return self.Orders.count
def min_item(self):
return self.Orders.min_item()
def eat(self,amount):
"""
use this only if amount <= self.amount
return OrderToPop,RestAmount
"""
current_amount = amount
to_pop_orders = []
while self.totalAmount>0 and current_amount>0:
if not self.Orders or self.Orders.count==0:
#print "amount:%.4f" % self.totalAmount
self.totalAmount = 0.0
return []
min_item = self.Orders.min_item()
min_i = min_item[1]
# 1.enough
# 2.not enough
if min_i.amount <= current_amount:
current_amount -= min_i.amount
self.totalAmount -= min_i.amount
to_pop_orders.append(self.Orders.pop_min()[1])
continue
# this is a hard case , need to take care of
# first we split the order , take amount from current order
# and make
elif min_i.amount > current_amount:
new_order = min_i.copy()
new_order.amount = current_amount
to_pop_orders.append(new_order)
min_i.amount -= current_amount
self.totalAmount -= current_amount
current_amount = 0
break
# return:
# 1.orders
# 2.amount to trade
# 3.amount left
return to_pop_orders
def remove(self,order):
k = order.timestamp
self.totalAmount -= self.Orders[k].amount
self.Orders.remove(k)
def get_price_depth(self):
return self.totalAmount
def is_empty(self):
return self.Orders.count == 0
class TDigest(object):
def __init__(self, delta=0.01, K=25):
self.C = RBTree()
self.n = 0
self.delta = delta
self.K = K
def __add__(self, other_digest):
data = list(chain(self.C.values(), other_digest.C.values()))
new_digest = TDigest(self.delta, self.K)
if len(data) > 0:
for c in pyudorandom.items(data):
new_digest.update(c.mean, c.count)
return new_digest
def __len__(self):
return len(self.C)
def __repr__(self):
return """<T-Digest: n=%d, centroids=%d>""" % (self.n, len(self))
def _add_centroid(self, centroid):
if centroid.mean not in self.C:
self.C.insert(centroid.mean, centroid)
else:
self.C[centroid.mean].update(centroid.mean, centroid.count)
def _compute_centroid_quantile(self, centroid):
denom = self.n
cumulative_sum = sum(
c_i.count
for c_i in self.C.value_slice(-float('Inf'), centroid.mean))
return (centroid.count / 2. + cumulative_sum) / denom
def _update_centroid(self, centroid, x, w):
self.C.pop(centroid.mean)
centroid.update(x, w)
self._add_centroid(centroid)
def _find_closest_centroids(self, x):
try:
ceil_key = self.C.ceiling_key(x)
except KeyError:
floor_key = self.C.floor_key(x)
return [self.C[floor_key]]
try:
floor_key = self.C.floor_key(x)
except KeyError:
ceil_key = self.C.ceiling_key(x)
return [self.C[ceil_key]]
if abs(floor_key - x) < abs(ceil_key - x):
return [self.C[floor_key]]
elif abs(floor_key - x) == abs(ceil_key -
x) and (ceil_key != floor_key):
return [self.C[ceil_key], self.C[floor_key]]
else:
return [self.C[ceil_key]]
def _theshold(self, q):
return 4 * self.n * self.delta * q * (1 - q)
def update(self, x, w=1):
"""
Update the t-digest with value x and weight w.
"""
self.n += w
if len(self) == 0:
self._add_centroid(Centroid(x, w))
return
S = self._find_closest_centroids(x)
while len(S) != 0 and w > 0:
j = choice(list(range(len(S))))
c_j = S[j]
q = self._compute_centroid_quantile(c_j)
# This filters the out centroids that do not satisfy the second part
# of the definition of S. See original paper by Dunning.
if c_j.count + w > self._theshold(q):
S.pop(j)
continue
delta_w = min(self._theshold(q) - c_j.count, w)
self._update_centroid(c_j, x, delta_w)
w -= delta_w
S.pop(j)
if w > 0:
self._add_centroid(Centroid(x, w))
if len(self) > self.K / self.delta:
self.compress()
return
def batch_update(self, values, w=1):
"""
Update the t-digest with an iterable of values. This assumes all points have the
same weight.
"""
for x in values:
self.update(x, w)
self.compress()
return
def compress(self):
T = TDigest(self.delta, self.K)
C = list(self.C.values())
for c_i in pyudorandom.items(C):
T.update(c_i.mean, c_i.count)
self.C = T.C
def percentile(self, p):
"""
Computes the percentile of a specific value in [0,100].
"""
if not (0 <= p <= 100):
raise ValueError("p must be between 0 and 100, inclusive.")
t = 0
p = float(p) / 100.
p *= self.n
for i, key in enumerate(self.C.keys()):
c_i = self.C[key]
k = c_i.count
if p < t + k:
if i == 0:
return c_i.mean
elif i == len(self) - 1:
return c_i.mean
else:
delta = (self.C.succ_item(key)[1].mean -
self.C.prev_item(key)[1].mean) / 2.
return c_i.mean + ((p - t) / k - 0.5) * delta
t += k
return self.C.max_item()[1].mean
def quantile(self, q):
"""
Computes the quantile of a specific value, ie. computes F(q) where F denotes
the CDF of the distribution.
"""
t = 0
N = float(self.n)
for i, key in enumerate(self.C.keys()):
c_i = self.C[key]
if i == len(self) - 1:
delta = (c_i.mean - self.C.prev_item(key)[1].mean) / 2.
else:
delta = (self.C.succ_item(key)[1].mean - c_i.mean) / 2.
z = max(-1, (q - c_i.mean) / delta)
if z < 1:
return t / N + c_i.count / N * (z + 1) / 2
t += c_i.count
return 1
def trimmed_mean(self, p1, p2):
"""
Computes the mean of the distribution between the two percentiles p1 and p2.
This is a modified algorithm than the one presented in the original t-Digest paper.
"""
if not (p1 < p2):
raise ValueError("p1 must be between 0 and 100 and less than p2.")
s = k = t = 0
p1 /= 100.
p2 /= 100.
p1 *= self.n
p2 *= self.n
for i, key in enumerate(self.C.keys()):
c_i = self.C[key]
k_i = c_i.count
if p1 < t + k_i:
if i == 0:
delta = self.C.succ_item(key)[1].mean - c_i.mean
elif i == len(self) - 1:
delta = c_i.mean - self.C.prev_item(key)[1].mean
else:
delta = (self.C.succ_item(key)[1].mean -
self.C.prev_item(key)[1].mean) / 2.
nu = ((p1 - t) / k_i - 0.5) * delta
s += nu * k_i * c_i.mean
k += nu * k_i
if p2 < t + k_i:
return s / k
t += k_i
return s / k
class TradeTree(object):
'''A red-black tree used to store TradeLists in price trade
The exchange will be using the TradeTree to hold bid and ask data (one TradeTree for each side).
Keeping the information in a red black tree makes it easier/faster to detect a match.
'''
def __init__(self):
self.price_tree = FastRBTree()
self.trade_map = {}
self.num_trades = 0 # Contains count of Orders in tree
self.depth = 0 # Number of different prices in tree (http://en.wikipedia.org/wiki/trade_book_(trading)#Book_depth)
def __len__(self):
return len(self.trade_map)
def get_price_list(self, price):
return self.price_tree.get(price, [])
def get_trade(self, trade_id):
return self.trade_map[trade_id] if trade_id in self.trade_map else None
def create_price(self, price):
self.depth += 1 # Add a price depth level to the tree
new_list = LinkedList()
self.price_tree.insert(price,
new_list) # Insert a new price into the tree
def remove_price(self, price):
self.depth -= 1 # Remove a price depth level
self.price_tree.remove(price)
def price_exists(self, price):
return self.price_tree.__contains__(price)
def trade_exists(self, trade_id):
return trade_id in self.trade_map
def insert_trade(self, xtrade):
if self.trade_exists(xtrade.id):
return
self.num_trades += 1
if not self.price_exists(xtrade.limit_price):
self.create_price(
xtrade.limit_price
) # If price not in Price Map, create a node in RBtree
self.trade_map[
trade.id] = self.price_tree[xtrade.limit_price].append_item(
xtrade
) # Add the trade to the TradeList in Price Map return the reference
def remove_trade(self, xtrade):
self.num_trades -= 1
trade_node = self.trade_map[trade.id]
self.price_tree[trade.limit_price].remove_item(trade_node)
if len(self.price_tree[trade.limit_price]) == 0:
self.remove_price(trade.limit_price)
self.trade_map.pop(trade.id, None)
def max_price(self):
if self.depth > 0:
return self.price_tree.max_key()
else:
return None
def min_price(self):
if self.depth > 0:
return self.price_tree.min_key()
else:
return None
def max_price_list(self):
if self.depth > 0:
return self.get_price_list(self.max_price())
else:
return None
def min_price_list(self):
if self.depth > 0:
return self.get_price_list(self.min_price())
else:
return None
class BookSide(object):
'''
A side of the lmit order book representation
'''
def __init__(self, s_side, fr_data, i_member=None):
'''
Initialize a BookSide object. Save all parameters as attributes
:param s_side: string. BID or ASK
:param fr_data: ZipExtFile object. data to read
:param i_member*: integer. Member number to be used as a filter
'''
if s_side not in ['BID', 'ASK']:
raise InvalidTypeException('side should be BID or ASK')
self.i_member = i_member
self.s_side = s_side
self.price_tree = FastRBTree()
self._i_idx = 0
self.fr_data = fr_data
self.parser = parser_data.LineParser(s_side)
self.d_order_map = {}
self.last_price = 0.
def how_many_rows_read(self):
'''
Return the number of rows processed
'''
return self._i_idx
def update(self, d_data, s_last_ident):
'''
Update the state of the order book given the data pased
:param d_data: dict. data from the last row
:param s_last_ident: string. last identification
'''
# check if the information should be processed
if s_last_ident != 'MSG':
return False
# check if should filter out member
if not self._should_use_it(d_data):
return False
# update the book information
order_aux = Order(d_data)
s_status = order_aux['order_status']
b_sould_update = True
# treat Bovespa files at the begining f the day
if s_status != 'New':
try:
i_old_id = self.d_order_map[order_aux]['main_id']
except KeyError:
if s_status == 'Canceled' or s_status == 'Filled':
b_sould_update = False
s_status = 'Invalid'
elif s_status == 'Replaced':
s_status = 'New'
# process
if s_status == 'New':
b_sould_update = self._new_order(order_aux)
elif s_status != 'Invalid':
i_old_id = self.d_order_map[order_aux]['main_id']
f_old_pr = self.d_order_map[order_aux]['price']
i_old_q = self.d_order_map[order_aux]['qty']
# hold the last traded price
if s_status in ['Partially Filled', 'Filled']:
self.last_price = order_aux['order_price']
# process message
if s_status in ['Canceled', 'Expired', 'Filled']:
b_sould_update = self._canc_expr_filled_order(order_aux,
i_old_id,
f_old_pr,
i_old_q)
elif s_status == 'Replaced':
b_sould_update = self._replaced_order(order_aux,
i_old_id,
f_old_pr,
i_old_q)
elif s_status == 'Partially Filled':
b_sould_update = self._partially_filled(order_aux,
i_old_id,
f_old_pr,
i_old_q)
# remove from order map
if s_status not in ['New', 'Invalid']:
self.d_order_map.pop(order_aux)
# update the order map
if b_sould_update:
f_qty = int(order_aux['total_qty_order'])
self.d_order_map[order_aux] = {}
self.d_order_map[order_aux]['price'] = d_data['order_price']
self.d_order_map[order_aux]['sec_order'] = order_aux.sec_order_id
self.d_order_map[order_aux]['qty'] = f_qty
self.d_order_map[order_aux]['main_id'] = order_aux.main_id
# return that the update was done
return True
def _should_use_it(self, d_data):
'''
Check if should use the passed row to update method
:param d_data: dict. data from the last row
'''
if self.i_member:
if d_data['member'] != self.i_member:
return False
return True
def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed canceled, expried or filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# remove from order map
return False
def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed replaced orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# remove from the old price
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# insert the order in the due price
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed partially filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# delete old price, if it is needed
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# add/modify order
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _new_order(self, order_obj):
'''
Update price_tree when passed new orders
:param order_obj: Order Object. The last order in the file
'''
# if it was already in the order map
if order_obj in self.d_order_map:
i_old_sec_id = self.d_order_map[order_obj]['main_id']
f_old_price = self.d_order_map[order_obj]['price']
i_old_qty = self.d_order_map[order_obj]['qty']
this_price = self.price_tree.get(f_old_price)
# remove from order map
self.d_order_map.pop(order_obj)
if this_price.delete(i_old_sec_id, i_old_qty):
self.price_tree.remove(f_old_price)
# insert a empty price level if it is needed
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# add the order
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def get_n_top_prices(self, n):
'''
Return a dataframe with the N top price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def get_n_botton_prices(self, n=5):
'''
Return a dataframe with the N botton price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def _readline(self):
'''
Return a line from the fr_data file if available. Return false
otherwiese
'''
row = self.fr_data.readline()
if row == '':
self.fr_data.close()
return False, False
self._i_idx += 1
d_aux = self.parser(row)
return d_aux, self.parser.last_identification
def __iter__(self):
'''
Return the self as an iterator object. Use next() to check the rows
'''
return self
def next(self):
'''
Return the next item from the fr_data in iter process. If there are no
further items, raise the StopIteration exception
'''
d_rtn, last_identification = self._readline()
if not d_rtn:
raise StopIteration
return d_rtn, last_identification
class Tree(object):
def __init__(self):
self.ptree = FastRBTree()
self.vol = 0
self.prmp = {}
self.order_map = {}
self.mip = None
self.mxp = None
def __len__(self):
return len(self.order_map)
def get_pri(self, pri):
return self.prmp[pri]
def get_order(self, id_num):
return self.order_map[id_num]
def create_pri(self, pri):
new_list = OrderList()
self.ptree.insert(pri, new_list)
self.prmp[pri] = new_list
if self.mxp == None or pri > self.mxp:
self.mxp = pri
if self.mip == None or pri < self.mip:
self.mip = pri
def remove_pri(self, pri):
self.ptree.remove(pri)
del self.prmp[pri]
if self.mxp == pri:
try:
self.mxp = max(self.ptree)
except ValueError:
self.mxp = None
if self.mip == pri:
try:
self.mip = min(self.ptree)
except ValueError:
self.mip = None
def pri_exists(self, pri):
return pri in self.prmp
def order_exists(self, id_num):
return id_num in self.order_map
def insert_tick(self, tick):
if tick.pri not in self.prmp:
self.create_pri(tick.pri)
order = Order(tick, self.prmp[tick.pri])
self.prmp[order.pri].append_order(order)
self.order_map[order.id_num] = order
self.vol += order.qty
def update_order(self, tick):
order = self.order_map[tick.id_num]
original_vol = order.qty
if tick.pri != order.pri:
order_list = self.prmp[order.pri]
order_list.remove_order(order)
if len(order_list) == 0:
self.remove_pri(order.pri)
self.insert_tick(tick)
self.vol -= original_vol
else:
order.update_qty(tick.qty, tick.pri)
self.vol += order.qty - original_vol
def remove_order_by_id(self, id_num):
order = self.order_map[id_num]
self.vol -= order.qty
order.order_list.remove_order(order)
if len(order.order_list) == 0:
self.remove_pri(order.pri)
del self.order_map[id_num]
def max(self):
return self.mxp
def min(self):
return self.mip
class PriceLevel(object):
'''
A representation of a Price level in the book
'''
def __init__(self, f_price):
'''
A representation of a PriceLevel object
'''
self.f_price = f_price
self.i_qty = 0
self.order_tree = FastRBTree()
def add(self, order_aux):
'''
Insert the information in the tree using the info in order_aux. Return
is should delete the Price level or not
:param order_aux: Order Object. The Order message to be updated
'''
# check if the order_aux price is the same of the self
s_status = order_aux['order_status']
if order_aux['order_price'] != self.f_price:
raise DifferentPriceException
elif s_status in ['New', 'Replaced', 'Partially Filled']:
self.order_tree.insert(order_aux.main_id, order_aux)
self.i_qty += int(order_aux['total_qty_order'])
# check if there is no object in the updated tree (should be deleted)
return self.order_tree.count == 0
def delete(self, i_last_id, i_old_qty):
'''
Delete the information in the tree using the info in order_aux. Return
is should delete the Price level or not
:param i_last_id: Integer. The previous secondary order id
:param i_old_qty: Integer. The previous order qty
'''
# check if the order_aux price is the same of the self
try:
self.order_tree.remove(i_last_id)
self.i_qty -= i_old_qty
except KeyError:
raise DifferentPriceException
# check if there is no object in the updated tree (should be deleted)
return self.order_tree.count == 0
def __str__(self):
'''
Return the name of the PriceLevel
'''
return '{:,.0f}'.format(self.i_qty)
def __repr__(self):
'''
Return the name of the PriceLevel
'''
return '{:,.0f}'.format(self.i_qty)
def __eq__(self, other):
'''
Return if a PriceLevel has equal price from the other
:param other: PriceLevel object. PriceLevel to be compared
'''
# just to make sure that there is no floating point discrepance
f_aux = other
if not isinstance(other, float):
f_aux = other.f_price
return abs(self.f_price - f_aux) < 1e-4
def __gt__(self, other):
'''
Return if a PriceLevel has a gerater price from the other.
Bintrees uses that to compare nodes
:param other: PriceLevel object. PriceLevel to be compared
'''
# just to make sure that there is no floating point discrepance
f_aux = other
if not isinstance(other, float):
f_aux = other.f_price
return (f_aux - self.f_price) > 1e-4
def __lt__(self, other):
'''
Return if a Order has smaller order_id from the other. Bintrees uses
that to compare nodes
:param other: Order object. Order to be compared
'''
f_aux = other
if not isinstance(other, float):
f_aux = other.f_price
return (f_aux - self.f_price) < -1e-4
def __ne__(self, other):
'''
Return if a Order has different order_id from the other
:param other: Order object. Order to be compared
'''
return not self.__eq__(other)
class Tree(object):
def __init__(self):
self.priceTree = FastRBTree()
self.volume = 0
self.priceMap = {} # Map from price -> orderList object
self.orderMap = {} # Order ID to Order object
def __len__(self):
return len(self.orderMap)
def getPrice(self, price):
return self.priceMap[price]
def getOrder(self, idNum):
return self.orderMap[idNum]
def createPrice(self, price):
newList = OrderList()
self.priceTree.insert(price, newList)
self.priceMap[price] = newList
def removePrice(self, price):
self.priceTree.remove(price)
del self.priceMap[price]
def priceExists(self, price):
return price in self.priceMap
def orderExists(self, idNum):
return idNum in self.orderMap
def insertTick(self, tick):
if tick.price not in self.priceMap:
self.createPrice(tick.price)
order = Order(tick, self.priceMap[tick.price])
self.priceMap[order.price].appendOrder(order)
self.orderMap[order.idNum] = order
self.volume += order.qty
def updateOrder(self, tick):
order = self.orderMap[tick.idNum]
originalVolume = order.qty
if tick.price != order.price:
# Price changed
orderList = self.priceMap[order.price]
orderList.removeOrder(order)
if len(orderList) == 0:
self.removePrice(order.price)
self.insertTick(tick)
else:
# Quantity changed
order.updateQty(tick.qty, tick.price)
self.volume += order.qty - originalVolume
def removeOrderById(self, idNum):
order = self.orderMap[idNum]
self.volume -= order.qty
order.orderList.removeOrder(order)
if len(order.orderList) == 0:
self.removePrice(order.price)
del self.orderMap[idNum]
def max(self):
return min(self.priceTree)
def min(self):
return max(self.priceTree)
for x in range(0, 3000):
alert = []
for y in range(0, 6):
alert.append(repr(randint(0, 8 - 1)) + "test")
for c in hcombinations:
label = []
key = alert[:]
for x in range(2):
key[c[x]] = "*"
label.append(alert[c[x]])
hyperkey = tuple(key)
hlabel = tuple(label)
if hyperkey in hyper_dict:
hyper_dict[hyperkey].nalerts += 1
result = hyper_dict[hyperkey].get_alert(hlabel)
if result is not None:
hyper_dict[hyperkey].insert_alert(hlabel, result + 1)
else:
hyper_dict[hyperkey].insert_alert(hlabel, 1)
else:
hyper_dict[hyperkey] = Hyperedge(key, c, hlabel)
for hyperedge in hyper_dict.values():
hypersize_list.insert((hyperedge.nalerts, hyperedge.hyperkey), hyperedge)
while not hypersize_list.is_empty():
(key, hyperedge) = hypersize_list.pop_max()
hyperedge.foreach_alert(treeloop)
print("Completed iteration %d of 10" % z)
class BookSide(object):
'''
A side of the lmit order book representation
'''
def __init__(self, s_side):
'''
Initialize a BookSide object. Save all parameters as attributes
:param s_side: string. BID or ASK
'''
if s_side not in ['BID', 'ASK']:
raise InvalidTypeException('side should be BID or ASK')
self.s_side = s_side
self.price_tree = FastRBTree()
self._i_idx = 0
self.d_order_map = {}
self.last_price = 0.
def update(self, d_data):
'''
Update the state of the order book given the data pased. Return if the
message was handle successfully
:param d_data: dict. data related to a single order
'''
# dont process aggresive trades
if d_data['agressor_indicator'] == 'Agressive':
return True
# update the book information
order_aux = Order(d_data)
s_status = order_aux['order_status']
b_sould_update = True
b_success = True
# check the order status
if s_status != 'New':
try:
i_old_id = self.d_order_map[order_aux]['main_id']
except KeyError:
if s_status == 'Canceled' or s_status == 'Filled':
b_sould_update = False
s_status = 'Invalid'
elif s_status == 'Replaced':
s_status = 'New'
# process the message
if s_status == 'New':
b_sould_update = self._new_order(order_aux)
elif s_status != 'Invalid':
i_old_id = self.d_order_map[order_aux]['main_id']
f_old_pr = self.d_order_map[order_aux]['price']
i_old_q = self.d_order_map[order_aux]['qty']
# hold the last traded price
if s_status in ['Partially Filled', 'Filled']:
self.last_price = order_aux['order_price']
# process message
if s_status in ['Canceled', 'Expired', 'Filled']:
b_sould_update = self._canc_expr_filled_order(
order_aux, i_old_id, f_old_pr, i_old_q)
if not b_sould_update:
b_success = False
elif s_status == 'Replaced':
b_sould_update = self._replaced_order(order_aux, i_old_id,
f_old_pr, i_old_q)
elif s_status == 'Partially Filled':
b_sould_update = self._partially_filled(
order_aux, i_old_id, f_old_pr, i_old_q)
# remove from order map
if s_status not in ['New', 'Invalid']:
self.d_order_map.pop(order_aux)
# update the order map
if b_sould_update:
f_qty = int(order_aux['total_qty_order'])
self.d_order_map[order_aux] = {}
self.d_order_map[order_aux]['price'] = d_data['order_price']
self.d_order_map[order_aux]['order_id'] = order_aux.order_id
self.d_order_map[order_aux]['qty'] = f_qty
self.d_order_map[order_aux]['main_id'] = order_aux.main_id
# return that the update was done
return True
def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed canceled, expried or filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# remove from order map
return False
def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed replaced orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# remove from the old price
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# insert the order in the due price
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed partially filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# delete old price, if it is needed
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# add/modify order
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _new_order(self, order_obj):
'''
Update price_tree when passed new orders
:param order_obj: Order Object. The last order in the file
'''
# if it was already in the order map
if order_obj in self.d_order_map:
i_old_sec_id = self.d_order_map[order_obj]['last_order_id']
f_old_price = self.d_order_map[order_obj]['price']
i_old_qty = self.d_order_map[order_obj]['qty']
this_price = self.price_tree.get(f_old_price)
# remove from order map
self.d_order_map.pop(order_obj)
if this_price.delete(i_old_sec_id, i_old_qty):
self.price_tree.remove(f_old_price)
# insert a empty price level if it is needed
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# add the order
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def get_n_top_prices(self, n):
'''
Return a dataframe with the N top price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def get_n_botton_prices(self, n=5):
'''
Return a dataframe with the N botton price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
class OrderQueue(object):
"""
"""
def __init__(self):
self.Orders = FastRBTree()
self.totalAmount = 0.0
def append(self, order):
k = order.timestamp
self.Orders.insert(k, order)
self.totalAmount += self.Orders[k].amount
def count(self):
return self.Orders.count
def min_item(self):
return self.Orders.min_item()
def eat(self, amount):
"""
use this only if amount <= self.amount
return OrderToPop,RestAmount
"""
current_amount = amount
to_pop_orders = []
while self.totalAmount > 0 and current_amount > 0:
if not self.Orders or self.Orders.count == 0:
#print "amount:%.4f" % self.totalAmount
self.totalAmount = 0.0
return []
min_item = self.Orders.min_item()
min_i = min_item[1]
# 1.enough
# 2.not enough
if min_i.amount <= current_amount:
current_amount -= min_i.amount
self.totalAmount -= min_i.amount
to_pop_orders.append(self.Orders.pop_min()[1])
continue
# this is a hard case , need to take care of
# first we split the order , take amount from current order
# and make
elif min_i.amount > current_amount:
new_order = min_i.copy()
new_order.amount = current_amount
to_pop_orders.append(new_order)
min_i.amount -= current_amount
self.totalAmount -= current_amount
current_amount = 0
break
# return:
# 1.orders
# 2.amount to trade
# 3.amount left
return to_pop_orders
def remove(self, order):
k = order.timestamp
self.totalAmount -= self.Orders[k].amount
self.Orders.remove(k)
def get_price_depth(self):
return self.totalAmount
def is_empty(self):
return self.Orders.count == 0
class TDigest(object):
def __init__(self, delta=0.01, K=25):
self.C = RBTree()
self.n = 0
self.delta = delta
self.K = K
def __add__(self, other_digest):
C1 = list(self.C.values())
C2 = list(other_digest.C.values())
shuffle(C1)
shuffle(C2)
data = C1 + C2
new_digest = TDigest(self.delta, self.K)
for c in data:
new_digest.update(c.mean, c.count)
return new_digest
def __len__(self):
return len(self.C)
def __repr__(self):
return """<T-Digest: n=%d, centroids=%d>""" % (self.n, len(self))
def _add_centroid(self, centroid):
if centroid.mean not in self.C:
self.C.insert(centroid.mean, centroid)
else:
self.C[centroid.mean].update(centroid.mean, centroid.count)
def _compute_centroid_quantile(self, centroid):
denom = self.n
cumulative_sum = sum(
c_i.count for c_i in self.C.value_slice(-float('Inf'), centroid.mean))
return (centroid.count / 2. + cumulative_sum) / denom
def _update_centroid(self, centroid, x, w):
self.C.pop(centroid.mean)
centroid.update(x, w)
self._add_centroid(centroid)
def _find_closest_centroids(self, x):
try:
ceil_key = self.C.ceiling_key(x)
except KeyError:
floor_key = self.C.floor_key(x)
return [self.C[floor_key]]
try:
floor_key = self.C.floor_key(x)
except KeyError:
ceil_key = self.C.ceiling_key(x)
return [self.C[ceil_key]]
if abs(floor_key - x) < abs(ceil_key - x):
return [self.C[floor_key]]
elif abs(floor_key - x) == abs(ceil_key - x) and (ceil_key != floor_key):
return [self.C[ceil_key], self.C[floor_key]]
else:
return [self.C[ceil_key]]
def _theshold(self, q):
return 4 * self.n * self.delta * q * (1 - q)
def update(self, x, w=1):
"""
Update the t-digest with value x and weight w.
"""
self.n += w
if len(self) == 0:
self._add_centroid(Centroid(x, w))
return
S = self._find_closest_centroids(x)
while len(S) != 0 and w > 0:
j = choice(list(range(len(S))))
c_j = S[j]
q = self._compute_centroid_quantile(c_j)
# This filters the out centroids that do not satisfy the second part
# of the definition of S. See original paper by Dunning.
if c_j.count + w > self._theshold(q):
S.pop(j)
continue
delta_w = min(self._theshold(q) - c_j.count, w)
self._update_centroid(c_j, x, delta_w)
w -= delta_w
S.pop(j)
if w > 0:
self._add_centroid(Centroid(x, w))
if len(self) > self.K / self.delta:
self.compress()
return
def batch_update(self, values, w=1):
"""
Update the t-digest with an iterable of values. This assumes all points have the
same weight.
"""
for x in values:
self.update(x, w)
self.compress()
return
def compress(self):
T = TDigest(self.delta, self.K)
C = list(self.C.values())
shuffle(C)
for c_i in C:
T.update(c_i.mean, c_i.count)
self.C = T.C
def percentile(self, q):
"""
Computes the percentile of a specific value in [0,1], ie. computes F^{-1}(q) where F^{-1} denotes
the inverse CDF of the distribution.
"""
if not (0 <= q <= 1):
raise ValueError("q must be between 0 and 1, inclusive.")
t = 0
q *= self.n
for i, key in enumerate(self.C.keys()):
c_i = self.C[key]
k = c_i.count
if q < t + k:
if i == 0:
return c_i.mean
elif i == len(self) - 1:
return c_i.mean
else:
delta = (self.C.succ_item(key)[1].mean - self.C.prev_item(key)[1].mean) / 2.
return c_i.mean + ((q - t) / k - 0.5) * delta
t += k
return self.C.max_item()[1].mean
def quantile(self, q):
"""
Computes the quantile of a specific value, ie. computes F(q) where F denotes
the CDF of the distribution.
"""
t = 0
N = float(self.n)
for i, key in enumerate(self.C.keys()):
c_i = self.C[key]
if i == len(self) - 1:
delta = (c_i.mean - self.C.prev_item(key)[1].mean) / 2.
else:
delta = (self.C.succ_item(key)[1].mean - c_i.mean) / 2.
z = max(-1, (q - c_i.mean) / delta)
if z < 1:
return t / N + c_i.count / N * (z + 1) / 2
t += c_i.count
return 1
def trimmed_mean(self, q1, q2):
"""
Computes the mean of the distribution between the two percentiles q1 and q2.
This is a modified algorithm than the one presented in the original t-Digest paper.
"""
if not (q1 < q2):
raise ValueError("q must be between 0 and 1, inclusive.")
s = k = t = 0
q1 *= self.n
q2 *= self.n
for i, key in enumerate(self.C.keys()):
c_i = self.C[key]
k_i = c_i.count
if q1 < t + k_i:
if i == 0:
delta = self.C.succ_item(key)[1].mean - c_i.mean
elif i == len(self) - 1:
delta = c_i.mean - self.C.prev_item(key)[1].mean
else:
delta = (self.C.succ_item(key)[1].mean - self.C.prev_item(key)[1].mean) / 2.
nu = ((q1 - t) / k_i - 0.5) * delta
s += nu * k_i * c_i.mean
k += nu * k_i
if q2 < t + k_i:
return s/k
t += k_i
return s/k
class OrderTree(object):
def __init__(self):
self.price_tree = FastRBTree()
self.min_price = None
self.max_price = None
def get_orders_at_price(self, price):
return self.price_tree.get(price)
def insert_price(self, price, amount, oid):
## ignore market order
if price == Decimal(0.0):
return
prev_val = self.get_orders_at_price(price)
if prev_val != None:
## price exists in local order book
if oid in prev_val:
## update to an existing order at price
prev_val['total'] = prev_val['total'] - prev_val[oid] + amount
prev_val[oid] = amount
else:
## new order at price
prev_val['total'] += amount
prev_val[oid] = amount
self.price_tree.insert(price, prev_val)
elif amount != 0.0:
## price did not exit in order book
val = {'total': amount, oid: amount}
self.price_tree.insert(price, val)
try:
val = self.price_tree.get(price)
if val['total'] > 0:
if self.max_price == None or price > self.max_price:
self.max_price = price
if self.min_price == None or price < self.min_price:
self.min_price = price
elif val['total'] == 0:
## price removed from orderbook
self.remove_price(price)
else:
## something has gone terribly wrong
logging.error(
"total amount at price %s went to negative amounts" %
(price))
except:
logging.error("price (%s) does not exist in orderbook" % (price))
def remove_price(self, price):
self.price_tree.remove(price)
if self.max_price == price:
try:
self.max_price = max(self.price_tree)
except ValueError:
self.max_price = None
if self.min_price == price:
try:
self.min_price = min(self.price_tree)
except ValueError:
self.min_price = None
class PriceTree(object):
def __init__(self, name):
self.tree = FastRBTree()
self.name = name
self.price_map = {} # Map price -> OrderList
self.order_map = {} # Map order_id -> Order
self.min_price = None
self.max_price = None
def insert_price(self, price):
"""
Add a new price TreedNode and associate it with an orderList
:param price:
:return:
"""
new_list = OrderList()
self.tree.insert(price, new_list)
self.price_map[price] = new_list
if self.max_price is None or price > self.max_price:
self.max_price = price
if self.min_price is None or price < self.min_price:
self.min_price = price
def remove_price(self, price):
"""
Remove price from the tree structure and the associated orderList
Update min and max prices if needed
:param price:
:return:
"""
self.tree.remove(price)
# Order-map will still contain all Orders emptied (with size 0)
# as we delete them on the List match_order which is fine for now
for to_del_order in self.price_map[price]:
del self.order_map[to_del_order.id]
# Delete the price from the price-map
del self.price_map[price]
if self.max_price == price:
try:
self.max_price = self.tree.max_key()
except ValueError:
self.max_price = None
if self.min_price == price:
try:
self.min_price = self.tree.min_key()
except ValueError:
self.min_price = None
def insert_price_order(self, order):
if order.price not in self.price_map:
self.insert_price(order.price)
# Add order to orderList
self.price_map[order.price].add(order)
# Also keep it in the order mapping
self.order_map[order.id] = order
def match_price_order(self, curr_order):
if len(self.price_map) == 0:
return []
# if bid -> sell_tree min
# if ask -> buy_tree max
best_price = self.min if curr_order.is_bid else self.max
complete_trades = []
while ((curr_order.is_bid and curr_order.price >= best_price)
or (not curr_order.is_bid and curr_order.price <= best_price)) \
and curr_order.peak_size > 0:
# Get price OrderList
matching_orders_list = self.get_price(best_price)
complete_trades.extend(
matching_orders_list.match_order(curr_order, self.order_map))
# Remove exhausted price
if matching_orders_list.size == 0:
self.remove_price(best_price)
if len(self.price_map) == 0:
break
# Try to find more price matches using the next price
best_price = self.min if curr_order.is_bid else self.max
return complete_trades
def price_exists(self, price):
return price in self.price_map
def order_exists(self, id_num):
return id_num in self.order_map
def get_price(self, price):
return self.price_map[price]
def get_order(self, id_num):
return self.order_map[id_num]
@property
def max(self):
return self.max_price
@property
def min(self):
return self.min_price
class SymbolTable():
#constructor
def __init__(self, SourceFile):
self.Table = [
] #declare table as a stack (list) containing an empty tree
self.TopScope = FastRBTree(
) # a place where the current top scope is held
self.ReadMode = False #Read or lookup mode
self.DebugMode = False
self.SourceFile = SourceFile
#Function: InsertSymbol
#Desc: Insert a symbol into the current top of the symbol table
# The symbol key string is a lexeme
#Content_dict: At present this will contain
# {Type: (the token adjacent to the SymbolKey i.e. >int< <SymbolKey>),
# Attribute: some modifier on the symbol 'static' 'const' etc.
# TokenLocation: tuple(line, char_in_file, char_in_line) }
def InsertSymbol(self, SymbolKey_str, Content_dict):
try:
if self.DebugMode == True:
print("Insert symbol is called: ", "In Read Mode?",
self.ReadMode, SymbolKey_str, Content_dict)
if self.ReadMode == False:
found = self.FindSymbolInCurrentScope(SymbolKey_str)
if not found:
found = self.FindSymbolInTable(SymbolKey_str)
if found:
for item in found:
for key in list(item.keys()):
self.PrettyErrorPrint(
"Warning: {0} on line {3} is a shadowded variable. Previous declaration in scope level {1} at line {2}."
.format(SymbolKey_str,
abs(key - len(self.Table)),
item[key]["TokenLocation"][0],
Content_dict['TokenLocation'][0]),
item[key]["TokenLocation"][0],
item[key]["TokenLocation"][2])
#perform deepcopy on passed in dictionary
self.TopScope.insert(SymbolKey_str, deepcopy(Content_dict))
else:
self.PrettyErrorPrint(
'''Error: Redeclaration of existing variable.\nPrior declaration is here at line {0}: \n'''
.format(found["TokenLocation"][0]),
found["TokenLocation"][0], found["TokenLocation"][2])
raise Exception(
'Redeclaration of exisitng variable in current scope.')
else:
# do nothing
pass
except Exception as e:
raise e
return True
#Function: FindSymbolInTable
#Desc: Search all scopes of the symbol table to find a specific symbol
#Return: [{Level_int: Content_dict}, False] (list of keys for possibility of many shadowed vars)
def FindSymbolInTable(self, SymbolKey_str):
T_list = []
Level_int = 0
#search the top of our stack
if self.FindSymbolInCurrentScope(SymbolKey_str):
T_list.append(self.FindSymbolInCurrentScope(SymbolKey_str))
Level_int += 1
#iterate over all trees
#add found isntances of symbols to list if present
for Tree in reversed(
self.Table): #reversed so appended items are at the front
if Tree.__contains__(SymbolKey_str):
T_list.append({Level_int: Tree.get(SymbolKey_str)})
Level_int += 1
if len(T_list) > 0:
return T_list
#nothing found case
return False
#Function: FindSymbolInCurrentScope
#Desc: Search only the top level of the symbol table for key
def FindSymbolInCurrentScope(self, SymbolKey_str):
return self.TopScope.get(SymbolKey_str, False)
#Function:PushNewScope
#Desc: Create a new scope and push it onto the table
def PushNewScope(self):
self.PushScope(FastRBTree())
return
#Function: PushScope
#Desc: Insert symbol tree (RBTree) onto our table
def PushScope(self, SymbolTree):
self.Table.append(self.TopScope)
self.TopScope = SymbolTree
return
#Function: PopScope
#Desc: Remove and return scope (RBtree) from the symbol table
def PopScope(self):
TScope = self.TopScope
if len(self.Table) > 0:
self.TopScope = self.Table.pop()
else:
self.TopScope = None
return TScope
#Function: WriteSymbolTableToFile
#Desc: Write the current contents of the symbol table to file
def WriteSymbolTableToFile(self, FileName_str):
T_Stack = []
i = 0
try:
with open(FileName_str, "w") as File:
File.write(
"\n**** Outputting Contents of Symbol Table **** \n\n")
while not self.TableIsEmpty():
File.write("Items in Tree at Level {}: \n".format(i))
self.PrettyPrintScope(self.TopScope, FilePtr=File)
T_Stack.append(self.PopScope())
i += 1
while len(T_Stack) > 0:
self.PushScope(T_Stack.pop())
except Exception as e:
raise
def PrettyPrintScope(self, Scope, FilePtr=None):
for Key in Scope.keys():
if FilePtr is not None:
FilePtr.write("\tKey: \"{}\" | Content: {}\n".format(
Key, Scope.get(Key)))
else:
print("\tKey: \"{}\" | Content: {}\n".format(
Key, Scope.get(Key)))
return
def ToggleDebugMode(self):
self.DebugMode = not self.DebugMode
return
def ReadModeOn(self):
self.ReadMode = True
# print("Insert Mode Toggled Off")
return
def InsertMode(self):
self.ReadMode = False
# print("Insert Mode Toggled On")
return
def ToggleReadMode(self):
self.ReadMode = not self.ReadMode
if self.ReadMode == False:
print("Insert Mode Toggled On")
elif self.ReadMode == True:
print("Insert Mode Toggled Off")
return
def TableIsEmpty(self):
if self.TopScope is None:
return True
return False
def PrettyErrorPrint(self, Message, Lineno, Column):
arrow = ""
print(Message)
#print line
with open(self.SourceFile) as file:
for i in range(0, Lineno):
source = file.readline()
print(source)
#build arrow
for i in range(0, Column - 1):
arrow += " "
arrow += "^\n"
print(arrow)
class OrderBook(object):
"""
Uses RBTrees to handle all types of orders and store them in their corresponding bucket
"""
def __init__(self, product_id: str):
self._asks = RBTree()
self._bids = RBTree()
self._product_id = product_id
@property
def product_id(self):
return self._product_id
def process_snapshot(self, message: Dict):
"""
Process a snapshot message
:param message: json
"""
# If a snapshot is sent reset trees
self._asks = RBTree()
self._bids = RBTree()
# Parse all asks and add them to tree
for ask in message['asks']:
price, size = ask
price = Decimal(price)
size = Decimal(size)
self._asks.insert(price, size)
# Parse all bids and add them to tree
for bid in message['bids']:
price, size = bid
price = Decimal(price)
size = Decimal(size)
self._bids.insert(price, size)
def process_update(self, message: Dict):
"""
Process a update message
:param message: json
"""
# Retrieve changes
changes = message['changes']
for change in changes:
side, price, size = change
# parse numbers and keep precision
price = Decimal(price)
size = Decimal(size)
if side == 'buy':
# If it is equal to 0 (or less than) the order no longer exists
if size <= 0:
self._bids.remove(price)
else:
self._bids.insert(price, size)
elif side == 'sell':
# If it is equal to 0 (or less than) the order no longer exists
if size <= 0:
self._asks.remove(price)
else:
self._asks.insert(price, size)
def process_message(self, message: Dict):
"""
Process all messages to identify next parser location
:param message: json
"""
# Read type
msg_type = message['type']
# dropped - not same product id
if message.get('product_id', None) != self._product_id:
return
if msg_type == 'snapshot':
self.process_snapshot(message)
elif msg_type == 'l2update':
self.process_update(message)
def get_asks(self) -> List[Tuple[float, float]]:
"""
Provides a list of asks and sizes in order of best price for the buyer
:return: a list of Tuple's corresponding to ask (price rate), and ask size
"""
asks = []
for ask in self._asks:
try:
size = self._asks[ask]
except KeyError:
continue
asks.append([float(ask), float(size)])
return asks
def get_bids(self) -> List[Tuple[float, float]]:
"""
Provides a list of bids and sizes in order of best price for the seller
:return: a list of Tuple's corresponding to ask (price rate), and ask size
"""
bids = []
for bid in self._bids:
try:
size = self._bids[bid]
except KeyError:
continue
# For bids the best value (for selling) is reversed so inserting at the beginning flips the order
bids.insert(0, [float(bid), float(size)])
return bids
def get_orders(self) -> Dict[str, List[Tuple[float, float]]]:
"""
Uses get_bids and get_asks to compile all orders
:return: both bids and asks
"""
return {'asks': self.get_asks(), 'bids': self.get_bids()}
def get_ask(self) -> Tuple[Decimal, Decimal]:
"""
Get the best asking price. If it does not exist it returns a size of 0
:return: the rate, and the size
"""
price = self._asks.min_key()
try:
size = self._asks[price]
except KeyError:
return price, Decimal(0)
return price, size
def get_bid(self) -> Tuple[Decimal, Decimal]:
"""
Get the best bid price. If it does not exist it returns a size of 0
:return: the rate, and the size
"""
price = self._bids.max_key()
try:
size = self._bids[price]
except KeyError:
return price, Decimal(0)
return price, size
class BookSide(object):
'''
A side of the lmit order book representation
'''
def __init__(self, s_side):
'''
Initialize a BookSide object. Save all parameters as attributes
:param s_side: string. BID or ASK
'''
if s_side not in ['BID', 'ASK']:
raise InvalidTypeException('side should be BID or ASK')
self.s_side = s_side
self.price_tree = FastRBTree()
self._i_idx = 0
self.d_order_map = {}
self.last_price = 0.
def update(self, d_data):
'''
Update the state of the order book given the data pased. Return if the
message was handle successfully
:param d_data: dict. data related to a single order
'''
# dont process aggresive trades
if d_data['agressor_indicator'] == 'Agressive':
return True
# update the book information
order_aux = Order(d_data)
s_status = order_aux['order_status']
b_sould_update = True
b_success = True
# check the order status
if s_status != 'New':
try:
i_old_id = self.d_order_map[order_aux]['main_id']
except KeyError:
if s_status == 'Canceled' or s_status == 'Filled':
b_sould_update = False
s_status = 'Invalid'
elif s_status == 'Replaced':
s_status = 'New'
# process the message
if s_status == 'New':
b_sould_update = self._new_order(order_aux)
elif s_status != 'Invalid':
i_old_id = self.d_order_map[order_aux]['main_id']
f_old_pr = self.d_order_map[order_aux]['price']
i_old_q = self.d_order_map[order_aux]['qty']
# hold the last traded price
if s_status in ['Partially Filled', 'Filled']:
self.last_price = order_aux['order_price']
# process message
if s_status in ['Canceled', 'Expired', 'Filled']:
b_sould_update = self._canc_expr_filled_order(order_aux,
i_old_id,
f_old_pr,
i_old_q)
if not b_sould_update:
b_success = False
elif s_status == 'Replaced':
b_sould_update = self._replaced_order(order_aux,
i_old_id,
f_old_pr,
i_old_q)
elif s_status == 'Partially Filled':
b_sould_update = self._partially_filled(order_aux,
i_old_id,
f_old_pr,
i_old_q)
# remove from order map
if s_status not in ['New', 'Invalid']:
self.d_order_map.pop(order_aux)
# update the order map
if b_sould_update:
f_qty = int(order_aux['total_qty_order'])
self.d_order_map[order_aux] = {}
self.d_order_map[order_aux]['price'] = d_data['order_price']
self.d_order_map[order_aux]['order_id'] = order_aux.order_id
self.d_order_map[order_aux]['qty'] = f_qty
self.d_order_map[order_aux]['main_id'] = order_aux.main_id
# return that the update was done
return True
def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed canceled, expried or filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# remove from order map
return False
def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed replaced orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# remove from the old price
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# insert the order in the due price
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed partially filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# delete old price, if it is needed
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# add/modify order
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _new_order(self, order_obj):
'''
Update price_tree when passed new orders
:param order_obj: Order Object. The last order in the file
'''
# if it was already in the order map
if order_obj in self.d_order_map:
i_old_sec_id = self.d_order_map[order_obj]['last_order_id']
f_old_price = self.d_order_map[order_obj]['price']
i_old_qty = self.d_order_map[order_obj]['qty']
this_price = self.price_tree.get(f_old_price)
# remove from order map
self.d_order_map.pop(order_obj)
if this_price.delete(i_old_sec_id, i_old_qty):
self.price_tree.remove(f_old_price)
# insert a empty price level if it is needed
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# add the order
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def get_n_top_prices(self, n):
'''
Return a dataframe with the N top price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def get_n_botton_prices(self, n=5):
'''
Return a dataframe with the N botton price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
class Tree(object):
def __init__(self):
self.price_tree = FastRBTree()
self.volume = 0
self.price_map = {} # Map from price -> order_list object
self.order_map = {} # Order ID to Order object
self.min_price = None
self.max_price = None
def __len__(self):
return len(self.order_map)
def get_price(self, price):
return self.price_map[price]
def get_order(self, id_num):
return self.order_map[id_num]
def create_price(self, price):
new_list = OrderList()
self.price_tree.insert(price, new_list)
self.price_map[price] = new_list
if self.max_price == None or price > self.max_price:
self.max_price = price
if self.min_price == None or price < self.min_price:
self.min_price = price
def remove_price(self, price):
self.price_tree.remove(price)
del self.price_map[price]
if self.max_price == price:
try:
self.max_price = max(self.price_tree)
except ValueError:
self.max_price = None
if self.min_price == price:
try:
self.min_price = min(self.price_tree)
except ValueError:
self.min_price = None
def price_exists(self, price):
return price in self.price_map
def order_exists(self, id_num):
return id_num in self.order_map
def insert_tick(self, tick):
if tick.price not in self.price_map:
self.create_price(tick.price)
order = Order(tick, self.price_map[tick.price])
self.price_map[order.price].append_order(order)
self.order_map[order.id_num] = order
self.volume += order.qty
def update_order(self, tick):
order = self.order_map[tick.id_num]
original_volume = order.qty
if tick.price != order.price:
# Price changed
order_list = self.price_map[order.price]
order_list.remove_order(order)
if len(order_list) == 0:
self.remove_price(order.price)
self.insert_tick(tick)
self.volume -= original_volume
else:
# Quantity changed
order.update_qty(tick.qty, tick.price)
self.volume += order.qty - original_volume
def remove_order_by_id(self, id_num):
order = self.order_map[id_num]
self.volume -= order.qty
order.order_list.remove_order(order)
if len(order.order_list) == 0:
self.remove_price(order.price)
del self.order_map[id_num]
def max(self):
return self.max_price
def min(self):
return self.min_price
class BidBook(object):
"""
A BidBook is used to store the order book's rates and amounts on the bid side with a defined depth.
To maintain a sorted order of rates, the BidBook uses a red-black tree to store rates and corresponding amounts.
For O(1) query of volume at a predetermined rate, the BidBook also uses a dictionary to store rate and amount.
"""
def __init__(self, max_depth, data):
# RBTree: maintains sorted order of rates
# every value inserted to RBTree must be a tuple, so we hard code the second value to be 0
self.rate_tree = FastRBTree()
# dict: Uses rate and amount for key value pairs
self.rate_dict = {}
# float: amounts summed across all rate levels in tree
self.volume = 0
# int: total number of rate levels in tree
self.depth = len(data)
# int: maximum number of rate levels in tree
self.max_depth = max_depth
# populate rate_tree and rate_dict from public API call data
# set volume
for level in data:
rate = float(level[0])
amount = float(level[1])
self.rate_tree.insert(rate, 0)
self.rate_dict[rate] = amount
self.volume += amount
def __len__(self):
return len(self.rate_dict)
def rate_exists(self, rate):
return rate in self.rate_dict
def get_amount_at_rate(self, rate):
return self.rate_dict.get(rate)
def max_rate_level(self):
if self.depth > 0:
rate = self.rate_tree.max_key()
amount = self.get_amount_at_rate(rate)
return rate, amount
else:
return None
def min_rate_level(self):
if self.depth > 0:
rate = self.rate_tree.min_key()
amount = self.get_amount_at_rate(rate)
return rate, amount
else:
return None
def modify(self, event):
# if the event's rate is already in the book, just modify the amount at the event's rate
rate = float(event[u'data'][u'rate'])
amount = float(event[u'data'][u'amount'])
if self.rate_exists(rate):
# print '~~~~~~~~~~~~~~~~~~~~~~ BID MODIFY ~~~~~~~~~~~~~~~~~~~~~~'
self.rate_dict[rate] = amount
# only rates not already in the book reach this logic
# if the max depth hasn't been reached, just insert the event's rate and amount
elif self.depth < self.max_depth:
# print '~~~~~~~~~~~~~~~~~~~~~~ BID MODIFY ~~~~~~~~~~~~~~~~~~~~~~'
self.rate_tree.insert(rate, 0)
self.rate_dict[rate] = amount
self.depth += 1
# only events being handled by a full order tree reach this logic
# if the event is a bid and the rate is greater than min rate, effectively replace min rate level with event
else:
min_rate = self.min_rate_level()[0]
if rate > min_rate:
# print '~~~~~~~~~~~~~~~~~~~~~~ BID MODIFY ~~~~~~~~~~~~~~~~~~~~~~'
self.rate_tree.remove(min_rate)
del self.rate_dict[min_rate]
self.rate_tree.insert(rate, 0)
self.rate_dict[rate] = amount
def remove(self, event):
# if the event's rate is in the book, delete it
rate = float(event[u'data'][u'rate'])
if self.rate_exists(rate):
# print '~~~~~~~~~~~~~~~~~~~~~~ BID REMOVE ~~~~~~~~~~~~~~~~~~~~~~'
self.rate_tree.remove(rate)
del self.rate_dict[rate]
self.depth -= 1
def __str__(self):
rate_tree_str = '[' + ','.join(rate[0]
for rate in self.rate_tree) + ']'
return 'BIDS: ' + rate_tree_str
class BookSide(object):
'''
A side of the lmit order book representation
'''
def __init__(self, s_side, fr_data, i_member=None):
'''
Initialize a BookSide object. Save all parameters as attributes
:param s_side: string. BID or ASK
:param fr_data: ZipExtFile object. data to read
:param i_member*: integer. Member number to be used as a filter
'''
if s_side not in ['BID', 'ASK']:
raise InvalidTypeException('side should be BID or ASK')
self.i_member = i_member
self.s_side = s_side
self.price_tree = FastRBTree()
self._i_idx = 0
self.fr_data = fr_data
self.parser = parser_data.LineParser(s_side)
self.d_order_map = {}
self.last_price = 0.
# control other statistics
self.best_queue = (None, None)
self.i_qty_rel = 0
self.i_cum_rel = 0
def set_last_best_queue(self, t_best):
'''
Set the best queue of this side
'''
self.best_queue = t_best
def update(self, d_data):
'''
Update the state of the order book given the data pased
:param d_data: dict. data from the last row
'''
# update the book information
order_aux = Order(d_data)
s_status = order_aux['order_status']
b_sould_update = True
i_rel_price = 0
# treat Bovespa files at the begining f the day
if s_status != 'New':
try:
i_old_id = self.d_order_map[order_aux]['main_id']
except KeyError:
# is not securing changes, also change part. filled status
l_check = ['Canceled', 'Filled']
if not self.b_secure_changes:
l_check = ['Canceled', 'Filled', 'Partially Filled']
# change order status when it is not found
if s_status in l_check:
b_sould_update = False
s_status = 'Invalid'
elif s_status == 'Replaced':
s_status = 'New'
# process
if s_status == 'New':
b_sould_update = self._new_order(order_aux)
i_rel_price = get_relative_price(self.best_queue, order_aux)
elif s_status != 'Invalid':
i_old_id = self.d_order_map[order_aux]['main_id']
f_old_pr = self.d_order_map[order_aux]['price']
i_old_q = self.d_order_map[order_aux]['qty']
i_rel_price = self.d_order_map[order_aux]['relative_price']
# hold the last traded price
if s_status in ['Partially Filled', 'Filled']:
self.last_price = order_aux['order_price']
# process message
if s_status in ['Canceled', 'Expired', 'Filled']:
b_sould_update = self._canc_expr_filled_order(order_aux,
i_old_id,
f_old_pr,
i_old_q)
elif s_status == 'Replaced':
i_rel_price = get_relative_price(self.best_queue, order_aux)
b_sould_update = self._replaced_order(order_aux,
i_old_id,
f_old_pr,
i_old_q)
elif s_status == 'Partially Filled':
b_sould_update = self._partially_filled(order_aux,
i_old_id,
f_old_pr,
i_old_q)
# remove from order map
if s_status not in ['New', 'Invalid']:
self.d_order_map.pop(order_aux)
# update the order map
if b_sould_update:
f_qty = int(order_aux['total_qty_order'])
f_prior_time = d_data['priority_seconds']
self.d_order_map[order_aux] = {}
self.d_order_map[order_aux]['price'] = d_data['order_price']
self.d_order_map[order_aux]['sec_order'] = order_aux.sec_order_id
self.d_order_map[order_aux]['qty'] = f_qty
self.d_order_map[order_aux]['main_id'] = order_aux.main_id
self.d_order_map[order_aux]['priority_seconds'] = f_prior_time
self.d_order_map[order_aux]['relative_price'] = i_rel_price
if s_status in ['New', 'Replaced']:
self.i_qty_rel += f_qty * 1.
self.i_cum_rel += i_rel_price * 1. * f_qty
# return that the update was done
return True
def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed canceled, expried or filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
b_break = False
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# remove from order map
if b_break:
raise NotImplementedError
return False
def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed replaced orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# remove from the old price
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# insert the order in the due price
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed partially filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# delete old price, if it is needed
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# add/modify order
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _new_order(self, order_obj):
'''
Update price_tree when passed new orders
:param order_obj: Order Object. The last order in the file
'''
# if it was already in the order map
if order_obj in self.d_order_map:
i_old_sec_id = self.d_order_map[order_obj]['main_id']
f_old_price = self.d_order_map[order_obj]['price']
i_old_qty = self.d_order_map[order_obj]['qty']
this_price = self.price_tree.get(f_old_price)
# remove from order map
self.d_order_map.pop(order_obj)
if this_price.delete(i_old_sec_id, i_old_qty):
self.price_tree.remove(f_old_price)
# insert a empty price level if it is needed
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# add the order
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def get_n_top_prices(self, n):
'''
Return a dataframe with the N top price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def get_n_botton_prices(self, n=5):
'''
Return a dataframe with the N botton price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def _readline(self):
'''
Return a line from the fr_data file if available. Return false
otherwiese
'''
row = self.fr_data.readline()
if row == '':
self.fr_data.close()
return False, False
self._i_idx += 1
d_aux = self.parser(row)
# treat when the line is zero
# TODO: I should move it to preprocessment step
if 'order_price' in d_aux:
if d_aux['order_price'] == 0.:
while True:
row = self.fr_data.readline()
d_aux = self.parser(row)
if d_aux['order_price'] != 0.:
break
return d_aux, self.parser.last_identification
def __iter__(self):
'''
Return the self as an iterator object. Use next() to check the rows
'''
return self
def next(self):
'''
Return the next item from the fr_data in iter process. If there are no
further items, raise the StopIteration exception
'''
d_rtn, last_identification = self._readline()
if not d_rtn:
raise StopIteration
return d_rtn, last_identification
class Tdigest(object):
def __init__(self, delta=0.01, K=25, CX=1.1):
self.delta = delta
self.K = K
self.CX = CX
self.centroids = RBTree()
self.nreset = 0
self.reset()
def reset(self):
self.centroids.clear()
self.n = 0
self.nreset += 1
self.last_cumulate = 0
self.compressing = False
def push(self, x, n=1):
if not isinstance(x, list):
x = [x]
for item in x:
self._digest(item, n)
def percentile(self, p):
if self.size() == 0:
return None
self._cumulate(True)
cumn = self.n * p
lower = self.centroids.min_item()[1]
upper = self.centroids.max_item()[1]
for c in self.centroids.values():
if c.cumn <= cumn:
lower = c
else:
upper = c
break
if lower == upper:
return lower.mean
return lower.mean + (cumn - lower.cumn) * (upper.mean - lower.mean) / \
(upper.cumn - lower.cumn)
def serialize(self):
result = '%s~%s~%s~' % (self.delta, self.K, self.size())
if self.size() == 0:
return result
self._cumulate(True)
means = []
counts = []
for c in self.centroids.values():
means.append(str(c.mean))
counts.append(str(c.n))
return '%s%s~%s' % (result, '~'.join(means), '~'.join(counts))
@classmethod
def deserialize(cls, serialized_str):
if not isinstance(serialized_str, basestring):
raise Exception(u'serialized_str must be str')
data = serialized_str.split('~')
t = Tdigest(delta=float(data[0]), K=int(data[1]))
size = int(data[2])
for i in xrange(size):
t.push(float(data[i + 3]), int(data[size + i + 3]))
t._cumulate(True)
return t
def _digest(self, x, n):
if self.size() == 0:
self._new_centroid(x, n, 0)
else:
_min = self.centroids.min_item()[1]
_max = self.centroids.max_item()[1]
nearest = self.find_nearest(x)
if nearest and nearest.mean == x:
self._addweight(nearest, x, n)
elif nearest == _min:
self._new_centroid(x, n, 0)
elif nearest == _max:
self._new_centroid(x, n, self.n)
else:
p = (nearest.cumn + nearest.n / 2.0) / self.n
max_n = int(4 * self.n * self.delta * p * (1 - p))
if max_n >= nearest.n + n:
self._addweight(nearest, x, n)
else:
self._new_centroid(x, n, nearest.cumn)
self._cumulate(False)
if self.K and self.size() > self.K / self.delta:
self.compress()
def find_nearest(self, x):
if self.size() == 0:
return None
try:
lower = self.centroids.ceiling_item(x)[1]
except KeyError:
lower = None
if lower and lower.mean == x:
return lower
try:
prev = self.centroids.floor_item(x)[1]
except KeyError:
prev = None
if not lower:
return prev
if not prev:
return lower
if abs(prev.mean - x) < abs(lower.mean - x):
return prev
else:
return lower
def size(self):
return len(self.centroids)
def compress(self):
if self.compressing:
return
points = self.toList()
self.reset()
self.compressing = True
for point in sorted(points, key=lambda x: random()):
self.push(point['mean'], point['n'])
self._cumulate(True)
self.compressing = False
def _cumulate(self, exact):
if self.n == self.last_cumulate:
return
if not exact and self.CX and self.last_cumulate and \
self.CX > (self.n / self.last_cumulate):
return
cumn = 0
for c in self.centroids.values():
cumn = c.cumn = cumn + c.n
self.n = self.last_cumulate = cumn
def toList(self):
return [dict(mean=c.mean, n=c.n, cumn=c.cumn) for
c in self.centroids.values()]
def _addweight(self, nearest, x, n):
if x != nearest.mean:
nearest.mean += n * (x - nearest.mean) / (nearest.n + n)
nearest.cumn += n
nearest.n += n
self.n += n
def _new_centroid(self, x, n, cumn):
c = Centroid(x, n, cumn)
self.centroids.insert(x, c)
self.n += n
return c
class PriceLevel(object):
'''
A representation of a Price level in the book
'''
def __init__(self, f_price):
'''
A representation of a PriceLevel object
'''
self.f_price = f_price
self.i_qty = 0
self.order_tree = FastRBTree()
def add(self, order_aux):
'''
Insert the information in the tree using the info in order_aux. Return
is should delete the Price level or not
:param order_aux: Order Object. The Order message to be updated
:param i_old_sec_order: Integer. The previous secondary order id
:param i_old_qty: Integer. The previous order qty
'''
# check if the order_aux price is the same of the self
s_status = order_aux['order_status']
if order_aux['order_price'] != self.f_price:
raise DifferentPriceException
elif s_status in ['New', 'Replaced', 'Partially Filled']:
self.order_tree.insert(order_aux.main_id, order_aux)
self.i_qty += int(order_aux['total_qty_order'])
# check if there is no object in the updated tree (should be deleted)
return self.order_tree.count == 0
def delete(self, i_old_sec_order, i_old_qty):
'''
Delete the information in the tree using the info in order_aux. Return
is should delete the Price level or not
:param order_aux: Order Object. The Order message to be updated
:param i_old_sec_order: Integer. The previous secondary order id
:param i_old_qty: Integer. The previous order qty
'''
# check if the order_aux price is the same of the self
try:
self.order_tree.remove(i_old_sec_order)
self.i_qty -= i_old_qty
except KeyError:
raise DifferentPriceException
# check if there is no object in the updated tree (should be deleted)
return self.order_tree.count == 0
def __str__(self):
'''
Return the name of the PriceLevel
'''
return '{:,.0f}'.format(self.i_qty)
def __repr__(self):
'''
Return the name of the PriceLevel
'''
return '{:,.0f}'.format(self.i_qty)
def __eq__(self, other):
'''
Return if a PriceLevel has equal price from the other
:param other: PriceLevel object. PriceLevel to be compared
'''
# just to make sure that there is no floating point discrepance
f_aux = other
if not isinstance(other, float):
f_aux = other.f_price
return abs(self.f_price - f_aux) < 1e-4
def __gt__(self, other):
'''
Return if a PriceLevel has a gerater price from the other.
Bintrees uses that to compare nodes
:param other: PriceLevel object. PriceLevel to be compared
'''
# just to make sure that there is no floating point discrepance
f_aux = other
if not isinstance(other, float):
f_aux = other.f_price
return (f_aux - self.f_price) > 1e-4
def __lt__(self, other):
'''
Return if a Order has smaller order_id from the other. Bintrees uses
that to compare nodes
:param other: Order object. Order to be compared
'''
f_aux = other
if not isinstance(other, float):
f_aux = other.f_price
return (f_aux - self.f_price) < -1e-4
def __ne__(self, other):
'''
Return if a Order has different order_id from the other
:param other: Order object. Order to be compared
'''
return not self.__eq__(other)
class Tree(object):
def __init__(self):
self.price_tree = FastRBTree()
self.volume = 0
self.price_map = {} # Map from price -> order_list object
self.order_map = {} # Order ID to Order object
self.min_price = None
self.max_price = None
def __len__(self):
return len(self.order_map)
def get_price(self, price):
return self.price_map[price]
def get_order(self, id_num):
return self.order_map[id_num]
def create_price(self, price):
new_list = OrderList()
self.price_tree.insert(price, new_list)
self.price_map[price] = new_list
if self.max_price == None or price > self.max_price:
self.max_price = price
if self.min_price == None or price < self.min_price:
self.min_price = price
def remove_price(self, price):
self.price_tree.remove(price)
del self.price_map[price]
if self.max_price == price:
try:
self.max_price = max(self.price_tree)
except ValueError:
self.max_price = None
if self.min_price == price:
try:
self.min_price = min(self.price_tree)
except ValueError:
self.min_price = None
def price_exists(self, price):
return price in self.price_map
def order_exists(self, id_num):
return id_num in self.order_map
def insert_tick(self, tick):
if tick.price not in self.price_map:
self.create_price(tick.price)
order = Order(tick, self.price_map[tick.price])
self.price_map[order.price].append_order(order)
self.order_map[order.id_num] = order
self.volume += order.qty
def update_order(self, tick):
order = self.order_map[tick.id_num]
original_volume = order.qty
if tick.price != order.price:
# Price changed
order_list = self.price_map[order.price]
order_list.remove_order(order)
if len(order_list) == 0:
self.remove_price(order.price)
self.insert_tick(tick)
self.volume -= original_volume
else:
# Quantity changed
order.update_qty(tick.qty, tick.price)
self.volume += order.qty - original_volume
def remove_order_by_id(self, id_num):
order = self.order_map[id_num]
self.volume -= order.qty
order.order_list.remove_order(order)
if len(order.order_list) == 0:
self.remove_price(order.price)
del self.order_map[id_num]
def max(self):
return self.max_price
def min(self):
return self.min_price
class BookSide(object):
'''
A side of the lmit order book representation
'''
def __init__(self, s_side, fr_data, i_member=None):
'''
Initialize a BookSide object. Save all parameters as attributes
:param s_side: string. BID or ASK
:param fr_data: ZipExtFile object. data to read
:param i_member*: integer. Member number to be used as a filter
'''
if s_side not in ['BID', 'ASK']:
raise InvalidTypeException('side should be BID or ASK')
self.i_member = i_member
self.s_side = s_side
self.price_tree = FastRBTree()
self._i_idx = 0
self.fr_data = fr_data
self.parser = parser_data.LineParser(s_side)
self.d_order_map = {}
self.last_price = 0.
def how_many_rows_read(self):
'''
Return the number of rows processed
'''
return self._i_idx
def update(self, d_data, s_last_ident):
'''
Update the state of the order book given the data pased
:param d_data: dict. data from the last row
:param s_last_ident: string. last identification
'''
# check if the information should be processed
if s_last_ident != 'MSG':
return False
# check if should filter out member
if not self._should_use_it(d_data):
return False
# update the book information
order_aux = Order(d_data)
s_status = order_aux['order_status']
b_sould_update = True
# treat Bovespa files at the begining f the day
if s_status != 'New':
try:
i_old_id = self.d_order_map[order_aux]['main_id']
except KeyError:
if s_status == 'Canceled' or s_status == 'Filled':
b_sould_update = False
s_status = 'Invalid'
elif s_status == 'Replaced':
s_status = 'New'
# process
if s_status == 'New':
b_sould_update = self._new_order(order_aux)
elif s_status != 'Invalid':
i_old_id = self.d_order_map[order_aux]['main_id']
f_old_pr = self.d_order_map[order_aux]['price']
i_old_q = self.d_order_map[order_aux]['qty']
# hold the last traded price
if s_status in ['Partially Filled', 'Filled']:
self.last_price = order_aux['order_price']
# process message
if s_status in ['Canceled', 'Expired', 'Filled']:
b_sould_update = self._canc_expr_filled_order(
order_aux, i_old_id, f_old_pr, i_old_q)
elif s_status == 'Replaced':
b_sould_update = self._replaced_order(order_aux, i_old_id,
f_old_pr, i_old_q)
elif s_status == 'Partially Filled':
b_sould_update = self._partially_filled(
order_aux, i_old_id, f_old_pr, i_old_q)
# remove from order map
if s_status not in ['New', 'Invalid']:
self.d_order_map.pop(order_aux)
# update the order map
if b_sould_update:
f_qty = int(order_aux['total_qty_order'])
self.d_order_map[order_aux] = {}
self.d_order_map[order_aux]['price'] = d_data['order_price']
self.d_order_map[order_aux]['sec_order'] = order_aux.sec_order_id
self.d_order_map[order_aux]['qty'] = f_qty
self.d_order_map[order_aux]['main_id'] = order_aux.main_id
# return that the update was done
return True
def _should_use_it(self, d_data):
'''
Check if should use the passed row to update method
:param d_data: dict. data from the last row
'''
if self.i_member:
if d_data['member'] != self.i_member:
return False
return True
def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed canceled, expried or filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# remove from order map
return False
def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed replaced orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# remove from the old price
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# insert the order in the due price
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed partially filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# delete old price, if it is needed
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# add/modify order
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _new_order(self, order_obj):
'''
Update price_tree when passed new orders
:param order_obj: Order Object. The last order in the file
'''
# if it was already in the order map
if order_obj in self.d_order_map:
i_old_sec_id = self.d_order_map[order_obj]['main_id']
f_old_price = self.d_order_map[order_obj]['price']
i_old_qty = self.d_order_map[order_obj]['qty']
this_price = self.price_tree.get(f_old_price)
# remove from order map
self.d_order_map.pop(order_obj)
if this_price.delete(i_old_sec_id, i_old_qty):
self.price_tree.remove(f_old_price)
# insert a empty price level if it is needed
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# add the order
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def get_n_top_prices(self, n):
'''
Return a dataframe with the N top price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def get_n_botton_prices(self, n=5):
'''
Return a dataframe with the N botton price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def _readline(self):
'''
Return a line from the fr_data file if available. Return false
otherwiese
'''
row = self.fr_data.readline()
if row == '':
self.fr_data.close()
return False, False
self._i_idx += 1
d_aux = self.parser(row)
return d_aux, self.parser.last_identification
def __iter__(self):
'''
Return the self as an iterator object. Use next() to check the rows
'''
return self
def next(self):
'''
Return the next item from the fr_data in iter process. If there are no
further items, raise the StopIteration exception
'''
d_rtn, last_identification = self._readline()
if not d_rtn:
raise StopIteration
return d_rtn, last_identification
