def solve(n, k): tree = FastRBTree() tree.insert(n, 1) ls = rs = n for i in range(k): key, val = tree.max_item() tree.remove(key) if val > 1: tree.insert(key, val - 1) if key % 2 == 1: key //= 2 ls = rs = key update_tree(tree, key) update_tree(tree, key) else: key //= 2 ls = key rs = key - 1 update_tree(tree, ls) update_tree(tree, rs) return str(ls) + " " + str(rs)
class Tree(object): def __init__(self): self.price_tree = FastRBTree() self.price_map = {} self.order_map = {} self.received_orders = {} def receive(self, order_id, size): self.received_orders[order_id] = size def create_price(self, price): new_list = [] self.price_tree.insert(price, new_list) self.price_map[price] = new_list def remove_price(self, price): self.price_tree.remove(price) del self.price_map[price] def insert_order(self, order_id, size, price, initial=False): if not initial: del self.received_orders[order_id] if price not in self.price_map: self.create_price(price) order = { 'order_id': order_id, 'size': size, 'price': price, 'price_map': self.price_map[price] } self.price_map[price].append(order) self.order_map[order_id] = order def match(self, maker_order_id, match_size): order = self.order_map[maker_order_id] original_size = order['size'] new_size = original_size - match_size order['size'] = new_size def change(self, order_id, new_size): order = self.order_map[order_id] order['size'] = new_size def remove_order(self, order_id): if order_id in self.order_map: order = self.order_map[order_id] self.price_map[order['price']] = [ o for o in self.price_map[order['price']] if o['order_id'] != order_id ] if not self.price_map[order['price']]: self.remove_price(order['price']) del self.order_map[order_id] else: del self.received_orders[order_id]
class Tree(object): def __init__(self): self.priceTree = FastRBTree() self.volume = 0 self.priceMap = {} # Map from price -> orderList object self.orderMap = {} # Order ID to Order object def __len__(self): return len(self.orderMap) def getPrice(self, price): return self.priceMap[price] def getOrder(self, idNum): return self.orderMap[idNum] def createPrice(self, price): newList = OrderList() self.priceTree.insert(price, newList) self.priceMap[price] = newList def removePrice(self, price): self.priceTree.remove(price) del self.priceMap[price] def priceExists(self, price): return price in self.priceMap def orderExists(self, idNum): return idNum in self.orderMap def insertTick(self, tick): if tick.price not in self.priceMap: self.createPrice(tick.price) order = Order(tick, self.priceMap[tick.price]) self.priceMap[order.price].appendOrder(order) self.orderMap[order.idNum] = order self.volume += order.qty def updateOrder(self, tick): order = self.orderMap[tick.idNum] originalVolume = order.qty if tick.price != order.price: # Price changed orderList = self.priceMap[order.price] orderList.removeOrder(order) if len(orderList) == 0: removePrice(order.price) self.insertTick(tick) else: # Quantity changed order.updateQty(tick.qty,tick.price) self.volume += order.qty - originalVolume def removeOrderById(self, idNum): order = self.orderMap[idNum] self.volume -= order.qty order.orderList.removeOrder(order) if len(order.orderList) == 0: self.removePrice(order.price) del self.orderMap[idNum] def max(self): return min(self.priceTree) def min(self): return max(self.priceTree)
def GetLeastNumbers2(l,k): count = 0 tree = FastRBTree() for i in l: if count <k: tree.insert(i,i) count += 1 else: maxVal = max(tree) if i < maxVal: tree.remove(maxVal) tree.insert(i,i) datas = [item for item in tree] return datas
def _update(self, p, seeds: FastRBTree): # since eps is INF, the neighbours are all the points in the dataset. for o in range(self.N): if self.processed[o]: continue new_reach_dist = max(self.core_dist[p], self.dist_mat[o][p]) if np.isinf(self.reachability_dist[o]): self.reachability_dist[o] = new_reach_dist seeds.insert((new_reach_dist, o), o) self.predecessor[o] = p elif new_reach_dist < self.reachability_dist[o]: seeds.remove((self.reachability_dist[o], o)) self.reachability_dist[o] = new_reach_dist seeds.insert((new_reach_dist, o), o) self.predecessor[o] = p
class Tree(object): def __init__(self): self.price_tree = FastRBTree() self.price_map = {} self.order_map = {} self.received_orders = {} def receive(self, order_id, size): self.received_orders[order_id] = size def create_price(self, price): new_list = [] self.price_tree.insert(price, new_list) self.price_map[price] = new_list def remove_price(self, price): self.price_tree.remove(price) del self.price_map[price] def insert_order(self, order_id, size, price, initial=False): if not initial: del self.received_orders[order_id] if price not in self.price_map: self.create_price(price) order = {'order_id': order_id, 'size': size, 'price': price, 'price_map': self.price_map[price]} self.price_map[price].append(order) self.order_map[order_id] = order def match(self, maker_order_id, match_size): order = self.order_map[maker_order_id] original_size = order['size'] new_size = original_size - match_size order['size'] = new_size def change(self, order_id, new_size): order = self.order_map[order_id] order['size'] = new_size def remove_order(self, order_id): if order_id in self.order_map: order = self.order_map[order_id] self.price_map[order['price']] = [o for o in self.price_map[order['price']] if o['order_id'] != order_id] if not self.price_map[order['price']]: self.remove_price(order['price']) del self.order_map[order_id] else: del self.received_orders[order_id]
class Tree(object): def __init__(self): self.price_tree = FastRBTree() self.price_map = {} # Map from price -> order_list object self.order_map = {} # Order ID to Order object self.received_orders = {} def receive(self, order_id, size): self.received_orders[order_id] = size def create_price(self, price): new_list = OrderList() self.price_tree.insert(price, new_list) self.price_map[price] = new_list def remove_price(self, price): self.price_tree.remove(price) del self.price_map[price] def insert_order(self, order_id, size, price, initial=False): if not initial: del self.received_orders[order_id] if price not in self.price_map: self.create_price(price) order = Order(order_id, size, price, self.price_map[price]) self.price_map[order.price].append_order(order) self.order_map[order.order_id] = order def match(self, maker_order_id, size): order = self.order_map[maker_order_id] original_size = order.size new_size = original_size - size order.update_size(new_size) def change(self, order_id, new_size): order = self.order_map[order_id] order.update_size(new_size) def remove_order(self, order_id): if order_id in self.order_map: order = self.order_map[order_id] order.order_list.remove_order(order) if len(order.order_list) == 0: self.remove_price(order.price) del self.order_map[order_id] else: del self.received_orders[order_id]
class OrderTree(object): def __init__(self): self.price_tree = FastRBTree() self.price_map = {} self.min_price = None self.max_price = None def insert_price(self, price, amount): self.price_tree.insert(price, amount) self.price_map[price] = amount if self.max_price == None or price > self.max_price: self.max_price = price if self.min_price == None or price < self.min_price: self.min_price = price def update_price(self, price, amount): self.price_tree.insert(price, amount) #updates if key exists self.price_map[price] = amount def remove_price(self, price): self.price_tree.remove(price) del self.price_map[price] if self.max_price == price: try: self.max_price = max(self.price_tree) except ValueError: self.max_price = None if self.min_price == price: try: self.min_price = min(self.price_tree) except ValueError: self.min_price = None def price_exists(self, price): return price in self.price_map def max(self): return self.max_price def min(self): return self.min_price
class OrderTree(object): def __init__(self): self.price_tree = FastRBTree() self.min_price = None self.max_price = None def get_orders_at_price(self, price): return self.price_tree.get(price) def insert_price(self, price, amount, oid): ## ignore market order if price == Decimal(0.0): return prev_val = self.get_orders_at_price(price) if prev_val != None: ## price exists in local order book if oid in prev_val: ## update to an existing order at price prev_val['total'] = prev_val['total'] - prev_val[oid] + amount prev_val[oid] = amount else: ## new order at price prev_val['total'] += amount prev_val[oid] = amount self.price_tree.insert(price, prev_val) elif amount != 0.0: ## price did not exit in order book val = {'total': amount, oid: amount} self.price_tree.insert(price, val) try: val = self.price_tree.get(price) if val['total'] > 0: if self.max_price == None or price > self.max_price: self.max_price = price if self.min_price == None or price < self.min_price: self.min_price = price elif val['total'] == 0: ## price removed from orderbook self.remove_price(price) else: ## something has gone terribly wrong logging.error( "total amount at price %s went to negative amounts" % (price)) except: logging.error("price (%s) does not exist in orderbook" % (price)) def remove_price(self, price): self.price_tree.remove(price) if self.max_price == price: try: self.max_price = max(self.price_tree) except ValueError: self.max_price = None if self.min_price == price: try: self.min_price = min(self.price_tree) except ValueError: self.min_price = None
class LOBTree: def __init__(self): ''' Limit order book tree implementation using Red-Black tree for self-balancing Each limit price level is a OrderLinkedlist, and each order contains information including id, price, timestamp, volume self.limit_level: dict key: price level; value: OrderLinkedlist object self.order_ids: dict key: order id; value: Order object helps to locate order by id ''' # tree that store price as keys and number of orders on that level as values self.price_tree = FastRBTree() self.max_price = None self.min_price = None self.limit_levels = {} self.order_ids = {} @property def max(self): return self.max_price @property def min(self): return self.min_price def _get_price(self, price): ''' price: int :return: OrderLinkedlist instance ''' return self.limit_levels[price] def insert_order(self, order: Order): ''' order: Order Instance If order price doesn't exist in the self.limit_levels, insert it into the price level, else update accordingly; Will be used as limit order submission :return: None ''' if order.id in self.order_ids: raise ValueError('order already exists in the book') return if order.price not in self.limit_levels: new_price_level = OrderLinkedlist() self.price_tree[order.price] = 1 self.limit_levels[order.price] = new_price_level self.limit_levels[order.price].set_head(order) self.limit_levels[order.price].size += order.size self.order_ids[order.id] = order if self.max_price is None or order.price > self.max_price: self.max_price = order.price if self.min_price is None or order.price < self.min_price: self.min_price = order.price else: self.limit_levels[order.price].set_head(order) self.limit_levels[order.price].size += order.size self.order_ids[order.id] = order self.price_tree[order.price] += 1 def update_existing_order_size(self, order_id: int, updated_size: int): ''' order_id: int size: int Update an existing order's size in a price level and its price level's overall size :return: None ''' delta = self.order_ids[order_id].size - updated_size try: self.order_ids[order_id].size = updated_size order_price = self.order_ids[order_id].price # updated order will be put at the front of the list self.limit_levels[order_price].set_head(self.order_ids[order_id]) self.limit_levels[order_price].size -= delta except Exception as e: LOG.info('Order is not in the book') def remove_order(self, order_id: int): ''' order: Order Instance Remove the order from the self.order_ids first, then remove it from the self.limit_levels; if the limit_levels is empty after removal, adjust the max_price and min_price accordingly from the self.price_tree :return: Order Instance | order removed from the book ''' popped = self.order_ids.pop(order_id) self.limit_levels[popped.price].remove(popped, decrement=True) self.price_tree[popped.price] -= 1 if self.limit_levels[popped.price].size == 0: self._remove_price_level(popped.price) return popped def _remove_price_level(self, price: int): ''' order: Order Instance Given a price level, remove the price level in the price_tree and limit_levels reset the max and min prices ''' del self.limit_levels[price] self.price_tree.remove(price) if self.max_price == price: try: self.max_price = self.price_tree.max_key() except KeyError or ValueError: self.max_price = None if self.min_price == price: try: self.min_price = self.price_tree.min_key() except KeyError or ValueError: self.min_price = None def market_order(self, order: Order): ''' order: Order Instance ''' if len(self.limit_levels) == 0: raise ValueError('No orders in the book') return if order.is_bid: best_price = self.min_price while order.size > 0 and best_price != None: price_level = self._get_price(best_price) order.size, number_of_orders_deleted = price_level._consume_orders( order, self.order_ids) self.price_tree[best_price] -= number_of_orders_deleted if price_level._head == None: self._remove_price_level(best_price) best_price = self.min_price if order.size != 0: LOG.warning('no more limit orders in the bid book') else: best_price = self.max_price while order.size > 0 and best_price != None: price_level = self._get_price(best_price) order.size, number_of_orders_deleted = price_level._consume_orders( order, self.order_ids) self.price_tree[best_price] -= number_of_orders_deleted if price_level._head == None: self._remove_price_level(best_price) best_price = self.max_price if order.size != 0: LOG.warning('no more orders in the ask book') def level_with_most_orders(self, range: int): ''' range: int Gives the price level with the most orders on the top levels ''' pass def iceberg(self): ''' Iceberg order type ''' pass
class Tree(object): def __init__(self): self.ptree = FastRBTree() self.vol = 0 self.prmp = {} self.order_map = {} self.mip = None self.mxp = None def __len__(self): return len(self.order_map) def get_pri(self, pri): return self.prmp[pri] def get_order(self, id_num): return self.order_map[id_num] def create_pri(self, pri): new_list = OrderList() self.ptree.insert(pri, new_list) self.prmp[pri] = new_list if self.mxp == None or pri > self.mxp: self.mxp = pri if self.mip == None or pri < self.mip: self.mip = pri def remove_pri(self, pri): self.ptree.remove(pri) del self.prmp[pri] if self.mxp == pri: try: self.mxp = max(self.ptree) except ValueError: self.mxp = None if self.mip == pri: try: self.mip = min(self.ptree) except ValueError: self.mip = None def pri_exists(self, pri): return pri in self.prmp def order_exists(self, id_num): return id_num in self.order_map def insert_tick(self, tick): if tick.pri not in self.prmp: self.create_pri(tick.pri) order = Order(tick, self.prmp[tick.pri]) self.prmp[order.pri].append_order(order) self.order_map[order.id_num] = order self.vol += order.qty def update_order(self, tick): order = self.order_map[tick.id_num] original_vol = order.qty if tick.pri != order.pri: order_list = self.prmp[order.pri] order_list.remove_order(order) if len(order_list) == 0: self.remove_pri(order.pri) self.insert_tick(tick) self.vol -= original_vol else: order.update_qty(tick.qty, tick.pri) self.vol += order.qty - original_vol def remove_order_by_id(self, id_num): order = self.order_map[id_num] self.vol -= order.qty order.order_list.remove_order(order) if len(order.order_list) == 0: self.remove_pri(order.pri) del self.order_map[id_num] def max(self): return self.mxp def min(self): return self.mip
class TradeTree(object): '''A red-black tree used to store TradeLists in price trade The exchange will be using the TradeTree to hold bid and ask data (one TradeTree for each side). Keeping the information in a red black tree makes it easier/faster to detect a match. ''' def __init__(self): self.price_tree = FastRBTree() self.trade_map = {} self.num_trades = 0 # Contains count of Orders in tree self.depth = 0 # Number of different prices in tree (http://en.wikipedia.org/wiki/trade_book_(trading)#Book_depth) def __len__(self): return len(self.trade_map) def get_price_list(self, price): return self.price_tree.get(price, []) def get_trade(self, trade_id): return self.trade_map[trade_id] if trade_id in self.trade_map else None def create_price(self, price): self.depth += 1 # Add a price depth level to the tree new_list = LinkedList() self.price_tree.insert(price, new_list) # Insert a new price into the tree def remove_price(self, price): self.depth -= 1 # Remove a price depth level self.price_tree.remove(price) def price_exists(self, price): return self.price_tree.__contains__(price) def trade_exists(self, trade_id): return trade_id in self.trade_map def insert_trade(self, xtrade): if self.trade_exists(xtrade.id): return self.num_trades += 1 if not self.price_exists(xtrade.limit_price): self.create_price( xtrade.limit_price ) # If price not in Price Map, create a node in RBtree self.trade_map[ trade.id] = self.price_tree[xtrade.limit_price].append_item( xtrade ) # Add the trade to the TradeList in Price Map return the reference def remove_trade(self, xtrade): self.num_trades -= 1 trade_node = self.trade_map[trade.id] self.price_tree[trade.limit_price].remove_item(trade_node) if len(self.price_tree[trade.limit_price]) == 0: self.remove_price(trade.limit_price) self.trade_map.pop(trade.id, None) def max_price(self): if self.depth > 0: return self.price_tree.max_key() else: return None def min_price(self): if self.depth > 0: return self.price_tree.min_key() else: return None def max_price_list(self): if self.depth > 0: return self.get_price_list(self.max_price()) else: return None def min_price_list(self): if self.depth > 0: return self.get_price_list(self.min_price()) else: return None
class BookSide(object): ''' A side of the lmit order book representation ''' def __init__(self, s_side, fr_data, i_member=None): ''' Initialize a BookSide object. Save all parameters as attributes :param s_side: string. BID or ASK :param fr_data: ZipExtFile object. data to read :param i_member*: integer. Member number to be used as a filter ''' if s_side not in ['BID', 'ASK']: raise InvalidTypeException('side should be BID or ASK') self.i_member = i_member self.s_side = s_side self.price_tree = FastRBTree() self._i_idx = 0 self.fr_data = fr_data self.parser = parser_data.LineParser(s_side) self.d_order_map = {} self.last_price = 0. def how_many_rows_read(self): ''' Return the number of rows processed ''' return self._i_idx def update(self, d_data, s_last_ident): ''' Update the state of the order book given the data pased :param d_data: dict. data from the last row :param s_last_ident: string. last identification ''' # check if the information should be processed if s_last_ident != 'MSG': return False # check if should filter out member if not self._should_use_it(d_data): return False # update the book information order_aux = Order(d_data) s_status = order_aux['order_status'] b_sould_update = True # treat Bovespa files at the begining f the day if s_status != 'New': try: i_old_id = self.d_order_map[order_aux]['main_id'] except KeyError: if s_status == 'Canceled' or s_status == 'Filled': b_sould_update = False s_status = 'Invalid' elif s_status == 'Replaced': s_status = 'New' # process if s_status == 'New': b_sould_update = self._new_order(order_aux) elif s_status != 'Invalid': i_old_id = self.d_order_map[order_aux]['main_id'] f_old_pr = self.d_order_map[order_aux]['price'] i_old_q = self.d_order_map[order_aux]['qty'] # hold the last traded price if s_status in ['Partially Filled', 'Filled']: self.last_price = order_aux['order_price'] # process message if s_status in ['Canceled', 'Expired', 'Filled']: b_sould_update = self._canc_expr_filled_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Replaced': b_sould_update = self._replaced_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Partially Filled': b_sould_update = self._partially_filled(order_aux, i_old_id, f_old_pr, i_old_q) # remove from order map if s_status not in ['New', 'Invalid']: self.d_order_map.pop(order_aux) # update the order map if b_sould_update: f_qty = int(order_aux['total_qty_order']) self.d_order_map[order_aux] = {} self.d_order_map[order_aux]['price'] = d_data['order_price'] self.d_order_map[order_aux]['sec_order'] = order_aux.sec_order_id self.d_order_map[order_aux]['qty'] = f_qty self.d_order_map[order_aux]['main_id'] = order_aux.main_id # return that the update was done return True def _should_use_it(self, d_data): ''' Check if should use the passed row to update method :param d_data: dict. data from the last row ''' if self.i_member: if d_data['member'] != self.i_member: return False return True def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed canceled, expried or filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # remove from order map return False def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed replaced orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # remove from the old price this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # insert the order in the due price this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed partially filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # delete old price, if it is needed this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # add/modify order # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _new_order(self, order_obj): ''' Update price_tree when passed new orders :param order_obj: Order Object. The last order in the file ''' # if it was already in the order map if order_obj in self.d_order_map: i_old_sec_id = self.d_order_map[order_obj]['main_id'] f_old_price = self.d_order_map[order_obj]['price'] i_old_qty = self.d_order_map[order_obj]['qty'] this_price = self.price_tree.get(f_old_price) # remove from order map self.d_order_map.pop(order_obj) if this_price.delete(i_old_sec_id, i_old_qty): self.price_tree.remove(f_old_price) # insert a empty price level if it is needed f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # add the order this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def get_n_top_prices(self, n): ''' Return a dataframe with the N top price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def get_n_botton_prices(self, n=5): ''' Return a dataframe with the N botton price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def _readline(self): ''' Return a line from the fr_data file if available. Return false otherwiese ''' row = self.fr_data.readline() if row == '': self.fr_data.close() return False, False self._i_idx += 1 d_aux = self.parser(row) return d_aux, self.parser.last_identification def __iter__(self): ''' Return the self as an iterator object. Use next() to check the rows ''' return self def next(self): ''' Return the next item from the fr_data in iter process. If there are no further items, raise the StopIteration exception ''' d_rtn, last_identification = self._readline() if not d_rtn: raise StopIteration return d_rtn, last_identification
class OrderQueue(object): """ """ def __init__(self): self.Orders = FastRBTree() self.totalAmount = 0.0 def append(self, order): k = order.timestamp self.Orders.insert(k, order) self.totalAmount += self.Orders[k].amount def count(self): return self.Orders.count def min_item(self): return self.Orders.min_item() def eat(self, amount): """ use this only if amount <= self.amount return OrderToPop,RestAmount """ current_amount = amount to_pop_orders = [] while self.totalAmount > 0 and current_amount > 0: if not self.Orders or self.Orders.count == 0: #print "amount:%.4f" % self.totalAmount self.totalAmount = 0.0 return [] min_item = self.Orders.min_item() min_i = min_item[1] # 1.enough # 2.not enough if min_i.amount <= current_amount: current_amount -= min_i.amount self.totalAmount -= min_i.amount to_pop_orders.append(self.Orders.pop_min()[1]) continue # this is a hard case , need to take care of # first we split the order , take amount from current order # and make elif min_i.amount > current_amount: new_order = min_i.copy() new_order.amount = current_amount to_pop_orders.append(new_order) min_i.amount -= current_amount self.totalAmount -= current_amount current_amount = 0 break # return: # 1.orders # 2.amount to trade # 3.amount left return to_pop_orders def remove(self, order): k = order.timestamp self.totalAmount -= self.Orders[k].amount self.Orders.remove(k) def get_price_depth(self): return self.totalAmount def is_empty(self): return self.Orders.count == 0
class BidBook(object): """ A BidBook is used to store the order book's rates and amounts on the bid side with a defined depth. To maintain a sorted order of rates, the BidBook uses a red-black tree to store rates and corresponding amounts. For O(1) query of volume at a predetermined rate, the BidBook also uses a dictionary to store rate and amount. """ def __init__(self, max_depth, data): # RBTree: maintains sorted order of rates # every value inserted to RBTree must be a tuple, so we hard code the second value to be 0 self.rate_tree = FastRBTree() # dict: Uses rate and amount for key value pairs self.rate_dict = {} # float: amounts summed across all rate levels in tree self.volume = 0 # int: total number of rate levels in tree self.depth = len(data) # int: maximum number of rate levels in tree self.max_depth = max_depth # populate rate_tree and rate_dict from public API call data # set volume for level in data: rate = float(level[0]) amount = float(level[1]) self.rate_tree.insert(rate, 0) self.rate_dict[rate] = amount self.volume += amount def __len__(self): return len(self.rate_dict) def rate_exists(self, rate): return rate in self.rate_dict def get_amount_at_rate(self, rate): return self.rate_dict.get(rate) def max_rate_level(self): if self.depth > 0: rate = self.rate_tree.max_key() amount = self.get_amount_at_rate(rate) return rate, amount else: return None def min_rate_level(self): if self.depth > 0: rate = self.rate_tree.min_key() amount = self.get_amount_at_rate(rate) return rate, amount else: return None def modify(self, event): # if the event's rate is already in the book, just modify the amount at the event's rate rate = float(event[u'data'][u'rate']) amount = float(event[u'data'][u'amount']) if self.rate_exists(rate): # print '~~~~~~~~~~~~~~~~~~~~~~ BID MODIFY ~~~~~~~~~~~~~~~~~~~~~~' self.rate_dict[rate] = amount # only rates not already in the book reach this logic # if the max depth hasn't been reached, just insert the event's rate and amount elif self.depth < self.max_depth: # print '~~~~~~~~~~~~~~~~~~~~~~ BID MODIFY ~~~~~~~~~~~~~~~~~~~~~~' self.rate_tree.insert(rate, 0) self.rate_dict[rate] = amount self.depth += 1 # only events being handled by a full order tree reach this logic # if the event is a bid and the rate is greater than min rate, effectively replace min rate level with event else: min_rate = self.min_rate_level()[0] if rate > min_rate: # print '~~~~~~~~~~~~~~~~~~~~~~ BID MODIFY ~~~~~~~~~~~~~~~~~~~~~~' self.rate_tree.remove(min_rate) del self.rate_dict[min_rate] self.rate_tree.insert(rate, 0) self.rate_dict[rate] = amount def remove(self, event): # if the event's rate is in the book, delete it rate = float(event[u'data'][u'rate']) if self.rate_exists(rate): # print '~~~~~~~~~~~~~~~~~~~~~~ BID REMOVE ~~~~~~~~~~~~~~~~~~~~~~' self.rate_tree.remove(rate) del self.rate_dict[rate] self.depth -= 1 def __str__(self): rate_tree_str = '[' + ','.join(rate[0] for rate in self.rate_tree) + ']' return 'BIDS: ' + rate_tree_str
class BookSide(object): ''' A side of the lmit order book representation ''' def __init__(self, s_side, fr_data, i_member=None): ''' Initialize a BookSide object. Save all parameters as attributes :param s_side: string. BID or ASK :param fr_data: ZipExtFile object. data to read :param i_member*: integer. Member number to be used as a filter ''' if s_side not in ['BID', 'ASK']: raise InvalidTypeException('side should be BID or ASK') self.i_member = i_member self.s_side = s_side self.price_tree = FastRBTree() self._i_idx = 0 self.fr_data = fr_data self.parser = parser_data.LineParser(s_side) self.d_order_map = {} self.last_price = 0. # control other statistics self.best_queue = (None, None) self.i_qty_rel = 0 self.i_cum_rel = 0 def set_last_best_queue(self, t_best): ''' Set the best queue of this side ''' self.best_queue = t_best def update(self, d_data): ''' Update the state of the order book given the data pased :param d_data: dict. data from the last row ''' # update the book information order_aux = Order(d_data) s_status = order_aux['order_status'] b_sould_update = True i_rel_price = 0 # treat Bovespa files at the begining f the day if s_status != 'New': try: i_old_id = self.d_order_map[order_aux]['main_id'] except KeyError: # is not securing changes, also change part. filled status l_check = ['Canceled', 'Filled'] if not self.b_secure_changes: l_check = ['Canceled', 'Filled', 'Partially Filled'] # change order status when it is not found if s_status in l_check: b_sould_update = False s_status = 'Invalid' elif s_status == 'Replaced': s_status = 'New' # process if s_status == 'New': b_sould_update = self._new_order(order_aux) i_rel_price = get_relative_price(self.best_queue, order_aux) elif s_status != 'Invalid': i_old_id = self.d_order_map[order_aux]['main_id'] f_old_pr = self.d_order_map[order_aux]['price'] i_old_q = self.d_order_map[order_aux]['qty'] i_rel_price = self.d_order_map[order_aux]['relative_price'] # hold the last traded price if s_status in ['Partially Filled', 'Filled']: self.last_price = order_aux['order_price'] # process message if s_status in ['Canceled', 'Expired', 'Filled']: b_sould_update = self._canc_expr_filled_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Replaced': i_rel_price = get_relative_price(self.best_queue, order_aux) b_sould_update = self._replaced_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Partially Filled': b_sould_update = self._partially_filled(order_aux, i_old_id, f_old_pr, i_old_q) # remove from order map if s_status not in ['New', 'Invalid']: self.d_order_map.pop(order_aux) # update the order map if b_sould_update: f_qty = int(order_aux['total_qty_order']) f_prior_time = d_data['priority_seconds'] self.d_order_map[order_aux] = {} self.d_order_map[order_aux]['price'] = d_data['order_price'] self.d_order_map[order_aux]['sec_order'] = order_aux.sec_order_id self.d_order_map[order_aux]['qty'] = f_qty self.d_order_map[order_aux]['main_id'] = order_aux.main_id self.d_order_map[order_aux]['priority_seconds'] = f_prior_time self.d_order_map[order_aux]['relative_price'] = i_rel_price if s_status in ['New', 'Replaced']: self.i_qty_rel += f_qty * 1. self.i_cum_rel += i_rel_price * 1. * f_qty # return that the update was done return True def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed canceled, expried or filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' b_break = False this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # remove from order map if b_break: raise NotImplementedError return False def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed replaced orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # remove from the old price this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # insert the order in the due price this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed partially filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # delete old price, if it is needed this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # add/modify order # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _new_order(self, order_obj): ''' Update price_tree when passed new orders :param order_obj: Order Object. The last order in the file ''' # if it was already in the order map if order_obj in self.d_order_map: i_old_sec_id = self.d_order_map[order_obj]['main_id'] f_old_price = self.d_order_map[order_obj]['price'] i_old_qty = self.d_order_map[order_obj]['qty'] this_price = self.price_tree.get(f_old_price) # remove from order map self.d_order_map.pop(order_obj) if this_price.delete(i_old_sec_id, i_old_qty): self.price_tree.remove(f_old_price) # insert a empty price level if it is needed f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # add the order this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def get_n_top_prices(self, n): ''' Return a dataframe with the N top price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def get_n_botton_prices(self, n=5): ''' Return a dataframe with the N botton price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def _readline(self): ''' Return a line from the fr_data file if available. Return false otherwiese ''' row = self.fr_data.readline() if row == '': self.fr_data.close() return False, False self._i_idx += 1 d_aux = self.parser(row) # treat when the line is zero # TODO: I should move it to preprocessment step if 'order_price' in d_aux: if d_aux['order_price'] == 0.: while True: row = self.fr_data.readline() d_aux = self.parser(row) if d_aux['order_price'] != 0.: break return d_aux, self.parser.last_identification def __iter__(self): ''' Return the self as an iterator object. Use next() to check the rows ''' return self def next(self): ''' Return the next item from the fr_data in iter process. If there are no further items, raise the StopIteration exception ''' d_rtn, last_identification = self._readline() if not d_rtn: raise StopIteration return d_rtn, last_identification
class Tree(object): def __init__(self): self.priceTree = FastRBTree() self.volume = 0 self.priceMap = {} # Map from price -> orderList object self.orderMap = {} # Order ID to Order object def __len__(self): return len(self.orderMap) def getPrice(self, price): return self.priceMap[price] def getOrder(self, idNum): return self.orderMap[idNum] def createPrice(self, price): newList = OrderList() self.priceTree.insert(price, newList) self.priceMap[price] = newList def removePrice(self, price): self.priceTree.remove(price) del self.priceMap[price] def priceExists(self, price): return price in self.priceMap def orderExists(self, idNum): return idNum in self.orderMap def insertTick(self, tick): if tick.price not in self.priceMap: self.createPrice(tick.price) order = Order(tick, self.priceMap[tick.price]) self.priceMap[order.price].appendOrder(order) self.orderMap[order.idNum] = order self.volume += order.qty def updateOrder(self, tick): order = self.orderMap[tick.idNum] originalVolume = order.qty if tick.price != order.price: # Price changed orderList = self.priceMap[order.price] orderList.removeOrder(order) if len(orderList) == 0: self.removePrice(order.price) self.insertTick(tick) else: # Quantity changed order.updateQty(tick.qty, tick.price) self.volume += order.qty - originalVolume def removeOrderById(self, idNum): order = self.orderMap[idNum] self.volume -= order.qty order.orderList.removeOrder(order) if len(order.orderList) == 0: self.removePrice(order.price) del self.orderMap[idNum] def max(self): return min(self.priceTree) def min(self): return max(self.priceTree)
class BookSide(object): ''' A side of the lmit order book representation ''' def __init__(self, s_side): ''' Initialize a BookSide object. Save all parameters as attributes :param s_side: string. BID or ASK ''' if s_side not in ['BID', 'ASK']: raise InvalidTypeException('side should be BID or ASK') self.s_side = s_side self.price_tree = FastRBTree() self._i_idx = 0 self.d_order_map = {} self.last_price = 0. def update(self, d_data): ''' Update the state of the order book given the data pased. Return if the message was handle successfully :param d_data: dict. data related to a single order ''' # dont process aggresive trades if d_data['agressor_indicator'] == 'Agressive': return True # update the book information order_aux = Order(d_data) s_status = order_aux['order_status'] b_sould_update = True b_success = True # check the order status if s_status != 'New': try: i_old_id = self.d_order_map[order_aux]['main_id'] except KeyError: if s_status == 'Canceled' or s_status == 'Filled': b_sould_update = False s_status = 'Invalid' elif s_status == 'Replaced': s_status = 'New' # process the message if s_status == 'New': b_sould_update = self._new_order(order_aux) elif s_status != 'Invalid': i_old_id = self.d_order_map[order_aux]['main_id'] f_old_pr = self.d_order_map[order_aux]['price'] i_old_q = self.d_order_map[order_aux]['qty'] # hold the last traded price if s_status in ['Partially Filled', 'Filled']: self.last_price = order_aux['order_price'] # process message if s_status in ['Canceled', 'Expired', 'Filled']: b_sould_update = self._canc_expr_filled_order( order_aux, i_old_id, f_old_pr, i_old_q) if not b_sould_update: b_success = False elif s_status == 'Replaced': b_sould_update = self._replaced_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Partially Filled': b_sould_update = self._partially_filled( order_aux, i_old_id, f_old_pr, i_old_q) # remove from order map if s_status not in ['New', 'Invalid']: self.d_order_map.pop(order_aux) # update the order map if b_sould_update: f_qty = int(order_aux['total_qty_order']) self.d_order_map[order_aux] = {} self.d_order_map[order_aux]['price'] = d_data['order_price'] self.d_order_map[order_aux]['order_id'] = order_aux.order_id self.d_order_map[order_aux]['qty'] = f_qty self.d_order_map[order_aux]['main_id'] = order_aux.main_id # return that the update was done return True def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed canceled, expried or filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # remove from order map return False def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed replaced orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # remove from the old price this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # insert the order in the due price this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed partially filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # delete old price, if it is needed this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # add/modify order # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _new_order(self, order_obj): ''' Update price_tree when passed new orders :param order_obj: Order Object. The last order in the file ''' # if it was already in the order map if order_obj in self.d_order_map: i_old_sec_id = self.d_order_map[order_obj]['last_order_id'] f_old_price = self.d_order_map[order_obj]['price'] i_old_qty = self.d_order_map[order_obj]['qty'] this_price = self.price_tree.get(f_old_price) # remove from order map self.d_order_map.pop(order_obj) if this_price.delete(i_old_sec_id, i_old_qty): self.price_tree.remove(f_old_price) # insert a empty price level if it is needed f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # add the order this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def get_n_top_prices(self, n): ''' Return a dataframe with the N top price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def get_n_botton_prices(self, n=5): ''' Return a dataframe with the N botton price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError
class PriceTree(object): def __init__(self, name): self.tree = FastRBTree() self.name = name self.price_map = {} # Map price -> OrderList self.order_map = {} # Map order_id -> Order self.min_price = None self.max_price = None def insert_price(self, price): """ Add a new price TreedNode and associate it with an orderList :param price: :return: """ new_list = OrderList() self.tree.insert(price, new_list) self.price_map[price] = new_list if self.max_price is None or price > self.max_price: self.max_price = price if self.min_price is None or price < self.min_price: self.min_price = price def remove_price(self, price): """ Remove price from the tree structure and the associated orderList Update min and max prices if needed :param price: :return: """ self.tree.remove(price) # Order-map will still contain all Orders emptied (with size 0) # as we delete them on the List match_order which is fine for now for to_del_order in self.price_map[price]: del self.order_map[to_del_order.id] # Delete the price from the price-map del self.price_map[price] if self.max_price == price: try: self.max_price = self.tree.max_key() except ValueError: self.max_price = None if self.min_price == price: try: self.min_price = self.tree.min_key() except ValueError: self.min_price = None def insert_price_order(self, order): if order.price not in self.price_map: self.insert_price(order.price) # Add order to orderList self.price_map[order.price].add(order) # Also keep it in the order mapping self.order_map[order.id] = order def match_price_order(self, curr_order): if len(self.price_map) == 0: return [] # if bid -> sell_tree min # if ask -> buy_tree max best_price = self.min if curr_order.is_bid else self.max complete_trades = [] while ((curr_order.is_bid and curr_order.price >= best_price) or (not curr_order.is_bid and curr_order.price <= best_price)) \ and curr_order.peak_size > 0: # Get price OrderList matching_orders_list = self.get_price(best_price) complete_trades.extend( matching_orders_list.match_order(curr_order, self.order_map)) # Remove exhausted price if matching_orders_list.size == 0: self.remove_price(best_price) if len(self.price_map) == 0: break # Try to find more price matches using the next price best_price = self.min if curr_order.is_bid else self.max return complete_trades def price_exists(self, price): return price in self.price_map def order_exists(self, id_num): return id_num in self.order_map def get_price(self, price): return self.price_map[price] def get_order(self, id_num): return self.order_map[id_num] @property def max(self): return self.max_price @property def min(self): return self.min_price
class PriceLevel(object): ''' A representation of a Price level in the book ''' def __init__(self, f_price): ''' A representation of a PriceLevel object ''' self.f_price = f_price self.i_qty = 0 self.order_tree = FastRBTree() def add(self, order_aux): ''' Insert the information in the tree using the info in order_aux. Return is should delete the Price level or not :param order_aux: Order Object. The Order message to be updated ''' # check if the order_aux price is the same of the self s_status = order_aux['order_status'] if order_aux['order_price'] != self.f_price: raise DifferentPriceException elif s_status in ['New', 'Replaced', 'Partially Filled']: self.order_tree.insert(order_aux.main_id, order_aux) self.i_qty += int(order_aux['total_qty_order']) # check if there is no object in the updated tree (should be deleted) return self.order_tree.count == 0 def delete(self, i_last_id, i_old_qty): ''' Delete the information in the tree using the info in order_aux. Return is should delete the Price level or not :param i_last_id: Integer. The previous secondary order id :param i_old_qty: Integer. The previous order qty ''' # check if the order_aux price is the same of the self try: self.order_tree.remove(i_last_id) self.i_qty -= i_old_qty except KeyError: raise DifferentPriceException # check if there is no object in the updated tree (should be deleted) return self.order_tree.count == 0 def __str__(self): ''' Return the name of the PriceLevel ''' return '{:,.0f}'.format(self.i_qty) def __repr__(self): ''' Return the name of the PriceLevel ''' return '{:,.0f}'.format(self.i_qty) def __eq__(self, other): ''' Return if a PriceLevel has equal price from the other :param other: PriceLevel object. PriceLevel to be compared ''' # just to make sure that there is no floating point discrepance f_aux = other if not isinstance(other, float): f_aux = other.f_price return abs(self.f_price - f_aux) < 1e-4 def __gt__(self, other): ''' Return if a PriceLevel has a gerater price from the other. Bintrees uses that to compare nodes :param other: PriceLevel object. PriceLevel to be compared ''' # just to make sure that there is no floating point discrepance f_aux = other if not isinstance(other, float): f_aux = other.f_price return (f_aux - self.f_price) > 1e-4 def __lt__(self, other): ''' Return if a Order has smaller order_id from the other. Bintrees uses that to compare nodes :param other: Order object. Order to be compared ''' f_aux = other if not isinstance(other, float): f_aux = other.f_price return (f_aux - self.f_price) < -1e-4 def __ne__(self, other): ''' Return if a Order has different order_id from the other :param other: Order object. Order to be compared ''' return not self.__eq__(other)
class PriceLevel(object): ''' A representation of a Price level in the book ''' def __init__(self, f_price): ''' A representation of a PriceLevel object ''' self.f_price = f_price self.i_qty = 0 self.order_tree = FastRBTree() def add(self, order_aux): ''' Insert the information in the tree using the info in order_aux. Return is should delete the Price level or not :param order_aux: Order Object. The Order message to be updated :param i_old_sec_order: Integer. The previous secondary order id :param i_old_qty: Integer. The previous order qty ''' # check if the order_aux price is the same of the self s_status = order_aux['order_status'] if order_aux['order_price'] != self.f_price: raise DifferentPriceException elif s_status in ['New', 'Replaced', 'Partially Filled']: self.order_tree.insert(order_aux.main_id, order_aux) self.i_qty += int(order_aux['total_qty_order']) # check if there is no object in the updated tree (should be deleted) return self.order_tree.count == 0 def delete(self, i_old_sec_order, i_old_qty): ''' Delete the information in the tree using the info in order_aux. Return is should delete the Price level or not :param order_aux: Order Object. The Order message to be updated :param i_old_sec_order: Integer. The previous secondary order id :param i_old_qty: Integer. The previous order qty ''' # check if the order_aux price is the same of the self try: self.order_tree.remove(i_old_sec_order) self.i_qty -= i_old_qty except KeyError: raise DifferentPriceException # check if there is no object in the updated tree (should be deleted) return self.order_tree.count == 0 def __str__(self): ''' Return the name of the PriceLevel ''' return '{:,.0f}'.format(self.i_qty) def __repr__(self): ''' Return the name of the PriceLevel ''' return '{:,.0f}'.format(self.i_qty) def __eq__(self, other): ''' Return if a PriceLevel has equal price from the other :param other: PriceLevel object. PriceLevel to be compared ''' # just to make sure that there is no floating point discrepance f_aux = other if not isinstance(other, float): f_aux = other.f_price return abs(self.f_price - f_aux) < 1e-4 def __gt__(self, other): ''' Return if a PriceLevel has a gerater price from the other. Bintrees uses that to compare nodes :param other: PriceLevel object. PriceLevel to be compared ''' # just to make sure that there is no floating point discrepance f_aux = other if not isinstance(other, float): f_aux = other.f_price return (f_aux - self.f_price) > 1e-4 def __lt__(self, other): ''' Return if a Order has smaller order_id from the other. Bintrees uses that to compare nodes :param other: Order object. Order to be compared ''' f_aux = other if not isinstance(other, float): f_aux = other.f_price return (f_aux - self.f_price) < -1e-4 def __ne__(self, other): ''' Return if a Order has different order_id from the other :param other: Order object. Order to be compared ''' return not self.__eq__(other)
class BookSide(object): ''' A side of the lmit order book representation ''' def __init__(self, s_side): ''' Initialize a BookSide object. Save all parameters as attributes :param s_side: string. BID or ASK ''' if s_side not in ['BID', 'ASK']: raise InvalidTypeException('side should be BID or ASK') self.s_side = s_side self.price_tree = FastRBTree() self._i_idx = 0 self.d_order_map = {} self.last_price = 0. def update(self, d_data): ''' Update the state of the order book given the data pased. Return if the message was handle successfully :param d_data: dict. data related to a single order ''' # dont process aggresive trades if d_data['agressor_indicator'] == 'Agressive': return True # update the book information order_aux = Order(d_data) s_status = order_aux['order_status'] b_sould_update = True b_success = True # check the order status if s_status != 'New': try: i_old_id = self.d_order_map[order_aux]['main_id'] except KeyError: if s_status == 'Canceled' or s_status == 'Filled': b_sould_update = False s_status = 'Invalid' elif s_status == 'Replaced': s_status = 'New' # process the message if s_status == 'New': b_sould_update = self._new_order(order_aux) elif s_status != 'Invalid': i_old_id = self.d_order_map[order_aux]['main_id'] f_old_pr = self.d_order_map[order_aux]['price'] i_old_q = self.d_order_map[order_aux]['qty'] # hold the last traded price if s_status in ['Partially Filled', 'Filled']: self.last_price = order_aux['order_price'] # process message if s_status in ['Canceled', 'Expired', 'Filled']: b_sould_update = self._canc_expr_filled_order(order_aux, i_old_id, f_old_pr, i_old_q) if not b_sould_update: b_success = False elif s_status == 'Replaced': b_sould_update = self._replaced_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Partially Filled': b_sould_update = self._partially_filled(order_aux, i_old_id, f_old_pr, i_old_q) # remove from order map if s_status not in ['New', 'Invalid']: self.d_order_map.pop(order_aux) # update the order map if b_sould_update: f_qty = int(order_aux['total_qty_order']) self.d_order_map[order_aux] = {} self.d_order_map[order_aux]['price'] = d_data['order_price'] self.d_order_map[order_aux]['order_id'] = order_aux.order_id self.d_order_map[order_aux]['qty'] = f_qty self.d_order_map[order_aux]['main_id'] = order_aux.main_id # return that the update was done return True def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed canceled, expried or filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # remove from order map return False def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed replaced orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # remove from the old price this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # insert the order in the due price this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed partially filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # delete old price, if it is needed this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # add/modify order # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _new_order(self, order_obj): ''' Update price_tree when passed new orders :param order_obj: Order Object. The last order in the file ''' # if it was already in the order map if order_obj in self.d_order_map: i_old_sec_id = self.d_order_map[order_obj]['last_order_id'] f_old_price = self.d_order_map[order_obj]['price'] i_old_qty = self.d_order_map[order_obj]['qty'] this_price = self.price_tree.get(f_old_price) # remove from order map self.d_order_map.pop(order_obj) if this_price.delete(i_old_sec_id, i_old_qty): self.price_tree.remove(f_old_price) # insert a empty price level if it is needed f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # add the order this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def get_n_top_prices(self, n): ''' Return a dataframe with the N top price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def get_n_botton_prices(self, n=5): ''' Return a dataframe with the N botton price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError
class BookSide(object): ''' A side of the lmit order book representation ''' def __init__(self, s_side, fr_data, i_member=None): ''' Initialize a BookSide object. Save all parameters as attributes :param s_side: string. BID or ASK :param fr_data: ZipExtFile object. data to read :param i_member*: integer. Member number to be used as a filter ''' if s_side not in ['BID', 'ASK']: raise InvalidTypeException('side should be BID or ASK') self.i_member = i_member self.s_side = s_side self.price_tree = FastRBTree() self._i_idx = 0 self.fr_data = fr_data self.parser = parser_data.LineParser(s_side) self.d_order_map = {} self.last_price = 0. def how_many_rows_read(self): ''' Return the number of rows processed ''' return self._i_idx def update(self, d_data, s_last_ident): ''' Update the state of the order book given the data pased :param d_data: dict. data from the last row :param s_last_ident: string. last identification ''' # check if the information should be processed if s_last_ident != 'MSG': return False # check if should filter out member if not self._should_use_it(d_data): return False # update the book information order_aux = Order(d_data) s_status = order_aux['order_status'] b_sould_update = True # treat Bovespa files at the begining f the day if s_status != 'New': try: i_old_id = self.d_order_map[order_aux]['main_id'] except KeyError: if s_status == 'Canceled' or s_status == 'Filled': b_sould_update = False s_status = 'Invalid' elif s_status == 'Replaced': s_status = 'New' # process if s_status == 'New': b_sould_update = self._new_order(order_aux) elif s_status != 'Invalid': i_old_id = self.d_order_map[order_aux]['main_id'] f_old_pr = self.d_order_map[order_aux]['price'] i_old_q = self.d_order_map[order_aux]['qty'] # hold the last traded price if s_status in ['Partially Filled', 'Filled']: self.last_price = order_aux['order_price'] # process message if s_status in ['Canceled', 'Expired', 'Filled']: b_sould_update = self._canc_expr_filled_order( order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Replaced': b_sould_update = self._replaced_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Partially Filled': b_sould_update = self._partially_filled( order_aux, i_old_id, f_old_pr, i_old_q) # remove from order map if s_status not in ['New', 'Invalid']: self.d_order_map.pop(order_aux) # update the order map if b_sould_update: f_qty = int(order_aux['total_qty_order']) self.d_order_map[order_aux] = {} self.d_order_map[order_aux]['price'] = d_data['order_price'] self.d_order_map[order_aux]['sec_order'] = order_aux.sec_order_id self.d_order_map[order_aux]['qty'] = f_qty self.d_order_map[order_aux]['main_id'] = order_aux.main_id # return that the update was done return True def _should_use_it(self, d_data): ''' Check if should use the passed row to update method :param d_data: dict. data from the last row ''' if self.i_member: if d_data['member'] != self.i_member: return False return True def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed canceled, expried or filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # remove from order map return False def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed replaced orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # remove from the old price this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # insert the order in the due price this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed partially filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # delete old price, if it is needed this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # add/modify order # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _new_order(self, order_obj): ''' Update price_tree when passed new orders :param order_obj: Order Object. The last order in the file ''' # if it was already in the order map if order_obj in self.d_order_map: i_old_sec_id = self.d_order_map[order_obj]['main_id'] f_old_price = self.d_order_map[order_obj]['price'] i_old_qty = self.d_order_map[order_obj]['qty'] this_price = self.price_tree.get(f_old_price) # remove from order map self.d_order_map.pop(order_obj) if this_price.delete(i_old_sec_id, i_old_qty): self.price_tree.remove(f_old_price) # insert a empty price level if it is needed f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # add the order this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def get_n_top_prices(self, n): ''' Return a dataframe with the N top price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def get_n_botton_prices(self, n=5): ''' Return a dataframe with the N botton price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def _readline(self): ''' Return a line from the fr_data file if available. Return false otherwiese ''' row = self.fr_data.readline() if row == '': self.fr_data.close() return False, False self._i_idx += 1 d_aux = self.parser(row) return d_aux, self.parser.last_identification def __iter__(self): ''' Return the self as an iterator object. Use next() to check the rows ''' return self def next(self): ''' Return the next item from the fr_data in iter process. If there are no further items, raise the StopIteration exception ''' d_rtn, last_identification = self._readline() if not d_rtn: raise StopIteration return d_rtn, last_identification
class Tree(object): def __init__(self): self.price_tree = FastRBTree() self.volume = 0 self.price_map = {} # Map from price -> order_list object self.order_map = {} # Order ID to Order object self.min_price = None self.max_price = None def __len__(self): return len(self.order_map) def get_price(self, price): return self.price_map[price] def get_order(self, id_num): return self.order_map[id_num] def create_price(self, price): new_list = OrderList() self.price_tree.insert(price, new_list) self.price_map[price] = new_list if self.max_price == None or price > self.max_price: self.max_price = price if self.min_price == None or price < self.min_price: self.min_price = price def remove_price(self, price): self.price_tree.remove(price) del self.price_map[price] if self.max_price == price: try: self.max_price = max(self.price_tree) except ValueError: self.max_price = None if self.min_price == price: try: self.min_price = min(self.price_tree) except ValueError: self.min_price = None def price_exists(self, price): return price in self.price_map def order_exists(self, id_num): return id_num in self.order_map def insert_tick(self, tick): if tick.price not in self.price_map: self.create_price(tick.price) order = Order(tick, self.price_map[tick.price]) self.price_map[order.price].append_order(order) self.order_map[order.id_num] = order self.volume += order.qty def update_order(self, tick): order = self.order_map[tick.id_num] original_volume = order.qty if tick.price != order.price: # Price changed order_list = self.price_map[order.price] order_list.remove_order(order) if len(order_list) == 0: self.remove_price(order.price) self.insert_tick(tick) self.volume -= original_volume else: # Quantity changed order.update_qty(tick.qty, tick.price) self.volume += order.qty - original_volume def remove_order_by_id(self, id_num): order = self.order_map[id_num] self.volume -= order.qty order.order_list.remove_order(order) if len(order.order_list) == 0: self.remove_price(order.price) del self.order_map[id_num] def max(self): return self.max_price def min(self): return self.min_price
class OrderQueue(object): """ """ def __init__(self): self.Orders = FastRBTree() self.totalAmount = 0.0 def append(self,order): k = order.timestamp self.Orders.insert(k,order) self.totalAmount += self.Orders[k].amount def count(self): return self.Orders.count def min_item(self): return self.Orders.min_item() def eat(self,amount): """ use this only if amount <= self.amount return OrderToPop,RestAmount """ current_amount = amount to_pop_orders = [] while self.totalAmount>0 and current_amount>0: if not self.Orders or self.Orders.count==0: #print "amount:%.4f" % self.totalAmount self.totalAmount = 0.0 return [] min_item = self.Orders.min_item() min_i = min_item[1] # 1.enough # 2.not enough if min_i.amount <= current_amount: current_amount -= min_i.amount self.totalAmount -= min_i.amount to_pop_orders.append(self.Orders.pop_min()[1]) continue # this is a hard case , need to take care of # first we split the order , take amount from current order # and make elif min_i.amount > current_amount: new_order = min_i.copy() new_order.amount = current_amount to_pop_orders.append(new_order) min_i.amount -= current_amount self.totalAmount -= current_amount current_amount = 0 break # return: # 1.orders # 2.amount to trade # 3.amount left return to_pop_orders def remove(self,order): k = order.timestamp self.totalAmount -= self.Orders[k].amount self.Orders.remove(k) def get_price_depth(self): return self.totalAmount def is_empty(self): return self.Orders.count == 0