def test_add_symbol(): ST = SymbolTable("") T = FastRBTree() Content = {'Type': "int" , 'Attribute': None , 'TokenLocation': (10,2) } ST.InsertSymbol("age", Content) T.insert("age", Content) Content = {'Type': "float" , 'Attribute': 'static' , 'TokenLocation': (11,2) } ST.InsertSymbol("temperature", Content) T.insert("temperature", Content) Content = {'Type': "char" , 'Attribute': 'const' , 'TokenLocation': (12,2) } ST.InsertSymbol("letter", Content) T.insert("letter", Content) keys = ST.TopScope.keys() for key in keys: assert(T.__contains__(key)) assert(T.get(key) == ST.TopScope.get(key)) assert(T.get(key) is not ST.TopScope.get(key)) #write test to prove that the returned item is a pointer return
class Node: def __init__(self, name): self.name = name self.children = {} self.visited = False self.ancestors = FastRBTree() def updateChildren(self, child, nameOfSequence): if child in self.children: self.children[child].add(nameOfSequence) else: self.children[child] = set() self.children[child].add(nameOfSequence) def getChild(self, nameOfChild): return filter(lambda x: x == nameOfChild, self.children.keys()) def updateAncestors(self, ancestors, nameOfSequence): for ancestor in ancestors: if self.name == ancestor.name: continue elementFromTree = self.ancestors.get(ancestor.name, None) if elementFromTree != None: elementFromTree.add(nameOfSequence) self.ancestors.__setitem__(ancestor.name, elementFromTree) else: # NIE JE V SEK tmpValue = set() tmpValue.add(nameOfSequence) self.ancestors.insert(ancestor.name, tmpValue)
class Hyperedge: def __init__(self, hyperkey, col, hlabel): self.hyperkey = hyperkey self.col = col self._alerts = Tree() self.insert_alert(hlabel, 1) self.nalerts = 1 def get_alert(self, key): return self._alerts.get(key) def insert_alert(self, alert_key, count): self._alerts.insert(alert_key, count) def foreach_alert(self, func): self._alerts.foreach(func) def pop_alert(self, key): return self._alerts.pop(key)
from bintrees import FastRBTree T = int(sys.stdin.readline().strip()) def split(n): n -= 1 return n // 2, n // 2 + (n % 2) assert (0, 0) == split(1) assert (0, 1) == split(2) assert (1, 1) == split(3) for t in range(1, T + 1): N, K = map(int, sys.stdin.readline().strip().split()) holes = FastRBTree({N: 1}) while K > 0: size, count = holes.max_item() del holes[size] for s in split(size): holes[s] = holes.get(s, 0) + count K -= count minD, maxD = split(size) print("Case #%d: %d %d" % (t, maxD, minD))
class BookSide(object): ''' A side of the lmit order book representation ''' def __init__(self, s_side): ''' Initialize a BookSide object. Save all parameters as attributes :param s_side: string. BID or ASK ''' if s_side not in ['BID', 'ASK']: raise InvalidTypeException('side should be BID or ASK') self.s_side = s_side self.price_tree = FastRBTree() self._i_idx = 0 self.d_order_map = {} self.last_price = 0. def update(self, d_data): ''' Update the state of the order book given the data pased. Return if the message was handle successfully :param d_data: dict. data related to a single order ''' # dont process aggresive trades if d_data['agressor_indicator'] == 'Agressive': return True # update the book information order_aux = Order(d_data) s_status = order_aux['order_status'] b_sould_update = True b_success = True # check the order status if s_status != 'New': try: i_old_id = self.d_order_map[order_aux]['main_id'] except KeyError: if s_status == 'Canceled' or s_status == 'Filled': b_sould_update = False s_status = 'Invalid' elif s_status == 'Replaced': s_status = 'New' # process the message if s_status == 'New': b_sould_update = self._new_order(order_aux) elif s_status != 'Invalid': i_old_id = self.d_order_map[order_aux]['main_id'] f_old_pr = self.d_order_map[order_aux]['price'] i_old_q = self.d_order_map[order_aux]['qty'] # hold the last traded price if s_status in ['Partially Filled', 'Filled']: self.last_price = order_aux['order_price'] # process message if s_status in ['Canceled', 'Expired', 'Filled']: b_sould_update = self._canc_expr_filled_order(order_aux, i_old_id, f_old_pr, i_old_q) if not b_sould_update: b_success = False elif s_status == 'Replaced': b_sould_update = self._replaced_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Partially Filled': b_sould_update = self._partially_filled(order_aux, i_old_id, f_old_pr, i_old_q) # remove from order map if s_status not in ['New', 'Invalid']: self.d_order_map.pop(order_aux) # update the order map if b_sould_update: f_qty = int(order_aux['total_qty_order']) self.d_order_map[order_aux] = {} self.d_order_map[order_aux]['price'] = d_data['order_price'] self.d_order_map[order_aux]['order_id'] = order_aux.order_id self.d_order_map[order_aux]['qty'] = f_qty self.d_order_map[order_aux]['main_id'] = order_aux.main_id # return that the update was done return True def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed canceled, expried or filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # remove from order map return False def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed replaced orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # remove from the old price this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # insert the order in the due price this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed partially filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # delete old price, if it is needed this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # add/modify order # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _new_order(self, order_obj): ''' Update price_tree when passed new orders :param order_obj: Order Object. The last order in the file ''' # if it was already in the order map if order_obj in self.d_order_map: i_old_sec_id = self.d_order_map[order_obj]['last_order_id'] f_old_price = self.d_order_map[order_obj]['price'] i_old_qty = self.d_order_map[order_obj]['qty'] this_price = self.price_tree.get(f_old_price) # remove from order map self.d_order_map.pop(order_obj) if this_price.delete(i_old_sec_id, i_old_qty): self.price_tree.remove(f_old_price) # insert a empty price level if it is needed f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # add the order this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def get_n_top_prices(self, n): ''' Return a dataframe with the N top price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def get_n_botton_prices(self, n=5): ''' Return a dataframe with the N botton price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError
class SymbolTable(): #constructor def __init__(self, SourceFile): self.Table = [ ] #declare table as a stack (list) containing an empty tree self.TopScope = FastRBTree( ) # a place where the current top scope is held self.ReadMode = False #Read or lookup mode self.DebugMode = False self.SourceFile = SourceFile #Function: InsertSymbol #Desc: Insert a symbol into the current top of the symbol table # The symbol key string is a lexeme #Content_dict: At present this will contain # {Type: (the token adjacent to the SymbolKey i.e. >int< <SymbolKey>), # Attribute: some modifier on the symbol 'static' 'const' etc. # TokenLocation: tuple(line, char_in_file, char_in_line) } def InsertSymbol(self, SymbolKey_str, Content_dict): try: if self.DebugMode == True: print("Insert symbol is called: ", "In Read Mode?", self.ReadMode, SymbolKey_str, Content_dict) if self.ReadMode == False: found = self.FindSymbolInCurrentScope(SymbolKey_str) if not found: found = self.FindSymbolInTable(SymbolKey_str) if found: for item in found: for key in list(item.keys()): self.PrettyErrorPrint( "Warning: {0} on line {3} is a shadowded variable. Previous declaration in scope level {1} at line {2}." .format(SymbolKey_str, abs(key - len(self.Table)), item[key]["TokenLocation"][0], Content_dict['TokenLocation'][0]), item[key]["TokenLocation"][0], item[key]["TokenLocation"][2]) #perform deepcopy on passed in dictionary self.TopScope.insert(SymbolKey_str, deepcopy(Content_dict)) else: self.PrettyErrorPrint( '''Error: Redeclaration of existing variable.\nPrior declaration is here at line {0}: \n''' .format(found["TokenLocation"][0]), found["TokenLocation"][0], found["TokenLocation"][2]) raise Exception( 'Redeclaration of exisitng variable in current scope.') else: # do nothing pass except Exception as e: raise e return True #Function: FindSymbolInTable #Desc: Search all scopes of the symbol table to find a specific symbol #Return: [{Level_int: Content_dict}, False] (list of keys for possibility of many shadowed vars) def FindSymbolInTable(self, SymbolKey_str): T_list = [] Level_int = 0 #search the top of our stack if self.FindSymbolInCurrentScope(SymbolKey_str): T_list.append(self.FindSymbolInCurrentScope(SymbolKey_str)) Level_int += 1 #iterate over all trees #add found isntances of symbols to list if present for Tree in reversed( self.Table): #reversed so appended items are at the front if Tree.__contains__(SymbolKey_str): T_list.append({Level_int: Tree.get(SymbolKey_str)}) Level_int += 1 if len(T_list) > 0: return T_list #nothing found case return False #Function: FindSymbolInCurrentScope #Desc: Search only the top level of the symbol table for key def FindSymbolInCurrentScope(self, SymbolKey_str): return self.TopScope.get(SymbolKey_str, False) #Function:PushNewScope #Desc: Create a new scope and push it onto the table def PushNewScope(self): self.PushScope(FastRBTree()) return #Function: PushScope #Desc: Insert symbol tree (RBTree) onto our table def PushScope(self, SymbolTree): self.Table.append(self.TopScope) self.TopScope = SymbolTree return #Function: PopScope #Desc: Remove and return scope (RBtree) from the symbol table def PopScope(self): TScope = self.TopScope if len(self.Table) > 0: self.TopScope = self.Table.pop() else: self.TopScope = None return TScope #Function: WriteSymbolTableToFile #Desc: Write the current contents of the symbol table to file def WriteSymbolTableToFile(self, FileName_str): T_Stack = [] i = 0 try: with open(FileName_str, "w") as File: File.write( "\n**** Outputting Contents of Symbol Table **** \n\n") while not self.TableIsEmpty(): File.write("Items in Tree at Level {}: \n".format(i)) self.PrettyPrintScope(self.TopScope, FilePtr=File) T_Stack.append(self.PopScope()) i += 1 while len(T_Stack) > 0: self.PushScope(T_Stack.pop()) except Exception as e: raise def PrettyPrintScope(self, Scope, FilePtr=None): for Key in Scope.keys(): if FilePtr is not None: FilePtr.write("\tKey: \"{}\" | Content: {}\n".format( Key, Scope.get(Key))) else: print("\tKey: \"{}\" | Content: {}\n".format( Key, Scope.get(Key))) return def ToggleDebugMode(self): self.DebugMode = not self.DebugMode return def ReadModeOn(self): self.ReadMode = True # print("Insert Mode Toggled Off") return def InsertMode(self): self.ReadMode = False # print("Insert Mode Toggled On") return def ToggleReadMode(self): self.ReadMode = not self.ReadMode if self.ReadMode == False: print("Insert Mode Toggled On") elif self.ReadMode == True: print("Insert Mode Toggled Off") return def TableIsEmpty(self): if self.TopScope is None: return True return False def PrettyErrorPrint(self, Message, Lineno, Column): arrow = "" print(Message) #print line with open(self.SourceFile) as file: for i in range(0, Lineno): source = file.readline() print(source) #build arrow for i in range(0, Column - 1): arrow += " " arrow += "^\n" print(arrow)
class OrderTree(object): def __init__(self): self.price_tree = FastRBTree() self.min_price = None self.max_price = None def get_orders_at_price(self, price): return self.price_tree.get(price) def insert_price(self, price, amount, oid): ## ignore market order if price == Decimal(0.0): return prev_val = self.get_orders_at_price(price) if prev_val != None: ## price exists in local order book if oid in prev_val: ## update to an existing order at price prev_val['total'] = prev_val['total'] - prev_val[oid] + amount prev_val[oid] = amount else: ## new order at price prev_val['total'] += amount prev_val[oid] = amount self.price_tree.insert(price, prev_val) elif amount != 0.0: ## price did not exit in order book val = {'total': amount, oid: amount} self.price_tree.insert(price, val) try: val = self.price_tree.get(price) if val['total'] > 0: if self.max_price == None or price > self.max_price: self.max_price = price if self.min_price == None or price < self.min_price: self.min_price = price elif val['total'] == 0: ## price removed from orderbook self.remove_price(price) else: ## something has gone terribly wrong logging.error( "total amount at price %s went to negative amounts" % (price)) except: logging.error("price (%s) does not exist in orderbook" % (price)) def remove_price(self, price): self.price_tree.remove(price) if self.max_price == price: try: self.max_price = max(self.price_tree) except ValueError: self.max_price = None if self.min_price == price: try: self.min_price = min(self.price_tree) except ValueError: self.min_price = None
class TradeTree(object): '''A red-black tree used to store TradeLists in price trade The exchange will be using the TradeTree to hold bid and ask data (one TradeTree for each side). Keeping the information in a red black tree makes it easier/faster to detect a match. ''' def __init__(self): self.price_tree = FastRBTree() self.trade_map = {} self.num_trades = 0 # Contains count of Orders in tree self.depth = 0 # Number of different prices in tree (http://en.wikipedia.org/wiki/trade_book_(trading)#Book_depth) def __len__(self): return len(self.trade_map) def get_price_list(self, price): return self.price_tree.get(price, []) def get_trade(self, trade_id): return self.trade_map[trade_id] if trade_id in self.trade_map else None def create_price(self, price): self.depth += 1 # Add a price depth level to the tree new_list = LinkedList() self.price_tree.insert(price, new_list) # Insert a new price into the tree def remove_price(self, price): self.depth -= 1 # Remove a price depth level self.price_tree.remove(price) def price_exists(self, price): return self.price_tree.__contains__(price) def trade_exists(self, trade_id): return trade_id in self.trade_map def insert_trade(self, xtrade): if self.trade_exists(xtrade.id): return self.num_trades += 1 if not self.price_exists(xtrade.limit_price): self.create_price( xtrade.limit_price ) # If price not in Price Map, create a node in RBtree self.trade_map[ trade.id] = self.price_tree[xtrade.limit_price].append_item( xtrade ) # Add the trade to the TradeList in Price Map return the reference def remove_trade(self, xtrade): self.num_trades -= 1 trade_node = self.trade_map[trade.id] self.price_tree[trade.limit_price].remove_item(trade_node) if len(self.price_tree[trade.limit_price]) == 0: self.remove_price(trade.limit_price) self.trade_map.pop(trade.id, None) def max_price(self): if self.depth > 0: return self.price_tree.max_key() else: return None def min_price(self): if self.depth > 0: return self.price_tree.min_key() else: return None def max_price_list(self): if self.depth > 0: return self.get_price_list(self.max_price()) else: return None def min_price_list(self): if self.depth > 0: return self.get_price_list(self.min_price()) else: return None
class BookSide(object): ''' A side of the lmit order book representation ''' def __init__(self, s_side, fr_data, i_member=None): ''' Initialize a BookSide object. Save all parameters as attributes :param s_side: string. BID or ASK :param fr_data: ZipExtFile object. data to read :param i_member*: integer. Member number to be used as a filter ''' if s_side not in ['BID', 'ASK']: raise InvalidTypeException('side should be BID or ASK') self.i_member = i_member self.s_side = s_side self.price_tree = FastRBTree() self._i_idx = 0 self.fr_data = fr_data self.parser = parser_data.LineParser(s_side) self.d_order_map = {} self.last_price = 0. def how_many_rows_read(self): ''' Return the number of rows processed ''' return self._i_idx def update(self, d_data, s_last_ident): ''' Update the state of the order book given the data pased :param d_data: dict. data from the last row :param s_last_ident: string. last identification ''' # check if the information should be processed if s_last_ident != 'MSG': return False # check if should filter out member if not self._should_use_it(d_data): return False # update the book information order_aux = Order(d_data) s_status = order_aux['order_status'] b_sould_update = True # treat Bovespa files at the begining f the day if s_status != 'New': try: i_old_id = self.d_order_map[order_aux]['main_id'] except KeyError: if s_status == 'Canceled' or s_status == 'Filled': b_sould_update = False s_status = 'Invalid' elif s_status == 'Replaced': s_status = 'New' # process if s_status == 'New': b_sould_update = self._new_order(order_aux) elif s_status != 'Invalid': i_old_id = self.d_order_map[order_aux]['main_id'] f_old_pr = self.d_order_map[order_aux]['price'] i_old_q = self.d_order_map[order_aux]['qty'] # hold the last traded price if s_status in ['Partially Filled', 'Filled']: self.last_price = order_aux['order_price'] # process message if s_status in ['Canceled', 'Expired', 'Filled']: b_sould_update = self._canc_expr_filled_order( order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Replaced': b_sould_update = self._replaced_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Partially Filled': b_sould_update = self._partially_filled( order_aux, i_old_id, f_old_pr, i_old_q) # remove from order map if s_status not in ['New', 'Invalid']: self.d_order_map.pop(order_aux) # update the order map if b_sould_update: f_qty = int(order_aux['total_qty_order']) self.d_order_map[order_aux] = {} self.d_order_map[order_aux]['price'] = d_data['order_price'] self.d_order_map[order_aux]['sec_order'] = order_aux.sec_order_id self.d_order_map[order_aux]['qty'] = f_qty self.d_order_map[order_aux]['main_id'] = order_aux.main_id # return that the update was done return True def _should_use_it(self, d_data): ''' Check if should use the passed row to update method :param d_data: dict. data from the last row ''' if self.i_member: if d_data['member'] != self.i_member: return False return True def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed canceled, expried or filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # remove from order map return False def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed replaced orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # remove from the old price this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # insert the order in the due price this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed partially filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # delete old price, if it is needed this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # add/modify order # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _new_order(self, order_obj): ''' Update price_tree when passed new orders :param order_obj: Order Object. The last order in the file ''' # if it was already in the order map if order_obj in self.d_order_map: i_old_sec_id = self.d_order_map[order_obj]['main_id'] f_old_price = self.d_order_map[order_obj]['price'] i_old_qty = self.d_order_map[order_obj]['qty'] this_price = self.price_tree.get(f_old_price) # remove from order map self.d_order_map.pop(order_obj) if this_price.delete(i_old_sec_id, i_old_qty): self.price_tree.remove(f_old_price) # insert a empty price level if it is needed f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # add the order this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def get_n_top_prices(self, n): ''' Return a dataframe with the N top price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def get_n_botton_prices(self, n=5): ''' Return a dataframe with the N botton price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def _readline(self): ''' Return a line from the fr_data file if available. Return false otherwiese ''' row = self.fr_data.readline() if row == '': self.fr_data.close() return False, False self._i_idx += 1 d_aux = self.parser(row) return d_aux, self.parser.last_identification def __iter__(self): ''' Return the self as an iterator object. Use next() to check the rows ''' return self def next(self): ''' Return the next item from the fr_data in iter process. If there are no further items, raise the StopIteration exception ''' d_rtn, last_identification = self._readline() if not d_rtn: raise StopIteration return d_rtn, last_identification
class BookSide(object): ''' A side of the lmit order book representation ''' def __init__(self, s_side): ''' Initialize a BookSide object. Save all parameters as attributes :param s_side: string. BID or ASK ''' if s_side not in ['BID', 'ASK']: raise InvalidTypeException('side should be BID or ASK') self.s_side = s_side self.price_tree = FastRBTree() self._i_idx = 0 self.d_order_map = {} self.last_price = 0. def update(self, d_data): ''' Update the state of the order book given the data pased. Return if the message was handle successfully :param d_data: dict. data related to a single order ''' # dont process aggresive trades if d_data['agressor_indicator'] == 'Agressive': return True # update the book information order_aux = Order(d_data) s_status = order_aux['order_status'] b_sould_update = True b_success = True # check the order status if s_status != 'New': try: i_old_id = self.d_order_map[order_aux]['main_id'] except KeyError: if s_status == 'Canceled' or s_status == 'Filled': b_sould_update = False s_status = 'Invalid' elif s_status == 'Replaced': s_status = 'New' # process the message if s_status == 'New': b_sould_update = self._new_order(order_aux) elif s_status != 'Invalid': i_old_id = self.d_order_map[order_aux]['main_id'] f_old_pr = self.d_order_map[order_aux]['price'] i_old_q = self.d_order_map[order_aux]['qty'] # hold the last traded price if s_status in ['Partially Filled', 'Filled']: self.last_price = order_aux['order_price'] # process message if s_status in ['Canceled', 'Expired', 'Filled']: b_sould_update = self._canc_expr_filled_order( order_aux, i_old_id, f_old_pr, i_old_q) if not b_sould_update: b_success = False elif s_status == 'Replaced': b_sould_update = self._replaced_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Partially Filled': b_sould_update = self._partially_filled( order_aux, i_old_id, f_old_pr, i_old_q) # remove from order map if s_status not in ['New', 'Invalid']: self.d_order_map.pop(order_aux) # update the order map if b_sould_update: f_qty = int(order_aux['total_qty_order']) self.d_order_map[order_aux] = {} self.d_order_map[order_aux]['price'] = d_data['order_price'] self.d_order_map[order_aux]['order_id'] = order_aux.order_id self.d_order_map[order_aux]['qty'] = f_qty self.d_order_map[order_aux]['main_id'] = order_aux.main_id # return that the update was done return True def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed canceled, expried or filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # remove from order map return False def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed replaced orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # remove from the old price this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # insert the order in the due price this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed partially filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # delete old price, if it is needed this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # add/modify order # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _new_order(self, order_obj): ''' Update price_tree when passed new orders :param order_obj: Order Object. The last order in the file ''' # if it was already in the order map if order_obj in self.d_order_map: i_old_sec_id = self.d_order_map[order_obj]['last_order_id'] f_old_price = self.d_order_map[order_obj]['price'] i_old_qty = self.d_order_map[order_obj]['qty'] this_price = self.price_tree.get(f_old_price) # remove from order map self.d_order_map.pop(order_obj) if this_price.delete(i_old_sec_id, i_old_qty): self.price_tree.remove(f_old_price) # insert a empty price level if it is needed f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # add the order this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def get_n_top_prices(self, n): ''' Return a dataframe with the N top price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def get_n_botton_prices(self, n=5): ''' Return a dataframe with the N botton price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError
class BookSide(object): ''' A side of the lmit order book representation ''' def __init__(self, s_side, fr_data, i_member=None): ''' Initialize a BookSide object. Save all parameters as attributes :param s_side: string. BID or ASK :param fr_data: ZipExtFile object. data to read :param i_member*: integer. Member number to be used as a filter ''' if s_side not in ['BID', 'ASK']: raise InvalidTypeException('side should be BID or ASK') self.i_member = i_member self.s_side = s_side self.price_tree = FastRBTree() self._i_idx = 0 self.fr_data = fr_data self.parser = parser_data.LineParser(s_side) self.d_order_map = {} self.last_price = 0. def how_many_rows_read(self): ''' Return the number of rows processed ''' return self._i_idx def update(self, d_data, s_last_ident): ''' Update the state of the order book given the data pased :param d_data: dict. data from the last row :param s_last_ident: string. last identification ''' # check if the information should be processed if s_last_ident != 'MSG': return False # check if should filter out member if not self._should_use_it(d_data): return False # update the book information order_aux = Order(d_data) s_status = order_aux['order_status'] b_sould_update = True # treat Bovespa files at the begining f the day if s_status != 'New': try: i_old_id = self.d_order_map[order_aux]['main_id'] except KeyError: if s_status == 'Canceled' or s_status == 'Filled': b_sould_update = False s_status = 'Invalid' elif s_status == 'Replaced': s_status = 'New' # process if s_status == 'New': b_sould_update = self._new_order(order_aux) elif s_status != 'Invalid': i_old_id = self.d_order_map[order_aux]['main_id'] f_old_pr = self.d_order_map[order_aux]['price'] i_old_q = self.d_order_map[order_aux]['qty'] # hold the last traded price if s_status in ['Partially Filled', 'Filled']: self.last_price = order_aux['order_price'] # process message if s_status in ['Canceled', 'Expired', 'Filled']: b_sould_update = self._canc_expr_filled_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Replaced': b_sould_update = self._replaced_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Partially Filled': b_sould_update = self._partially_filled(order_aux, i_old_id, f_old_pr, i_old_q) # remove from order map if s_status not in ['New', 'Invalid']: self.d_order_map.pop(order_aux) # update the order map if b_sould_update: f_qty = int(order_aux['total_qty_order']) self.d_order_map[order_aux] = {} self.d_order_map[order_aux]['price'] = d_data['order_price'] self.d_order_map[order_aux]['sec_order'] = order_aux.sec_order_id self.d_order_map[order_aux]['qty'] = f_qty self.d_order_map[order_aux]['main_id'] = order_aux.main_id # return that the update was done return True def _should_use_it(self, d_data): ''' Check if should use the passed row to update method :param d_data: dict. data from the last row ''' if self.i_member: if d_data['member'] != self.i_member: return False return True def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed canceled, expried or filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # remove from order map return False def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed replaced orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # remove from the old price this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # insert the order in the due price this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed partially filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # delete old price, if it is needed this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # add/modify order # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _new_order(self, order_obj): ''' Update price_tree when passed new orders :param order_obj: Order Object. The last order in the file ''' # if it was already in the order map if order_obj in self.d_order_map: i_old_sec_id = self.d_order_map[order_obj]['main_id'] f_old_price = self.d_order_map[order_obj]['price'] i_old_qty = self.d_order_map[order_obj]['qty'] this_price = self.price_tree.get(f_old_price) # remove from order map self.d_order_map.pop(order_obj) if this_price.delete(i_old_sec_id, i_old_qty): self.price_tree.remove(f_old_price) # insert a empty price level if it is needed f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # add the order this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def get_n_top_prices(self, n): ''' Return a dataframe with the N top price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def get_n_botton_prices(self, n=5): ''' Return a dataframe with the N botton price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def _readline(self): ''' Return a line from the fr_data file if available. Return false otherwiese ''' row = self.fr_data.readline() if row == '': self.fr_data.close() return False, False self._i_idx += 1 d_aux = self.parser(row) return d_aux, self.parser.last_identification def __iter__(self): ''' Return the self as an iterator object. Use next() to check the rows ''' return self def next(self): ''' Return the next item from the fr_data in iter process. If there are no further items, raise the StopIteration exception ''' d_rtn, last_identification = self._readline() if not d_rtn: raise StopIteration return d_rtn, last_identification
class BookSide(object): ''' A side of the lmit order book representation ''' def __init__(self, s_side, fr_data, i_member=None): ''' Initialize a BookSide object. Save all parameters as attributes :param s_side: string. BID or ASK :param fr_data: ZipExtFile object. data to read :param i_member*: integer. Member number to be used as a filter ''' if s_side not in ['BID', 'ASK']: raise InvalidTypeException('side should be BID or ASK') self.i_member = i_member self.s_side = s_side self.price_tree = FastRBTree() self._i_idx = 0 self.fr_data = fr_data self.parser = parser_data.LineParser(s_side) self.d_order_map = {} self.last_price = 0. # control other statistics self.best_queue = (None, None) self.i_qty_rel = 0 self.i_cum_rel = 0 def set_last_best_queue(self, t_best): ''' Set the best queue of this side ''' self.best_queue = t_best def update(self, d_data): ''' Update the state of the order book given the data pased :param d_data: dict. data from the last row ''' # update the book information order_aux = Order(d_data) s_status = order_aux['order_status'] b_sould_update = True i_rel_price = 0 # treat Bovespa files at the begining f the day if s_status != 'New': try: i_old_id = self.d_order_map[order_aux]['main_id'] except KeyError: # is not securing changes, also change part. filled status l_check = ['Canceled', 'Filled'] if not self.b_secure_changes: l_check = ['Canceled', 'Filled', 'Partially Filled'] # change order status when it is not found if s_status in l_check: b_sould_update = False s_status = 'Invalid' elif s_status == 'Replaced': s_status = 'New' # process if s_status == 'New': b_sould_update = self._new_order(order_aux) i_rel_price = get_relative_price(self.best_queue, order_aux) elif s_status != 'Invalid': i_old_id = self.d_order_map[order_aux]['main_id'] f_old_pr = self.d_order_map[order_aux]['price'] i_old_q = self.d_order_map[order_aux]['qty'] i_rel_price = self.d_order_map[order_aux]['relative_price'] # hold the last traded price if s_status in ['Partially Filled', 'Filled']: self.last_price = order_aux['order_price'] # process message if s_status in ['Canceled', 'Expired', 'Filled']: b_sould_update = self._canc_expr_filled_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Replaced': i_rel_price = get_relative_price(self.best_queue, order_aux) b_sould_update = self._replaced_order(order_aux, i_old_id, f_old_pr, i_old_q) elif s_status == 'Partially Filled': b_sould_update = self._partially_filled(order_aux, i_old_id, f_old_pr, i_old_q) # remove from order map if s_status not in ['New', 'Invalid']: self.d_order_map.pop(order_aux) # update the order map if b_sould_update: f_qty = int(order_aux['total_qty_order']) f_prior_time = d_data['priority_seconds'] self.d_order_map[order_aux] = {} self.d_order_map[order_aux]['price'] = d_data['order_price'] self.d_order_map[order_aux]['sec_order'] = order_aux.sec_order_id self.d_order_map[order_aux]['qty'] = f_qty self.d_order_map[order_aux]['main_id'] = order_aux.main_id self.d_order_map[order_aux]['priority_seconds'] = f_prior_time self.d_order_map[order_aux]['relative_price'] = i_rel_price if s_status in ['New', 'Replaced']: self.i_qty_rel += f_qty * 1. self.i_cum_rel += i_rel_price * 1. * f_qty # return that the update was done return True def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed canceled, expried or filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' b_break = False this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # remove from order map if b_break: raise NotImplementedError return False def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed replaced orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # remove from the old price this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # insert the order in the due price this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q): ''' Update price_tree when passed partially filled orders :param order_obj: Order Object. The last order in the file :param i_old_id: integer. Old id of the order_obj :param f_old_pr: float. Old price of the order_obj :param i_old_q: integer. Old qty of the order_obj ''' # delete old price, if it is needed this_price = self.price_tree.get(f_old_pr) if this_price.delete(i_old_id, i_old_q): self.price_tree.remove(f_old_pr) # add/modify order # insert in the new price f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def _new_order(self, order_obj): ''' Update price_tree when passed new orders :param order_obj: Order Object. The last order in the file ''' # if it was already in the order map if order_obj in self.d_order_map: i_old_sec_id = self.d_order_map[order_obj]['main_id'] f_old_price = self.d_order_map[order_obj]['price'] i_old_qty = self.d_order_map[order_obj]['qty'] this_price = self.price_tree.get(f_old_price) # remove from order map self.d_order_map.pop(order_obj) if this_price.delete(i_old_sec_id, i_old_qty): self.price_tree.remove(f_old_price) # insert a empty price level if it is needed f_price = order_obj['order_price'] if not self.price_tree.get(f_price): self.price_tree.insert(f_price, PriceLevel(f_price)) # add the order this_price = self.price_tree.get(f_price) this_price.add(order_obj) return True def get_n_top_prices(self, n): ''' Return a dataframe with the N top price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def get_n_botton_prices(self, n=5): ''' Return a dataframe with the N botton price levels :param n: integer. Number of price levels desired ''' raise NotImplementedError def _readline(self): ''' Return a line from the fr_data file if available. Return false otherwiese ''' row = self.fr_data.readline() if row == '': self.fr_data.close() return False, False self._i_idx += 1 d_aux = self.parser(row) # treat when the line is zero # TODO: I should move it to preprocessment step if 'order_price' in d_aux: if d_aux['order_price'] == 0.: while True: row = self.fr_data.readline() d_aux = self.parser(row) if d_aux['order_price'] != 0.: break return d_aux, self.parser.last_identification def __iter__(self): ''' Return the self as an iterator object. Use next() to check the rows ''' return self def next(self): ''' Return the next item from the fr_data in iter process. If there are no further items, raise the StopIteration exception ''' d_rtn, last_identification = self._readline() if not d_rtn: raise StopIteration return d_rtn, last_identification