def test_add_symbol():
ST = SymbolTable("")
T = FastRBTree()
Content = {'Type': "int" , 'Attribute': None , 'TokenLocation': (10,2) }
ST.InsertSymbol("age", Content)
T.insert("age", Content)
Content = {'Type': "float" , 'Attribute': 'static' , 'TokenLocation': (11,2) }
ST.InsertSymbol("temperature", Content)
T.insert("temperature", Content)
Content = {'Type': "char" , 'Attribute': 'const' , 'TokenLocation': (12,2) }
ST.InsertSymbol("letter", Content)
T.insert("letter", Content)
keys = ST.TopScope.keys()
for key in keys:
assert(T.__contains__(key))
assert(T.get(key) == ST.TopScope.get(key))
assert(T.get(key) is not ST.TopScope.get(key))
#write test to prove that the returned item is a pointer
return
class Node:
def __init__(self, name):
self.name = name
self.children = {}
self.visited = False
self.ancestors = FastRBTree()
def updateChildren(self, child, nameOfSequence):
if child in self.children:
self.children[child].add(nameOfSequence)
else:
self.children[child] = set()
self.children[child].add(nameOfSequence)
def getChild(self, nameOfChild):
return filter(lambda x: x == nameOfChild, self.children.keys())
def updateAncestors(self, ancestors, nameOfSequence):
for ancestor in ancestors:
if self.name == ancestor.name:
continue
elementFromTree = self.ancestors.get(ancestor.name, None)
if elementFromTree != None:
elementFromTree.add(nameOfSequence)
self.ancestors.__setitem__(ancestor.name, elementFromTree)
else: # NIE JE V SEK
tmpValue = set()
tmpValue.add(nameOfSequence)
self.ancestors.insert(ancestor.name, tmpValue)
class Hyperedge:
def __init__(self, hyperkey, col, hlabel):
self.hyperkey = hyperkey
self.col = col
self._alerts = Tree()
self.insert_alert(hlabel, 1)
self.nalerts = 1
def get_alert(self, key):
return self._alerts.get(key)
def insert_alert(self, alert_key, count):
self._alerts.insert(alert_key, count)
def foreach_alert(self, func):
self._alerts.foreach(func)
def pop_alert(self, key):
return self._alerts.pop(key)
class Hyperedge:
def __init__(self, hyperkey, col, hlabel):
self.hyperkey = hyperkey
self.col = col
self._alerts = Tree()
self.insert_alert(hlabel, 1)
self.nalerts = 1
def get_alert(self, key):
return self._alerts.get(key)
def insert_alert(self, alert_key, count):
self._alerts.insert(alert_key, count)
def foreach_alert(self, func):
self._alerts.foreach(func)
def pop_alert(self, key):
return self._alerts.pop(key)
from bintrees import FastRBTree
T = int(sys.stdin.readline().strip())
def split(n):
n -= 1
return n // 2, n // 2 + (n % 2)
assert (0, 0) == split(1)
assert (0, 1) == split(2)
assert (1, 1) == split(3)
for t in range(1, T + 1):
N, K = map(int, sys.stdin.readline().strip().split())
holes = FastRBTree({N: 1})
while K > 0:
size, count = holes.max_item()
del holes[size]
for s in split(size):
holes[s] = holes.get(s, 0) + count
K -= count
minD, maxD = split(size)
print("Case #%d: %d %d" % (t, maxD, minD))
class BookSide(object):
'''
A side of the lmit order book representation
'''
def __init__(self, s_side):
'''
Initialize a BookSide object. Save all parameters as attributes
:param s_side: string. BID or ASK
'''
if s_side not in ['BID', 'ASK']:
raise InvalidTypeException('side should be BID or ASK')
self.s_side = s_side
self.price_tree = FastRBTree()
self._i_idx = 0
self.d_order_map = {}
self.last_price = 0.
def update(self, d_data):
'''
Update the state of the order book given the data pased. Return if the
message was handle successfully
:param d_data: dict. data related to a single order
'''
# dont process aggresive trades
if d_data['agressor_indicator'] == 'Agressive':
return True
# update the book information
order_aux = Order(d_data)
s_status = order_aux['order_status']
b_sould_update = True
b_success = True
# check the order status
if s_status != 'New':
try:
i_old_id = self.d_order_map[order_aux]['main_id']
except KeyError:
if s_status == 'Canceled' or s_status == 'Filled':
b_sould_update = False
s_status = 'Invalid'
elif s_status == 'Replaced':
s_status = 'New'
# process the message
if s_status == 'New':
b_sould_update = self._new_order(order_aux)
elif s_status != 'Invalid':
i_old_id = self.d_order_map[order_aux]['main_id']
f_old_pr = self.d_order_map[order_aux]['price']
i_old_q = self.d_order_map[order_aux]['qty']
# hold the last traded price
if s_status in ['Partially Filled', 'Filled']:
self.last_price = order_aux['order_price']
# process message
if s_status in ['Canceled', 'Expired', 'Filled']:
b_sould_update = self._canc_expr_filled_order(order_aux,
i_old_id,
f_old_pr,
i_old_q)
if not b_sould_update:
b_success = False
elif s_status == 'Replaced':
b_sould_update = self._replaced_order(order_aux,
i_old_id,
f_old_pr,
i_old_q)
elif s_status == 'Partially Filled':
b_sould_update = self._partially_filled(order_aux,
i_old_id,
f_old_pr,
i_old_q)
# remove from order map
if s_status not in ['New', 'Invalid']:
self.d_order_map.pop(order_aux)
# update the order map
if b_sould_update:
f_qty = int(order_aux['total_qty_order'])
self.d_order_map[order_aux] = {}
self.d_order_map[order_aux]['price'] = d_data['order_price']
self.d_order_map[order_aux]['order_id'] = order_aux.order_id
self.d_order_map[order_aux]['qty'] = f_qty
self.d_order_map[order_aux]['main_id'] = order_aux.main_id
# return that the update was done
return True
def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed canceled, expried or filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# remove from order map
return False
def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed replaced orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# remove from the old price
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# insert the order in the due price
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed partially filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# delete old price, if it is needed
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# add/modify order
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _new_order(self, order_obj):
'''
Update price_tree when passed new orders
:param order_obj: Order Object. The last order in the file
'''
# if it was already in the order map
if order_obj in self.d_order_map:
i_old_sec_id = self.d_order_map[order_obj]['last_order_id']
f_old_price = self.d_order_map[order_obj]['price']
i_old_qty = self.d_order_map[order_obj]['qty']
this_price = self.price_tree.get(f_old_price)
# remove from order map
self.d_order_map.pop(order_obj)
if this_price.delete(i_old_sec_id, i_old_qty):
self.price_tree.remove(f_old_price)
# insert a empty price level if it is needed
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# add the order
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def get_n_top_prices(self, n):
'''
Return a dataframe with the N top price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def get_n_botton_prices(self, n=5):
'''
Return a dataframe with the N botton price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
class SymbolTable():
#constructor
def __init__(self, SourceFile):
self.Table = [
] #declare table as a stack (list) containing an empty tree
self.TopScope = FastRBTree(
) # a place where the current top scope is held
self.ReadMode = False #Read or lookup mode
self.DebugMode = False
self.SourceFile = SourceFile
#Function: InsertSymbol
#Desc: Insert a symbol into the current top of the symbol table
# The symbol key string is a lexeme
#Content_dict: At present this will contain
# {Type: (the token adjacent to the SymbolKey i.e. >int< <SymbolKey>),
# Attribute: some modifier on the symbol 'static' 'const' etc.
# TokenLocation: tuple(line, char_in_file, char_in_line) }
def InsertSymbol(self, SymbolKey_str, Content_dict):
try:
if self.DebugMode == True:
print("Insert symbol is called: ", "In Read Mode?",
self.ReadMode, SymbolKey_str, Content_dict)
if self.ReadMode == False:
found = self.FindSymbolInCurrentScope(SymbolKey_str)
if not found:
found = self.FindSymbolInTable(SymbolKey_str)
if found:
for item in found:
for key in list(item.keys()):
self.PrettyErrorPrint(
"Warning: {0} on line {3} is a shadowded variable. Previous declaration in scope level {1} at line {2}."
.format(SymbolKey_str,
abs(key - len(self.Table)),
item[key]["TokenLocation"][0],
Content_dict['TokenLocation'][0]),
item[key]["TokenLocation"][0],
item[key]["TokenLocation"][2])
#perform deepcopy on passed in dictionary
self.TopScope.insert(SymbolKey_str, deepcopy(Content_dict))
else:
self.PrettyErrorPrint(
'''Error: Redeclaration of existing variable.\nPrior declaration is here at line {0}: \n'''
.format(found["TokenLocation"][0]),
found["TokenLocation"][0], found["TokenLocation"][2])
raise Exception(
'Redeclaration of exisitng variable in current scope.')
else:
# do nothing
pass
except Exception as e:
raise e
return True
#Function: FindSymbolInTable
#Desc: Search all scopes of the symbol table to find a specific symbol
#Return: [{Level_int: Content_dict}, False] (list of keys for possibility of many shadowed vars)
def FindSymbolInTable(self, SymbolKey_str):
T_list = []
Level_int = 0
#search the top of our stack
if self.FindSymbolInCurrentScope(SymbolKey_str):
T_list.append(self.FindSymbolInCurrentScope(SymbolKey_str))
Level_int += 1
#iterate over all trees
#add found isntances of symbols to list if present
for Tree in reversed(
self.Table): #reversed so appended items are at the front
if Tree.__contains__(SymbolKey_str):
T_list.append({Level_int: Tree.get(SymbolKey_str)})
Level_int += 1
if len(T_list) > 0:
return T_list
#nothing found case
return False
#Function: FindSymbolInCurrentScope
#Desc: Search only the top level of the symbol table for key
def FindSymbolInCurrentScope(self, SymbolKey_str):
return self.TopScope.get(SymbolKey_str, False)
#Function:PushNewScope
#Desc: Create a new scope and push it onto the table
def PushNewScope(self):
self.PushScope(FastRBTree())
return
#Function: PushScope
#Desc: Insert symbol tree (RBTree) onto our table
def PushScope(self, SymbolTree):
self.Table.append(self.TopScope)
self.TopScope = SymbolTree
return
#Function: PopScope
#Desc: Remove and return scope (RBtree) from the symbol table
def PopScope(self):
TScope = self.TopScope
if len(self.Table) > 0:
self.TopScope = self.Table.pop()
else:
self.TopScope = None
return TScope
#Function: WriteSymbolTableToFile
#Desc: Write the current contents of the symbol table to file
def WriteSymbolTableToFile(self, FileName_str):
T_Stack = []
i = 0
try:
with open(FileName_str, "w") as File:
File.write(
"\n**** Outputting Contents of Symbol Table **** \n\n")
while not self.TableIsEmpty():
File.write("Items in Tree at Level {}: \n".format(i))
self.PrettyPrintScope(self.TopScope, FilePtr=File)
T_Stack.append(self.PopScope())
i += 1
while len(T_Stack) > 0:
self.PushScope(T_Stack.pop())
except Exception as e:
raise
def PrettyPrintScope(self, Scope, FilePtr=None):
for Key in Scope.keys():
if FilePtr is not None:
FilePtr.write("\tKey: \"{}\" | Content: {}\n".format(
Key, Scope.get(Key)))
else:
print("\tKey: \"{}\" | Content: {}\n".format(
Key, Scope.get(Key)))
return
def ToggleDebugMode(self):
self.DebugMode = not self.DebugMode
return
def ReadModeOn(self):
self.ReadMode = True
# print("Insert Mode Toggled Off")
return
def InsertMode(self):
self.ReadMode = False
# print("Insert Mode Toggled On")
return
def ToggleReadMode(self):
self.ReadMode = not self.ReadMode
if self.ReadMode == False:
print("Insert Mode Toggled On")
elif self.ReadMode == True:
print("Insert Mode Toggled Off")
return
def TableIsEmpty(self):
if self.TopScope is None:
return True
return False
def PrettyErrorPrint(self, Message, Lineno, Column):
arrow = ""
print(Message)
#print line
with open(self.SourceFile) as file:
for i in range(0, Lineno):
source = file.readline()
print(source)
#build arrow
for i in range(0, Column - 1):
arrow += " "
arrow += "^\n"
print(arrow)
class OrderTree(object):
def __init__(self):
self.price_tree = FastRBTree()
self.min_price = None
self.max_price = None
def get_orders_at_price(self, price):
return self.price_tree.get(price)
def insert_price(self, price, amount, oid):
## ignore market order
if price == Decimal(0.0):
return
prev_val = self.get_orders_at_price(price)
if prev_val != None:
## price exists in local order book
if oid in prev_val:
## update to an existing order at price
prev_val['total'] = prev_val['total'] - prev_val[oid] + amount
prev_val[oid] = amount
else:
## new order at price
prev_val['total'] += amount
prev_val[oid] = amount
self.price_tree.insert(price, prev_val)
elif amount != 0.0:
## price did not exit in order book
val = {'total': amount, oid: amount}
self.price_tree.insert(price, val)
try:
val = self.price_tree.get(price)
if val['total'] > 0:
if self.max_price == None or price > self.max_price:
self.max_price = price
if self.min_price == None or price < self.min_price:
self.min_price = price
elif val['total'] == 0:
## price removed from orderbook
self.remove_price(price)
else:
## something has gone terribly wrong
logging.error(
"total amount at price %s went to negative amounts" %
(price))
except:
logging.error("price (%s) does not exist in orderbook" % (price))
def remove_price(self, price):
self.price_tree.remove(price)
if self.max_price == price:
try:
self.max_price = max(self.price_tree)
except ValueError:
self.max_price = None
if self.min_price == price:
try:
self.min_price = min(self.price_tree)
except ValueError:
self.min_price = None
class TradeTree(object):
'''A red-black tree used to store TradeLists in price trade
The exchange will be using the TradeTree to hold bid and ask data (one TradeTree for each side).
Keeping the information in a red black tree makes it easier/faster to detect a match.
'''
def __init__(self):
self.price_tree = FastRBTree()
self.trade_map = {}
self.num_trades = 0 # Contains count of Orders in tree
self.depth = 0 # Number of different prices in tree (http://en.wikipedia.org/wiki/trade_book_(trading)#Book_depth)
def __len__(self):
return len(self.trade_map)
def get_price_list(self, price):
return self.price_tree.get(price, [])
def get_trade(self, trade_id):
return self.trade_map[trade_id] if trade_id in self.trade_map else None
def create_price(self, price):
self.depth += 1 # Add a price depth level to the tree
new_list = LinkedList()
self.price_tree.insert(price,
new_list) # Insert a new price into the tree
def remove_price(self, price):
self.depth -= 1 # Remove a price depth level
self.price_tree.remove(price)
def price_exists(self, price):
return self.price_tree.__contains__(price)
def trade_exists(self, trade_id):
return trade_id in self.trade_map
def insert_trade(self, xtrade):
if self.trade_exists(xtrade.id):
return
self.num_trades += 1
if not self.price_exists(xtrade.limit_price):
self.create_price(
xtrade.limit_price
) # If price not in Price Map, create a node in RBtree
self.trade_map[
trade.id] = self.price_tree[xtrade.limit_price].append_item(
xtrade
) # Add the trade to the TradeList in Price Map return the reference
def remove_trade(self, xtrade):
self.num_trades -= 1
trade_node = self.trade_map[trade.id]
self.price_tree[trade.limit_price].remove_item(trade_node)
if len(self.price_tree[trade.limit_price]) == 0:
self.remove_price(trade.limit_price)
self.trade_map.pop(trade.id, None)
def max_price(self):
if self.depth > 0:
return self.price_tree.max_key()
else:
return None
def min_price(self):
if self.depth > 0:
return self.price_tree.min_key()
else:
return None
def max_price_list(self):
if self.depth > 0:
return self.get_price_list(self.max_price())
else:
return None
def min_price_list(self):
if self.depth > 0:
return self.get_price_list(self.min_price())
else:
return None
class BookSide(object):
'''
A side of the lmit order book representation
'''
def __init__(self, s_side, fr_data, i_member=None):
'''
Initialize a BookSide object. Save all parameters as attributes
:param s_side: string. BID or ASK
:param fr_data: ZipExtFile object. data to read
:param i_member*: integer. Member number to be used as a filter
'''
if s_side not in ['BID', 'ASK']:
raise InvalidTypeException('side should be BID or ASK')
self.i_member = i_member
self.s_side = s_side
self.price_tree = FastRBTree()
self._i_idx = 0
self.fr_data = fr_data
self.parser = parser_data.LineParser(s_side)
self.d_order_map = {}
self.last_price = 0.
def how_many_rows_read(self):
'''
Return the number of rows processed
'''
return self._i_idx
def update(self, d_data, s_last_ident):
'''
Update the state of the order book given the data pased
:param d_data: dict. data from the last row
:param s_last_ident: string. last identification
'''
# check if the information should be processed
if s_last_ident != 'MSG':
return False
# check if should filter out member
if not self._should_use_it(d_data):
return False
# update the book information
order_aux = Order(d_data)
s_status = order_aux['order_status']
b_sould_update = True
# treat Bovespa files at the begining f the day
if s_status != 'New':
try:
i_old_id = self.d_order_map[order_aux]['main_id']
except KeyError:
if s_status == 'Canceled' or s_status == 'Filled':
b_sould_update = False
s_status = 'Invalid'
elif s_status == 'Replaced':
s_status = 'New'
# process
if s_status == 'New':
b_sould_update = self._new_order(order_aux)
elif s_status != 'Invalid':
i_old_id = self.d_order_map[order_aux]['main_id']
f_old_pr = self.d_order_map[order_aux]['price']
i_old_q = self.d_order_map[order_aux]['qty']
# hold the last traded price
if s_status in ['Partially Filled', 'Filled']:
self.last_price = order_aux['order_price']
# process message
if s_status in ['Canceled', 'Expired', 'Filled']:
b_sould_update = self._canc_expr_filled_order(
order_aux, i_old_id, f_old_pr, i_old_q)
elif s_status == 'Replaced':
b_sould_update = self._replaced_order(order_aux, i_old_id,
f_old_pr, i_old_q)
elif s_status == 'Partially Filled':
b_sould_update = self._partially_filled(
order_aux, i_old_id, f_old_pr, i_old_q)
# remove from order map
if s_status not in ['New', 'Invalid']:
self.d_order_map.pop(order_aux)
# update the order map
if b_sould_update:
f_qty = int(order_aux['total_qty_order'])
self.d_order_map[order_aux] = {}
self.d_order_map[order_aux]['price'] = d_data['order_price']
self.d_order_map[order_aux]['sec_order'] = order_aux.sec_order_id
self.d_order_map[order_aux]['qty'] = f_qty
self.d_order_map[order_aux]['main_id'] = order_aux.main_id
# return that the update was done
return True
def _should_use_it(self, d_data):
'''
Check if should use the passed row to update method
:param d_data: dict. data from the last row
'''
if self.i_member:
if d_data['member'] != self.i_member:
return False
return True
def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed canceled, expried or filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# remove from order map
return False
def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed replaced orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# remove from the old price
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# insert the order in the due price
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed partially filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# delete old price, if it is needed
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# add/modify order
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _new_order(self, order_obj):
'''
Update price_tree when passed new orders
:param order_obj: Order Object. The last order in the file
'''
# if it was already in the order map
if order_obj in self.d_order_map:
i_old_sec_id = self.d_order_map[order_obj]['main_id']
f_old_price = self.d_order_map[order_obj]['price']
i_old_qty = self.d_order_map[order_obj]['qty']
this_price = self.price_tree.get(f_old_price)
# remove from order map
self.d_order_map.pop(order_obj)
if this_price.delete(i_old_sec_id, i_old_qty):
self.price_tree.remove(f_old_price)
# insert a empty price level if it is needed
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# add the order
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def get_n_top_prices(self, n):
'''
Return a dataframe with the N top price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def get_n_botton_prices(self, n=5):
'''
Return a dataframe with the N botton price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def _readline(self):
'''
Return a line from the fr_data file if available. Return false
otherwiese
'''
row = self.fr_data.readline()
if row == '':
self.fr_data.close()
return False, False
self._i_idx += 1
d_aux = self.parser(row)
return d_aux, self.parser.last_identification
def __iter__(self):
'''
Return the self as an iterator object. Use next() to check the rows
'''
return self
def next(self):
'''
Return the next item from the fr_data in iter process. If there are no
further items, raise the StopIteration exception
'''
d_rtn, last_identification = self._readline()
if not d_rtn:
raise StopIteration
return d_rtn, last_identification
class BookSide(object):
'''
A side of the lmit order book representation
'''
def __init__(self, s_side):
'''
Initialize a BookSide object. Save all parameters as attributes
:param s_side: string. BID or ASK
'''
if s_side not in ['BID', 'ASK']:
raise InvalidTypeException('side should be BID or ASK')
self.s_side = s_side
self.price_tree = FastRBTree()
self._i_idx = 0
self.d_order_map = {}
self.last_price = 0.
def update(self, d_data):
'''
Update the state of the order book given the data pased. Return if the
message was handle successfully
:param d_data: dict. data related to a single order
'''
# dont process aggresive trades
if d_data['agressor_indicator'] == 'Agressive':
return True
# update the book information
order_aux = Order(d_data)
s_status = order_aux['order_status']
b_sould_update = True
b_success = True
# check the order status
if s_status != 'New':
try:
i_old_id = self.d_order_map[order_aux]['main_id']
except KeyError:
if s_status == 'Canceled' or s_status == 'Filled':
b_sould_update = False
s_status = 'Invalid'
elif s_status == 'Replaced':
s_status = 'New'
# process the message
if s_status == 'New':
b_sould_update = self._new_order(order_aux)
elif s_status != 'Invalid':
i_old_id = self.d_order_map[order_aux]['main_id']
f_old_pr = self.d_order_map[order_aux]['price']
i_old_q = self.d_order_map[order_aux]['qty']
# hold the last traded price
if s_status in ['Partially Filled', 'Filled']:
self.last_price = order_aux['order_price']
# process message
if s_status in ['Canceled', 'Expired', 'Filled']:
b_sould_update = self._canc_expr_filled_order(
order_aux, i_old_id, f_old_pr, i_old_q)
if not b_sould_update:
b_success = False
elif s_status == 'Replaced':
b_sould_update = self._replaced_order(order_aux, i_old_id,
f_old_pr, i_old_q)
elif s_status == 'Partially Filled':
b_sould_update = self._partially_filled(
order_aux, i_old_id, f_old_pr, i_old_q)
# remove from order map
if s_status not in ['New', 'Invalid']:
self.d_order_map.pop(order_aux)
# update the order map
if b_sould_update:
f_qty = int(order_aux['total_qty_order'])
self.d_order_map[order_aux] = {}
self.d_order_map[order_aux]['price'] = d_data['order_price']
self.d_order_map[order_aux]['order_id'] = order_aux.order_id
self.d_order_map[order_aux]['qty'] = f_qty
self.d_order_map[order_aux]['main_id'] = order_aux.main_id
# return that the update was done
return True
def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed canceled, expried or filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# remove from order map
return False
def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed replaced orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# remove from the old price
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# insert the order in the due price
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed partially filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# delete old price, if it is needed
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# add/modify order
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _new_order(self, order_obj):
'''
Update price_tree when passed new orders
:param order_obj: Order Object. The last order in the file
'''
# if it was already in the order map
if order_obj in self.d_order_map:
i_old_sec_id = self.d_order_map[order_obj]['last_order_id']
f_old_price = self.d_order_map[order_obj]['price']
i_old_qty = self.d_order_map[order_obj]['qty']
this_price = self.price_tree.get(f_old_price)
# remove from order map
self.d_order_map.pop(order_obj)
if this_price.delete(i_old_sec_id, i_old_qty):
self.price_tree.remove(f_old_price)
# insert a empty price level if it is needed
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# add the order
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def get_n_top_prices(self, n):
'''
Return a dataframe with the N top price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def get_n_botton_prices(self, n=5):
'''
Return a dataframe with the N botton price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
class BookSide(object):
'''
A side of the lmit order book representation
'''
def __init__(self, s_side, fr_data, i_member=None):
'''
Initialize a BookSide object. Save all parameters as attributes
:param s_side: string. BID or ASK
:param fr_data: ZipExtFile object. data to read
:param i_member*: integer. Member number to be used as a filter
'''
if s_side not in ['BID', 'ASK']:
raise InvalidTypeException('side should be BID or ASK')
self.i_member = i_member
self.s_side = s_side
self.price_tree = FastRBTree()
self._i_idx = 0
self.fr_data = fr_data
self.parser = parser_data.LineParser(s_side)
self.d_order_map = {}
self.last_price = 0.
def how_many_rows_read(self):
'''
Return the number of rows processed
'''
return self._i_idx
def update(self, d_data, s_last_ident):
'''
Update the state of the order book given the data pased
:param d_data: dict. data from the last row
:param s_last_ident: string. last identification
'''
# check if the information should be processed
if s_last_ident != 'MSG':
return False
# check if should filter out member
if not self._should_use_it(d_data):
return False
# update the book information
order_aux = Order(d_data)
s_status = order_aux['order_status']
b_sould_update = True
# treat Bovespa files at the begining f the day
if s_status != 'New':
try:
i_old_id = self.d_order_map[order_aux]['main_id']
except KeyError:
if s_status == 'Canceled' or s_status == 'Filled':
b_sould_update = False
s_status = 'Invalid'
elif s_status == 'Replaced':
s_status = 'New'
# process
if s_status == 'New':
b_sould_update = self._new_order(order_aux)
elif s_status != 'Invalid':
i_old_id = self.d_order_map[order_aux]['main_id']
f_old_pr = self.d_order_map[order_aux]['price']
i_old_q = self.d_order_map[order_aux]['qty']
# hold the last traded price
if s_status in ['Partially Filled', 'Filled']:
self.last_price = order_aux['order_price']
# process message
if s_status in ['Canceled', 'Expired', 'Filled']:
b_sould_update = self._canc_expr_filled_order(order_aux,
i_old_id,
f_old_pr,
i_old_q)
elif s_status == 'Replaced':
b_sould_update = self._replaced_order(order_aux,
i_old_id,
f_old_pr,
i_old_q)
elif s_status == 'Partially Filled':
b_sould_update = self._partially_filled(order_aux,
i_old_id,
f_old_pr,
i_old_q)
# remove from order map
if s_status not in ['New', 'Invalid']:
self.d_order_map.pop(order_aux)
# update the order map
if b_sould_update:
f_qty = int(order_aux['total_qty_order'])
self.d_order_map[order_aux] = {}
self.d_order_map[order_aux]['price'] = d_data['order_price']
self.d_order_map[order_aux]['sec_order'] = order_aux.sec_order_id
self.d_order_map[order_aux]['qty'] = f_qty
self.d_order_map[order_aux]['main_id'] = order_aux.main_id
# return that the update was done
return True
def _should_use_it(self, d_data):
'''
Check if should use the passed row to update method
:param d_data: dict. data from the last row
'''
if self.i_member:
if d_data['member'] != self.i_member:
return False
return True
def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed canceled, expried or filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# remove from order map
return False
def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed replaced orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# remove from the old price
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# insert the order in the due price
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed partially filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# delete old price, if it is needed
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# add/modify order
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _new_order(self, order_obj):
'''
Update price_tree when passed new orders
:param order_obj: Order Object. The last order in the file
'''
# if it was already in the order map
if order_obj in self.d_order_map:
i_old_sec_id = self.d_order_map[order_obj]['main_id']
f_old_price = self.d_order_map[order_obj]['price']
i_old_qty = self.d_order_map[order_obj]['qty']
this_price = self.price_tree.get(f_old_price)
# remove from order map
self.d_order_map.pop(order_obj)
if this_price.delete(i_old_sec_id, i_old_qty):
self.price_tree.remove(f_old_price)
# insert a empty price level if it is needed
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# add the order
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def get_n_top_prices(self, n):
'''
Return a dataframe with the N top price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def get_n_botton_prices(self, n=5):
'''
Return a dataframe with the N botton price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def _readline(self):
'''
Return a line from the fr_data file if available. Return false
otherwiese
'''
row = self.fr_data.readline()
if row == '':
self.fr_data.close()
return False, False
self._i_idx += 1
d_aux = self.parser(row)
return d_aux, self.parser.last_identification
def __iter__(self):
'''
Return the self as an iterator object. Use next() to check the rows
'''
return self
def next(self):
'''
Return the next item from the fr_data in iter process. If there are no
further items, raise the StopIteration exception
'''
d_rtn, last_identification = self._readline()
if not d_rtn:
raise StopIteration
return d_rtn, last_identification
class BookSide(object):
'''
A side of the lmit order book representation
'''
def __init__(self, s_side, fr_data, i_member=None):
'''
Initialize a BookSide object. Save all parameters as attributes
:param s_side: string. BID or ASK
:param fr_data: ZipExtFile object. data to read
:param i_member*: integer. Member number to be used as a filter
'''
if s_side not in ['BID', 'ASK']:
raise InvalidTypeException('side should be BID or ASK')
self.i_member = i_member
self.s_side = s_side
self.price_tree = FastRBTree()
self._i_idx = 0
self.fr_data = fr_data
self.parser = parser_data.LineParser(s_side)
self.d_order_map = {}
self.last_price = 0.
# control other statistics
self.best_queue = (None, None)
self.i_qty_rel = 0
self.i_cum_rel = 0
def set_last_best_queue(self, t_best):
'''
Set the best queue of this side
'''
self.best_queue = t_best
def update(self, d_data):
'''
Update the state of the order book given the data pased
:param d_data: dict. data from the last row
'''
# update the book information
order_aux = Order(d_data)
s_status = order_aux['order_status']
b_sould_update = True
i_rel_price = 0
# treat Bovespa files at the begining f the day
if s_status != 'New':
try:
i_old_id = self.d_order_map[order_aux]['main_id']
except KeyError:
# is not securing changes, also change part. filled status
l_check = ['Canceled', 'Filled']
if not self.b_secure_changes:
l_check = ['Canceled', 'Filled', 'Partially Filled']
# change order status when it is not found
if s_status in l_check:
b_sould_update = False
s_status = 'Invalid'
elif s_status == 'Replaced':
s_status = 'New'
# process
if s_status == 'New':
b_sould_update = self._new_order(order_aux)
i_rel_price = get_relative_price(self.best_queue, order_aux)
elif s_status != 'Invalid':
i_old_id = self.d_order_map[order_aux]['main_id']
f_old_pr = self.d_order_map[order_aux]['price']
i_old_q = self.d_order_map[order_aux]['qty']
i_rel_price = self.d_order_map[order_aux]['relative_price']
# hold the last traded price
if s_status in ['Partially Filled', 'Filled']:
self.last_price = order_aux['order_price']
# process message
if s_status in ['Canceled', 'Expired', 'Filled']:
b_sould_update = self._canc_expr_filled_order(order_aux,
i_old_id,
f_old_pr,
i_old_q)
elif s_status == 'Replaced':
i_rel_price = get_relative_price(self.best_queue, order_aux)
b_sould_update = self._replaced_order(order_aux,
i_old_id,
f_old_pr,
i_old_q)
elif s_status == 'Partially Filled':
b_sould_update = self._partially_filled(order_aux,
i_old_id,
f_old_pr,
i_old_q)
# remove from order map
if s_status not in ['New', 'Invalid']:
self.d_order_map.pop(order_aux)
# update the order map
if b_sould_update:
f_qty = int(order_aux['total_qty_order'])
f_prior_time = d_data['priority_seconds']
self.d_order_map[order_aux] = {}
self.d_order_map[order_aux]['price'] = d_data['order_price']
self.d_order_map[order_aux]['sec_order'] = order_aux.sec_order_id
self.d_order_map[order_aux]['qty'] = f_qty
self.d_order_map[order_aux]['main_id'] = order_aux.main_id
self.d_order_map[order_aux]['priority_seconds'] = f_prior_time
self.d_order_map[order_aux]['relative_price'] = i_rel_price
if s_status in ['New', 'Replaced']:
self.i_qty_rel += f_qty * 1.
self.i_cum_rel += i_rel_price * 1. * f_qty
# return that the update was done
return True
def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed canceled, expried or filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
b_break = False
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# remove from order map
if b_break:
raise NotImplementedError
return False
def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed replaced orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# remove from the old price
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# insert the order in the due price
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _partially_filled(self, order_obj, i_old_id, f_old_pr, i_old_q):
'''
Update price_tree when passed partially filled orders
:param order_obj: Order Object. The last order in the file
:param i_old_id: integer. Old id of the order_obj
:param f_old_pr: float. Old price of the order_obj
:param i_old_q: integer. Old qty of the order_obj
'''
# delete old price, if it is needed
this_price = self.price_tree.get(f_old_pr)
if this_price.delete(i_old_id, i_old_q):
self.price_tree.remove(f_old_pr)
# add/modify order
# insert in the new price
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def _new_order(self, order_obj):
'''
Update price_tree when passed new orders
:param order_obj: Order Object. The last order in the file
'''
# if it was already in the order map
if order_obj in self.d_order_map:
i_old_sec_id = self.d_order_map[order_obj]['main_id']
f_old_price = self.d_order_map[order_obj]['price']
i_old_qty = self.d_order_map[order_obj]['qty']
this_price = self.price_tree.get(f_old_price)
# remove from order map
self.d_order_map.pop(order_obj)
if this_price.delete(i_old_sec_id, i_old_qty):
self.price_tree.remove(f_old_price)
# insert a empty price level if it is needed
f_price = order_obj['order_price']
if not self.price_tree.get(f_price):
self.price_tree.insert(f_price, PriceLevel(f_price))
# add the order
this_price = self.price_tree.get(f_price)
this_price.add(order_obj)
return True
def get_n_top_prices(self, n):
'''
Return a dataframe with the N top price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def get_n_botton_prices(self, n=5):
'''
Return a dataframe with the N botton price levels
:param n: integer. Number of price levels desired
'''
raise NotImplementedError
def _readline(self):
'''
Return a line from the fr_data file if available. Return false
otherwiese
'''
row = self.fr_data.readline()
if row == '':
self.fr_data.close()
return False, False
self._i_idx += 1
d_aux = self.parser(row)
# treat when the line is zero
# TODO: I should move it to preprocessment step
if 'order_price' in d_aux:
if d_aux['order_price'] == 0.:
while True:
row = self.fr_data.readline()
d_aux = self.parser(row)
if d_aux['order_price'] != 0.:
break
return d_aux, self.parser.last_identification
def __iter__(self):
'''
Return the self as an iterator object. Use next() to check the rows
'''
return self
def next(self):
'''
Return the next item from the fr_data in iter process. If there are no
further items, raise the StopIteration exception
'''
d_rtn, last_identification = self._readline()
if not d_rtn:
raise StopIteration
return d_rtn, last_identification
