class GeoDB(object): """ Container object for the geo tree that allows applications to easily lookup country information by IP address. """ def __init__(self, csvfile=None, parser=MaxMindGeoLiteCSVParser()): treelist = parser.parse(csvfile) self.tree = FastAVLTree(treelist) # Done with the list, remove it since it can be rather large del treelist def lookup(self, ip, default=None): """ Get the GeoValue object for a given IP address IP addresses can be either a Long or a dotted-decimal formatted string. """ return self.tree.get(ip, default)
class GeoDB(object): """ Container object for the geo tree that allows applications to easily lookup country information by IP address. """ def __init__(self, csvfile=None, parser=MaxMindGeoLiteCSVParser()): treelist = parser.parse(csvfile) self.tree = FastAVLTree(treelist) # Done with the list, remove it since it can be rather large del treelist def lookup(self, ip, default=None): """ Get the GeoValue object for a given IP address IP addresses can be either a Long or a dotted-decimal formatted string. """ return self.tree.get(ip, default)
class AkashicRecord(object): """ Database of Tumblr Post get by id sorted with unix_timestamp viewed flag. new note check """ def __init__(self): self.impl = FastAVLTree() def get(self, post_id): return self.impl.get(post_id) def put(self, post): assert isinstance(post, Post) return self.impl.insert(post.id, post) def get_after(self, start): pass
class OrderBook: """Limit order book able to process LMT and MKT orders MKT orders are disassembled to LMT orders up to current liquidity situation """ def __init__(self): # AVL trees are used as a main structure due its optimal performance features for this purpose self._participants = FastAVLTree() self._order_owners = FastAVLTree() # Assigning ID -> Owner self._asks = FastAVLTree() # MIN Heap self._bids = FastAVLTree() # MAX Heap self._price_ids = FastAVLTree() # Assigning ID -> Price self._total_ask_size = 0 # For monitoring purpose self._total_bid_size = 0 # For monitoring purpose self._last_order_id = 0 # Increases with each order processed self._cleared_orders_count = 0 # For monitoring purpose self._total_volume_trad = 0 # For monitoring purpose self._total_volume_pending = 0 # For monitoring purpose def __getstate__(self): """ Whole book could be repopulated from dict containing class attributes """ return self.__dict__ def __setstate__(self, state): """ Book repopulation (recovery) """ for at_name, at_val in state.items(): setattr(self, at_name, at_val) def _get_order_id(self): """ Orders id managment """ self._last_order_id += 1 return self._last_order_id def _balance(self, trades_stack): """ Executes trades if it finds liquidity for them """ # No liquidity at all if self._asks.is_empty() or self._bids.is_empty(): return trades_stack min_ask = self._asks.min_key() max_bid = self._bids.max_key() # Check liquidity situation if max_bid >= min_ask: ask_orders = self._asks.get(min_ask) bid_orders = self._bids.get(max_bid) for ask_order in ask_orders: for bid_order in bid_orders: if not ask_order in ask_orders or not bid_order in bid_orders: continue trad = min(ask_orders[ask_order], bid_orders[bid_order]) ask_orders[ask_order] -= traded bid_orders[bid_order] -= traded self._total_ask_size -= traded self._total_bid_size -= traded self._total_volume_trad += traded self._total_volume_pending -= 2 * traded ask_owner = self._order_owners[ask_order] bid_owner = self._order_owners[bid_order] # Buy side order fully liquidated if bid_orders[bid_order] == 0: # print("BID ORDER LIQUIDATED") self._cleared_orders_count += 1 del bid_orders[bid_order] del self._price_ids[bid_order] del self._order_owners[bid_order] owner_ids = self._participants[bid_owner] owner_ids.remove(bid_order) del self._participants[bid_owner] self._participants.insert(bid_owner, owner_ids) # Sell side order fully liquidated if ask_orders[ask_order] == 0: # print("ASK ORDER LIQUIDATED") self._cleared_orders_count += 1 del ask_orders[ask_order] del self._price_ids[ask_order] del self._order_owners[ask_order] owner_ids = self._participants[ask_owner] owner_ids.remove(ask_order) del self._participants[ask_owner] self._participants.insert(ask_owner, owner_ids) # Inform sides about state of their orders trades_stack.append((0, traded, max_bid)) trades_stack.append((1, ask_order, traded, max_bid, ask_owner, 'ask')) trades_stack.append((1, bid_order, traded, max_bid, bid_owner, 'bid')) # Whole ASK price level were liquidated, remove it from three and let it rebalance if self._asks[min_ask].is_empty(): # print("ASK level liquidated") del self._asks[min_ask] # Whole BID price level were liquidated, remove it from three and let it rebalance if self._bids[max_bid].is_empty(): # print("BID level liquidated") del self._bids[max_bid] else: return trades_stack return self._balance(trades_stack) def _submit_mkt(self, side, size, pi_d): """ Find liquidity for mkt order - put multiple lmt orders to extract liquidity for order execution """ olst = [] trades_stack = [] while size > 0: if side == 'ask': second_side_size = self.bid_size second_side_price = self.bid else: second_side_size = self.ask_size second_side_price = self.ask # We could only taky liquidity which exists trade_size = min([second_side_size, size]) olst.append(self._submit_lmt(side, trade_size, second_side_price, pi_d)) trades_stack = self._balance(trades_stack) size -= trade_size return 0, trades_stack def _submit_lmt(self, side, size, price, pi_d): """ Submits LMT order to book """ # Assign order ID order_id = self._get_order_id() # Pending volume monitoring self._total_volume_pending += size self._price_ids.insert(order_id, (price, side)) # Keep track of participant orders, book will be asked for sure if pi_d not in self._participants: self._participants.insert(pi_d, [order_id]) else: owner_trades = self._participants.get(pi_d, []) owner_trades.append(order_id) self._order_owners.insert(order_id, pi_d) # Assign to right (correct) side if side == 'ask': self._total_ask_size += size ask_level = self._asks.get(price, FastAVLTree()) ask_level.insert(order_id, size) if price not in self._asks: self._asks.insert(price, ask_level) else: # bid self._total_bid_size += size bid_level = self._bids.get(price, FastAVLTree()) bid_level.insert(order_id, size) if price not in self._bids: self._bids.insert(price, bid_level) return order_id def cancel(self, order_id): """ Cancel order """ # Finds and cancels order order = self._price_ids[order_id] if order[1] == 'ask': del self._asks[order[0]][order_id] if self._asks[order[0]].is_empty(): del self._asks[order[0]] else: del self._bids[order[0]][order_id] if self._bids[order[0]].is_empty(): del self._bids[order[0]] @property def ask_size(self): """ Volume waiting on ask side bottom level - liquidity level size for ask price """ best_ask = self.gmd(1)[0] if len(best_ask) == 0: return 0 else: return best_ask[0][1] @property def total_ask_size(self): return self._total_ask_size @property def bid_size(self): """ Volume waiting on bid side top level - liquidity level size for bid price """ best_bid = self.gmd(1)[1] if len(best_bid) == 0: return 0 else: return best_bid[0][1] @property def total_volume_traded(self): """ Total trad volume """ return self._total_volume_traded @property def total_volume_pending(self): """ Total size of orders in whole book """ return self._total_volume_pending @property def total_bid_size(self): return self._total_bid_size @property def ask(self): """ Best ask """ try: return self.gmd(1)[0][0][0] except: return -1 @property def bid(self): """ Best bid """ try: return self.gmd(1)[1][0][0] except: return -1 @property def spread(self): """ Difference between ask and bid """ return self.ask - self.bid def get_participant_orders(self, pi_d): """ Orders of given participant """ olst = self._participants.get_value(pi_d) order_prices = {} for order_id in olst: order = self._price_ids.get_value(order_id) if order[1] == 'ask': order_size = self._asks.get_value(order[0]).get_value(order_id) else: order_size = self._bids.get_value(order[0]).get_value(order_id) # price, side, size order_prices[order_id] = (order[0], order[1], order_size) return olst, order_prices def submit_order(self, order_type, side, size, price, pi_d): """ Abstraction on order placement - boht LMT and MKT """ if order_type == 'lmt': order_id = self._submit_lmt(side, size, price, pi_d) trades = self._balance([]) return order_id, trades if order_type == 'mkt': second_side_ask = 0 if side != 'ask': second_side_ask = self._total_ask_size else: second_side_ask = self._total_bid_size if second_side_ask >= size: return self._submit_mkt(side, size, pi_d) else: # Insufficient liquidity return -1, [] def gmd(self, depth): """ Liquidity levels size for both bid and ask """ ask_side = [] if not self._asks.is_empty(): for price in self._asks.keys(): ask_level = self._asks.get(price) ask_size = 0 for order_id in ask_level.keys(): # print(ask_size, order_id, ask_level.get(order_id)) ask_size += ask_level.get(order_id) ask_side.append([price, ask_size]) if len(ask_side) >= depth: break bid_side = [] if not self._bids.is_empty(): for price in self._bids.keys(reverse=True): bid_level = self._bids.get(price) bid_size = 0 for order_id in bid_level.keys(): bid_size += bid_level.get(order_id) bid_side.append([price, bid_size]) if len(bid_side) >= depth: break return [ask_side, bid_side]
class PriorityQueue(object): """ Combined priority queue and set data structure. Acts like a priority queue, except that its items are guaranteed to be unique. Provides O(1) membership test, O(log N) insertion and O(log N) removal of the smallest item. Important: the items of this data structure must be both comparable and hashable (i.e. must implement __cmp__ and __hash__). This is true of Python's built-in objects, but you should implement those methods if you want to use the data structure for custom objects. """ def __init__(self, items=[], key = None , maxitems=None, maxkey=None): """ Create a new PriorityQueueSet. items: An initial item list - it can be unsorted and non-unique. The data structure will be created in O(N). """ if key == None: self.key=lambda x: x else: self.key=key self.tree = FastAVLTree() #self.tree = AVLTree() self.maxitems = maxitems self.maxkey = maxkey for x in items: self.add(x) def has_item(self, item): """ Check if *item* exists in the queue """ return bool(self.tree.get(self.key(item), False)) def pop_smallest(self): return self.tree.pop_min() def peek(self, d = None): try: return self.tree.min_item()[1] except: return d def __setitem__(self, key, value): self.tree[self.key(key)]=value def __getitem__(self, item): return self.tree[self.key(item)] # updateing by removing and reinserting # i cant find a anode by object ?? # i hate your data structures ... index in O(n) :( def update(self, item): itemsbykey = self.tree[self.key(item):self.key(item)] del self.tree[self.key(item):self.key(item)] for x in itemsbykey: #if not (x is item): self.add(x) def add(self, item): """ Add *item* to the queue. The item will be added only if it doesn't already exist in the queue. """ #print "PriorityQue add " + str(item) if self.maxkey and self.key(item) > self.maxkey: return if self.tree.get(self.key(item), None) is None: self.tree[self.key(item)]=item # sholdnt it be pop biggest??? [yes we need a tree] if self.maxitems and self.tree.__len__() > self.maxitems: self.tree.pop_max() #print "PriorityQue add peek " + str(self.peek()) def prettyprint(self): pp = operator.methodcaller('prettyprint') return "".join(map(pp,self.tree.values())) """