def build_minimal_height_bst_bintrees(nums):
# here SelfBalancingBST would be your implementaion of BST with self balancing property.
# tree = SelfBalancingBST()
tree = RBTree()
for i in nums:
tree.insert(i, i)
return tree
def calc(arr, max_size):
import heapq;
print max_size
nxt = arr[:];
pos = {};
for i in range(len(arr)):
pos[arr[i]] = len(arr);
for i in reversed(range(len(arr))):
nxt[i] = pos[arr[i]];
pos[arr[i]] = i;
print nxt;
prio_queue = RBTree();
in_queue = {};
cache_miss = 0;
cur_in_queue_cnt = 0;
print arr;
for i in range(len(arr)):
out_str = str(i)+" ";
print prio_queue;
#cache missed
if in_queue.has_key(arr[i]) == False:
#while( space need):
out_str += "Missed: ";
cache_miss +=1;
#add into cache
if (cur_in_queue_cnt+1>max_size):
del in_queue[prio_queue.min_item()[1]];
cur_in_queue_cnt-=1;
out_str+= str(prio_queue.min_item()[1])+" is kicked. And ";
prio_queue.discard(prio_queue.min_key());
out_str+= str(arr[i])+" is added into Q.";
prio_queue.insert((-nxt[i],i),arr[i]);
cur_in_queue_cnt+=1;
in_queue[arr[i]] = 1;
else :
prio_queue.discard(prio_queue.ceiling_key((-i,0)));
prio_queue.insert((-nxt[i],i),arr[i]);
out_str += str(arr[i])+" hit.";
print out_str;
print cur_in_queue_cnt;
#print prio_queue.q;
print "Cache hit: "+str(len(arr)-cache_miss);
print "Cache miss: "+str(cache_miss);
print "Ratio: "+ str((len(arr)-cache_miss)*1.0/len(arr));
def median_maintenance_rbtree(arr):
median_sum = arr[0]
rbtree = RBTree([(arr[0], 1)]) # (number, subtree_size)
for i in range(1, len(arr)):
ele = arr[i]
rbtree.insert(ele, 1)
post_order_traverse(rbtree._root)
# assert rbtree._root.value == (i+1)
median_order = int(i / 2) + 1
median_sum += select_order_statisic(rbtree._root, median_order)
return median_sum
def __sweep_graph(self, graph):
tree = RBTree()
last_y = -1
for entry in graph.entries:
if entry.vertex.point.y != last_y:
self.stripes.append(Stripe(last_y, [edge for edge in tree]))
last_y = entry.vertex.point.y
for edge in entry.outbound_edges:
tree.insert(edge, edge)
for edge in entry.inbound_edges:
tree.remove(edge)
self.stripes.append(Stripe(last_y, []))
class BrownianVariableHistory(object):
''' Represents the set of known time value pairs for a particular Brownian variable '''
def __init__(self):
self._historyTree = RBTree()
def insertData(self, t, val):
'''
Inserts a data point into the history object
t (float) : time
val (float) : value
'''
self._historyTree.insert(t, val)
def getMartingaleRelevantPoints(self, t):
'''
Returns 2 data points. The first will be the data point with the
largest 't' in the history that is still smaller than the given user
provided argument 't'. The second will be the datapoint with the
smallest 't' that is still larger than the user provided 't'. If one
or both of the data points do not exist, this function will return
None in that data point's place.
t (float) : time
returns ((t1,val1), (t2,val2)) where t1, t2, val1, val2 are floats : 2 data points
Ex: bh.getMartingaleRelevantPoints(3.1) == ((3.0, 0.07), (3.5, 0.21))
bh.getMartingaleRelevantPoints(3.6) == ((3.5, 0.21), None)
'''
if self._historyTree.is_empty():
return None, None
leftPoint = None
rightPoint = None
if self._historyTree.min_key() <= t:
leftPoint = self._historyTree.floor_item(t)
if self._historyTree.max_key() >= t:
rightPoint = self._historyTree.ceiling_item(t)
return leftPoint, rightPoint
def max_sum_mod(array, size, m):
sum = [0] * size
# Első elem létrehozása, ez után ciklus kell
sum[0] = array[0] % m
RB = RBTree()
RB.insert(sum[0], sum[0]) #key, value páros
result = sum[0]
for i in range(1, size):
#flag azért kell, mert implementáció szerint ha egy elemnek nincs rákövetkezője,
#akkor kivételt dobna
flag = True
sum[i] = sum[i - 1] + array[i]
sum[i] %= m
try:
(k, v) = RB.ceiling_item(sum[i]) #következő elem
except KeyError as e:
#ha nincs rákövetkező akkor nem tudjuk a különbségét venni
print(e)
result = max(sum[i], result)
flag = False
if flag:
#pythonban a % művelet negatív számokra is úgy működik, ahogy azt elvárjuk
#C-ben például a sum[i] -k -hoz hozzá kéne adni az m-et.
result = max((sum[i] - k) % m, result)
#ez egy potenciális gyorsítás, ha elértük a maximális értéket (mod m), ami m-1
#akkor nincs értelme tovább keresni, kiléphetünk a ciklusból
if result == (m - 1):
break
RB.insert(sum[i], sum[i])
#RB.foreach(print, 0)
return result
def calculate_medians_with_rbt(numbers):
"""Calculate the sum of all 10,000 medians, modulo 10000"""
smaller = RBTree() # For storing the smaller half of numbers
larger = RBTree() # For storing the larger half of numbers
medians = []
for number in numbers:
if not len(smaller) or smaller.max_item()[0] > number:
smaller.insert(number, None)
if len(smaller) > len(larger) + 1:
larger.insert(smaller.max_item()[0], None)
else:
larger.insert(number, None)
if len(larger) > len(smaller) + 1:
smaller.insert(larger.min_item()[0], None)
if len(smaller) >= len(larger):
medians.append(smaller.max_item()[0])
else:
medians.append(larger.min_item()[0])
return medians
def __init__(self, rules):
t = RBTree()
## to avoid exceptions during LE lookup
t.insert(0, [])
nets = []
for rule in rules:
net = IPNetwork(rule.net)
t.insert(net.first, [])
t.insert(net.last+1, [])
for prio, rule in enumerate(rules):
net = IPNetwork(rule.net)
for k,v in t.iter_items(net.first, net.last+1):
v.append((prio, rule))
self.t = t
print(seen)
PrintUnique1(input) #1,5
#Code: Solution 2 (using using design #1 and red black tree)
'''
Why red and black trees are better than hashtables?
Source: http://www.quora.com/Why-would-anyone-like-to-use-a-red-black-tree-when-a-hash-table-can-do-the-job-perfectly
1. Simple memory management - simplifies error handling in concurrent code, less I/O hits,
2. Consistent performance because rehashing (expanding the hash table's array) happens on some insertions.
This is important in real-time systems where you want to provide bounds on how long each operation takes
3. Keys are sorted
'''
#https://bitbucket.org/mozman/bintrees
from bintrees import RBTree
def PrintUnique2(k,v):
if (v == 1): #unique
print(k) #print unique
#Initialize
tree = RBTree()
#Insert
for x in input:
if (tree.get(x)): #duplicate
tree.insert(x,tree.get(x)+1) #increment occurrence
else: #new
tree.insert(x,1)
#Iterate
tree.foreach(PrintUnique2,0) #0=inorder
class BitMEXBook:
def __init__(self,
endpoint="https://www.bitmex.com/api/v1",
symbol='XBTUSD'):
'''Connect to the websocket and initialize data stores.'''
self.logger = logging.getLogger(__name__)
self.logger.debug("Initializing WebSocket.")
self.endpoint = endpoint
self.symbol = symbol
self.data = {}
self.keys = {}
self.exited = False
self._asks = RBTree()
self._bids = RBTree()
# We can subscribe right in the connection querystring, so let's build that.
# Subscribe to all pertinent endpoints
wsURL = self.__get_url()
self.logger.debug("Connecting to %s" % wsURL)
self.__connect(wsURL, symbol)
self.logger.info('Connected to WS, waiting for partials.')
# Connected. Wait for partials
self.__wait_for_symbol(symbol)
self.logger.info('Got all market data. Starting.')
def init(self):
self.logger.debug("Initializing WebSocket...")
self.data = {}
self.keys = {}
self.exited = False
wsURL = self.__get_url()
self.logger.debug("Connecting to URL -- %s" % wsURL)
self.__connect(wsURL, self.symbol)
self.logger.info('Connected to WS, waiting for partials.')
# Connected. Wait for partials
self.__wait_for_symbol(self.symbol)
self.logger.info('Got all market data. Starting.')
def error(self, err):
self._error = err
self.logger.error(err)
#self.exit()
def __del__(self):
self.exit()
def reset(self):
self.logger.warning('Websocket resetting...')
self.ws.close()
self.logger.info('Weboscket closed.')
self.logger.info('Restarting...')
self.init()
def exit(self):
'''Call this to exit - will close websocket.'''
self.exited = True
self.ws.close()
### Main orderbook function
def get_current_book(self):
result = {'asks': [], 'bids': []}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'],
order['id']]) #(order['size']/Decimal(order['price']))
# Same procedure for bids
for bid in self._bids:
try:
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'],
order['id']]) #(order['size']/Decimal(order['price']))
return result
# -----------------------------------------------------------------------------------------
# ----------------------RBTrees Handling---------------------------------------------------
# -----------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------
# Get current minimum ask price from tree
def get_ask(self):
return self._asks.min_key()
# Get ask given id
def get_asks(self, id):
return self._asks.get(id)
# Remove ask form tree
def remove_asks(self, id):
self._asks.remove(id)
# Insert ask into tree
def set_asks(self, id, asks):
self._asks.insert(id, asks)
# Get current maximum bid price from tree
def get_bid(self):
return self._bids.max_key()
# Get bid given id
def get_bids(self, id):
return self._bids.get(id)
# Remove bid form tree
def remove_bids(self, id):
self._bids.remove(id)
# Insert bid into tree
def set_bids(self, id, bids):
self._bids.insert(id, bids)
# Add order to out watched orders
def add(self, order):
order = {
'id': order['id'], # Order id data
'side': order['side'], # Order side data
'size': Decimal(order['size']), # Order size data
'price': order['price'] # Order price data
}
if order['side'] == 'Buy':
bids = self.get_bids(order['id'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['id'], bids)
else:
asks = self.get_asks(order['id'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['id'], asks)
# Order is done, remove it from watched orders
def remove(self, order):
oid = order['id']
if order['side'] == 'Buy':
bids = self.get_bids(oid)
if bids is not None:
bids = [o for o in bids if o['id'] != order['id']]
if len(bids) > 0:
self.set_bids(oid, bids)
else:
self.remove_bids(oid)
else:
asks = self.get_asks(oid)
if asks is not None:
asks = [o for o in asks if o['id'] != order['id']]
if len(asks) > 0:
self.set_asks(oid, asks)
else:
self.remove_asks(oid)
# Updating order price and size
def change(self, order):
new_size = Decimal(order['size'])
# Bitmex updates don't come with price, so we use the id to match it instead
oid = order['id']
if order['side'] == 'Buy':
bids = self.get_bids(oid)
if bids is None or not any(o['id'] == order['id'] for o in bids):
return
index = list(map(itemgetter('id'), bids)).index(order['id'])
bids[index]['size'] = new_size
self.set_bids(oid, bids)
else:
asks = self.get_asks(oid)
if asks is None or not any(o['id'] == order['id'] for o in asks):
return
index = list(map(itemgetter('id'), asks)).index(order['id'])
asks[index]['size'] = new_size
self.set_asks(oid, asks)
tree = self._asks if order['side'] == 'Sell' else self._bids
node = tree.get(oid)
if node is None or not any(o['id'] == order['id'] for o in node):
return
# -----------------------------------------------------------------------------------------
# ----------------------WS Private Methods-------------------------------------------------
# -----------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------
def __connect(self, wsURL, symbol):
'''Connect to the websocket in a thread.'''
self.logger.debug("Starting thread")
self.ws = websocket.WebSocketApp(wsURL,
on_message=self.__on_message,
on_close=self.__on_close,
on_open=self.__on_open,
on_error=self.__on_error)
self.wst = threading.Thread(target=lambda: self.ws.run_forever())
self.wst.daemon = True
self.wst.start()
self.logger.debug("Started thread")
# Wait for connect before continuing
conn_timeout = CONN_TIMEOUT
while (not self.ws.sock
or not self.ws.sock.connected) and conn_timeout:
sleep(1)
conn_timeout -= 1
if not conn_timeout:
self.logger.error("Couldn't connect to WS! Exiting.")
# self.exit()
# raise websocket.WebSocketTimeoutException('Couldn\'t connect to WS! Exiting.')
self.reset()
def __get_url(self):
'''
Generate a connection URL. We can define subscriptions right in the querystring.
Most subscription topics are scoped by the symbol we're listening to.
'''
symbolSubs = ["orderBookL2"]
subscriptions = [sub + ':' + self.symbol for sub in symbolSubs]
urlParts = list(urllib.parse.urlparse(self.endpoint))
urlParts[0] = urlParts[0].replace('http', 'ws')
urlParts[2] = "/realtime?subscribe={}".format(','.join(subscriptions))
return urllib.parse.urlunparse(urlParts)
def __wait_for_symbol(self, symbol):
'''On subscribe, this data will come down. Wait for it.'''
pbar = tqdm(total=160)
# Wait until data reaches our RBTrees
while self._asks.is_empty() and self._bids.is_empty():
sleep(0.1)
pbar.update(3)
pbar.close()
def __send_command(self, command, args=None):
'''Send a raw command.'''
if args is None:
args = []
self.ws.send(json.dumps({"op": command, "args": args}))
def __on_message(self, message):
'''Handler for parsing WS messages.'''
message = json.loads(message)
# self.logger.debug(json.dumps(message))
table = message['table'] if 'table' in message else None
action = message['action'] if 'action' in message else None
# table = message.get("table")
# action = message.get("action")
try:
# RBTrees for orderBook
if table == 'orderBookL2':
# For every order received
try:
for order in message['data']:
if action == 'partial':
self.logger.debug('%s: adding partial %s' %
(table, order))
self.add(order)
elif action == 'insert':
self.logger.debug('%s: inserting %s' %
(table, order))
self.add(order)
elif action == 'update':
self.logger.debug('%s: updating %s' %
(table, order))
self.change(order)
elif action == 'delete':
self.logger.debug('%s: deleting %s' %
(table, order))
self.remove(order)
else:
raise Exception("Unknown action: %s" % action)
except:
self.logger.error('Error handling RBTrees: %s' %
traceback.format_exc())
# Uncomment this to watch RBTrees evolution in real time
# self.logger.info('==============================================================')
# self.logger.info('=============================ASKS=============================')
# self.logger.info('==============================================================')
# self.logger.info(self._asks)
# self.logger.info('==============================================================')
# self.logger.info('=============================BIDS=============================')
# self.logger.info('==============================================================')
# self.logger.info(self._bids)
except:
self.logger.error(traceback.format_exc())
def __on_error(self, error):
'''Called on fatal websocket errors. We exit on these.'''
if not self.exited:
self.logger.error("Error : %s" % error)
raise websocket.WebSocketException(error)
def __on_open(self):
'''Called when the WS opens.'''
self.logger.debug("Websocket Opened.")
def __on_close(self):
'''Called on websocket close.'''
self.logger.info('Websocket Closed')
class GdaxBookFeed(GdaxWebsocketClient):
def __init__(self,
product_id='LTC-USD',
log_to=None,
gdax=None,
auth=True):
if gdax is None:
from stocklook.crypto.gdax.api import Gdax
gdax = Gdax()
super(GdaxBookFeed, self).__init__(products=product_id,
auth=auth,
api_key=gdax.api_key,
api_secret=gdax.api_secret,
api_passphrase=gdax.api_passphrase)
self._asks = None
self._bids = None
self._client = gdax
self._sequence = -1
self._log_to = log_to
if self._log_to:
assert hasattr(self._log_to, 'write')
self._current_ticker = None
self._key_errs = 0
self._errs = 0
self.message_count = 0
@property
def product_id(self):
'''
Currently OrderBook only supports a single product even
though it is stored as a list of products.
'''
return self.products[0]
def on_message(self, message):
self.message_count += 1
if self._log_to:
pickle.dump(message, self._log_to)
try:
sequence = message['sequence']
except KeyError:
print("Error: {}".format(message))
self._key_errs += 1
if self._key_errs >= 3:
print("3 errors retrieving sequence. Restarting....")
self.close()
self._key_errs = 0
self.start()
return
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.get_book(self.product_id, level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': float(bid[0]),
'size': float(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': float(ask[0]),
'size': float(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book
# initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
print('Error: messages missing ({} - {}). '
'Re-initializing websocket.'.format(sequence,
self._sequence))
self.close()
self.start()
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
self._current_ticker = message
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
# bid = self.get_bid()
# bids = self.get_bids(bid)
# bid_depth = sum([b['size'] for b in bids])
# ask = self.get_ask()
# asks = self.get_asks(ask)
# ask_depth = sum([a['size'] for a in asks])
# print('bid: %f @ %f - ask: %f @ %f' % (bid_depth, bid, ask_depth, ask))
def on_error(self, e):
self._sequence = -1
self._errs += 1
self.close()
if self._errs >= 3:
raise Exception(e)
self.start()
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': float(order['price']),
'size': float(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = float(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = float(order['size'])
price = float(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = float(order['new_size'])
except KeyError:
return
price = float(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here,
# where a price point is removed
# between these two ops.
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
bit = [order['price'], order['size'], order['id']]
result['asks'].append(bit)
for bid in self._bids:
try:
# There can be a race condition here,
# where a price point is removed
# between these two ops.
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'], order['id']])
return result
def get_orders_matching_ids(self, order_ids):
return [a for a in self._asks
if a['id'] in order_ids] +\
[b for b in self._bids
if b['id'] in order_ids
]
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class Dijkstra:
def __init__(self, edgesDict, uToV, origNode, destNode, estimationModel):
#self.pointList = []
self.passedNodesSet = set()
self.notPassedNodeDict = dict()
#self.notPassedNodeHeapq = []
#self.notPassedNodeDict = dict()
self.edgesDict = edgesDict
self.uToV = uToV
self.dummyWindow = Window(-1, -1, -1, -1)
self.origNode = origNode
self.destNode = destNode
self.estimationModel = estimationModel
self.dummyOriNodeInPathGraph = self.generateDummyOirNode()
self.notPassedNodeDict[self.dummyOriNodeInPathGraph] = 0
self.dummyDestNodeInPathGraph = self.generateDummyDestNode()
self.notPassedNodeQ = RBTree()
self.notPassedNodeQ.insert(self.dummyOriNodeInPathGraph, 0)
listOfNodesFromOrig = self.uToV[self.origNode]
self.initializeQ(listOfNodesFromOrig)
if not len(listOfNodesFromOrig):
print("not path from node:", self.origNode)
self.__initializedStatus = False
else:
self.__initializedStatus = True
def generateDummyOirNode(self):
return NodeInPathGraph(self.dummyWindow, self.origNode, None, -1, 0)
def generateDummyDestNode(self):
return NodeInPathGraph(self.dummyWindow, -1, None, -1, inf)
def initializeQ(self, listOfNodesFromOrig):
# print(edgesGdfFromNode.iloc[0])
for edgeIdAndV in listOfNodesFromOrig:
edgeIdInGdf = edgeIdAndV[0]
nextNodeId = edgeIdAndV[1]
nextWindow = Window(self.dummyWindow.prevSeg,
self.dummyWindow.midSeg,
self.dummyWindow.sucSeg, edgeIdInGdf)
nextNode = NodeInPathGraph(nextWindow, nextNodeId,
self.dummyOriNodeInPathGraph,
edgeIdInGdf, 0)
self.notPassedNodeQ.insert(nextNode, 0)
self.notPassedNodeDict[nextNode] = 0
return
def routing(self):
steps = 0
if self.checkIniStatus():
while not self.ifFinished():
if self.noNodeToExplore():
print("No route")
return
else:
steps += 1
self.onePaceUpdate()
pathWitMinVal = self.generateMinValNodePath()
edgePathWithMinVal = self.generateMinValEdgePath()
print("num of steps:", steps)
return pathWitMinVal, self.minVal, edgePathWithMinVal
return
def checkIniStatus(self):
return self.__initializedStatus
def ifFinished(self):
return self.dummyDestNodeInPathGraph in self.passedNodesSet
def noNodeToExplore(self):
return self.notPassedNodeQ.is_empty()
def onePaceUpdate(self):
curNodeInPathGraph, valOfCurNode = self.notPassedNodeQ.min_item()
self.notPassedNodeQ.remove(curNodeInPathGraph)
_ = self.notPassedNodeDict.pop(curNodeInPathGraph)
self.passedNodesSet.add(curNodeInPathGraph)
if self.isDestNode(curNodeInPathGraph):
self.minVal = valOfCurNode
self.destNodeGenerated = curNodeInPathGraph
return
self.exploreNextNode(curNodeInPathGraph, valOfCurNode)
def isDestNode(self, curNodeInPathGraph):
return curNodeInPathGraph == self.dummyDestNodeInPathGraph
def exploreNextNode(self, curNodeInPathGraph, valOfCurNode):
if self.isNoSucNode(curNodeInPathGraph):
self.exploreDummySucNode(curNodeInPathGraph, valOfCurNode)
else:
self.exploreNextNodeFromEdge(curNodeInPathGraph, valOfCurNode)
return
def isNoSucNode(self, curNodeInPathGraph):
return curNodeInPathGraph.node == -1 or curNodeInPathGraph.node == self.destNode
def exploreDummySucNode(self, curNodeInPathGraph, valOfCurNode):
nextNodeId = -1
nextWindow = Window(curNodeInPathGraph.window.prevSeg,
curNodeInPathGraph.window.midSeg,
curNodeInPathGraph.window.sucSeg, -1)
nextNodes = NodeInPathGraph(nextWindow, nextNodeId, curNodeInPathGraph,
-1)
valOfNextNode = self.calVal(nextNodes)
self.updateQ(nextNodes, valOfCurNode + valOfNextNode)
return
def exploreNextNodeFromEdge(self, curNodeInPathGraph, valOfCurNode):
listOfNodes = self.uToV[curNodeInPathGraph.node]
for edgeIdAndV in listOfNodes:
edgeIdInGdf = edgeIdAndV[0]
nextNodeId = edgeIdAndV[1]
nextWindow = Window(curNodeInPathGraph.window.prevSeg,
curNodeInPathGraph.window.midSeg,
curNodeInPathGraph.window.sucSeg, edgeIdInGdf)
nextNodeInPathGraph = NodeInPathGraph(nextWindow, nextNodeId,
curNodeInPathGraph,
edgeIdInGdf)
if nextWindow.valid(
) and nextNodeInPathGraph not in self.passedNodesSet:
valOfNextNode = self.calVal(nextNodeInPathGraph)
self.updateQ(nextNodeInPathGraph, valOfNextNode + valOfCurNode)
def updateQ(self, nextNodeInPathGraph, curShortPathEst):
if nextNodeInPathGraph not in self.notPassedNodeDict:
nextNodeInPathGraph.shortPathEst = curShortPathEst
self.notPassedNodeQ.insert(nextNodeInPathGraph,
nextNodeInPathGraph.shortPathEst)
self.notPassedNodeDict[nextNodeInPathGraph] = curShortPathEst
else:
nextNodeInPathGraph.shortPathEst = self.notPassedNodeDict[
nextNodeInPathGraph]
if curShortPathEst < nextNodeInPathGraph.shortPathEst:
self.notPassedNodeQ.remove(nextNodeInPathGraph)
nextNodeInPathGraph.shortPathEst = curShortPathEst
self.notPassedNodeQ.insert(nextNodeInPathGraph,
nextNodeInPathGraph.shortPathEst)
self.notPassedNodeDict[nextNodeInPathGraph] = curShortPathEst
def calVal(self, curNodeInPathGraph):
if curNodeInPathGraph.window.midSeg == -1:
return 0
else:
numericalFeatures, categoricalFeatures = curNodeInPathGraph.window.extractFeatures(
self.edgesDict)
return self.estimationModel.predictFromData(
numericalFeatures, categoricalFeatures)
def generateMinValNodePath(self):
dNode = self.destNodeGenerated
pathWitMinVal = [dNode.node]
while dNode.prevNode:
dNode = dNode.prevNode
pathWitMinVal.append(dNode.node)
return pathWitMinVal[:2:-1]
def generateMinValEdgePath(self):
dNode = self.destNodeGenerated
edgePathWithMinVal = [dNode.edge]
while dNode.prevNode:
dNode = dNode.prevNode
edgePathWithMinVal.append(dNode.edge)
return edgePathWithMinVal[-2:2:-1]
class OrderBook(object):
def __init__(self, product_id, log_to=None):
self._asks = RBTree()
self._bids = RBTree()
self._client = PublicClient()
self._sequence = -1
self.sync = -1
self.product_id = product_id
self._log_to = log_to
if self._log_to:
assert hasattr(self._log_to, 'write')
self._current_ticker = None
def on_open(self):
self._sequence = -1
#print("-- Subscribed to " + self.product_id + " OrderBook! --\n")
def on_close(self):
print("\n-- OrderBook " + self.product_id + " Socket Closed! --")
def reset_book(self):
self._asks = RBTree()
self._bids = RBTree()
res = self._client.get_product_order_book(product_id=self.product_id,
level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
def on_message(self, message):
if self._log_to:
pickle.dump(message, self._log_to)
sequence = message['sequence']
if self._sequence == -1:
self.sync = -1
self.reset_book()
return
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
self.on_sequence_gap(self._sequence, sequence)
self.sync = -1
return
self.sync = 1
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
self._current_ticker = message
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
def on_sequence_gap(self, gap_start, gap_end):
self.reset_book()
print(
'Error: messages missing ({} - {}). Re-initializing book at sequence.'
.format(gap_start, gap_end, self._sequence))
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
try:
price = Decimal(order['price'])
except KeyError:
return
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_ask_size(self):
return sum(order['size'] for order in self._asks.min_item()[1])
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bid_size(self):
return sum(order['size'] for order in self._bids.min_item()[1])
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class CompletedKeys(object):
def __init__(self, max_index, min_index=0):
self._max_index = max_index
self._min_index = min_index
self.num_remaining = max_index - min_index
self._slabs = RBTree()
def _get_previous_or_none(self, index):
try:
return self._slabs.floor_item(index)
except KeyError:
return None
def is_available(self, index):
logger.debug("Testing index %s", index)
if index >= self._max_index or index < self._min_index:
logger.debug("Index out of range")
return False
try:
prev_start, prev_length = self._slabs.floor_item(index)
logger.debug("Prev range: %s-%s", prev_start,
prev_start + prev_length)
return (prev_start + prev_length) <= index
except KeyError:
return True
def mark_completed(self, start_index, past_last_index):
logger.debug("Marking the range completed: %s-%s", start_index,
past_last_index)
num_completed = min(past_last_index, self._max_index) - max(
start_index, self._min_index)
# Find the item directly before this and see if there is overlap
to_discard = set()
try:
prev_start, prev_length = self._slabs.floor_item(start_index)
max_prev_completed = prev_start + prev_length
if max_prev_completed >= start_index:
# we are going to merge with the range before us
logger.debug("Merging with the prev range: %s-%s", prev_start,
prev_start + prev_length)
to_discard.add(prev_start)
num_completed = max(
num_completed - (max_prev_completed - start_index), 0)
start_index = prev_start
past_last_index = max(past_last_index,
prev_start + prev_length)
except KeyError:
pass
# Find all keys between the start and last index and merge them into one block
for merge_start, merge_length in self._slabs.iter_items(
start_index, past_last_index + 1):
if merge_start in to_discard:
logger.debug("Already merged with block %s-%s", merge_start,
merge_start + merge_length)
continue
candidate_next_index = merge_start + merge_length
logger.debug("Merging with block %s-%s", merge_start,
candidate_next_index)
num_completed -= merge_length - max(
candidate_next_index - past_last_index, 0)
to_discard.add(merge_start)
past_last_index = max(past_last_index, candidate_next_index)
# write the new block which is fully merged
discard = False
if past_last_index >= self._max_index:
logger.debug("Discarding block and setting new max to: %s",
start_index)
self._max_index = start_index
discard = True
if start_index <= self._min_index:
logger.debug("Discarding block and setting new min to: %s",
past_last_index)
self._min_index = past_last_index
discard = True
if to_discard:
logger.debug("Discarding %s obsolete blocks", len(to_discard))
self._slabs.remove_items(to_discard)
if not discard:
logger.debug("Writing new block with range: %s-%s", start_index,
past_last_index)
self._slabs.insert(start_index, past_last_index - start_index)
# Update the number of remaining items with the adjustments we've made
assert num_completed >= 0
self.num_remaining -= num_completed
logger.debug("Total blocks: %s", len(self._slabs))
def get_block_start_index(self, block_size_estimate):
logger.debug("Total range: %s-%s", self._min_index, self._max_index)
if self._max_index <= self._min_index:
raise NoAvailableKeysError(
"All indexes have been marked completed")
num_holes = len(self._slabs) + 1
random_hole = random.randint(0, num_holes - 1)
logger.debug("Selected random hole %s with %s total holes",
random_hole, num_holes)
hole_start = self._min_index
past_hole_end = self._max_index
# Now that we have picked a hole, we need to define the bounds
if random_hole > 0:
# There will be a slab before this hole, find where it ends
bound_entries = self._slabs.nsmallest(random_hole + 1)[-2:]
left_index, left_len = bound_entries[0]
logger.debug("Left range %s-%s", left_index, left_index + left_len)
hole_start = left_index + left_len
if len(bound_entries) > 1:
right_index, right_len = bound_entries[1]
logger.debug("Right range %s-%s", right_index,
right_index + right_len)
past_hole_end, _ = bound_entries[1]
elif not self._slabs.is_empty():
right_index, right_len = self._slabs.nsmallest(1)[0]
logger.debug("Right range %s-%s", right_index,
right_index + right_len)
past_hole_end, _ = self._slabs.nsmallest(1)[0]
# Now that we have our hole bounds, select a random block from [0:len - block_size_estimate]
logger.debug("Selecting from hole range: %s-%s", hole_start,
past_hole_end)
rand_max_bound = max(hole_start, past_hole_end - block_size_estimate)
logger.debug("Rand max bound: %s", rand_max_bound)
return random.randint(hole_start, rand_max_bound)
class MarinerOrderBook(gdax.OrderBook):
current_milli_time = lambda self: int(time.time() * 1000)
def __init__(self, ticker, threshold):
gdax.OrderBook.__init__(self, product_id=ticker)
self._threshold = threshold
self.bookChanged = None
self.whaleEnteredMarket = None
self.whaleExitedMarket = None
def registerHandlers(self, bookChangedHandler, whaleEnteredMarketHandler,
whaleExitedMarketHandler, whaleChangedHandler):
Logging.logger.info("registering callbacks...")
self.bookChanged = bookChangedHandler
self.whaleEnteredMarket = whaleEnteredMarketHandler
self.whaleExitedMarket = whaleExitedMarketHandler
self.whaleChanged = whaleChangedHandler
def onMessage(self, message):
#Logging.logger.info(message)
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.getProductOrderBook(level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
Logging.logger.error(
'Error: messages missing ({} - {}). Re-initializing websocket.'
.format(sequence, self._sequence))
self.close()
self.start()
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
if not self.bookChanged is None:
self.bookChanged()
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
if not self.whaleEnteredMarket == None and self.isWhale(order['size']):
self.whaleEnteredMarket(order)
def remove(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
if not self.whaleExitedMarket == None and self.isWhale(order['size']):
self.whaleExitedMarket(order)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
price = Decimal(order['price'])
new_volume = Decimal(order['new_size'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = map(itemgetter('id'), bids).index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = map(itemgetter('id'), asks).index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
if not self.whaleChanged == None and self.isWhale(order['size']):
self.whaleChanged(order)
def get_current_book(self,
num_levels=10
): #fetch only 100 levels off book either way
book = []
for index, bid_entry in enumerate(self._bids.items([True])):
if index == num_levels:
break
else:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
thisBid = bid_entry[1]
except KeyError:
continue
for order in thisBid:
order['price'] = Decimal128(order['price'])
order['size'] = Decimal128(order['size'])
book.append(order)
for index, ask_entry in enumerate(self._asks.items()):
if index == num_levels:
break
else:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
thisAsk = ask_entry[1]
except KeyError:
continue
for order in thisAsk:
order['price'] = Decimal128(order['price'])
order['size'] = Decimal128(order['size'])
book.append(order)
book_frame = pd.DataFrame(book)
return book_frame
def isWhale(self, aVolume):
return aVolume >= self._threshold
#For general book management purposes
def get_bid(self):
if not self._bids.is_empty():
return self._bids.max_key()
else:
return None
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
def get_ask(self):
if not self._asks.is_empty():
return self._asks.min_key()
else:
return None
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
class OrderBook(WebsocketClient):
def __init__(self, product_id='BTC-USD'):
WebsocketClient.__init__(self, products=product_id)
self._asks = RBTree()
self._bids = RBTree()
self._client = PublicClient(product_id=product_id)
self._sequence = -1
def onMessage(self, message):
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.getProductOrderBook(level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': float(bid[0]),
'size': float(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': float(ask[0]),
'size': float(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
return
elif sequence > self._sequence + 1:
self.close()
self.start()
return
# print(message)
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
# bid = self.get_bid()
# bids = self.get_bids(bid)
# bid_depth = sum([b['size'] for b in bids])
# ask = self.get_ask()
# asks = self.get_asks(ask)
# ask_depth = sum([a['size'] for a in asks])
# print('bid: %f @ %f - ask: %f @ %f' % (bid_depth, bid, ask_depth, ask))
def add(self, order):
order = {
'id':
order['order_id'] if 'order_id' in order else order['id'],
'side':
order['side'],
'price':
float(order['price']),
'size':
float(order['size'])
if 'size' in order else float(order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = float(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = float(order['size'])
price = float(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
new_size = float(order['new_size'])
price = float(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = map(itemgetter('id'), bids).index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = map(itemgetter('id'), asks).index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class OrderBook(WebsocketClient):
def __init__(self, product_id='BTC-USD'):
super(OrderBook, self).__init__(products=product_id)
self._asks = RBTree()
self._bids = RBTree()
self._client = PublicClient(product_id=product_id)
self._sequence = -1
def on_message(self, message):
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.get_product_order_book(level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
print('Error: messages missing ({} - {}). Re-initializing websocket.'.format(sequence, self._sequence))
self.close()
self.start()
return
# print(message)
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
# bid = self.get_bid()
# bids = self.get_bids(bid)
# bid_depth = sum([b['size'] for b in bids])
# ask = self.get_ask()
# asks = self.get_asks(ask)
# ask_depth = sum([a['size'] for a in asks])
# print('bid: %f @ %f - ask: %f @ %f' % (bid_depth, bid, ask_depth, ask))
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
new_size = Decimal(order['new_size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id'] for o in bids):
return
index = map(itemgetter('id'), bids).index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id'] for o in asks):
return
index = map(itemgetter('id'), asks).index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append([order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append([order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class OrderTree(object):
'''A red-black tree used to store std::list<std::shared_ptr<Order>>s in price order
The exchange will be using the OrderTree to hold bid and ask data (one OrderTree for each side).
Keeping the information in a red black tree makes it easier/faster to detect a match.
'''
def __init__(self):
self.price_tree = RBTree()
self.price_map = {} # Dictionary containing price : std::list<std::shared_ptr<Order>> object
self.order_map = {} # Dictionary containing order_id : Order object
self.volume = 0 # Contains total quantity from all Orders in tree
self.num_orders = 0 # Contains count of Orders in tree
self.depth = 0 # Number of different prices in tree (http://en.wikipedia.org/wiki/Order_book_(trading)#Book_depth)
def __len__(self):
return len(self.order_map)
def get_price_list(self, price):
return self.price_map[price]
def get_order(self, order_id):
return self.order_map[order_id]
def create_price(self, price):
self.depth += 1 # Add a price depth level to the tree
new_list = std::list<std::shared_ptr<Order>>()
self.price_tree.insert(price, new_list) # Insert a new price into the tree
self.price_map[price] = new_list # Can i just get this by using self.price_tree.get_value(price)? Maybe this is faster though.
def remove_price(self, price):
self.depth -= 1 # Remove a price depth level
self.price_tree.remove(price)
del self.price_map[price]
def price_exists(self, price):
return price in self.price_map
def order_exists(self, order):
return order in self.order_map
def insert_order(self, quote):
if self.order_exists(quote['order_id']):
self.remove_order_by_id(quote['order_id'])
self.num_orders += 1
if quote['price'] not in self.price_map:
self.create_price(quote['price']) # If price not in Price Map, create a node in RBtree
order = Order(quote, self.price_map[quote['price']]) # Create an order
self.price_map[order.price].append_order(order) # Add the order to the std::list<std::shared_ptr<Order>> in Price Map
self.order_map[order.order_id] = order
self.volume += order.quantity
def update_order(self, order_update):
order = self.order_map[order_update['order_id']]
original_quantity = order.quantity
if order_update['price'] != order.price:
# Price changed. Remove order and update tree.
order_list = self.price_map[order.price]
order_list.remove_order(order)
if len(order_list) == 0: # If there is nothing else in the std::list<std::shared_ptr<Order>>, remove the price from RBtree
self.remove_price(order.price)
self.insert_order(order_update)
else:
# Quantity changed. Price is the same.
order.update_quantity(order_update['quantity'], order_update['timestamp'])
self.volume += order.quantity - original_quantity
def remove_order_by_id(self, order_id):
self.num_orders -= 1
order = self.order_map[order_id]
self.volume -= order.quantity
order.order_list.remove_order(order)
if len(order.order_list) == 0:
self.remove_price(order.price)
del self.order_map[order_id]
def max_price(self):
if self.depth > 0:
return self.price_tree.max_key()
else:
return None
def min_price(self):
if self.depth > 0:
return self.price_tree.min_key()
else:
return None
def max_price_list(self):
if self.depth > 0:
return self.get_price_list(self.max_price())
else:
return None
def min_price_list(self):
if self.depth > 0:
return self.get_price_list(self.min_price())
else:
return None
class OrderBook():
def __init__(self, market=None, bids=None, asks=None, log_to=None):
self._asks = RBTree()
self._bids = RBTree()
self.book_valid = False
self.new_book(bids, asks)
self._sequence = -1
self.market = market
self._log_to = log_to
if self._log_to:
assert hasattr(self._log_to, 'write')
# My order Details
self.total_order_count = 0
self.total_open_order_count = 0
self.pending_buy_orders_db = {}
self.pending_sell_orders_db = {}
self.traded_buy_orders_db = {}
self.traded_sell_orders_db = {}
# positions
self.all_positions = []
self.open_positions = []
self.close_pending_positions = {}
self.closed_positions = []
self.orderDb = db.OrderDb(Order, market.exchange_name,
market.product_id)
self.positionsDb = db.PositionDb(Position, market.exchange_name,
market.product_id)
#stop loss handling
self.sl_dict = sorteddict.SortedDict()
#take profit handling
self.tp_dict = sorteddict.SortedDict()
#trade Reqs
self.pending_trade_req = [
] # TODO: FIXME: jork: this better be a nice AVL tree of sort
def __str__(self):
return """
{"position_all": %d, "open": %d, "close_pending": %d, "closed": %d}""" % (
len(self.all_positions), len(self.open_positions),
len(self.close_pending_positions), len(self.closed_positions))
def add_pending_trade_req(self, trade_req):
self.pending_trade_req.append(trade_req)
def remove_pending_trade_req(self, trade_req):
# primitive
self.pending_trade_req.remove(trade_req)
def open_position(self, buy_order):
#open positions with buy orders only(we don't support 'short' now)
# log.debug("open_position order: %s"%(buy_order))
position = Position(id=buy_order.id)
position.add_buy(buy_order)
if self.market.tradeConfig["stop_loss_enabled"]:
self.add_stop_loss_position(position, buy_order.get_price(),
self.market, buy_order.stop)
if self.market.tradeConfig["take_profit_enabled"]:
self.add_take_profit_position(position, buy_order.get_price(),
self.market, buy_order.profit)
self.all_positions.append(position)
self.open_positions.append(position)
# log.debug("\n\n\n***open_position: open(%d) closed(%d) close_pend(%d)"%(len(self.open_positions), len(self.closed_positions), len(self.close_pending_positions)))
# save pos to db
self.positionsDb.db_save_position(position)
def get_closable_position(self):
log.info("get_closable_position ")
#get last open position for now
# TODO: FIXME: This may not be the best way. might cause race with below api with multi thread/multi exch
pos = None
if len(self.open_positions):
if self.market.tradeConfig["stop_loss_enabled"]:
log.info("Finding closable position from _stop_loss_ pool.")
pos = self.pop_stop_loss_position()
if pos == None:
try:
pos = self.open_positions.pop()
except IndexError:
log.error("unable to find open position to close")
return None
log.info(
"Found closable position from _regular_ pool. pos: %s" %
(str(pos)))
else:
log.info(
"Found closable position from _stop_loss_ pool. pos: %s" %
(str(pos)))
self.open_positions.remove(pos)
if (self.close_pending_positions.get(pos.id)):
log.critical("""Position already close pending \npos:%s
close_pending_positions: %s
open_positions: %s""" %
(str(pos), str(self.close_pending_positions),
str(self.open_positions)))
raise Exception("Duplicate close pending position")
if self.market.tradeConfig["take_profit_enabled"]:
self.pop_take_profit_position(pos)
self.close_pending_positions[pos.id] = pos
# log.debug("\n\n\n***get_closable_position: open(%d) closed(%d) close_pend(%d) \n pos: %s"%(
# len(self.open_positions), len(self.closed_positions), len(self.close_pending_positions), pos))
return pos
def close_position_pending(self, sell_order):
# Intentionally _not_ updating the pos_db here. Yes, There is a case of wrong pos state if we go down come back during this state
# TODO: FIXME: This may not be the best way. might cause race with below api with multi thread/multi exch
log.info(" order:%s" % (sell_order.id))
# log.debug("\n\n\n***: open(%d) closed(%d) close_pend(%d)\n"%(
# len(self.open_positions), len(self.closed_positions), len(self.close_pending_positions)))
pos = self.close_pending_positions.get(sell_order.id)
if pos:
log.info(": sell order(%s) already in pending_list. do nothing" %
(sell_order.id))
return pos
#find a close_pending pos without sell attached.
k = sell_order._pos_id
if not k:
log.critical(
"*****Invalid pos_id attached to order:%s**** \n may be an outside order"
% (sell_order.id))
raise Exception("Invalid pos_id attached to order")
pos = self.close_pending_positions.get(k)
if pos:
#find the pos without sell attached. and reinsert after attach
# log.debug("pos:\n%s"%(pos))
if pos.sell == None:
pos.add_sell(sell_order)
pos.update_state("close_pending")
del (self.close_pending_positions[k])
self.close_pending_positions[sell_order.id] = pos
# log.debug("\n\n\n***: open(%d) closed(%d) close_pend(%d)"%(len(self.open_positions), len(self.closed_positions), len(self.close_pending_positions)))
return pos
else:
log.critical("Wrong sell attached to pos:%s" % (pos))
raise Exception("Wrong sell attached to pos")
else:
#something is very wrong
log.critical("Unable to find pending position for close id: %s" %
(sell_order.id))
raise Exception("Unable to find pending position for close")
def close_position_failed(self, pos_id):
log.info("close_position_failed order: %s" % (pos_id))
# log.debug("\n\n\n***close_position_failed: open(%d) closed(%d) close_pend(%d)"%(len(self.open_positions), len(self.closed_positions), len(self.close_pending_positions)))
# id = sell_order.id
position = self.close_pending_positions.get(pos_id)
if position:
position.sell = None
self.close_pending_positions.pop(pos_id, None)
self.open_positions.append(position)
if self.market.tradeConfig["stop_loss_enabled"]:
self.add_stop_loss_position(position, position.buy.get_price(),
self.market,
position.get_stop_loss())
if self.market.tradeConfig["take_profit_enabled"]:
self.add_take_profit_position(position,
position.buy.get_price(),
self.market,
position.get_take_profit())
else:
log.critical("Unable to get close_pending position. order_id: %s" %
(pos_id))
def close_position(self, sell_order):
log.info("close_position order: %s" % (sell_order.id))
id = sell_order.id
position = self.close_pending_positions.pop(id, None)
if position:
position.add_sell(sell_order)
position.update_state("closed")
profit = position.get_profit()
self.market.fund.current_realized_profit += profit
if profit > 0:
self.market.num_success_trade += 1
else:
self.market.num_failed_trade += 1
self.closed_positions.append(position)
#update position
self.positionsDb.db_save_position(position)
log.info("position closed pos: %s" % (str(position)))
else:
log.critical("Unable to get close_pending position. order_id: %s" %
(sell_order.id))
# log.debug("\n\n\n***close_position: open(%d) closed(%d) close_pend(%d)\n pos:%s"%(
# len(self.open_positions), len(self.closed_positions), len(self.close_pending_positions), position))
def add_stop_loss_position(self,
position,
market_rate,
market,
stop_price=0):
sl_kind = market.tradeConfig['stop_loss_kind']
if sl_kind in ['trailing', 'simple']:
sl_rate = market.tradeConfig["stop_loss_rate"]
stop_price = float(
round(market_rate * (1 - sl_rate * float(.01)), 4))
elif 'ATR' in sl_kind:
atr = market.get_cur_indicators()[
market.tradeConfig['stop_loss_kind']]
stop_price = float(round(market_rate - 2 * atr, 4))
elif sl_kind == "strategy":
if stop_price == 0:
log.critical("strategy provided invalid stop loss value")
return
else:
raise Exception("Unknown stop_loss kind - " +
market.tradeConfig['stop_loss_kind'])
position.set_stop_loss(stop_price)
pos_list = self.sl_dict.get(stop_price, None)
if pos_list == None:
pos_list = []
self.sl_dict[stop_price] = pos_list
pos_list.append(position)
def add_stop_loss_position_list(self, stop_price, position_l):
[pos.set_stop_loss(stop_price) for pos in position_l]
pos_list = self.sl_dict.get(stop_price, None)
if pos_list == None:
pos_list = []
self.sl_dict[stop_price] = pos_list
pos_list += position_l
def remove_all_positions_at_stop(self, stop_price):
return self.sl_dict.pop(stop_price, None)
def smart_stop_loss_update_positions(self, cur_indicators, market_rate,
tcfg):
if tcfg['stop_loss_kind'] == 'trailing':
sl_rate = tcfg["stop_loss_rate"]
new_sl = float(round(market_rate * (1 - sl_rate * float(.01)), 4))
elif 'ATR' in tcfg['stop_loss_kind']:
atr = cur_indicators[tcfg['stop_loss_kind']]
new_sl = float(round(market_rate - 2 * atr, 4))
else:
raise Exception("Unknown smart stop_loss kind %s" %
(tcfg['stop_loss_kind']))
key_list = list(
self.sl_dict.irange(maximum=new_sl, inclusive=(False, False)))
for key in key_list:
# log.critical("new_sl: %d key: %d"%(new_sl, key))
pos_list = self.sl_dict.pop(key)
self.add_stop_loss_position_list(new_sl, pos_list)
def db_commit_dirty_positions(self):
dirty_pos_list = []
for pos in self.all_positions:
if pos._dirty == True:
pos._dirty = False
dirty_pos_list.append(pos)
if len(dirty_pos_list):
log.debug("commit positions to db")
# save pos to db
self.positionsDb.db_save_positions(dirty_pos_list)
def pop_stop_loss_position(self, pos=None):
try:
sl_price, pos_list = 0, None
if pos:
sl_price = pos.get_stop_loss()
pos_list = self.sl_dict.get(sl_price)
else:
#get the lowest in the sorted SL list
sl_price, pos_list = self.sl_dict.peekitem(index=0)
# log.debug("pop position at sl_price:%d"%(sl_price))
if pos_list and len(pos_list):
if pos:
pos_list.remove(pos)
else:
pos = pos_list.pop()
if len(pos_list) == 0:
del (self.sl_dict[sl_price])
return pos
except (IndexError, ValueError):
return None
def get_stop_loss_positions(self, market_rate):
sl_pos_list = []
key_list = list(
self.sl_dict.irange(minimum=market_rate, inclusive=(True, True)))
# log.critical("slPrice: %d"%market_rate)
# log.critical("key_list :%s"%(key_list))
for key in key_list:
pos_list = self.sl_dict.pop(key)
sl_pos_list += pos_list
for pos in pos_list:
self.open_positions.remove(pos)
if (self.close_pending_positions.get(pos.id)):
log.critical("""Position already close pending \npos:%s
close_pending_positions: %s
open_positions: %s""" %
(str(pos), str(self.close_pending_positions),
str(self.open_positions)))
raise Exception("Duplicate close pending position")
self.close_pending_positions[pos.id] = pos
# remove pos from take profit points
self.pop_take_profit_position(pos)
self.market.num_stop_loss_hit += len(sl_pos_list)
# if len(sl_pos_list):
# log.critical("num_stop_loss_hit: %d slPrice: %d"%(len(sl_pos_list), market_rate))
return sl_pos_list
def get_take_profit_positions(self, market_rate):
tp_pos_list = []
key_list = list(
self.tp_dict.irange(maximum=market_rate, inclusive=(True, True)))
for key in key_list:
pos_list = self.tp_dict.pop(key)
tp_pos_list += pos_list
for pos in pos_list:
self.open_positions.remove(pos)
if (self.close_pending_positions.get(pos.id)):
log.critical("""Position already close pending \npos:%s
close_pending_positions: %s
open_positions: %s""" %
(str(pos), str(self.close_pending_positions),
str(self.open_positions)))
raise Exception("Duplicate close pending position")
self.close_pending_positions[pos.id] = pos
# remove pos from take profit points
self.pop_stop_loss_position(pos)
self.market.num_take_profit_hit += len(tp_pos_list)
return tp_pos_list
def add_take_profit_position(self,
position,
market_rate,
market,
new_tp=0):
tp_kind = market.tradeConfig['take_profit_kind']
if tp_kind == 'simple':
tp_rate = market.tradeConfig["take_profit_rate"]
new_tp = float(round(market_rate * (1 + tp_rate * float(.01)), 4))
elif tp_kind == 'strategy':
if new_tp == 0:
log.critical("strategy provided invalid take profit value")
return
else:
raise Exception("Unknown take profit kind - " +
market.tradeConfig['take_profit_kind'])
position.set_take_profit(new_tp)
pos_list = self.tp_dict.get(new_tp, None)
if pos_list == None:
pos_list = []
self.tp_dict[new_tp] = pos_list
pos_list.append(position)
log.debug("add take profit(%d) market_rate:(%d)" %
(new_tp, market_rate))
def pop_take_profit_position(self, pos=None):
try:
tp_price, pos_list = 0, None
if pos:
tp_price = pos.get_take_profit()
pos_list = self.tp_dict.get(tp_price)
else:
tp_price, pos_list = self.tp_dict.peekitem(0)
# log.debug("pop position at sl_price:%d"%(sl_price))
if pos_list and len(pos_list):
if pos:
pos_list.remove(pos)
else:
pos = pos_list.pop()
if len(pos_list) == 0:
del (self.tp_dict[tp_price])
return pos
except (IndexError, ValueError):
return None
def get_all_pending_orders(self):
pending_order_list = []
pending_order_list += list(self.pending_buy_orders_db.values())
pending_order_list += list(self.pending_sell_orders_db.values())
return pending_order_list
def add_or_update_pending_buy_order(self, order):
id = order.id
cur_order = self.pending_buy_orders_db.get(id)
if not cur_order:
self.total_open_order_count += 1
self.total_order_count += 1
else:
#copy required fields
order.stop = cur_order.stop
order.profit = cur_order.profit
self.pending_buy_orders_db[id] = order
def get_pending_buy_order(self, order_id):
return self.pending_buy_orders_db.get(order_id)
def add_traded_buy_order(self, order):
cur_order = self.pending_buy_orders_db.get(order.id)
if cur_order:
#copy required fields
order.stop = cur_order.stop
order.profit = cur_order.profit
self.total_open_order_count -= 1
del (self.pending_buy_orders_db[order.id])
self.traded_buy_orders_db[order.id] = order
#if this is a successful order, we have a new position open
if order.status == "filled":
self.open_position(order)
def get_traded_buy_order(self, order_id):
return self.traded_buy_orders_db.get(order_id)
def add_or_update_pending_sell_order(self, order):
id = order.id
if not self.pending_sell_orders_db.get(id):
self.total_open_order_count += 1
self.total_order_count += 1
self.pending_sell_orders_db[id] = order
def get_pending_sell_order(self, order_id):
self.pending_sell_orders_db.get(order_id)
def add_traded_sell_order(self, order):
del (self.pending_sell_orders_db[order.id])
self.total_open_order_count -= 1
self.traded_sell_orders_db[order.id] = order
#close/reopen position
#TODO: TBD: more checks required??
if order.status == "filled":
log.debug("closed position order: %s" % (order.id))
self.close_position(order)
else:
log.critical("closed position failed order: %s" % (order))
self.close_position_failed(order.id)
def get_traded_sell_order(self, order_id):
return self.traded_sell_orders_db.get(order_id)
def add_order_list(self, bids, asks):
if (asks):
self.add_asks(asks)
if (bids):
self.add_bids(bids)
def clear_order_book(self):
log.info("clearing older states")
self.orderDb.clear_order_db()
self.positionsDb.clear_position_db()
def restore_order_book(self):
# TODO: FIXME: Not considering pending state orders
if (sims.simulator_on):
#don't do order db init for sim
return None
log.info("Restoring positions and orders")
#1. Retrieve states back from Db
order_list = self.orderDb.get_all_orders()
if not order_list:
log.info("no orders to restore")
else:
# restore orders
log.info("Restoring %d orders" % (len(order_list)))
for order in order_list:
# order_status = order.status_type
order_side = order.side
log.info("restoring order: %s side: %s" %
(order.id, order_side))
self.total_order_count += 1
if order_side == 'buy':
self.traded_buy_orders_db[order.id] = order
else:
self.traded_sell_orders_db[order.id] = order
# restore positions
pos_list = self.positionsDb.db_get_all_positions(self.orderDb)
# log.critical("mapping: %s"%(str(self.positionsDb.mapping)))
if not pos_list:
log.info("no positions to restore")
return
log.info("Restoring %d positions" % (len(pos_list)))
for pos in pos_list:
log.debug("restoring position(%s)" % (pos.id))
self.all_positions.append(pos)
if pos.status == "open":
self.open_positions.append(pos)
if self.market.tradeConfig["stop_loss_enabled"]:
self.add_stop_loss_position(pos, pos.buy.get_price(),
self.market,
pos.get_stop_loss())
if self.market.tradeConfig["take_profit_enabled"]:
self.add_take_profit_position(pos, pos.buy.get_price(),
self.market,
pos.get_take_profit())
else:
self.closed_positions.append(pos)
log.info("all positions and orders are restored")
# sys.exit()
def get_positions(self, from_time=0, to_time=0):
log.info("get positions ", from_time, to_time)
return self.all_positions[:]
def dump_traded_orders(self, fd=sys.stdout):
traded = str(
list(self.traded_buy_orders_db.values()) +
list(self.traded_sell_orders_db.values()))
fd.write(traded)
def dump_positions(self, fd=sys.stdout):
fd.write(str(self.all_positions))
#
# def on_sequence_gap(self, gap_start, gap_end):
# self.reset_book()
# print('Error: messages missing({} - {}). Re-initializing book at sequence.'.format(
# gap_start, gap_end, self._sequence))
####### Public API #######
def add_or_update_my_order(self, order):
# simplified order state machine :[open, filled, canceled]
# this rework is assumed an abstraction and handles only simplified order status
# if there are more order states, it should be handled/translated in the exch impl.
# linked to market.order_status_update()
# '''
# Handle a new order update msg
# return : order
# '''
if (not order):
return None
order_id = order.id
order_status = order.status
order_side = order.side
if (not order_id):
log.critical("Invalid order_id: status:%s side: %s" %
(order_status, order_side))
raise Exception("Invalid order_id: status:%s side: %s" %
(order_status, order_side))
if (order_side == 'buy'):
# see if this is a late/mixed up state msg for an already done order. What we do here, may not be correct
if self.get_traded_buy_order(order_id):
log.critical(
"********(%s) order done already, but(%s) state msg recvd, ignore for now, FIXME: FIXME:"
% (order_side, order_status))
return None
# insert/replace the order
if (order_status == 'filled' or order_status == 'canceled'):
# a previously placed order is completed, remove from open order, add to completed orderlist
self.add_traded_buy_order(order)
log.debug("Buy order Done: total_order_count: %d "
"total_open_order_count: %d "
"traded_buy_orders_count: %d" %
(self.total_order_count, self.total_open_order_count,
len(self.traded_buy_orders_db)))
elif (order_status == 'open'):
# Nothing much to do for us here
log.info("Buy order_id(%s) Status: %s" %
(str(order_id), order_status))
self.add_or_update_pending_buy_order(order)
else:
log.critical("UNKNOWN buy order status: %s" % (order_status))
raise Exception("UNKNOWN buy order status: %s" %
(order_status))
elif (order_side == 'sell'):
# see if this is a late/mixed up state msg for an already done order. What we do here, may not be correct
if self.get_traded_sell_order(order_id):
log.critical(
"********(%s) order done already, but(%s) state msg recvd, ignore for now, FIXME: FIXME:"
% (order_side, order_status))
return None
# insert/replace the order
if (order_status == 'filled' or order_status == 'canceled'):
# a previously placed order is completed, remove from open order, add to completed orderlist
self.add_traded_sell_order(order)
log.debug("Sell order Done: total_order_count: %d "
"total_open_order_count: %d "
"traded_sell_orders_count: %d" %
(self.total_order_count, self.total_open_order_count,
len(self.traded_sell_orders_db)))
elif (order_status == 'open'):
# Nothing much to do for us here
log.info("Sell order_id(%s) Status: %s" %
(str(order_id), order_status))
self.add_or_update_pending_sell_order(order)
self.close_position_pending(order)
else:
log.critical("UNKNOWN sell order status: %s" % (order_status))
raise Exception("UNKNOWN buy order status: %s" %
(order_status))
else:
log.critical("Invalid order :%s" % (order))
raise Exception("Invalid order :%s" % (order))
# log.debug("Order: %s\n"%(str(order)))
#Add the successful order to the db
self.orderDb.db_add_or_update_order(order)
stats.stats_update_order(self.market, order)
return order
def new_book(self, bids, asks):
log.info("Building new order book")
if (bids and len(bids)) or (asks and len(asks)):
self.add_order_list(bids, asks)
self.book_valid = True
else:
self.book_valid = False
def reset_book(self):
self._asks = RBTree()
self._bids = RBTree()
res = self.market.exchange.get_product_order_book(
self.market.product_id, level=3)
# log.debug("%s"%(str(res)))
if res == None:
log.error("Unable to get orderbook for exchange(%s) product: %s" %
(self.market.exchange.name, self.market.product_id))
return
for bid in res['bids']:
new_size = float(bid[1])
price = float(bid[0])
new_size += float((self.get_bids(price) or 0))
self.set_bids(price, new_size)
for ask in res['asks']:
new_size = float(ask[1])
price = float(ask[0])
new_size += float((self.get_asks(price) or 0))
self.set_asks(price, new_size)
self._sequence = float(res['sequence'])
self.book_valid = True
# print("asks: %s"%(str(self._asks)))
# print("bids: %s"%(str(self._bids)))
def add_asks(self, asks):
'''
asks =[[price, size]]
'''
for ask in asks:
price = float(ask[0])
size = float(ask[1])
if size > 0: # size > 0 add, size = 0 remove
self.set_asks(price, size)
else:
if (self.get_asks(price)):
self.remove_asks(price)
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
price = round(price, 8)
asks = round(asks, 8)
log.debug("set_asks: price: %g size: %g" % (price, asks))
self._asks.insert(price, asks)
def add_bids(self, bids):
'''
bids =[[price, size]]
'''
for bid in bids:
price = float(bid[0])
size = float(bid[1])
if size > 0: # size > 0 add, size = 0 remove
self.set_bids(price, size)
else:
if (self.get_bids(price)):
self.remove_bids(price)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
price = round(price, 8)
bids = round(bids, 8)
log.debug("set_bid: price: %g size: %g" % (price, bids))
self._bids.insert(
price, bids) # insert on RBtree is a replace for existing keys
class OrderBookL2():
def __init__(self, product_id='BTC-USD'):
self._asks = RBTree()
self._bids = RBTree()
self._product_id = product_id
@property
def product_id(self):
return self._product_id
def process_snapshot(self, message):
''' Process a snapshot message '''
self._asks = RBTree()
self._bids = RBTree()
for ask in message['asks']:
price, size = ask
price = Decimal(price)
size = Decimal(size)
self._asks.insert(price, size)
for bid in message['bids']:
price, size = bid
price = Decimal(price)
size = Decimal(size)
self._bids.insert(price, size)
def process_update(self, message):
''' Process an update message '''
changes = message['changes']
for change in changes:
side, price, size = change
price = Decimal(price)
size = Decimal(size)
if side == 'buy':
if size <= 0:
self._bids.remove(price)
else:
self._bids.insert(price, size)
elif side == 'sell':
if size <= 0:
self._asks.remove(price)
else:
self._asks.insert(price, size)
def process_message(self, message):
msg_type = message['type']
# dropped - not same product id
if (message.get('product_id', None) != self._product_id):
return
if msg_type == 'snapshot':
self.process_snapshot(message)
elif msg_type == 'l2update':
self.process_update(message)
def get_current_book(self):
result = {'asks': [], 'bids': []}
for ask in self._asks:
try:
size = self._asks[ask]
except KeyError:
continue
result['asks'].append([ask, size])
for bid in self._bids:
try:
size = self._bids[bid]
except KeyError:
continue
result['bids'].append([bid, size])
return result
def get_ask(self):
price = self._asks.min_key()
try:
size = self._asks[price]
except KeyError:
return (price, 0)
return (price, size)
def get_bid(self):
price = self._bids.max_key()
try:
size = self._bids[price]
except KeyError:
return (price, 0)
return (price, size)
class OrderBook(object):
def __init__(self, product_id='BTC-USD', feed=None, log_to=None):
self._product_id = product_id
self._asks = RBTree()
self._bids = RBTree()
self._sequence = -1
self._current_ticker = None
self._feed = feed
@property
def product_id(self):
""" Currently OrderBook only supports a single product even though it is stored as a list of products. """
return self.product_id
def reset_book(self, snapshot):
self._asks = RBTree()
self._bids = RBTree()
for bid in snapshot['bids']:
self._add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in snapshot['asks']:
self._add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = snapshot['sequence']
def on_message(self, message):
sequence = message['sequence']
if self._sequence == -1:
logger.error("Expected snapshot before any message")
sys.exit()
# self.reset_book()
return
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
self.on_sequence_gap(self._sequence, sequence)
# return
msg_type = message['type']
if msg_type == 'open':
self._add(message)
elif msg_type == 'done' and 'price' in message:
self._remove(message)
elif msg_type == 'match':
self._match(message)
self._current_ticker = message
elif msg_type == 'change':
self._change(message)
self._sequence = sequence
def on_sequence_gap(self, gap_start, gap_end):
# self.reset_book()
logger.error(
'Error: messages missing ({} - {}). ignoring the gap.'.format(
gap_start, gap_end, self._sequence))
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
@staticmethod
def meet_min_diff_price(low_price, high_price, min_diff_price):
if low_price is None or high_price is None or min_diff_price is None:
return False
return high_price - low_price > min_diff_price
@staticmethod
def meet_max_size(total_size, max_size):
if total_size is None or max_size is None:
return False
return total_size > max_size
def get_aggr_bids(self, min_diff_price=None, max_size=None):
max_bid = self.get_bid()
total_size = 0
aggr_bids = list()
for price, bid_orders in reversed(list(self._bids.items())):
if (OrderBook.meet_min_diff_price(price, max_bid, min_diff_price)
and OrderBook.meet_max_size(total_size, max_size)):
break
size = sum(bid_order['size'] for bid_order in bid_orders)
num_orders = len(bid_orders)
aggr_bids.append({
'price': price,
'size': size,
'num_orders': num_orders
})
total_size += size
return aggr_bids
def get_aggr_asks(self, min_diff_price=None, max_size=None):
min_ask = self.get_ask()
total_size = 0
aggr_asks = list()
for price, ask_orders in self._asks.items():
if (OrderBook.meet_min_diff_price(min_ask, price, min_diff_price)
and OrderBook.meet_max_size(total_size, max_size)):
break
size = sum(ask_order['size'] for ask_order in ask_orders)
num_orders = len(ask_orders)
aggr_asks.append({
'price': price,
'size': size,
'num_orders': num_orders
})
total_size += size
return aggr_asks
# Internal operations
def _add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def _remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def _match(self, order):
if self._feed is not None:
self._feed._match(order)
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def _change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
try:
price = Decimal(order['price'])
except KeyError:
return
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
class Book(object):
def __init__(self):
self.tree = RBTree()
self.prices = dict() # { price : Queue } objects
self.orders = dict() # { order_id : Order } objects
self.volume = 0 # Contains total quantity from all Orders in tree
self.num_orders = 0 # Contains count of Orders in tree
self.depth = 0 # Number of different prices in tree (http://en.wikipedia.org/wiki/Order_book_(trading)#Book_depth)
def get_price_list(self, price):
return self.prices[price]
def get_order(self, order_id):
return self.orders[order_id]
def create_price(self, price):
self.depth += 1
new_price = Queue()
self.tree.insert(price, new_price)
self.prices[price] = new_price
def remove_price(self, price):
self.depth -= 1
self.tree.remove(price)
del self.prices[price]
def price_exists(self, price):
return price in self.prices
def order_exists(self, order):
return order in self.orders
def insert_order(self, quote):
if self.order_exists(quote['order_id']):
self.remove_order_by_id(quote['order_id'])
self.num_orders += 1
if quote['price'] not in self.prices:
# If price not in price list, create a node in tree
self.create_price(quote['price'])
order = Order(quote, self.prices[quote['price']]) # Create an order
self.prices[order.price].append(order) # Add the order to the prices queue
self.orders[order.order_id] = order
self.volume += order.quantity
def update_order(self, order_update):
order = self.orders[order_update['order_id']]
original_quantity = order.quantity
if order_update['price'] != order.price:
# Price changed. Remove order and update tree.
order_list = self.prices[order.price]
order_list.remove(order)
if len(order_list) == 0: # If there is nothing else in the OrderList, remove the price from RBtree
self.remove_price(order.price)
self.insert_order(order_update)
else:
# Quantity changed. Price is the same.
order.update_quantity(order_update['quantity'], order_update['timestamp'])
self.volume += order.quantity - original_quantity
def remove_order_by_id(self, order_id):
self.num_orders -= 1
order = self.orders[order_id]
self.volume -= order.quantity
order.order_list.remove(order)
if len(order.order_list) == 0:
self.remove_price(order.price)
del self.orders[order_id]
def max_price(self):
if self.depth > 0:
return self.tree.max_key()
else:
return None
def min_price(self):
if self.depth > 0:
return self.tree.min_key()
else:
return None
def max_price_list(self):
if self.depth > 0:
return self.get_price_list(self.max_price())
else:
return None
def min_price_list(self):
if self.depth > 0:
return self.get_price_list(self.min_price())
else:
return None
def __len__(self):
return len(self.orders)
class OrderTree(object):
''' An RBtree used to store OrderLists in price order '''
def __init__(self):
'''
@summary: initialization of Ordertree class
@mem price_tree: a red-black tree
@mem price_map: Dictionary containing price : OrderList object
@mem order_map: Dictionary containing order_id : Order object
@mem volume: Contains total quantity from all Orders in tree
@mem num_orders: Contains count of Orders in tree
@mem depth: Number of different prices in tree (http://en.wikipedia.org/wiki/Order_book_(trading)#Book_depth)
'''
self.price_tree = RBTree()
self.price_map = {}
self.order_map = {}
self.volume = 0
self.num_orders = 0
self.depth = 0
def __len__(self):
''' return the length of order_map '''
return len(self.order_map)
def get_price_list(self, price):
''' get the price list from the price map '''
return self.price_map[price]
def get_order(self, order_id):
''' get order list from order map '''
return self.order_map[order_id]
def create_price(self, price):
''' create a new price if adding an order without price in the tree '''
self.depth += 1
new_list = OrderList()
self.price_tree.insert(price, new_list)
self.price_map[price] = new_list
def remove_price(self, price):
''' remove a price from the tree '''
self.depth -= 1
self.price_tree.remove(price)
del self.price_map[price]
def price_exists(self, price):
''' check whether price exists in price map '''
return price in self.price_map
def order_exists(self, order):
''' check whether an order exists in order map '''
return order in self.order_map
def insert_order(self, quote):
''' insert an order into the order map '''
if self.order_exists(quote['order_id']):
self.remove_order_by_id(quote['order_id'])
self.num_orders += 1
if quote['price'] not in self.price_map:
self.create_price(
quote['price']
) # If price not in Price Map, create a node in RBtree
order = Order(quote, self.price_map[quote['price']]) # Create an order
self.price_map[order.price].append_order(
order) # Add the order to the OrderList in Price Map
self.order_map[order.order_id] = order
self.volume += order.quantity
def remove_order_by_id(self, order_id):
''' remove an order from the order map '''
self.num_orders -= 1
order = self.order_map[order_id]
self.volume -= order.quantity
order.order_list.remove_order(order)
if len(order.order_list) == 0:
self.remove_price(order.price)
del self.order_map[order_id]
def max_price(self):
''' return max price in price tree '''
if self.depth > 0:
return self.price_tree.max_key()
else:
return None
def min_price(self):
''' return min price in price tree '''
if self.depth > 0:
return self.price_tree.min_key()
else:
return None
def max_price_list(self):
''' return max price in price tree '''
if self.depth > 0:
return self.get_price_list(self.max_price())
else:
return None
def min_price_list(self):
''' return min price in price tree '''
if self.depth > 0:
return self.get_price_list(self.min_price())
else:
return None
class OrderBook(WebsocketClient):
def __init__(self,
product_id='BTC-USD',
log_to=None,
autoreconnect=True,
max_retries=3):
super(OrderBook, self).__init__(products=[product_id])
self.channels = ['full']
self._asks = RBTree()
self._bids = RBTree()
self._client = PublicClient()
self._sequence = -1
self._log_to = log_to
self._retries = 0
self._max_retries = max_retries
self._autoreconnect = autoreconnect
self._is_ready = False
if self._log_to:
assert hasattr(self._log_to, 'write')
self._current_ticker = None
@property
def product_id(self):
''' Currently OrderBook only supports a single product even though it is stored as a list of products. '''
return self.products[0]
def is_ready(self):
return self._is_ready
def on_open(self):
self._sequence = -1
self._retries = 0
print("-- Subscribed to OrderBook! --\n")
def on_close(self):
print("\n-- OrderBook Socket Closed! --")
self._is_ready = False
if not self.ws.sock or not self.ws.sock.connected and self._autoreconnect:
self._sequence = -1
if self._retries < self._max_retries:
print('-- OrderBook Reconnecting... --')
self._retries += 1
self.start()
else:
print('-- Could not reconnect, stopping... --')
def reset_book(self):
self._is_ready = False
self._asks = RBTree()
self._bids = RBTree()
res = self._client.get_product_order_book(product_id=self.product_id,
level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
print("\n-- OrderBook Data Initialized --")
self._is_ready = True
self._sequence = res['sequence']
def on_message(self, message):
if self._log_to:
pickle.dump(message, self._log_to)
if 'sequence' not in message:
return
sequence = message['sequence']
if self._sequence == -1:
print("\n-- Initializing OrderBook --")
self.reset_book()
return
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
self.on_sequence_gap(self._sequence, sequence)
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
self._current_ticker = message
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
def on_error(self, ws, e):
super(OrderBook, self).on_error(ws, e)
if not ws.sock or not ws.sock.connected:
self._sequence = -1
self._is_ready = False
if self._retries < self._max_retries:
print('-- OrderBook Disconnected, reconnecting... --')
self._retries += 1
self.start()
else:
print('-- Could not reconnect, stopping... --')
def on_sequence_gap(self, gap_start, gap_end):
self._sequence = -1
self.reset_book()
print('-- Error: messages missing, re-initializing book. --'.format(
gap_start, gap_end, self._sequence))
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
try:
price = Decimal(order['price'])
except KeyError:
return
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class MarinerOrderBook(GDAX.OrderBook):
def __init__(self, ticker, threshold):
GDAX.OrderBook.__init__(self, product_id = ticker)
self._threshold = threshold
self._buyWhales = RBTree()
self._sellWhales = RBTree()
self._topBuyWhale = None
self._topSellWhale = None
def registerHandlers(self, topWhaleChangedHandler):
print(" registering callbacks...")
self.topWhaleChangedHandler = topWhaleChangedHandler
def onMessage(self, message):
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.getProductOrderBook(level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
print('Error: messages missing ({} - {}). Re-initializing websocket.'.format(sequence, self._sequence))
self.close()
self.start()
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
self.checkWhaleAddBuy(order)
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
self.checkWhaleAddSell(order)
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def checkWhaleAddBuy(self, order):
price = Decimal(order['price'])
volume = Decimal(order['size'])
if self.isWhale(volume): #too low in practice
print("WHALE ENTERED (BUY): price=" + str(price) + " volume=" + str(volume))
whale_bid_level = self.get_whale_bids(price)
if whale_bid_level is None:
whale_bid_level = WhaleLevel(price, volume)
else:
whale_bid_level.add_volume(volume)
self.set_whale_bids(price, whale_bid_level)
top_whale_price = self.get_whale_bid()
if self._topBuyWhale is None or top_whale_price != self._topBuyWhale.get_price():
self._topBuyWhale = self._buyWhales.get(top_whale_price)
self.topWhaleChangedHandler(self._topBuyWhale, "BUY")
else:
print(" top BUY whale is still " + str(self._topBuyWhale))
def checkWhaleAddSell(self, order):
price = Decimal(order['price'])
volume = Decimal(order['size'])
if self.isWhale(volume): #too low in practice
print("WHALE ENTERED (SELL): price=" + str(price) + " volume=" + str(volume))
whale_ask_level = self.get_whale_asks(price)
if whale_ask_level is None:
whale_ask_level = WhaleLevel(price, volume)
else:
whale_ask_level.add_volume(volume)
self.set_whale_asks(price, whale_ask_level)
top_whale_price = self.get_whale_ask()
if self._topSellWhale is None or top_whale_price != self._topSellWhale.get_price():
self._topSellWhale = self._sellWhales.get(top_whale_price)
self.topWhaleChangedHandler(self._topSellWhale, "SELL")
else:
print(" top SELL whale is still " + str(self._topSellWhale))
def remove(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
price = Decimal(order['price'])
if order['side'] == 'buy':
self.checkWhaleRemoveBuy(order)
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
self.checkWhaleRemoveSell(order)
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def checkWhaleRemoveBuy(self, order):
price = Decimal(order['price'])
volume = Decimal(order['size'])
whale_bid_level = self.get_whale_bids(price)
if whale_bid_level is not None:
whale_bid_level.remove_volume(volume)
print("WHALE LEFT (BUY): price=" + str(price) + " volume=" + str(volume))
if not self.isWhale(whale_bid_level.get_volume()):
self.remove_whale_bids(price)
top_whale_price = self.get_whale_bid()
self._topBuyWhale = self._buyWhales.get(top_whale_price)
self.topWhaleChangedHandler(self._topBuyWhale, "BUY")
else:
print(" top BUY whale is still " + str(self._topBuyWhale))
def checkWhaleRemoveSell(self, order):
price = Decimal(order['price'])
volume = Decimal(order['size'])
whale_ask_level = self.get_whale_asks(price)
if whale_ask_level is not None:
whale_ask_level.remove_volume(volume)
print("WHALE ENTERED (SELL): price=" + str(price) + " volume=" + str(volume))
if not self.isWhale(whale_ask_level.get_volume()):
self.remove_whale_asks(price)
top_whale_price = self.get_whale_ask()
self._topSellWhale = self._sellWhales.get(top_whale_price)
self.topWhaleChangedHandler(self._topSellWhale, "SELL")
else:
print(" top SELL whale is still " + str(self._topSellWhale))
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
price = Decimal(order['price'])
new_volume = Decimal(order['new_size'])
if order['side'] == 'buy':
self.checkWhaleChangeBuy(order)
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id'] for o in bids):
return
index = map(itemgetter('id'), bids).index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
self.checkWhaleChangeSell(order)
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id'] for o in asks):
return
index = map(itemgetter('id'), asks).index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def checkWhaleChangeBuy(self, order):
price = Decimal(order['price'])
new_volume = order['new_size']
whale_bid_level = self.get_whale_bids(price)
if not whale_bid_level == None:
print("BUY WHALE CHANGED: price=" + price + " new_volume=" + new_volume)
whale_bid_level.setVolume(new_volume)
self.set_whale_bids(price, whale_bid_level)
def checkWhaleChangeSell(self, order):
price = Decimal(order['price'])
new_volume = order['new_size']
whale_ask_level = self.get_whale_asks(price)
if not whale_ask_level == None:
print("SELL WHALE CHANGED: price=" + price + " new_volume=" + new_volume)
whale_ask_level.setVolume(new_volume)
self.set_whale_asks(price, whale_ask_level)
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
thisAsk = self._asks[ask]
except KeyError:
continue
for order in thisAsk:
result['asks'].append([
order['price'],
order['size'],
order['id'],
])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
thisBid = self._bids[bid]
except KeyError:
continue
for order in thisBid:
result['bids'].append([
order['price'],
order['size'],
order['id'],
])
return result
def isWhale(self, aVolume):
return aVolume >= self._threshold
#For general book management purposes
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
#For whale book management purposes
def get_whale_ask(self):
return self._sellWhales.min_key()
def get_whale_asks(self, price):
return self._sellWhales.get(price)
def remove_whale_asks(self, price):
self._sellWhales.remove(price)
def set_whale_asks(self, price, asks):
self._sellWhales.insert(price, asks)
def get_whale_bid(self):
return self._buyWhales.max_key()
def get_whale_bids(self, price):
return self._buyWhales.get(price)
def remove_whale_bids(self, price):
self._buyWhales.remove(price)
def set_whale_bids(self, price, bids):
self._buyWhales.insert(price, bids)
class WhaleTracker():
current_milli_time = lambda self: int(time.time() * 1000)
def __init__(self, ticker):
self._ticker = ticker
self._bid_whales = RBTree()
self._ask_whales = RBTree()
def whaleEnteredMarketHandler(self, order):
if order['side'] == 'buy':
self.addBidWhale(order)
else:
self.addAskWhale(order)
def whaleExitedMarketHandler(self, order):
if order['side'] == 'buy':
self.removeBidWhale(order)
else:
self.removeAskWhale(order)
def whaleChangedHandler(self, order):
if order['side'] == 'buy':
self.changeBidWhale(order)
else:
self.changeAskWhale(order)
def addBidWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
bid_whale_order = self.get_bid_whale(price)
if bid_whale_order is None or volume > bid_whale_order.get_volume():
#Logging.logger.info("NEW WHALE ENTERED (BID): price=" + str(price) + " volume=" + str(volume))
bid_whale_order = WhaleOrder(ID, price, volume)
self.set_bid_whale(price, bid_whale_order)
def addAskWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
ask_whale_order = self.get_ask_whale(price)
if ask_whale_order is None or volume > ask_whale_order.get_volume():
#Logging.logger.info("NEW WHALE ENTERED (ASK): price=" + str(price) + " volume=" + str(volume))
ask_whale_order = WhaleOrder(ID, price, volume)
self.set_ask_whale(price, ask_whale_order)
def removeBidWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
bid_whale_order = self.get_bid_whale(price)
if bid_whale_order is not None and bid_whale_order.get_id() == ID:
#Logging.logger.info("WHALE LEFT (BID): price=" + str(price) + " volume=" + str(volume))
self.remove_bid_whale(price)
def removeAskWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
ask_whale_order = self.get_ask_whale(price)
if ask_whale_order is not None and ask_whale_order.get_id() == ID:
#Logging.logger.info("WHALE LEFT (ASK): price=" + str(price) + " volume=" + str(volume))
self.remove_ask_whale(price)
def changeBidWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
new_volume = order['new_size']
bid_whale_order = self.get_bid_whale(price)
if not bid_whale_order == None:
#Logger.logging.info("WHALE CHANGED (BID): price=" + price + " new_volume=" + new_volume)
if self.isWhale(new_volume):
bid_whale_order.setVolume(new_volume)
self.set_bid_whale(price, bid_whale_order)
else:
self.removeBidWhale(order)
def changeAskWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
new_volume = order['new_size']
ask_whale_order = self.get_ask_whale(price)
if not ask_whale_order == None:
#Logger.logging.info("WHALE CHANGED (ASK): price=" + price + " new_volume=" + new_volume)
if self.isWhale(new_volume):
ask_whale_order.setVolume(new_volume)
self.set_ask_whale(price, ask_whale_order)
else:
self.removeAskWhale(order)
def get_current_whales(
self,
num_whales=30
): #by default, fetch only 30 whales off the book either way
whales = []
for index, bid_whale in self._bid_whales.items(True):
if index == num_whales:
break
else:
whales.append({
'price': Decimal128(bid_whale.get_price()),
'volume': Decimal128(bid_whale.get_volume()),
'id': bid_whale.get_id(),
})
for index, ask_whale in self._ask_whales.items():
if index == num_whales:
break
else:
whales.append({
'price': Decimal128(ask_whale.get_price()),
'volume': Decimal128(ask_whale.get_volume()),
'id': ask_whale.get_id(),
})
whales_frame = pd.DataFrame(whales)
return whales_frame
def get_top_bid_whale(self):
if not self._bid_whales.is_empty():
top_bid = self._bid_whales.max_key()
return self.get_bid_whale(top_bid)
else:
return None
def get_bid_whale(self, price):
return self._bid_whales.get(price)
def remove_bid_whale(self, price):
self._bid_whales.remove(price)
def set_bid_whale(self, price, whale):
self._bid_whales.insert(price, whale)
def get_top_ask_whale(self):
if not self._ask_whales.is_empty():
top_ask = self._ask_whales.min_key()
return self.get_ask_whale(top_ask)
else:
return None
def get_ask_whale(self, price):
return self._ask_whales.get(price)
def remove_ask_whale(self, price):
self._ask_whales.remove(price)
def set_ask_whale(self, price, whale):
self._ask_whales.insert(price, whale)
class OrderBook(WebsocketClient):
def __init__(self, product_id='BTC-USD', log_to=None):
super(OrderBook, self).__init__(products=product_id)
self._asks = RBTree()
self._bids = RBTree()
self._client = PublicClient()
self._sequence = -1
self._log_to = log_to
if self._log_to:
assert hasattr(self._log_to, 'write')
self._current_ticker = None
@property
def product_id(self):
''' Currently OrderBook only supports a single product even though it is stored as a list of products. '''
return self.products[0]
def on_open(self):
self._sequence = -1
print("-- Subscribed to OrderBook! --\n")
def on_close(self):
print("\n-- OrderBook Socket Closed! --")
def reset_book(self):
self._asks = RBTree()
self._bids = RBTree()
res = self._client.get_product_order_book(
product_id=self.product_id, level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
def on_message(self, message):
if self._log_to:
pickle.dump(message, self._log_to)
sequence = message['sequence']
if self._sequence == -1:
self.reset_book()
return
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
self.on_sequence_gap(self._sequence, sequence)
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
self._current_ticker = message
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
def on_sequence_gap(self, gap_start, gap_end):
self.reset_book()
print('Error: messages missing ({} - {}). Re-initializing book at sequence.'.format(
gap_start, gap_end, self._sequence))
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = collections.OrderedDict()
bids[order['id']] = order
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = collections.OrderedDict()
asks[order['id']] = order
self.set_asks(order['price'], asks)
def remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
if order['order_id'] in bids:
del bids[order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
if order['order_id'] in asks:
del asks[order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
top_order = bids.items()[0]
assert top_order[1]['id'] == order['maker_order_id']
if top_order[1]['size'] == size:
del bids[order['maker_order_id']]
self.set_bids(price, bids)
else:
top_order[1]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
top_order = asks.items()[0]
assert top_order[1]['id'] == order['maker_order_id']
if top_order[1]['size'] == size:
del asks[order['maker_order_id']]
self.set_asks(price, asks)
else:
top_order[1]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
try:
price = Decimal(order['price'])
except KeyError:
return
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or order['order_id'] not in bids:
return
bids[order['order_id']]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or order['order_id'] not in asks:
return
asks[order['order_id']]['size'] = new_size
self.set_asks(price, asks)
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class OrderTree(object):
def __init__(self):
self.priceTree = RBTree()
self.priceMap = {} # Map from price -> orderList object
self.orderMap = {} # Order ID to Order object
self.volume = 0 # How much volume on this side?
self.nOrders = 0 # How many orders?
self.lobDepth = 0 # How many different prices on lob?
def __len__(self):
return len(self.orderMap)
def getPrice(self, price):
return self.priceMap[price]
def getOrder(self, idNum):
return self.orderMap[idNum]
def createPrice(self, price):
self.lobDepth += 1
newList = OrderList()
self.priceTree.insert(price, newList)
self.priceMap[price] = newList
def removePrice(self, price):
self.lobDepth -= 1
self.priceTree.remove(price)
del self.priceMap[price]
def priceExists(self, price):
return price in self.priceMap
def orderExists(self, idNum):
return idNum in self.orderMap
def insertOrder(self, quote):
if self.orderExists(quote['idNum']):
self.removeOrderById(quote['idNum'])
self.nOrders += 1
if quote['price'] not in self.priceMap:
self.createPrice(quote['price'])
order = Order(quote, self.priceMap[quote['price']])
self.priceMap[order.price].appendOrder(order)
self.orderMap[order.idNum] = order
self.volume += order.qty
def updateOrder(self, orderUpdate):
order = self.orderMap[orderUpdate['idNum']]
originalVolume = order.qty
if orderUpdate['price'] != order.price:
# Price changed
orderList = self.priceMap[order.price]
orderList.removeOrder(order)
if len(orderList) == 0:
self.removePrice(order.price)
self.insertOrder(orderUpdate)
else:
# Quantity changed
order.updateQty(orderUpdate['qty'], orderUpdate['timestamp'])
self.volume += order.qty - originalVolume
def removeOrderById(self, idNum):
self.nOrders -= 1
order = self.orderMap[idNum]
self.volume -= order.qty
order.orderList.removeOrder(order)
if len(order.orderList) == 0:
self.removePrice(order.price)
del self.orderMap[idNum]
def maxPrice(self):
if self.lobDepth > 0:
return self.priceTree.max_key()
else:
return None
def minPrice(self):
if self.lobDepth > 0:
return self.priceTree.min_key()
else:
return None
def maxPriceList(self):
if self.lobDepth > 0:
return self.getPrice(self.maxPrice())
else:
return None
def minPriceList(self):
if self.lobDepth > 0:
return self.getPrice(self.minPrice())
else:
return None
class OrderBook:
def __init__(self,
API_KEY,
API_SECRET,
API_PASS,
API_URL,
WSS_URL,
products,
M_bidask,
auth_user_msg_q,
M_get_depth,
authclient,
message_type="subscribe",
auth=True,
level=True):
self.authclient = authclient
self.WSS_URL = WSS_URL
self.API_URL = API_URL
self.products = products
self.type = message_type
self.stop = False
self.ws = None
self.thread = None
self.lock = Lock()
self.auth = auth
self.API_KEY = API_KEY
self.API_SECRET = API_SECRET
self.API_PASS = API_PASS
self._sequence = -1
self._bid = None
self._ask = None
self._bid_depth = None
self._ask_depth = None
self.price = None
self.level = level
self.pd_data_set = None
self.buyop = 0
self.sellop = 0
self._bid = None
self._ask = None
self.M_bidask = M_bidask
self.auth_user_msg_q = auth_user_msg_q
self.M_get_depth = M_get_depth
self.count01 = 0
self.manager = Manager()
def start(self):
logging.info('start called')
self.on_open()
self.process = Process(target=self._go,
args=(self.M_bidask, self.M_get_depth))
self.process.start()
def _go(self, M_bidask, M_get_depth):
self.stop = False
self._sequence = -1
self.M_get_depth = M_get_depth
self._connect()
self._listen(M_bidask)
def _connect(self):
logging.info('_connect called')
if self.WSS_URL[-1] == "/":
self.WSS_URL = self.WSS_URL[:-1] # If url ends in a "/" strip it
sub_params = {'type': 'subscribe', 'product_ids': [self.products]}
if self.auth:
timestamp = str(time.time())
message = timestamp + 'GET' + '/users/self'
message = message.encode('ascii')
hmac_key = base64.b64decode(self.API_SECRET)
signature = hmac.new(hmac_key, message, hashlib.sha256)
digest = signature.digest()
signature_b64 = base64.b64encode(digest).decode('utf-8').rstrip(
'\n')
sub_params['signature'] = signature_b64
sub_params['key'] = self.API_KEY
sub_params['passphrase'] = self.API_PASS
sub_params['timestamp'] = timestamp
self.ws = create_connection(self.WSS_URL)
self.ws.send(json.dumps(sub_params))
if self.level:
sub_params = {
"type": "subscribe",
"product_ids": [self.products],
"channels": ["matches"]
}
self.ws.send(json.dumps(sub_params))
def _listen(self, M_bidask):
self.M_bidask2 = M_bidask # setting self.p1_w2 within this process so as to not confuse with self.p1_w. they are the came Pipe object
logging.info('_listen called')
while not self.stop:
try:
msg = self.ws.recv()
except Exception as e:
print('exception 1')
gv.logger.exception(e)
gv.logger.info('sleeping for 5 seconds and running _connect()')
self.close()
time.sleep(5)
self._go(self.M_bidask, self.M_get_depth)
else:
if msg:
msg = json.loads(msg)
self.on_message(msg)
def on_message(self, message):
try:
if 'user_id' in message:
self.auth_user_msg_q.put(message)
if 'sequence' not in message:
return
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self.get_product_order_book(level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
print(
'Error: messages missing ({} - {}). Re-initializing websocket.'
.format(sequence, self._sequence))
self.close()
time.sleep(5)
self._go(self.M_bidask, self.M_get_depth)
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
bid = self.get_bid(
) # Update Multiprocessing.Value variables with the new bid and ask
ask = self.get_ask()
if self.M_get_depth.value == 'bid':
bids = self.get_bids(bid)
bid_depth = sum([b['size'] for b in bids])
self.M_get_depth.value = bid_depth
if self.M_get_depth.value == 'ask':
asks = self.get_asks(ask)
ask_depth = sum([a['size'] for a in asks])
self.M_get_depth.value = ask_depth
if not self._bid == bid or not self._ask == ask:
# lock these vars
self.M_bidask2['bid'] = float(bid)
self.M_bidask2['ask'] = float(ask)
self._bid = bid
self._ask = ask
except Exception as e:
print(e)
print('exception 2')
gv.logger.exception(e)
self.close()
time.sleep(5)
self._go(self.M_bidask, self.M_get_depth)
# bids = self.get_bids(bid)
# bid_depth = sum([b['size'] for b in bids])
# asks = self.get_asks(ask)
# ask_depth = sum([a['size'] for a in asks])
# print('bid: %f @ %f - ask: %f @ %f' % (bid_depth, bid, ask_depth, ask))
def on_error(self, e):
self._sequence.value = -1
print('on_error called')
self.close()
self._go(self.M_bidask, self.M_get_depth)
def close(self):
logging.info('close called')
if not self.stop:
print('in not self.stop')
self.on_close()
self.stop = True
#self.process.join()
try:
if self.ws:
print('if self.ws')
self.ws.close()
except WebSocketConnectionClosedException as e:
gv.logger.debug(e)
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(
self, order
): # order is the instance of one message receded from the websockit
price = Decimal(
order['price']
) # get the price from the order and saves it as a decimal in variable price
if order['side'] == 'buy': # if its a buy order continue
bids = self.get_bids(
price
) # returns all the bids at specified (price) from orderbook, geting ready to compare with current order.
if bids is not None: # bids being None does not happen often
bidslist = [
o for o in bids if o['id'] != order['order_id']
] # creates a list minus the current order to be removed
if len(
bidslist
) > 0: # when bidslist still contains items in its list send list and price to set_bids
self.set_bids(price, bidslist)
else:
self.remove_bids(
price
) # when bidslist returns nothing in its list send price to remove_bids(price) to be removed from orderbook.
else:
asks = self.get_asks(
price
) # same as the first part of this function except on the ask side
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
# An order was matched so it needs to be removed from our orderbook \
# and keep the remaining
def match(
self, order
): # order is the instance of one message receded from the websockit
size = Decimal(order['size']) # order size how many ETH or BTC etc..
price = Decimal(order['price'])
if order['side'] == 'buy': # if the order is a buy order \
bids = self.get_bids(
price
) # returns all the bids at specified (price) from orderbook \
if not bids: # if there are no orders at this price on the bid/buy side than just return
return
assert bids[0]['id'] == order[
'maker_order_id'] # if condition returns False trigger an error
# this is taking the trade id of the order on our orderbook and comparing \
# it to the market makers order id to varify the data.
# this should always return True if it returns false are market data \
# we have saved is incorrect
if bids[0][
'size'] == size: # The first order on our books at this pirce is bids[0] and if the size \
# of that order matches the incoming match order then we need to remove \
# the order from our books at this price
# we use bids[0] because the market works on a first come first serve \
# and bids[0] has been here the longest
self.set_bids(
price, bids[1:]
) # excludes the first order in the list and sets the remaining \
# back in the orderbook list
else:
bids[0][
'size'] -= size # subtract size of current trade from the position held on the orderbook "bids[0]"
self.set_bids(price, bids)
else: # same as above for the match function but on the ask/sell side
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError as e:
gv.logger.debug(e)
return
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
# _bids is the is the binary data structure of the entire buy side of the overbook
# this function is returning every order from the buy side with a specific price
# get is a funcntion of an RBTree data structure
def get_product_order_book(self, level):
params = {'level': level}
r = requests.get(self.API_URL +
'/products/{}/book'.format(self.products),
params=params,
timeout=30)
return r.json()
def on_open(self):
print("-- Subscribed! --\n")
def on_close(self):
print("\n-- Socket Closed --")
def bid_ask_spread(self):
lock.acquire()
try:
bid = self.get_bid()
ask = self.get_ask()
print((bid - ask) * -1)
except Exception as e:
gv.logger.debug(e)
finally:
lock.release()
#def bid_ask(self):
# try:
# bid = self.get_bid()
# bids = self.get_bids(bid)
# ask = self.get_ask()
# asks = self.get_asks(ask)
#
# bid_depth = sum([b['size'] for b in bids]) # If there are no changes to the bid-ask spread \
# ask_depth = sum([a['size'] for a in asks]) # since the last update, no need to print
#
# # if we end up logging a type error we may need to \
# # lock the rbtree(_bid and _ask)
# if gv._bid == bid and gv._ask == ask and self._bid_depth == bid_depth and self._ask_depth == ask_depth:
# pass
# else: # If there are differences, update the cache
# gv._bid = bid
# gv._ask = ask
# self._bid_depth = bid_depth
# self._ask_depth = ask_depth
# print('{}\tbid: {:.3f} @ {:.2f}\task: {:.3f} @ {:.2f}'.format(dt.datetime.now(), bid_depth, bid, ask_depth, ask))
# except Exception as e:
# gv.logger.debug(e)
def user_orders_on_book(self):
self.lock.acquire()
try:
print('user orders on book {}'.format(gv.transactions_onbook))
finally:
self.lock.release()
def test2(self):
return 'test'
def market_price(self):
self.lock.acquire()
try:
if gv.market_price is not None:
print('match {}'.format(gv.market_price))
pass
finally:
self.lock.release()
class OrderTree(object):
'''A red-black tree used to store OrderLists in price order
The exchange will be using the OrderTree to hold bid and ask data (one OrderTree for each side).
Keeping the information in a red black tree makes it easier/faster to detect a match.
'''
def __init__(self):
self.price_tree = RBTree()
self.price_map = {} # Dictionary containing price : OrderList object
self.order_map = {} # Dictionary containing order_id : Order object
self.volume = 0 # Contains total quantity from all Orders in tree
self.num_orders = 0 # Contains count of Orders in tree
self.depth = 0 # Number of different prices in tree (http://en.wikipedia.org/wiki/Order_book_(trading)#Book_depth)
def __len__(self):
return len(self.order_map)
def get_price_list(self, price):
return self.price_map[price]
def get_order(self, order_id):
return self.order_map[order_id]
def create_price(self, price):
self.depth += 1 # Add a price depth level to the tree
new_list = OrderList()
self.price_tree.insert(price, new_list) # Insert a new price into the tree
self.price_map[price] = new_list # Can i just get this by using self.price_tree.get_value(price)? Maybe this is faster though.
def remove_price(self, price):
self.depth -= 1 # Remove a price depth level
self.price_tree.remove(price)
del self.price_map[price]
def price_exists(self, price):
return price in self.price_map
def order_exists(self, order):
return order in self.order_map
def insert_order(self, quote):
print quote
if self.order_exists(quote['order_id']):
self.remove_order_by_id(quote['order_id'])
self.num_orders += 1
if quote['price'] not in self.price_map:
self.create_price(quote['price']) # If price not in Price Map, create a node in RBtree
order = Order(quote, self.price_map[quote['price']]) # Create an order
self.price_map[order.price].append_order(order) # Add the order to the OrderList in Price Map
self.order_map[order.order_id] = order
self.volume += order.quantity
def update_order(self, order_update):
order = self.order_map[order_update['order_id']]
original_quantity = order.quantity
if order_update['price'] != order.price:
# Price changed. Remove order and update tree.
order_list = self.price_map[order.price]
order_list.remove_order(order)
if len(order_list) == 0: # If there is nothing else in the OrderList, remove the price from RBtree
self.remove_price(order.price)
self.insert_order(order_update)
else:
# Quantity changed. Price is the same.
order.update_quantity(order_update['quantity'], order_update['timestamp'])
self.volume += order.quantity - original_quantity
def remove_order_by_id(self, order_id):
self.num_orders -= 1
order = self.order_map[order_id]
self.volume -= order.quantity
order.order_list.remove_order(order)
if len(order.order_list) == 0:
self.remove_price(order.price)
del self.order_map[order_id]
def max_price(self):
if self.depth > 0:
return self.price_tree.max_key()
else:
return None
def min_price(self):
if self.depth > 0:
return self.price_tree.min_key()
else:
return None
def max_price_list(self):
if self.depth > 0:
return self.get_price_list(self.max_price())
else:
return None
def min_price_list(self):
if self.depth > 0:
return self.get_price_list(self.min_price())
else:
return None
1. Simple memory management - simplifies error handling in concurrent code, less I/O hits,
2. Consistent performance because rehashing (expanding the hash table's array) happens on some insertions.
This is important in real-time systems where you want to provide bounds on how long each operation takes
3. Keys are sorted
'''
#https://bitbucket.org/mozman/bintrees
from bintrees import RBTree
def PrintUnique2(k,v):
if (v == 1): #unique
print(k) #print unique
#Initialize
tree = RBTree()
#Insert
for x in input:
if (tree.get(x)): #duplicate
tree.insert(x,tree.get(x)+1) #increment occurrence
else: #new
tree.insert(x,1)
#Iterate
tree.foreach(PrintUnique2,0) #0=inorder
#Solution 3
'''
REFERENCE:
http://www.geeksforgeeks.org/find-duplicates-in-on-time-and-constant-extra-space/
LIMITATIONS:
1. n elements with value > 0 and < n
2. Logic needs to be modified if 0 is one of the values
random.shuffle(next_orders)
for order in next_orders:
x_traded = [initial_price]
y_traded = [0]
order_side = order['side'] if order['type'] == 'market' else ''
# market_order = order['side'] if order['type'] == 'market' else ''
trades, order = lob.processOrder(order, False, False)
total_orders += 1
if order:
# order was created or not fully executed
idNum = order['idNum']
order['time_limit'] = t + np.random.geometric(
float(df[df['price'] == order['price']]['gamma']))
limit_orders.insert(idNum, order)
born_and_dead_history[idNum] = {
'idNum': idNum,
'born': t,
'dead': order['time_limit']
}
if trades:
for trade in trades:
idNum = trade['party1'][
2] # trade['party1'] = [tid, side, idNum]
born_and_dead_history[idNum]['dead'] = t
x_traded.append(trade['price'])
y_traded.append(trade['qty'])
total_trades += 1
total_traded = add_trades(total_traded, trades)
class OrderTree(object):
''' An RBtree used to store OrderLists in price order '''
def __init__(self):
'''
@summary: initialization of Ordertree class
@mem price_tree: a red-black tree
@mem price_map: Dictionary containing price : OrderList object
@mem order_map: Dictionary containing order_id : Order object
@mem volume: Contains total quantity from all Orders in tree
@mem num_orders: Contains count of Orders in tree
@mem depth: Number of different prices in tree (http://en.wikipedia.org/wiki/Order_book_(trading)#Book_depth)
'''
self.price_tree = RBTree()
self.price_map = {}
self.order_map = {}
self.volume = 0
self.num_orders = 0
self.depth = 0
def __len__(self):
''' return the length of order_map '''
return len(self.order_map)
def get_price_list(self, price):
''' get the price list from the price map '''
return self.price_map[price]
def get_order(self, order_id):
''' get order list from order map '''
return self.order_map[order_id]
def create_price(self, price):
''' create a new price if adding an order without price in the tree '''
self.depth += 1
new_list = OrderList()
self.price_tree.insert(price, new_list)
self.price_map[price] = new_list
def remove_price(self, price):
''' remove a price from the tree '''
self.depth -= 1
self.price_tree.remove(price)
del self.price_map[price]
def price_exists(self, price):
''' check whether price exists in price map '''
return price in self.price_map
def order_exists(self, order):
''' check whether an order exists in order map '''
return order in self.order_map
def insert_order(self, quote):
''' insert an order into the order map '''
if self.order_exists(quote['order_id']):
self.remove_order_by_id(quote['order_id'])
self.num_orders += 1
if quote['price'] not in self.price_map:
self.create_price(quote['price']) # If price not in Price Map, create a node in RBtree
order = Order(quote, self.price_map[quote['price']]) # Create an order
self.price_map[order.price].append_order(order) # Add the order to the OrderList in Price Map
self.order_map[order.order_id] = order
self.volume += order.quantity
def remove_order_by_id(self, order_id):
''' remove an order from the order map '''
self.num_orders -= 1
order = self.order_map[order_id]
self.volume -= order.quantity
order.order_list.remove_order(order)
if len(order.order_list) == 0:
self.remove_price(order.price)
del self.order_map[order_id]
def max_price(self):
''' return max price in price tree '''
if self.depth > 0:
return self.price_tree.max_key()
else:
return None
def min_price(self):
''' return min price in price tree '''
if self.depth > 0:
return self.price_tree.min_key()
else:
return None
def max_price_list(self):
''' return max price in price tree '''
if self.depth > 0:
return self.get_price_list(self.max_price())
else:
return None
def min_price_list(self):
''' return min price in price tree '''
if self.depth > 0:
return self.get_price_list(self.min_price())
else:
return None
class OrderBook(WebsocketClient):
def __init__(self, product_id='BTC-USD', log_to=None):
super(OrderBook, self).__init__(products=product_id)
self._asks = RBTree()
self._bids = RBTree()
self._client = PublicClient()
self._sequence = -1
self._log_to = log_to
if self._log_to:
assert hasattr(self._log_to, 'write')
self._current_ticker = None
@property
def product_id(self):
''' Currently OrderBook only supports a single product even though it is stored as a list of products. '''
return self.products[0]
def on_message(self, message):
if self._log_to:
pickle.dump(message, self._log_to)
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.get_product_order_book(product_id=self.product_id, level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
print('Error: messages missing ({} - {}). Re-initializing websocket.'.format(sequence, self._sequence))
self.close()
self.start()
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
self._current_ticker = message
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
# bid = self.get_bid()
# bids = self.get_bids(bid)
# bid_depth = sum([b['size'] for b in bids])
# ask = self.get_ask()
# asks = self.get_asks(ask)
# ask_depth = sum([a['size'] for a in asks])
# print('bid: %f @ %f - ask: %f @ %f' % (bid_depth, bid, ask_depth, ask))
def on_error(self, e):
self._sequence = -1
self.close()
self.start()
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id'] for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id'] for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append([order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append([order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class OrderTree(object):
"""
A red-black tree used to store OrderLists in price order
The exchange will be using the OrderTree to hold bid and ask data (one OrderTree for each side).
Keeping the information in a red black tree makes it easier/faster to detect a match.
"""
def __init__(self):
self.price_tree = RBTree()
self.price_map = {} # Dictionary containing price : OrderList object
self.order_map = {} # Dictionary containing order_id : Order object
self.volume = 0 # Contains total quantity from all Orders in tree
self.num_orders = 0 # Contains count of Orders in tree
self.depth = 0 # Number of different prices in tree
def __len__(self):
return len(self.order_map)
def get_price_list(self, price):
return self.price_map[price]
def get_order(self, order_id):
return self.order_map[order_id]
def create_price(self, price):
# Add a price depth level to the tree
self.depth += 1
new_list = OrderList()
# Insert a new price into the tree
self.price_tree.insert(price, new_list)
self.price_map[price] = new_list
def remove_price(self, price):
self.depth -= 1 # Remove a price depth level
self.price_tree.remove(price)
del self.price_map[price]
def price_exists(self, price):
return price in self.price_map
def order_exists(self, order):
return order in self.order_map
def insert_order(self, quote):
if self.order_exists(quote['order_id']):
self.remove_order_by_id(quote['order_id'])
self.num_orders += 1
# If price not in Price Map, create a node in RBtree
# Create an order
# Add the order to the OrderList in Price Map
if quote['price'] not in self.price_map:
self.create_price(quote['price'])
order = Order(quote, self.price_map[quote['price']])
self.price_map[order.price].append_order(order)
self.order_map[order.order_id] = order
self.volume += order.quantity
def update_order(self, order_update):
order = self.order_map[order_update['order_id']]
original_quantity = order.quantity
if order_update['price'] != order.price:
# Price changed. Remove order and update tree.
order_list = self.price_map[order.price]
order_list.remove_order(order)
# If there is nothing else in the OrderList, remove the price from RBtree
if len(order_list) == 0:
self.remove_price(order.price)
self.insert_order(order_update)
else:
# Quantity changed. Price is the same.
order.update_quantity(order_update['quantity'],
order_update['timestamp'])
self.volume += order.quantity - original_quantity
def remove_order_by_id(self, order_id):
self.num_orders -= 1
order = self.order_map[order_id]
self.volume -= order.quantity
order.order_list.remove_order(order)
if len(order.order_list) == 0:
self.remove_price(order.price)
del self.order_map[order_id]
def max_price(self):
if self.depth > 0:
return self.price_tree.max_key()
else:
return None
def min_price(self):
if self.depth > 0:
return self.price_tree.min_key()
else:
return None
def max_price_list(self):
if self.depth > 0:
return self.get_price_list(self.max_price())
else:
return None
def min_price_list(self):
if self.depth > 0:
return self.get_price_list(self.min_price())
else:
return None
