class BrownianVariableHistory(object):
''' Represents the set of known time value pairs for a particular Brownian variable '''
def __init__(self):
self._historyTree = RBTree()
def insertData(self, t, val):
'''
Inserts a data point into the history object
t (float) : time
val (float) : value
'''
self._historyTree.insert(t, val)
def getMartingaleRelevantPoints(self, t):
'''
Returns 2 data points. The first will be the data point with the
largest 't' in the history that is still smaller than the given user
provided argument 't'. The second will be the datapoint with the
smallest 't' that is still larger than the user provided 't'. If one
or both of the data points do not exist, this function will return
None in that data point's place.
t (float) : time
returns ((t1,val1), (t2,val2)) where t1, t2, val1, val2 are floats : 2 data points
Ex: bh.getMartingaleRelevantPoints(3.1) == ((3.0, 0.07), (3.5, 0.21))
bh.getMartingaleRelevantPoints(3.6) == ((3.5, 0.21), None)
'''
if self._historyTree.is_empty():
return None, None
leftPoint = None
rightPoint = None
if self._historyTree.min_key() <= t:
leftPoint = self._historyTree.floor_item(t)
if self._historyTree.max_key() >= t:
rightPoint = self._historyTree.ceiling_item(t)
return leftPoint, rightPoint
class BitMEXBook:
def __init__(self,
endpoint="https://www.bitmex.com/api/v1",
symbol='XBTUSD'):
'''Connect to the websocket and initialize data stores.'''
self.logger = logging.getLogger(__name__)
self.logger.debug("Initializing WebSocket.")
self.endpoint = endpoint
self.symbol = symbol
self.data = {}
self.keys = {}
self.exited = False
self._asks = RBTree()
self._bids = RBTree()
# We can subscribe right in the connection querystring, so let's build that.
# Subscribe to all pertinent endpoints
wsURL = self.__get_url()
self.logger.debug("Connecting to %s" % wsURL)
self.__connect(wsURL, symbol)
self.logger.info('Connected to WS, waiting for partials.')
# Connected. Wait for partials
self.__wait_for_symbol(symbol)
self.logger.info('Got all market data. Starting.')
def init(self):
self.logger.debug("Initializing WebSocket...")
self.data = {}
self.keys = {}
self.exited = False
wsURL = self.__get_url()
self.logger.debug("Connecting to URL -- %s" % wsURL)
self.__connect(wsURL, self.symbol)
self.logger.info('Connected to WS, waiting for partials.')
# Connected. Wait for partials
self.__wait_for_symbol(self.symbol)
self.logger.info('Got all market data. Starting.')
def error(self, err):
self._error = err
self.logger.error(err)
#self.exit()
def __del__(self):
self.exit()
def reset(self):
self.logger.warning('Websocket resetting...')
self.ws.close()
self.logger.info('Weboscket closed.')
self.logger.info('Restarting...')
self.init()
def exit(self):
'''Call this to exit - will close websocket.'''
self.exited = True
self.ws.close()
### Main orderbook function
def get_current_book(self):
result = {'asks': [], 'bids': []}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'],
order['id']]) #(order['size']/Decimal(order['price']))
# Same procedure for bids
for bid in self._bids:
try:
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'],
order['id']]) #(order['size']/Decimal(order['price']))
return result
# -----------------------------------------------------------------------------------------
# ----------------------RBTrees Handling---------------------------------------------------
# -----------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------
# Get current minimum ask price from tree
def get_ask(self):
return self._asks.min_key()
# Get ask given id
def get_asks(self, id):
return self._asks.get(id)
# Remove ask form tree
def remove_asks(self, id):
self._asks.remove(id)
# Insert ask into tree
def set_asks(self, id, asks):
self._asks.insert(id, asks)
# Get current maximum bid price from tree
def get_bid(self):
return self._bids.max_key()
# Get bid given id
def get_bids(self, id):
return self._bids.get(id)
# Remove bid form tree
def remove_bids(self, id):
self._bids.remove(id)
# Insert bid into tree
def set_bids(self, id, bids):
self._bids.insert(id, bids)
# Add order to out watched orders
def add(self, order):
order = {
'id': order['id'], # Order id data
'side': order['side'], # Order side data
'size': Decimal(order['size']), # Order size data
'price': order['price'] # Order price data
}
if order['side'] == 'Buy':
bids = self.get_bids(order['id'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['id'], bids)
else:
asks = self.get_asks(order['id'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['id'], asks)
# Order is done, remove it from watched orders
def remove(self, order):
oid = order['id']
if order['side'] == 'Buy':
bids = self.get_bids(oid)
if bids is not None:
bids = [o for o in bids if o['id'] != order['id']]
if len(bids) > 0:
self.set_bids(oid, bids)
else:
self.remove_bids(oid)
else:
asks = self.get_asks(oid)
if asks is not None:
asks = [o for o in asks if o['id'] != order['id']]
if len(asks) > 0:
self.set_asks(oid, asks)
else:
self.remove_asks(oid)
# Updating order price and size
def change(self, order):
new_size = Decimal(order['size'])
# Bitmex updates don't come with price, so we use the id to match it instead
oid = order['id']
if order['side'] == 'Buy':
bids = self.get_bids(oid)
if bids is None or not any(o['id'] == order['id'] for o in bids):
return
index = list(map(itemgetter('id'), bids)).index(order['id'])
bids[index]['size'] = new_size
self.set_bids(oid, bids)
else:
asks = self.get_asks(oid)
if asks is None or not any(o['id'] == order['id'] for o in asks):
return
index = list(map(itemgetter('id'), asks)).index(order['id'])
asks[index]['size'] = new_size
self.set_asks(oid, asks)
tree = self._asks if order['side'] == 'Sell' else self._bids
node = tree.get(oid)
if node is None or not any(o['id'] == order['id'] for o in node):
return
# -----------------------------------------------------------------------------------------
# ----------------------WS Private Methods-------------------------------------------------
# -----------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------
def __connect(self, wsURL, symbol):
'''Connect to the websocket in a thread.'''
self.logger.debug("Starting thread")
self.ws = websocket.WebSocketApp(wsURL,
on_message=self.__on_message,
on_close=self.__on_close,
on_open=self.__on_open,
on_error=self.__on_error)
self.wst = threading.Thread(target=lambda: self.ws.run_forever())
self.wst.daemon = True
self.wst.start()
self.logger.debug("Started thread")
# Wait for connect before continuing
conn_timeout = CONN_TIMEOUT
while (not self.ws.sock
or not self.ws.sock.connected) and conn_timeout:
sleep(1)
conn_timeout -= 1
if not conn_timeout:
self.logger.error("Couldn't connect to WS! Exiting.")
# self.exit()
# raise websocket.WebSocketTimeoutException('Couldn\'t connect to WS! Exiting.')
self.reset()
def __get_url(self):
'''
Generate a connection URL. We can define subscriptions right in the querystring.
Most subscription topics are scoped by the symbol we're listening to.
'''
symbolSubs = ["orderBookL2"]
subscriptions = [sub + ':' + self.symbol for sub in symbolSubs]
urlParts = list(urllib.parse.urlparse(self.endpoint))
urlParts[0] = urlParts[0].replace('http', 'ws')
urlParts[2] = "/realtime?subscribe={}".format(','.join(subscriptions))
return urllib.parse.urlunparse(urlParts)
def __wait_for_symbol(self, symbol):
'''On subscribe, this data will come down. Wait for it.'''
pbar = tqdm(total=160)
# Wait until data reaches our RBTrees
while self._asks.is_empty() and self._bids.is_empty():
sleep(0.1)
pbar.update(3)
pbar.close()
def __send_command(self, command, args=None):
'''Send a raw command.'''
if args is None:
args = []
self.ws.send(json.dumps({"op": command, "args": args}))
def __on_message(self, message):
'''Handler for parsing WS messages.'''
message = json.loads(message)
# self.logger.debug(json.dumps(message))
table = message['table'] if 'table' in message else None
action = message['action'] if 'action' in message else None
# table = message.get("table")
# action = message.get("action")
try:
# RBTrees for orderBook
if table == 'orderBookL2':
# For every order received
try:
for order in message['data']:
if action == 'partial':
self.logger.debug('%s: adding partial %s' %
(table, order))
self.add(order)
elif action == 'insert':
self.logger.debug('%s: inserting %s' %
(table, order))
self.add(order)
elif action == 'update':
self.logger.debug('%s: updating %s' %
(table, order))
self.change(order)
elif action == 'delete':
self.logger.debug('%s: deleting %s' %
(table, order))
self.remove(order)
else:
raise Exception("Unknown action: %s" % action)
except:
self.logger.error('Error handling RBTrees: %s' %
traceback.format_exc())
# Uncomment this to watch RBTrees evolution in real time
# self.logger.info('==============================================================')
# self.logger.info('=============================ASKS=============================')
# self.logger.info('==============================================================')
# self.logger.info(self._asks)
# self.logger.info('==============================================================')
# self.logger.info('=============================BIDS=============================')
# self.logger.info('==============================================================')
# self.logger.info(self._bids)
except:
self.logger.error(traceback.format_exc())
def __on_error(self, error):
'''Called on fatal websocket errors. We exit on these.'''
if not self.exited:
self.logger.error("Error : %s" % error)
raise websocket.WebSocketException(error)
def __on_open(self):
'''Called when the WS opens.'''
self.logger.debug("Websocket Opened.")
def __on_close(self):
'''Called on websocket close.'''
self.logger.info('Websocket Closed')
class CompletedKeys(object):
def __init__(self, max_index, min_index=0):
self._max_index = max_index
self._min_index = min_index
self.num_remaining = max_index - min_index
self._slabs = RBTree()
def _get_previous_or_none(self, index):
try:
return self._slabs.floor_item(index)
except KeyError:
return None
def is_available(self, index):
logger.debug("Testing index %s", index)
if index >= self._max_index or index < self._min_index:
logger.debug("Index out of range")
return False
try:
prev_start, prev_length = self._slabs.floor_item(index)
logger.debug("Prev range: %s-%s", prev_start,
prev_start + prev_length)
return (prev_start + prev_length) <= index
except KeyError:
return True
def mark_completed(self, start_index, past_last_index):
logger.debug("Marking the range completed: %s-%s", start_index,
past_last_index)
num_completed = min(past_last_index, self._max_index) - max(
start_index, self._min_index)
# Find the item directly before this and see if there is overlap
to_discard = set()
try:
prev_start, prev_length = self._slabs.floor_item(start_index)
max_prev_completed = prev_start + prev_length
if max_prev_completed >= start_index:
# we are going to merge with the range before us
logger.debug("Merging with the prev range: %s-%s", prev_start,
prev_start + prev_length)
to_discard.add(prev_start)
num_completed = max(
num_completed - (max_prev_completed - start_index), 0)
start_index = prev_start
past_last_index = max(past_last_index,
prev_start + prev_length)
except KeyError:
pass
# Find all keys between the start and last index and merge them into one block
for merge_start, merge_length in self._slabs.iter_items(
start_index, past_last_index + 1):
if merge_start in to_discard:
logger.debug("Already merged with block %s-%s", merge_start,
merge_start + merge_length)
continue
candidate_next_index = merge_start + merge_length
logger.debug("Merging with block %s-%s", merge_start,
candidate_next_index)
num_completed -= merge_length - max(
candidate_next_index - past_last_index, 0)
to_discard.add(merge_start)
past_last_index = max(past_last_index, candidate_next_index)
# write the new block which is fully merged
discard = False
if past_last_index >= self._max_index:
logger.debug("Discarding block and setting new max to: %s",
start_index)
self._max_index = start_index
discard = True
if start_index <= self._min_index:
logger.debug("Discarding block and setting new min to: %s",
past_last_index)
self._min_index = past_last_index
discard = True
if to_discard:
logger.debug("Discarding %s obsolete blocks", len(to_discard))
self._slabs.remove_items(to_discard)
if not discard:
logger.debug("Writing new block with range: %s-%s", start_index,
past_last_index)
self._slabs.insert(start_index, past_last_index - start_index)
# Update the number of remaining items with the adjustments we've made
assert num_completed >= 0
self.num_remaining -= num_completed
logger.debug("Total blocks: %s", len(self._slabs))
def get_block_start_index(self, block_size_estimate):
logger.debug("Total range: %s-%s", self._min_index, self._max_index)
if self._max_index <= self._min_index:
raise NoAvailableKeysError(
"All indexes have been marked completed")
num_holes = len(self._slabs) + 1
random_hole = random.randint(0, num_holes - 1)
logger.debug("Selected random hole %s with %s total holes",
random_hole, num_holes)
hole_start = self._min_index
past_hole_end = self._max_index
# Now that we have picked a hole, we need to define the bounds
if random_hole > 0:
# There will be a slab before this hole, find where it ends
bound_entries = self._slabs.nsmallest(random_hole + 1)[-2:]
left_index, left_len = bound_entries[0]
logger.debug("Left range %s-%s", left_index, left_index + left_len)
hole_start = left_index + left_len
if len(bound_entries) > 1:
right_index, right_len = bound_entries[1]
logger.debug("Right range %s-%s", right_index,
right_index + right_len)
past_hole_end, _ = bound_entries[1]
elif not self._slabs.is_empty():
right_index, right_len = self._slabs.nsmallest(1)[0]
logger.debug("Right range %s-%s", right_index,
right_index + right_len)
past_hole_end, _ = self._slabs.nsmallest(1)[0]
# Now that we have our hole bounds, select a random block from [0:len - block_size_estimate]
logger.debug("Selecting from hole range: %s-%s", hole_start,
past_hole_end)
rand_max_bound = max(hole_start, past_hole_end - block_size_estimate)
logger.debug("Rand max bound: %s", rand_max_bound)
return random.randint(hole_start, rand_max_bound)
class MarinerOrderBook(gdax.OrderBook):
current_milli_time = lambda self: int(time.time() * 1000)
def __init__(self, ticker, threshold):
gdax.OrderBook.__init__(self, product_id=ticker)
self._threshold = threshold
self.bookChanged = None
self.whaleEnteredMarket = None
self.whaleExitedMarket = None
def registerHandlers(self, bookChangedHandler, whaleEnteredMarketHandler,
whaleExitedMarketHandler, whaleChangedHandler):
Logging.logger.info("registering callbacks...")
self.bookChanged = bookChangedHandler
self.whaleEnteredMarket = whaleEnteredMarketHandler
self.whaleExitedMarket = whaleExitedMarketHandler
self.whaleChanged = whaleChangedHandler
def onMessage(self, message):
#Logging.logger.info(message)
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.getProductOrderBook(level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
Logging.logger.error(
'Error: messages missing ({} - {}). Re-initializing websocket.'
.format(sequence, self._sequence))
self.close()
self.start()
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
if not self.bookChanged is None:
self.bookChanged()
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
if not self.whaleEnteredMarket == None and self.isWhale(order['size']):
self.whaleEnteredMarket(order)
def remove(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
if not self.whaleExitedMarket == None and self.isWhale(order['size']):
self.whaleExitedMarket(order)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
price = Decimal(order['price'])
new_volume = Decimal(order['new_size'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = map(itemgetter('id'), bids).index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = map(itemgetter('id'), asks).index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
if not self.whaleChanged == None and self.isWhale(order['size']):
self.whaleChanged(order)
def get_current_book(self,
num_levels=10
): #fetch only 100 levels off book either way
book = []
for index, bid_entry in enumerate(self._bids.items([True])):
if index == num_levels:
break
else:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
thisBid = bid_entry[1]
except KeyError:
continue
for order in thisBid:
order['price'] = Decimal128(order['price'])
order['size'] = Decimal128(order['size'])
book.append(order)
for index, ask_entry in enumerate(self._asks.items()):
if index == num_levels:
break
else:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
thisAsk = ask_entry[1]
except KeyError:
continue
for order in thisAsk:
order['price'] = Decimal128(order['price'])
order['size'] = Decimal128(order['size'])
book.append(order)
book_frame = pd.DataFrame(book)
return book_frame
def isWhale(self, aVolume):
return aVolume >= self._threshold
#For general book management purposes
def get_bid(self):
if not self._bids.is_empty():
return self._bids.max_key()
else:
return None
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
def get_ask(self):
if not self._asks.is_empty():
return self._asks.min_key()
else:
return None
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
class Dijkstra:
def __init__(self, edgesDict, uToV, origNode, destNode, estimationModel):
#self.pointList = []
self.passedNodesSet = set()
self.notPassedNodeDict = dict()
#self.notPassedNodeHeapq = []
#self.notPassedNodeDict = dict()
self.edgesDict = edgesDict
self.uToV = uToV
self.dummyWindow = Window(-1, -1, -1, -1)
self.origNode = origNode
self.destNode = destNode
self.estimationModel = estimationModel
self.dummyOriNodeInPathGraph = self.generateDummyOirNode()
self.notPassedNodeDict[self.dummyOriNodeInPathGraph] = 0
self.dummyDestNodeInPathGraph = self.generateDummyDestNode()
self.notPassedNodeQ = RBTree()
self.notPassedNodeQ.insert(self.dummyOriNodeInPathGraph, 0)
listOfNodesFromOrig = self.uToV[self.origNode]
self.initializeQ(listOfNodesFromOrig)
if not len(listOfNodesFromOrig):
print("not path from node:", self.origNode)
self.__initializedStatus = False
else:
self.__initializedStatus = True
def generateDummyOirNode(self):
return NodeInPathGraph(self.dummyWindow, self.origNode, None, -1, 0)
def generateDummyDestNode(self):
return NodeInPathGraph(self.dummyWindow, -1, None, -1, inf)
def initializeQ(self, listOfNodesFromOrig):
# print(edgesGdfFromNode.iloc[0])
for edgeIdAndV in listOfNodesFromOrig:
edgeIdInGdf = edgeIdAndV[0]
nextNodeId = edgeIdAndV[1]
nextWindow = Window(self.dummyWindow.prevSeg,
self.dummyWindow.midSeg,
self.dummyWindow.sucSeg, edgeIdInGdf)
nextNode = NodeInPathGraph(nextWindow, nextNodeId,
self.dummyOriNodeInPathGraph,
edgeIdInGdf, 0)
self.notPassedNodeQ.insert(nextNode, 0)
self.notPassedNodeDict[nextNode] = 0
return
def routing(self):
steps = 0
if self.checkIniStatus():
while not self.ifFinished():
if self.noNodeToExplore():
print("No route")
return
else:
steps += 1
self.onePaceUpdate()
pathWitMinVal = self.generateMinValNodePath()
edgePathWithMinVal = self.generateMinValEdgePath()
print("num of steps:", steps)
return pathWitMinVal, self.minVal, edgePathWithMinVal
return
def checkIniStatus(self):
return self.__initializedStatus
def ifFinished(self):
return self.dummyDestNodeInPathGraph in self.passedNodesSet
def noNodeToExplore(self):
return self.notPassedNodeQ.is_empty()
def onePaceUpdate(self):
curNodeInPathGraph, valOfCurNode = self.notPassedNodeQ.min_item()
self.notPassedNodeQ.remove(curNodeInPathGraph)
_ = self.notPassedNodeDict.pop(curNodeInPathGraph)
self.passedNodesSet.add(curNodeInPathGraph)
if self.isDestNode(curNodeInPathGraph):
self.minVal = valOfCurNode
self.destNodeGenerated = curNodeInPathGraph
return
self.exploreNextNode(curNodeInPathGraph, valOfCurNode)
def isDestNode(self, curNodeInPathGraph):
return curNodeInPathGraph == self.dummyDestNodeInPathGraph
def exploreNextNode(self, curNodeInPathGraph, valOfCurNode):
if self.isNoSucNode(curNodeInPathGraph):
self.exploreDummySucNode(curNodeInPathGraph, valOfCurNode)
else:
self.exploreNextNodeFromEdge(curNodeInPathGraph, valOfCurNode)
return
def isNoSucNode(self, curNodeInPathGraph):
return curNodeInPathGraph.node == -1 or curNodeInPathGraph.node == self.destNode
def exploreDummySucNode(self, curNodeInPathGraph, valOfCurNode):
nextNodeId = -1
nextWindow = Window(curNodeInPathGraph.window.prevSeg,
curNodeInPathGraph.window.midSeg,
curNodeInPathGraph.window.sucSeg, -1)
nextNodes = NodeInPathGraph(nextWindow, nextNodeId, curNodeInPathGraph,
-1)
valOfNextNode = self.calVal(nextNodes)
self.updateQ(nextNodes, valOfCurNode + valOfNextNode)
return
def exploreNextNodeFromEdge(self, curNodeInPathGraph, valOfCurNode):
listOfNodes = self.uToV[curNodeInPathGraph.node]
for edgeIdAndV in listOfNodes:
edgeIdInGdf = edgeIdAndV[0]
nextNodeId = edgeIdAndV[1]
nextWindow = Window(curNodeInPathGraph.window.prevSeg,
curNodeInPathGraph.window.midSeg,
curNodeInPathGraph.window.sucSeg, edgeIdInGdf)
nextNodeInPathGraph = NodeInPathGraph(nextWindow, nextNodeId,
curNodeInPathGraph,
edgeIdInGdf)
if nextWindow.valid(
) and nextNodeInPathGraph not in self.passedNodesSet:
valOfNextNode = self.calVal(nextNodeInPathGraph)
self.updateQ(nextNodeInPathGraph, valOfNextNode + valOfCurNode)
def updateQ(self, nextNodeInPathGraph, curShortPathEst):
if nextNodeInPathGraph not in self.notPassedNodeDict:
nextNodeInPathGraph.shortPathEst = curShortPathEst
self.notPassedNodeQ.insert(nextNodeInPathGraph,
nextNodeInPathGraph.shortPathEst)
self.notPassedNodeDict[nextNodeInPathGraph] = curShortPathEst
else:
nextNodeInPathGraph.shortPathEst = self.notPassedNodeDict[
nextNodeInPathGraph]
if curShortPathEst < nextNodeInPathGraph.shortPathEst:
self.notPassedNodeQ.remove(nextNodeInPathGraph)
nextNodeInPathGraph.shortPathEst = curShortPathEst
self.notPassedNodeQ.insert(nextNodeInPathGraph,
nextNodeInPathGraph.shortPathEst)
self.notPassedNodeDict[nextNodeInPathGraph] = curShortPathEst
def calVal(self, curNodeInPathGraph):
if curNodeInPathGraph.window.midSeg == -1:
return 0
else:
numericalFeatures, categoricalFeatures = curNodeInPathGraph.window.extractFeatures(
self.edgesDict)
return self.estimationModel.predictFromData(
numericalFeatures, categoricalFeatures)
def generateMinValNodePath(self):
dNode = self.destNodeGenerated
pathWitMinVal = [dNode.node]
while dNode.prevNode:
dNode = dNode.prevNode
pathWitMinVal.append(dNode.node)
return pathWitMinVal[:2:-1]
def generateMinValEdgePath(self):
dNode = self.destNodeGenerated
edgePathWithMinVal = [dNode.edge]
while dNode.prevNode:
dNode = dNode.prevNode
edgePathWithMinVal.append(dNode.edge)
return edgePathWithMinVal[-2:2:-1]
class WhaleTracker():
current_milli_time = lambda self: int(time.time() * 1000)
def __init__(self, ticker):
self._ticker = ticker
self._bid_whales = RBTree()
self._ask_whales = RBTree()
def whaleEnteredMarketHandler(self, order):
if order['side'] == 'buy':
self.addBidWhale(order)
else:
self.addAskWhale(order)
def whaleExitedMarketHandler(self, order):
if order['side'] == 'buy':
self.removeBidWhale(order)
else:
self.removeAskWhale(order)
def whaleChangedHandler(self, order):
if order['side'] == 'buy':
self.changeBidWhale(order)
else:
self.changeAskWhale(order)
def addBidWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
bid_whale_order = self.get_bid_whale(price)
if bid_whale_order is None or volume > bid_whale_order.get_volume():
#Logging.logger.info("NEW WHALE ENTERED (BID): price=" + str(price) + " volume=" + str(volume))
bid_whale_order = WhaleOrder(ID, price, volume)
self.set_bid_whale(price, bid_whale_order)
def addAskWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
ask_whale_order = self.get_ask_whale(price)
if ask_whale_order is None or volume > ask_whale_order.get_volume():
#Logging.logger.info("NEW WHALE ENTERED (ASK): price=" + str(price) + " volume=" + str(volume))
ask_whale_order = WhaleOrder(ID, price, volume)
self.set_ask_whale(price, ask_whale_order)
def removeBidWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
bid_whale_order = self.get_bid_whale(price)
if bid_whale_order is not None and bid_whale_order.get_id() == ID:
#Logging.logger.info("WHALE LEFT (BID): price=" + str(price) + " volume=" + str(volume))
self.remove_bid_whale(price)
def removeAskWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
ask_whale_order = self.get_ask_whale(price)
if ask_whale_order is not None and ask_whale_order.get_id() == ID:
#Logging.logger.info("WHALE LEFT (ASK): price=" + str(price) + " volume=" + str(volume))
self.remove_ask_whale(price)
def changeBidWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
new_volume = order['new_size']
bid_whale_order = self.get_bid_whale(price)
if not bid_whale_order == None:
#Logger.logging.info("WHALE CHANGED (BID): price=" + price + " new_volume=" + new_volume)
if self.isWhale(new_volume):
bid_whale_order.setVolume(new_volume)
self.set_bid_whale(price, bid_whale_order)
else:
self.removeBidWhale(order)
def changeAskWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
new_volume = order['new_size']
ask_whale_order = self.get_ask_whale(price)
if not ask_whale_order == None:
#Logger.logging.info("WHALE CHANGED (ASK): price=" + price + " new_volume=" + new_volume)
if self.isWhale(new_volume):
ask_whale_order.setVolume(new_volume)
self.set_ask_whale(price, ask_whale_order)
else:
self.removeAskWhale(order)
def get_current_whales(
self,
num_whales=30
): #by default, fetch only 30 whales off the book either way
whales = []
for index, bid_whale in self._bid_whales.items(True):
if index == num_whales:
break
else:
whales.append({
'price': Decimal128(bid_whale.get_price()),
'volume': Decimal128(bid_whale.get_volume()),
'id': bid_whale.get_id(),
})
for index, ask_whale in self._ask_whales.items():
if index == num_whales:
break
else:
whales.append({
'price': Decimal128(ask_whale.get_price()),
'volume': Decimal128(ask_whale.get_volume()),
'id': ask_whale.get_id(),
})
whales_frame = pd.DataFrame(whales)
return whales_frame
def get_top_bid_whale(self):
if not self._bid_whales.is_empty():
top_bid = self._bid_whales.max_key()
return self.get_bid_whale(top_bid)
else:
return None
def get_bid_whale(self, price):
return self._bid_whales.get(price)
def remove_bid_whale(self, price):
self._bid_whales.remove(price)
def set_bid_whale(self, price, whale):
self._bid_whales.insert(price, whale)
def get_top_ask_whale(self):
if not self._ask_whales.is_empty():
top_ask = self._ask_whales.min_key()
return self.get_ask_whale(top_ask)
else:
return None
def get_ask_whale(self, price):
return self._ask_whales.get(price)
def remove_ask_whale(self, price):
self._ask_whales.remove(price)
def set_ask_whale(self, price, whale):
self._ask_whales.insert(price, whale)
class Exchange(Entity):
def __init__(self, coin_type, price_type, bids=None, asks=None):
self.coin_type = coin_type
self.price_type = price_type
self.bids = RBTree(bids or {})
self.asks = RBTree(asks or {})
def __eq__(self, other):
return self.coin_type == other.coin_type and \
self.price_type == other.price_type and \
list(self.bids) == list(self.asks)
@property
def id(self):
return "%s-%s" % (self.coin_type, self.price_type)
def enqueue(self, order):
rbtree = self._find_rbtree(order)
queue = rbtree.setdefault(order.price, collections.deque())
queue.append(order.id)
rbtree[order.price] = queue
def dequeue(self, order):
rbtree = self._find_rbtree(order)
self._discard(rbtree, order.price, order.id)
def dequeue_if_completed(self, order):
if order.is_completed():
self.dequeue(order)
def is_empty(self):
return self.bids.is_empty() and self.asks.is_empty()
# 最高买价大于等于最低卖价
def match(self, pop=False):
bid_price, ask_price = None, None
try:
bid_price = self.bids.max_key()
ask_price = self.asks.min_key()
except ValueError:
return (None, None)
if bid_price >= ask_price:
bids_queue = self.bids[bid_price]
asks_queue = self.asks[ask_price]
bid_id = bids_queue[0]
ask_id = asks_queue[0]
if pop:
self._discard(self.bids, bid_price, bid_id)
self._discard(self.asks, ask_price, ask_id)
return (bid_id, ask_id)
return (None, None)
@classmethod
def compute_deals(cls, bid, ask):
assert type(bid) is BidOrder
assert type(ask) is AskOrder
assert bid.price >= ask.price
assert bid.rest_amount > 0
assert ask.rest_amount > 0
timestamp = int(time.time())
# 卖出申报价格低于即时揭示的最高买入申报价格时，以即时揭示的最高买入申报价格为成交价。
# 买入申报价格高于即时揭示的最低卖出申报价格时，以即时揭示的最低卖出申报价格为成交价。
if bid.timestamp > ask.timestamp:
deal_price = ask.price
else:
deal_price = bid.price
# 这里需要 round(:down)，不然会导致成交额大于委托额
deal_amount = min(bid.rest_amount, ask.rest_amount).quantize(PRECISION_EXP, ROUND_DOWN)
ask_outcome = deal_amount
bid_outcome_origin = (ask_outcome * deal_price).quantize(PRECISION_EXP, ROUND_DOWN)
# 买单手续费 = 买单支出部分 * 买单手续费率，加在买单支出上
# 卖单手续费 = 卖单收入部分 * 卖单手续费率，扣在卖单收入里
bid_fee = (bid_outcome_origin * bid.fee_rate).quantize(PRECISION_EXP, ROUND_DOWN)
ask_fee = (bid_outcome_origin * ask.fee_rate).quantize(PRECISION_EXP, ROUND_DOWN)
bid_outcome = bid_outcome_origin + bid_fee
# 买单收入 = 卖单支出
# 卖单收入 = 买单支出 - 卖单手续费
bid_income = ask_outcome
ask_income = bid_outcome_origin - ask_fee
# 记录订单未结清的额度
bid_rest_amount = bid.rest_amount - deal_amount
ask_rest_amount = ask.rest_amount - deal_amount
bid_rest_freeze_amount = bid.rest_freeze_amount - bid_outcome
ask_rest_freeze_amount = ask.rest_freeze_amount - ask_outcome
bid_deal = Deal(bid.id, ask.id, deal_price, deal_amount, bid_rest_amount, bid_rest_freeze_amount, bid_income, bid_outcome, bid_fee, timestamp)
ask_deal = Deal(ask.id, bid.id, deal_price, deal_amount, ask_rest_amount, ask_rest_freeze_amount, ask_income, ask_outcome, ask_fee, timestamp)
return (bid_deal, ask_deal)
def match_and_compute_deals(self, repo):
bid_id, ask_id = self.match()
if not bid_id or not ask_id:
return (None, None)
bid = repo.orders.find(bid_id)
ask = repo.orders.find(ask_id)
return Exchange.compute_deals(bid, ask)
def _find_rbtree(self, order):
if order.exchange_id != self.id:
raise ValueError("Order#exchange_id<%s> mismatch with Exchange<%s>" % (order.exchange_id, self.id))
if type(order) is BidOrder:
return self.bids
elif type(order) is AskOrder:
return self.asks
else:
raise ValueError("argument is not an Order")
# 当同价格的队列为空时，删除红黑树中的键
def _discard(self, rbtree, price, order_id):
queue = rbtree.get(price)
if not queue or not order_id in queue:
return
queue.remove(order_id)
if queue == collections.deque():
del rbtree[price]
