class Scope(object):
# Static constants uses for indicating the result of the inserting
# a symbol into the Scope.
INSERT_SUCCESS = 0
INSERT_SHADOWED = 1
INSERT_REDECL = 2
# Initializes the Scope with a balanced Red-Black tree.
def __init__(self):
self.map = RBTree()
# Inserts a symbol into the Scope.
#
# @return INSERT_SUCCESS or INSERT_REDECL.
def insert(self, symbol):
if not isinstance(symbol, Symbol):
raise TypeError("'symbol' is not an instance of Symbol.")
if symbol.identifier not in self.map:
self.map[symbol.identifier] = symbol
return Scope.INSERT_SUCCESS
else:
return Scope.INSERT_REDECL
# Finds a symbol in the Scope.
#
# @param name String identifier for the Symbol to find.
#
# @return Symbol if found or None if not found.
def find(self, name):
return self.map.get(name, None)
# Size of the scope.
#
# @return Number of Symbols in the Scope.
def size(self):
return len(self.map)
# Clones the current Scope and returns a deep copy.
#
# @return Copy of the Scope.
def clone(self):
result = Scope()
for identifier, value in self.map.items():
result.map[identifier] = value.clone()
return result
# @return String representation of the current Scope.
def __repr__(self):
symbols = []
for key, value in self.map.items():
symbols.append(' ' + key + ' : ' + repr(value))
return '\n'.join(symbols)
class OrderBook(WebsocketClient):
def __init__(self, product_id='BTC-USD'):
WebsocketClient.__init__(self, products=product_id)
self._asks = RBTree()
self._bids = RBTree()
self._client = PublicClient(product_id=product_id)
self._sequence = -1
def onMessage(self, message):
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.getProductOrderBook(level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': float(bid[0]),
'size': float(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': float(ask[0]),
'size': float(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
return
elif sequence > self._sequence + 1:
self.close()
self.start()
return
# print(message)
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
# bid = self.get_bid()
# bids = self.get_bids(bid)
# bid_depth = sum([b['size'] for b in bids])
# ask = self.get_ask()
# asks = self.get_asks(ask)
# ask_depth = sum([a['size'] for a in asks])
# print('bid: %f @ %f - ask: %f @ %f' % (bid_depth, bid, ask_depth, ask))
def add(self, order):
order = {
'id':
order['order_id'] if 'order_id' in order else order['id'],
'side':
order['side'],
'price':
float(order['price']),
'size':
float(order['size'])
if 'size' in order else float(order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = float(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = float(order['size'])
price = float(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
new_size = float(order['new_size'])
price = float(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = map(itemgetter('id'), bids).index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = map(itemgetter('id'), asks).index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class OrderBook(WebsocketClient):
def __init__(self,
product_id='BTC-USD',
log_to=None,
autoreconnect=True,
max_retries=3):
super(OrderBook, self).__init__(products=[product_id])
self.channels = ['full']
self._asks = RBTree()
self._bids = RBTree()
self._client = PublicClient()
self._sequence = -1
self._log_to = log_to
self._retries = 0
self._max_retries = max_retries
self._autoreconnect = autoreconnect
self._is_ready = False
if self._log_to:
assert hasattr(self._log_to, 'write')
self._current_ticker = None
@property
def product_id(self):
''' Currently OrderBook only supports a single product even though it is stored as a list of products. '''
return self.products[0]
def is_ready(self):
return self._is_ready
def on_open(self):
self._sequence = -1
self._retries = 0
print("-- Subscribed to OrderBook! --\n")
def on_close(self):
print("\n-- OrderBook Socket Closed! --")
self._is_ready = False
if not self.ws.sock or not self.ws.sock.connected and self._autoreconnect:
self._sequence = -1
if self._retries < self._max_retries:
print('-- OrderBook Reconnecting... --')
self._retries += 1
self.start()
else:
print('-- Could not reconnect, stopping... --')
def reset_book(self):
self._is_ready = False
self._asks = RBTree()
self._bids = RBTree()
res = self._client.get_product_order_book(product_id=self.product_id,
level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
print("\n-- OrderBook Data Initialized --")
self._is_ready = True
self._sequence = res['sequence']
def on_message(self, message):
if self._log_to:
pickle.dump(message, self._log_to)
if 'sequence' not in message:
return
sequence = message['sequence']
if self._sequence == -1:
print("\n-- Initializing OrderBook --")
self.reset_book()
return
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
self.on_sequence_gap(self._sequence, sequence)
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
self._current_ticker = message
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
def on_error(self, ws, e):
super(OrderBook, self).on_error(ws, e)
if not ws.sock or not ws.sock.connected:
self._sequence = -1
self._is_ready = False
if self._retries < self._max_retries:
print('-- OrderBook Disconnected, reconnecting... --')
self._retries += 1
self.start()
else:
print('-- Could not reconnect, stopping... --')
def on_sequence_gap(self, gap_start, gap_end):
self._sequence = -1
self.reset_book()
print('-- Error: messages missing, re-initializing book. --'.format(
gap_start, gap_end, self._sequence))
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
try:
price = Decimal(order['price'])
except KeyError:
return
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class BitMEXBook:
def __init__(self,
endpoint="https://www.bitmex.com/api/v1",
symbol='XBTUSD'):
'''Connect to the websocket and initialize data stores.'''
self.logger = logging.getLogger(__name__)
self.logger.debug("Initializing WebSocket.")
self.endpoint = endpoint
self.symbol = symbol
self.data = {}
self.keys = {}
self.exited = False
self._asks = RBTree()
self._bids = RBTree()
# We can subscribe right in the connection querystring, so let's build that.
# Subscribe to all pertinent endpoints
wsURL = self.__get_url()
self.logger.debug("Connecting to %s" % wsURL)
self.__connect(wsURL, symbol)
self.logger.info('Connected to WS, waiting for partials.')
# Connected. Wait for partials
self.__wait_for_symbol(symbol)
self.logger.info('Got all market data. Starting.')
def init(self):
self.logger.debug("Initializing WebSocket...")
self.data = {}
self.keys = {}
self.exited = False
wsURL = self.__get_url()
self.logger.debug("Connecting to URL -- %s" % wsURL)
self.__connect(wsURL, self.symbol)
self.logger.info('Connected to WS, waiting for partials.')
# Connected. Wait for partials
self.__wait_for_symbol(self.symbol)
self.logger.info('Got all market data. Starting.')
def error(self, err):
self._error = err
self.logger.error(err)
#self.exit()
def __del__(self):
self.exit()
def reset(self):
self.logger.warning('Websocket resetting...')
self.ws.close()
self.logger.info('Weboscket closed.')
self.logger.info('Restarting...')
self.init()
def exit(self):
'''Call this to exit - will close websocket.'''
self.exited = True
self.ws.close()
### Main orderbook function
def get_current_book(self):
result = {'asks': [], 'bids': []}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'],
order['id']]) #(order['size']/Decimal(order['price']))
# Same procedure for bids
for bid in self._bids:
try:
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'],
order['id']]) #(order['size']/Decimal(order['price']))
return result
# -----------------------------------------------------------------------------------------
# ----------------------RBTrees Handling---------------------------------------------------
# -----------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------
# Get current minimum ask price from tree
def get_ask(self):
return self._asks.min_key()
# Get ask given id
def get_asks(self, id):
return self._asks.get(id)
# Remove ask form tree
def remove_asks(self, id):
self._asks.remove(id)
# Insert ask into tree
def set_asks(self, id, asks):
self._asks.insert(id, asks)
# Get current maximum bid price from tree
def get_bid(self):
return self._bids.max_key()
# Get bid given id
def get_bids(self, id):
return self._bids.get(id)
# Remove bid form tree
def remove_bids(self, id):
self._bids.remove(id)
# Insert bid into tree
def set_bids(self, id, bids):
self._bids.insert(id, bids)
# Add order to out watched orders
def add(self, order):
order = {
'id': order['id'], # Order id data
'side': order['side'], # Order side data
'size': Decimal(order['size']), # Order size data
'price': order['price'] # Order price data
}
if order['side'] == 'Buy':
bids = self.get_bids(order['id'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['id'], bids)
else:
asks = self.get_asks(order['id'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['id'], asks)
# Order is done, remove it from watched orders
def remove(self, order):
oid = order['id']
if order['side'] == 'Buy':
bids = self.get_bids(oid)
if bids is not None:
bids = [o for o in bids if o['id'] != order['id']]
if len(bids) > 0:
self.set_bids(oid, bids)
else:
self.remove_bids(oid)
else:
asks = self.get_asks(oid)
if asks is not None:
asks = [o for o in asks if o['id'] != order['id']]
if len(asks) > 0:
self.set_asks(oid, asks)
else:
self.remove_asks(oid)
# Updating order price and size
def change(self, order):
new_size = Decimal(order['size'])
# Bitmex updates don't come with price, so we use the id to match it instead
oid = order['id']
if order['side'] == 'Buy':
bids = self.get_bids(oid)
if bids is None or not any(o['id'] == order['id'] for o in bids):
return
index = list(map(itemgetter('id'), bids)).index(order['id'])
bids[index]['size'] = new_size
self.set_bids(oid, bids)
else:
asks = self.get_asks(oid)
if asks is None or not any(o['id'] == order['id'] for o in asks):
return
index = list(map(itemgetter('id'), asks)).index(order['id'])
asks[index]['size'] = new_size
self.set_asks(oid, asks)
tree = self._asks if order['side'] == 'Sell' else self._bids
node = tree.get(oid)
if node is None or not any(o['id'] == order['id'] for o in node):
return
# -----------------------------------------------------------------------------------------
# ----------------------WS Private Methods-------------------------------------------------
# -----------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------
def __connect(self, wsURL, symbol):
'''Connect to the websocket in a thread.'''
self.logger.debug("Starting thread")
self.ws = websocket.WebSocketApp(wsURL,
on_message=self.__on_message,
on_close=self.__on_close,
on_open=self.__on_open,
on_error=self.__on_error)
self.wst = threading.Thread(target=lambda: self.ws.run_forever())
self.wst.daemon = True
self.wst.start()
self.logger.debug("Started thread")
# Wait for connect before continuing
conn_timeout = CONN_TIMEOUT
while (not self.ws.sock
or not self.ws.sock.connected) and conn_timeout:
sleep(1)
conn_timeout -= 1
if not conn_timeout:
self.logger.error("Couldn't connect to WS! Exiting.")
# self.exit()
# raise websocket.WebSocketTimeoutException('Couldn\'t connect to WS! Exiting.')
self.reset()
def __get_url(self):
'''
Generate a connection URL. We can define subscriptions right in the querystring.
Most subscription topics are scoped by the symbol we're listening to.
'''
symbolSubs = ["orderBookL2"]
subscriptions = [sub + ':' + self.symbol for sub in symbolSubs]
urlParts = list(urllib.parse.urlparse(self.endpoint))
urlParts[0] = urlParts[0].replace('http', 'ws')
urlParts[2] = "/realtime?subscribe={}".format(','.join(subscriptions))
return urllib.parse.urlunparse(urlParts)
def __wait_for_symbol(self, symbol):
'''On subscribe, this data will come down. Wait for it.'''
pbar = tqdm(total=160)
# Wait until data reaches our RBTrees
while self._asks.is_empty() and self._bids.is_empty():
sleep(0.1)
pbar.update(3)
pbar.close()
def __send_command(self, command, args=None):
'''Send a raw command.'''
if args is None:
args = []
self.ws.send(json.dumps({"op": command, "args": args}))
def __on_message(self, message):
'''Handler for parsing WS messages.'''
message = json.loads(message)
# self.logger.debug(json.dumps(message))
table = message['table'] if 'table' in message else None
action = message['action'] if 'action' in message else None
# table = message.get("table")
# action = message.get("action")
try:
# RBTrees for orderBook
if table == 'orderBookL2':
# For every order received
try:
for order in message['data']:
if action == 'partial':
self.logger.debug('%s: adding partial %s' %
(table, order))
self.add(order)
elif action == 'insert':
self.logger.debug('%s: inserting %s' %
(table, order))
self.add(order)
elif action == 'update':
self.logger.debug('%s: updating %s' %
(table, order))
self.change(order)
elif action == 'delete':
self.logger.debug('%s: deleting %s' %
(table, order))
self.remove(order)
else:
raise Exception("Unknown action: %s" % action)
except:
self.logger.error('Error handling RBTrees: %s' %
traceback.format_exc())
# Uncomment this to watch RBTrees evolution in real time
# self.logger.info('==============================================================')
# self.logger.info('=============================ASKS=============================')
# self.logger.info('==============================================================')
# self.logger.info(self._asks)
# self.logger.info('==============================================================')
# self.logger.info('=============================BIDS=============================')
# self.logger.info('==============================================================')
# self.logger.info(self._bids)
except:
self.logger.error(traceback.format_exc())
def __on_error(self, error):
'''Called on fatal websocket errors. We exit on these.'''
if not self.exited:
self.logger.error("Error : %s" % error)
raise websocket.WebSocketException(error)
def __on_open(self):
'''Called when the WS opens.'''
self.logger.debug("Websocket Opened.")
def __on_close(self):
'''Called on websocket close.'''
self.logger.info('Websocket Closed')
print(seen)
PrintUnique1(input) #1,5
#Code: Solution 2 (using using design #1 and red black tree)
'''
Why red and black trees are better than hashtables?
Source: http://www.quora.com/Why-would-anyone-like-to-use-a-red-black-tree-when-a-hash-table-can-do-the-job-perfectly
1. Simple memory management - simplifies error handling in concurrent code, less I/O hits,
2. Consistent performance because rehashing (expanding the hash table's array) happens on some insertions.
This is important in real-time systems where you want to provide bounds on how long each operation takes
3. Keys are sorted
'''
#https://bitbucket.org/mozman/bintrees
from bintrees import RBTree
def PrintUnique2(k,v):
if (v == 1): #unique
print(k) #print unique
#Initialize
tree = RBTree()
#Insert
for x in input:
if (tree.get(x)): #duplicate
tree.insert(x,tree.get(x)+1) #increment occurrence
else: #new
tree.insert(x,1)
#Iterate
tree.foreach(PrintUnique2,0) #0=inorder
class GdaxBookFeed(GdaxWebsocketClient):
def __init__(self,
product_id='LTC-USD',
log_to=None,
gdax=None,
auth=True):
if gdax is None:
from stocklook.crypto.gdax.api import Gdax
gdax = Gdax()
super(GdaxBookFeed, self).__init__(products=product_id,
auth=auth,
api_key=gdax.api_key,
api_secret=gdax.api_secret,
api_passphrase=gdax.api_passphrase)
self._asks = None
self._bids = None
self._client = gdax
self._sequence = -1
self._log_to = log_to
if self._log_to:
assert hasattr(self._log_to, 'write')
self._current_ticker = None
self._key_errs = 0
self._errs = 0
self.message_count = 0
@property
def product_id(self):
'''
Currently OrderBook only supports a single product even
though it is stored as a list of products.
'''
return self.products[0]
def on_message(self, message):
self.message_count += 1
if self._log_to:
pickle.dump(message, self._log_to)
try:
sequence = message['sequence']
except KeyError:
print("Error: {}".format(message))
self._key_errs += 1
if self._key_errs >= 3:
print("3 errors retrieving sequence. Restarting....")
self.close()
self._key_errs = 0
self.start()
return
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.get_book(self.product_id, level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': float(bid[0]),
'size': float(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': float(ask[0]),
'size': float(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book
# initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
print('Error: messages missing ({} - {}). '
'Re-initializing websocket.'.format(sequence,
self._sequence))
self.close()
self.start()
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
self._current_ticker = message
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
# bid = self.get_bid()
# bids = self.get_bids(bid)
# bid_depth = sum([b['size'] for b in bids])
# ask = self.get_ask()
# asks = self.get_asks(ask)
# ask_depth = sum([a['size'] for a in asks])
# print('bid: %f @ %f - ask: %f @ %f' % (bid_depth, bid, ask_depth, ask))
def on_error(self, e):
self._sequence = -1
self._errs += 1
self.close()
if self._errs >= 3:
raise Exception(e)
self.start()
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': float(order['price']),
'size': float(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = float(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = float(order['size'])
price = float(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = float(order['new_size'])
except KeyError:
return
price = float(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here,
# where a price point is removed
# between these two ops.
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
bit = [order['price'], order['size'], order['id']]
result['asks'].append(bit)
for bid in self._bids:
try:
# There can be a race condition here,
# where a price point is removed
# between these two ops.
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'], order['id']])
return result
def get_orders_matching_ids(self, order_ids):
return [a for a in self._asks
if a['id'] in order_ids] +\
[b for b in self._bids
if b['id'] in order_ids
]
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
Source: http://www.quora.com/Why-would-anyone-like-to-use-a-red-black-tree-when-a-hash-table-can-do-the-job-perfectly
1. Simple memory management - simplifies error handling in concurrent code, less I/O hits,
2. Consistent performance because rehashing (expanding the hash table's array) happens on some insertions.
This is important in real-time systems where you want to provide bounds on how long each operation takes
3. Keys are sorted
'''
#https://bitbucket.org/mozman/bintrees
from bintrees import RBTree
def PrintUnique2(k,v):
if (v == 1): #unique
print(k) #print unique
#Initialize
tree = RBTree()
#Insert
for x in input:
if (tree.get(x)): #duplicate
tree.insert(x,tree.get(x)+1) #increment occurrence
else: #new
tree.insert(x,1)
#Iterate
tree.foreach(PrintUnique2,0) #0=inorder
#Solution 3
'''
REFERENCE:
http://www.geeksforgeeks.org/find-duplicates-in-on-time-and-constant-extra-space/
LIMITATIONS:
1. n elements with value > 0 and < n
class MarinerOrderBook(GDAX.OrderBook):
def __init__(self, ticker, threshold):
GDAX.OrderBook.__init__(self, product_id = ticker)
self._threshold = threshold
self._buyWhales = RBTree()
self._sellWhales = RBTree()
self._topBuyWhale = None
self._topSellWhale = None
def registerHandlers(self, topWhaleChangedHandler):
print(" registering callbacks...")
self.topWhaleChangedHandler = topWhaleChangedHandler
def onMessage(self, message):
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.getProductOrderBook(level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
print('Error: messages missing ({} - {}). Re-initializing websocket.'.format(sequence, self._sequence))
self.close()
self.start()
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
self.checkWhaleAddBuy(order)
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
self.checkWhaleAddSell(order)
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def checkWhaleAddBuy(self, order):
price = Decimal(order['price'])
volume = Decimal(order['size'])
if self.isWhale(volume): #too low in practice
print("WHALE ENTERED (BUY): price=" + str(price) + " volume=" + str(volume))
whale_bid_level = self.get_whale_bids(price)
if whale_bid_level is None:
whale_bid_level = WhaleLevel(price, volume)
else:
whale_bid_level.add_volume(volume)
self.set_whale_bids(price, whale_bid_level)
top_whale_price = self.get_whale_bid()
if self._topBuyWhale is None or top_whale_price != self._topBuyWhale.get_price():
self._topBuyWhale = self._buyWhales.get(top_whale_price)
self.topWhaleChangedHandler(self._topBuyWhale, "BUY")
else:
print(" top BUY whale is still " + str(self._topBuyWhale))
def checkWhaleAddSell(self, order):
price = Decimal(order['price'])
volume = Decimal(order['size'])
if self.isWhale(volume): #too low in practice
print("WHALE ENTERED (SELL): price=" + str(price) + " volume=" + str(volume))
whale_ask_level = self.get_whale_asks(price)
if whale_ask_level is None:
whale_ask_level = WhaleLevel(price, volume)
else:
whale_ask_level.add_volume(volume)
self.set_whale_asks(price, whale_ask_level)
top_whale_price = self.get_whale_ask()
if self._topSellWhale is None or top_whale_price != self._topSellWhale.get_price():
self._topSellWhale = self._sellWhales.get(top_whale_price)
self.topWhaleChangedHandler(self._topSellWhale, "SELL")
else:
print(" top SELL whale is still " + str(self._topSellWhale))
def remove(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
price = Decimal(order['price'])
if order['side'] == 'buy':
self.checkWhaleRemoveBuy(order)
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
self.checkWhaleRemoveSell(order)
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def checkWhaleRemoveBuy(self, order):
price = Decimal(order['price'])
volume = Decimal(order['size'])
whale_bid_level = self.get_whale_bids(price)
if whale_bid_level is not None:
whale_bid_level.remove_volume(volume)
print("WHALE LEFT (BUY): price=" + str(price) + " volume=" + str(volume))
if not self.isWhale(whale_bid_level.get_volume()):
self.remove_whale_bids(price)
top_whale_price = self.get_whale_bid()
self._topBuyWhale = self._buyWhales.get(top_whale_price)
self.topWhaleChangedHandler(self._topBuyWhale, "BUY")
else:
print(" top BUY whale is still " + str(self._topBuyWhale))
def checkWhaleRemoveSell(self, order):
price = Decimal(order['price'])
volume = Decimal(order['size'])
whale_ask_level = self.get_whale_asks(price)
if whale_ask_level is not None:
whale_ask_level.remove_volume(volume)
print("WHALE ENTERED (SELL): price=" + str(price) + " volume=" + str(volume))
if not self.isWhale(whale_ask_level.get_volume()):
self.remove_whale_asks(price)
top_whale_price = self.get_whale_ask()
self._topSellWhale = self._sellWhales.get(top_whale_price)
self.topWhaleChangedHandler(self._topSellWhale, "SELL")
else:
print(" top SELL whale is still " + str(self._topSellWhale))
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
price = Decimal(order['price'])
new_volume = Decimal(order['new_size'])
if order['side'] == 'buy':
self.checkWhaleChangeBuy(order)
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id'] for o in bids):
return
index = map(itemgetter('id'), bids).index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
self.checkWhaleChangeSell(order)
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id'] for o in asks):
return
index = map(itemgetter('id'), asks).index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def checkWhaleChangeBuy(self, order):
price = Decimal(order['price'])
new_volume = order['new_size']
whale_bid_level = self.get_whale_bids(price)
if not whale_bid_level == None:
print("BUY WHALE CHANGED: price=" + price + " new_volume=" + new_volume)
whale_bid_level.setVolume(new_volume)
self.set_whale_bids(price, whale_bid_level)
def checkWhaleChangeSell(self, order):
price = Decimal(order['price'])
new_volume = order['new_size']
whale_ask_level = self.get_whale_asks(price)
if not whale_ask_level == None:
print("SELL WHALE CHANGED: price=" + price + " new_volume=" + new_volume)
whale_ask_level.setVolume(new_volume)
self.set_whale_asks(price, whale_ask_level)
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
thisAsk = self._asks[ask]
except KeyError:
continue
for order in thisAsk:
result['asks'].append([
order['price'],
order['size'],
order['id'],
])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
thisBid = self._bids[bid]
except KeyError:
continue
for order in thisBid:
result['bids'].append([
order['price'],
order['size'],
order['id'],
])
return result
def isWhale(self, aVolume):
return aVolume >= self._threshold
#For general book management purposes
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
#For whale book management purposes
def get_whale_ask(self):
return self._sellWhales.min_key()
def get_whale_asks(self, price):
return self._sellWhales.get(price)
def remove_whale_asks(self, price):
self._sellWhales.remove(price)
def set_whale_asks(self, price, asks):
self._sellWhales.insert(price, asks)
def get_whale_bid(self):
return self._buyWhales.max_key()
def get_whale_bids(self, price):
return self._buyWhales.get(price)
def remove_whale_bids(self, price):
self._buyWhales.remove(price)
def set_whale_bids(self, price, bids):
self._buyWhales.insert(price, bids)
class OrderBook():
def __init__(self, market=None, bids=None, asks=None, log_to=None):
self._asks = RBTree()
self._bids = RBTree()
self.book_valid = False
self.new_book(bids, asks)
self._sequence = -1
self.market = market
self._log_to = log_to
if self._log_to:
assert hasattr(self._log_to, 'write')
# My order Details
self.total_order_count = 0
self.total_open_order_count = 0
self.pending_buy_orders_db = {}
self.pending_sell_orders_db = {}
self.traded_buy_orders_db = {}
self.traded_sell_orders_db = {}
# positions
self.all_positions = []
self.open_positions = []
self.close_pending_positions = {}
self.closed_positions = []
self.orderDb = db.OrderDb(Order, market.exchange_name,
market.product_id)
self.positionsDb = db.PositionDb(Position, market.exchange_name,
market.product_id)
#stop loss handling
self.sl_dict = sorteddict.SortedDict()
#take profit handling
self.tp_dict = sorteddict.SortedDict()
#trade Reqs
self.pending_trade_req = [
] # TODO: FIXME: jork: this better be a nice AVL tree of sort
def __str__(self):
return """
{"position_all": %d, "open": %d, "close_pending": %d, "closed": %d}""" % (
len(self.all_positions), len(self.open_positions),
len(self.close_pending_positions), len(self.closed_positions))
def add_pending_trade_req(self, trade_req):
self.pending_trade_req.append(trade_req)
def remove_pending_trade_req(self, trade_req):
# primitive
self.pending_trade_req.remove(trade_req)
def open_position(self, buy_order):
#open positions with buy orders only(we don't support 'short' now)
# log.debug("open_position order: %s"%(buy_order))
position = Position(id=buy_order.id)
position.add_buy(buy_order)
if self.market.tradeConfig["stop_loss_enabled"]:
self.add_stop_loss_position(position, buy_order.get_price(),
self.market, buy_order.stop)
if self.market.tradeConfig["take_profit_enabled"]:
self.add_take_profit_position(position, buy_order.get_price(),
self.market, buy_order.profit)
self.all_positions.append(position)
self.open_positions.append(position)
# log.debug("\n\n\n***open_position: open(%d) closed(%d) close_pend(%d)"%(len(self.open_positions), len(self.closed_positions), len(self.close_pending_positions)))
# save pos to db
self.positionsDb.db_save_position(position)
def get_closable_position(self):
log.info("get_closable_position ")
#get last open position for now
# TODO: FIXME: This may not be the best way. might cause race with below api with multi thread/multi exch
pos = None
if len(self.open_positions):
if self.market.tradeConfig["stop_loss_enabled"]:
log.info("Finding closable position from _stop_loss_ pool.")
pos = self.pop_stop_loss_position()
if pos == None:
try:
pos = self.open_positions.pop()
except IndexError:
log.error("unable to find open position to close")
return None
log.info(
"Found closable position from _regular_ pool. pos: %s" %
(str(pos)))
else:
log.info(
"Found closable position from _stop_loss_ pool. pos: %s" %
(str(pos)))
self.open_positions.remove(pos)
if (self.close_pending_positions.get(pos.id)):
log.critical("""Position already close pending \npos:%s
close_pending_positions: %s
open_positions: %s""" %
(str(pos), str(self.close_pending_positions),
str(self.open_positions)))
raise Exception("Duplicate close pending position")
if self.market.tradeConfig["take_profit_enabled"]:
self.pop_take_profit_position(pos)
self.close_pending_positions[pos.id] = pos
# log.debug("\n\n\n***get_closable_position: open(%d) closed(%d) close_pend(%d) \n pos: %s"%(
# len(self.open_positions), len(self.closed_positions), len(self.close_pending_positions), pos))
return pos
def close_position_pending(self, sell_order):
# Intentionally _not_ updating the pos_db here. Yes, There is a case of wrong pos state if we go down come back during this state
# TODO: FIXME: This may not be the best way. might cause race with below api with multi thread/multi exch
log.info(" order:%s" % (sell_order.id))
# log.debug("\n\n\n***: open(%d) closed(%d) close_pend(%d)\n"%(
# len(self.open_positions), len(self.closed_positions), len(self.close_pending_positions)))
pos = self.close_pending_positions.get(sell_order.id)
if pos:
log.info(": sell order(%s) already in pending_list. do nothing" %
(sell_order.id))
return pos
#find a close_pending pos without sell attached.
k = sell_order._pos_id
if not k:
log.critical(
"*****Invalid pos_id attached to order:%s**** \n may be an outside order"
% (sell_order.id))
raise Exception("Invalid pos_id attached to order")
pos = self.close_pending_positions.get(k)
if pos:
#find the pos without sell attached. and reinsert after attach
# log.debug("pos:\n%s"%(pos))
if pos.sell == None:
pos.add_sell(sell_order)
pos.update_state("close_pending")
del (self.close_pending_positions[k])
self.close_pending_positions[sell_order.id] = pos
# log.debug("\n\n\n***: open(%d) closed(%d) close_pend(%d)"%(len(self.open_positions), len(self.closed_positions), len(self.close_pending_positions)))
return pos
else:
log.critical("Wrong sell attached to pos:%s" % (pos))
raise Exception("Wrong sell attached to pos")
else:
#something is very wrong
log.critical("Unable to find pending position for close id: %s" %
(sell_order.id))
raise Exception("Unable to find pending position for close")
def close_position_failed(self, pos_id):
log.info("close_position_failed order: %s" % (pos_id))
# log.debug("\n\n\n***close_position_failed: open(%d) closed(%d) close_pend(%d)"%(len(self.open_positions), len(self.closed_positions), len(self.close_pending_positions)))
# id = sell_order.id
position = self.close_pending_positions.get(pos_id)
if position:
position.sell = None
self.close_pending_positions.pop(pos_id, None)
self.open_positions.append(position)
if self.market.tradeConfig["stop_loss_enabled"]:
self.add_stop_loss_position(position, position.buy.get_price(),
self.market,
position.get_stop_loss())
if self.market.tradeConfig["take_profit_enabled"]:
self.add_take_profit_position(position,
position.buy.get_price(),
self.market,
position.get_take_profit())
else:
log.critical("Unable to get close_pending position. order_id: %s" %
(pos_id))
def close_position(self, sell_order):
log.info("close_position order: %s" % (sell_order.id))
id = sell_order.id
position = self.close_pending_positions.pop(id, None)
if position:
position.add_sell(sell_order)
position.update_state("closed")
profit = position.get_profit()
self.market.fund.current_realized_profit += profit
if profit > 0:
self.market.num_success_trade += 1
else:
self.market.num_failed_trade += 1
self.closed_positions.append(position)
#update position
self.positionsDb.db_save_position(position)
log.info("position closed pos: %s" % (str(position)))
else:
log.critical("Unable to get close_pending position. order_id: %s" %
(sell_order.id))
# log.debug("\n\n\n***close_position: open(%d) closed(%d) close_pend(%d)\n pos:%s"%(
# len(self.open_positions), len(self.closed_positions), len(self.close_pending_positions), position))
def add_stop_loss_position(self,
position,
market_rate,
market,
stop_price=0):
sl_kind = market.tradeConfig['stop_loss_kind']
if sl_kind in ['trailing', 'simple']:
sl_rate = market.tradeConfig["stop_loss_rate"]
stop_price = float(
round(market_rate * (1 - sl_rate * float(.01)), 4))
elif 'ATR' in sl_kind:
atr = market.get_cur_indicators()[
market.tradeConfig['stop_loss_kind']]
stop_price = float(round(market_rate - 2 * atr, 4))
elif sl_kind == "strategy":
if stop_price == 0:
log.critical("strategy provided invalid stop loss value")
return
else:
raise Exception("Unknown stop_loss kind - " +
market.tradeConfig['stop_loss_kind'])
position.set_stop_loss(stop_price)
pos_list = self.sl_dict.get(stop_price, None)
if pos_list == None:
pos_list = []
self.sl_dict[stop_price] = pos_list
pos_list.append(position)
def add_stop_loss_position_list(self, stop_price, position_l):
[pos.set_stop_loss(stop_price) for pos in position_l]
pos_list = self.sl_dict.get(stop_price, None)
if pos_list == None:
pos_list = []
self.sl_dict[stop_price] = pos_list
pos_list += position_l
def remove_all_positions_at_stop(self, stop_price):
return self.sl_dict.pop(stop_price, None)
def smart_stop_loss_update_positions(self, cur_indicators, market_rate,
tcfg):
if tcfg['stop_loss_kind'] == 'trailing':
sl_rate = tcfg["stop_loss_rate"]
new_sl = float(round(market_rate * (1 - sl_rate * float(.01)), 4))
elif 'ATR' in tcfg['stop_loss_kind']:
atr = cur_indicators[tcfg['stop_loss_kind']]
new_sl = float(round(market_rate - 2 * atr, 4))
else:
raise Exception("Unknown smart stop_loss kind %s" %
(tcfg['stop_loss_kind']))
key_list = list(
self.sl_dict.irange(maximum=new_sl, inclusive=(False, False)))
for key in key_list:
# log.critical("new_sl: %d key: %d"%(new_sl, key))
pos_list = self.sl_dict.pop(key)
self.add_stop_loss_position_list(new_sl, pos_list)
def db_commit_dirty_positions(self):
dirty_pos_list = []
for pos in self.all_positions:
if pos._dirty == True:
pos._dirty = False
dirty_pos_list.append(pos)
if len(dirty_pos_list):
log.debug("commit positions to db")
# save pos to db
self.positionsDb.db_save_positions(dirty_pos_list)
def pop_stop_loss_position(self, pos=None):
try:
sl_price, pos_list = 0, None
if pos:
sl_price = pos.get_stop_loss()
pos_list = self.sl_dict.get(sl_price)
else:
#get the lowest in the sorted SL list
sl_price, pos_list = self.sl_dict.peekitem(index=0)
# log.debug("pop position at sl_price:%d"%(sl_price))
if pos_list and len(pos_list):
if pos:
pos_list.remove(pos)
else:
pos = pos_list.pop()
if len(pos_list) == 0:
del (self.sl_dict[sl_price])
return pos
except (IndexError, ValueError):
return None
def get_stop_loss_positions(self, market_rate):
sl_pos_list = []
key_list = list(
self.sl_dict.irange(minimum=market_rate, inclusive=(True, True)))
# log.critical("slPrice: %d"%market_rate)
# log.critical("key_list :%s"%(key_list))
for key in key_list:
pos_list = self.sl_dict.pop(key)
sl_pos_list += pos_list
for pos in pos_list:
self.open_positions.remove(pos)
if (self.close_pending_positions.get(pos.id)):
log.critical("""Position already close pending \npos:%s
close_pending_positions: %s
open_positions: %s""" %
(str(pos), str(self.close_pending_positions),
str(self.open_positions)))
raise Exception("Duplicate close pending position")
self.close_pending_positions[pos.id] = pos
# remove pos from take profit points
self.pop_take_profit_position(pos)
self.market.num_stop_loss_hit += len(sl_pos_list)
# if len(sl_pos_list):
# log.critical("num_stop_loss_hit: %d slPrice: %d"%(len(sl_pos_list), market_rate))
return sl_pos_list
def get_take_profit_positions(self, market_rate):
tp_pos_list = []
key_list = list(
self.tp_dict.irange(maximum=market_rate, inclusive=(True, True)))
for key in key_list:
pos_list = self.tp_dict.pop(key)
tp_pos_list += pos_list
for pos in pos_list:
self.open_positions.remove(pos)
if (self.close_pending_positions.get(pos.id)):
log.critical("""Position already close pending \npos:%s
close_pending_positions: %s
open_positions: %s""" %
(str(pos), str(self.close_pending_positions),
str(self.open_positions)))
raise Exception("Duplicate close pending position")
self.close_pending_positions[pos.id] = pos
# remove pos from take profit points
self.pop_stop_loss_position(pos)
self.market.num_take_profit_hit += len(tp_pos_list)
return tp_pos_list
def add_take_profit_position(self,
position,
market_rate,
market,
new_tp=0):
tp_kind = market.tradeConfig['take_profit_kind']
if tp_kind == 'simple':
tp_rate = market.tradeConfig["take_profit_rate"]
new_tp = float(round(market_rate * (1 + tp_rate * float(.01)), 4))
elif tp_kind == 'strategy':
if new_tp == 0:
log.critical("strategy provided invalid take profit value")
return
else:
raise Exception("Unknown take profit kind - " +
market.tradeConfig['take_profit_kind'])
position.set_take_profit(new_tp)
pos_list = self.tp_dict.get(new_tp, None)
if pos_list == None:
pos_list = []
self.tp_dict[new_tp] = pos_list
pos_list.append(position)
log.debug("add take profit(%d) market_rate:(%d)" %
(new_tp, market_rate))
def pop_take_profit_position(self, pos=None):
try:
tp_price, pos_list = 0, None
if pos:
tp_price = pos.get_take_profit()
pos_list = self.tp_dict.get(tp_price)
else:
tp_price, pos_list = self.tp_dict.peekitem(0)
# log.debug("pop position at sl_price:%d"%(sl_price))
if pos_list and len(pos_list):
if pos:
pos_list.remove(pos)
else:
pos = pos_list.pop()
if len(pos_list) == 0:
del (self.tp_dict[tp_price])
return pos
except (IndexError, ValueError):
return None
def get_all_pending_orders(self):
pending_order_list = []
pending_order_list += list(self.pending_buy_orders_db.values())
pending_order_list += list(self.pending_sell_orders_db.values())
return pending_order_list
def add_or_update_pending_buy_order(self, order):
id = order.id
cur_order = self.pending_buy_orders_db.get(id)
if not cur_order:
self.total_open_order_count += 1
self.total_order_count += 1
else:
#copy required fields
order.stop = cur_order.stop
order.profit = cur_order.profit
self.pending_buy_orders_db[id] = order
def get_pending_buy_order(self, order_id):
return self.pending_buy_orders_db.get(order_id)
def add_traded_buy_order(self, order):
cur_order = self.pending_buy_orders_db.get(order.id)
if cur_order:
#copy required fields
order.stop = cur_order.stop
order.profit = cur_order.profit
self.total_open_order_count -= 1
del (self.pending_buy_orders_db[order.id])
self.traded_buy_orders_db[order.id] = order
#if this is a successful order, we have a new position open
if order.status == "filled":
self.open_position(order)
def get_traded_buy_order(self, order_id):
return self.traded_buy_orders_db.get(order_id)
def add_or_update_pending_sell_order(self, order):
id = order.id
if not self.pending_sell_orders_db.get(id):
self.total_open_order_count += 1
self.total_order_count += 1
self.pending_sell_orders_db[id] = order
def get_pending_sell_order(self, order_id):
self.pending_sell_orders_db.get(order_id)
def add_traded_sell_order(self, order):
del (self.pending_sell_orders_db[order.id])
self.total_open_order_count -= 1
self.traded_sell_orders_db[order.id] = order
#close/reopen position
#TODO: TBD: more checks required??
if order.status == "filled":
log.debug("closed position order: %s" % (order.id))
self.close_position(order)
else:
log.critical("closed position failed order: %s" % (order))
self.close_position_failed(order.id)
def get_traded_sell_order(self, order_id):
return self.traded_sell_orders_db.get(order_id)
def add_order_list(self, bids, asks):
if (asks):
self.add_asks(asks)
if (bids):
self.add_bids(bids)
def clear_order_book(self):
log.info("clearing older states")
self.orderDb.clear_order_db()
self.positionsDb.clear_position_db()
def restore_order_book(self):
# TODO: FIXME: Not considering pending state orders
if (sims.simulator_on):
#don't do order db init for sim
return None
log.info("Restoring positions and orders")
#1. Retrieve states back from Db
order_list = self.orderDb.get_all_orders()
if not order_list:
log.info("no orders to restore")
else:
# restore orders
log.info("Restoring %d orders" % (len(order_list)))
for order in order_list:
# order_status = order.status_type
order_side = order.side
log.info("restoring order: %s side: %s" %
(order.id, order_side))
self.total_order_count += 1
if order_side == 'buy':
self.traded_buy_orders_db[order.id] = order
else:
self.traded_sell_orders_db[order.id] = order
# restore positions
pos_list = self.positionsDb.db_get_all_positions(self.orderDb)
# log.critical("mapping: %s"%(str(self.positionsDb.mapping)))
if not pos_list:
log.info("no positions to restore")
return
log.info("Restoring %d positions" % (len(pos_list)))
for pos in pos_list:
log.debug("restoring position(%s)" % (pos.id))
self.all_positions.append(pos)
if pos.status == "open":
self.open_positions.append(pos)
if self.market.tradeConfig["stop_loss_enabled"]:
self.add_stop_loss_position(pos, pos.buy.get_price(),
self.market,
pos.get_stop_loss())
if self.market.tradeConfig["take_profit_enabled"]:
self.add_take_profit_position(pos, pos.buy.get_price(),
self.market,
pos.get_take_profit())
else:
self.closed_positions.append(pos)
log.info("all positions and orders are restored")
# sys.exit()
def get_positions(self, from_time=0, to_time=0):
log.info("get positions ", from_time, to_time)
return self.all_positions[:]
def dump_traded_orders(self, fd=sys.stdout):
traded = str(
list(self.traded_buy_orders_db.values()) +
list(self.traded_sell_orders_db.values()))
fd.write(traded)
def dump_positions(self, fd=sys.stdout):
fd.write(str(self.all_positions))
#
# def on_sequence_gap(self, gap_start, gap_end):
# self.reset_book()
# print('Error: messages missing({} - {}). Re-initializing book at sequence.'.format(
# gap_start, gap_end, self._sequence))
####### Public API #######
def add_or_update_my_order(self, order):
# simplified order state machine :[open, filled, canceled]
# this rework is assumed an abstraction and handles only simplified order status
# if there are more order states, it should be handled/translated in the exch impl.
# linked to market.order_status_update()
# '''
# Handle a new order update msg
# return : order
# '''
if (not order):
return None
order_id = order.id
order_status = order.status
order_side = order.side
if (not order_id):
log.critical("Invalid order_id: status:%s side: %s" %
(order_status, order_side))
raise Exception("Invalid order_id: status:%s side: %s" %
(order_status, order_side))
if (order_side == 'buy'):
# see if this is a late/mixed up state msg for an already done order. What we do here, may not be correct
if self.get_traded_buy_order(order_id):
log.critical(
"********(%s) order done already, but(%s) state msg recvd, ignore for now, FIXME: FIXME:"
% (order_side, order_status))
return None
# insert/replace the order
if (order_status == 'filled' or order_status == 'canceled'):
# a previously placed order is completed, remove from open order, add to completed orderlist
self.add_traded_buy_order(order)
log.debug("Buy order Done: total_order_count: %d "
"total_open_order_count: %d "
"traded_buy_orders_count: %d" %
(self.total_order_count, self.total_open_order_count,
len(self.traded_buy_orders_db)))
elif (order_status == 'open'):
# Nothing much to do for us here
log.info("Buy order_id(%s) Status: %s" %
(str(order_id), order_status))
self.add_or_update_pending_buy_order(order)
else:
log.critical("UNKNOWN buy order status: %s" % (order_status))
raise Exception("UNKNOWN buy order status: %s" %
(order_status))
elif (order_side == 'sell'):
# see if this is a late/mixed up state msg for an already done order. What we do here, may not be correct
if self.get_traded_sell_order(order_id):
log.critical(
"********(%s) order done already, but(%s) state msg recvd, ignore for now, FIXME: FIXME:"
% (order_side, order_status))
return None
# insert/replace the order
if (order_status == 'filled' or order_status == 'canceled'):
# a previously placed order is completed, remove from open order, add to completed orderlist
self.add_traded_sell_order(order)
log.debug("Sell order Done: total_order_count: %d "
"total_open_order_count: %d "
"traded_sell_orders_count: %d" %
(self.total_order_count, self.total_open_order_count,
len(self.traded_sell_orders_db)))
elif (order_status == 'open'):
# Nothing much to do for us here
log.info("Sell order_id(%s) Status: %s" %
(str(order_id), order_status))
self.add_or_update_pending_sell_order(order)
self.close_position_pending(order)
else:
log.critical("UNKNOWN sell order status: %s" % (order_status))
raise Exception("UNKNOWN buy order status: %s" %
(order_status))
else:
log.critical("Invalid order :%s" % (order))
raise Exception("Invalid order :%s" % (order))
# log.debug("Order: %s\n"%(str(order)))
#Add the successful order to the db
self.orderDb.db_add_or_update_order(order)
stats.stats_update_order(self.market, order)
return order
def new_book(self, bids, asks):
log.info("Building new order book")
if (bids and len(bids)) or (asks and len(asks)):
self.add_order_list(bids, asks)
self.book_valid = True
else:
self.book_valid = False
def reset_book(self):
self._asks = RBTree()
self._bids = RBTree()
res = self.market.exchange.get_product_order_book(
self.market.product_id, level=3)
# log.debug("%s"%(str(res)))
if res == None:
log.error("Unable to get orderbook for exchange(%s) product: %s" %
(self.market.exchange.name, self.market.product_id))
return
for bid in res['bids']:
new_size = float(bid[1])
price = float(bid[0])
new_size += float((self.get_bids(price) or 0))
self.set_bids(price, new_size)
for ask in res['asks']:
new_size = float(ask[1])
price = float(ask[0])
new_size += float((self.get_asks(price) or 0))
self.set_asks(price, new_size)
self._sequence = float(res['sequence'])
self.book_valid = True
# print("asks: %s"%(str(self._asks)))
# print("bids: %s"%(str(self._bids)))
def add_asks(self, asks):
'''
asks =[[price, size]]
'''
for ask in asks:
price = float(ask[0])
size = float(ask[1])
if size > 0: # size > 0 add, size = 0 remove
self.set_asks(price, size)
else:
if (self.get_asks(price)):
self.remove_asks(price)
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
price = round(price, 8)
asks = round(asks, 8)
log.debug("set_asks: price: %g size: %g" % (price, asks))
self._asks.insert(price, asks)
def add_bids(self, bids):
'''
bids =[[price, size]]
'''
for bid in bids:
price = float(bid[0])
size = float(bid[1])
if size > 0: # size > 0 add, size = 0 remove
self.set_bids(price, size)
else:
if (self.get_bids(price)):
self.remove_bids(price)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
price = round(price, 8)
bids = round(bids, 8)
log.debug("set_bid: price: %g size: %g" % (price, bids))
self._bids.insert(
price, bids) # insert on RBtree is a replace for existing keys
class OrderBook:
def __init__(self,
API_KEY,
API_SECRET,
API_PASS,
API_URL,
WSS_URL,
products,
M_bidask,
auth_user_msg_q,
M_get_depth,
authclient,
message_type="subscribe",
auth=True,
level=True):
self.authclient = authclient
self.WSS_URL = WSS_URL
self.API_URL = API_URL
self.products = products
self.type = message_type
self.stop = False
self.ws = None
self.thread = None
self.lock = Lock()
self.auth = auth
self.API_KEY = API_KEY
self.API_SECRET = API_SECRET
self.API_PASS = API_PASS
self._sequence = -1
self._bid = None
self._ask = None
self._bid_depth = None
self._ask_depth = None
self.price = None
self.level = level
self.pd_data_set = None
self.buyop = 0
self.sellop = 0
self._bid = None
self._ask = None
self.M_bidask = M_bidask
self.auth_user_msg_q = auth_user_msg_q
self.M_get_depth = M_get_depth
self.count01 = 0
self.manager = Manager()
def start(self):
logging.info('start called')
self.on_open()
self.process = Process(target=self._go,
args=(self.M_bidask, self.M_get_depth))
self.process.start()
def _go(self, M_bidask, M_get_depth):
self.stop = False
self._sequence = -1
self.M_get_depth = M_get_depth
self._connect()
self._listen(M_bidask)
def _connect(self):
logging.info('_connect called')
if self.WSS_URL[-1] == "/":
self.WSS_URL = self.WSS_URL[:-1] # If url ends in a "/" strip it
sub_params = {'type': 'subscribe', 'product_ids': [self.products]}
if self.auth:
timestamp = str(time.time())
message = timestamp + 'GET' + '/users/self'
message = message.encode('ascii')
hmac_key = base64.b64decode(self.API_SECRET)
signature = hmac.new(hmac_key, message, hashlib.sha256)
digest = signature.digest()
signature_b64 = base64.b64encode(digest).decode('utf-8').rstrip(
'\n')
sub_params['signature'] = signature_b64
sub_params['key'] = self.API_KEY
sub_params['passphrase'] = self.API_PASS
sub_params['timestamp'] = timestamp
self.ws = create_connection(self.WSS_URL)
self.ws.send(json.dumps(sub_params))
if self.level:
sub_params = {
"type": "subscribe",
"product_ids": [self.products],
"channels": ["matches"]
}
self.ws.send(json.dumps(sub_params))
def _listen(self, M_bidask):
self.M_bidask2 = M_bidask # setting self.p1_w2 within this process so as to not confuse with self.p1_w. they are the came Pipe object
logging.info('_listen called')
while not self.stop:
try:
msg = self.ws.recv()
except Exception as e:
print('exception 1')
gv.logger.exception(e)
gv.logger.info('sleeping for 5 seconds and running _connect()')
self.close()
time.sleep(5)
self._go(self.M_bidask, self.M_get_depth)
else:
if msg:
msg = json.loads(msg)
self.on_message(msg)
def on_message(self, message):
try:
if 'user_id' in message:
self.auth_user_msg_q.put(message)
if 'sequence' not in message:
return
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self.get_product_order_book(level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
print(
'Error: messages missing ({} - {}). Re-initializing websocket.'
.format(sequence, self._sequence))
self.close()
time.sleep(5)
self._go(self.M_bidask, self.M_get_depth)
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
bid = self.get_bid(
) # Update Multiprocessing.Value variables with the new bid and ask
ask = self.get_ask()
if self.M_get_depth.value == 'bid':
bids = self.get_bids(bid)
bid_depth = sum([b['size'] for b in bids])
self.M_get_depth.value = bid_depth
if self.M_get_depth.value == 'ask':
asks = self.get_asks(ask)
ask_depth = sum([a['size'] for a in asks])
self.M_get_depth.value = ask_depth
if not self._bid == bid or not self._ask == ask:
# lock these vars
self.M_bidask2['bid'] = float(bid)
self.M_bidask2['ask'] = float(ask)
self._bid = bid
self._ask = ask
except Exception as e:
print(e)
print('exception 2')
gv.logger.exception(e)
self.close()
time.sleep(5)
self._go(self.M_bidask, self.M_get_depth)
# bids = self.get_bids(bid)
# bid_depth = sum([b['size'] for b in bids])
# asks = self.get_asks(ask)
# ask_depth = sum([a['size'] for a in asks])
# print('bid: %f @ %f - ask: %f @ %f' % (bid_depth, bid, ask_depth, ask))
def on_error(self, e):
self._sequence.value = -1
print('on_error called')
self.close()
self._go(self.M_bidask, self.M_get_depth)
def close(self):
logging.info('close called')
if not self.stop:
print('in not self.stop')
self.on_close()
self.stop = True
#self.process.join()
try:
if self.ws:
print('if self.ws')
self.ws.close()
except WebSocketConnectionClosedException as e:
gv.logger.debug(e)
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(
self, order
): # order is the instance of one message receded from the websockit
price = Decimal(
order['price']
) # get the price from the order and saves it as a decimal in variable price
if order['side'] == 'buy': # if its a buy order continue
bids = self.get_bids(
price
) # returns all the bids at specified (price) from orderbook, geting ready to compare with current order.
if bids is not None: # bids being None does not happen often
bidslist = [
o for o in bids if o['id'] != order['order_id']
] # creates a list minus the current order to be removed
if len(
bidslist
) > 0: # when bidslist still contains items in its list send list and price to set_bids
self.set_bids(price, bidslist)
else:
self.remove_bids(
price
) # when bidslist returns nothing in its list send price to remove_bids(price) to be removed from orderbook.
else:
asks = self.get_asks(
price
) # same as the first part of this function except on the ask side
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
# An order was matched so it needs to be removed from our orderbook \
# and keep the remaining
def match(
self, order
): # order is the instance of one message receded from the websockit
size = Decimal(order['size']) # order size how many ETH or BTC etc..
price = Decimal(order['price'])
if order['side'] == 'buy': # if the order is a buy order \
bids = self.get_bids(
price
) # returns all the bids at specified (price) from orderbook \
if not bids: # if there are no orders at this price on the bid/buy side than just return
return
assert bids[0]['id'] == order[
'maker_order_id'] # if condition returns False trigger an error
# this is taking the trade id of the order on our orderbook and comparing \
# it to the market makers order id to varify the data.
# this should always return True if it returns false are market data \
# we have saved is incorrect
if bids[0][
'size'] == size: # The first order on our books at this pirce is bids[0] and if the size \
# of that order matches the incoming match order then we need to remove \
# the order from our books at this price
# we use bids[0] because the market works on a first come first serve \
# and bids[0] has been here the longest
self.set_bids(
price, bids[1:]
) # excludes the first order in the list and sets the remaining \
# back in the orderbook list
else:
bids[0][
'size'] -= size # subtract size of current trade from the position held on the orderbook "bids[0]"
self.set_bids(price, bids)
else: # same as above for the match function but on the ask/sell side
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError as e:
gv.logger.debug(e)
return
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
# _bids is the is the binary data structure of the entire buy side of the overbook
# this function is returning every order from the buy side with a specific price
# get is a funcntion of an RBTree data structure
def get_product_order_book(self, level):
params = {'level': level}
r = requests.get(self.API_URL +
'/products/{}/book'.format(self.products),
params=params,
timeout=30)
return r.json()
def on_open(self):
print("-- Subscribed! --\n")
def on_close(self):
print("\n-- Socket Closed --")
def bid_ask_spread(self):
lock.acquire()
try:
bid = self.get_bid()
ask = self.get_ask()
print((bid - ask) * -1)
except Exception as e:
gv.logger.debug(e)
finally:
lock.release()
#def bid_ask(self):
# try:
# bid = self.get_bid()
# bids = self.get_bids(bid)
# ask = self.get_ask()
# asks = self.get_asks(ask)
#
# bid_depth = sum([b['size'] for b in bids]) # If there are no changes to the bid-ask spread \
# ask_depth = sum([a['size'] for a in asks]) # since the last update, no need to print
#
# # if we end up logging a type error we may need to \
# # lock the rbtree(_bid and _ask)
# if gv._bid == bid and gv._ask == ask and self._bid_depth == bid_depth and self._ask_depth == ask_depth:
# pass
# else: # If there are differences, update the cache
# gv._bid = bid
# gv._ask = ask
# self._bid_depth = bid_depth
# self._ask_depth = ask_depth
# print('{}\tbid: {:.3f} @ {:.2f}\task: {:.3f} @ {:.2f}'.format(dt.datetime.now(), bid_depth, bid, ask_depth, ask))
# except Exception as e:
# gv.logger.debug(e)
def user_orders_on_book(self):
self.lock.acquire()
try:
print('user orders on book {}'.format(gv.transactions_onbook))
finally:
self.lock.release()
def test2(self):
return 'test'
def market_price(self):
self.lock.acquire()
try:
if gv.market_price is not None:
print('match {}'.format(gv.market_price))
pass
finally:
self.lock.release()
class OrderBook(object):
def __init__(self, product_id='BTC-USD', feed=None, log_to=None):
self._product_id = product_id
self._asks = RBTree()
self._bids = RBTree()
self._sequence = -1
self._current_ticker = None
self._feed = feed
@property
def product_id(self):
""" Currently OrderBook only supports a single product even though it is stored as a list of products. """
return self.product_id
def reset_book(self, snapshot):
self._asks = RBTree()
self._bids = RBTree()
for bid in snapshot['bids']:
self._add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in snapshot['asks']:
self._add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = snapshot['sequence']
def on_message(self, message):
sequence = message['sequence']
if self._sequence == -1:
logger.error("Expected snapshot before any message")
sys.exit()
# self.reset_book()
return
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
self.on_sequence_gap(self._sequence, sequence)
# return
msg_type = message['type']
if msg_type == 'open':
self._add(message)
elif msg_type == 'done' and 'price' in message:
self._remove(message)
elif msg_type == 'match':
self._match(message)
self._current_ticker = message
elif msg_type == 'change':
self._change(message)
self._sequence = sequence
def on_sequence_gap(self, gap_start, gap_end):
# self.reset_book()
logger.error(
'Error: messages missing ({} - {}). ignoring the gap.'.format(
gap_start, gap_end, self._sequence))
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
@staticmethod
def meet_min_diff_price(low_price, high_price, min_diff_price):
if low_price is None or high_price is None or min_diff_price is None:
return False
return high_price - low_price > min_diff_price
@staticmethod
def meet_max_size(total_size, max_size):
if total_size is None or max_size is None:
return False
return total_size > max_size
def get_aggr_bids(self, min_diff_price=None, max_size=None):
max_bid = self.get_bid()
total_size = 0
aggr_bids = list()
for price, bid_orders in reversed(list(self._bids.items())):
if (OrderBook.meet_min_diff_price(price, max_bid, min_diff_price)
and OrderBook.meet_max_size(total_size, max_size)):
break
size = sum(bid_order['size'] for bid_order in bid_orders)
num_orders = len(bid_orders)
aggr_bids.append({
'price': price,
'size': size,
'num_orders': num_orders
})
total_size += size
return aggr_bids
def get_aggr_asks(self, min_diff_price=None, max_size=None):
min_ask = self.get_ask()
total_size = 0
aggr_asks = list()
for price, ask_orders in self._asks.items():
if (OrderBook.meet_min_diff_price(min_ask, price, min_diff_price)
and OrderBook.meet_max_size(total_size, max_size)):
break
size = sum(ask_order['size'] for ask_order in ask_orders)
num_orders = len(ask_orders)
aggr_asks.append({
'price': price,
'size': size,
'num_orders': num_orders
})
total_size += size
return aggr_asks
# Internal operations
def _add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def _remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def _match(self, order):
if self._feed is not None:
self._feed._match(order)
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def _change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
try:
price = Decimal(order['price'])
except KeyError:
return
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
class MarinerOrderBook(gdax.OrderBook):
current_milli_time = lambda self: int(time.time() * 1000)
def __init__(self, ticker, threshold):
gdax.OrderBook.__init__(self, product_id=ticker)
self._threshold = threshold
self.bookChanged = None
self.whaleEnteredMarket = None
self.whaleExitedMarket = None
def registerHandlers(self, bookChangedHandler, whaleEnteredMarketHandler,
whaleExitedMarketHandler, whaleChangedHandler):
Logging.logger.info("registering callbacks...")
self.bookChanged = bookChangedHandler
self.whaleEnteredMarket = whaleEnteredMarketHandler
self.whaleExitedMarket = whaleExitedMarketHandler
self.whaleChanged = whaleChangedHandler
def onMessage(self, message):
#Logging.logger.info(message)
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.getProductOrderBook(level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
Logging.logger.error(
'Error: messages missing ({} - {}). Re-initializing websocket.'
.format(sequence, self._sequence))
self.close()
self.start()
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
if not self.bookChanged is None:
self.bookChanged()
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
if not self.whaleEnteredMarket == None and self.isWhale(order['size']):
self.whaleEnteredMarket(order)
def remove(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
if not self.whaleExitedMarket == None and self.isWhale(order['size']):
self.whaleExitedMarket(order)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
price = Decimal(order['price'])
new_volume = Decimal(order['new_size'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = map(itemgetter('id'), bids).index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = map(itemgetter('id'), asks).index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
if not self.whaleChanged == None and self.isWhale(order['size']):
self.whaleChanged(order)
def get_current_book(self,
num_levels=10
): #fetch only 100 levels off book either way
book = []
for index, bid_entry in enumerate(self._bids.items([True])):
if index == num_levels:
break
else:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
thisBid = bid_entry[1]
except KeyError:
continue
for order in thisBid:
order['price'] = Decimal128(order['price'])
order['size'] = Decimal128(order['size'])
book.append(order)
for index, ask_entry in enumerate(self._asks.items()):
if index == num_levels:
break
else:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
thisAsk = ask_entry[1]
except KeyError:
continue
for order in thisAsk:
order['price'] = Decimal128(order['price'])
order['size'] = Decimal128(order['size'])
book.append(order)
book_frame = pd.DataFrame(book)
return book_frame
def isWhale(self, aVolume):
return aVolume >= self._threshold
#For general book management purposes
def get_bid(self):
if not self._bids.is_empty():
return self._bids.max_key()
else:
return None
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
def get_ask(self):
if not self._asks.is_empty():
return self._asks.min_key()
else:
return None
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
class OrderBook(WebsocketClient):
def __init__(self, product_id='BTC-USD', log_to=None):
super(OrderBook, self).__init__(products=product_id)
self._asks = RBTree()
self._bids = RBTree()
self._client = PublicClient()
self._sequence = -1
self._log_to = log_to
if self._log_to:
assert hasattr(self._log_to, 'write')
self._current_ticker = None
@property
def product_id(self):
''' Currently OrderBook only supports a single product even though it is stored as a list of products. '''
return self.products[0]
def on_open(self):
self._sequence = -1
print("-- Subscribed to OrderBook! --\n")
def on_close(self):
print("\n-- OrderBook Socket Closed! --")
def reset_book(self):
self._asks = RBTree()
self._bids = RBTree()
res = self._client.get_product_order_book(
product_id=self.product_id, level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
def on_message(self, message):
if self._log_to:
pickle.dump(message, self._log_to)
sequence = message['sequence']
if self._sequence == -1:
self.reset_book()
return
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
self.on_sequence_gap(self._sequence, sequence)
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
self._current_ticker = message
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
def on_sequence_gap(self, gap_start, gap_end):
self.reset_book()
print('Error: messages missing ({} - {}). Re-initializing book at sequence.'.format(
gap_start, gap_end, self._sequence))
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = collections.OrderedDict()
bids[order['id']] = order
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = collections.OrderedDict()
asks[order['id']] = order
self.set_asks(order['price'], asks)
def remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
if order['order_id'] in bids:
del bids[order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
if order['order_id'] in asks:
del asks[order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
top_order = bids.items()[0]
assert top_order[1]['id'] == order['maker_order_id']
if top_order[1]['size'] == size:
del bids[order['maker_order_id']]
self.set_bids(price, bids)
else:
top_order[1]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
top_order = asks.items()[0]
assert top_order[1]['id'] == order['maker_order_id']
if top_order[1]['size'] == size:
del asks[order['maker_order_id']]
self.set_asks(price, asks)
else:
top_order[1]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
try:
price = Decimal(order['price'])
except KeyError:
return
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or order['order_id'] not in bids:
return
bids[order['order_id']]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or order['order_id'] not in asks:
return
asks[order['order_id']]['size'] = new_size
self.set_asks(price, asks)
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class WhaleTracker():
current_milli_time = lambda self: int(time.time() * 1000)
def __init__(self, ticker):
self._ticker = ticker
self._bid_whales = RBTree()
self._ask_whales = RBTree()
def whaleEnteredMarketHandler(self, order):
if order['side'] == 'buy':
self.addBidWhale(order)
else:
self.addAskWhale(order)
def whaleExitedMarketHandler(self, order):
if order['side'] == 'buy':
self.removeBidWhale(order)
else:
self.removeAskWhale(order)
def whaleChangedHandler(self, order):
if order['side'] == 'buy':
self.changeBidWhale(order)
else:
self.changeAskWhale(order)
def addBidWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
bid_whale_order = self.get_bid_whale(price)
if bid_whale_order is None or volume > bid_whale_order.get_volume():
#Logging.logger.info("NEW WHALE ENTERED (BID): price=" + str(price) + " volume=" + str(volume))
bid_whale_order = WhaleOrder(ID, price, volume)
self.set_bid_whale(price, bid_whale_order)
def addAskWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
ask_whale_order = self.get_ask_whale(price)
if ask_whale_order is None or volume > ask_whale_order.get_volume():
#Logging.logger.info("NEW WHALE ENTERED (ASK): price=" + str(price) + " volume=" + str(volume))
ask_whale_order = WhaleOrder(ID, price, volume)
self.set_ask_whale(price, ask_whale_order)
def removeBidWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
bid_whale_order = self.get_bid_whale(price)
if bid_whale_order is not None and bid_whale_order.get_id() == ID:
#Logging.logger.info("WHALE LEFT (BID): price=" + str(price) + " volume=" + str(volume))
self.remove_bid_whale(price)
def removeAskWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
volume = Decimal(order['size'])
ask_whale_order = self.get_ask_whale(price)
if ask_whale_order is not None and ask_whale_order.get_id() == ID:
#Logging.logger.info("WHALE LEFT (ASK): price=" + str(price) + " volume=" + str(volume))
self.remove_ask_whale(price)
def changeBidWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
new_volume = order['new_size']
bid_whale_order = self.get_bid_whale(price)
if not bid_whale_order == None:
#Logger.logging.info("WHALE CHANGED (BID): price=" + price + " new_volume=" + new_volume)
if self.isWhale(new_volume):
bid_whale_order.setVolume(new_volume)
self.set_bid_whale(price, bid_whale_order)
else:
self.removeBidWhale(order)
def changeAskWhale(self, order):
ID = order['id']
price = Decimal(order['price'])
new_volume = order['new_size']
ask_whale_order = self.get_ask_whale(price)
if not ask_whale_order == None:
#Logger.logging.info("WHALE CHANGED (ASK): price=" + price + " new_volume=" + new_volume)
if self.isWhale(new_volume):
ask_whale_order.setVolume(new_volume)
self.set_ask_whale(price, ask_whale_order)
else:
self.removeAskWhale(order)
def get_current_whales(
self,
num_whales=30
): #by default, fetch only 30 whales off the book either way
whales = []
for index, bid_whale in self._bid_whales.items(True):
if index == num_whales:
break
else:
whales.append({
'price': Decimal128(bid_whale.get_price()),
'volume': Decimal128(bid_whale.get_volume()),
'id': bid_whale.get_id(),
})
for index, ask_whale in self._ask_whales.items():
if index == num_whales:
break
else:
whales.append({
'price': Decimal128(ask_whale.get_price()),
'volume': Decimal128(ask_whale.get_volume()),
'id': ask_whale.get_id(),
})
whales_frame = pd.DataFrame(whales)
return whales_frame
def get_top_bid_whale(self):
if not self._bid_whales.is_empty():
top_bid = self._bid_whales.max_key()
return self.get_bid_whale(top_bid)
else:
return None
def get_bid_whale(self, price):
return self._bid_whales.get(price)
def remove_bid_whale(self, price):
self._bid_whales.remove(price)
def set_bid_whale(self, price, whale):
self._bid_whales.insert(price, whale)
def get_top_ask_whale(self):
if not self._ask_whales.is_empty():
top_ask = self._ask_whales.min_key()
return self.get_ask_whale(top_ask)
else:
return None
def get_ask_whale(self, price):
return self._ask_whales.get(price)
def remove_ask_whale(self, price):
self._ask_whales.remove(price)
def set_ask_whale(self, price, whale):
self._ask_whales.insert(price, whale)
class OrderBook(object):
def __init__(self, product_id, log_to=None):
self._asks = RBTree()
self._bids = RBTree()
self._client = PublicClient()
self._sequence = -1
self.sync = -1
self.product_id = product_id
self._log_to = log_to
if self._log_to:
assert hasattr(self._log_to, 'write')
self._current_ticker = None
def on_open(self):
self._sequence = -1
#print("-- Subscribed to " + self.product_id + " OrderBook! --\n")
def on_close(self):
print("\n-- OrderBook " + self.product_id + " Socket Closed! --")
def reset_book(self):
self._asks = RBTree()
self._bids = RBTree()
res = self._client.get_product_order_book(product_id=self.product_id,
level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
def on_message(self, message):
if self._log_to:
pickle.dump(message, self._log_to)
sequence = message['sequence']
if self._sequence == -1:
self.sync = -1
self.reset_book()
return
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
self.on_sequence_gap(self._sequence, sequence)
self.sync = -1
return
self.sync = 1
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
self._current_ticker = message
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
def on_sequence_gap(self, gap_start, gap_end):
self.reset_book()
print(
'Error: messages missing ({} - {}). Re-initializing book at sequence.'
.format(gap_start, gap_end, self._sequence))
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
try:
price = Decimal(order['price'])
except KeyError:
return
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_ask_size(self):
return sum(order['size'] for order in self._asks.min_item()[1])
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bid_size(self):
return sum(order['size'] for order in self._bids.min_item()[1])
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class OrderBook(WebsocketClient):
def __init__(self, product_id='BTC-USD'):
super(OrderBook, self).__init__(products=product_id)
self._asks = RBTree()
self._bids = RBTree()
self._client = PublicClient(product_id=product_id)
self._sequence = -1
def on_message(self, message):
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.get_product_order_book(level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
print('Error: messages missing ({} - {}). Re-initializing websocket.'.format(sequence, self._sequence))
self.close()
self.start()
return
# print(message)
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
# bid = self.get_bid()
# bids = self.get_bids(bid)
# bid_depth = sum([b['size'] for b in bids])
# ask = self.get_ask()
# asks = self.get_asks(ask)
# ask_depth = sum([a['size'] for a in asks])
# print('bid: %f @ %f - ask: %f @ %f' % (bid_depth, bid, ask_depth, ask))
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
new_size = Decimal(order['new_size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id'] for o in bids):
return
index = map(itemgetter('id'), bids).index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id'] for o in asks):
return
index = map(itemgetter('id'), asks).index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append([order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append([order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class OrderBook(WebsocketClient):
def __init__(self, product_id='BTC-USD', log_to=None):
super(OrderBook, self).__init__(products=product_id)
self._asks = RBTree()
self._bids = RBTree()
self._client = PublicClient()
self._sequence = -1
self._log_to = log_to
if self._log_to:
assert hasattr(self._log_to, 'write')
self._current_ticker = None
@property
def product_id(self):
''' Currently OrderBook only supports a single product even though it is stored as a list of products. '''
return self.products[0]
def on_message(self, message):
if self._log_to:
pickle.dump(message, self._log_to)
sequence = message['sequence']
if self._sequence == -1:
self._asks = RBTree()
self._bids = RBTree()
res = self._client.get_product_order_book(product_id=self.product_id, level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
print('Error: messages missing ({} - {}). Re-initializing websocket.'.format(sequence, self._sequence))
self.close()
self.start()
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
self._current_ticker = message
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
# bid = self.get_bid()
# bids = self.get_bids(bid)
# bid_depth = sum([b['size'] for b in bids])
# ask = self.get_ask()
# asks = self.get_asks(ask)
# ask_depth = sum([a['size'] for a in asks])
# print('bid: %f @ %f - ask: %f @ %f' % (bid_depth, bid, ask_depth, ask))
def on_error(self, e):
self._sequence = -1
self.close()
self.start()
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id'] for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id'] for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append([order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append([order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.min_key()
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
self._asks.remove(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
def get_bid(self):
return self._bids.max_key()
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
self._bids.remove(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
class obook(object):
def __init__(self, saveMessage=0, product_id='BTC-USD'):
self.url = "wss://ws-feed.gdax.com/"
self.ws = create_connection(self.url)
self.product_id = product_id
self.sequence = -1
self.stop = False
self._sequence = -1
self._client = PublicClient()
self._asks = RBTree()
self._bids = RBTree()
self._current_ticker = None
self.saveMessage = saveMessage
self.t1 = None
self.t2 = None
self.t3 = None
def saveMessage(self, msg):
### To be done - save live message in file
path = 'xxx'
with open(pathOUT, 'a') as fileIO:
pathOUT.write(json.dumps(msg))
a = 4
def demo(self):
self.connect()
self.listen(demo=1)
self.disconnect()
def connect(self):
print('connessione')
sub_params = {'type': 'subscribe', 'product_ids': [product_id]}
self.ws.send(json.dumps(sub_params))
sub_params = {"type": "heartbeat", "on": False}
self.ws.send(json.dumps(sub_params))
c = 0
def listen(self, demo=0):
print('in ascolto')
count = 0
while not self.stop:
data = self.ws.recv()
msg = json.loads(data)
self.onmessage(msg)
print msg['sequence']
count = count + 1
if count > 10 and demo:
self.stop = True
def disconnect(self):
print('Fine Connessione')
#self.reset_book()
def get_snapshot(self):
self.reset_book()
def reset_book(self):
self._asks = RBTree()
self._bids = RBTree()
self.tref = datetime.datetime.now()
# res = self._client.get_product_order_book(product_id=self.product_id, level=3)
##################
params = {'level': 3}
r = requests.get('https://api.gdax.com/products/{}/book'.format(
self.product_id),
params=params,
timeout=30)
try:
res = r.json()
except:
res['bids'] = {}
res['asks'] = {}
res['sequence'] = 0
# r.raise_for_status()
##################
for bid in res['bids']:
self.add1({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add1({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
def onmessage(self, msg):
if self.saveMessage:
saveMessage(msg)
message = msg
sequence = message['sequence']
if self._sequence == -1:
self.reset_book()
return
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
self.on_sequence_gap(self._sequence, sequence)
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
self._current_ticker = message
elif msg_type == 'change':
self.change(message)
def get_current_book_serializable(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[float(order['price']),
float(order['size']), order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[float(order['price']),
float(order['size']), order['id']])
return result
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append(
[order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append(
[order['price'], order['size'], order['id']])
return result
def on_sequence_gap(self, gap_start, gap_end):
self.reset_book()
print(
'Error: messages missing ({} - {}). Re-initializing book at sequence.'
.format(gap_start, gap_end, self._sequence))
#### UTILS
def remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
try:
price = Decimal(order['price'])
except KeyError:
return
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id']
for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id']
for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_current_ticker(self):
return self._current_ticker
########################################
def add1(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': float(order['price']),
'size': float(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
########################################
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def get_bids(self, price):
return self._bids.get(price)
def set_bids(self, price, bids):
self._bids.insert(price, bids)
def get_asks(self, price):
return self._asks.get(price)
def set_asks(self, price, asks):
self._asks.insert(price, asks)
