def insert_order_book(self, instmt):
"""
Insert order book row into the database client
:param instmt: Instrument
"""
# If local timestamp indicator is on, assign the local timestamp again
if self.is_local_timestamp:
instmt.get_l2_depth().date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f")
# Update the snapshot
if self.data_mode & ExchangeGateway.DataMode.SNAPSHOT_ONLY and \
instmt.get_l2_depth() is not None:
self.db_client.insert(table=self.get_snapshot_table_name(),
columns=Snapshot.columns(),
types=Snapshot.types(),
values=Snapshot.values(instmt.get_exchange_name(),
instmt.get_instmt_name(),
instmt.get_l2_depth(),
Trade() if instmt.get_last_trade() is None else instmt.get_last_trade(),
Snapshot.UpdateType.ORDER_BOOK),
primary_key_index=[0,1],
is_orreplace=True,
is_commit=not(self.data_mode & ExchangeGateway.DataMode.ORDER_BOOK_ONLY))
# Update its order book table
if self.data_mode & ExchangeGateway.DataMode.ORDER_BOOK_ONLY:
self.db_client.insert(table=instmt.get_order_book_table_name(),
columns=['id'] + L2Depth.columns(),
types=['int'] + L2Depth.types(),
values=[instmt.get_order_book_id()] + instmt.get_l2_depth().values())
def start(self, instmt):
"""
Start the exchange gateway
:param instmt: Instrument
:return List of threads
"""
instmt.set_prev_l2_depth(L2Depth(20))
instmt.set_l2_depth(L2Depth(20))
instmt.set_order_book_table_name(
self.get_order_book_table_name(instmt.get_exchange_name(),
instmt.get_instmt_name()))
instmt.set_trades_table_name(
self.get_trades_table_name(instmt.get_exchange_name(),
instmt.get_instmt_name()))
instmt.set_order_book_id(self.get_order_book_init(instmt))
trade_id, last_exch_trade_id = self.get_trades_init(instmt)
instmt.set_trade_id(trade_id)
instmt.set_exch_trade_id(last_exch_trade_id)
return [
self.api_socket.connect(
self.api_socket.get_link(),
on_message_handler=partial(self.on_message_handler, instmt),
on_open_handler=partial(self.on_open_handler, instmt),
on_close_handler=partial(self.on_close_handler, instmt))
]
def __init__(self, name, info):
self.name = name
self.info = info
self.prev_l2_depth = L2Depth()
self.l2_depth = L2Depth()
self.last_trade = Trade()
self.trade_history = []
self.trade_history_max_sec = datetime.timedelta(seconds=60)
def init_order_book_table(self, instmt):
if self.data_mode & ExchangeGateway.DataMode.ORDER_BOOK_ONLY:
table_name = self.get_order_book_table_name(instmt.get_exchange_name(),
instmt.get_instmt_name())
self.db_client.create(table_name,
['id'] + L2Depth.columns(),
['int'] + L2Depth.types(),
[0])
def start(self, instmt):
"""
Start the exchange gateway
:param instmt: Instrument
:return List of threads
"""
instmt.set_l2_depth(L2Depth(5))
instmt.set_prev_l2_depth(L2Depth(5))
instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(),
instmt.get_instmt_name()))
self.init_instmt_snapshot_table(instmt)
return [self.api_socket.connect(self.api_socket.get_link(),
on_message_handler=partial(self.on_message_handler, instmt),
on_open_handler=partial(self.on_open_handler, instmt),
on_close_handler=partial(self.on_close_handler, instmt))]
def parse_l2_depth(cls, instmt, raw):
"""
Parse raw data to L2 depth
:param instmt: Instrument
:param raw: Raw data in JSON
"""
l2_depth = L2Depth()
keys = list(raw.keys())
if (cls.get_bids_field_name() in keys and
cls.get_asks_field_name() in keys):
# Date time
date_time = float(raw[cls.get_order_book_timestamp_field_name()])
date_time = date_time / cls.get_timestamp_offset()
l2_depth.date_time = datetime.utcfromtimestamp(date_time).strftime("%Y%m%d %H:%M:%S.%f")
# Bids
bids = raw[cls.get_bids_field_name()]
bids = sorted(bids, key=lambda x: x[0], reverse=True)
for i in range(0, 5):
l2_depth.bids[i].price = float(bids[i][0]) if not isinstance(bids[i][0], float) else bids[i][0]
l2_depth.bids[i].volume = float(bids[i][1]) if not isinstance(bids[i][1], float) else bids[i][1]
# Asks
asks = raw[cls.get_asks_field_name()]
asks = sorted(asks, key=lambda x: x[0])
for i in range(0, 5):
l2_depth.asks[i].price = float(asks[i][0]) if not isinstance(asks[i][0], float) else asks[i][0]
l2_depth.asks[i].volume = float(asks[i][1]) if not isinstance(asks[i][1], float) else asks[i][1]
else:
raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \
(instmt.get_exchange_name(), instmt.get_instmt_name(), \
raw))
return l2_depth
def parse_l2_depth(cls, instmt, raw):
"""
Parse raw data to L2 depth
:param instmt: Instrument
:param raw: Raw data in JSON
"""
l2_depth = L2Depth()
keys = list(raw.keys())
if cls.get_bids_field_name() in keys and \
cls.get_asks_field_name() in keys:
# No Date time information, has update id only
l2_depth.date_time = datetime.now().strftime("%Y%m%d %H:%M:%S.%f")
# Bids
bids = raw[cls.get_bids_field_name()]
bids = sorted(bids, key=lambda x: x['price'], reverse=True)
for i in range(0, 5):
l2_depth.bids[i].price = float(bids[i]['price']) if type(bids[i]['price']) != float else bids[i]['price']
l2_depth.bids[i].volume = float(bids[i]['qty']) if type(bids[i]['qty']) != float else bids[i]['qty']
# Asks
asks = raw[cls.get_asks_field_name()]
asks = sorted(asks, key=lambda x: x['price'])
for i in range(0, 5):
l2_depth.asks[i].price = float(asks[i]['price']) if type(asks[i]['price']) != float else asks[i]['price']
l2_depth.asks[i].volume = float(asks[i]['qty']) if type(asks[i]['qty']) != float else asks[i]['qty']
else:
raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \
(instmt.get_exchange_name(), instmt.get_instmt_name(), \
raw))
return l2_depth
def parse_l2_depth(cls, instmt, raw):
"""
Parse raw data to L2 depth
:param instmt: Instrument
:param raw: Raw data in JSON
"""
l2_depth = L2Depth()
raw = raw["result"]
keys = list(raw.keys())
if cls.get_bids_field_name() in keys and \
cls.get_asks_field_name() in keys:
# Date time
l2_depth.date_time = datetime.utcnow().strftime(
"%Y%m%d %H:%M:%S.%f")
# Bids
bids = raw[cls.get_bids_field_name()]
for i in range(0, 5):
l2_depth.bids[i].price = bids[i][cls.get_price_field_name()]
l2_depth.bids[i].volume = bids[i][cls.get_volume_field_name()]
# Asks
asks = raw[cls.get_asks_field_name()]
for i in range(0, 5):
l2_depth.asks[i].price = asks[i][cls.get_price_field_name()]
l2_depth.asks[i].volume = asks[i][cls.get_volume_field_name()]
else:
raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \
(instmt.get_exchange_name(), instmt.get_instmt_name(), \
raw))
return l2_depth
def parse_l2_depth(cls, instmt, raw):
"""
Parse raw data to L2 depth
:param instmt: Instrument
:param raw: Raw data in JSON
"""
l2_depth = L2Depth()
keys = list(raw.keys())
if cls.get_bids_field_name() in keys and \
cls.get_asks_field_name() in keys:
# Bids
bids = raw[cls.get_bids_field_name()]
bids = sorted(bids, key=lambda x: x[0], reverse=True)
for i in range(0, len(bids)):
l2_depth.bids[i].price = float(
bids[i][0]) if type(bids[i][0]) != float else bids[i][0]
l2_depth.bids[i].volume = float(
bids[i][1]) if type(bids[i][1]) != float else bids[i][1]
# Asks
asks = raw[cls.get_asks_field_name()]
asks = sorted(asks, key=lambda x: x[0])
for i in range(0, len(asks)):
l2_depth.asks[i].price = float(
asks[i][0]) if type(asks[i][0]) != float else asks[i][0]
l2_depth.asks[i].volume = float(
asks[i][1]) if type(asks[i][1]) != float else asks[i][1]
return l2_depth
def start(self, instmt):
"""
Start the exchange gateway
:param instmt: Instrument
:return List of threads
"""
instmt.set_l2_depth(L2Depth(5))
instmt.set_prev_l2_depth(L2Depth(5))
instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(),
instmt.get_instmt_name()))
self.init_instmt_snapshot_table(instmt)
instmt.set_recovered(False)
t1 = threading.Thread(target=partial(self.get_order_book_worker, instmt))
t2 = threading.Thread(target=partial(self.get_trades_worker, instmt))
t1.start()
t2.start()
return [t1, t2]
def start(self, instmt):
"""
Start the exchange gateway
:param instmt: Instrument
:return List of threads
"""
instmt.set_l2_depth(L2Depth(50))
instmt.set_prev_l2_depth(L2Depth(50))
instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(),
instmt.get_instmt_name()))
self.init_instmt_snapshot_table(instmt)
t_trades = self.api_socket.connect(url=self.api_socket.get_link(),
on_message_handler=partial(self.on_message_handler, instmt),
on_open_handler=partial(self.on_open_handler, instmt),
on_close_handler=partial(self.on_close_handler, instmt))
t_order_book = threading.Thread(target=partial(self.get_order_book_worker, instmt))
t_order_book.start()
return [t_order_book, t_trades]
def parse_l2_depth(cls, instmt, raw):
"""
Parse raw data to L2 depth
:param instmt: Instrument
:param raw: Raw data in JSON
"""
keys = list(raw.keys())
if cls.get_bids_field_name() in keys and \
cls.get_asks_field_name() in keys:
l2_depth = L2Depth()
# Bids
bids = raw[cls.get_bids_field_name()]
bid_level = -1
for bid in bids:
price = bid[cls.get_order_book_price_field_name()]
volume = bid[cls.get_order_book_volume_field_name()]
if bid_level == -1 or l2_depth.bids[bid_level].price != price:
bid_level += 1
if bid_level < 5:
l2_depth.bids[bid_level].price = float(price)
else:
break
l2_depth.bids[bid_level].volume += float(volume)
# Asks
asks = raw[cls.get_asks_field_name()]
ask_level = -1
for ask in asks:
price = ask[cls.get_order_book_price_field_name()]
volume = ask[cls.get_order_book_volume_field_name()]
if ask_level == -1 or l2_depth.asks[ask_level].price != price:
ask_level += 1
if ask_level < 5:
l2_depth.asks[ask_level].price = float(price)
else:
break
l2_depth.asks[ask_level].volume += float(volume)
return l2_depth
else:
raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \
(instmt.get_exchange_name(), instmt.get_instmt_name(), \
raw))
def parse_l2_depth(cls, instmt, raw):
"""
Parse raw data to L2 depth
:param instmt: Instrument
:param raw: Raw data in JSON
"""
l2_depth = L2Depth()
raw = raw[instmt.instmt_code]
keys = list(raw.keys())
if (cls.get_bids_field_name() in keys
and cls.get_asks_field_name() in keys):
# Date time
l2_depth.date_time = datetime.utcnow().strftime(
"%Y%m%d %H:%M:%S.%f")
# Bids
bids = raw[cls.get_bids_field_name()]
for i in range(0, 5):
l2_depth.bids[i].price = float(bids[i][0]) if not isinstance(
bids[i][0], float) else bids[i][0]
l2_depth.bids[i].volume = float(bids[i][1]) if not isinstance(
bids[i][1], float) else bids[i][1]
# Asks
asks = raw[cls.get_asks_field_name()]
for i in range(0, 5):
l2_depth.asks[i].price = float(asks[i][0]) if not isinstance(
asks[i][0], float) else asks[i][0]
l2_depth.asks[i].volume = float(asks[i][1]) if not isinstance(
asks[i][1], float) else asks[i][1]
else:
raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \
(instmt.get_exchange_name(), instmt.get_instmt_name(), \
raw))
return l2_depth
instmt.set_l2_depth(L2Depth(5))
instmt.set_prev_l2_depth(L2Depth(5))
instmt.set_order_book_table_name(self.get_order_book_table_name(instmt.get_exchange_name(),
instmt.get_instmt_name()))
instmt.set_trades_table_name(self.get_trades_table_name(instmt.get_exchange_name(),
instmt.get_instmt_name()))
instmt.set_recovered(False)
t1 = threading.Thread(target=partial(self.get_order_book_worker, instmt))
t1.start()
t2 = threading.Thread(target=partial(self.get_trades_worker, instmt))
t2.start()
return [t1, t2]
if __name__ == '__main__':
Logger.init_log()
exchange_name = 'Gatecoin'
instmt_name = 'BTCHKD'
instmt_code = 'BTCHKD'
instmt = Instrument(exchange_name, instmt_name, instmt_code)
db_client = SqlClientTemplate()
exch = ExchGwGatecoin(db_client)
instmt.set_l2_depth(L2Depth(5))
instmt.set_prev_l2_depth(L2Depth(5))
instmt.set_order_book_table_name(exch.get_order_book_table_name(instmt.get_exchange_name(),
instmt.get_instmt_name()))
instmt.set_trades_table_name(exch.get_trades_table_name(instmt.get_exchange_name(),
instmt.get_instmt_name()))
instmt.set_recovered(False)
# exch.get_order_book_worker(instmt)
exch.get_trades_worker(instmt)
def parse_l2_depth(cls, instmt, raw):
"""
Parse raw data to L2 depth
:param instmt: Instrument
:param raw: Raw data in JSON
"""
def get_order_info(data):
return (
data['price'] if 'price' in data.keys() else None,
(0 if data['side'] == "Buy" else 1),
data['id'],
data['size'] if 'size' in data.keys() else None
)
if raw['action'] in ('partial', 'insert'):
# Order book initialization or insertion
for data in raw['data']:
if data['symbol'] != instmt.get_instmt_code():
continue
price, side, id, volume = get_order_info(data)
instmt.realtime_order_book_ids[side][id] = price
if price not in instmt.realtime_order_book_prices[side].keys():
instmt.realtime_order_book_prices[side][price] = { id: volume }
else:
instmt.realtime_order_book_prices[side][price][id] = volume
elif raw['action'] == 'update':
# Order book update
for data in raw['data']:
if data['symbol'] != instmt.get_instmt_code():
continue
_, side, id, volume = get_order_info(data)
price = instmt.realtime_order_book_ids[side][id]
instmt.realtime_order_book_ids[side][id] = price
instmt.realtime_order_book_prices[side][price][id] = volume
elif raw['action'] == 'delete':
# Order book delete
for data in raw['data']:
if data['symbol'] != instmt.get_instmt_code():
continue
_, side, id, _ = get_order_info(data)
price = instmt.realtime_order_book_ids[side][id]
del instmt.realtime_order_book_prices[side][price][id]
if len(instmt.realtime_order_book_prices[side][price]) == 0:
del instmt.realtime_order_book_prices[side][price]
# return l2_depth
l2_depth = L2Depth()
l2_depth.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f")
bids_px = sorted(list(instmt.realtime_order_book_prices[0].keys()), reverse=True)[:5]
asks_px = sorted(list(instmt.realtime_order_book_prices[1].keys()))[:5]
bids_qty = [sum(instmt.realtime_order_book_prices[0][px].values()) for px in bids_px]
asks_qty = [sum(instmt.realtime_order_book_prices[1][px].values()) for px in asks_px]
for i in range(0, 5):
l2_depth.bids[i].price = bids_px[i]
l2_depth.bids[i].volume = bids_qty[i]
l2_depth.asks[i].price = asks_px[i]
l2_depth.asks[i].volume = asks_qty[i]
return l2_depth
def parse_l2_depth(cls, instmt, raw):
"""
Parse raw data to L2 depth
:param instmt: Instrument
:param raw: Raw data in JSON
"""
l2_depth = instmt.get_l2_depth()
keys = list(raw.keys())
bids_field = cls.get_bids_field_name()
asks_field = cls.get_asks_field_name()
if bids_field in keys and asks_field in keys:
# Date time
l2_depth.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f")
# Bids
bids = raw[bids_field]
bids_len = min(l2_depth.depth, len(bids))
for i in range(0, bids_len):
l2_depth.bids[i].price = float(bids[i]['price']) \
if not isinstance(bids[i]['price'], float) else bids[i][0]
l2_depth.bids[i].volume = float(bids[i]['volume']) \
if not isinstance(bids[i]['volume'], float) else bids[i][1]
l2_depth.bids[i].id = bids[i]['id']
# Asks
asks = raw[asks_field]
asks_len = min(l2_depth.depth, len(asks))
for i in range(0, asks_len):
l2_depth.asks[i].price = float(asks[i]['price']) \
if not isinstance(asks[i]['price'], float) else asks[i][0]
l2_depth.asks[i].volume = float(asks[i]['volume']) \
if not isinstance(asks[i]['volume'], float) else asks[i][1]
l2_depth.asks[i].id = asks[i]['id']
elif "order_id" in keys:
if 'type' in keys:
# Insertion
order_id = raw['order_id']
price = float(raw['price'])
volume = float(raw['volume'])
update_type = raw['type']
if update_type == "BID":
l2_depth.bids.append(L2Depth.Depth(price=price,
volume=volume))
l2_depth.bids[-1].id = order_id
l2_depth.sort_bids()
if len(l2_depth.bids) > l2_depth.depth * 2:
del l2_depth.bids[l2_depth.depth:]
elif update_type == "ASK":
l2_depth.asks.append(L2Depth.Depth(price=price,
volume=volume))
l2_depth.asks[-1].id = order_id
l2_depth.sort_asks()
if len(l2_depth.asks) > l2_depth.depth * 2:
del l2_depth.asks[l2_depth.depth:]
elif 'base' in keys:
# Update
order_id = raw['order_id']
volume = float(raw['base'])
price = float(raw['counter']) / float(raw['base'])
for i in range(0, len(l2_depth.bids)):
if l2_depth.bids[i].id == order_id:
if l2_depth.bids[i].price == price:
l2_depth.bids[i].volume -= volume
break
else:
# Deletion
order_id = raw['order_id']
found = False
for i in range(0, len(l2_depth.bids)):
if l2_depth.bids[i].id == order_id:
found = True
del l2_depth.bids[i]
break
if not found:
for i in range(0, len(l2_depth.asks)):
if l2_depth.asks[i].id == order_id:
del l2_depth.asks[i]
break
else:
raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' %
(instmt.get_exchange_name(), instmt.get_instmt_name(),
raw))
return l2_depth
def parse_l2_depth(cls, instmt, raw):
"""
Parse raw data to L2 depth
:param instmt: Instrument
:param raw: Raw data in JSON
"""
# No order book mapping from config. Need to decode here.
l2_depth = instmt.get_l2_depth()
l2_depth.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f")
if isinstance(raw[0], list):
# Start subscription
for i in range(0, 25):
bid = raw[i]
ask = raw[25 + i]
l2_depth.bids[i] = L2Depth.Depth(price=bid[0],
count=bid[1],
volume=bid[2])
l2_depth.asks[i] = L2Depth.Depth(price=ask[0],
count=ask[1],
volume=-ask[2])
else:
price = raw[1]
count = raw[2]
volume = raw[3]
found = False
if count == 0:
# Deletion
if volume > 0:
for i in range(0, len(l2_depth.bids)):
if price == l2_depth.bids[i].price:
found = True
del l2_depth.bids[i]
break
else:
for i in range(0, len(l2_depth.asks)):
if price == l2_depth.asks[i].price:
found = True
del l2_depth.asks[i]
break
if not found:
depth_text = ""
for i in range(0, l2_depth.depth):
if i < len(l2_depth.bids):
depth_text += "%.4f,%d,%.4f" % \
(l2_depth.bids[i].volume, \
l2_depth.bids[i].count, \
l2_depth.bids[i].price)
else:
depth_text += " "
depth_text += "<--->"
if i < len(l2_depth.asks):
depth_text += "%.4f,%d,%.4f" % \
(l2_depth.asks[i].volume, \
l2_depth.asks[i].count, \
l2_depth.asks[i].price)
else:
depth_text += " "
depth_text += "\n"
Logger.info(cls.__name__, "Cannot find the deletion of the message: %s\nDepth:\n%s\n" % \
(raw, depth_text))
else:
# Insertion/Update
if volume > 0:
# Update
for i in range(0, len(l2_depth.bids)):
if price == l2_depth.bids[i].price:
l2_depth.bids[i].count = count
l2_depth.bids[i].volume = volume
found = True
break
if not found:
# Insertion
l2_depth.bids.append(
L2Depth.Depth(price=price,
count=count,
volume=volume))
l2_depth.sort_bids()
if len(l2_depth.bids) > l2_depth.depth * 2:
del l2_depth.bids[l2_depth.depth:]
else:
for i in range(0, len(l2_depth.asks)):
# Update
if price == l2_depth.asks[i].price:
l2_depth.asks[i].count = count
l2_depth.asks[i].volume = -volume
found = True
break
if not found:
# Insertion
l2_depth.asks.append(
L2Depth.Depth(price=price,
count=count,
volume=-volume))
l2_depth.sort_asks()
if len(l2_depth.asks) > l2_depth.depth * 2:
del l2_depth.asks[l2_depth.depth:]
return l2_depth
