def insert_order_book(self, instmt): """ Insert order book row into the database client :param instmt: Instrument """ # If local timestamp indicator is on, assign the local timestamp again if self.is_local_timestamp: instmt.get_l2_depth().date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f") # Update the snapshot if self.data_mode & ExchangeGateway.DataMode.SNAPSHOT_ONLY and \ instmt.get_l2_depth() is not None: self.db_client.insert(table=self.get_snapshot_table_name(), columns=Snapshot.columns(), types=Snapshot.types(), values=Snapshot.values(instmt.get_exchange_name(), instmt.get_instmt_name(), instmt.get_l2_depth(), Trade() if instmt.get_last_trade() is None else instmt.get_last_trade(), Snapshot.UpdateType.ORDER_BOOK), primary_key_index=[0,1], is_orreplace=True, is_commit=not(self.data_mode & ExchangeGateway.DataMode.ORDER_BOOK_ONLY)) # Update its order book table if self.data_mode & ExchangeGateway.DataMode.ORDER_BOOK_ONLY: self.db_client.insert(table=instmt.get_order_book_table_name(), columns=['id'] + L2Depth.columns(), types=['int'] + L2Depth.types(), values=[instmt.get_order_book_id()] + instmt.get_l2_depth().values())
def start(self, instmt): """ Start the exchange gateway :param instmt: Instrument :return List of threads """ instmt.set_prev_l2_depth(L2Depth(20)) instmt.set_l2_depth(L2Depth(20)) instmt.set_order_book_table_name( self.get_order_book_table_name(instmt.get_exchange_name(), instmt.get_instmt_name())) instmt.set_trades_table_name( self.get_trades_table_name(instmt.get_exchange_name(), instmt.get_instmt_name())) instmt.set_order_book_id(self.get_order_book_init(instmt)) trade_id, last_exch_trade_id = self.get_trades_init(instmt) instmt.set_trade_id(trade_id) instmt.set_exch_trade_id(last_exch_trade_id) return [ self.api_socket.connect( self.api_socket.get_link(), on_message_handler=partial(self.on_message_handler, instmt), on_open_handler=partial(self.on_open_handler, instmt), on_close_handler=partial(self.on_close_handler, instmt)) ]
def __init__(self, name, info): self.name = name self.info = info self.prev_l2_depth = L2Depth() self.l2_depth = L2Depth() self.last_trade = Trade() self.trade_history = [] self.trade_history_max_sec = datetime.timedelta(seconds=60)
def init_order_book_table(self, instmt): if self.data_mode & ExchangeGateway.DataMode.ORDER_BOOK_ONLY: table_name = self.get_order_book_table_name(instmt.get_exchange_name(), instmt.get_instmt_name()) self.db_client.create(table_name, ['id'] + L2Depth.columns(), ['int'] + L2Depth.types(), [0])
def start(self, instmt): """ Start the exchange gateway :param instmt: Instrument :return List of threads """ instmt.set_l2_depth(L2Depth(5)) instmt.set_prev_l2_depth(L2Depth(5)) instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), instmt.get_instmt_name())) self.init_instmt_snapshot_table(instmt) return [self.api_socket.connect(self.api_socket.get_link(), on_message_handler=partial(self.on_message_handler, instmt), on_open_handler=partial(self.on_open_handler, instmt), on_close_handler=partial(self.on_close_handler, instmt))]
def parse_l2_depth(cls, instmt, raw): """ Parse raw data to L2 depth :param instmt: Instrument :param raw: Raw data in JSON """ l2_depth = L2Depth() keys = list(raw.keys()) if (cls.get_bids_field_name() in keys and cls.get_asks_field_name() in keys): # Date time date_time = float(raw[cls.get_order_book_timestamp_field_name()]) date_time = date_time / cls.get_timestamp_offset() l2_depth.date_time = datetime.utcfromtimestamp(date_time).strftime("%Y%m%d %H:%M:%S.%f") # Bids bids = raw[cls.get_bids_field_name()] bids = sorted(bids, key=lambda x: x[0], reverse=True) for i in range(0, 5): l2_depth.bids[i].price = float(bids[i][0]) if not isinstance(bids[i][0], float) else bids[i][0] l2_depth.bids[i].volume = float(bids[i][1]) if not isinstance(bids[i][1], float) else bids[i][1] # Asks asks = raw[cls.get_asks_field_name()] asks = sorted(asks, key=lambda x: x[0]) for i in range(0, 5): l2_depth.asks[i].price = float(asks[i][0]) if not isinstance(asks[i][0], float) else asks[i][0] l2_depth.asks[i].volume = float(asks[i][1]) if not isinstance(asks[i][1], float) else asks[i][1] else: raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ (instmt.get_exchange_name(), instmt.get_instmt_name(), \ raw)) return l2_depth
def parse_l2_depth(cls, instmt, raw): """ Parse raw data to L2 depth :param instmt: Instrument :param raw: Raw data in JSON """ l2_depth = L2Depth() keys = list(raw.keys()) if cls.get_bids_field_name() in keys and \ cls.get_asks_field_name() in keys: # No Date time information, has update id only l2_depth.date_time = datetime.now().strftime("%Y%m%d %H:%M:%S.%f") # Bids bids = raw[cls.get_bids_field_name()] bids = sorted(bids, key=lambda x: x['price'], reverse=True) for i in range(0, 5): l2_depth.bids[i].price = float(bids[i]['price']) if type(bids[i]['price']) != float else bids[i]['price'] l2_depth.bids[i].volume = float(bids[i]['qty']) if type(bids[i]['qty']) != float else bids[i]['qty'] # Asks asks = raw[cls.get_asks_field_name()] asks = sorted(asks, key=lambda x: x['price']) for i in range(0, 5): l2_depth.asks[i].price = float(asks[i]['price']) if type(asks[i]['price']) != float else asks[i]['price'] l2_depth.asks[i].volume = float(asks[i]['qty']) if type(asks[i]['qty']) != float else asks[i]['qty'] else: raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ (instmt.get_exchange_name(), instmt.get_instmt_name(), \ raw)) return l2_depth
def parse_l2_depth(cls, instmt, raw): """ Parse raw data to L2 depth :param instmt: Instrument :param raw: Raw data in JSON """ l2_depth = L2Depth() raw = raw["result"] keys = list(raw.keys()) if cls.get_bids_field_name() in keys and \ cls.get_asks_field_name() in keys: # Date time l2_depth.date_time = datetime.utcnow().strftime( "%Y%m%d %H:%M:%S.%f") # Bids bids = raw[cls.get_bids_field_name()] for i in range(0, 5): l2_depth.bids[i].price = bids[i][cls.get_price_field_name()] l2_depth.bids[i].volume = bids[i][cls.get_volume_field_name()] # Asks asks = raw[cls.get_asks_field_name()] for i in range(0, 5): l2_depth.asks[i].price = asks[i][cls.get_price_field_name()] l2_depth.asks[i].volume = asks[i][cls.get_volume_field_name()] else: raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ (instmt.get_exchange_name(), instmt.get_instmt_name(), \ raw)) return l2_depth
def parse_l2_depth(cls, instmt, raw): """ Parse raw data to L2 depth :param instmt: Instrument :param raw: Raw data in JSON """ l2_depth = L2Depth() keys = list(raw.keys()) if cls.get_bids_field_name() in keys and \ cls.get_asks_field_name() in keys: # Bids bids = raw[cls.get_bids_field_name()] bids = sorted(bids, key=lambda x: x[0], reverse=True) for i in range(0, len(bids)): l2_depth.bids[i].price = float( bids[i][0]) if type(bids[i][0]) != float else bids[i][0] l2_depth.bids[i].volume = float( bids[i][1]) if type(bids[i][1]) != float else bids[i][1] # Asks asks = raw[cls.get_asks_field_name()] asks = sorted(asks, key=lambda x: x[0]) for i in range(0, len(asks)): l2_depth.asks[i].price = float( asks[i][0]) if type(asks[i][0]) != float else asks[i][0] l2_depth.asks[i].volume = float( asks[i][1]) if type(asks[i][1]) != float else asks[i][1] return l2_depth
def start(self, instmt): """ Start the exchange gateway :param instmt: Instrument :return List of threads """ instmt.set_l2_depth(L2Depth(5)) instmt.set_prev_l2_depth(L2Depth(5)) instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), instmt.get_instmt_name())) self.init_instmt_snapshot_table(instmt) instmt.set_recovered(False) t1 = threading.Thread(target=partial(self.get_order_book_worker, instmt)) t2 = threading.Thread(target=partial(self.get_trades_worker, instmt)) t1.start() t2.start() return [t1, t2]
def start(self, instmt): """ Start the exchange gateway :param instmt: Instrument :return List of threads """ instmt.set_l2_depth(L2Depth(50)) instmt.set_prev_l2_depth(L2Depth(50)) instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), instmt.get_instmt_name())) self.init_instmt_snapshot_table(instmt) t_trades = self.api_socket.connect(url=self.api_socket.get_link(), on_message_handler=partial(self.on_message_handler, instmt), on_open_handler=partial(self.on_open_handler, instmt), on_close_handler=partial(self.on_close_handler, instmt)) t_order_book = threading.Thread(target=partial(self.get_order_book_worker, instmt)) t_order_book.start() return [t_order_book, t_trades]
def parse_l2_depth(cls, instmt, raw): """ Parse raw data to L2 depth :param instmt: Instrument :param raw: Raw data in JSON """ keys = list(raw.keys()) if cls.get_bids_field_name() in keys and \ cls.get_asks_field_name() in keys: l2_depth = L2Depth() # Bids bids = raw[cls.get_bids_field_name()] bid_level = -1 for bid in bids: price = bid[cls.get_order_book_price_field_name()] volume = bid[cls.get_order_book_volume_field_name()] if bid_level == -1 or l2_depth.bids[bid_level].price != price: bid_level += 1 if bid_level < 5: l2_depth.bids[bid_level].price = float(price) else: break l2_depth.bids[bid_level].volume += float(volume) # Asks asks = raw[cls.get_asks_field_name()] ask_level = -1 for ask in asks: price = ask[cls.get_order_book_price_field_name()] volume = ask[cls.get_order_book_volume_field_name()] if ask_level == -1 or l2_depth.asks[ask_level].price != price: ask_level += 1 if ask_level < 5: l2_depth.asks[ask_level].price = float(price) else: break l2_depth.asks[ask_level].volume += float(volume) return l2_depth else: raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ (instmt.get_exchange_name(), instmt.get_instmt_name(), \ raw))
def parse_l2_depth(cls, instmt, raw): """ Parse raw data to L2 depth :param instmt: Instrument :param raw: Raw data in JSON """ l2_depth = L2Depth() raw = raw[instmt.instmt_code] keys = list(raw.keys()) if (cls.get_bids_field_name() in keys and cls.get_asks_field_name() in keys): # Date time l2_depth.date_time = datetime.utcnow().strftime( "%Y%m%d %H:%M:%S.%f") # Bids bids = raw[cls.get_bids_field_name()] for i in range(0, 5): l2_depth.bids[i].price = float(bids[i][0]) if not isinstance( bids[i][0], float) else bids[i][0] l2_depth.bids[i].volume = float(bids[i][1]) if not isinstance( bids[i][1], float) else bids[i][1] # Asks asks = raw[cls.get_asks_field_name()] for i in range(0, 5): l2_depth.asks[i].price = float(asks[i][0]) if not isinstance( asks[i][0], float) else asks[i][0] l2_depth.asks[i].volume = float(asks[i][1]) if not isinstance( asks[i][1], float) else asks[i][1] else: raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ (instmt.get_exchange_name(), instmt.get_instmt_name(), \ raw)) return l2_depth
instmt.set_l2_depth(L2Depth(5)) instmt.set_prev_l2_depth(L2Depth(5)) instmt.set_order_book_table_name(self.get_order_book_table_name(instmt.get_exchange_name(), instmt.get_instmt_name())) instmt.set_trades_table_name(self.get_trades_table_name(instmt.get_exchange_name(), instmt.get_instmt_name())) instmt.set_recovered(False) t1 = threading.Thread(target=partial(self.get_order_book_worker, instmt)) t1.start() t2 = threading.Thread(target=partial(self.get_trades_worker, instmt)) t2.start() return [t1, t2] if __name__ == '__main__': Logger.init_log() exchange_name = 'Gatecoin' instmt_name = 'BTCHKD' instmt_code = 'BTCHKD' instmt = Instrument(exchange_name, instmt_name, instmt_code) db_client = SqlClientTemplate() exch = ExchGwGatecoin(db_client) instmt.set_l2_depth(L2Depth(5)) instmt.set_prev_l2_depth(L2Depth(5)) instmt.set_order_book_table_name(exch.get_order_book_table_name(instmt.get_exchange_name(), instmt.get_instmt_name())) instmt.set_trades_table_name(exch.get_trades_table_name(instmt.get_exchange_name(), instmt.get_instmt_name())) instmt.set_recovered(False) # exch.get_order_book_worker(instmt) exch.get_trades_worker(instmt)
def parse_l2_depth(cls, instmt, raw): """ Parse raw data to L2 depth :param instmt: Instrument :param raw: Raw data in JSON """ def get_order_info(data): return ( data['price'] if 'price' in data.keys() else None, (0 if data['side'] == "Buy" else 1), data['id'], data['size'] if 'size' in data.keys() else None ) if raw['action'] in ('partial', 'insert'): # Order book initialization or insertion for data in raw['data']: if data['symbol'] != instmt.get_instmt_code(): continue price, side, id, volume = get_order_info(data) instmt.realtime_order_book_ids[side][id] = price if price not in instmt.realtime_order_book_prices[side].keys(): instmt.realtime_order_book_prices[side][price] = { id: volume } else: instmt.realtime_order_book_prices[side][price][id] = volume elif raw['action'] == 'update': # Order book update for data in raw['data']: if data['symbol'] != instmt.get_instmt_code(): continue _, side, id, volume = get_order_info(data) price = instmt.realtime_order_book_ids[side][id] instmt.realtime_order_book_ids[side][id] = price instmt.realtime_order_book_prices[side][price][id] = volume elif raw['action'] == 'delete': # Order book delete for data in raw['data']: if data['symbol'] != instmt.get_instmt_code(): continue _, side, id, _ = get_order_info(data) price = instmt.realtime_order_book_ids[side][id] del instmt.realtime_order_book_prices[side][price][id] if len(instmt.realtime_order_book_prices[side][price]) == 0: del instmt.realtime_order_book_prices[side][price] # return l2_depth l2_depth = L2Depth() l2_depth.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f") bids_px = sorted(list(instmt.realtime_order_book_prices[0].keys()), reverse=True)[:5] asks_px = sorted(list(instmt.realtime_order_book_prices[1].keys()))[:5] bids_qty = [sum(instmt.realtime_order_book_prices[0][px].values()) for px in bids_px] asks_qty = [sum(instmt.realtime_order_book_prices[1][px].values()) for px in asks_px] for i in range(0, 5): l2_depth.bids[i].price = bids_px[i] l2_depth.bids[i].volume = bids_qty[i] l2_depth.asks[i].price = asks_px[i] l2_depth.asks[i].volume = asks_qty[i] return l2_depth
def parse_l2_depth(cls, instmt, raw): """ Parse raw data to L2 depth :param instmt: Instrument :param raw: Raw data in JSON """ l2_depth = instmt.get_l2_depth() keys = list(raw.keys()) bids_field = cls.get_bids_field_name() asks_field = cls.get_asks_field_name() if bids_field in keys and asks_field in keys: # Date time l2_depth.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f") # Bids bids = raw[bids_field] bids_len = min(l2_depth.depth, len(bids)) for i in range(0, bids_len): l2_depth.bids[i].price = float(bids[i]['price']) \ if not isinstance(bids[i]['price'], float) else bids[i][0] l2_depth.bids[i].volume = float(bids[i]['volume']) \ if not isinstance(bids[i]['volume'], float) else bids[i][1] l2_depth.bids[i].id = bids[i]['id'] # Asks asks = raw[asks_field] asks_len = min(l2_depth.depth, len(asks)) for i in range(0, asks_len): l2_depth.asks[i].price = float(asks[i]['price']) \ if not isinstance(asks[i]['price'], float) else asks[i][0] l2_depth.asks[i].volume = float(asks[i]['volume']) \ if not isinstance(asks[i]['volume'], float) else asks[i][1] l2_depth.asks[i].id = asks[i]['id'] elif "order_id" in keys: if 'type' in keys: # Insertion order_id = raw['order_id'] price = float(raw['price']) volume = float(raw['volume']) update_type = raw['type'] if update_type == "BID": l2_depth.bids.append(L2Depth.Depth(price=price, volume=volume)) l2_depth.bids[-1].id = order_id l2_depth.sort_bids() if len(l2_depth.bids) > l2_depth.depth * 2: del l2_depth.bids[l2_depth.depth:] elif update_type == "ASK": l2_depth.asks.append(L2Depth.Depth(price=price, volume=volume)) l2_depth.asks[-1].id = order_id l2_depth.sort_asks() if len(l2_depth.asks) > l2_depth.depth * 2: del l2_depth.asks[l2_depth.depth:] elif 'base' in keys: # Update order_id = raw['order_id'] volume = float(raw['base']) price = float(raw['counter']) / float(raw['base']) for i in range(0, len(l2_depth.bids)): if l2_depth.bids[i].id == order_id: if l2_depth.bids[i].price == price: l2_depth.bids[i].volume -= volume break else: # Deletion order_id = raw['order_id'] found = False for i in range(0, len(l2_depth.bids)): if l2_depth.bids[i].id == order_id: found = True del l2_depth.bids[i] break if not found: for i in range(0, len(l2_depth.asks)): if l2_depth.asks[i].id == order_id: del l2_depth.asks[i] break else: raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % (instmt.get_exchange_name(), instmt.get_instmt_name(), raw)) return l2_depth
def parse_l2_depth(cls, instmt, raw): """ Parse raw data to L2 depth :param instmt: Instrument :param raw: Raw data in JSON """ # No order book mapping from config. Need to decode here. l2_depth = instmt.get_l2_depth() l2_depth.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f") if isinstance(raw[0], list): # Start subscription for i in range(0, 25): bid = raw[i] ask = raw[25 + i] l2_depth.bids[i] = L2Depth.Depth(price=bid[0], count=bid[1], volume=bid[2]) l2_depth.asks[i] = L2Depth.Depth(price=ask[0], count=ask[1], volume=-ask[2]) else: price = raw[1] count = raw[2] volume = raw[3] found = False if count == 0: # Deletion if volume > 0: for i in range(0, len(l2_depth.bids)): if price == l2_depth.bids[i].price: found = True del l2_depth.bids[i] break else: for i in range(0, len(l2_depth.asks)): if price == l2_depth.asks[i].price: found = True del l2_depth.asks[i] break if not found: depth_text = "" for i in range(0, l2_depth.depth): if i < len(l2_depth.bids): depth_text += "%.4f,%d,%.4f" % \ (l2_depth.bids[i].volume, \ l2_depth.bids[i].count, \ l2_depth.bids[i].price) else: depth_text += " " depth_text += "<--->" if i < len(l2_depth.asks): depth_text += "%.4f,%d,%.4f" % \ (l2_depth.asks[i].volume, \ l2_depth.asks[i].count, \ l2_depth.asks[i].price) else: depth_text += " " depth_text += "\n" Logger.info(cls.__name__, "Cannot find the deletion of the message: %s\nDepth:\n%s\n" % \ (raw, depth_text)) else: # Insertion/Update if volume > 0: # Update for i in range(0, len(l2_depth.bids)): if price == l2_depth.bids[i].price: l2_depth.bids[i].count = count l2_depth.bids[i].volume = volume found = True break if not found: # Insertion l2_depth.bids.append( L2Depth.Depth(price=price, count=count, volume=volume)) l2_depth.sort_bids() if len(l2_depth.bids) > l2_depth.depth * 2: del l2_depth.bids[l2_depth.depth:] else: for i in range(0, len(l2_depth.asks)): # Update if price == l2_depth.asks[i].price: l2_depth.asks[i].count = count l2_depth.asks[i].volume = -volume found = True break if not found: # Insertion l2_depth.asks.append( L2Depth.Depth(price=price, count=count, volume=-volume)) l2_depth.sort_asks() if len(l2_depth.asks) > l2_depth.depth * 2: del l2_depth.asks[l2_depth.depth:] return l2_depth