def btfxwss(): log = logging.getLogger(__name__) fh = logging.FileHandler('test.log') fh.setLevel(logging.DEBUG) sh = logging.StreamHandler(sys.stdout) sh.setLevel(logging.DEBUG) log.addHandler(sh) log.addHandler(fh) logging.basicConfig(level=logging.DEBUG, handlers=[fh, sh]) wss = BtfxWss() wss.start() while not wss.conn.connected.is_set(): time.sleep(1) # Subscribe to some channels wss.subscribe_to_ticker('BTCUSD') wss.subscribe_to_order_book('BTCUSD') wss.subscribe_to_order_book('ETHUSD') # Wait for subscription... time.sleep(2) mbl = MBL() book = Book() recorder = Recorder(open("output.txt", mode='w')) # Accessing data stored in BtfxWss: books_queue = wss.books('ETHUSD') # returns a Queue object for the pair. while True: book_update = books_queue.get()[0][0] if len(book_update) > 3: print("snapshot") mbl_snapshot = MBLSnapshot(book_update) book = mbl.from_snapshot(mbl_snapshot) recorder.on_mbl(book) elif len(book_update) == 3: print("update: " + str(book_update)) update = MBLUpdate(book_update[0], book_update[1], book_update[2]) book = mbl.update(book, update) recorder.on_mbl(book) # Unsubscribing from channels: wss.unsubscribe_from_ticker('BTCUSD') wss.unsubscribe_from_order_book('BTCUSD') # Shutting down the client: wss.stop()
def test_subscribing_to_data_works(self): wss = BtfxWss(log_level=logging.DEBUG) wss.start() time.sleep(1) wss.subscribe_to_ticker('BTCUSD') wss.subscribe_to_candles('BTCUSD') wss.subscribe_to_order_book('BTCUSD') wss.subscribe_to_raw_order_book('BTCUSD') wss.subscribe_to_trades('BTCUSD') time.sleep(10) try: wss.tickers('BTCUSD').get(block=False) except Empty: self.fail("No ticker data arrived!") try: wss.candles('BTCUSD', '1m').get(block=False) except Empty: self.fail("No candles data arrived!") except KeyError: self.fail("No candles data arrived, key not found! %s" % list(wss.queue_processor.candles.keys())) try: wss.books('BTCUSD').get(block=False) except Empty: self.fail("No book data arrived!") try: wss.raw_books('BTCUSD').get(block=False) except Empty: self.fail("No war book data arrived!") try: wss.trades('BTCUSD').get(block=False) except Empty: self.fail("No trades data arrived!") wss.stop()
def test_subscribing_to_data_works(self): wss = BtfxWss() wss.start() time.sleep(1) wss.subscribe_to_ticker('BTCUSD') wss.subscribe_to_candles('BTCUSD') wss.subscribe_to_order_book('BTCUSD') wss.subscribe_to_raw_order_book('BTCUSD') wss.subscribe_to_trades('BTCUSD') time.sleep(10) try: wss.tickers('BTCUSD').get(block=False) except Empty: self.fail("No ticker data arrived!") try: wss.candles('BTCUSD', '1m').get(block=False) except Empty: self.fail("No candles data arrived!") try: wss.books('BTCUSD').get(block=False) except Empty: self.fail("No book data arrived!") try: wss.raw_books('BTCUSD').get(block=False) except Empty: self.fail("No war book data arrived!") try: wss.trades('BTCUSD').get(block=False) except Empty: self.fail("No trades data arrived!") wss.stop()
class Market(object): """ This is the market of the cryptocurrency given symbol. Data source is Bitfinex Websocket API. """ def __init__(self, symbol='BTCUSD'): self.symbol = symbol self.wss = BtfxWss() self.wss.start() def check_connection(self, pat=60): """ Check connection with Bitfinex API. If consecutively failing to connect, the program will send an alert message :param pat: the maximum patience time before sending an alert notification of disrupted connection. """ init = time.time() # Wait until either connected or exceeding patience while not self.wss.conn.connected.is_set() or time.time() > init + pat: time.sleep(1) # If connection is not set, send a note if self.wss.conn.connected.is_set(): pass else: message(txt_msg='The connection is disrupted.') logger.error('Connection failed.') exit() def initialize_api(self): """ Initialize the API for the crypto market and subscribe to trades and order books. Data is stored as Queues. """ self.check_connection() # Subscribe to some channels self.wss.subscribe_to_trades(self.symbol) self.wss.subscribe_to_order_book(pair=self.symbol, len=100) # Initialize a DataBase object self.db = DataBase(symbol=self.symbol) logger.info('API connection initialized.') def create_database(self): """ This function will build connection with Bitfinex Websocket API and AWS MySQL DB. Then initialize the csv files and sql databases for trades and orderbook. """ # Check connection self.check_connection() # Prepare csv databases for trades and quotes self.db.initialize_trade_csv() self.db.initialize_quote_csv() # Prepare sql database and tables self.db.initialize_sql_db() self.db.initialize_trade_sql() self.db.initialize_quote_sql() # Access data from BtfxWss and return a Queue object for the pair: self.trade_q = self.wss.trades(self.symbol) self.quote_q = self.wss.books(self.symbol) # Take a snapshot of the orderbook quote_snapshot = self.quote_q.get() # Input the snapshot to database self.db.create_order_book(quote_snapshot) self.db.create_quote_csv(quote_snapshot) self.db.create_quote_sql(quote_snapshot) logger.info('Databases created.') def stream_data(self): """ This function access the live data with established Bitfinex API, and keep streaming new trade and quote data. Then update the csv files and sql databases accordingly. """ # Check connection self.check_connection() # Update trade info new_trade = self.trade_q.get() self.db.update_trade_csv(new_trade) self.db.update_trade_sql(new_trade) # Update order book quote_change = self.quote_q.get() self.db.update_quote_csv(quote_change) self.db.update_quote_sql(quote_change) logger.info('New trade and quote updated.')
class BitfinexBookWatcher: def __init__(self): self.bids = dict() self.asks = dict() self.bid_depth = 0 self.bid_ma_slow = 0 self.bid_ma_fast = 0 self.ask_depth = 0 self.ask_ma_slow = 0 self.ask_ma_fast = 0 self.wss = BtfxWss() def start(self): if not self.wss.conn.connected.is_set(): logger.info("Starting Bitfinex websocket client") self.wss.start() while not self.wss.conn.connected.is_set(): time.sleep(1) # for P1 precision usual width of order book is ~100 USD (for BTC/USD price ~6500) # for wider range (about 1500 USD) use P2 precision. But the price will be rounded to tens self.wss.subscribe_to_order_book('BTCUSD', prec="P1", len=100) logger.info("Subscribed to Order Book WSS channel") # waiting for a bit to receive the book snapshot time.sleep(2) # call get_updates regularly to clear the queue!! Otherwise you may get OutOfMemory errors def get_updates(self): book_q = self.wss.books( 'BTCUSD') # returns a Queue object for the pair. # result = [] while not book_q.empty(): get = book_q.get() # print(get) # result.append(get) book_item, tag = get self.fill_the_book(book_item) self.bid_depth = sum(self.bids.values()) self.bid_ma_fast = update_ma(self.bid_depth, self.bid_ma_fast, 5) self.bid_ma_slow = update_ma(self.bid_depth, self.bid_ma_slow, 90) self.ask_depth = sum(self.asks.values()) self.ask_ma_fast = update_ma(self.ask_depth, self.ask_ma_fast, 5) self.ask_ma_slow = update_ma(self.ask_depth, self.ask_ma_slow, 90) # logger.debug("Market depth: bids: {0}, ma5: {1} ma90: {2} | asks: {3}, ma5: {4} ma90: {5}".format( # round(self.bid_depth), # round(self.bid_ma_fast), # round(self.bid_ma_slow), # round(self.ask_depth), # round(self.ask_ma_fast), # round(self.ask_ma_slow) # )) # return result # better call stop() at the end of the program (and on TERM signal) def stop(self): if self.wss.conn.connected.is_set() and self.wss.channel_configs: logger.debug("unsubscribe") self.wss.unsubscribe_from_order_book('BTCUSD') if self.wss.conn.connected.is_set(): logger.debug("stopping the socket") self.wss.stop() logger.debug("stopped.") def fill_the_book(self, input_list): for row in input_list: if isinstance(row[0], list): self.fill_the_book(row) else: price = str(row[0]) count = row[1] amt = row[2] if count != 0: if amt > 0: self.bids[price] = amt else: self.asks[price] = abs(amt) else: if amt > 0: if price in self.bids.keys(): del (self.bids[price]) elif amt < 0: if price in self.asks.keys(): del (self.asks[price])