def btfxwss():
log = logging.getLogger(__name__)
fh = logging.FileHandler('test.log')
fh.setLevel(logging.DEBUG)
sh = logging.StreamHandler(sys.stdout)
sh.setLevel(logging.DEBUG)
log.addHandler(sh)
log.addHandler(fh)
logging.basicConfig(level=logging.DEBUG, handlers=[fh, sh])
wss = BtfxWss()
wss.start()
while not wss.conn.connected.is_set():
time.sleep(1)
# Subscribe to some channels
wss.subscribe_to_ticker('BTCUSD')
wss.subscribe_to_order_book('BTCUSD')
wss.subscribe_to_order_book('ETHUSD')
# Wait for subscription...
time.sleep(2)
mbl = MBL()
book = Book()
recorder = Recorder(open("output.txt", mode='w'))
# Accessing data stored in BtfxWss:
books_queue = wss.books('ETHUSD') # returns a Queue object for the pair.
while True:
book_update = books_queue.get()[0][0]
if len(book_update) > 3:
print("snapshot")
mbl_snapshot = MBLSnapshot(book_update)
book = mbl.from_snapshot(mbl_snapshot)
recorder.on_mbl(book)
elif len(book_update) == 3:
print("update: " + str(book_update))
update = MBLUpdate(book_update[0], book_update[1], book_update[2])
book = mbl.update(book, update)
recorder.on_mbl(book)
# Unsubscribing from channels:
wss.unsubscribe_from_ticker('BTCUSD')
wss.unsubscribe_from_order_book('BTCUSD')
# Shutting down the client:
wss.stop()
def test_subscribing_to_data_works(self):
wss = BtfxWss(log_level=logging.DEBUG)
wss.start()
time.sleep(1)
wss.subscribe_to_ticker('BTCUSD')
wss.subscribe_to_candles('BTCUSD')
wss.subscribe_to_order_book('BTCUSD')
wss.subscribe_to_raw_order_book('BTCUSD')
wss.subscribe_to_trades('BTCUSD')
time.sleep(10)
try:
wss.tickers('BTCUSD').get(block=False)
except Empty:
self.fail("No ticker data arrived!")
try:
wss.candles('BTCUSD', '1m').get(block=False)
except Empty:
self.fail("No candles data arrived!")
except KeyError:
self.fail("No candles data arrived, key not found! %s" %
list(wss.queue_processor.candles.keys()))
try:
wss.books('BTCUSD').get(block=False)
except Empty:
self.fail("No book data arrived!")
try:
wss.raw_books('BTCUSD').get(block=False)
except Empty:
self.fail("No war book data arrived!")
try:
wss.trades('BTCUSD').get(block=False)
except Empty:
self.fail("No trades data arrived!")
wss.stop()
def test_subscribing_to_data_works(self):
wss = BtfxWss()
wss.start()
time.sleep(1)
wss.subscribe_to_ticker('BTCUSD')
wss.subscribe_to_candles('BTCUSD')
wss.subscribe_to_order_book('BTCUSD')
wss.subscribe_to_raw_order_book('BTCUSD')
wss.subscribe_to_trades('BTCUSD')
time.sleep(10)
try:
wss.tickers('BTCUSD').get(block=False)
except Empty:
self.fail("No ticker data arrived!")
try:
wss.candles('BTCUSD', '1m').get(block=False)
except Empty:
self.fail("No candles data arrived!")
try:
wss.books('BTCUSD').get(block=False)
except Empty:
self.fail("No book data arrived!")
try:
wss.raw_books('BTCUSD').get(block=False)
except Empty:
self.fail("No war book data arrived!")
try:
wss.trades('BTCUSD').get(block=False)
except Empty:
self.fail("No trades data arrived!")
wss.stop()
class Market(object):
"""
This is the market of the cryptocurrency given symbol.
Data source is Bitfinex Websocket API.
"""
def __init__(self, symbol='BTCUSD'):
self.symbol = symbol
self.wss = BtfxWss()
self.wss.start()
def check_connection(self, pat=60):
"""
Check connection with Bitfinex API. If consecutively failing to connect,
the program will send an alert message
:param pat:
the maximum patience time before sending an alert notification of
disrupted connection.
"""
init = time.time()
# Wait until either connected or exceeding patience
while not self.wss.conn.connected.is_set() or time.time() > init + pat:
time.sleep(1)
# If connection is not set, send a note
if self.wss.conn.connected.is_set():
pass
else:
message(txt_msg='The connection is disrupted.')
logger.error('Connection failed.')
exit()
def initialize_api(self):
"""
Initialize the API for the crypto market and subscribe to trades and order
books. Data is stored as Queues.
"""
self.check_connection()
# Subscribe to some channels
self.wss.subscribe_to_trades(self.symbol)
self.wss.subscribe_to_order_book(pair=self.symbol, len=100)
# Initialize a DataBase object
self.db = DataBase(symbol=self.symbol)
logger.info('API connection initialized.')
def create_database(self):
"""
This function will build connection with Bitfinex Websocket API and AWS
MySQL DB. Then initialize the csv files and sql databases
for trades and orderbook.
"""
# Check connection
self.check_connection()
# Prepare csv databases for trades and quotes
self.db.initialize_trade_csv()
self.db.initialize_quote_csv()
# Prepare sql database and tables
self.db.initialize_sql_db()
self.db.initialize_trade_sql()
self.db.initialize_quote_sql()
# Access data from BtfxWss and return a Queue object for the pair:
self.trade_q = self.wss.trades(self.symbol)
self.quote_q = self.wss.books(self.symbol)
# Take a snapshot of the orderbook
quote_snapshot = self.quote_q.get()
# Input the snapshot to database
self.db.create_order_book(quote_snapshot)
self.db.create_quote_csv(quote_snapshot)
self.db.create_quote_sql(quote_snapshot)
logger.info('Databases created.')
def stream_data(self):
"""
This function access the live data with established Bitfinex API, and
keep streaming new trade and quote data. Then update the csv files and
sql databases accordingly.
"""
# Check connection
self.check_connection()
# Update trade info
new_trade = self.trade_q.get()
self.db.update_trade_csv(new_trade)
self.db.update_trade_sql(new_trade)
# Update order book
quote_change = self.quote_q.get()
self.db.update_quote_csv(quote_change)
self.db.update_quote_sql(quote_change)
logger.info('New trade and quote updated.')
class BitfinexBookWatcher:
def __init__(self):
self.bids = dict()
self.asks = dict()
self.bid_depth = 0
self.bid_ma_slow = 0
self.bid_ma_fast = 0
self.ask_depth = 0
self.ask_ma_slow = 0
self.ask_ma_fast = 0
self.wss = BtfxWss()
def start(self):
if not self.wss.conn.connected.is_set():
logger.info("Starting Bitfinex websocket client")
self.wss.start()
while not self.wss.conn.connected.is_set():
time.sleep(1)
# for P1 precision usual width of order book is ~100 USD (for BTC/USD price ~6500)
# for wider range (about 1500 USD) use P2 precision. But the price will be rounded to tens
self.wss.subscribe_to_order_book('BTCUSD', prec="P1", len=100)
logger.info("Subscribed to Order Book WSS channel")
# waiting for a bit to receive the book snapshot
time.sleep(2)
# call get_updates regularly to clear the queue!! Otherwise you may get OutOfMemory errors
def get_updates(self):
book_q = self.wss.books(
'BTCUSD') # returns a Queue object for the pair.
# result = []
while not book_q.empty():
get = book_q.get()
# print(get)
# result.append(get)
book_item, tag = get
self.fill_the_book(book_item)
self.bid_depth = sum(self.bids.values())
self.bid_ma_fast = update_ma(self.bid_depth, self.bid_ma_fast, 5)
self.bid_ma_slow = update_ma(self.bid_depth, self.bid_ma_slow, 90)
self.ask_depth = sum(self.asks.values())
self.ask_ma_fast = update_ma(self.ask_depth, self.ask_ma_fast, 5)
self.ask_ma_slow = update_ma(self.ask_depth, self.ask_ma_slow, 90)
# logger.debug("Market depth: bids: {0}, ma5: {1} ma90: {2} | asks: {3}, ma5: {4} ma90: {5}".format(
# round(self.bid_depth),
# round(self.bid_ma_fast),
# round(self.bid_ma_slow),
# round(self.ask_depth),
# round(self.ask_ma_fast),
# round(self.ask_ma_slow)
# ))
# return result
# better call stop() at the end of the program (and on TERM signal)
def stop(self):
if self.wss.conn.connected.is_set() and self.wss.channel_configs:
logger.debug("unsubscribe")
self.wss.unsubscribe_from_order_book('BTCUSD')
if self.wss.conn.connected.is_set():
logger.debug("stopping the socket")
self.wss.stop()
logger.debug("stopped.")
def fill_the_book(self, input_list):
for row in input_list:
if isinstance(row[0], list):
self.fill_the_book(row)
else:
price = str(row[0])
count = row[1]
amt = row[2]
if count != 0:
if amt > 0:
self.bids[price] = amt
else:
self.asks[price] = abs(amt)
else:
if amt > 0:
if price in self.bids.keys():
del (self.bids[price])
elif amt < 0:
if price in self.asks.keys():
del (self.asks[price])