class CoinbaseClient(Client):
def __init__(self, **kwargs):
"""
Constructor for Coinbase Client.
"""
super(CoinbaseClient, self).__init__(exchange='coinbase', **kwargs)
self.request = json.dumps(
dict(type='subscribe', product_ids=[self.sym], channels=['full']))
self.request_unsubscribe = json.dumps(
dict(type='unsubscribe', product_ids=[self.sym],
channels=['full']))
self.book = CoinbaseOrderBook(sym=self.sym)
self.trades_request = None
self.ws_endpoint = COINBASE_ENDPOINT
def run(self):
"""
Handle incoming level 3 data on a separate thread or process.
Returns
-------
"""
super(CoinbaseClient, self).run()
while True:
msg = self.queue.get()
if self.book.new_tick(msg) is False:
self.book.load_book()
self.retry_counter += 1
LOGGER.info('\n[%s - %s] ...going to try and reload the order '
'book\n' % (self.exchange.upper(), self.sym))
continue
def __init__(self, **kwargs):
"""
Constructor for Coinbase Client.
"""
super(CoinbaseClient, self).__init__(exchange='coinbase', **kwargs)
self.request = json.dumps(
dict(type='subscribe', product_ids=[self.sym], channels=['full']))
self.request_unsubscribe = json.dumps(
dict(type='unsubscribe', product_ids=[self.sym],
channels=['full']))
self.book = CoinbaseOrderBook(sym=self.sym)
self.trades_request = None
self.ws_endpoint = COINBASE_ENDPOINT
def __init__(self, ccy, exchange):
super(Client, self).__init__(name=ccy, daemon=True)
self.sym = ccy
self.exchange = exchange
self.ws = None
self.retry_counter = 0
self.max_retries = MAX_RECONNECTION_ATTEMPTS
self.last_subscribe_time = None
self.queue = Queue(maxsize=0)
if self.exchange == 'coinbase':
self.request = json.dumps(dict(type='subscribe', product_ids=[self.sym], channels=['full']))
self.request_unsubscribe = json.dumps(dict(type='unsubscribe', product_ids=[self.sym], channels=['full']))
self.book = CoinbaseOrderBook(self.sym)
self.trades_request = None
self.ws_endpoint = COINBASE_ENDPOINT
elif self.exchange == 'bitfinex':
self.request = json.dumps({
"event": "subscribe",
"channel": "book",
"prec": "R0",
"freq": "F0",
"symbol": self.sym,
"len": "100"
})
self.request_unsubscribe = None
self.trades_request = json.dumps({
"event": "subscribe",
"channel": "trades",
"symbol": self.sym
})
self.book = BitfinexOrderBook(self.sym)
self.ws_endpoint = BITFINEX_ENDPOINT
def __init__(self, ccy: str):
"""
Constructor for Coinbase Client.
Parameters
----------
ccy : str
Name of instrument or cryptocurrency pair.
"""
super(CoinbaseClient, self).__init__(ccy=ccy, exchange='coinbase')
self.request = json.dumps(
dict(type='subscribe', product_ids=[self.sym], channels=['full']))
self.request_unsubscribe = json.dumps(
dict(type='unsubscribe', product_ids=[self.sym],
channels=['full']))
self.book = CoinbaseOrderBook(sym=self.sym)
self.trades_request = None
self.ws_endpoint = COINBASE_ENDPOINT
def get_orderbook_snapshot_history(self, query):
"""
Function to replay historical market data and generate
the features used for reinforcement learning & training.
NOTE:
The query can either be a single Coinbase CCY, or both Coinbase and Bitfinex,
but it cannot be only a Biftinex CCY. Later releases of this repo will
support Bitfinex only orderbook reconstruction.
:param query: (dict) query for finding tick history in Arctic TickStore
:return: (list of arrays) snapshots of limit order books using a
stationary feature set
"""
tick_history = self.get_tick_history(query=query)
loop_length = tick_history.shape[0]
coinbase_tick_counter = 0
snapshot_list = list()
last_snapshot_time = None
symbols = query['ccy']
print('querying {}'.format(symbols))
include_bitfinex = len(symbols) > 1
if include_bitfinex:
print('\n\nIncluding Bitfinex data in feature set.\n\n')
coinbase_order_book = CoinbaseOrderBook(symbols[0])
bitfinex_order_book = BitfinexOrderBook(symbols[1]) if include_bitfinex \
else None
start_time = dt.now(TIMEZONE)
print(
'Starting get_orderbook_snapshot_history() loop with %i ticks for %s'
% (loop_length, query['ccy']))
for idx, tx in enumerate(tick_history.itertuples()):
tick = tx._asdict()
# determine if incoming tick is from coinbase or bitfinex
coinbase = True if tick['product_id'] == coinbase_order_book.sym else \
False
if 'type' not in tick:
# filter out bad ticks
continue
if tick['type'] in ['load_book', 'book_loaded', 'preload']:
# flag for a order book reset
if coinbase:
coinbase_order_book.new_tick(tick)
else:
bitfinex_order_book.new_tick(tick)
# skip to next loop
continue
if coinbase: # incoming tick is from Coinbase exchange
if coinbase_order_book.done_warming_up():
new_tick_time = parse(tick.get('time'))
# timestamp for incoming tick
if new_tick_time is None:
print('No tick time: {}'.format(tick))
continue
coinbase_tick_counter += 1
coinbase_order_book.new_tick(tick)
if coinbase_tick_counter == 1:
# start tracking snapshot timestamps
# and keep in mind that snapshots are tethered to
# coinbase timestamps
last_snapshot_time = new_tick_time
print('%s first tick: %s | Sequence: %i' %
(coinbase_order_book.sym, str(new_tick_time),
coinbase_order_book.sequence))
# skip to next loop
continue
# calculate the amount of time between the incoming
# tick and tick received before that
diff = (new_tick_time - last_snapshot_time).microseconds
# multiple = diff // 250000
multiple = diff // 500000 # 500000 is 500 milliseconds,
# or 2x a second
# if there is a pause in incoming data,
# continue to create order book snapshots
if multiple >= 1:
# check to include Bitfinex data in features
if include_bitfinex:
for _ in range(multiple):
if coinbase_order_book.done_warming_up() & \
bitfinex_order_book.done_warming_up():
coinbase_order_book_snapshot = \
coinbase_order_book.render_book()
bitfinex_order_book_snapshot = \
bitfinex_order_book.render_book()
midpoint_delta = coinbase_order_book.midpoint - \
bitfinex_order_book.midpoint
snapshot_list.append(
list(
np.hstack((
new_tick_time, # tick time
coinbase_order_book.
midpoint, # midpoint price
midpoint_delta, # price delta between exchanges
coinbase_order_book_snapshot,
bitfinex_order_book_snapshot
)))) # longs/shorts
last_snapshot_time += timedelta(
milliseconds=500)
else:
last_snapshot_time += timedelta(
milliseconds=500)
else: # do not include bitfinex
for _ in range(multiple):
if coinbase_order_book.done_warming_up():
coinbase_order_book_snapshot = \
coinbase_order_book.render_book()
snapshot_list.append(
list(
np.hstack((
new_tick_time, # tick time
coinbase_order_book.midpoint,
coinbase_order_book_snapshot))))
last_snapshot_time += timedelta(
milliseconds=500)
else:
last_snapshot_time += timedelta(
milliseconds=500)
# incoming tick is from Bitfinex exchange
elif include_bitfinex & bitfinex_order_book.done_warming_up():
bitfinex_order_book.new_tick(tick)
# periodically print number of steps completed
if idx % 250000 == 0:
elapsed = (dt.now(TIMEZONE) - start_time).seconds
print('...completed %i loops in %i seconds' % (idx, elapsed))
elapsed = (dt.now(TIMEZONE) - start_time).seconds
print('Completed run_simulation() with %i ticks in %i seconds '
'at %i ticks/second' %
(loop_length, elapsed, loop_length // elapsed))
orderbook_snapshot_history = pd.DataFrame(
snapshot_list,
columns=self.get_feature_labels(include_system_time=True,
include_bitfinex=include_bitfinex))
orderbook_snapshot_history = orderbook_snapshot_history.dropna(axis=0)
return orderbook_snapshot_history
class CoinbaseClient(Client):
def __init__(self, **kwargs):
"""
Constructor for Coinbase Client.
"""
super(CoinbaseClient, self).__init__(exchange='coinbase', **kwargs)
self.request = json.dumps(
dict(type='subscribe', product_ids=[self.sym], channels=['full']))
self.request_unsubscribe = json.dumps(
dict(type='unsubscribe', product_ids=[self.sym],
channels=['full']))
self.book = CoinbaseOrderBook(sym=self.sym)
self.trades_request = None
self.ws_endpoint = COINBASE_ENDPOINT
def run(self):
"""
Handle incoming level 3 data on a separate thread or process.
Returns
-------
"""
curr_tim = dt.now(tz=TIMEZONE)
next_min = curr_tim.hour + 1
super(CoinbaseClient, self).run()
while True:
msg = self.queue.get()
curr_tim = dt.now(tz=TIMEZONE)
curr_min = curr_tim.hour
# Min = hour in this case
if curr_min == next_min == 0:
self.book.load_book()
print("RELOADING BOOK")
next_min = 4
continue
if curr_min == next_min == 2:
self.book.load_book()
print("RELOADING BOOK")
next_min = 6
continue
if curr_min == next_min == 4:
self.book.load_book()
print("RELOADING BOOK")
next_min = 8
continue
if curr_min == next_min == 6:
self.book.load_book()
print("RELOADING BOOK")
next_min = 10
continue
if curr_min == next_min == 8:
self.book.load_book()
print("RELOADING BOOK")
next_min = 12
continue
if curr_min == next_min == 10:
self.book.load_book()
print("RELOADING BOOK")
next_min = 14
continue
if curr_min == next_min == 12:
self.book.load_book()
print("RELOADING BOOK")
next_min = 16
continue
if curr_min == next_min == 14:
self.book.load_book()
print("RELOADING BOOK")
next_min = 18
continue
if curr_min == next_min == 16:
self.book.load_book()
print("RELOADING BOOK")
next_min = 20
continue
if curr_min == next_min == 18:
self.book.load_book()
print("RELOADING BOOK")
next_min = 22
continue
if curr_min == next_min == 20:
self.book.load_book()
print("RELOADING BOOK")
next_min = 0
continue
if curr_min == next_min == 22:
self.book.load_book()
print("RELOADING BOOK")
next_min = 2
continue
if self.book.new_tick(msg) is False:
# Coinbase requires a REST call to GET the initial LOB snapshot
self.book.load_book()
self.retry_counter += 1
LOGGER.info('\n[%s - %s] ...going to try and reload the order '
'book\n' % (self.exchange.upper(), self.sym))
continue
def get_orderbook_snapshot_history(self, query: dict):
"""
Function to replay historical market data and generate
the features used for reinforcement learning & training.
NOTE:
The query can either be a single Coinbase CCY, or both Coinbase and Bitfinex,
but it cannot be only a Biftinex CCY. Later releases of this repo will
support Bitfinex only orderbook reconstruction.
:param query: (dict) query for finding tick history in Arctic TickStore
:return: (pd.DataFrame) snapshots of limit order books using a
stationary feature set
"""
self.db.init_db_connection()
tick_history = self.db.get_tick_history(query=query)
if tick_history is None:
print("Query returned no data: {}".format(query))
return None
loop_length = tick_history.shape[0]
# number of microseconds between LOB snapshots
snapshot_interval_milliseconds = SNAPSHOT_RATE_IN_MICROSECONDS // 1000
snapshot_list = list()
last_snapshot_time = None
symbols = query['ccy']
print('querying {}'.format(symbols))
include_bitfinex = len(symbols) > 1
if include_bitfinex:
print('\n\nIncluding Bitfinex data in feature set.\n\n')
coinbase_order_book = CoinbaseOrderBook(symbols[0])
bitfinex_order_book = BitfinexOrderBook(symbols[1]) if include_bitfinex \
else None
start_time = dt.now(TIMEZONE)
print(
'Starting get_orderbook_snapshot_history() loop with %i ticks for %s'
% (loop_length, query['ccy']))
# loop through all ticks returned from the Arctic Tick Store query.
for count, tx in enumerate(tick_history.itertuples()):
# periodically print number of steps completed
if count % 250000 == 0:
elapsed = (dt.now(TIMEZONE) - start_time).seconds
print('...completed %i loops in %i seconds' % (count, elapsed))
# convert to dictionary for processing
tick = tx._asdict()
# determine if incoming tick is from coinbase or bitfinex
coinbase = True if tick['product_id'] == coinbase_order_book.sym else \
False
# filter out bad ticks
if 'type' not in tick:
continue
# flags for a order book reset
if tick['type'] in ['load_book', 'book_loaded', 'preload']:
if coinbase:
coinbase_order_book.new_tick(tick)
else:
bitfinex_order_book.new_tick(tick)
# skip to next loop
continue
# incoming tick is for coinbase LOB
if coinbase:
# check if the LOB is pre-loaded, if not skip message and do NOT process.
if coinbase_order_book.done_warming_up() is False:
print("coinbase_order_book not done warming up: {}".format(
tick))
continue
# timestamp for incoming tick
new_tick_time = parse(tick.get('time'))
# remove ticks without timestamps (should not exist/happen)
if new_tick_time is None:
print('No tick time: {}'.format(tick))
continue
# initialize the LOB snapshot timer
if last_snapshot_time is None:
# process first ticks and check if they're stale ticks; if so,
# skip to the next loop.
coinbase_order_book.new_tick(tick)
last_coinbase_tick_time = coinbase_order_book.last_tick_time
if last_coinbase_tick_time is None:
continue
last_coinbase_tick_time_dt = parse(last_coinbase_tick_time)
last_snapshot_time = last_coinbase_tick_time_dt
print('{} first tick: {} | Sequence: {}'.format(
coinbase_order_book.sym, new_tick_time,
coinbase_order_book.sequence))
# skip to next loop
continue
# calculate the amount of time between the incoming
# tick and tick received before that
diff = self._get_microsecond_delta(new_tick_time,
last_snapshot_time)
# update the LOB, but do not take a LOB snapshot if the tick time is
# out of sequence. This occurs when pre-loading a LOB with stale tick
# times in general.
if diff == -1:
coinbase_order_book.new_tick(tick)
continue
# derive the number of LOB snapshot insertions for the data buffer.
multiple = diff // SNAPSHOT_RATE_IN_MICROSECONDS # 1000000 is 1 second
# proceed if we have one or more insertions to make
if multiple <= 0:
coinbase_order_book.new_tick(tick)
continue
# check to include Bitfinex data in features.
if include_bitfinex:
# if bitfinex's LOB is still loading, do NOT export snapshots
# of coinbase in the meantime and continue to next loop.
if bitfinex_order_book.done_warming_up() is False:
print("bitfinex_order_book not done warming up: {}".
format(tick))
coinbase_order_book.new_tick(tick)
# update the LOB snapshot tracker.
for _ in range(multiple):
last_snapshot_time += timedelta(
milliseconds=snapshot_interval_milliseconds)
# move to next loop and see if bitfinex's LOB is ready then.
continue
# since both coinbase and bitfinex LOBs are assumed to be
# pre-loaded at this point, we can proceed to export snapshots
# of the LOB, even if there has been a 'long' duration between
# consecutive ticks.
coinbase_order_book_snapshot = coinbase_order_book.render_book(
)
bitfinex_order_book_snapshot = bitfinex_order_book.render_book(
)
midpoint_delta = coinbase_order_book.midpoint - \
bitfinex_order_book.midpoint
# update the LOB snapshot time-delta AND add LOB snapshots to the
# data buffer.
for i in range(multiple):
last_snapshot_time += timedelta(
milliseconds=snapshot_interval_milliseconds)
snapshot_list.append(
np.hstack((
last_snapshot_time,
coinbase_order_book.midpoint, # midpoint price
midpoint_delta, # price delta between exchanges
coinbase_order_book_snapshot,
bitfinex_order_book_snapshot))) # longs/shorts
# update order book with most recent tick now, so the snapshots
# are up to date for the next iteration of the loop.
coinbase_order_book.new_tick(tick)
continue
else: # do not include bitfinex
coinbase_order_book_snapshot = coinbase_order_book.render_book(
)
for i in range(multiple):
last_snapshot_time += timedelta(
milliseconds=snapshot_interval_milliseconds)
snapshot_list.append(
np.hstack((last_snapshot_time,
coinbase_order_book.midpoint,
coinbase_order_book_snapshot)))
# update order book with most recent tick now, so the snapshots
# are up to date for the next iteration of the loop.
coinbase_order_book.new_tick(tick)
continue
# incoming tick is from Bitfinex exchange
elif include_bitfinex and bitfinex_order_book.done_warming_up():
bitfinex_order_book.new_tick(tick)
continue
elapsed = (dt.now(TIMEZONE) - start_time).seconds
print('Completed run_simulation() with %i ticks in %i seconds '
'at %i ticks/second' %
(loop_length, elapsed, loop_length // elapsed))
orderbook_snapshot_history = pd.DataFrame(
snapshot_list,
columns=self.get_feature_labels(
include_system_time=True,
include_spread=False,
include_bitfinex=include_bitfinex,
include_order_flow=INCLUDE_ORDERFLOW,
include_imbalances=False,
include_ema=self.alpha))
orderbook_snapshot_history = orderbook_snapshot_history.dropna(axis=0)
return orderbook_snapshot_history