def __init__(self, ccy, exchange):
super(Client, self).__init__(name=ccy, daemon=True)
self.sym = ccy
self.exchange = exchange
self.ws = None
self.retry_counter = 0
self.max_retries = configs.MAX_RECONNECTION_ATTEMPTS
self.last_subscribe_time = None
self.queue = Queue(maxsize=0)
if self.exchange == 'coinbase':
self.request = json.dumps(dict(type='subscribe', product_ids=[self.sym], channels=['full']))
self.request_unsubscribe = json.dumps(dict(type='unsubscribe', product_ids=[self.sym], channels=['full']))
self.book = CoinbaseOrderBook(self.sym)
self.trades_request = None
self.ws_endpoint = configs.COINBASE_ENDPOINT
elif self.exchange == 'bitfinex':
self.request = json.dumps({
"event": "subscribe",
"channel": "book",
"prec": "R0",
"freq": "F0",
"symbol": self.sym,
"len": "100"
})
self.request_unsubscribe = None
self.trades_request = json.dumps({
"event": "subscribe",
"channel": "trades",
"symbol": self.sym
})
self.book = BitfinexOrderBook(self.sym)
self.ws_endpoint = configs.BITFINEX_ENDPOINT
def main():
start_time = dt.now(TIMEZONE)
query = {
'ccy': ['BCH-USD', 'tBCHUSD'],
'start_date': 20181105,
'end_date': 20181106
}
coinbaseOrderBook = CoinbaseOrderBook(query['ccy'][0])
bitfinexOrderBook = BitfinexOrderBook(query['ccy'][1])
sim = Simulator()
tick_history = sim.get_tick_history(query)
orderbook_snapshot_history = sim.get_orderbook_snapshot_history(
coinbaseOrderBook, bitfinexOrderBook, tick_history)
# Export to CSV to verify if order book reconstruction is accurate/good
# NOTE: this is only to show that the functionality works and
# should be fed into an Environment for reinforcement learning.
sim.export_snapshots_to_csv(sim.get_feature_labels(),
orderbook_snapshot_history)
elapsed = (dt.now(TIMEZONE) - start_time).seconds
print('Completed %s in %i seconds' % (__name__, elapsed))
print('DONE. EXITING %s' % __name__)
def _get_env_states(self, query):
start_time = dt.now(TIMEZONE)
tick_history = self.get_tick_history(query)
coinbaseOrderBook = CoinbaseOrderBook(query['ccy'][0])
bitfinexOrderBook = BitfinexOrderBook(query['ccy'][1])
orderbook_snapshot_history = self.get_orderbook_snapshot_history(
coinbaseOrderBook, bitfinexOrderBook, tick_history)
elapsed = (dt.now(TIMEZONE) - start_time).seconds
print('Sim.get_env_states() executed in %i seconds' % elapsed)
return orderbook_snapshot_history
def __init__(self, ccy, exchange):
super(Client, self).__init__(name=ccy, daemon=False)
self.sym = ccy
self.exchange = exchange
self.ws = None
self.retry_counter = 0
self.max_retries = configs.MAX_RECONNECTION_ATTEMPTS
self.last_subscribe_time = None
self.queue = Queue(maxsize=0)
# print('\nClient __init__ - Process ID: %s | Thread: %s' % (str(os.getpid()), threading.current_thread().name))
if self.exchange == 'gdax':
self.request = json.dumps(
dict(type='subscribe',
product_ids=[self.sym],
channels=['full']))
self.request_unsubscribe = json.dumps(
dict(type='unsubscribe',
product_ids=[self.sym],
channels=['full']))
self.book = GdaxOrderBook(self.sym)
self.trades_request = None
self.ws_endpoint = configs.GDAX_ENDPOINT
elif self.exchange == 'bitfinex':
self.request = json.dumps({
"event": "subscribe",
"channel": "book",
"prec": "R0",
"freq": "F0",
"symbol": self.sym,
"len": "100"
})
self.request_unsubscribe = None
self.trades_request = json.dumps({
"event": "subscribe",
"channel": "trades",
"symbol": self.sym
})
self.book = BitfinexOrderBook(self.sym)
self.ws_endpoint = configs.BITFINEX_ENDPOINT
def test_get_orderbook_snapshot_history(query):
start_time = dt.now(TIMEZONE)
coinbaseOrderBook = CoinbaseOrderBook(query['ccy'][0])
bitfinexOrderBook = BitfinexOrderBook(query['ccy'][1])
sim = Simulator()
tick_history = sim.get_tick_history(query)
if tick_history is None:
print('Exiting due to no data being available.')
return
orderbook_snapshot_history = sim.get_orderbook_snapshot_history(coinbaseOrderBook,
bitfinexOrderBook,
tick_history)
# Export to CSV to verify if order book reconstruction is accurate/good
# NOTE: this is only to show that the functionality works and
# should be fed into an Environment for reinforcement learning.
sim.export_to_csv(data=orderbook_snapshot_history, filename='./orderbook_snapshot_history.csv')
elapsed = (dt.now(TIMEZONE) - start_time).seconds
print('Completed %s in %i seconds' % (__name__, elapsed))
print('DONE. EXITING %s' % __name__)
def get_orderbook_snapshot_history(self, query):
"""
Function to replay historical market data and generate
the features used for reinforcement learning & training.
NOTE:
The query can either be a single Coinbase CCY, or both Coinbase and Bitfinex,
but it cannot be only a Biftinex CCY. Later releases of this repo will
support Bitfinex only orderbook reconstruction.
:param query: (dict) query for finding tick history in Arctic TickStore
:return: (list of arrays) snapshots of limit order books using a stationary feature set
"""
tick_history = self.get_tick_history(query=query)
loop_length = tick_history.shape[0]
coinbase_tick_counter = 0
snapshot_list = list()
last_snapshot_time = None
symbols = query['ccy']
print('querying {}'.format(symbols))
include_bitfinex = len(symbols) > 1
if include_bitfinex:
print('\n\nIncluding Bitfinex data in feature set.\n\n')
coinbase_order_book = CoinbaseOrderBook(symbols[0])
bitfinex_order_book = BitfinexOrderBook(symbols[1]) if include_bitfinex else None
start_time = dt.now(TIMEZONE)
print('Starting get_orderbook_snapshot_history() loop with %i ticks for %s' %
(loop_length, query['ccy']))
for idx, tx in enumerate(tick_history.itertuples()):
tick = tx._asdict()
# determine if incoming tick is from coinbase or bitfinex
coinbase = True if tick['product_id'] == coinbase_order_book.sym else False
if 'type' not in tick:
# filter out bad ticks
continue
if tick['type'] in ['load_book', 'book_loaded', 'preload']:
# flag for a order book reset
if coinbase:
coinbase_order_book.new_tick(tick)
else:
bitfinex_order_book.new_tick(tick)
# skip to next loop
continue
if coinbase: # incoming tick is from Coinbase exchange
if coinbase_order_book.done_warming_up():
new_tick_time = parse(tick.get('time')) # timestamp for incoming tick
if new_tick_time is None:
print('No tick time: {}'.format(tick))
continue
coinbase_tick_counter += 1
coinbase_order_book.new_tick(tick)
if coinbase_tick_counter == 1:
# start tracking snapshot timestamps
# and keep in mind that snapshots are tethered to coinbase timestamps
last_snapshot_time = new_tick_time
print('%s first tick: %s | Sequence: %i' %
(coinbase_order_book.sym, str(new_tick_time), coinbase_order_book.sequence))
# skip to next loop
continue
# calculate the amount of time between the incoming tick and tick received before that
diff = (new_tick_time - last_snapshot_time).microseconds
# multiple = diff // 250000 # 250000 is 250 milliseconds, or 4x a second
multiple = diff // 500000 # 500000 is 500 milliseconds, or 2x a second
# if there is a pause in incoming data, continue to create order book snapshots
if multiple >= 1:
# check to include Bitfinex data in features
if include_bitfinex:
for _ in range(multiple):
if coinbase_order_book.done_warming_up() & bitfinex_order_book.done_warming_up():
coinbase_order_book_snapshot = coinbase_order_book.render_book()
bitfinex_order_book_snapshot = bitfinex_order_book.render_book()
midpoint_delta = coinbase_order_book.midpoint - bitfinex_order_book.midpoint
snapshot_list.append(list(np.hstack((new_tick_time, # tick time
coinbase_order_book.midpoint, # midpoint price
midpoint_delta, # price delta between exchanges
coinbase_order_book_snapshot,
bitfinex_order_book_snapshot)))) # longs/shorts
last_snapshot_time += timedelta(milliseconds=500) # 250)
else:
last_snapshot_time += timedelta(milliseconds=500) # 250)
else: # do not include bitfinex
for _ in range(multiple):
if coinbase_order_book.done_warming_up():
coinbase_order_book_snapshot = coinbase_order_book.render_book()
snapshot_list.append(list(np.hstack((new_tick_time, # tick time
coinbase_order_book.midpoint, # midpoint price
coinbase_order_book_snapshot)))) # longs/shorts
last_snapshot_time += timedelta(milliseconds=500) # 250)
else:
last_snapshot_time += timedelta(milliseconds=500) # 250)
# incoming tick is from Bitfinex exchange
elif include_bitfinex & bitfinex_order_book.done_warming_up():
bitfinex_order_book.new_tick(tick)
# periodically print number of steps completed
if idx % 250000 == 0:
elapsed = (dt.now(TIMEZONE) - start_time).seconds
print('...completed %i loops in %i seconds' % (idx, elapsed))
elapsed = (dt.now(TIMEZONE) - start_time).seconds
print('Completed run_simulation() with %i ticks in %i seconds at %i ticks/second' %
(loop_length, elapsed, loop_length//elapsed))
orderbook_snapshot_history = pd.DataFrame(snapshot_list,
columns=self.get_feature_labels(
include_system_time=True,
include_bitfinex=include_bitfinex))
orderbook_snapshot_history = orderbook_snapshot_history.dropna(axis=0)
return orderbook_snapshot_history
def get_orderbook_snapshot_history(self, query):
"""
Function to replay historical market data and generate
the features used for reinforcement learning & training
:param tick_history: (list of dicts) historical tick data
:return: (list of arrays) snapshots of limit order books using a stationary feature set
"""
tick_history = self.get_tick_history(query=query)
loop_length = len(tick_history)
coinbase_tick_counter = 0
snapshot_list = list()
last_snapshot_time = None
coinbase_order_book = CoinbaseOrderBook(query['ccy'][0])
bitfinex_order_book = BitfinexOrderBook(query['ccy'][1])
start_time = dt.now(TIMEZONE)
print('Starting get_orderbook_snapshot_history() loop with %i ticks for %s and %s' %
(loop_length, coinbase_order_book.sym, bitfinex_order_book.sym))
for idx, tick in enumerate(tick_history):
# determine if incoming tick is from coinbase or bitfinex
coinbase = True if tick['product_id'] == coinbase_order_book.sym else False
if 'type' not in tick:
# filter out bad ticks
continue
if tick['type'] in ['load_book', 'book_loaded', 'preload']:
# flag for a order book reset
if coinbase:
coinbase_order_book.new_tick(tick)
else:
bitfinex_order_book.new_tick(tick)
# skip to next loop
continue
if coinbase: # incoming tick is from Coinbase exchange
if coinbase_order_book.done_warming_up():
new_tick_time = parse(tick['time']) # timestamp for incoming tick
coinbase_tick_counter += 1
coinbase_order_book.new_tick(tick)
if coinbase_tick_counter == 1:
# start tracking snapshot timestamps
# and keep in mind that snapshots are tethered to coinbase timestamps
last_snapshot_time = new_tick_time
print('%s first tick: %s | Sequence: %i' %
(coinbase_order_book.sym, str(new_tick_time), coinbase_order_book.sequence))
# skip to next loop
continue
# calculate the amount of time between the incoming tick and tick received before that
diff = (new_tick_time - last_snapshot_time).microseconds
# multiple = diff // 250000 # 250000 is 250 milliseconds, or 4x a second
multiple = diff // 500000 # 500000 is 500 milliseconds, or 2x a second
if multiple >= 1: # if there is a pause in incoming data, continue to create order book snapshots
for _ in range(multiple):
if coinbase_order_book.done_warming_up() & bitfinex_order_book.done_warming_up():
coinbase_order_book_snapshot = coinbase_order_book.render_book()
bitfinex_order_book_snapshot = bitfinex_order_book.render_book()
midpoint_delta = coinbase_order_book.midpoint - bitfinex_order_book.midpoint
snapshot_list.append(list(np.hstack((new_tick_time, # tick time
coinbase_order_book.midpoint, # midpoint price
midpoint_delta, # price delta between exchanges
coinbase_order_book_snapshot,
bitfinex_order_book_snapshot)))) # longs/shorts
last_snapshot_time += timedelta(milliseconds=500) # 250)
else:
last_snapshot_time += timedelta(milliseconds=500) # 250)
# incoming tick is from Bitfinex exchange
elif bitfinex_order_book.done_warming_up():
bitfinex_order_book.new_tick(tick)
# periodically print number of steps completed
if idx % 250000 == 0:
elapsed = (dt.now(TIMEZONE) - start_time).seconds
print('...completed %i loops in %i seconds' % (idx, elapsed))
elapsed = (dt.now(TIMEZONE) - start_time).seconds
print('Completed run_simulation() with %i ticks in %i seconds at %i ticks/second' %
(loop_length, elapsed, loop_length//elapsed))
orderbook_snapshot_history = pd.DataFrame(snapshot_list, columns=self.get_feature_labels())
orderbook_snapshot_history = orderbook_snapshot_history.dropna(axis=0)
return orderbook_snapshot_history