def import_csv(filename: str) -> pd.DataFrame:
"""
Import an historical tick file created from the export_to_csv() function.
:param filename: Full file path including filename
:return: (panda.DataFrame) historical limit order book data
"""
start_time = dt.now(tz=TIMEZONE)
if 'xz' in filename:
data = pd.read_csv(filepath_or_buffer=filename,
index_col=0,
compression='xz',
engine='c')
elif 'csv' in filename:
data = pd.read_csv(filepath_or_buffer=filename,
index_col=0,
engine='c')
else:
LOGGER.warn('Error: file must be a csv or xz')
data = None
elapsed = (dt.now(tz=TIMEZONE) - start_time).seconds
LOGGER.info('Imported %s from a csv in %i seconds' %
(filename[-25:], elapsed))
return data
def remove_order(self, msg: dict) -> None:
"""
Done messages result in the order being removed from map.
:param msg: incoming order message
"""
msg_order_id = msg.get('order_id', None)
if msg_order_id in self.order_map:
old_order = self.order_map[msg_order_id]
price = old_order.get('price', None)
if price in self.price_dict:
if msg.get('reason', None) == 'canceled':
self.price_dict[price].add_cancel(quantity=float(
msg.get('remaining_size')),
price=price)
self.price_dict[price].remove_quantity(
quantity=old_order['size'], price=price)
self.price_dict[price].remove_count()
if self.price_dict[price].count == 0:
self.remove_price(price)
elif RECORD_DATA:
LOGGER.info('%s remove_order: price not in price_map [%s]' %
(msg['product_id'], str(price)))
del self.order_map[msg_order_id]
def render_lob_feature_names(include_orderflow: bool = INCLUDE_ORDERFLOW) -> list:
"""
Get the column names for the LOB render features.
:param include_orderflow: if TRUE, order flow imbalance stats are included in set
:return: list containing features names
"""
feature_names = list()
feature_names.append('midpoint')
feature_names.append('spread')
feature_names.append('buys')
feature_names.append('sells')
feature_types = ['distance', 'notional']
if include_orderflow:
feature_types += ['cancel_notional', 'limit_notional', 'market_notional']
for side in ['bid', 'ask']:
for feature in feature_types:
for row in range(MAX_BOOK_ROWS):
feature_names.append("{}_{}_{}".format(side, feature, row))
LOGGER.info("render_feature_names() has {} features".format(len(feature_names)))
return feature_names
def match(self, msg: dict) -> None:
"""
Change volume of book.
:param msg: incoming order message
"""
msg_order_id = msg.get('maker_order_id', None)
if msg_order_id in self.order_map:
old_order = self.order_map[msg_order_id]
order = {
'order_id': msg_order_id,
'price': float(msg['price']),
'size': float(msg['size']),
'side': msg['side'],
'time': msg['time'],
'type': msg['type'],
'product_id': msg['product_id']
}
price = order['price']
if price in self.price_dict:
remove_size = order['size']
remaining_size = old_order['size'] - remove_size
order['size'] = remaining_size
self.order_map[old_order['order_id']] = order
old_order_price = old_order.get('price', None)
self.price_dict[price].add_market(quantity=remove_size,
price=old_order_price)
self.price_dict[price].remove_quantity(quantity=remove_size,
price=old_order_price)
else:
LOGGER.info('\nmatch: price not in tree already [%s]\n' % msg)
elif RECORD_DATA:
LOGGER.warn('\n%s match: order id cannot be found for %s\n' %
(self.sym, msg))
def extract_features(self, query: dict) -> None:
"""
Create and export limit order book data to csv. This function
exports multiple days of data and ensures each day starts and
ends exactly on time.
:param query: (dict) ccy=sym, daterange=(YYYYMMDD,YYYYMMDD)
:return: void
"""
start_time = dt.now(tz=TIMEZONE)
order_book_data = self.get_orderbook_snapshot_history(query=query)
if order_book_data is not None:
dates = order_book_data['system_time'].dt.date.unique()
LOGGER.info('dates: {}'.format(dates))
for date in dates[:]:
tmp = order_book_data.loc[
order_book_data['system_time'].dt.date == date]
self.export_to_csv(tmp,
filename='{}_{}'.format(
query['ccy'][0], date),
compress=True)
elapsed = (dt.now(tz=TIMEZONE) - start_time).seconds
LOGGER.info(
'***\nSimulator.extract_features() executed in %i seconds\n***' %
elapsed)
def _load_book(self, book):
"""
Load initial limit order book snapshot
:param book: order book snapshot
:return: void
"""
start_time = time()
self.db.new_tick({'type': 'load_book', 'product_id': self.sym})
for row in book[1]:
order = {
"order_id": int(row[0]),
"price": float(row[1]),
"size": float(abs(row[2])),
"side": 'sell' if float(row[2]) < float(0) else 'buy',
"product_id": self.sym,
"type": 'preload'
}
self.db.new_tick(order)
if order['side'] == 'buy':
self.bids.insert_order(order)
else:
self.asks.insert_order(order)
self.db.new_tick({'type': 'book_loaded', 'product_id': self.sym})
self.bids.warming_up = self.asks.warming_up = False
elapsed = time() - start_time
LOGGER.info('%s: book loaded..............in %f seconds\n' %
(self.sym, elapsed))
def update_metrics(self, price: float, step: int) -> None:
"""
Update specific position metrics per each order.
:param price: (float) current midpoint price
:param step: (int) current time step
:return: (void)
"""
self.metrics.steps_in_position = step - self.step
if self.is_filled:
if self.side == 'long':
unrealized_pnl = (price - self.average_execution_price) / \
self.average_execution_price
elif self.side == 'short':
unrealized_pnl = (self.average_execution_price - price) / \
self.average_execution_price
else:
unrealized_pnl = 0.0
LOGGER.warning('alert: unknown order.step() side %s' %
self.side)
if unrealized_pnl < self.metrics.drawdown_max:
self.metrics.drawdown_max = unrealized_pnl
if unrealized_pnl > self.metrics.upside_max:
self.metrics.upside_max = unrealized_pnl
def get_tick_history(self, query: dict) -> Union[pd.DataFrame, None]:
"""
Function to query the Arctic Tick Store and...
1. Return the specified historical data for a given set of securities
over a specified amount of time
2. Convert the data returned from the query from a panda to a list of dicts
and while doing so, allocate the work across all available CPU cores
:param query: (dict) of the query parameters
- ccy: list of symbols
- startDate: int YYYYMMDD start date
- endDate: int YYYYMMDD end date
:return: list of dicts, where each dict is a tick that was recorded
"""
start_time = dt.now(tz=self.tz)
assert self.recording is False, "RECORD_DATA must be set to FALSE to replay data"
cursor = self._query_arctic(**query)
if cursor is None:
LOGGER.info(
'\nNothing returned from Arctic for the query: %s\n...Exiting...'
% str(query))
return
elapsed = (dt.now(tz=self.tz) - start_time).seconds
LOGGER.info('***Completed get_tick_history() in %i seconds***' %
elapsed)
return cursor
def _add_market_order(self, order: MarketOrder) -> bool:
"""
Add a MARKET order.
:param order: (Order) New order to be used for updating existing order or
placing a new order
"""
if self.full_inventory:
LOGGER.debug(' %s inventory max' % order.side)
return False
# Create a hypothetical average execution price incorporating a fixed slippage
order.average_execution_price = order.price
order.executed = order.DEFAULT_SIZE
# Update position inventory attributes
self.cancel_limit_order() # remove any unfilled limit orders
self.positions.append(order) # execute and save the market order
self.total_exposure += order.average_execution_price
self.average_price = self.total_exposure / self.position_count
self.full_inventory = self.position_count >= self.max_position_count
self.total_trade_count += 1
# deduct transaction fees whenever an order gets filled
if self.transaction_fee:
self.realized_pnl -= MARKET_ORDER_FEE
# update statistics
self.statistics.market_orders += 1
LOGGER.debug(' %s @ %.2f | step %i' %
(order.side, order.average_execution_price, order.step))
return True
def _process_trades(self, msg):
"""
Internal method to process trade messages
:param msg: incoming tick
:return: False if a re-subscribe is required
"""
if len(msg) == 2:
# historical trades
return True
msg_type = msg[1]
side = 'upticks' if msg[2][2] > 0.0 else 'downticks'
if msg_type == 'hb':
LOGGER.info('Heartbeat for trades')
return True
elif msg_type == 'te':
trade = {
'price': float(msg[2][3]),
'size': float(msg[2][2]),
'side': side,
'type': msg_type,
"product_id": self.sym
}
self.db.new_tick(trade)
return self._process_trades_replay(msg=trade)
return True
def remove(self, netting_order: MarketOrder or LimitOrder) -> float:
"""
Remove position from inventory and return position PnL.
:param netting_order: order object used to net position
:return: (bool) TRUE if position removed successfully
"""
pnl = 0.
if self.position_count < 1:
LOGGER.info('Error. No {} positions to remove.'.format(self.side))
return pnl
order = self.positions.popleft()
# Calculate PnL
if self.side == 'long':
pnl = (netting_order.price / order.average_execution_price) - 1.
elif self.side == 'short':
pnl = (order.average_execution_price / netting_order.price) - 1.
# Add Profit and Loss to realized gains/losses
self.realized_pnl += pnl
# Update positions attributes
self.total_exposure -= order.average_execution_price
self.average_price = self.total_exposure / self.position_count if \
self.position_count > 0 else 0.
self.full_inventory = self.position_count >= self.max_position_count
LOGGER.debug(
'remove-> Netted {} position #{} with {} trade #{} PnL = {:.4f}'.
format(self.side, order.id, netting_order.side, netting_order.id,
pnl))
return pnl
def flatten_inventory(self, price: float) -> float:
"""
Flatten all positions held in inventory.
:param price: (float) current bid or ask price
:return: (float) PnL from flattening inventory
"""
LOGGER.debug('{} is flattening inventory of {}'.format(
self.side, self.position_count))
if self.position_count < 1:
return -ENCOURAGEMENT
pnl = 0.
# Need to reverse the side to reflect the correct direction of
# the flatten_order()
side = 'long' if self.side == 'short' else 'short'
while self.position_count > 0:
order = MarketOrder(ccy=None, side=side, price=price)
pnl += self.remove(netting_order=order)
self.total_trade_count += 1
# Deduct transaction fee based on order type
if self.transaction_fee:
pnl -= MARKET_ORDER_FEE
# Update statistics
self.statistics.market_orders += 1
return pnl
def run(self) -> None:
"""
Thread to override in Coinbase or Bitfinex or Bitmex implementation class.
:return:
"""
LOGGER.info("run() initiated on : {}".format(self.name))
self.last_worker_time = dt.now()
def load_book(self) -> None:
"""
Load initial limit order book snapshot.
"""
book = self._get_book()
start_time = time()
self.sequence = book['sequence']
now = dt.now(tz=TIMEZONE)
load_time = str(now)
self.db.new_tick({
'type': 'load_book',
'product_id': self.sym,
'sequence': self.sequence
})
for bid in book['bids']:
msg = {
'price': float(bid[0]),
'size': float(bid[1]),
'order_id': bid[2],
'side': 'buy',
'product_id': self.sym,
'type': 'preload',
'sequence': self.sequence,
'time': load_time,
}
self.db.new_tick(msg)
self.bids.insert_order(msg)
for ask in book['asks']:
msg = {
'price': float(ask[0]),
'size': float(ask[1]),
'order_id': ask[2],
'side': 'sell',
'product_id': self.sym,
'type': 'preload',
'sequence': self.sequence,
'time': load_time,
}
self.db.new_tick(msg)
self.asks.insert_order(msg)
self.db.new_tick({
'type': 'book_loaded',
'product_id': self.sym,
'sequence': self.sequence
})
del book
self.bids.warming_up = self.asks.warming_up = False
elapsed = time() - start_time
LOGGER.info('%s: book loaded................in %f seconds' %
(self.sym, elapsed))
def _process_book(self, msg):
"""
Internal method to process FULL BOOK market data
:param msg: incoming tick
:return: False if re-subscribe is required
"""
# check for a heartbeat
if msg[1] == 'hb':
# render_book('heart beat %s' % msg)
return True
# order book message (initial snapshot)
elif np.shape(msg[1])[0] > 3:
LOGGER.info('%s loading book...' % self.sym)
self.clear_book()
self._load_book(msg)
return True
else:
# else, the incoming message is a order update
order = {
"order_id": int(msg[1][0]),
"price": float(msg[1][1]),
"size": float(abs(msg[1][2])),
"side": 'sell' if float(msg[1][2]) < float(0) else 'buy',
"product_id": self.sym,
"type": 'update'
}
self.db.new_tick(order)
# order should be removed from the book
if order['price'] == 0.:
if order['side'] == 'buy':
self.bids.remove_order(order)
elif order['side'] == 'sell':
self.asks.remove_order(order)
# order is a new order or size update for bids
elif order['side'] == 'buy':
if order['order_id'] in self.bids.order_map:
self.bids.change(order)
else:
self.bids.insert_order(order)
# order is a new order or size update for asks
elif order['side'] == 'sell':
if order['order_id'] in self.asks.order_map:
self.asks.change(order)
else:
self.asks.insert_order(order)
# unhandled msg
else:
LOGGER.warn('\nUnhandled list msg %s' % msg)
return True
def clear_book(self) -> None:
"""
Method to reset the limit order book.
:return: (void)
"""
self.bids.clear() # warming_up flag reset in `Position` class
self.asks.clear() # warming_up flag reset in `Position` class
self.last_tick_time = None
LOGGER.info("{}'s order book cleared.".format(self.sym))
def step_limit_order_pnl(self, bid_price: float, ask_price: float,
buy_volume: float, sell_volume: float,
step: int) -> (float, bool, bool):
"""
Update PnL & positions every time step in the environment.
:param bid_price: (float) current time step bid price
:param ask_price: (float) current time step ask price
:param buy_volume: (float) current time step buy volume
:param sell_volume: (float) current time step sell volume
:param step: (int) current time step number
:return: (float) PnL for current time step due to limit order fill and netting
"""
pnl = 0.
is_long_order_filled = self.long_inventory.step(
bid_price=bid_price,
ask_price=ask_price,
buy_volume=buy_volume,
sell_volume=sell_volume,
step=step)
is_short_order_filled = self.short_inventory.step(
bid_price=bid_price,
ask_price=ask_price,
buy_volume=buy_volume,
sell_volume=sell_volume,
step=step)
if is_long_order_filled and is_short_order_filled:
# protection in case Long and Short orders get filled in the same time step.
# Although this shouldn't happen, it prevents an error from occurring if it
# does happen.
LOGGER.info(
"WARNING: Long and Short orders filled in the same step")
LOGGER.info(
'bid={} | ask={} | buy_vol={} | sell_vol={} | step={}'.format(
bid_price, ask_price, buy_volume, sell_volume, step))
is_short_order_filled = False
if is_long_order_filled:
# check if we can net the inventory
if self.short_inventory_count > 0:
# net out the inventory
new_position = self.long_inventory.pop_position()
pnl += self.short_inventory.remove(netting_order=new_position)
if is_short_order_filled:
# check if we can net the inventory
if self.long_inventory_count > 0:
# net out the inventory
new_position = self.short_inventory.pop_position()
pnl += self.long_inventory.remove(netting_order=new_position)
return pnl, is_long_order_filled, is_short_order_filled
def create_model(self, name: str = 'cnn') -> Sequential:
"""
Helper function get create and get the default MLP or CNN model.
:param name: Neural network type ['mlp' or 'cnn']
:return: neural network
"""
LOGGER.info("creating model for {}".format(name))
if name == 'cnn':
return self._create_cnn_model()
elif name == 'mlp':
return self._create_mlp_model()
def _process_book_replay(self, order):
"""
Internal method to process FULL BOOK market data
:param order: incoming tick
:return: False if resubscription in required
"""
# clean up the datatypes
order['price'] = float(order['price'])
order['size'] = float(order['size'])
if order['type'] == 'update':
# order should be removed from the book
if order['price'] == float(0):
if order['side'] == 'buy':
self.bids.remove_order(order)
elif order['side'] == 'sell':
self.asks.remove_order(order)
# order is a new order or size update for bids
elif order['side'] == 'buy':
if order['order_id'] in self.bids.order_map:
self.bids.change(order)
else:
self.bids.insert_order(order)
# order is a new order or size update for asks
elif order['side'] == 'sell':
if order['order_id'] in self.asks.order_map:
self.asks.change(order)
else:
self.asks.insert_order(order)
# unhandled tick message
else:
LOGGER.warn('_process_book_replay: unhandled message\n%s' % str(order))
elif order['type'] == 'preload':
if order['side'] == 'buy':
self.bids.insert_order(order)
else:
self.asks.insert_order(order)
elif order['type'] == 'te':
trade_notional = order['price'] * order['size']
if order['side'] == 'upticks':
self.buy_tracker.add(notional=trade_notional)
self.asks.match(order)
else:
self.sell_tracker.add(notional=trade_notional)
self.bids.match(order)
else:
LOGGER.warn('\n_process_book_replay() Unhandled list msg %s' % order)
return True
def flatten_inventory(self, bid_price: float, ask_price: float) -> float:
"""
Flatten all positions held in inventory.
:param bid_price: (float) current bid price
:param ask_price: (float) current ask price
:return: (float) PnL from flattening inventory
"""
LOGGER.debug('Flattening inventory. {} longs / {} shorts'.format(
self.long_inventory_count, self.short_inventory_count))
long_pnl = self.long_inventory.flatten_inventory(price=bid_price)
short_pnl = self.short_inventory.flatten_inventory(price=ask_price)
return long_pnl + short_pnl
def cancel_limit_order(self) -> bool:
"""
Cancel a limit order.
:return: (bool) TRUE if cancel was successful
"""
if self.order is None:
LOGGER.debug('No {} open orders to cancel.'.format(self.side))
return False
LOGGER.debug('Cancelling order ({})'.format(self.order))
self.order = None
return True
def init_db_connection(self) -> None:
"""
Initiate database connection to Arctic.
:return: (void)
"""
LOGGER.info("init_db_connection for {}...".format(self.sym))
try:
self.db = Arctic(MONGO_ENDPOINT)
self.db.initialize_library(ARCTIC_NAME, lib_type=TICK_STORE)
self.collection = self.db[ARCTIC_NAME]
except PyMongoError as e:
LOGGER.warn("Database.PyMongoError() --> {}".format(e))
def _add_limit_order(self, order: LimitOrder) -> bool:
"""
Add / update a LIMIT order.
:param order: (Order) New order to be used for updating existing order or
placing a new order
"""
if self.order is None:
if self.full_inventory:
LOGGER.debug(
"{} order rejected. Already at max position limit ({})".
format(self.side, self.max_position_count))
return False
self.order = order
# update statistics
self.statistics.orders_placed += 1
LOGGER.debug('\nOpened new order={}'.format(order))
elif self.order.price != order.price:
self.order.price = order.price
self.order.queue_ahead = order.queue_ahead
self.order.id = order.id
self.order.step = order.step
# update statistics
self.statistics.orders_updated += 1
LOGGER.debug('\nUpdating order{} --> \n{}'.format(
order, self.order))
else:
LOGGER.debug("\nNothing to update about the order {}".format(
self.order))
return True
def _create_mlp_model(self) -> Sequential:
"""
Create a DENSE neural network with dense layer at the end
:return: keras model
"""
features_shape = (self.memory_frame_stack,
*self.env.observation_space.shape)
model = Sequential()
model.add(
Dense(units=256, input_shape=features_shape, activation='relu'))
model.add(Dense(units=256, activation='relu'))
model.add(Flatten())
model.add(Dense(self.env.action_space.n, activation='softmax'))
LOGGER.info(model.summary())
return model
def _get_book(self) -> dict:
"""
Get order book snapshot.
:return: order book
"""
LOGGER.info('%s get_book request made.' % self.sym)
start_time = time()
self.clear_book()
path = (COINBASE_BOOK_ENDPOINT % self.sym)
book = requests.get(path, params={'level': 3}).json()
elapsed = time() - start_time
LOGGER.info('%s get_book request completed in %f seconds.' % (self.sym, elapsed))
return book
def _step_limit_order(self, bid_price: float, ask_price: float,
buy_volume: float, sell_volume: float,
step: int) -> bool:
"""
Step in environment and update LIMIT order inventories.
:param bid_price: best bid price
:param ask_price: best ask price
:param buy_volume: executions initiated by buyers (in notional terms)
:param sell_volume: executions initiated by sellers (in notional terms)
:param step: current time step
:return: (bool) TRUE if a limit order was filled, otherwise FALSE
"""
if self.order is None:
return False
if self.order.side == 'long':
if bid_price <= self.order.price:
self._process_transaction_volume(volume=sell_volume)
elif self.order.side == 'short':
if ask_price >= self.order.price:
self._process_transaction_volume(volume=buy_volume)
if self.order.is_filled:
avg_execution_px = self.order.get_average_execution_price()
self.positions.append(self.order)
self.total_exposure += avg_execution_px
self.average_price = self.total_exposure / self.position_count
self.full_inventory = self.position_count >= self.max_position_count
self.total_trade_count += 1
LOGGER.debug(
'FILLED {} order #{} at {:.3f} after {} steps on {}.'.format(
self.order.side, self.order.id, avg_execution_px,
self.order.metrics.steps_in_position, step))
self.order = None # set the slot back to no open orders
self.statistics.orders_executed += 1
# deduct transaction fees when the LIMIT order gets filled
if self.transaction_fee:
self.realized_pnl -= LIMIT_ORDER_FEE
return True
return False
def reset_ema(ema) -> None:
"""
Reset EMA manager.
:param ema: EMA manager to be reset
:return: EMA manager that has been reset
"""
if ema is None:
pass
elif isinstance(ema, ExponentialMovingAverage):
ema.reset()
LOGGER.info("Reset EMA data.")
elif isinstance(ema, list):
for e in ema:
e.reset()
LOGGER.info("Reset EMA data.")
return ema
def __init__(self,
sym: str,
exchange: str,
record_data: bool = RECORD_DATA):
"""
Database constructor.
"""
self.counter = 0
self.data = list()
self.tz = TIMEZONE
self.sym = sym
self.exchange = exchange
self.recording = record_data
self.db = self.collection = None
if self.recording:
LOGGER.info('\nDatabase: [%s is recording %s]\n' %
(self.exchange, self.sym))
def run(self) -> None:
"""
Thread to override in Coinbase or Bitfinex or Bitmex implementation class.
"""
LOGGER.info("run() initiated on : {}".format(self.name))
self.last_worker_time = dt.now()
# Used for debugging exchanges individually
# Timer(4.0, _timer_worker, args=(self.book, self.last_worker_time,)).start()
# from data_recorder.connector_components.orderbook import OrderBook
# Used for debugging exchanges individually
# def _timer_worker(orderbook: OrderBook, last_worker_time: dt) -> None:
# """
# Thread worker to be invoked every N seconds
# (e.g., configs.SNAPSHOT_RATE)
#
# :param orderbook: OrderBook
# :return: void
# """
# now = dt.now()
# delta = now - last_worker_time
# print('\n{} - {} with delta {}\n{}'.format(orderbook.sym, now, delta.microseconds,
# orderbook))
# last_worker_time = now
#
# Timer(SNAPSHOT_RATE, _timer_worker, args=(orderbook, last_worker_time,)).start()
#
# if orderbook.done_warming_up:
# """
# This is the place to insert a trading model.
# You'll have to create your own.
#
# Example:
# orderbook_data = tuple(coinbaseClient.book, bitfinexClient.book)
# model = agent.dqn.Agent()
# fix_api = SomeFixAPI()
# action = model(orderbook_data)
# if action is buy:
# buy_order = create_order(pair, price, etc.)
# fix_api.send_order(buy_order)
#
# """
# _ = orderbook.render_book()
def run(self):
"""
Handle incoming level 3 data on a separate thread or process.
Returns
-------
"""
super(CoinbaseClient, self).run()
while True:
msg = self.queue.get()
if self.book.new_tick(msg) is False:
self.book.load_book()
self.retry_counter += 1
LOGGER.info('\n[%s - %s] ...going to try and reload the order '
'book\n' % (self.exchange.upper(), self.sym))
continue