def _fill_order(self, order: OrderCommon, qty_to_fill, price_fill, fee=0.):
assert price_fill >= 0, "Price has to be positive"
assert abs(order.leaves_qty
) >= 1, "Order with leaves_qty < 1 should be closed"
assert abs(order.signed_qty
) >= 1, "Order signed_qty should not be less than 1"
assert abs(qty_to_fill) >= 1, "Can not fill order with qty less than 1"
fully_filled = order.fill(qty_to_fill)
side = order.side()
fill = Fill(
symbol=order.symbol,
side=order.side_str(),
qty_filled=qty_to_fill,
price_fill=price_fill,
fill_time=self.current_timestamp,
fill_type=FillType.complete if fully_filled else FillType.partial,
order_id=order.client_id)
self._log_fill(fill)
self.volume[order.symbol] += abs(qty_to_fill) * price_fill
self.n_fills[order.symbol] += 1
position = self.positions[order.symbol]
position.update(signed_qty=qty_to_fill * side,
price=price_fill,
leverage=self.leverage[order.symbol],
current_timestamp=self.current_timestamp,
fee=fee)
method = self._get_tactic(order.client_id).handle_fill
self._queue_append(self.current_timestamp + WS_DELAY, method, fill)
return fully_filled
def _exec_market_order(self, order: OrderCommon):
assert order.type == OrderType.Market
assert order.status == OrderStatus.Pending
order.status = OrderStatus.New
tick_size = order.symbol.tick_size
# TODO: watch for self-cross
# We only fill at two price levels: best price and second best price
if order.side() > 0:
best_price = self.current_quote.ask_price
second_best = best_price + tick_size
size = self.current_quote.ask_size
else:
best_price = self.current_quote.bid_price
second_best = best_price - tick_size
size = self.current_quote.bid_size
qty_fill1 = min(size, order.leaves_qty)
qty_fill2 = order.leaves_qty - qty_fill1
fully_filled = self._fill_order(order=order,
qty_to_fill=qty_fill1,
price_fill=best_price)
if not fully_filled:
assert qty_fill2 > 0
self._fill_order(order=order,
qty_to_fill=qty_fill2,
price_fill=second_best)
return
def _create_orders(self, learning: float, fcst: float, quote: Quote):
# TODO: improve to use full book information
buy_best = quote.bid_price if quote.bid_size >= self.min_liq else quote.bid_price - 0.5
sell_best = quote.ask_price if quote.ask_size >= self.min_liq else quote.ask_price + 0.5
buy_best -= self.spread
sell_best += self.spread
price = fcst - learning / self.alpha
buy = min(buy_best - 0.5, self._floor(price - 0.25))
sell = max(sell_best + 0.5, self._ceil(price + 0.25))
qty_buy = min(self.max_qty, round((price - buy) * self.alpha))
qty_sel = min(self.max_qty, round((sell - price) * self.alpha))
if qty_buy >= 1:
self.buy_order = OrderCommon(symbol=self.symbol,
type=OrderType.Limit,
client_id=self.gen_order_id(),
signed_qty=qty_buy,
price=buy)
if qty_sel >= 1:
self.sell_order = OrderCommon(symbol=self.symbol,
type=OrderType.Limit,
client_id=self.gen_order_id(),
signed_qty=-qty_sel,
price=sell)
def on_order(self, raw: dict, action: str):
# DEV NOTE: this is the only method who can delete orders from self.all_orders
# DEV NOTE: this is the only method who broadcast orders to tactics
order_id = raw.get('clOrdID')
if not order_id:
self.logger.error(
'Got an order without "clOrdID", probably a bitmex order (e.g., liquidation)'
)
self.logger.error('order: {}'.format(raw))
raise AttributeError(
'If we were liquidated, we should change our tactics')
if OrderStatus[raw['ordStatus']] == OrderStatus.Rejected:
raise ValueError(
'A order should never be rejected, please fix the tactic. Order: {}'
.format(raw))
order = self.all_orders.get(order_id)
if order:
if action != 'update':
raise AttributeError(
'Got action "{}". This should be an "update", since this order already '
'exist in our data and we don\'t expect a "delete"'.format(
action))
if not order.is_open():
# This is the case when a closed (e.g. Canceled) order where inserted, now we clean it up in the insert
# We also assume that bitmex will not update a canceled/closed order twice
del self.all_orders[order.client_id]
return
order.update_from_bitmex(raw) # type: OrderCommon
if not order.is_open():
del self.all_orders[order.client_id]
self._notify_cancel_if_the_case(order)
else:
symbol = Symbol[raw['symbol']]
type_ = OrderType[raw['ordType']]
order = OrderCommon(symbol=symbol,
type=type_).update_from_bitmex(raw)
if action != 'insert' and order.is_open():
raise AttributeError(
'Got action "{}". This should be an "insert", since this data does\'nt exist '
'in memory and is open. Order: {}'.format(action, raw))
# yes, always add an order to the list, even if it is closed. If so, it will be removed in the "update" feed
self.all_orders[order_id] = order
self._notify_cancel_if_the_case(order)
def _liq_pos(self, pos: float):
self.exchange.send_orders([
OrderCommon(symbol=self.symbol,
type=OrderType.Market,
client_id=self.gen_order_id(),
signed_qty=-pos)
])
def send_orders(self, orders: List[OrderCommon]):
# DEV NOTE: it logs all sent orders, good or bad orders
# DEV NOTE: it should NOT change any internal state
# sanity check
if not all([o.status == OrderStatus.Pending for o in orders]):
raise ValueError(
"New orders should have status OrderStatus.Pending")
for o in orders:
if o.client_id:
tokens = o.client_id.split('_')
tactic_id = tokens[0]
if not o.client_id or len(
tokens) < 2 or tactic_id not in self.tactics_map:
raise ValueError(
'The order client_id should be a string in the form TACTICID_HASH'
)
converted = [
self.convert_order_to_bitmex(self.check_order_sanity(order))
for order in orders
]
raws = self._curl_bitmex(path='order/bulk',
postdict={'orders': converted},
verb='POST')
for r in raws:
symbol = Symbol[r['symbol']]
type_ = OrderType[r['ordType']]
tactic_id = r['clOrdID'].split('_')[0]
tactic = self.tactics_map[tactic_id]
self._log_order(OrderCommon(symbol, type_).update_from_bitmex(r))
def _init_files(self, log_dir):
self.fills_file = open(os.path.join(log_dir, 'fills.csv'), 'w')
self.orders_file = open(os.path.join(log_dir, 'orders.csv'), 'w')
self.pnl_file = open(os.path.join(log_dir, 'pnl.csv'), 'w')
self.fills_file.write(Fill.get_header() + '\n')
self.orders_file.write(OrderCommon.get_header() + '\n')
self.pnl_file.write('timestamp,symbol,pnl,cum_pnl\n')
def _create_orders(self):
quote = self.exchange.get_quote(self.symbol)
bid = quote.bid_price
ask = quote.ask_price
# sending two orders at same time to check if we get two different fills
self.orders_to_send = [
OrderCommon(symbol=self.symbol,
type=OrderType.Limit,
price=bid - 0.5,
client_id=self.gen_order_id(),
signed_qty=+self.qty),
OrderCommon(symbol=self.symbol,
type=OrderType.Limit,
price=ask + 0.5,
client_id=self.gen_order_id(),
signed_qty=-self.qty)
]
def _create_orders(self):
self.orders_to_send = []
self.expected_position = []
# step 0
self.orders_to_send.append([
OrderCommon(symbol=self.symbol,
type=OrderType.Market,
client_id=self.gen_my_id('a'),
signed_qty=-self.qty)
])
self.expected_position.append(-self.qty)
# step 1
# sending two orders at same time to check if we get two different fills
self.orders_to_send.append([
OrderCommon(symbol=self.symbol,
type=OrderType.Market,
client_id=self.gen_my_id('b'),
signed_qty=-self.qty),
OrderCommon(symbol=self.symbol,
type=OrderType.Market,
client_id=self.gen_my_id('c'),
signed_qty=-self.qty)
])
self.expected_position.append(-3 * self.qty)
# step 2
self.orders_to_send.append([
OrderCommon(symbol=self.symbol,
type=OrderType.Market,
client_id=self.gen_my_id('d'),
signed_qty=+4 * self.qty)
])
self.expected_position.append(+self.qty)
# step 3
self.orders_to_send.append([
OrderCommon(symbol=self.symbol,
type=OrderType.Market,
client_id=self.gen_my_id('e'),
signed_qty=-self.qty)
])
self.expected_position.append(0)
def _exec_limit_order(self, order: OrderCommon):
assert order.type == OrderType.Limit
bid = self.current_quote.bid_price
ask = self.current_quote.ask_price
order_side = order.side()
self._check_price_sanity(order)
violated_post_only = order_side > 0 and order.price >= ask or order_side < 0 and order.price <= bid
if violated_post_only:
order.status = OrderStatus.Canceled
order.status_msg = OrderCancelReason.cross_during_post_only
self._exec_order_cancels([order])
return
order.status = OrderStatus.New
self.active_orders[order.client_id] = order
order._made_spread = order_side > 0 and order.price > bid or order_side < 0 and order.price < ask
def close_position(self, symbol: Symbol):
# TODO: it should also close limit orders on the same side as the position
pos = self.positions[symbol]
oid = self.liquidator_tactic.gen_order_id().split('_')
oid = oid[0] + '_closing_' + oid[1]
if pos.is_open:
self.send_orders([
OrderCommon(symbol=symbol,
type=OrderType.Market,
client_id=oid,
signed_qty=-pos.signed_qty)
])
def _execute_liquidation(self, symbol, order_cancel_reason=OrderCancelReason.liquidation):
self.can_call_handles = False
orders = filter_symbol(self.active_orders, symbol)
cancelled = self._cancel_orders_helper(orders, reason=order_cancel_reason)
self.active_orders = drop_orders(self.active_orders, cancelled)
self.can_call_handles = True
try:
assert len(filter_symbol(self.active_orders, symbol)) == 0
except:
for i in filter_symbol(self.active_orders, symbol):
print("-" + str(i[1].status))
print("-" + str(len(cancelled)))
raise AttributeError()
position = self.positions.get(symbol, None)
if not position or not position.is_open:
return
assert position.is_open
order = OrderCommon(symbol=symbol,
signed_qty=-position.signed_qty,
type=OrderType.Market,
tactic=self.liquidator)
order.status_msg = order_cancel_reason
self.can_call_handles = False
self.send_orders([order])
self.can_call_handles = True
assert order.status == OrderStatus.Filled
if position.is_open:
raise AttributeError("position was not close during liquidation. position = %f" % position.signed_qty)
if not self.is_last_tick():
self.n_liquidations[symbol.name] += 1
if order_cancel_reason == OrderCancelReason.liquidation:
closed = self.closed_positions_hist[symbol][-1] # type: PositionSim
if closed.realized_pnl >= 0:
raise AttributeError("Liquidation caused profit! position = {},\n current price = {}"
.format(str(position), self._estimate_price()))
assert len(filter_symbol(self.active_orders, symbol)) == 0
def handle_1m_candles(self, candles1m: pd.DataFrame) -> None:
if not self.sell_id:
opened_orders = self.exchange.get_opened_orders(self.symbol, self.id())
if len(opened_orders) != 0:
raise ValueError('This test has to start with no opened orders')
# wait for quotes
time.sleep(0.1)
price = self.exchange.get_quote(self.symbol).ask_price
if not price:
raise AttributeError("We should have quote by now")
orders = [
# First order should be accepted
OrderCommon(symbol=self.symbol,
type=OrderType.Limit,
client_id=self.gen_order_id(),
signed_qty=-self.qty,
price=price + 1000),
# First order should be cancelled because we will try to sell bellow the bid
# Note that our Limit orders should be post-only, i.e., ParticipateDoNotInitiate
OrderCommon(symbol=self.symbol,
type=OrderType.Limit,
client_id=self.gen_order_id(),
signed_qty=-self.qty,
price=max(price - 500, 1)),
]
self.sell_id = [o.client_id for o in orders]
self.exchange.send_orders(orders)
pass
def get_opened_orders(self, symbol: Symbol,
client_id_prefix: str) -> OrderContainerType:
raws = self._curl_bitmex(path='order',
query={
'symbol': symbol.name,
'filter': '{"open": true}'
},
verb='GET')
def is_owner(client_id: str):
return client_id.split('_')[0] == client_id_prefix
return {
r['clOrdID']:
OrderCommon(symbol, OrderType[r['ordType']]).update_from_bitmex(r)
for r in raws if r['leavesQty'] != 0 and is_owner(r['clOrdID'])
}
def handle_1m_candles(self, candles1m: pd.DataFrame) -> None:
if not self.candle_last_ts:
self.candle_last_ts = candles1m.index[-1]
elif not isinstance(candles1m, str):
assert self.candle_last_ts < candles1m.index[-1]
if self.n_trades > 0 and not self.buy_id[0] and self.n_positions > 0:
logger.info("opening a position")
self.initial_pos = self.exchange.get_position(
self.symbol).signed_qty
self.buy_id[0] = self.gen_order_id()
self.next_action = 1
logger.info("sending buy order {}".format(self.buy_id[0]))
self.exchange.send_orders([
OrderCommon(symbol=self.symbol,
type=OrderType.Market,
client_id=self.buy_id[0],
signed_qty=+self.qty)
])
pass
def print_output_files(self, input_args):
if self.log_dir is None:
raise ValueError("asked to print results, but log_dir is None")
print("printing results to " + self.log_dir)
fills_file = open(os.path.join(self.log_dir, 'fills.csv'), 'w')
orders_file = open(os.path.join(self.log_dir, 'orders.csv'), 'w')
pnl_file = open(os.path.join(self.log_dir, 'pnl.csv'), 'w')
pars_used_file = open(os.path.join(self.log_dir, 'parameters_used'), 'w')
summary_file = open(os.path.join(self.log_dir, 'summary'), 'w')
fills_file.write(Fill.get_header() + '\n')
orders_file.write(OrderCommon.get_header() + '\n')
for f in self.fills_hist: # type: Fill
fills_file.write(f.to_line() + '\n')
for o in self.order_hist: # type: OrderCommon
orders_file.write(o.to_line() + '\n')
pnl_file.write('time,symbol,pnl,cum_pnl\n')
for s in self.SYMBOLS:
sum = 0
for p in self.closed_positions_hist[s]: # type: PositionSim
sum += p.realized_pnl
pnl_file.write(','.join([str(p.current_timestamp.strftime('%Y-%m-%dT%H:%M:%S')),
s.name,
str(p.realized_pnl),
str(sum)])
+ '\n')
pars_used_file.write(str(input_args))
pars_used_file.write("")
summary_file.write(self.summary.to_str())
pars_used_file.close()
pnl_file.close()
fills_file.close()
orders_file.close()
summary_file.close()
def _exec_liquidation(self,
symbol: Symbol,
reason=OrderCancelReason.liquidation):
orders_to_cancel = [
o for o in self.active_orders.values() if o.symbol == symbol
]
for order in orders_to_cancel:
order.status_msg = reason
self._exec_order_cancels(orders_to_cancel)
if reason == OrderCancelReason.liquidation:
self.n_liquidations[symbol] += 1
if self.positions[symbol].is_open:
order = OrderCommon(
symbol=symbol,
type=OrderType.Market,
client_id=self.liquidator_tactic.gen_order_id(),
signed_qty=-self.positions[symbol].signed_qty)
self._exec_market_order(order)
return
def handle_fill(self, fill: Fill) -> None:
if fill.side.lower()[0] == 'b':
i = self.buy_id_to_index(fill.order_id)
if i == -1:
raise AttributeError("We didn't send order with id {}".format(
fill.order_id))
self.buy_leaves[i] -= fill.qty
else:
if fill.side.lower()[0] != 's':
raise AttributeError(
'side should be buy or sell, got {}'.format(fill.side))
i = self.sell_id_to_index(fill.order_id)
if i == -1:
raise AttributeError("We didn't send order with id {}".format(
fill.order_id))
self.sell_leaves[i] -= fill.qty
if fill.fill_type == FillType.complete:
# we need this little delays because it seems that bitmex takes a while to update the position
time.sleep(.3)
pos = self.exchange.get_position(self.symbol)
assertEqual(
pos.signed_qty, self.expected_position(),
"n_buys={}, n_sells={}, init_pos={}".format(
min(self.next_action, self.n_trades),
max(self.next_action - self.n_trades, 0),
self.initial_pos))
if self.next_action < self.n_trades:
client_id = self.gen_order_id()
self.buy_id[self.next_action] = client_id
logger.info("sending buy order {}".format(
self.buy_id[self.next_action]))
self.next_action += 1
self.exchange.send_orders([
OrderCommon(symbol=self.symbol,
type=OrderType.Market,
client_id=client_id,
signed_qty=+self.qty)
])
elif self.n_trades <= self.next_action < 2 * self.n_trades:
client_id = self.gen_order_id()
self.sell_id[self.next_action - self.n_trades] = client_id
self.next_action += 1
logger.info("sending sell order {}".format(self.sell_id))
self.exchange.send_orders([
OrderCommon(symbol=self.symbol,
type=OrderType.Market,
client_id=client_id,
signed_qty=-self.qty)
])
else:
time.sleep(.3)
logger.info(
"checking position, it should be = initial position ...")
pos = self.exchange.get_position(self.symbol)
assertEqual(pos.signed_qty, self.initial_pos)
self.n_closed_positions += 1
pnls = self.exchange.get_pnl_history(self.symbol)
if len(pnls) != self.n_closed_positions:
raise AttributeError(
"Expected to have {} closed position, got {}".format(
self.n_closed_positions, len(pnls)))
self.n_positions -= 1
if self.n_positions > 0:
self.__init__(self.n_trades, self.n_positions)
self.handle_1m_candles('skip_candle')
pass
def _execute_order(self, order: OrderCommon) -> Fill:
assert self.tick_num < len(self.ohlcv)
if order.status != OrderStatus.Pending and not order.is_open():
raise ValueError("expected order to be opened, but got " + str(order.status) + ". Order = \n"
+ order.get_header() + "\n" + str(order))
current_candle = self.current_candle()
current_time = current_candle.name # pd.Timestamp
high = current_candle.high
low = current_candle.low
open = current_candle.open
close = current_candle.close
position = self.positions[order.symbol] # type: PositionSim
current_qty = position.signed_qty
qty_fill = qty_to_close = outstanding_qty = None
crossed = False
if order.type is OrderType.Market:
crossed = True
price_fill = self._estimate_price()
qty_fill = order.signed_qty
elif order.type is OrderType.Limit:
price_fill = order.price
max_qty_fill = order.leaves_qty * order.side()
# clip fill
if (open <= order.price <= close) or (close <= order.price <= open):
qty_fill = max_qty_fill
elif high == low == order.price:
qty_fill = round(0.5 * max_qty_fill)
else:
if low < order.price < high:
if order.is_sell():
factor = max((high - order.price) / (high - low), 0.50001)
assert factor >= 0
else:
factor = max((low - order.price) / (low - high), 0.50001)
assert factor >= 0
qty_fill = round(factor * max_qty_fill)
if qty_fill is not None:
crossed = True
else:
raise ValueError("order type " + str(order.type) + " not supported")
if not crossed:
return None # type: Fill
if position.is_open and position.would_change_side(qty_fill):
qty_to_close = float(sign(qty_fill)) * min(abs(current_qty), abs(qty_fill))
outstanding_qty = qty_fill - qty_to_close
if order.fill(qty_fill) or order.type == OrderType.Market:
order.status = OrderStatus.Filled
order.fill_price = price_fill
if order.price is not None and ((open <= order.price <= close) or (close <= order.price <= open)):
assert order.status == OrderStatus.Filled
fee = self.FEE[order.type]
if outstanding_qty:
position = self._update_position(order.symbol,
qty=qty_to_close,
price=price_fill,
leverage=self.leverage[order.symbol],
current_timestamp=current_time,
fee=fee)
assert not position.is_open
position = self._update_position(order.symbol,
qty=outstanding_qty,
price=price_fill,
leverage=self.leverage[order.symbol],
current_timestamp=current_time,
fee=fee)
assert position.is_open
else:
self._update_position(order.symbol,
qty=qty_fill,
price=price_fill,
leverage=self.leverage[order.symbol],
current_timestamp=current_time,
fee=fee)
fill = Fill(order=order,
qty_filled=qty_fill,
price_fill=price_fill,
fill_time=current_time,
fill_type=FillType.complete if (order.status == OrderStatus.Filled) else FillType.partial)
self.fills_hist += [fill]
self.active_orders = drop_closed_orders_dict(self.active_orders)
if self.can_call_handles:
order.tactic.handle_fill(fill)
return fill
def _log_order(self, order: OrderCommon):
self.orders_file.write(order.to_line() + '\n')