def __init__(self, strategy_id, order_sockets):
"""order_sockets maps venue ids to zmq DEALER sockets"""
logger.info("Initializing OrderManager")
self.orders = {}
self.live_order_ids = set([])
# use the strategy id when constructing protobuffers
self.strategy_id = strategy_id
self.order_sockets = order_sockets
self.positions = {}
self.pending = OneToManyDict()
import sys
import one_to_many
from one_to_many import OneToManyDict
m = OneToManyDict()
def create_map():
m.add("a", 1)
m.add("a", 2)
m.add("a", 3)
m.add("b", 4)
m.add("c", 5)
print m.dbg_string()
if __name__ == "__main__":
create_map()
print "remove_value(2)"
m.remove_value(2)
print m.dbg_string()
print "remove_key(b)"
m.remove_key("b")
print m.dbg_string()
print "remove_key(c)"
m.remove_key("c")
print m.dbg_string()
class OrderManager:
def __init__(self, strategy_id, order_sockets):
"""order_sockets maps venue ids to zmq DEALER sockets"""
logger.info("Initializing OrderManager")
self.orders = {}
self.live_order_ids = set([])
# use the strategy id when constructing protobuffers
self.strategy_id = strategy_id
self.order_sockets = order_sockets
self.positions = {}
self.pending = OneToManyDict()
def DBG_ORDER_MAP(self):
logger.debug("******************** <ORDER MAP> *********************")
logger.debug("ALIVE:\n")
for o1 in self.live_order_ids:
logger.debug("\t%s = %s", o1, self.get_order(o1))
logger.debug("ORDER:\n")
for o2 in self.orders:
logger.debug("\t%s = %s", o2, self.get_order(o2))
logger.debug("PENDING:\n")
logger.debug(self.pending.dbg_string())
logger.debug("******************** </ORDER MAP> *********************")
# KTK should use setdefault? E.g. orders.setdefault(order_did, None)
def get_order(self, order_id):
assert order_id in self.orders, "Couldn't find order id %s" % str(order_id)
return self.orders[order_id]
# def pending_id_accepted(self, order_id, pending_id):
# logger.info('pending id accepted %s %s', order_id, pending_id)
# assert self.pending.has_value(pending_id), \
# 'Unexpected pending ID %s for order %s' % (pending_id,
# order_id)
# self.pending.remove_value(pending_id)
# self.get_order(order_id).id = pending_id
def pending_id_rejected(self, order_id, pending_id):
logger.info("pending id REJECTED order_id=%s pending_id=%s", order_id, pending_id)
assert self.pending.has_value(pending_id), "Unexpected pending ID %s for order %s" % (pending_id, order_id)
self.pending.remove_value(pending_id)
def is_pending(self, pending_id):
return self.pending.has_value(pending_id)
def is_alive(self, order_id):
return order_id in self.live_order_ids
# def _add_order(self, order):
# order_id = order.id
# assert order_id not in self.orders
# assert order_id not in self.live_order_ids
# self.orders[order_id] = order
# # just assume that if we're ever adding a new order that means
# # it's about to be sent to an ECN-- it doesn't really make sense
# # to add dead orders
# self.live_order_ids.add(order_id)
# def _remove_order(self, order_id):
# """Remove order from orders & live_order_ids and return """
# assert order_id in self.orders, "Can't find order %s" % order_id
# order = self.orders[order_id]
# del self.orders[order_id]
# if order_id in self.live_order_ids:
# self.live_order_ids.remove(order_id)
# return order
# KTK TODO removed as dead code 20121106 - seems never to be called from anywhere
def _rename(self, old_id, new_id):
logger.debug("_rename %s to %s", old_id, new_id)
assert old_id in self.orders, "Can't rename non-existent order %s" % str(old_id)
assert old_id in self.live_order_ids, "Can't rename dead order %s" % str(old_id)
# WE MIGHT NEED TO REFER TO THE ORDERS BY THEIR OLD NAMES!
# For now just remove the old IDs from live_order_ids
# order = self._remove_order(old_id)
order = self.orders[old_id]
order.id = new_id
self.orders[new_id] = order
if old_id in self.live_order_ids:
self.live_order_ids.remove(old_id)
self.live_order_ids.add(new_id)
return order
# def _cancel(self, order_id, cancel_id):
# """We need the ID of the cancel request itself since we may get future messages
# about the underlying order but only under the ID of the cancel request"""
#
# assert order_id in self.live_order_ids, \
# "Can't cancel %s if it's not in live orders" % order_id
#
# self.live_order_ids.remove(order_id)
#
# # have to rename the live order first and then kill it once its ID is the
# # same as that of the cancel request
# order = self._rename(order_id, cancel_id)
# order.set_status(ORDER_STATUS.CANCELLED)
def _update_order(self, order, price, qty, cum_qty, leaves_qty, avg_price, last_shares, last_price, status):
changed = False
if order.price != price:
order.price = price
changed = True
if order.last_price != last_price:
order.last_price = last_price
changed = True
if order.qty != qty:
order.qty = qty
changed = True
if order.leaves_qty != leaves_qty:
order.leaves_qty = leaves_qty
changed = True
if avg_price and avg_price > 0 and order.avg_price != avg_price:
order.avg_price = avg_price
changed = True
if order.cum_qty != cum_qty:
order.cum_qty = cum_qty
# KTK - could we use last_shares here from ER?
# old_filled = order.cum_qty
# old_pos = self.positions.get(order.symbol, 0)
# if order.side == BID:
# new_pos = old_pos - old_filled + cum_qty
# else:
# new_pos = old_pos + old_filled - cum_qty
# self.positions[order.symbol] = new_pos
# order.cum_qty = cum_qty
changed = True
if order.last_price != last_price:
order.last_price = last_price
changed = True
if order.last_shares != last_shares:
order.last_shares = last_shares
changed = True
order.status = status
if changed:
order.last_update_time = time.time()
def print_position(self):
logger.info("POSITION INFO")
for k in self.positions:
logger.info("%s", self.positions[k])
def handle_fill(self, order):
# logger.debug('HANDLE_FILL(%s)', order)
pos = self.positions.get(order.symbol, Position(order.symbol))
if order.side == BID:
pos.long_pos += order.last_shares
pos.long_val += order.last_shares * order.last_price
if order.side == ASK:
pos.short_pos += order.last_shares
pos.short_val += order.last_shares * order.last_price
self.positions[order.symbol] = pos
self.print_position()
def _handle_execution_report(self, er):
cl_order_id = uuid.UUID(bytes=er.cl_order_id)
# only used for cancel and cancel/replace
if er.orig_cl_order_id is not "":
orig_cl_order_id = uuid.UUID(bytes=er.orig_cl_order_id)
else:
logger.warning("ORIG_CL_ORDER_ID IS NOT SET IN THIS ER - using cl_order_id")
# KTK - is this OK? Fast doesn't fill in orig on new order ack
orig_cl_order_id = cl_order_id
venue_id = er.venue_id
status = er.order_status
exec_type = er.exec_type
symbol = er.symbol
transaction_type = er.exec_trans_type
side = er.side
price = er.price
qty = er.order_qty
cum_qty = er.cum_qty
leaves_qty = er.leaves_qty
avg_price = er.avg_price
last_price = er.last_price
last_shares = er.last_shares
logger.info(
"Execution: venue_id = %d, cl_order_id = %s, orig_cl_order_id = %s, status = %s, exec_type = %s, transaction_type= %s, side = %s, symbol = %s, price = %f, qty = %d, cum_qty = %d, leaves_qty = %d, avg_price = %f, last_shares= %f, last_price = %f",
venue_id,
cl_order_id,
orig_cl_order_id,
EXEC_TYPE.to_str(status),
EXEC_TYPE.to_str(exec_type),
transaction_type,
side,
symbol,
price,
qty,
cum_qty,
leaves_qty,
avg_price,
last_shares,
last_price,
)
# NEW transactions are updates to our state and STATUS transactions just
# repeat the values of the most recent transaction. The trickier cases
# are CANCEL and CORRECT, which refer to the exec_id of the previous
# transaction they undo or modify via exec_ref_id
# For now we crash if the ECN tries to cancel or correct a previous
# transaction
unsupported_transaction_types = [EXEC_TRANS_TYPE.CANCEL, EXEC_TRANS_TYPE.CORRECT, EXEC_TRANS_TYPE.STATUS]
assert (
transaction_type not in unsupported_transaction_types
), "Unsupported exec_trans_type %s " % EXEC_TRANS_TYPE.to_str(transaction_type)
# Catch exec_types which we don't support
# regardless of the transaction type)
unsupported_exec_types = [EXEC_TYPE.STOPPED, EXEC_TYPE.SUSPENDED, EXEC_TYPE.RESTATED, EXEC_TYPE.CALCULATED]
assert exec_type not in unsupported_exec_types, (
"Unsupported exec_type = %s, order status = %s, cl_order_id = %s, orig_cl_order_id = %s"
% (EXEC_TYPE.to_str(exec_type), EXEC_TYPE.to_str(status), cl_order_id, orig_cl_order_id)
)
# #################################
# Update Order fields #
# #################################
assert cl_order_id in self.orders, (
"Received unknown order: venue=%d, cl_order_id=%s, price=%f, side=%d, qty=%s, filled=%s"
% (venue_id, cl_order_id, price, side, qty, cum_qty)
)
# Get the order - throws exception if not found
order = self.get_order(orig_cl_order_id)
# Update with new fields from ER
self._update_order(order, price, qty, cum_qty, leaves_qty, avg_price, last_shares, last_price, status)
# It's possible to get unsolicited cancels and corrects, in which case
# there is only cl_order_id and an undefined value in orig_cl_order_id
# cancels and restatements should come in with exec_trans_type = CANCEL
# or exec_trans_type = CORRECT
if transaction_type == EXEC_TRANS_TYPE.NEW:
if exec_type == EXEC_TYPE.NEW:
assert self.pending.has_value(cl_order_id), (
"Received new order for unknown ID <orig_cl_order_id=%s, cl_order_id=%s>"
% (orig_cl_order_id, cl_order_id)
)
self.pending.remove_value(cl_order_id)
self.live_order_ids.add(cl_order_id)
elif exec_type == EXEC_TYPE.CANCELLED:
if not self.is_pending(cl_order_id):
logger.warning(
"Unknown cancel for <orig_cl_order_id=%s, cl_order_id=%s> - NOT IN PENDING",
str(orig_cl_order_id),
str(cl_order_id),
)
else:
assert orig_cl_order_id in self.live_order_ids, (
"Order: %s cancelled not in live orders" % orig_cl_order_id
)
assert orig_cl_order_id in self.orders, "Can't rename non-existent order %s" % str(orig_cl_order_id)
assert orig_cl_order_id in self.live_order_ids, "Can't rename dead order %s" % str(orig_cl_order_id)
# WE MIGHT NEED TO REFER TO THE ORDERS BY THEIR OLD NAMES!
# For now just remove the old IDs from live_order_ids
# order = self._remove_order(old_id)
# WE COULD REMOVE FROM ORDERS TOO...
# del self.orders[orig_cl_order_id]
order.id = cl_order_id
self.orders[cl_order_id] = order
if orig_cl_order_id in self.live_order_ids:
self.live_order_ids.remove(orig_cl_order_id)
self.pending.remove_value(cl_order_id)
self.pending.remove_key(orig_cl_order_id)
elif exec_type == EXEC_TYPE.FILL:
logger.info("RECEIVED FILL: %s", order)
self.handle_fill(order)
# The check for ord_status == FILL reflects FXCM's STUPID F*****G HANDLING of
# ExecType which uses ExecType=FILL even for partial orders so according to
# the docs if:
# OrdStatus == PARTIAL_FILL && ExecType==FILL then it's a partial
# OrdStatus == FILL && ExecType == FILL then it's a full fill
# F**K THAT BITCHES
if status == ORDER_STATUS.FILL:
logger.info(" --FILL WAS FULL")
self.live_order_ids.remove(cl_order_id)
elif status == ORDER_STATUS.PARTIAL_FILL:
logger.info(" --FILL WAS PARTIAL")
elif exec_type == EXEC_TYPE.PARTIAL_FILL:
logger.info("RECEIVED PARTIAL FILL")
# %s', order)
self.handle_fill(order)
elif exec_type == EXEC_TYPE.REJECTED:
# NEW order can be rejected (bad price, size, etc...)
# assert order.id is None
assert self.pending.has_value(
cl_order_id
), "Rejected order not in pending <orig_cl_order_id=%s cl_order_id=%s" % (
orig_cl_order_id,
cl_order_id,
)
self.pending.remove_value(cl_order_id)
# NB - removing the key here will cause all pending messages relating to the order
# to be removed from pending as well. Thus, if a new order (e.g.) is rejected then
# the cancel will be rejected as well but not found in pending since the "key" value
# (i.e. the original rejected order id) has been removed.
# Sequence is as follows:
# 1) Send new that will be rejected cl_orde_id = ABC
# 2) Immediately send cancel on that order so cl_order_id = DEF, orig_cl_order_id = ABC
# 3) When reject is received key ABC will be removed
# 4) Removing ABC also removes value DEF from pending map
# 5) When cancel reject is received the DEF order id is unknown
self.pending.remove_key(cl_order_id)
elif exec_type == EXEC_TYPE.PENDING_CANCEL:
logger.info(
"RECEIVED PENDING CANCEL <orig_cl_order_id=%s, cl_order_id=%s>",
str(orig_cl_order_id),
str(cl_order_id),
)
elif exec_type == EXEC_TYPE.REPLACE:
if not self.is_pending(cl_order_id):
logger.warning(
"Replace not in pending <orig_cl_order_id=%s, cl_order_id=%s>",
str(orig_cl_order_id),
str(cl_order_id),
)
assert orig_cl_order_id in self.live_order_ids, "%s not in LIVE orders" % str(orig_cl_order_id)
assert orig_cl_order_id in self.orders, "%s not in orders" % str(orig_cl_order_id)
order = self.orders[orig_cl_order_id]
order.id = cl_order_id
self.orders[cl_order_id] = order
if orig_cl_order_id in self.live_order_ids:
self.live_order_ids.remove(orig_cl_order_id)
self.live_order_ids.add(cl_order_id)
# print "AFTER REPLACE RECEIVED - does pending contain orig=%s or cl=%s???????????????????????????????" % (orig_cl_order_id, cl_order_id)
# TODO It's the orig order id that is the key in the pending map - but I don't think this is correct
# since the request that is pending is the cl_order_id and the order it relates to is the orig_cl_order_id
# If you do change it then chagne the send_cancel_replace(...) fcn to add to pending correctly as well
self.pending.remove_key(orig_cl_order_id)
elif transaction_type == EXEC_TRANS_TYPE.CANCEL:
logger.warning("Unsolicited CANCEL (busted exec) request on %s", str(cl_order_id))
logger.warning("Order was: %s", self.get_order(cl_order_id))
logger.warning("New exec is: %f, %f, %f, %f, %f, %s", price, qty, cum_qty, leaves_qty, avg_price, status)
elif transaction_type == EXEC_TRANS_TYPE.CORRECT:
logger.warning("Unsolicited CORRECT request on %s", str(cl_order_id))
logger.warning("Order was: %s", self.get_order(cl_order_id))
logger.warning("New exec is: %f, %f, %f, %f, %f, %s", price, qty, cum_qty, leaves_qty, avg_price, status)
# ###############################################
# Is the order in a terminal state? #
# ###############################################
terminal_states = [ORDER_STATUS.FILL, ORDER_STATUS.CANCELLED, ORDER_STATUS.REJECTED, ORDER_STATUS.EXPIRED]
if status in terminal_states:
if cl_order_id in self.live_order_ids:
logger.warning("Removing %s from live order ids", str(cl_order_id))
self.live_order_ids.remove(cl_order_id)
# elif transaction_type == EXEC_TRANS_TYPE.NEW:
# logger.warning("Order %s should have been alive before entering terminal state %s", str(cl_order_id), ORDER_STATUS.to_str(status))
def _handle_cancel_reject(self, cr):
# cl_order_id is the cancel request id
cl_order_id = uuid.UUID(bytes=cr.cl_order_id)
# orig_cl_order_id is the order id of the order we're trying to cancel
orig_cl_order_id = uuid.UUID(bytes=cr.orig_cl_order_id)
logger.warning(
"Cancel reject: cl_order_id = %s, orig_cl_order_id = %s, reason =%s",
cl_order_id,
orig_cl_order_id,
cr.cancel_reject_reason,
)
assert orig_cl_order_id in self.orders, "Cancel reject for unknown original order ID %s" % str(orig_cl_order_id)
# make sure the cancel is no longer "live" if cl_order_id in self.live_order_ids:
# N.B. Cancel replace may be rejected and never appear as alive since live orders
# must have been ack'd at least once (NEW acknowlegement for example) to be moved
# into live orders - otherwise they are simply pending
if self.is_alive(cl_order_id):
self.live_order_ids.remove(cl_order_id)
if self.is_pending(cl_order_id):
self.pending_id_rejected(orig_cl_order_id, cl_order_id)
else:
logger.warning(
"Got unexpected cancel rejection - maybe NEW or REPLACE were rejected or order already cancelled?"
)
def received_message_from_order_engine(self, tag, msg):
if tag == order_engine_constants.EXEC_RPT:
er = execution_report()
er.ParseFromString(msg)
# logger.debug(er.__str__())
self._handle_execution_report(er)
elif tag == order_engine_constants.ORDER_CANCEL_REJ:
cr = order_cancel_reject()
# logger.debug(cr.__str__())
cr.ParseFromString(msg)
self._handle_cancel_reject(cr)
else:
raise RuntimeError("Unsupported order engine message: %s" % order_engine_constants.to_str(tag))
def _make_cancel_request(self, request_id, order):
"""Takes an order id to cancel, constructs a proto buf which can be sent
to the order engine.
"""
pb = order_cancel()
pb.cl_order_id = request_id.bytes
pb.orig_order_id = order.id.bytes
pb.strategy_id = self.strategy_id
pb.symbol = order.symbol
pb.side = order.side
pb.order_qty = order.qty
return pb
def _make_new_order_request(self, order):
order_pb = new_order_single()
order_pb.order_id = order.id.bytes
order_pb.strategy_id = self.strategy_id
order_pb.symbol = order.symbol
order_pb.side = order.side
order_pb.order_qty = order.qty
order_pb.ord_type = order.order_type
order_pb.price = order.price
order_pb.time_in_force = order.time_in_force
# order_pb.account = None
order_pb.venue_id = order.venue
return order_pb
def _make_cancel_replace_request(self, request_id, order, price, qty):
pb = order_cancel_replace()
pb.orig_order_id = order.id.bytes
pb.cl_order_id = request_id.bytes
pb.strategy_id = self.strategy_id
# hard-coded for baxter-- won't work with FAST or FXCM
pb.handl_inst = HANDLING_INSTRUCTION.AUTOMATED_INTERVENTION_OK
pb.symbol = order.symbol
pb.side = order.side
pb.order_qty = qty
pb.price = price
pb.transact_time = datetime.datetime.utcnow().strftime("%Y%M%D-%H:%M:%S")
pb.ord_type = order.order_type
pb.time_in_force = order.time_in_force
return pb
def send_new_order(self, venue, symbol, side, price, qty, order_type=LIM, time_in_force=GFD):
# print "Attempting to create new order: venue = %s, symbol = %s, side = %s, price = %s, size = %s" % \
# (venue, symbol, side, price, qty)
order_id = fresh_id()
logger.info(
"send_new_order: id=%s, venue=%s, symbol=%s, side=%s, price=%f, size=%s",
order_id,
venue,
symbol,
side,
price,
qty,
)
order = Order(order_id, venue, symbol, side, price, qty, order_type=order_type, time_in_force=time_in_force)
pb = self._make_new_order_request(order)
socket = self.order_sockets[venue]
tag = int_to_bytes(order_engine_constants.ORDER_NEW)
bytes = pb.SerializeToString()
socket.send_multipart([tag, self.strategy_id, order_id.bytes, bytes])
self.orders[order_id] = order
self.live_order_ids.add(order_id)
self.pending.add(order_id, order_id)
return order_id
def send_cancel_replace(self, order_id, price, qty):
# print "Attempting to cancel/replace %s to price=%s qty=%s" % (order_id, price, qty)
assert order_id in self.orders
assert order_id in self.live_order_ids
order = self.orders[order_id]
assert (
order.price != price or order.qty != qty
), "Trying to cancel/replace without changing anything for order %s" % str(order_id)
request_id = fresh_id()
logger.info(
"send_cancel_replace: orig_id=%s, replace_id=%s price=%s qty=%s" % (order_id, request_id, price, qty)
)
pb = self._make_cancel_replace_request(request_id, order, price, qty)
venue = order.venue
socket = self.order_sockets[venue]
tag = int_to_bytes(order_engine_constants.ORDER_REPLACE)
bytes = pb.SerializeToString()
socket.send_multipart([tag, self.strategy_id, request_id.bytes, bytes])
self.orders[request_id] = order
self.pending.add(order_id, request_id)
return request_id
def send_synth_cancel_replace(self, order_id, price, qty):
logger.info("send_synth_cancel_replace: %s to price=%s qty=%s" % (order_id, price, qty))
assert order_id in self.orders
assert order_id in self.live_order_ids
order = self.orders[order_id]
assert (
order.price != price or order.qty != qty
), "Trying to cancel/replace without changing anything for order %s" % str(order_id)
# cancel the original order
cancel_request_id = self.send_cancel(order_id)
# send the new order with the modified price
new_order_request_id = self.send_new_order(order.venue, order.symbol, order.side, price, qty)
logger.info("Sent synthetic cancel/replace")
logger.info(
"1) Sent cancel to %s: orig_id = %s, new_id = %s", order.venue, str(order_id), str(cancel_request_id)
)
logger.info(
"2) Sent new order to %s: new_id = %s, price = %f, qty= %s",
order.venue,
str(new_order_request_id),
price,
qty,
)
return new_order_request_id
def send_cancel(self, order_id):
assert order_id in self.orders, "send_cancel: Unknown order %s" % str(order_id)
# assert order_id in self.live_order_ids, "send_cancel: Can't cancel dead order %s" % str(order_id)
order = self.orders[order_id]
request_id = fresh_id()
logger.info("Sending cancel for order_id=%s, cancel_request_id=%s", str(order_id), str(request_id))
pb = self._make_cancel_request(request_id, order)
tag = int_to_bytes(order_engine_constants.ORDER_CANCEL)
bytes = pb.SerializeToString()
socket = self.order_sockets[order.venue]
socket.send_multipart([tag, self.strategy_id, request_id.bytes, bytes])
self.orders[request_id] = order
self.live_order_ids.add(request_id)
self.pending.add(order_id, request_id)
return request_id
def cancel_if_alive(self, order_id):
"""The cancel method is intentionally dumb and will try to cancel an order
even if it's in a terminal state. This is the smarter wrapper which
first checks if the order is alive and only sends a cancel request
if it is.
"""
alive = order_id in self.live_order_ids
if alive:
logger.debug("Sending cancel for live order: %s", order_id)
self.send_cancel(order_id)
return alive
def cancel_everything(self):
for order_id in self.live_order_ids:
self.send_cancel(order_id)
def open_orders(self):
return [self.orders[order_id] for order_id in self.live_order_ids]
def liquidate_all_open_orders(self, md):
"""Takes MarketData, mapping symbol -> venue -> entry,
replaces all open orders with significantly worse prices
likely to transact """
logger.info("Attempting to liquidate all %d open orders", len(self.live_order_ids))
request_ids = []
for order_id in self.live_order_ids:
request_id = self.liquidate_order(md, order_id)
request_ids.append(request_id)
return request_ids
def liquidate_order(self, md, order_id, qty=None):
order = self.get_order(order_id)
qty = qty if qty is not None else order.qty
price = md.liquidation_price(order.side, order.symbol, order.venue)
if venue_attrs.venue_specifics[order.venue].use_synthetic_cancel_replace == True:
return self.send_synth_cancel_replace(order.id, price=price, qty=qty)
else:
return self.send_cancel_replace(order.id, price=price, qty=qty)
