class IBBroker(BaseBroker):
def __init__(self):
basicConfig()
# These two variables are initialized in Connect method
self._connection = None
self._wrapper = None
self._request_id = 0
def _get_next_request_id(self):
self._request_id += 1
return self._request_id
def get_security(self, symbol):
contract = IBSecurity()
contract.Symbol = symbol
contract.symbol_id = 0
contract.Currency = 'USD'
return contract
def _get_next_valid_order_id(self):
"""
You must assign a unique order ID to each order you place. IB's servers
keep track of the next available order ID you can use; this function
requests that value from IB's servers, waits until IB sends a response,
then returns the ID.
"""
last_time = self._wrapper._time_received_next_valid_order_id
self._connection.reqIds(1)
# Wait until IB sends the next valid ID
while last_time == self._wrapper._time_received_next_valid_order_id:
very_short_sleep()
return self._wrapper._next_valid_order_id
def _request_current_time(self):
self._connection.reqCurrentTime()
def connect(self):
self._wrapper = ReferenceWrapper()
self._connection = EClientSocket(self._wrapper)
self._connection.eConnect(IB_HOST, IB_PORT, IB_CLIENT_ID)
def disconnect(self):
if self._connection.isConnected():
self._connection.eDisconnect()
def send_pre_trade(self, trade_info): # trade info is fa profile
self._connection.requestFA(self._connection.PROFILES)
self._connection.replaceFA(self._connection.PROFILES, trade_info)
def send_order(self, order):
order.__class__ = IBOrder # casting to IBOrder
order.prepare_IB_order()
order_id = self._get_next_valid_order_id()
contract = self.get_security(order.Symbol)
#order.m_transmit = True # forces IB to transmit order straight away
self._connection.placeOrder(order_id, contract, order) # places order
order.Status = Order.StatusChoice.Sent.value # order status is set to SENT
order.Order_Id = order_id # sets broker specific ID
while not self._wrapper.isOpeningOfOrdersFinished(order_id):
err = self._wrapper.getError(order_id)
if err is not None:
raise Exception(err)
very_short_sleep()
#if(self._wrapper.isError(id)):
# raise Exception(self.wrapper.isError(id))
def update_orders(self, orders):
requestId = self._get_next_request_id()
exf = ExecutionFilter()
distribution = {}
self._connection.reqExecutions(requestId, exf) #
while not self._wrapper.isExecutionRequestFinished(requestId):
err = self._wrapper.getError(requestId)
if err is not None:
raise Exception(err)
very_short_sleep()
executions = self._wrapper.getExecutions(requestId)
for order in orders:
price = 0
shares = 0
if executions is not None:
for execution in executions:
if execution.m_shares > 0 and execution.m_orderId == order.Order_Id and not execution.m_acctNumber.startswith('DF'):
price = execution.m_price
if order.Symbol not in distribution:
distribution[order.Symbol] = {}
if execution.m_acctNumber not in distribution[order.Symbol]:
distribution[order.Symbol][execution.m_acctNumber] = 0
distribution[order.Symbol][execution.m_acctNumber] += execution.m_shares
shares += execution.m_shares
if price != 0:
order.setFills(price, shares)
return distribution
def get_account_info(self, broker_account):
requestId = self._get_next_request_id()
self._connection.reqAccountSummary(requestId, 'All', 'AccountType,TotalCashValue')
while not self._wrapper.isExecutionRequestFinished(requestId):
err = self._wrapper.getError(requestId)
max_resp = self._wrapper.getMaxRequestFailureError()
if err is not None:
raise Exception(err)
if max_resp:
raise Exception("Maximum number of account summary requests exceeded")
very_short_sleep()
return self._wrapper.getAccountInfo(broker_account.ib_account)
# Account and Portfolio ##############################################################
# reqAccountUpdates ---> updateAccountTime self.update_AccountTime
# updateAccountValue self.update_AccountValue
# updatePortfolio self.update_Portfolio
# accountDownloadEnd self.accountDownloadEnd_flag
# reqAccountSummary ---> accountSummary self.account_Summary
# cancelAccountSummary
# accountSummaryEnd self.account_SummaryEnd_flag
# reqPositions ---> position self.update_Position
# cancelPositions
# positionEnd self.positionEnd_flag
###################################################################################'''
print "Testing Account and Portfolio \n"
tws.reqAccountUpdates(1, accountName)
tws.reqAccountSummary(1, "All", "NetLiquidation")
#tws.cancelAccountSummary(1)
tws.reqPositions()
#tws.cancelPositions()
# Orders #############################################################################
# placeOrder ---> orderStatus** self.order_Status
# cancelorder
# ---> openOrderEnd self.open_OrderEnd_flag
# reqOpenOrders ---> openOrder* self.open_Order
# ---> orderStatus**
# reqAllOpenOrders ---> openOrder*
# ---> orderStatus**
# reqAutoOpenOrders ---> openOrder*
# ---> orderStatus**
# reqIds ---> nextValidId self.next_ValidId
import numpy as np
from IBWrapper import IBWrapper, contract
accountName = "asd781820"
callback = IBWrapper()
tws = EClientSocket(callback)
host = ""
port = 4002
clientId = 9
tws.eConnect(host, port, clientId)
create = contract()
callback.initiate_variables()
tws.reqAccountSummary(8,'All','TotalCashValue')
callback.accountSummary()
globalid = 138756199
def buy(globalid,contract_Details):
oid = globalid
order = create_order('MKT', 100000, 'BUY')
tws.placeOrder(oid, contract_Details, order)
time.sleep(2)
print('--------Account value before buy------')
print(callback.account_Summary[-1][3])
time.sleep(300)
oid += 1
order = create_order('MKT', 100000, 'SELL')
tws.placeOrder(oid, contract_Details, order)
print('buy order finished')
# Account and Portfolio ##############################################################
# reqAccountUpdates ---> updateAccountTime self.update_AccountTime
# updateAccountValue self.update_AccountValue
# updatePortfolio self.update_Portfolio
# accountDownloadEnd self.accountDownloadEnd_flag
# reqAccountSummary ---> accountSummary self.account_Summary
# cancelAccountSummary
# accountSummaryEnd self.account_SummaryEnd_flag
# reqPositions ---> position self.update_Position
# cancelPositions
# positionEnd self.positionEnd_flag
###################################################################################'''
print "Testing Account and Portfolio \n"
tws.reqAccountUpdates(1, accountName)
tws.reqAccountSummary(1,"All","NetLiquidation")
#tws.cancelAccountSummary(1)
tws.reqPositions()
#tws.cancelPositions()
# Orders #############################################################################
# placeOrder ---> orderStatus** self.order_Status
# cancelorder
# ---> openOrderEnd self.open_OrderEnd_flag
# reqOpenOrders ---> openOrder* self.open_Order
# ---> orderStatus**
class orderRunnerBeta:
def __init__(self, datadecoder):
self.accountName = "asd781820"
self.callback = IBWrapper()
self.tws = EClientSocket(self.callback)
self.host = ""
self.port = 4002
self.clientId = 8
self.tws.eConnect(self.host, self.port, self.clientId)
self.dd = datadecoder
self.create = contract()
self.callback.initiate_variables()
self.callback.initiate_variables()
self.tws.reqAccountSummary(8, 'All', 'TotalCashValue')
self.filedf = datadecoder.usabledf
self.datadecoder = datadecoder
contract_Details = self.create.create_contract('EUR', 'CASH',
'IDEALPRO', 'USD')
tickerId = 10000
self.tws.reqRealTimeBars(tickerId, contract_Details, 5, "MIDPOINT", 0)
self.tickerId = 9010
self.df = pd.read_csv('df.csv')
self.RbinDict = bqe.makeRDf(self.df)
self.column_index_value_dict, self.binDict = bqe.makeDf(self.df)
def create_trailing_order(self, action, quantity, trailingPercent=0.006):
order = Order()
order.m_orderType = "TRAIL"
order.m_totalQuantity = quantity
order.m_trailingPercent = trailingPercent
order.m_action = action
return order
def create_order(self, order_type, quantity, action):
"""Create an Order object (Market/Limit) to go long/short.
order_type - 'MKT', 'LMT' for Market or Limit orders
quantity - Integral number of assets to order
action - 'BUY' or 'SELL'"""
order = Order()
order.m_orderType = order_type
order.m_totalQuantity = quantity
order.m_action = action
return order
def buy(self, contract_Details, limit, stop):
self.tws.reqIds(-1)
time.sleep(0.5)
oid = self.callback.next_ValidId
parent = Order()
parent.orderId = oid
parent.m_action = "BUY"
parent.m_orderType = "MKT"
parent.m_totalQuantity = 300000
parent.m_transmit = False
print('|==========执行触发=================|')
print('|执行时期:%s|' % datetime.now())
print('|操作: BUY |')
print('|执行前总额:%s' % self.callback.account_Summary[-1][3])
print('|===================================|')
self.tws.reqIds(-1)
takeProfit = Order()
takeProfit.orderId = parent.orderId + 1
takeProfit.m_action = "SELL"
takeProfit.m_orderType = "LMT"
takeProfit.m_totalQuantity = 300000
takeProfit.m_lmtPrice = limit
takeProfit.m_parentId = oid
takeProfit.m_transmit = False
stopLoss = Order()
stopLoss.orderId = parent.orderId + 2
stopLoss.m_action = "SELL"
stopLoss.m_orderType = "STP"
#Stop trigger price
stopLoss.m_auxPrice = stop
stopLoss.m_totalQuantity = 300000
stopLoss.m_parentId = oid
stopLoss.m_transmit = True
bracketOrder = [parent, takeProfit, stopLoss]
for o in bracketOrder:
self.tws.placeOrder(o.orderId, contract_Details, o)
self.tws.reqIds(-1)
time.sleep(1)
time.sleep(2)
def sell(self, contract_Details, limit, stop):
self.tws.reqIds(-1)
time.sleep(0.5)
oid = self.callback.next_ValidId
parent = Order()
parent.orderId = oid
parent.m_action = "SELL"
parent.m_orderType = "MKT"
parent.m_totalQuantity = 300000
parent.m_transmit = False
print('|==========执行触发=================|')
print('|执行时期:%s|' % datetime.now())
print('|操作: BUY |')
print('|执行前总额:%s' % self.callback.account_Summary[-1][3])
print('|===================================|')
self.tws.reqIds(-1)
takeProfit = Order()
takeProfit.orderId = parent.orderId + 1
takeProfit.m_action = "BUY"
takeProfit.m_orderType = "LMT"
takeProfit.m_totalQuantity = 300000
takeProfit.m_lmtPrice = limit
takeProfit.m_parentId = oid
takeProfit.m_transmit = False
stopLoss = Order()
stopLoss.orderId = parent.orderId + 2
stopLoss.m_action = "BUY"
stopLoss.m_orderType = "STP"
#Stop trigger price
stopLoss.m_auxPrice = stop
stopLoss.m_totalQuantity = 300000
stopLoss.m_parentId = oid
stopLoss.m_transmit = True
bracketOrder = [parent, takeProfit, stopLoss]
for o in bracketOrder:
self.tws.placeOrder(o.orderId, contract_Details, o)
self.tws.reqIds(-1)
time.sleep(1)
time.sleep(2)
def job(self):
contract_Details = self.create.create_contract('EUR', 'CASH',
'IDEALPRO', 'USD')
#self.tws.reqCurrentTime()
#time.sleep(1)
#ts = self.callback.current_Time
#data_endtime = datetime.utcfromtimestamp(ts).strftime('%Y%m%d %H:%M:%S')
self.tws.reqHistoricalData(tickerId=self.tickerId,
contract=contract_Details,
endDateTime='',
durationStr="1 D",
barSizeSetting="1 min",
whatToShow="BID",
useRTH=0,
formatDate=1)
time.sleep(2)
data = pd.DataFrame(self.callback.historical_Data,
columns=[
"reqId", "date", "open", "high", "low",
"close", "volume", "count", "WAP", "hasGaps"
])[-500:-1]
self.dd.findMatch(data)
print(data)
print(datetime.now())
for b in self.dd.reversedBucket:
print('|=============================================|')
print('|检测到基因:%s|' % b)
print('|=============================================|')
if (self.filedf.index.contains(b)):
stra = bqe.get_sharpe_stra(b)
if (stra > 0):
threading.Thread(target=self.buy,
args=[contract_Details,
stra + 1]).start()
elif (stra < 0):
threading.Thread(target=self.sell,
args=[contract_Details,
abs(stra - 1)]).start()
self.tws.reqIds(-1)
print('|===================================|')
print('|总额:%s |' % self.callback.account_Summary[-1][3])
print('|===================================|')
def run(self):
print('|===========================================|')
print('|=======欢迎使用beta版本，祝您好运==========|')
print('|====新特性：1。止损定单 2。智能订单执行====|')
print('|===========================================|')
print('|===========================================|')
self.tickerId = 9010
while (1):
now = datetime.now()
if (now.second == 5):
contract_Details = self.create.create_contract(
'EUR', 'CASH', 'IDEALPRO', 'USD')
#self.tws.reqCurrentTime()
#time.sleep(1)
#ts = self.callback.current_Time
#data_endtime = datetime.utcfromtimestamp(ts).strftime('%Y%m%d %H:%M:%S')
self.tws.reqHistoricalData(tickerId=self.tickerId,
contract=contract_Details,
endDateTime='',
durationStr="1 D",
barSizeSetting="1 min",
whatToShow="BID",
useRTH=0,
formatDate=1)
time.sleep(2)
data = pd.DataFrame(self.callback.historical_Data,
columns=[
"reqId", "date", "open", "high", "low",
"close", "volume", "count", "WAP",
"hasGaps"
])[-500:-1]
self.dd.findMatch(data)
last_close = data.iloc[-1].close
for b in self.dd.reversedBucket:
if (self.filedf.index.contains(b)):
print('usable gene: %s' % b)
limit, stop, action = bqe.get_monte_carlo_stra(
ast.literal_eval(b), self.column_index_value_dict,
self.binDict, self.df, self.RbinDict)
if (action > 0):
limit_price = last_close + limit
stop_price = last_close - stop
threading.Thread(target=self.buy,
args=[
contract_Details, limit_price,
stop_price
]).start()
elif (action < 0):
limit_price = last_close - limit
stop_price = last_close + stop
threading.Thread(
target=self.sell,
args=[contract_Details, limit, stop]).start()
self.tws.reqIds(-1)
print('|===================================|')
print('|总额:%s |' %
self.callback.account_Summary[-1][3])
print('|===================================|')
def dc(self):
self.tws.eDisconnect()
class TWS_gateway(threading.Thread):
# config
config = None
# redis connection
rs = None
# channel clients' requests to IB/TWS
cli_request_handler = None
# manage conID / contracts mapping
contract_subscription_mgr = None
connection = None
# handler to process incoming IB/TWS messages and echo back to clients
tws_event_handler = None
# monitor IB connection / heart beat
ibh = None
tlock = None
ib_conn_status = None
ib_order_transmit = False
def __init__(self, host, port, clientId, kafka_host, kafka_port, config):
super(TWS_gateway, self).__init__()
self.config = config
self.host = host
self.port = port
self.clientId = clientId
self.ib_order_transmit = config.get("tws_gateway", "tws_gateway.order_transmit").strip('"').strip("'") if \
config.get("tws_gateway", "tws_gateway.order_transmit").strip('"').strip("'") <> None\
else False
logging.info('starting up TWS_gateway...')
logging.info('Order straight through (no-touch) flag = %s' %
('True' if self.ib_order_transmit == True else 'False'))
logging.info('connecting to Redis server...')
self.initialize_redis(config)
logging.info('starting up TWS_event_handler...')
self.tws_event_handler = TWS_event_handler(kafka_host, kafka_port)
logging.info('starting up IB EClientSocket...')
self.connection = EClientSocket(self.tws_event_handler)
logging.info('starting up client request handler - kafkaConsumer...')
self.cli_request_handler = KafkaConsumer( *[(v,0) for v in list(TWS_Protocol.topicMethods) + list(TWS_Protocol.gatewayMethods) ], \
metadata_broker_list=['%s:%s' % (kafka_host, kafka_port)],\
group_id = 'epc.tws_gateway',\
auto_commit_enable=True,\
auto_commit_interval_ms=30 * 1000,\
auto_offset_reset='largest') # discard old ones
self.reset_message_offset()
if not self.eConnect():
logging.error(
'TWS_gateway: unable to establish connection to IB %s:%d' %
(self.host, self.port))
sys.exit(-1)
else:
# start heart beat monitor
logging.info('starting up IB heart beat monitor...')
self.tlock = Lock()
self.ibh = IbHeartBeat(config)
self.ibh.register_listener([self.on_ib_conn_broken])
self.ibh.run()
logging.info('starting up subscription manager...')
self.initialize_subscription_mgr()
def initialize_subscription_mgr(self):
self.contract_subscription_mgr = SubscriptionManager(self)
self.contract_subscription_mgr.register_persistence_callback(
self.persist_subscriptions)
key = self.config.get(
"tws_gateway",
"subscription_manager.subscriptions.redis_key").strip('"').strip(
"'")
if self.rs.get(key):
#contracts = map(lambda x: ContractHelper.kvstring2contract(x), json.loads(self.rs.get(key)))
def is_outstanding(c):
today = time.strftime('%Y%m%d')
if c.m_expiry < today:
logging.info(
'initialize_subscription_mgr: ignoring expired contract %s%s%s'
% (c.m_expiry, c.m_strike, c.m_right))
return False
return True
contracts = filter(
lambda x: is_outstanding(x),
map(lambda x: ContractHelper.kvstring2object(x, Contract),
json.loads(self.rs.get(key))))
self.contract_subscription_mgr.load_subscription(contracts)
def persist_subscriptions(self, contracts):
key = self.config.get(
"tws_gateway",
"subscription_manager.subscriptions.redis_key").strip('"').strip(
"'")
#cs = json.dumps(map(lambda x: ContractHelper.contract2kvstring(x) if x <> None else None, contracts))
cs = json.dumps(
map(
lambda x: ContractHelper.object2kvstring(x)
if x <> None else None, contracts))
logging.debug(
'Tws_gateway: updating subscription table to redis store %s' % cs)
self.rs.set(key, cs)
def initialize_redis(self, config):
r_host = config.get("redis", "redis.server").strip('"').strip("'")
r_port = config.get("redis", "redis.port")
r_db = config.get("redis", "redis.db")
self.rs = redis.Redis(r_host, r_port, r_db)
try:
self.rs.client_list()
except redis.ConnectionError:
logging.error(
'TWS_gateway: unable to connect to redis server using these settings: %s port:%d db:%d'
% (r_host, r_port, r_db))
logging.error('aborting...')
sys.exit(-1)
def reset_message_offset(self):
topic_offsets = map(
lambda topic: (topic,
self.cli_request_handler.get_partition_offsets(
topic, 0, -1, 999)),
TWS_Protocol.topicMethods + TWS_Protocol.gatewayMethods)
topic_offsets = filter(
lambda x: x <> None,
map(lambda x: (x[0], x[1][1], x[1][0])
if len(x[1]) > 1 else None, topic_offsets))
logging.info('TWS_gateway set topic offset to the latest point\n%s' %
(''.join('%s,%s,%s\n' % (x[0], x[1], x[2])
for x in topic_offsets)))
# the set_topic_partitions method clears out all previous settings when executed
# therefore it's not possible to call the function multiple times:
# self.consumer.set_topic_partitions(('gw_subscriptions', 0, 114,)
# self.consumer.set_topic_partitions(('tickPrice', 0, 27270,))
# as the second call will wipe out whatever was done previously
self.cli_request_handler.set_topic_partitions(*topic_offsets)
def run(self):
for message in self.cli_request_handler:
logging.info("%s:%d:%d: key=%s value=%s" %
(message.topic, message.partition, message.offset,
message.key, message.value))
# print ("TWS_gateway: received client request %s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
# message.offset, message.key,
# message.value))
getattr(self, message.topic, None)(message.value)
#self.cli_request_handler.task_done(message)
def on_ib_conn_broken(self, msg):
logging.error('TWS_gateway: detected broken IB connection!')
self.ib_conn_status = 'ERROR'
self.tlock.acquire() # this function may get called multiple times
try: # block until another party finishes executing
if self.ib_conn_status == 'OK': # check status
return # if already fixed up while waiting, return
self.eDisconnect()
self.eConnect()
while not self.connection.isConnected():
logging.error('TWS_gateway: attempt to reconnect...')
self.eConnect()
sleep(2)
# we arrived here because the connection has been restored
# resubscribe tickers again!
logging.info(
'TWS_gateway: IB connection restored...resubscribe contracts')
self.contract_subscription_mgr.force_resubscription()
finally:
self.tlock.release()
def eConnect(self):
logging.info('TWS_gateway - eConnect. Connecting to %s:%s App Id: %s' %
(self.host, self.port, self.clientId))
self.connection.eConnect(self.host, self.port, self.clientId)
return self.connection.isConnected()
def reqAccountUpdates(self, value=None):
logging.info('TWS_gateway - reqAccountUpdates value=%s' % value)
self.connection.reqAccountUpdates(1, '')
def reqAccountSummary(self, value):
logging.info('TWS_gateway - reqAccountSummary value=%s' % value)
vals = map(
lambda x: x.encode('ascii') if isinstance(x, unicode) else x,
json.loads(value))
self.connection.reqAccountSummary(vals[0], vals[1], vals[2])
def reqOpenOrders(self, value=None):
self.connection.reqOpenOrders()
def reqPositions(self, value=None):
self.connection.reqPositions()
def reqExecutions(self, value):
try:
filt = ExecutionFilter(
) if value == '' else ExecutionFilterHelper.kvstring2object(
value, ExecutionFilter)
self.connection.reqExecutions(0, filt)
except:
logging.error(traceback.format_exc())
def reqIds(self, value=None):
self.connection.reqIds(1)
def reqNewsBulletins(self):
self.connection.reqNewsBulletins(1)
def cancelNewsBulletins(self):
self.connection.cancelNewsBulletins()
def setServerLogLevel(self):
self.connection.setServerLogLevel(3)
def reqAutoOpenOrders(self):
self.connection.reqAutoOpenOrders(1)
def reqAllOpenOrders(self):
self.connection.reqAllOpenOrders()
def reqManagedAccts(self):
self.connection.reqManagedAccts()
def requestFA(self):
self.connection.requestFA(1)
def reqMktData(self, sm_contract):
logging.info('TWS Gateway received reqMktData request: %s' %
sm_contract)
try:
#self.contract_subscription_mgr.reqMktData(ContractHelper.kvstring2contract(sm_contract))
self.contract_subscription_mgr.reqMktData(
ContractHelper.kvstring2object(sm_contract, Contract))
except:
pass
def reqHistoricalData(self):
contract = Contract()
contract.m_symbol = 'QQQQ'
contract.m_secType = 'STK'
contract.m_exchange = 'SMART'
endtime = strftime('%Y%m%d %H:%M:%S')
self.connection.reqHistoricalData(tickerId=1,
contract=contract,
endDateTime=endtime,
durationStr='1 D',
barSizeSetting='1 min',
whatToShow='TRADES',
useRTH=0,
formatDate=1)
def placeOrder(self, value=None):
logging.info('TWS_gateway - placeOrder value=%s' % value)
try:
vals = json.loads(value)
except ValueError:
logging.error('TWS_gateway - placeOrder Exception %s' %
traceback.format_exc())
return
# c = ContractHelper.kvstring2contract(vals[1])
o = OrderHelper.kvstring2object(vals[2], Order)
o.__dict__['transmit'] = self.ib_order_transmit
# print c.__dict__
# print o.__dict__
# print '---------------------'
#self.connection.placeOrder(vals[0], ContractHelper.kvstring2contract(vals[1]), OrderHelper.kvstring2object(vals[2], Order))
self.connection.placeOrder(
vals[0], ContractHelper.kvstring2object(vals[1], Contract),
OrderHelper.kvstring2object(vals[2], Order))
# self.connection.placeOrder(orderId, contract, newOptOrder)
def eDisconnect(self, value=None):
sleep(2)
self.connection.eDisconnect()
####################################################################3
# Gateway commands
def gw_req_subscriptions(self, value=None):
#subm = map(lambda i: ContractHelper.contract2kvstring(self.contract_subscription_mgr.handle[i]), range(len(self.contract_subscription_mgr.handle)))
#subm = map(lambda i: ContractHelper.object2kvstring(self.contract_subscription_mgr.handle[i]), range(len(self.contract_subscription_mgr.handle)))
subm = map(
lambda i: (i,
ContractHelper.object2kvstring(
self.contract_subscription_mgr.handle[i])),
range(len(self.contract_subscription_mgr.handle)))
print subm
if subm:
self.tws_event_handler.broadcast_event('gw_subscriptions',
{'subscriptions': subm},
source='GW')