class ReferenceApp: def __init__(self, host='localhost', port=7496, clientId=0): self.host = host self.port = port self.clientId = clientId self.wrapper = ReferenceWrapper() self.connection = EClientSocket(self.wrapper) @ref def eConnect(self): self.connection.eConnect(self.host, self.port, self.clientId) @ref def reqAccountUpdates(self): self.connection.reqAccountUpdates(1, '') @ref def reqOpenOrders(self): self.connection.reqOpenOrders() @ref def reqExecutions(self): filt = ExecutionFilter() self.connection.reqExecutions(0, filt) @ref def reqIds(self): self.connection.reqIds(10) @ref def reqNewsBulletins(self): self.connection.reqNewsBulletins(1) @ref def cancelNewsBulletins(self): self.connection.cancelNewsBulletins() @ref def setServerLogLevel(self): self.connection.setServerLogLevel(3) @ref def reqAutoOpenOrders(self): self.connection.reqAutoOpenOrders(1) @ref def reqAllOpenOrders(self): self.connection.reqAllOpenOrders() @ref def reqManagedAccts(self): self.connection.reqManagedAccts() @ref def requestFA(self): self.connection.requestFA(1) @ref def reqMktData(self): contract = Contract() # contract.m_symbol = 'AUD' contract.m_currency = 'USD' contract.m_secType = 'CASH' contract.m_exchange = 'IDEALPRO' self.connection.reqMktData(1, contract, '', False) @ref def reqHistoricalData(self): contract = Contract() contract.m_symbol = 'QQQQ' contract.m_secType = 'STK' contract.m_exchange = 'SMART' endtime = strftime('%Y%m%d %H:%M:%S') self.connection.reqHistoricalData( tickerId=1, contract=contract, endDateTime=endtime, durationStr='1 D', barSizeSetting='1 min', whatToShow='TRADES', useRTH=0, formatDate=1) @ref def eDisconnect(self): sleep(5) self.connection.eDisconnect()
# News Bulletin ###################################################################### # reqNewsBulletins ---> updateNewsBulletin self.update_NewsBulletin_msgId # self.update_NewsBulletin_msgType # self.update_NewsBulletin_message # self.update_NewsBulletin_origExchange ###################################################################################''' print "Testing News Bulletin Group \n" tws.reqNewsBulletins(1) time.sleep(20) tws.cancelNewsBulletins() # Financial Advisors Group ########################################################### # reqManagedAccts ---> managedAccounts self.managed_Accounts ###################################################################################''' print "Testing Financial Advisors Group \n" tws.reqManagedAccts() #tws.requestFA() # non FA account. Unable to test. # Historical Data #################################################################### # reqHistoricalData ---> historicalData self.historical_Data ###################################################################################''' print "Testing Historical Data Group \n" contract_Details10 = create.create_contract('EUR', 'CASH', 'IDEALPRO', 'USD') data_endtime = datetime.now().strftime("%Y%m%d %H:%M:%S") tws.reqHistoricalData(9000, contract_Details10, data_endtime, "1 M", "1 day", "BID", 0, 1) time.sleep(3) tws.cancelHistoricalData(9000) # Market Scanners ####################################################################
class IBClient(object): fields = {'trades': 'TRADES', 'midpoint': 'MIDPOINT', 'bid': 'BID', 'ask': 'ASK', 'bid_ask': 'BID_ASK', 'hist_vol': 'HISTORICAL_VOLATILITY', 'imp_vol': 'OPTION_IMPLIED_VOLATILITY'} def __init__(self, name=None, call_msg=True, host=None, port=None, client_id=None): self.name = name self.host = host self.port = port self.client_id = client_id self.ref_nums = [0] self.wrapper = SyncWrapper() self.connection = EClientSocket(self.wrapper) self.account = self.wrapper.account self.contracts = self.wrapper.contracts self.executions_ = self.wrapper.executions self.order_messages = self.wrapper.order_messages if self.host is None: self.host = 'localhost' if self.port is None: self.port = 7496 if call_msg is False: self.wrapper.suppress = True if self.client_id is None: self.client_id = 0 # listen to execution #self.wrapper.register(self.method, events='execution') self.__connect__ = self.connection.eConnect(self.host, self.port, self.client_id) self.__gen_order_id__(1) sleep(.2) def request_reference_id(self, integer=False): if not integer: ref_id = uuid.uuid4().hex if ref_id in self.ref_nums: return self.request_reference() else: self.ref_nums.append(ref_id) return ref_id else: ref_id = '{0:09d}'.format(np.random.randint(0, 999999999)) if ref_id > max([x for x in self.ref_nums if type(x) is int]): return int(ref_id) else: return self.request_reference(integer=True) def __gen_order_id__(self, num): self.connection.reqIds(num) return self.wrapper.order_id def method(self, sender, event, msg=None): print "[{0}] got event {1} with message {2}".format(self.name, event, msg) def __track_orders__(self): self.connection.reqAutoOpenOrders(1) def managed_accounts(self): if self.account.child_accounts: return self.account.child_accounts else: self.connection.reqManagedAccts() sleep(1) if self.account.child_accounts: return self.account.child_accounts return ['REQUEST FAILED'] def account_updates(self, acct): #get a unique id reference = self.request_reference_id() #append a new packet container to account self.account.append_request(reference, AccountPacket(acct), PortfolioPacket(acct)) self.wrapper.ref_id = reference self.connection.reqAccountUpdates(1, acct) sleep(1) return reference def place_order(self, contract, order): self.wrapper.order_id += 100 ref = self.wrapper.order_id self.connection.placeOrder(ref, contract, order) sleep(.2) return ref def cancel_order(self, id): self.connection.cancelOrder(id) def get_executions(self, efilter=None): if not efilter: efilter = ExecutionFilter() ref = self.request_reference_id(integer=True) self.connection.reqExecutions(reqId=ref, filter=efilter) sleep(3) # Todo: This is a ridiculous bottleneck return ref def get_contract(self, contract): ref = self.request_reference_id(integer=True) self.connection.reqContractDetails(ref, contract) sleep(1) return ref def portfolio(self, account): ref = self.account_updates(account) return self.account[ref]['portfolio'].messages def account_details(self, account): ref = self.account_updates(account) return self.account[ref]['account'].messages def executions(self, efilter=None): ref = self.get_executions(efilter) return self.executions_[ref] def order_status(self, order_id): sleep(.2) return [msg for msg in self.order_messages if msg['orderId'] == order_id] def disconnect(self): self.connection.eDisconnect()
class IBClient(object): fields = { 'trades': 'TRADES', 'midpoint': 'MIDPOINT', 'bid': 'BID', 'ask': 'ASK', 'bid_ask': 'BID_ASK', 'hist_vol': 'HISTORICAL_VOLATILITY', 'imp_vol': 'OPTION_IMPLIED_VOLATILITY' } def __init__(self, name=None, call_msg=True, host=None, port=None, client_id=None): self.name = name self.host = host self.port = port self.client_id = client_id self.ref_nums = [0] self.wrapper = SyncWrapper() self.connection = EClientSocket(self.wrapper) self.account = self.wrapper.account self.contracts = self.wrapper.contracts self.executions_ = self.wrapper.executions self.order_messages = self.wrapper.order_messages if self.host is None: self.host = 'localhost' if self.port is None: self.port = 7496 if call_msg is False: self.wrapper.suppress = True if self.client_id is None: self.client_id = 0 # listen to execution #self.wrapper.register(self.method, events='execution') self.__connect__ = self.connection.eConnect(self.host, self.port, self.client_id) self.__gen_order_id__(1) sleep(.2) def request_reference_id(self, integer=False): if not integer: ref_id = uuid.uuid4().hex if ref_id in self.ref_nums: return self.request_reference() else: self.ref_nums.append(ref_id) return ref_id else: ref_id = '{0:09d}'.format(np.random.randint(0, 999999999)) if ref_id > max([x for x in self.ref_nums if type(x) is int]): return int(ref_id) else: return self.request_reference(integer=True) def __gen_order_id__(self, num): self.connection.reqIds(num) return self.wrapper.order_id def method(self, sender, event, msg=None): print "[{0}] got event {1} with message {2}".format( self.name, event, msg) def __track_orders__(self): self.connection.reqAutoOpenOrders(1) def managed_accounts(self): if self.account.child_accounts: return self.account.child_accounts else: self.connection.reqManagedAccts() sleep(1) if self.account.child_accounts: return self.account.child_accounts return ['REQUEST FAILED'] def account_updates(self, acct): #get a unique id reference = self.request_reference_id() #append a new packet container to account self.account.append_request(reference, AccountPacket(acct), PortfolioPacket(acct)) self.wrapper.ref_id = reference self.connection.reqAccountUpdates(1, acct) sleep(1) return reference def place_order(self, contract, order): self.wrapper.order_id += 100 ref = self.wrapper.order_id self.connection.placeOrder(ref, contract, order) sleep(.2) return ref def cancel_order(self, id): self.connection.cancelOrder(id) def get_executions(self, efilter=None): if not efilter: efilter = ExecutionFilter() ref = self.request_reference_id(integer=True) self.connection.reqExecutions(reqId=ref, filter=efilter) sleep(3) # Todo: This is a ridiculous bottleneck return ref def get_contract(self, contract): ref = self.request_reference_id(integer=True) self.connection.reqContractDetails(ref, contract) sleep(1) return ref def portfolio(self, account): ref = self.account_updates(account) return self.account[ref]['portfolio'].messages def account_details(self, account): ref = self.account_updates(account) return self.account[ref]['account'].messages def executions(self, efilter=None): ref = self.get_executions(efilter) return self.executions_[ref] def order_status(self, order_id): sleep(.2) return [ msg for msg in self.order_messages if msg['orderId'] == order_id ] def disconnect(self): self.connection.eDisconnect()
# self.update_NewsBulletin_origExchange ###################################################################################''' print "Testing News Bulletin Group \n" tws.reqNewsBulletins(1) time.sleep(20) tws.cancelNewsBulletins() # Financial Advisors Group ########################################################### # reqManagedAccts ---> managedAccounts self.managed_Accounts ###################################################################################''' print "Testing Financial Advisors Group \n" tws.reqManagedAccts() #tws.requestFA() # non FA account. Unable to test. # Historical Data #################################################################### # reqHistoricalData ---> historicalData self.historical_Data ###################################################################################''' print "Testing Historical Data Group \n" contract_Details10 = create.create_contract('EUR', 'CASH', 'IDEALPRO', 'USD') data_endtime = datetime.now().strftime("%Y%m%d %H:%M:%S") tws.reqHistoricalData(9000, contract_Details10, data_endtime, "1 M", "1 day", "BID", 0, 1)
class ReferenceApp: parameters = None def __init__(self, host='localhost', port=7496, clientId=0): self.host = host self.port = port self.clientId = clientId self.parameters = settings.TradeParameters() self.wrapper = ReferenceWrapper(self.parameters) self.connection = EClientSocket(self.wrapper) def eConnect(self): self.connection.eConnect(self.host, self.port, self.clientId) def reqAccountUpdates(self): self.connection.reqAccountUpdates(1, '') def reqOpenOrders(self): self.connection.reqOpenOrders() def reqExecutions(self): filt = ExecutionFilter() self.connection.reqExecutions(filt) def reqIds(self): self.connection.reqIds(10) def reqNewsBulletins(self): self.connection.reqNewsBulletins(1) def cancelNewsBulletins(self): self.connection.cancelNewsBulletins() def setServerLogLevel(self): self.connection.setServerLogLevel(3) def reqAutoOpenOrders(self): self.connection.reqAutoOpenOrders(1) def reqAllOpenOrders(self): self.connection.reqAllOpenOrders() def reqManagedAccts(self): self.connection.reqManagedAccts() def requestFA(self): self.connection.requestFA(1) def reqMktData(self): for tick_id, symbol in self.parameters.tickers().iteritems(): contract = app.wrapper.makeContract(symbol) self.connection.reqMktData(tick_id, contract, [], False) def reqHistoricalData(self): contract = Contract() contract.m_symbol = 'QQQQ' contract.m_secType = 'STK' contract.m_exchange = 'SMART' endtime = strftime('%Y%m%d %H:%M:%S') self.connection.reqHistoricalData(tickerId=1, contract=contract, endDateTime=endtime, durationStr='1 D', barSizeSetting='1 min', whatToShow='TRADES', useRTH=0, formatDate=1) def eDisconnect(self): sleep(5) self.connection.eDisconnect()
class TWS_gateway(threading.Thread): # config config = None # redis connection rs = None # channel clients' requests to IB/TWS cli_request_handler = None # manage conID / contracts mapping contract_subscription_mgr = None connection = None # handler to process incoming IB/TWS messages and echo back to clients tws_event_handler = None # monitor IB connection / heart beat ibh = None tlock = None ib_conn_status = None ib_order_transmit = False def __init__(self, host, port, clientId, kafka_host, kafka_port, config): super(TWS_gateway, self).__init__() self.config = config self.host = host self.port = port self.clientId = clientId self.ib_order_transmit = config.get("tws_gateway", "tws_gateway.order_transmit").strip('"').strip("'") if \ config.get("tws_gateway", "tws_gateway.order_transmit").strip('"').strip("'") <> None\ else False logging.info('starting up TWS_gateway...') logging.info('Order straight through (no-touch) flag = %s' % ('True' if self.ib_order_transmit == True else 'False')) logging.info('connecting to Redis server...') self.initialize_redis(config) logging.info('starting up TWS_event_handler...') self.tws_event_handler = TWS_event_handler(kafka_host, kafka_port) logging.info('starting up IB EClientSocket...') self.connection = EClientSocket(self.tws_event_handler) logging.info('starting up client request handler - kafkaConsumer...') self.cli_request_handler = KafkaConsumer( *[(v,0) for v in list(TWS_Protocol.topicMethods) + list(TWS_Protocol.gatewayMethods) ], \ metadata_broker_list=['%s:%s' % (kafka_host, kafka_port)],\ group_id = 'epc.tws_gateway',\ auto_commit_enable=True,\ auto_commit_interval_ms=30 * 1000,\ auto_offset_reset='largest') # discard old ones self.reset_message_offset() if not self.eConnect(): logging.error( 'TWS_gateway: unable to establish connection to IB %s:%d' % (self.host, self.port)) sys.exit(-1) else: # start heart beat monitor logging.info('starting up IB heart beat monitor...') self.tlock = Lock() self.ibh = IbHeartBeat(config) self.ibh.register_listener([self.on_ib_conn_broken]) self.ibh.run() logging.info('starting up subscription manager...') self.initialize_subscription_mgr() def initialize_subscription_mgr(self): self.contract_subscription_mgr = SubscriptionManager(self) self.contract_subscription_mgr.register_persistence_callback( self.persist_subscriptions) key = self.config.get( "tws_gateway", "subscription_manager.subscriptions.redis_key").strip('"').strip( "'") if self.rs.get(key): #contracts = map(lambda x: ContractHelper.kvstring2contract(x), json.loads(self.rs.get(key))) def is_outstanding(c): today = time.strftime('%Y%m%d') if c.m_expiry < today: logging.info( 'initialize_subscription_mgr: ignoring expired contract %s%s%s' % (c.m_expiry, c.m_strike, c.m_right)) return False return True contracts = filter( lambda x: is_outstanding(x), map(lambda x: ContractHelper.kvstring2object(x, Contract), json.loads(self.rs.get(key)))) self.contract_subscription_mgr.load_subscription(contracts) def persist_subscriptions(self, contracts): key = self.config.get( "tws_gateway", "subscription_manager.subscriptions.redis_key").strip('"').strip( "'") #cs = json.dumps(map(lambda x: ContractHelper.contract2kvstring(x) if x <> None else None, contracts)) cs = json.dumps( map( lambda x: ContractHelper.object2kvstring(x) if x <> None else None, contracts)) logging.debug( 'Tws_gateway: updating subscription table to redis store %s' % cs) self.rs.set(key, cs) def initialize_redis(self, config): r_host = config.get("redis", "redis.server").strip('"').strip("'") r_port = config.get("redis", "redis.port") r_db = config.get("redis", "redis.db") self.rs = redis.Redis(r_host, r_port, r_db) try: self.rs.client_list() except redis.ConnectionError: logging.error( 'TWS_gateway: unable to connect to redis server using these settings: %s port:%d db:%d' % (r_host, r_port, r_db)) logging.error('aborting...') sys.exit(-1) def reset_message_offset(self): topic_offsets = map( lambda topic: (topic, self.cli_request_handler.get_partition_offsets( topic, 0, -1, 999)), TWS_Protocol.topicMethods + TWS_Protocol.gatewayMethods) topic_offsets = filter( lambda x: x <> None, map(lambda x: (x[0], x[1][1], x[1][0]) if len(x[1]) > 1 else None, topic_offsets)) logging.info('TWS_gateway set topic offset to the latest point\n%s' % (''.join('%s,%s,%s\n' % (x[0], x[1], x[2]) for x in topic_offsets))) # the set_topic_partitions method clears out all previous settings when executed # therefore it's not possible to call the function multiple times: # self.consumer.set_topic_partitions(('gw_subscriptions', 0, 114,) # self.consumer.set_topic_partitions(('tickPrice', 0, 27270,)) # as the second call will wipe out whatever was done previously self.cli_request_handler.set_topic_partitions(*topic_offsets) def run(self): for message in self.cli_request_handler: logging.info("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition, message.offset, message.key, message.value)) # print ("TWS_gateway: received client request %s:%d:%d: key=%s value=%s" % (message.topic, message.partition, # message.offset, message.key, # message.value)) getattr(self, message.topic, None)(message.value) #self.cli_request_handler.task_done(message) def on_ib_conn_broken(self, msg): logging.error('TWS_gateway: detected broken IB connection!') self.ib_conn_status = 'ERROR' self.tlock.acquire() # this function may get called multiple times try: # block until another party finishes executing if self.ib_conn_status == 'OK': # check status return # if already fixed up while waiting, return self.eDisconnect() self.eConnect() while not self.connection.isConnected(): logging.error('TWS_gateway: attempt to reconnect...') self.eConnect() sleep(2) # we arrived here because the connection has been restored # resubscribe tickers again! logging.info( 'TWS_gateway: IB connection restored...resubscribe contracts') self.contract_subscription_mgr.force_resubscription() finally: self.tlock.release() def eConnect(self): logging.info('TWS_gateway - eConnect. Connecting to %s:%s App Id: %s' % (self.host, self.port, self.clientId)) self.connection.eConnect(self.host, self.port, self.clientId) return self.connection.isConnected() def reqAccountUpdates(self, value=None): logging.info('TWS_gateway - reqAccountUpdates value=%s' % value) self.connection.reqAccountUpdates(1, '') def reqAccountSummary(self, value): logging.info('TWS_gateway - reqAccountSummary value=%s' % value) vals = map( lambda x: x.encode('ascii') if isinstance(x, unicode) else x, json.loads(value)) self.connection.reqAccountSummary(vals[0], vals[1], vals[2]) def reqOpenOrders(self, value=None): self.connection.reqOpenOrders() def reqPositions(self, value=None): self.connection.reqPositions() def reqExecutions(self, value): try: filt = ExecutionFilter( ) if value == '' else ExecutionFilterHelper.kvstring2object( value, ExecutionFilter) self.connection.reqExecutions(0, filt) except: logging.error(traceback.format_exc()) def reqIds(self, value=None): self.connection.reqIds(1) def reqNewsBulletins(self): self.connection.reqNewsBulletins(1) def cancelNewsBulletins(self): self.connection.cancelNewsBulletins() def setServerLogLevel(self): self.connection.setServerLogLevel(3) def reqAutoOpenOrders(self): self.connection.reqAutoOpenOrders(1) def reqAllOpenOrders(self): self.connection.reqAllOpenOrders() def reqManagedAccts(self): self.connection.reqManagedAccts() def requestFA(self): self.connection.requestFA(1) def reqMktData(self, sm_contract): logging.info('TWS Gateway received reqMktData request: %s' % sm_contract) try: #self.contract_subscription_mgr.reqMktData(ContractHelper.kvstring2contract(sm_contract)) self.contract_subscription_mgr.reqMktData( ContractHelper.kvstring2object(sm_contract, Contract)) except: pass def reqHistoricalData(self): contract = Contract() contract.m_symbol = 'QQQQ' contract.m_secType = 'STK' contract.m_exchange = 'SMART' endtime = strftime('%Y%m%d %H:%M:%S') self.connection.reqHistoricalData(tickerId=1, contract=contract, endDateTime=endtime, durationStr='1 D', barSizeSetting='1 min', whatToShow='TRADES', useRTH=0, formatDate=1) def placeOrder(self, value=None): logging.info('TWS_gateway - placeOrder value=%s' % value) try: vals = json.loads(value) except ValueError: logging.error('TWS_gateway - placeOrder Exception %s' % traceback.format_exc()) return # c = ContractHelper.kvstring2contract(vals[1]) o = OrderHelper.kvstring2object(vals[2], Order) o.__dict__['transmit'] = self.ib_order_transmit # print c.__dict__ # print o.__dict__ # print '---------------------' #self.connection.placeOrder(vals[0], ContractHelper.kvstring2contract(vals[1]), OrderHelper.kvstring2object(vals[2], Order)) self.connection.placeOrder( vals[0], ContractHelper.kvstring2object(vals[1], Contract), OrderHelper.kvstring2object(vals[2], Order)) # self.connection.placeOrder(orderId, contract, newOptOrder) def eDisconnect(self, value=None): sleep(2) self.connection.eDisconnect() ####################################################################3 # Gateway commands def gw_req_subscriptions(self, value=None): #subm = map(lambda i: ContractHelper.contract2kvstring(self.contract_subscription_mgr.handle[i]), range(len(self.contract_subscription_mgr.handle))) #subm = map(lambda i: ContractHelper.object2kvstring(self.contract_subscription_mgr.handle[i]), range(len(self.contract_subscription_mgr.handle))) subm = map( lambda i: (i, ContractHelper.object2kvstring( self.contract_subscription_mgr.handle[i])), range(len(self.contract_subscription_mgr.handle))) print subm if subm: self.tws_event_handler.broadcast_event('gw_subscriptions', {'subscriptions': subm}, source='GW')