class IBStore(with_metaclass(MetaSingleton, object)): '''Singleton class wrapping an ibpy ibConnection instance. The parameters can also be specified in the classes which use this store, like ``IBData`` and ``IBBroker`` Params: - ``host`` (default:``127.0.0.1``): where IB TWS or IB Gateway are actually running. And although this will usually be the localhost, it must not be - ``port`` (default: ``7496``): port to connect to. The demo system uses ``7497`` - ``clientId`` (default: ``None``): which clientId to use to connect to TWS. ``None``: generates a random id between 1 and 65535 An ``integer``: will be passed as the value to use. - ``notifyall`` (default: ``False``) If ``False`` only ``error`` messages will be sent to the ``notify_store`` methods of ``Cerebro`` and ``Strategy``. If ``True``, each and every message received from TWS will be notified - ``_debug`` (default: ``False``) Print all messages received from TWS to standard output - ``reconnect`` (default: ``3``) Number of attempts to try to reconnect after the 1st connection attempt fails Set it to a ``-1`` value to keep on reconnecting forever - ``timeout`` (default: ``3.0``) Time in seconds between reconnection attemps - ``timeoffset`` (default: ``True``) If True, the time obtained from ``reqCurrentTime`` (IB Server time) will be used to calculate the offset to localtime and this offset will be used for the price notifications (tickPrice events, for example for CASH markets) to modify the locally calculated timestamp. The time offset will propagate to other parts of the ``backtrader`` ecosystem like the **resampling** to align resampling timestamps using the calculated offset. - ``timerefresh`` (default: ``60.0``) Time in seconds: how often the time offset has to be refreshed - ``indcash`` (default: ``True``) Manage IND codes as if they were cash for price retrieval ''' # Set a base for the data requests (historical/realtime) to distinguish the # id in the error notifications from orders, where the basis (usually # starting at 1) is set by TWS REQIDBASE = 0x01000000 BrokerCls = None #getattr(sys.modules["cerebro.strategies." +classname.split('.')[0]], classname.split('.')[1])IBBroker #None # broker class will autoregister DataCls = None # data class will auto register params = ( ('host', '127.0.0.1'), ('port', 7496), ('clientId', None), # None generates a random clientid 1 -> 2^16 ('notifyall', False), # NOT IMPLEMENTED ('_debug', False), ('reconnect', 3), # -1 forever, 0 No, > 0 number of retries ('timeout', 3.0), # timeout between reconnections ('timeoffset', True), # Use offset to server for timestamps if needed ('timerefresh', 60.0), # How often to refresh the timeoffset ('indcash', True), # Treat IND codes as CASH elements ('readonly', False), # Set to True when IB API is in read-only mode ('account', ''), # Main account to receive updates for ) @classmethod def getdata(cls, *args, **kwargs): '''Returns ``DataCls`` with args, kwargs''' return cls.DataCls(*args, **kwargs) @classmethod def getbroker(cls, *args, **kwargs): '''Returns broker with *args, **kwargs from registered ``BrokerCls``''' return cls.BrokerCls(*args, **kwargs) def __init__(self): super(IBStore, self).__init__() self._env = None # reference to cerebro for general notifications self.broker = None # broker instance self.datas = list() # datas that have registered over start # self.ccount = 0 # requests to start (from cerebro or datas) # self._lock_tmoffset = threading.Lock() # self.tmoffset = timedelta() # to control time difference with server # # Structures to hold datas requests # self.qs = collections.OrderedDict() # key: tickerId -> queues # self.ts = collections.OrderedDict() # key: queue -> tickerId self.iscash = dict() # tickerIds from cash products (for ex: EUR.JPY) self.acc_cash = AutoDict() # current total cash per account self.acc_value = AutoDict() # current total value per account self.acc_upds = AutoDict() # current account valueinfos per account self.positions = collections.defaultdict(Position) # actual positions self.orderid = None # next possible orderid (will be itertools.count) self.managed_accounts = list() # received via managedAccounts self.notifs = queue.Queue() # store notifications for cerebro self.orders = collections.OrderedDict() # orders by order ided self.opending = collections.defaultdict(list) # pending transmission self.brackets = dict() # confirmed brackets self.last_tick = None # Use the provided clientId or a random one if self.p.clientId is None: self.clientId = random.randint(1, pow(2, 16) - 1) else: self.clientId = self.p.clientId if self.p.timeout is None: self.timeout = 2 else: self.timeout = self.p.timeout if self.p.readonly is None: self.readonly = False else: self.readonly = self.p.readonly if self.p.account is None: self.account = "" else: self.account = self.p.account if self.p._debug: util.logToConsole(level=logging.DEBUG) util.patchAsyncio() util.startLoop() self.ib = IB() self.ib.connect( host=self.p.host, port=self.p.port, clientId=self.clientId, timeout=self.timeout, readonly=self.readonly, account=self.account, ) # This utility key function transforms a barsize into a: # (Timeframe, Compression) tuple which can be sorted def keyfn(x): n, t = x.split() tf, comp = self._sizes[t] return (tf, int(n) * comp) # This utility key function transforms a duration into a: # (Timeframe, Compression) tuple which can be sorted def key2fn(x): n, d = x.split() tf = self._dur2tf[d] return (tf, int(n)) # Generate a table of reverse durations self.revdur = collections.defaultdict(list) # The table (dict) is a ONE to MANY relation of # duration -> barsizes # Here it is reversed to get a ONE to MANY relation of # barsize -> durations for duration, barsizes in self._durations.items(): for barsize in barsizes: self.revdur[keyfn(barsize)].append(duration) # Once managed, sort the durations according to real duration and not # to the text form using the utility key above for barsize in self.revdur: self.revdur[barsize].sort(key=key2fn) def start(self, data=None, broker=None): #self.reconnect(fromstart=True) # reconnect should be an invariant # Datas require some processing to kickstart data reception if data is not None: self._env = data._env # For datas simulate a queue with None to kickstart co self.datas.append(data) # if connection fails, get a fakeation that will force the # datas to try to reconnect or else bail out return self.getTickerQueue(start=True) elif broker is not None: self.broker = broker def stop(self): try: self.ib.disconnect() # disconnect should be an invariant except AttributeError: pass # conn may have never been connected and lack "disconnect" def get_notifications(self): '''Return the pending "store" notifications''' # The background thread could keep on adding notifications. The None # mark allows to identify which is the last notification to deliver self.notifs.put(None) # put a mark notifs = list() while True: notif = self.notifs.get() if notif is None: # mark is reached break notifs.append(notif) return notifs def managedAccounts(self): # 1st message in the stream self.managed_accounts = self.ib.managedAccounts() # Request time to avoid synchronization issues self.reqCurrentTime() def currentTime(self,msg): if not self.p.timeoffset: # only if requested ... apply timeoffset return curtime = datetime.fromtimestamp(float(msg.time)) with self._lock_tmoffset: self.tmoffset = curtime - datetime.now() threading.Timer(self.p.timerefresh, self.reqCurrentTime).start() def timeoffset(self): with self._lock_tmoffset: return self.tmoffset def reqCurrentTime(self): self.ib.reqCurrentTime() def nextOrderId(self): # Get the next ticker using a new request value from TWS self.orderid = self.ib.client.getReqId() return self.orderid def getTickerQueue(self, start=False): '''Creates ticker/Queue for data delivery to a data feed''' q = queue.Queue() if start: q.put(None) return q return q def getContractDetails(self, contract, maxcount=None): #cds = list() cds = self.ib.reqContractDetails(contract) #cds.append(cd) if not cds or (maxcount and len(cds) > maxcount): err = 'Ambiguous contract: none/multiple answers received' self.notifs.put((err, cds, {})) return None return cds def reqHistoricalDataEx(self, contract, enddate, begindate, timeframe, compression, what=None, useRTH=False, tz='', sessionend=None, #tickerId=None ): ''' Extension of the raw reqHistoricalData proxy, which takes two dates rather than a duration, barsize and date It uses the IB published valid duration/barsizes to make a mapping and spread a historical request over several historical requests if needed ''' # Keep a copy for error reporting purposes kwargs = locals().copy() kwargs.pop('self', None) # remove self, no need to report it if timeframe < TimeFrame.Seconds: # Ticks are not supported return self.getTickerQueue(start=True) if enddate is None: enddate = datetime.now() if begindate is None: duration = self.getmaxduration(timeframe, compression) if duration is None: err = ('No duration for historical data request for ' 'timeframe/compresison') self.notifs.put((err, (), kwargs)) return self.getTickerQueue(start=True) barsize = self.tfcomp_to_size(timeframe, compression) if barsize is None: err = ('No supported barsize for historical data request for ' 'timeframe/compresison') self.notifs.put((err, (), kwargs)) return self.getTickerQueue(start=True) return self.reqHistoricalData(contract=contract, enddate=enddate, duration=duration, barsize=barsize, what=what, useRTH=useRTH, tz=tz, sessionend=sessionend) # Check if the requested timeframe/compression is supported by IB durations = self.getdurations(timeframe, compression) # if not durations: # return a queue and put a None in it # return self.getTickerQueue(start=True) # Get or reuse a queue # if tickerId is None: # tickerId, q = self.getTickerQueue() # else: # tickerId, q = self.reuseQueue(tickerId) # reuse q for old tickerId # Get the best possible duration to reduce number of requests duration = None # for dur in durations: # intdate = self.dt_plus_duration(begindate, dur) # if intdate >= enddate: # intdate = enddate # duration = dur # begin -> end fits in single request # break intdate = begindate if duration is None: # no duration large enough to fit the request duration = durations[-1] # Store the calculated data # self.histexreq[tickerId] = dict( # contract=contract, enddate=enddate, begindate=intdate, # timeframe=timeframe, compression=compression, # what=what, useRTH=useRTH, tz=tz, sessionend=sessionend) barsize = self.tfcomp_to_size(timeframe, compression) if contract.secType in ['CASH', 'CFD']: #self.iscash[tickerId] = 1 # msg.field code if not what: what = 'BID' # default for cash unless otherwise specified elif contract.secType in ['IND'] and self.p.indcash: #self.iscash[tickerId] = 4 # msg.field code pass what = what or 'TRADES' q = self.getTickerQueue() histdata = self.ib.reqHistoricalData( contract, intdate.strftime('%Y%m%d %H:%M:%S') + ' GMT', duration, barsize, what, useRTH, 2) # dateformat 1 for string, 2 for unix time in seconds for msg in histdata: q.put(msg) return q def reqHistoricalData(self, contract, enddate, duration, barsize, what=None, useRTH=False, tz='', sessionend=None): '''Proxy to reqHistorical Data''' # get a ticker/queue for identification/data delivery q = self.getTickerQueue() if contract.secType in ['CASH', 'CFD']: #self.iscash[tickerId] = True if not what: what = 'BID' # TRADES doesn't work elif what == 'ASK': #self.iscash[tickerId] = 2 pass else: what = what or 'TRADES' # split barsize "x time", look in sizes for (tf, comp) get tf #tframe = self._sizes[barsize.split()[1]][0] # self.histfmt[tickerId] = tframe >= TimeFrame.Days # self.histsend[tickerId] = sessionend # self.histtz[tickerId] = tz histdata = self.ib.reqHistoricalData( contract, enddate.strftime('%Y%m%d %H:%M:%S') + ' GMT', duration, barsize, what, useRTH, 2) # dateformat 1 for string, 2 for unix time in seconds for msg in histdata: q.put(msg) return q def reqRealTimeBars(self, contract, useRTH=False, duration=5): '''Creates a request for (5 seconds) Real Time Bars Params: - contract: a ib.ext.Contract.Contract intance - useRTH: (default: False) passed to TWS - duration: (default: 5) passed to TWS Returns: - a Queue the client can wait on to receive a RTVolume instance ''' # get a ticker/queue for identification/data delivery q = self.getTickerQueue() rtb = self.ib.reqRealTimeBars(contract, duration, 'MIDPOINT', useRTH=useRTH) self.ib.sleep(duration) for bar in rtb: q.put(bar) return q def reqMktData(self, contract, what=None): '''Creates a MarketData subscription Params: - contract: a ib.ext.Contract.Contract intance Returns: - a Queue the client can wait on to receive a RTVolume instance ''' # get a ticker/queue for identification/data delivery q = self.getTickerQueue() ticks = '233' # request RTVOLUME tick delivered over tickString if contract.secType in ['CASH', 'CFD']: #self.iscash[tickerId] = True ticks = '' # cash markets do not get RTVOLUME if what == 'ASK': #self.iscash[tickerId] = 2 pass # q.put(None) # to kickstart backfilling # Can request 233 also for cash ... nothing will arrive md = MktData() q_ticks = queue.Queue() util.run(md.update_ticks(self.ib, contract, ticks, q_ticks)) while not q_ticks.empty(): ticker = q_ticks.get() for tick in ticker.ticks: # https://interactivebrokers.github.io/tws-api/tick_types.html if tick != self.last_tick: #last price #print(str(tick.time) +" >> " + str(tick.price)) self.last_tick = tick q.put(tick) return q # The _durations are meant to calculate the needed historical data to # perform backfilling at the start of a connetion or a connection is lost. # Using a timedelta as a key allows to quickly find out which bar size # bar size (values in the tuples int the dict) can be used. _durations = dict([ # 60 seconds - 1 min ('60 S', ('1 secs', '5 secs', '10 secs', '15 secs', '30 secs', '1 min')), # 120 seconds - 2 mins ('120 S', ('1 secs', '5 secs', '10 secs', '15 secs', '30 secs', '1 min', '2 mins')), # 180 seconds - 3 mins ('180 S', ('1 secs', '5 secs', '10 secs', '15 secs', '30 secs', '1 min', '2 mins', '3 mins')), # 300 seconds - 5 mins ('300 S', ('1 secs', '5 secs', '10 secs', '15 secs', '30 secs', '1 min', '2 mins', '3 mins', '5 mins')), # 600 seconds - 10 mins ('600 S', ('1 secs', '5 secs', '10 secs', '15 secs', '30 secs', '1 min', '2 mins', '3 mins', '5 mins', '10 mins')), # 900 seconds - 15 mins ('900 S', ('1 secs', '5 secs', '10 secs', '15 secs', '30 secs', '1 min', '2 mins', '3 mins', '5 mins', '10 mins', '15 mins')), # 1200 seconds - 20 mins ('1200 S', ('1 secs', '5 secs', '10 secs', '15 secs', '30 secs', '1 min', '2 mins', '3 mins', '5 mins', '10 mins', '15 mins', '20 mins')), # 1800 seconds - 30 mins ('1800 S', ('1 secs', '5 secs', '10 secs', '15 secs', '30 secs', '1 min', '2 mins', '3 mins', '5 mins', '10 mins', '15 mins', '20 mins', '30 mins')), # 3600 seconds - 1 hour ('3600 S', ('5 secs', '10 secs', '15 secs', '30 secs', '1 min', '2 mins', '3 mins', '5 mins', '10 mins', '15 mins', '20 mins', '30 mins', '1 hour')), # 7200 seconds - 2 hours ('7200 S', ('5 secs', '10 secs', '15 secs', '30 secs', '1 min', '2 mins', '3 mins', '5 mins', '10 mins', '15 mins', '20 mins', '30 mins', '1 hour', '2 hours')), # 10800 seconds - 3 hours ('10800 S', ('10 secs', '15 secs', '30 secs', '1 min', '2 mins', '3 mins', '5 mins', '10 mins', '15 mins', '20 mins', '30 mins', '1 hour', '2 hours', '3 hours')), # 14400 seconds - 4 hours ('14400 S', ('15 secs', '30 secs', '1 min', '2 mins', '3 mins', '5 mins', '10 mins', '15 mins', '20 mins', '30 mins', '1 hour', '2 hours', '3 hours', '4 hours')), # 28800 seconds - 8 hours ('28800 S', ('30 secs', '1 min', '2 mins', '3 mins', '5 mins', '10 mins', '15 mins', '20 mins', '30 mins', '1 hour', '2 hours', '3 hours', '4 hours', '8 hours')), # 1 days ('1 D', ('1 min', '2 mins', '3 mins', '5 mins', '10 mins', '15 mins', '20 mins', '30 mins', '1 hour', '2 hours', '3 hours', '4 hours', '8 hours', '1 day')), # 2 days ('2 D', ('2 mins', '3 mins', '5 mins', '10 mins', '15 mins', '20 mins', '30 mins', '1 hour', '2 hours', '3 hours', '4 hours', '8 hours', '1 day')), # 1 weeks ('1 W', ('3 mins', '5 mins', '10 mins', '15 mins', '20 mins', '30 mins', '1 hour', '2 hours', '3 hours', '4 hours', '8 hours', '1 day', '1 W')), # 2 weeks ('2 W', ('15 mins', '20 mins', '30 mins', '1 hour', '2 hours', '3 hours', '4 hours', '8 hours', '1 day', '1 W')), # 1 months ('1 M', ('30 mins', '1 hour', '2 hours', '3 hours', '4 hours', '8 hours', '1 day', '1 W', '1 M')), # 2+ months ('2 M', ('1 day', '1 W', '1 M')), ('3 M', ('1 day', '1 W', '1 M')), ('4 M', ('1 day', '1 W', '1 M')), ('5 M', ('1 day', '1 W', '1 M')), ('6 M', ('1 day', '1 W', '1 M')), ('7 M', ('1 day', '1 W', '1 M')), ('8 M', ('1 day', '1 W', '1 M')), ('9 M', ('1 day', '1 W', '1 M')), ('10 M', ('1 day', '1 W', '1 M')), ('11 M', ('1 day', '1 W', '1 M')), # 1+ years ('1 Y', ('1 day', '1 W', '1 M')), ]) # Sizes allow for quick translation from bar sizes above to actual # timeframes to make a comparison with the actual data _sizes = { 'secs': (TimeFrame.Seconds, 1), 'min': (TimeFrame.Minutes, 1), 'mins': (TimeFrame.Minutes, 1), 'hour': (TimeFrame.Minutes, 60), 'hours': (TimeFrame.Minutes, 60), 'day': (TimeFrame.Days, 1), 'W': (TimeFrame.Weeks, 1), 'M': (TimeFrame.Months, 1), } _dur2tf = { 'S': TimeFrame.Seconds, 'D': TimeFrame.Days, 'W': TimeFrame.Weeks, 'M': TimeFrame.Months, 'Y': TimeFrame.Years, } def getdurations(self, timeframe, compression): key = (timeframe, compression) if key not in self.revdur: return [] return self.revdur[key] def getmaxduration(self, timeframe, compression): key = (timeframe, compression) try: return self.revdur[key][-1] except (KeyError, IndexError): pass return None def tfcomp_to_size(self, timeframe, compression): if timeframe == TimeFrame.Months: return '{} M'.format(compression) if timeframe == TimeFrame.Weeks: return '{} W'.format(compression) if timeframe == TimeFrame.Days: if not compression % 7: return '{} W'.format(compression // 7) return '{} day'.format(compression) if timeframe == TimeFrame.Minutes: if not compression % 60: hours = compression // 60 return ('{} hour'.format(hours)) + ('s' * (hours > 1)) return ('{} min'.format(compression)) + ('s' * (compression > 1)) if timeframe == TimeFrame.Seconds: return '{} secs'.format(compression) # Microseconds or ticks return None def dt_plus_duration(self, dt, duration): size, dim = duration.split() size = int(size) if dim == 'S': return dt + timedelta(seconds=size) if dim == 'D': return dt + timedelta(days=size) if dim == 'W': return dt + timedelta(days=size * 7) if dim == 'M': month = dt.month - 1 + size # -1 to make it 0 based, readd below years, month = divmod(month, 12) return dt.replace(year=dt.year + years, month=month + 1) if dim == 'Y': return dt.replace(year=dt.year + size) return dt # could do nothing with it ... return it intact # def histduration(self, dt1, dt2): # # Given two dates calculates the smallest possible duration according # # to the table from the Historical Data API limitations provided by IB # # # # Seconds: 'x S' (x: [60, 120, 180, 300, 600, 900, 1200, 1800, 3600, # # 7200, 10800, 14400, 28800]) # # Days: 'x D' (x: [1, 2] # # Weeks: 'x W' (x: [1, 2]) # # Months: 'x M' (x: [1, 11]) # # Years: 'x Y' (x: [1]) # td = dt2 - dt1 # get a timedelta for calculations # # First: array of secs # tsecs = td.total_seconds() # secs = [60, 120, 180, 300, 600, 900, 1200, 1800, 3600, 7200, 10800, # 14400, 28800] # idxsec = bisect.bisect_left(secs, tsecs) # if idxsec < len(secs): # return '{} S'.format(secs[idxsec]) # tdextra = bool(td.seconds or td.microseconds) # over days/weeks # # Next: 1 or 2 days # days = td.days + tdextra # if td.days <= 2: # return '{} D'.format(days) # # Next: 1 or 2 weeks # weeks, d = divmod(td.days, 7) # weeks += bool(d or tdextra) # if weeks <= 2: # return '{} W'.format(weeks) # # Get references to dt components # y2, m2, d2 = dt2.year, dt2.month, dt2.day # y1, m1, d1 = dt1.year, dt1.month, dt2.day # H2, M2, S2, US2 = dt2.hour, dt2.minute, dt2.second, dt2.microsecond # H1, M1, S1, US1 = dt1.hour, dt1.minute, dt1.second, dt1.microsecond # # Next: 1 -> 11 months (11 incl) # months = (y2 * 12 + m2) - (y1 * 12 + m1) + ( # (d2, H2, M2, S2, US2) > (d1, H1, M1, S1, US1)) # if months <= 1: # months <= 11 # return '1 M' # return '{} M'.format(months) # elif months <= 11: # return '2 M' # cap at 2 months to keep the table clean # # Next: years # # y = y2 - y1 + (m2, d2, H2, M2, S2, US2) > (m1, d1, H1, M1, S1, US1) # # return '{} Y'.format(y) # return '1 Y' # to keep the table clean def makecontract(self, symbol, sectype, exch, curr, expiry='', strike=0.0, right='', mult=1): '''returns a contract from the parameters without check''' contract = Contract() contract.symbol = symbol contract.secType = sectype contract.exchange = exch if curr: contract.currency = curr if sectype in ['FUT', 'OPT', 'FOP']: contract.lastTradeDateOrContractMonth = expiry if sectype in ['OPT', 'FOP']: contract.strike = strike contract.right = right if mult: contract.multiplier = mult return contract def cancelOrder(self, orderid): '''Proxy to cancelOrder''' self.ib.cancelOrder(orderid) def placeOrder(self, orderid, contract, order): '''Proxy to placeOrder''' trade = self.ib.placeOrder(contract, order) while not trade.isDone(): self.ib.waitOnUpdate() return trade def reqTrades(self): '''Proxy to Trades''' return self.ib.trades() def reqPositions(self): '''Proxy to reqPositions''' return self.ib.reqPositions() def getposition(self, contract, clone=False): # Lock access to the position dicts. This is called from main thread # and updates could be happening in the background #with self._lock_pos: position = self.positions[contract.conId] if clone: return copy(position) return position def reqAccountUpdates(self, subscribe=True, account=None): '''Proxy to reqAccountUpdates If ``account`` is ``None``, wait for the ``managedAccounts`` message to set the account codes ''' if account is None: #self._event_managed_accounts.wait() self.managedAccounts() account = self.managed_accounts[0] #self.ib.reqAccountUpdates(subscribe, bytes(account)) self.updateAccountValue() def updateAccountValue(self): # Lock access to the dicts where values are updated. This happens in a # sub-thread and could kick it at anytime #with self._lock_accupd: #if self.connected(): ret = self.ib.accountValues() for msg in ret: try: value = float(msg.value) except ValueError: value = msg.value self.acc_upds[msg.account][msg.tag][msg.currency] = value if msg.tag == 'NetLiquidation': # NetLiquidationByCurrency and currency == 'BASE' is the same self.acc_value[msg.account] = value elif msg.tag == 'TotalCashBalance' and msg.currency == 'BASE': self.acc_cash[msg.account] = value def get_acc_values(self, account=None): '''Returns all account value infos sent by TWS during regular updates Waits for at least 1 successful download If ``account`` is ``None`` then a dictionary with accounts as keys will be returned containing all accounts If account is specified or the system has only 1 account the dictionary corresponding to that account is returned ''' # Wait for at least 1 account update download to have been finished # before the account infos can be returned to the calling client # if self.connected(): # self._event_accdownload.wait() # Lock access to acc_cash to avoid an event intefering #with self._updacclock: if account is None: # wait for the managedAccount Messages # if self.connected(): # self._event_managed_accounts.wait() if not self.managed_accounts: return self.acc_upds.copy() elif len(self.managed_accounts) > 1: return self.acc_upds.copy() # Only 1 account, fall through to return only 1 account = self.managed_accounts[0] try: return self.acc_upds[account].copy() except KeyError: pass return self.acc_upds.copy() def get_acc_value(self, account=None): '''Returns the net liquidation value sent by TWS during regular updates Waits for at least 1 successful download If ``account`` is ``None`` then a dictionary with accounts as keys will be returned containing all accounts If account is specified or the system has only 1 account the dictionary corresponding to that account is returned ''' # Wait for at least 1 account update download to have been finished # before the value can be returned to the calling client # if self.connected(): # self._event_accdownload.wait() # Lock access to acc_cash to avoid an event intefering #with self._lock_accupd: if account is None: # wait for the managedAccount Messages # if self.connected(): # self._event_managed_accounts.wait() if not self.managed_accounts: return float() elif len(self.managed_accounts) > 1: return sum(self.acc_value.values()) # Only 1 account, fall through to return only 1 account = self.managed_accounts[0] try: return self.acc_value[account] except KeyError: pass return float() def get_acc_cash(self, account=None): '''Returns the total cash value sent by TWS during regular updates Waits for at least 1 successful download If ``account`` is ``None`` then a dictionary with accounts as keys will be returned containing all accounts If account is specified or the system has only 1 account the dictionary corresponding to that account is returned ''' # Wait for at least 1 account update download to have been finished # before the cash can be returned to the calling client' # if self.connected(): # self._event_accdownload.wait() # result = [v for v in self.ib.accountValues() \ # if v.tag == 'TotalCashBalance' and v.currency == 'BASE'] # Lock access to acc_cash to avoid an event intefering #with self._lock_accupd: if account is None: #wait for the managedAccount Messages # if self.connected(): # self._event_managed_accounts.wait() if not self.managed_accounts: return float() elif len(self.managed_accounts) > 1: return sum(self.acc_cash.values()) # Only 1 account, fall through to return only 1 account = self.managed_accounts[0] try: return self.acc_cash[account] except KeyError: pass
class LiveTrading(MethodManager_): def __init__(self, strat_name, client_id, runtime_tm, debugging, manager_, heartbeat_q, traded_instr): self.strat_name = strat_name self.client_id = client_id self.runtime_tm = runtime_tm self.debugging = debugging self.manager_ = manager_ self.heartbeat_q = heartbeat_q self.traded_instr = traded_instr self.account_curr = ACCOUNT_CURR self.tax_rate_account_country = TAX_RATE_ACCOUNT_COUNTRY self.simult_reqs_interval = SIMULT_REQS_INTERVAL self.potential_mkt_data_lines_already_in_use = POTENTIAL_MKT_DATA_LINES_ALREADY_IN_USE self.ib = IB() # self.ib.setCallback('error', self.on_ib_error) self.ib.errorEvent += self.on_ib_error self.err_ = None self.report = Reporting(self) self.cash = CashManagement(debugging) self.hlprs = ClientSharedMemory() self.access_type = 'tws' if self.access_type == 'gateway': self.__port = 4001 elif self.access_type == 'tws': self.__port = 7497 self.__host = '127.0.0.1' self.ib.connect(self.__host, self.__port, clientId=self.client_id) self.portfolio = [] self.portfolio_instr = [] self.cnt_trades_per_instr_per_day = {} self.req_tracker = { 'total': 0, 'open_market_data_reqs': 0, 'open_market_data_lines': 0, 'hist_data_prior_time': None } self.start_cet = datetime.datetime.now() self.manager_[ 'active_mkt_data_lines'] = POTENTIAL_MKT_DATA_LINES_ALREADY_IN_USE self.funda_data_req_max = IB_PACING['funda_data_reqs']['reqs'] if self.debugging.get_meths: self.print_meths(str(self.__class__.__name__), dir(self)) _inst_ = self self.fin_ratios = FinRatios(_inst_) @staticmethod def us_tz_op(dt_obj): """ Converting time :param dt_obj: datetime obj :return: Eastern datetime """ # log_start = datetime.datetime.now() if not isinstance(dt_obj, datetime.datetime): raise TypeError if dt_obj.year == 1900: curr_year = datetime.datetime.now().year else: curr_year = dt_obj.year # http://www.webexhibits.org/daylightsaving/b2.html dst = { 2018: [[3, 11], [11, 4]], 2019: [[3, 10], [11, 3]], 2020: [[3, 8], [11, 1]] } window = dst[curr_year] #disclaimer kind_of_difference = HOUR_DAY_BOUNDARY_FAIL_CHECK if dt_obj.hour < kind_of_difference: raise Exception("determination difficult, try later in the day") dst_start = datetime.datetime(curr_year, window[0][0], window[0][1]) dst_end = datetime.datetime(curr_year, window[1][0], window[1][1]) if dt_obj > dst_start and dt_obj < dst_end: utc2est = -4 else: utc2est = -5 this_tz_from_uts = '+0100' if time.strftime("%z", time.gmtime()) != this_tz_from_uts: raise exceptions_.WrongTimeZone( 'Wrong time zone, code is fixed to Danish winter time, might have to be adjusted' ) cet2utc = -2 dt_obj_utc = dt_obj + datetime.timedelta( hours=cet2utc) #.replace(tzinfo = datetime.timezone.utc) dt_obj_est = dt_obj_utc + datetime.timedelta(hours=utc2est) # logging = rk_logging.Logging() # logging.log( # runtime_tm, log_start, str('' + ',' + sys._getframe().f_code.co_name) # ,dt_obj_utc # ,dt_obj_est # ) return dt_obj_est def _manipulated_time(self, cet_in): delta = cet_in - self.start_cet current_manipulated_time = datetime.datetime( self.start_cet.year, self.start_cet.month, self.start_cet.day, self.debugging.dummy_hour, self.debugging.dummy_minute, self.debugging.dummy_second) current_manipulated_time += delta return current_manipulated_time def buy(self, ticker, rank, size, order_type): log_start = datetime.datetime.now() log = logging_.Logging( self.runtime_tm, log_start, str(self.__class__.__name__ + ',' + sys._getframe().f_code.co_name)) self.hlprs.add_to_manager(self.strat_name, *log.monitor()) order_log_msg = None symbol = ticker.contract.symbol if self._ticker_len_n_type_check(ticker.domBids) > 1: offer_price = ticker.domBids[rank].price # offer_size = ticker.domBids[rank].size else: offer_price = ticker.domBids.price # offer_size = ticker.domBids.size cnt = 0 max_order_filled_cnts = int(ORDER_CANCEL_IF_NOT_FILLED_SECS / ORDER_FILLED_CHECKED_CYCLE_SECS) if self.debugging.dummy_data: order_success = True else: if order_type == 'MKT': order = MarketOrder('BUY', size) trade = self.ib.placeOrder(ticker.contract, order) while cnt < max_order_filled_cnts: # a bit complicated but it does actually make sense order_log_msg = trade.log self.ib.sleep(ORDER_FILLED_CHECKED_CYCLE_SECS) cnt += 1 if not trade.orderStatus.status != 'Filled': # PendingSubmit = 'PendingSubmit' # PendingCancel = 'PendingCancel' # PreSubmitted = 'PreSubmitted' # Submitted = 'Submitted' # ApiPending = 'ApiPending' # undocumented, can be returned from req(All)OpenOrders # ApiCancelled = 'ApiCancelled' # Cancelled = 'Cancelled' # Filled = 'Filled' # Inactive = 'Inactive' order_success = True break else: self.ib.cancelOrder(trade) # while not trade.isDone(): # print("not sure if this makes sense") # self.ib.waitOnUpdate() elif order_type == 'LMT': raise Exception("order type not defined") else: raise Exception("order type not defined") if order_success: if self.debugging.dummy_time: _now = self._manipulated_time(datetime.datetime.now()) else: _now = datetime.datetime.now() filled_tm = self.us_tz_op(_now) status = 0 self.portfolio.append([symbol, filled_tm, offer_price, size]) self.hlprs.add_to_manager(self.strat_name, 'portfolio', self.portfolio) self.portfolio_instr.append(symbol) # self.add_to_monitor('portfolio_instr', 'self.portfolio_instr') print('not sure if this works') self.cnt_trades_per_instr_per_day[ticker.contract.symbol] += 1 self.hlprs.add_to_manager(self.strat_name, 'cnt_trades_per_instr_per_day', self.cnt_trades_per_instr_per_day) self.heartbeat_q.put([ self.strat_name, 'cnt_trades_per_instr_per_day', self.cnt_trades_per_instr_per_day ]) self.hlprs.add_to_manager( self.strat_name, 'cap_usd', self.cash.available_funds(self.report.pnl(), self.account_curr)) else: status = 1 log.log(self.portfolio, self.portfolio_instr, self.cnt_trades_per_instr_per_day, order_log_msg) return status def sell(self, ticker, rank, order_type): log_start = datetime.datetime.now() log = logging_.Logging( self.runtime_tm, log_start, str(self.__class__.__name__ + ',' + sys._getframe().f_code.co_name)) self.hlprs.add_to_manager(self.strat_name, *log.monitor()) order_log_msg = None symbol = ticker.contract.symbol try: ix = self.portfolio_instr.index(symbol) except ValueError: status = -1 print( 'not quite sure why this error sometimes occurs and if this is maybe to early to leave?' ) return status # if isinstance(ticker.domBids, str) == True: # offer_price = ticker.domAsks.price # offer_size = ticker.domAsks.size # else: # offer_price = ticker.domAsks[rank].price # offer_size = ticker.domAsks[rank].size price_ix = 2 size_ix = 3 # held_price = self.held[ix][price_ix] held_size = self.portfolio[ix][size_ix] cnt = 0 max_order_filled_cnts = int(ORDER_CANCEL_IF_NOT_FILLED_SECS / ORDER_FILLED_CHECKED_CYCLE_SECS) if not self.debugging.dummy_data: if order_type == 'MKT': order = MarketOrder('SELL', held_size) # https://github.com/erdewit/ib_insync/blob/master/notebooks/ordering.ipynb trade = self.ib.placeOrder(ticker.contract, order) while cnt < max_order_filled_cnts: order_log_msg = trade.log self.ib.sleep(ORDER_FILLED_CHECKED_CYCLE_SECS) cnt += 1 if trade.orderStatus.status == 'Filled': order_success = True break self.ib.cancelOrder(trade) while not trade.isDone(): print("not sure if this makes sense") self.ib.waitOnUpdate() elif order_type == 'LMT': raise Exception("order type not defined") else: raise Exception("order type not defined") else: order_success = True order_success = True if order_success: status = 0 del self.portfolio[ix] self.hlprs.add_to_manager(self.strat_name, 'portfolio', self.portfolio) del self.portfolio_instr[ix] # self.add_to_monitor('portfolio_instr', self.portfolio_instr) print('not sure if this works') self.cnt_trades_per_instr_per_day[ticker.contract.symbol] += 1 self.hlprs.add_to_manager(self.strat_name, 'cnt_trades_per_instr_per_day', self.cnt_trades_per_instr_per_day) self.heartbeat_q.put([ self.strat_name, 'cnt_trades_per_instr_per_day', self.cnt_trades_per_instr_per_day ]) self.hlprs.add_to_manager( self.strat_name, 'cap_usd', self.cash.available_funds(self.report.pnl(), self.account_curr)) else: status = 1 log.log(self.portfolio, self.portfolio_instr, self.cnt_trades_per_instr_per_day, order_log_msg) return status @staticmethod def _ticker_len_n_type_check(bid_or_ask_li): # log_start = datetime.datetime.now() n_ticks = len(bid_or_ask_li) if isinstance(bid_or_ask_li, str) == False else 1 # log = rk_logging.Logging() # log.log( # self.runtime_tm, log_start, str('' + ',' + sys._getframe().f_code.co_name) # ) return n_ticks def req_handling(self, ticker, type_): log_start = datetime.datetime.now() log = logging_.Logging( self.runtime_tm, log_start, str(self.__class__.__name__ + ',' + sys._getframe().f_code.co_name)) self.hlprs.add_to_manager(self.strat_name, *log.monitor()) # unix_ts = int(time.time()) self.req_tracker['total'] += 1 self.req_tracker['open_market_data_reqs'] += 1 self.req_tracker['open_market_data_lines'] += 1 # status = -1 means immediately kill connections status = self.pacing_violations(type_) log.log() if status == -1: print('get connections here') # do sth return elif status == 0: return def req_handler(self, toggle): if toggle == 'increase': self.manager_['active_mkt_data_lines'] += 1 elif toggle == 'decrease': self.manager_['active_mkt_data_lines'] -= 1 else: raise Exception('not defined') def req_mkt_dpt_ticker_(self, contract_): _cnt_ = 0 status = 1 while _cnt_ < N_TRIES_IF_SIMULT_REQS: if self.manager_['active_mkt_data_lines'] < IB_PACING[ 'mkt_data_lines']: self.req_handler('increase') ticker = self.ib.reqMktDepth(contract_) self.ib.sleep( SIMULT_REQS_INTERVAL ) # wait here because ticker needs updateEvent might not be fast enough else: self.ib.sleep( SIMULT_REQS_INTERVAL ) # wait here because ticker needs updateEvent might not be fast enough _cnt_ += 1 continue if self.err_ != exceptions_.IbPacingError: status = 0 break return ticker, status def cancel_mkt_dpt_ticker_(self, contract_): self.ib.cancelMktDepth(contract_) self.req_handler('decrease') @staticmethod def _get_ib_pacing(): return IB_PACING def pacing_violations(self, type_): log_start = datetime.datetime.now() log = logging_.Logging( self.runtime_tm, log_start, str(self.__class__.__name__ + ',' + sys._getframe().f_code.co_name)) self.hlprs.add_to_manager(self.strat_name, *log.monitor()) # histData, mktDepth, scannerData # TODO: far from done here, continue at some point status = 0 if type_ == 'histData': if self.req_tracker[ 'total'] == 1: #first time is already incremented self.req_tracker['hist_data_prior_time'] = int(time.time()) status = 0 elif self.req_tracker['total'] > 1 and self.req_tracker[ 'open_market_data_lines'] < MARKET_DATA_LINES: status = 0 elif self.req_tracker['total'] > 1 \ and self.req_tracker['open_market_data_lines'] == MARKET_DATA_LINES \ and int(time.time()) - self.req_tracker['hist_data_prior_time'] >= IB_PACING['hist_data_similar']['secs']: print('CRITICAL: market data lines limit reached') status = -1 elif type_ == 'mktDepth': status = 0 elif type_ == 'scannerData': status = 0 log.log(status, self.req_tracker) self.hlprs.add_to_manager(self.strat_name, *log.monitor()) return status def check_network_requirements(self): # s = socket.socket() # #e.g. # address, port = '10.8.8.19', '53141' # address, port = '208.245.107.3', '4000' # try: # s.connect((address, port)) # return True # except socket.error: # return False # #or # rows = [] # lc = psutil.net_connections('inet') # for c in lc: # (ip, port) = c.laddr # 0.0.0.0 # if ip == '10.8.8.19': # or ip == '::' # if c.type == socket.SOCK_STREAM and c.status == psutil.CONN_LISTEN: # proto_s = 'tcp' # elif c.type == socket.SOCK_DGRAM: # proto_s = 'udp' # else: # continue # pid_s = str(c.pid) if c.pid else '(unknown)' # msg = 'PID {} is listening on port {}/{} for all IPs.' # msg = msg.format(pid_s, port, proto_s) # print(msg) status = True if not status: raise Exception('cannot connect to all necessary servers') # def on_ib_error(self, reqId, errorCode, errorString, errorSomething): """ https://groups.io/g/insync/topic/how_to_capture_error_trapped/7718294?p=,,,20,0,0,0::recentpostdate%2Fsticky,,,20,1,80,7718294 """ max_n_mkt_depth_reqs = 309 # ERROR:ib_insync.wrapper:Error 309, reqId 39: Max number (3) of market depth requests has been reached, contract: Stock(symbol='PETZ', exchange='ISLAND', currency='USD') if errorCode == max_n_mkt_depth_reqs: self.err_ = exceptions_.IbPacingError
class IbManager(object): log_file = 'ib_manager' def __init__(self, ip: str, port: int, client_id: int): self._ib = IB() self._ib_ip: str = ip self._ib_port: int = port self._client_id: int = client_id self._subscribed_mkt_contracts: List[str] = [] self._subscribed_mkt_depth_contracts: List[str] = [] self._log: Log = Log.create(Log.path(self.log_file)) self._logger = self._log.get_logger('ibmanager') self._recorder: Recorder = Recorder(self._log) self._market_recorder: MarketRecorder = MarketRecorder( self._ib, self._recorder) self._account_recorder: AccountRecorder = AccountRecorder( self._ib, self._recorder) self._keep_connection_task: asyncio.Task = None self._ib.connectedEvent += self.on_ib_connected self._ib.disconnectedEvent += self.on_ib_disconnected self._reconnect_flag: bool = False def on_ib_connected(self) -> None: self._logger.info('connected with ib') self._reconnect_flag = False self._recover_subscriptions() def on_ib_disconnected(self) -> None: self._logger.warning('disconnected with ib') self._reconnect_flag = True if self._keep_connection_task is None: self._keep_connection_task = asyncio.create_task(self._reconnect()) async def _reconnect(self) -> None: while self._reconnect_flag: await asyncio.sleep(20) self._logger.info('try to reconnect ib gateway') await self.initialize() self._keep_connection_task = None def _recover_subscriptions(self) -> None: for contract in self._subscribed_mkt_contracts: self._logger.info(f'recover subscribe {str(contract)}') self._ib.reqMktData(contract) for contract in self._subscribed_mkt_depth_contracts: self._logger.info(f'recover subscribe depth {str(contract)}') self._ib.reqMktDepth(contract) async def initialize(self): if self._ib.isConnected(): return try: await self._ib.connectAsync(self._ib_ip, self._ib_port, clientId=self._client_id) accounts = self._ib.managedAccounts() if len(accounts) > 0: self._account_recorder.update_account(accounts[0]) self.update_account() except Exception: pass def update_account(self): self._ib.reqAccountSummaryAsync() async def find_symbols(self, pattern: str) -> List[str]: symbols = await self._ib.reqMatchingSymbolsAsync(pattern) contracts = [symbol.contract.nonDefaults() for symbol in symbols] return contracts def make_contract(self, **kwargs) -> Contract: return Contract.create(**kwargs) def sub_market(self, contract: Contract) -> str: if contract in self._subscribed_mkt_contracts: return 'already subscribe {}'.format(str(contract)) self._subscribed_mkt_contracts.append(contract) self._ib.reqMktData(contract) return 'subscribe {} success'.format(str(contract)) def unsub_market(self, contract: Contract) -> str: if contract not in self._subscribed_mkt_contracts: return 'not ever subscribe {}'.format(str(contract)) self._subscribed_mkt_contracts.append(contract) self._ib.cancelMktData(contract) return 'unsubscribe {} success'.format(str(contract)) def sub_market_depth(self, contract: Contract) -> str: if contract in self._subscribed_mkt_depth_contracts: return 'already subscribe depth {}'.format(str(contract)) self._subscribed_mkt_depth_contracts.append(contract) self._ib.reqMktDepth(contract) return 'subscribe depth {} success'.format(str(contract)) def unsub_market_depth(self, contract: Contract) -> str: if contract not in self._subscribed_mkt_depth_contracts: return 'not ever subscribe depth {}'.format(str(contract)) self._subscribed_mkt_contracts.remove(contract) self._ib.cancelMktDepth(contract) return 'unsubscribe depth {} success'.format(str(contract)) def place_order(self, contract: Contract, side: str, size: int, price: float) -> str: trade = self._place_order(contract, side, size, price) return str(trade) def _place_order(self, contract: Contract, side: str, size: int, price: float) -> Trade: side = side.upper() if side not in ('SELL', 'BUY'): return [f'invalid order type: {side}'] price = float(f'{round(float(price), 3):.3f}') order = LimitOrder(side, size, price, tif='GTC') trade = self._ib.placeOrder(contract, order) return trade def cancel_order(self, order_id: int) -> str: order_id = int(order_id) order = Order(orderId=order_id) trade = self._ib.cancelOrder(order) return str(trade) async def orders(self) -> List[str]: orders = await self._ib.reqOpenOrdersAsync() return [str(order) for order in orders] def portfolio(self) -> List[str]: results = self._ib.portfolio() return [str(value) for value in results]
class Hsi(object): '''HTISEC to Web Trade''' _singleton = None ib = None def __new__(cls, *args, **kwargs): if not cls._singleton: cls._singleton = super(Hsi, cls).__new__(cls, *args, **kwargs) return cls._singleton def __init__(self): urllib3.disable_warnings() self.status = 0 self.error = 0 #允许不止损的交易单数 self.nostop = 0 #默认止损的点数 self.stop_point = 50 # 是否去除不对应不全的交易记录 self.is_clear = False def verify_error(self, html): '''检验是否有错误提示 ''' reg = re.search(self.reg['error'], html.encode('utf-8')) if reg: self.error = reg.group(1).decode('utf-8') return 1 else: return 0 # auto login def login_a(self): self.login(self.account['user'], self.account['pwd']) #log in # def login(self,user,pwd): # '''auto login for htisec''' # self.account={'user':user,'pwd':pwd} # self._s.cookies.clear() # try: # get1=self._s.get(self.url['login_jsp'] ,verify=False) # #print('Login Request Status:',get1.status_code) # jid1=get1.cookies['JSESSIONID'] # self._headers['Cookie']="JSESSIONID="+jid1 # login_data = {'login_id': user, 'pwd': pwd} # post1 = self._s.post(self.url['login_do'] , data=login_data,verify=False,timeout=5) # if self.verify_error(post1.text): # self.status=0 # print("Login Error:",self.error) # return 0 # self._jid=post1.cookies.values()[0] # self._headers['Cookie']="JSESSIONID="+self._jid # print("Login OK,JSESSIONID:", self._jid) # self.status=1 # self.login_time=time.time() # print("Login Time:",time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) # return self._jid # except Exception as e: # self.status=0 # print("Login Try_Error:",e) def login_1(self, user, pwd, clientId): ''' 登陆 ''' # self.account={'user':user,'pwd':pwd} # self._s.cookies.clear() # #login jsp # get1 = self._s.get(self.url['login_jsp'], verify=False) # # print('Login Request Status:',get1.status_code) # jid1 = get1.cookies['JSESSIONID'] # self._headers['Cookie'] = "JSESSIONID=" + jid1 # login_data = {'login_id': user, 'pwd': pwd} # post1 = self._s.post(self.url['login_do'], data=login_data, verify=False, timeout=20) # print(post1.status_code) # self.login1_post=post1.text # return post1.text try: if self.ib is None: self.ib = IB() # self.ib.connect('192.168.2.117', 7496, clientId=8, timeout=3) self.ib.connect(user, pwd, clientId=clientId, timeout=3) except Exception as exc: self.ib = None # raise Exception(exc) return self.ib def get_tradelist(self): """ 获取交易记录 """ if self.ib is None: return None # ib.trades() # ib.orders() # ib.fills() # util.startLoop() # hsi = Future? # hsi = Future('HSI', '201901') # ib.qualifyContracts(hsi) # hsi # ticker = ib.reqMktData(hsi) data = self.ib.fills() df = [[ i.execution.shares if i.execution.side == 'BOT' else -i.execution.shares, i.execution.price, str(i.execution.time + datetime.timedelta(hours=8))[:19], i.contract.localSymbol, i.commissionReport.commission, int(i.contract.multiplier), i.execution.orderRef ] for i in data] df = pd.DataFrame(df, columns=[ 'bs', 'price', 'time', 'code', 'sxf', 'hy', 'operationPeople' ]) df = df.sort_values('time') # 去除信息不全的单 if self.is_clear: ccs = [[i.contract.localSymbol, i.position] for i in self.ib.portfolio()] codes = defaultdict(int) for i in ccs: codes[i[0]] += i[1] _s = [] for code in set(df.code): k = df[df.code == code] ints = int(k.bs.sum() - codes[code]) try: for ind in k.index: ind_bs = k.iloc[ind]['bs'] if (ints < 0 and ind_bs < 0 and ind_bs >= ints) or (ints > 0 and ind_bs > 0 and ind_bs <= ints): ints -= ind_bs _s.append(ind) if ints == 0: break except: pass df = df.drop(_s) return df def get_gd(self): """ 获取当前挂单的记录 """ if self.ib is None: return [] data = self.ib.openTrades() gd = [[ i.contract.localSymbol, str(i.log[0].time + datetime.timedelta(hours=8))[:19], i.order.lmtPrice, i.order.action, i.order.totalQuantity if i.order.action == 'BUY' else -i.order.totalQuantity, i.order.orderType, i.order.ocaGroup, i.order.orderRef ] for i in data] gd = pd.DataFrame( gd, columns=['合约', '时间', '价格', '买卖', '数量', '订单类型', '分组名称', '订单发起人']) return gd def get_cc(self): """ 获取当前持仓的记录 """ if self.ib is None: return [] data = self.ib.portfolio() cc = [[ i.contract.conId, i.contract.localSymbol, i.contract.currency, i.contract.multiplier, i.position, i.marketPrice, i.account, i.contract.lastTradeDateOrContractMonth ] for i in data] cc = pd.DataFrame(cc, columns=[ '合约ID', '合约代码', '货币', '合约乘数', '数量', '价格', '账户', ' 合约过期时间' ]) return cc def get_lots(self, types): """ 获取当前持仓与挂单的占比 types: LMT, STP LMT """ cc = self.get_cc() gd = self.get_gd() res = defaultdict(list) # groupID = set() # for i in gd.values: # if i[6] not in groupID: # res[i[0]].append(i[4]) # groupID.add(i[6]) [res[i[0]].append(i[4]) for i in gd.values if i[5] == types] [res[i[1]].append(i[4]) for i in cc.values] res = {i: -sum(j) for i, j in res.items()} return res def _post_order(self): try: orderHtml = self._s.post(self.url['confirmBS'], data=self.orderdata, verify=False, timeout=5) if self.verify_out(orderHtml): self._post_order() #print("_Post_Order Status:",orderHtml.status_code) self.ptext = orderHtml.text forms = self.get_forms(orderHtml.text) forms['login_pwd'] = self.account['pwd'] #print("taken:",forms['token']) orderHtml = self._s.post(self.url['order'], data=forms, verify=False, timeout=5) print("Order Request Status:", orderHtml.status_code) if self.verify_error(orderHtml.text): self.status = 2 print("_Post_order MSG:", self.error) except Exception as e: print("System Error:", e) self.error = "There was an error in post_order" self.status = 2 def order_buy(self, price, qty=1, prod_code='HSI'): if not prod_code: prod_code = self.orderdata['prod_code'] self.orderdata['price'] = str(price) self.orderdata['qty'] = str(qty) self.orderdata['prod_code'] = prod_code self.orderdata['valid_type'] = '0' self.orderdata['buy_sell'] = 'B' self.orderdata['stop_type'] = '' self.orderdata['stop_price'] = '' self.orderdata['cond_text'] = '' self._post_order() def order_sell(self, price, qty=1, prod_code='HSI'): print('prod code:', prod_code) if not prod_code: prod_code = self.orderdata['prod_code'] self.orderdata['price'] = str(price) self.orderdata['qty'] = str(qty) self.orderdata['prod_code'] = prod_code self.orderdata['valid_type'] = '0' self.orderdata['buy_sell'] = 'S' self.orderdata['stop_type'] = '' self.orderdata['stop_price'] = '' self.orderdata['cond_text'] = '' self._post_order() def order_stopB(self, stop_price, qty, product, add=0): '''限价多单止损,跌破止损价->触发空单''' price = stop_price - add prod_code = product[0:3] prod_month = product[3:5] self.orderdata['prod_code'] = prod_code self.orderdata['contract_mth'] = prod_month self.orderdata['price'] = str(price) self.orderdata['qty'] = str(qty) self.orderdata['valid_type'] = '0' self.orderdata['buy_sell'] = 'S' self.orderdata['stop_type'] = 'L' self.orderdata['stop_price'] = str(stop_price) self.orderdata['cond_text'] = '' self._post_order() def order_stopS(self, stop_price, qty, product, add=0): '''限价空单止损,升破止损价->触发多单''' price = stop_price + add prod_code = product[0:3] prod_month = product[3:5] self.orderdata['prod_code'] = prod_code self.orderdata['contract_mth'] = prod_month self.orderdata['price'] = str(price) self.orderdata['qty'] = str(qty) self.orderdata['valid_type'] = '0' self.orderdata['buy_sell'] = 'B' self.orderdata['stop_type'] = 'L' self.orderdata['stop_price'] = str(stop_price) self.orderdata['cond_text'] = '' self._post_order() def get_forms(self, html): '''获得提交表单数据''' d = pq(html) s = d('INPUT') para = {} for i in s.items(): if i.attr('type') == 'hidden': para[i.attr('name')] = i.attr('value') return para #del all def order_delAll(self): """ 取消所有订单 """ self.ib.reqGlobalCancel() #del order ref def order_del(self, orderid): '''del order''' try: gethtml = self.get_url(self.url['orderdetail'] % orderid) forms = self.get_forms(gethtml) #print("Orderdetail Status:",gethtml.status_code) forms['login_pwd'] = self.account['pwd'] posthtml = self._s.post(self.url['cancelorder'], data=forms, verify=False, timeout=5) print("Cancel Order Status:", posthtml.status_code) if self.verify_error(posthtml.text): self.status = 2 print("Order_del MSG:", self.error) except Exception as e: print("Order del Try_Error:", e) def get_hold(self, hh): '''得到最新执仓''' dfList = self.get_tradelist(hh) df2 = dfList.groupby('product').trade_qty.sum() dictH = {x[0]: 0 for x in df2.iteritems()} dfH = pd.DataFrame(columns=[ 'refno', 'product', 'trade_price', 'trade_qty', 'order_time', 'filled_qty', 'price', 'trade_time' ]) for index, row in dfList.iterrows(): product = row['product'] trade_qty = row.trade_qty if (dictH[product] >= 0 and trade_qty > 0) or (dictH[product] <= 0 and trade_qty < 0): dfH = dfH.append(row) dictH[product] += trade_qty #print("go add %d %d" %(row.refno,trade_qty)) else: #print("go calc %d %d" %(row.refno,trade_qty)) dfH = self.calc_hold(dfH, row, dictH) #print("len dfH:",len(dfH),dictH) print(dictH) self.dictHold = dictH return dfH def calc_hold(self, dfHold, row1, dict1): '''for ge hold''' if not len(dfHold): return dfHold proc = row1['product'] calc_qty = row1.trade_qty # print("func hold record:%d close:%d" %(len(dfHold),calc_qty)) for index, row in dfHold[dfHold['product'] == proc].sort_index( ascending=False).iterrows(): hold_qty = row.trade_qty if abs(hold_qty) == abs(calc_qty): dict1[proc] += calc_qty dfHold.drop(row.name, axis=0, inplace=True) # print("a Proc:%s Hold: %s %d,Close:%s %d" %(proc,row.refno,hold_qty,row1.refno,calc_qty)) return dfHold elif abs(hold_qty) > abs(calc_qty): dict1[proc] += calc_qty dfHold.loc[row.name, 'trade_qty'] = hold_qty + calc_qty # print("b Proc:%s Hold: %s %d,Close:%s %d" %(proc,row.refno,hold_qty,row1.refno,calc_qty)) return dfHold elif abs(hold_qty) < abs(calc_qty): dfHold.drop(row.name, axis=0, inplace=True) dict1[proc] += -hold_qty calc_qty += hold_qty # print("c Proc:%s Hold: %s %d,Close:%s %d | calc:%d,hold:%d" # %(proc,row.refno,hold_qty,row1.refno,-hold_qty,calc_qty,dict1[proc])) if abs(calc_qty) > 0 and dict1[proc] == 0: row1.trade_qty = calc_qty dict1[proc] = calc_qty dfHold = dfHold.append(row1) # print("d go add %d %d %d" %(row1.refno,calc_qty,len(dfHold))) return dfHold def gostop(self): # self.login_a() orderlist = self.get_orderlist() if self.status == 999: print("没有正常反回数据!") return -1 #tradelist=self.get_tradelist(orderlist) holdlist = self.get_hold(orderlist) for hh in self.dictHold: self._gostop(orderlist, holdlist, hh) def _gostop(self, orderlist, holdlist, proc): '''auto stop''' nostop = self.nostop stop_point = self.stop_point hh = orderlist[orderlist['product'] == proc] holdlist = holdlist[holdlist['product'] == proc] holdlist = holdlist.sort_values("trade_time", ascending=False) s_stop = hh.loc[(hh.status == '等待中') & (hh.cond.str.find('SL>=') > -1), 'r_qty'].sum() b_stop = hh.loc[(hh.status == '等待中') & (hh.cond.str.find('SL<=') > -1), 'r_qty'].sum() holdQty = self.dictHold[proc] if holdQty < 0: add_lots = -holdQty - s_stop - nostop else: add_lots = holdQty + b_stop - nostop print("%s@%d hold stop:%d@SL>=price,%d@SL<=price,Add Stop---%d@price" % (proc, holdQty, s_stop, b_stop, add_lots)) if holdQty == 0 or add_lots <= 0: return cnt = 0 for index, row in holdlist.iterrows(): qty = row['trade_qty'] price = row['trade_price'] if qty < 0 and cnt <= add_lots: print("%s:%d@SL>=%d" % (proc, qty, price + stop_point)) self.order_stopS(price + stop_point, qty=int(-qty), product=proc, add=0) elif qty > 0 and cnt <= add_lots: print("%s:%d@SL<=%d" % (proc, qty, price - stop_point)) self.order_stopB(price - stop_point, qty=int(qty), product=proc, add=0) cnt += abs(qty) if cnt >= add_lots: return def get_prod(self, txt): '''Get the New Product List''' r1 = re.search(self.reg['prodNameF'], txt) #print(r1) self.prodNameF = eval(r1.group(1)) r2 = re.search(self.reg['prodMonthF'], txt) self.prodMonthF = eval(r2.group(1)) #print(r2) #del 多余的止损 def del_stop(self): """ 删除多余的止损单 """ data = self.ib.openOrders() [ self.ib.cancelOrder(data[j]) for j, i in enumerate(data) if 'STP' in i.orderType ] # del指定品种多余的止损单 def _delstop(self, hh, proc): nostop = self.nostop stoplist = hh.loc[(hh['product'] == proc) & (hh.cond.str.find("SL") > -1) & (hh.status == '等待中')] stop_S = stoplist.loc[stoplist.r_qty > 0, 'r_qty'].sum() stop_B = stoplist.loc[stoplist.r_qty < 0, 'r_qty'].sum() if proc in self.dictHold: holdQty = self.dictHold[proc] stopQty = holdQty - nostop if holdQty >= nostop else 0 if holdQty > 0: delQty = abs(stop_B) - stopQty self.del_stopB(delQty, stoplist) self.del_stopAll('S', stoplist) elif holdQty < 0: delQty = stop_S - stopQty self.del_stopS(delQty, stoplist) self.del_stopAll('B', stoplist) else: self.del_stopAll('B', stoplist) self.del_stopAll('S', stoplist) else: self.del_stopAll('B', stoplist) self.del_stopAll('S', stoplist) #del stop order def del_stopB(self, delqty, hh): cont = "SL<=" delpd = hh.loc[(hh.status == '等待中') & (hh.cond.str.find(cont) > -1)].copy() if delqty <= 0 or not len(delpd): return delpd = delpd.sort_index(ascending=True) for index, rows in delpd.iterrows(): if rows.sqty <= delqty: print("del order %s refno:%d %d" % (rows['product'], rows.refno, rows.sqty)) self.order_del(rows.refno) elif rows.sqty > delqty: self.order_del(rows.refno) print("del order refno:", rows.refno, delqty) stop_price = int(rows.cond[4:].replace(',', '')) qty = rows.sqty - delqty product = rows['product'] add = stop_price - rows.price self.order_stopB(stop_price, qty, product, add=add) print('add stopB:stop_price=%d,qty=%d,product=%s,add=%d' % (stop_price, qty, product, add)) delqty = delqty - rows.sqty if delqty <= 0: break #del stop order def del_stopS(self, delqty, hh): cont = "SL>=" delpd = hh.loc[(hh.status == '等待中') & (hh.cond.str.find(cont) > -1)].copy() if delqty <= 0 or not len(delpd): return delpd = delpd.sort_index(ascending=True) for index, rows in delpd.iterrows(): if rows.bqty <= delqty: print("del order refno:", rows.refno, rows.bqty) self.order_del(rows.refno) elif rows.bqty > delqty: self.order_del(rows.refno) print("del order refno:", rows.refno, delqty) stop_price = int(rows.cond[4:].replace(',', '')) qty = rows.bqty - delqty product = rows['product'] add = rows.price - stop_price self.order_stopS(stop_price, qty, product, add=add) print('add stopB:stop_price=%d,qty=%d,product=%s,add=%d' % (stop_price, qty, product, add)) delqty = delqty - rows.bqty if delqty <= 0: break # del stop all def del_stopAll(self, buy_sell): data = self.ib.openOrders() if buy_sell == 'B': cont = 'SELL' else: cont = 'BUY' [ self.ib.cancelOrder(data[j]) for j, i in enumerate(data) if i.action == cont and 'STP' in i.orderType ]
class trade_ES(): def __init__(self): self.ib = IB() self.ib.connect('127.0.0.1', 7497, clientId=np.random.randint(10, 1000)) self.tickers_ret = {} self.endDateTime = '' self.No_days = '43200 S' self.interval = '30 secs' self.tickers_signal = "Hold" self.ES = Future(symbol='ES', lastTradeDateOrContractMonth='20200619', exchange='GLOBEX', currency='USD') self.ib.qualifyContracts(self.ES) self.ES_df = self.ib.reqHistoricalData(contract=self.ES, endDateTime=self.endDateTime, durationStr=self.No_days, barSizeSetting=self.interval, whatToShow='TRADES', useRTH=False, keepUpToDate=True) self.tickers_ret = [] self.options_ret = [] self.option = {'call': FuturesOption, 'put': FuturesOption} self.options_history = {} self.trade_options = {'call': [], 'put': []} self.price = 0 self.i = -1 self.ES_df.updateEvent += self.make_clean_df self.Buy = True self.Sell = False self.ib.positionEvent += self.order_verify self.waitTimeInSeconds = 120 self.tradeTime = 0 def run(self): while self.ib.waitOnUpdate(): util.allowCtrlC() self.ib.setCallback('error', x.checkError) self.make_clean_df(self.ES_df) def next_exp_weekday(self): weekdays = {2: [6, 0], 4: [0, 1, 2], 0: [3, 4]} today = datetime.date.today().weekday() for exp, day in weekdays.items(): if today in day: return exp def next_weekday(self, d, weekday): days_ahead = weekday - d.weekday() if days_ahead <= 0: # Target day already happened this week days_ahead += 7 date_to_return = d + datetime.timedelta(days_ahead) # 0 = Monday, 1=Tuself.ESday, 2=Wednself.ESday... return date_to_return.strftime('%Y%m%d') def get_strikes_and_expiration(self): expiration = self.next_weekday(datetime.date.today(), self.next_exp_weekday()) chains = self.ib.reqSecDefOptParams(underlyingSymbol='ES', futFopExchange='GLOBEX', underlyingSecType='FUT', underlyingConId=self.ES.conId) chain = util.df(chains) strikes = chain[chain['expirations'].astype(str).str.contains(expiration)].loc[:, 'strikes'].values[0] [ESValue] = self.ib.reqTickers(self.ES) ES_price = ESValue.marketPrice() strikes = [strike for strike in strikes if strike % 5 == 0 and ES_price - 10 < strike < ES_price + 10] return strikes, expiration def get_contract(self, right, net_liquidation): strikes, expiration = self.get_strikes_and_expiration() for strike in strikes: contract = FuturesOption(symbol='ES', lastTradeDateOrContractMonth=expiration, strike=strike, right=right, exchange='GLOBEX') self.ib.qualifyContracts(contract) self.price = self.ib.reqMktData(contract, "", False, False) if float(self.price.last) * 50 >= net_liquidation: continue else: return contract def make_clean_df(self, ES_df, hashbar=None): ES_df = util.df(ES_df) ES_df['RSI'] = ta.RSI(ES_df['close']) ES_df['macd'], ES_df['macdsignal'], ES_df['macdhist'] = ta.MACD(ES_df['close'], fastperiod=12, slowperiod=26, signalperiod=9) ES_df['MA_9'] = ta.MA(ES_df['close'], timeperiod=9) ES_df['MA_21'] = ta.MA(ES_df['close'], timeperiod=21) ES_df['MA_200'] = ta.MA(ES_df['close'], timeperiod=200) ES_df['EMA_9'] = ta.EMA(ES_df['close'], timeperiod=9) ES_df['EMA_21'] = ta.EMA(ES_df['close'], timeperiod=21) ES_df['EMA_200'] = ta.EMA(ES_df['close'], timeperiod=200) ES_df['ATR'] = ta.ATR(ES_df['high'], ES_df['low'], ES_df['close']) ES_df['roll_max_cp'] = ES_df['high'].rolling(20).max() ES_df['roll_min_cp'] = ES_df['low'].rolling(20).min() ES_df['roll_max_vol'] = ES_df['volume'].rolling(20).max() ES_df.dropna(inplace=True) self.loop_function(ES_df) def placeOrder(self, contract, order): trade = self.ib.placeOrder(contract, order) tradeTime = datetime.datetime.now() return([trade, contract, tradeTime]) def sell(self, contract, position): self.ib.qualifyContracts(contract) if position.position>0: order = 'Sell' else: order = 'Buy' marketorder = MarketOrder(order, abs(position.position)) if self.tradeTime!=0: timeDelta = datetime.datetime.now() - self.tradeTime if timeDelta.seconds > self.waitTimeInSeconds: marketTrade, contract, self.tradeTime = self.placeOrder(contract, marketorder) else: marketTrade, contract, tradeTime = self.placeOrder(contract, marketorder) condition = marketTrade.isDone timeout = 20 for c in self.ib.loopUntil(condition=condition, timeout=timeout): marketorder = MarketOrder('Sell', position.position) marketTrade = self.ib.placeOrder(contract, marketorder) if not condition == 'Filled': self.ib.cancelOrder(marketorder) marketorder = MarketOrder('Sell', position.position) marketTrade = self.ib.placeOrder(contract, marketorder) def buy(self, contract): self.ib.qualifyContracts(contract) marketorder = MarketOrder('Buy', 1) if self.tradeTime!=0: timeDelta = datetime.datetime.now() - self.tradeTime if timeDelta.seconds > self.waitTimeInSeconds: marketTrade, contract, self.tradeTime = self.placeOrder(contract, marketorder) else: marketTrade, contract, tradeTime = self.placeOrder(contract, marketorder) condition = marketTrade.isDone timeout = 10 for c in self.ib.loopUntil(condition=condition, timeout=timeout): marketorder = MarketOrder('Buy', 1) marketTrade = self.ib.placeOrder(contract, marketorder) if not condition == 'Filled': self.ib.cancelOrder(marketorder) marketorder = MarketOrder('Buy', 1) marketTrade = self.ib.placeOrder(contract, marketorder) def order_verify(self, order): if order.position == 0.0 or order.position < 0: self.Buy= True self.Sell= False elif order.position > 0: self.Buy = False self.Sell = True else: self.Buy = False self.Sell = False print(f'Buy= {self.Buy}, sell = {self.Sell}') def loop_function(self, ES_df): df = ES_df[ ['high', 'low', 'volume', 'close', 'RSI', 'ATR', 'roll_max_cp', 'roll_min_cp', 'roll_max_vol', 'EMA_9', 'EMA_21', 'macd', 'macdsignal']] if self.tickers_signal == "Hold": print('Hold') if df["high"].iloc[self.i] >= df["roll_max_cp"].iloc[self.i] and \ df["volume"].iloc[self.i] > df["roll_max_vol"].iloc[self.i - 1] and df['RSI'].iloc[self.i] > 30 \ and df['macd'].iloc[self.i] > df['macdsignal'].iloc[self.i] : self.tickers_signal = "Buy" return elif df["low"].iloc[self.i] <= df["roll_min_cp"].iloc[self.i] and \ df["volume"].iloc[self.i] > df["roll_max_vol"].iloc[self.i - 1] and df['RSI'].iloc[self.i] < 70 \ and df['macd'].iloc[self.i] < df['macdsignal'].iloc[self.i]: self.tickers_signal = "Sell" return elif self.tickers_signal == "Buy": print('BUY SIGNAL') if df["close"].iloc[self.i] > df["close"].iloc[self.i - 1] - (0.75 * df["ATR"].iloc[self.i - 1]) and self.Sell and not self.Buy: self.tickers_signal = "Hold" positions = self.ib.positions() for position in positions: if position.contract.right == 'C': self.sell(position.contract, position) return elif df["low"].iloc[self.i] <= df["roll_min_cp"].iloc[self.i] and \ df["volume"].iloc[self.i] > df["roll_max_vol"].iloc[self.i - 1] and df['RSI'].iloc[self.i] < 70 \ and df['macd'].iloc[self.i] < df['macdsignal'].iloc[self.i] and self.Sell and not self.Buy: self.tickers_signal = "Sell" positions = self.ib.positions() for position in positions: if position.contract.right == 'C': self.sell(position.contract, position) return self.option['put'] = self.get_contract(right="P", net_liquidation=2000) self.buy(self.option['put']) elif not self.Sell and self.Buy: self.option['call'] = self.get_contract(right="C", net_liquidation=2000) self.buy(self.option['call']) elif self.tickers_signal == "Sell": print('SELL SIGNAL') if df["close"].iloc[self.i] < df["close"].iloc[self.i - 1] + (0.75 * df["ATR"].iloc[self.i - 1]) and self.Sell and not self.Buy: self.tickers_signal = "Hold" positions = self.ib.positions() for position in positions: if position.contract.right == 'P': self.sell(position.contract, position) return elif df["high"].iloc[self.i] >= df["roll_max_cp"].iloc[self.i] and \ df["volume"].iloc[self.i] > df["roll_max_vol"].iloc[self.i - 1] and df['RSI'].iloc[self.i] > 30 \ and df['macd'].iloc[self.i] > df['macdsignal'].iloc[self.i] and self.Sell and not self.Buy: self.tickers_signal = "Buy" positions = self.ib.positions() for position in positions: if position.contract.right == 'P': self.sell(position.contract, position) return self.option['call'] = self.get_contract(right="C", net_liquidation=2000) self.buy(self.option['call']) elif not self.Sell and self.Buy: self.option['put'] = self.get_contract(right="P", net_liquidation=2000) self.buy(self.option['put']) def checkError(self, errCode, errString): print('Error Callback', errCode, errString) if errCode == 2104: print('re-connect after 5 secs') self.ib.sleep(5) self.ib.disconnect() self.ib.connect('127.0.0.1', 7497, clientId=np.random.randint(10, 1000)) self.make_clean_df(self.ES)