def _get_position(self, accountCode, security): """ Guarantee to return a PositionRecord even if there is no current position. :param accountCode: :param security: :return: Guarantee to return a PositionRecord """ adj_security = stripe_exchange_primaryExchange_from_security(security) self._log.debug(__name__ + '::_get_position: adj_security=%s' % (adj_security.full_print(),)) return self._singleTrader.get_position(self.brokerName, accountCode, adj_security)
def provide_historical_data_from_local_variable_hist(self, security, endTime, goBack, barSize, whatToShow, useRTH, formatDate): """ :param security: IBridgePy::quantopian::Security :param endTime: request's ending time with format yyyyMMdd HH:mm:ss {TMZ} ---from IB api doc :param goBack: 'x S' 'x D' 'x W' 'x M' 'x Y' :param barSize: string 1 sec, 5 secs, 15 secs, 30 secs, 1 min, 2 mins, 3 mins, 5 mins, 15 mins, 30 mins, 1 hour, 1 day :param whatToShow: :param useRTH: :param formatDate: :return: """ self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: endTime=%s goBack=%s barSize=%s' % (endTime, goBack, barSize)) if not self._histIngested: self._log.error(__name__ + '::provide_historical_data_from_local_variable_hist: EXIT, hist has not been ingested.') exit() # Read in self.hist to provide data and check errors security_no_exchange_primaryExchange = stripe_exchange_primaryExchange_from_security(security) securityFullPrint = security_no_exchange_primaryExchange.full_print() barSize = barSize.lower() if securityFullPrint not in self._hist: self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: hist of %s is not ingested.' % (securityFullPrint,)) raise NotEnoughHist() if barSize not in self._hist[securityFullPrint]: self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: hist of %s exists but barSize=%s is not ingested.' % (securityFullPrint, barSize)) # print(self._hist[securityFullPrint].keys()) raise NotEnoughHist() hist = self._hist[securityFullPrint][barSize] if not isinstance(hist, pd.DataFrame): self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: hist is empty') raise NotEnoughHist() endTime = adjust_endTime(endTime, barSize) endTimePosition = _search_index_location_in_hist(hist, endTime) startTimePosition = endTimePosition - convert_goBack_barSize(goBack, barSize) if startTimePosition >= len(hist) - 2: self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: EXIT, Not enough hist security=%s barSize=%s is provided to backtest.' % (securityFullPrint, barSize)) self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: first line in hist=%s' % (epoch_to_dt(hist.index[0]))) self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: last line in hist=%s' % (epoch_to_dt(hist.index[-1]))) self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: backtest at this spot time=%s' % (endTime,)) raise NotEnoughHist() if startTimePosition < endTimePosition: return hist.iloc[startTimePosition:endTimePosition + 1] else: self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: Incorrect endTime=%s or goBack=%s when barSize=%s' % (endTime, goBack, barSize)) self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: first line in hist=%s' % (epoch_to_dt(hist.index[0]))) self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: last line in hist=%s' % (epoch_to_dt(hist.index[-1]))) self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: Hint: Based on the 1st line and last line of the hist, Are hist data too short?') raise NotEnoughHist()
def _ingest_hists(self, histIngestionPlan, funcToFetchHist): """ Read in ingestion plan and use funcToFetchHist to fetch hist and Then, save them to self._hist. :param histIngestionPlan: data_provider_factor::data_loading_plan::HistIngestionPlan :param funcToFetchHist: the passed in function is responsible to convert the format to the required format :return: None """ self._log.debug(__name__ + '::_ingest_hists: loadingPlan=%s client=%s' % (histIngestionPlan, self._dataProviderClient)) if not histIngestionPlan: return for plan in histIngestionPlan.finalPlan: security_no_exchange_primaryExchange = stripe_exchange_primaryExchange_from_security(plan.security) str_security_no_exchange_primaryExchange = security_no_exchange_primaryExchange.full_print() barSize = plan.barSize.lower() if str_security_no_exchange_primaryExchange not in self._hist: self._hist[str_security_no_exchange_primaryExchange] = {} if barSize not in self._hist[str_security_no_exchange_primaryExchange]: self._hist[str_security_no_exchange_primaryExchange][barSize] = {} if plan.dataSourceName == DataSourceName.SIMULATED_BY_DAILY_BARS: self._simulatedByDailyBars.add((str_security_no_exchange_primaryExchange, barSize)) continue df_hist = funcToFetchHist(plan) if not len(df_hist): raise NotEnoughHist() self._hist[str_security_no_exchange_primaryExchange][barSize] = df_hist if histIngestionPlan.saveToFile: PROJECT_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) targetDir = os.path.join(PROJECT_ROOT_DIR, 'Input') if not os.path.exists(targetDir): os.mkdir(targetDir) targetFileFullName = os.path.join(targetDir, '%s_%s_%s.csv' % (plan.security, plan.barSize, plan.goBack)) if os.path.exists(targetFileFullName): print(__name__ + '::_ingest_hists: %s exists !!! Please delete it or rename it and try backtest again!' % (targetFileFullName,)) exit() df_hist.to_csv(targetFileFullName) self._log.info('Saved hist to %s' % (targetFileFullName,)) self._log.info('ingested hist of security=%s barSize=%s from %s' % (str_security_no_exchange_primaryExchange, barSize, self.name)) try: self._log.info('1st line=%s' % (epoch_to_dt(self._hist[str_security_no_exchange_primaryExchange][barSize].index[0], str_timezone='US/Eastern'))) self._log.info('last line=%s' % (epoch_to_dt(self._hist[str_security_no_exchange_primaryExchange][barSize].index[-1], str_timezone='US/Eastern'))) except ValueError: self._log.error('security=%s barSize=%s index=%s is not valid. The format of index should be epoch time. Refer to https://www.epochconverter.com/' % (str_security_no_exchange_primaryExchange, barSize, self._hist[str_security_no_exchange_primaryExchange][barSize].index[0])) self._log.error('If you need help on coding, please refer to the well known Rent-a-Coder service https://ibridgepy.com/rent-a-coder/') exit()
def _get_one_real_time_price_from_local_variable_hist( self, security, timeNow, tickType, freq='dummy'): # print('_get_one_real_time_price_from_local_variable_hist', security, timeNow, tickType) # Store the last value, be prepared that this function is fired twice at one timeNow security_no_exchange_primaryExchange = stripe_exchange_primaryExchange_from_security( security) str_security = security_no_exchange_primaryExchange.full_print() if (str_security, timeNow, tickType) in self._lastRealTimePrice: self._log.notset( __name__ + '::_get_one_real_time_price_from_local_variable_hist: str_security=%s timeNow=%s tickType=%s returnedValue=SavedValue' % (str_security, timeNow, tickType)) return self._lastRealTimePrice[(str_security, timeNow, tickType)] ans = -1 if tickType in [ IBCpp.TickType.ASK, IBCpp.TickType.BID, IBCpp.TickType.LAST, IBCpp.TickType.OPEN, IBCpp.TickType.HIGH, IBCpp.TickType.LOW, IBCpp.TickType.CLOSE ]: ans = roundToMinTick(random.uniform(50, 100)) elif tickType in [ IBCpp.TickType.VOLUME, IBCpp.TickType.BID_SIZE, IBCpp.TickType.ASK_SIZE ]: ans = roundToMinTick(random.uniform(10000, 50000), 1) else: self._log.error( __name__ + '::_get_one_real_time_price_from_local_variable_hist: EXIT, do not support tickType=%s' % (str(tickType), )) exit() self._lastRealTimePrice[(str_security, timeNow, tickType)] = ans if tickType == IBCpp.TickType.CLOSE: self._lastRealTimePrice[(str_security, timeNow, IBCpp.TickType.ASK)] = ans self._lastRealTimePrice[(str_security, timeNow, IBCpp.TickType.BID)] = ans self._log.notset( __name__ + '::_get_one_real_time_price_from_local_variable_hist: str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, timeNow, tickType, ans)) return ans
def _get_one_real_time_price_from_local_variable_hist(self, security, timeNow, tickType, freq='1 min'): """ Both of prices and volume will be provided. :param freq: :param security: :param timeNow: :param tickType: string ONLY :return: """ if not self._histIngested: self._log.error(__name__ + '::_get_one_real_time_price_from_local_variable_hist: EXIT, hist has not been ingested.') exit() fieldName = None if tickType in [IBCpp.TickType.ASK, IBCpp.TickType.BID, IBCpp.TickType.LAST, IBCpp.TickType.OPEN]: fieldName = 'open' elif tickType == IBCpp.TickType.HIGH: fieldName = 'high' elif tickType == IBCpp.TickType.LOW: fieldName = 'low' elif tickType == IBCpp.TickType.CLOSE: fieldName = 'close' elif tickType == IBCpp.TickType.VOLUME: fieldName = 'volume' else: self._log.error(__name__ + '::_get_one_real_time_price_from_local_variable_hist: EXIT, cannot handle tickType=%s' % (tickType,)) exit() security_no_exchange_primaryExchange = stripe_exchange_primaryExchange_from_security(security) str_security = security_no_exchange_primaryExchange.full_print() # IB only accepts str_endTime with a timezone of UTC. timeNow = timeNow.astimezone(pytz.utc) # If 1 minute bar is labeled as simulatedByDailyBars, it means to get daily bar that contains the timeNow and then, # use "close" price of the daily bar to simulate minute price. if (str_security, freq) in self._simulatedByDailyBars: int_timeNow = int(dt_to_epoch(timeNow)) hist = self._hist[str_security]['1 day'] if int_timeNow in hist.index: ans = hist.loc[int_timeNow, self._useColumnNameWhenSimulatedByDailyBar] self._log.debug(__name__ + '::_get_one_real_time_price_from_local_variable_hist: simulatedByDailyBars str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, int_timeNow, tickType, ans)) return ans else: # bisect_left is to find the loc and insert a row on the left of the loc. # Therefore, "-1" is needed to find the correct daily bar # for example, timeNow = 2020-12-23 15:59:00 -0500, then, timeNowPosition should be 2020-12-23 timeNowPosition = bisect.bisect_left(hist.index, int_timeNow) - 1 if timeNowPosition >= len(hist.index): timeNowPosition -= 1 ans = hist.iloc[timeNowPosition][self._useColumnNameWhenSimulatedByDailyBar] self._log.debug(__name__ + '::_get_one_real_time_price_from_local_variable_hist: simulatedByDailyBars str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, int_timeNow, tickType, ans)) return ans if str_security not in self._hist: str_timeNow = dt.datetime.strftime(timeNow, "%Y%m%d %H:%M:%S %Z") # datetime -> string if (security, str_timeNow) in self._1minHist: return self._1minHist[(security, str_timeNow)].iloc[-1][fieldName] if self.name in [DataProviderName.IB, DataProviderName.ROBINHOOD, DataProviderName.ROBINHOOD, DataProviderName.IBRIDGEPY]: self._log.info('Do not have hist for str_security=%s' % (str_security,)) if security.secType != 'CASH': return self._helper(security, str_timeNow, fieldName) else: return self._helper(security, str_timeNow, fieldName, 'ASK') else: self._log.error(__name__ + '::_get_one_real_time_price_from_local_variable_hist: EXIT, not enough hist %s, %s from dataProviderName=%s' % (security, str_timeNow, self.name,)) raise NotEnoughHist() if freq not in self._hist[str_security]: str_timeNow = dt.datetime.strftime(timeNow, "%Y%m%d %H:%M:%S %Z") # datetime -> string if (security, str_timeNow) in self._1minHist: return self._1minHist[(security, str_timeNow)].iloc[-1][fieldName] self._log.info(__name__ + '::_get_one_real_time_price_from_local_variable_hist: hist of %s does not have freq=%s' % (str_security, freq)) if security.secType != 'CASH': return self._helper(security, str_timeNow, fieldName) else: return self._helper(security, str_timeNow, fieldName, 'ASK') int_timeNow = int(dt_to_epoch(timeNow)) hist = self._hist[str_security][freq] if int_timeNow in hist.index: ans = hist.loc[int_timeNow, fieldName] self._log.debug(__name__ + '::_get_one_real_time_price_from_local_variable_hist: str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, int_timeNow, tickType, ans)) return ans else: # Solution: if timeNow is not in hist, then raise Exception. Maybe it is not a good idea # timeNow = epoch_to_dt(timeNow).astimezone(self.showTimeZone) # default is UTC # time1st = epoch_to_dt(self.hist[str_security][freq].index[0]).astimezone(self.showTimeZone) # timeLast = epoch_to_dt(self.hist[str_security][freq].index[-1]).astimezone(self.showTimeZone) # self._log.error(__name__ + '::get_one_real_time_prices: loaded hist does not have timeNow=%s' % (str(timeNow),)) # self._log.error(__name__ + '::get_one_real_time_prices: loaded hist of security=%s from %s to %s' # % (str_security, time1st, timeLast)) # raise AssertionError # AssertionError will be caught by broker_client_factory::BrokerClient_Local.py::processMessagesWrapper # Solution 2: look for the timeBar immediately before timeNow, and use its value. timeNowPosition = bisect.bisect_left(hist.index, int_timeNow) if timeNowPosition >= len(hist.index): timeNowPosition -= 1 ans = hist.iloc[timeNowPosition][fieldName] self._log.debug(__name__ + '::_get_one_real_time_price_from_local_variable_hist: str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, int_timeNow, tickType, ans)) return ans