def make_local_time_generator(): """ :return: a datetime with timezone """ while True: yield epoch_to_dt(int(time.time()))
def _ingest_hists(self, histIngestionPlan, funcToFetchHist): """ Read in ingestion plan and use funcToFetchHist to fetch hist and Then, save them to self._hist. :param histIngestionPlan: data_provider_factor::data_loading_plan::HistIngestionPlan :param funcToFetchHist: the passed in function is responsible to convert the format to the required format :return: None """ self._log.debug(__name__ + '::_ingest_hists: loadingPlan=%s client=%s' % (histIngestionPlan, self._dataProviderClient)) if not histIngestionPlan: return for plan in histIngestionPlan.finalPlan: security_no_exchange_primaryExchange = stripe_exchange_primaryExchange_from_security(plan.security) str_security_no_exchange_primaryExchange = security_no_exchange_primaryExchange.full_print() barSize = plan.barSize.lower() if str_security_no_exchange_primaryExchange not in self._hist: self._hist[str_security_no_exchange_primaryExchange] = {} if barSize not in self._hist[str_security_no_exchange_primaryExchange]: self._hist[str_security_no_exchange_primaryExchange][barSize] = {} if plan.dataSourceName == DataSourceName.SIMULATED_BY_DAILY_BARS: self._simulatedByDailyBars.add((str_security_no_exchange_primaryExchange, barSize)) continue df_hist = funcToFetchHist(plan) if not len(df_hist): raise NotEnoughHist() self._hist[str_security_no_exchange_primaryExchange][barSize] = df_hist if histIngestionPlan.saveToFile: PROJECT_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) targetDir = os.path.join(PROJECT_ROOT_DIR, 'Input') if not os.path.exists(targetDir): os.mkdir(targetDir) targetFileFullName = os.path.join(targetDir, '%s_%s_%s.csv' % (plan.security, plan.barSize, plan.goBack)) if os.path.exists(targetFileFullName): print(__name__ + '::_ingest_hists: %s exists !!! Please delete it or rename it and try backtest again!' % (targetFileFullName,)) exit() df_hist.to_csv(targetFileFullName) self._log.info('Saved hist to %s' % (targetFileFullName,)) self._log.info('ingested hist of security=%s barSize=%s from %s' % (str_security_no_exchange_primaryExchange, barSize, self.name)) try: self._log.info('1st line=%s' % (epoch_to_dt(self._hist[str_security_no_exchange_primaryExchange][barSize].index[0], str_timezone='US/Eastern'))) self._log.info('last line=%s' % (epoch_to_dt(self._hist[str_security_no_exchange_primaryExchange][barSize].index[-1], str_timezone='US/Eastern'))) except ValueError: self._log.error('security=%s barSize=%s index=%s is not valid. The format of index should be epoch time. Refer to https://www.epochconverter.com/' % (str_security_no_exchange_primaryExchange, barSize, self._hist[str_security_no_exchange_primaryExchange][barSize].index[0])) self._log.error('If you need help on coding, please refer to the well known Rent-a-Coder service https://ibridgepy.com/rent-a-coder/') exit()
def reqHistoricalDataWrapper(self, reqId, contract, endTime, goBack, barSize, whatToShow, useRTH, formatDate): if barSize not in ['1 day', '5 mins', '10 mins']: self._log.error( __name__ + '::reqHistoricalDataWrapper: EXIT, Robinhood does not support barSize=%s' % (barSize, )) interval = BarSizeConverter().fromIBtoRB(barSize) span = GoBackConverter().fromIBtoRB(goBack) self._log.notset( __name__ + '::reqHistoricalDataWrapper: contract=%s interval=%s span=%s' % (print_IBCpp_contract(contract), interval, span)) ans = self._robinhoodClient.get_historical_quotes( contract.symbol, interval, span) hist = None try: hist = ans['results'][0]['historicals'] except KeyError: self._log.error(__name__ + '::reqHistoricalDataWrapper: EXIT, ans=%s' % (ans, )) exit() for row in hist: t = dt.datetime.strptime(row['begins_at'], "%Y-%m-%dT%H:%M:%SZ") epoc = dt_to_epoch(t) if formatDate == 2: date = epoc else: date = epoch_to_dt(float(epoc)) date = date.strftime( "%Y%m%d %H:%M:%S" ) # Must be UTC because requested time was cast to UTC self.simulateHistoricalData( reqId, str(date), float(row['open_price']), float(row['high_price']), float(row['low_price']), float(row['close_price']), int(float(row['volume'])), 1, # barCount 0.0, # WAP 1) # hasGap self.simulateHistoricalData(reqId, 'finished', 0.0, 0.0, 0.0, 0.0, 1, 1, 0.0, 1)
def _sendHistoricalData(self, hist, reqId, formatDate): self._log.debug(__name__ + '::_sendHistoricalData: reqId=%s formatDate=%s' % (str(reqId), str(formatDate))) for idx in hist.index: if formatDate == 1: # asked to return datetime in string date = epoch_to_dt(float(idx)) # int ---> datetime date = date.strftime("%Y%m%d %H:%M:%S") # Must be UTC because requested time was cast to UTC else: # asked to return UTC int date = str(idx) self.simulateHistoricalData(reqId, date, float(hist.loc[idx, 'open']), float(hist.loc[idx, 'high']), float(hist.loc[idx, 'low']), float(hist.loc[idx, 'close']), int(hist.loc[idx, 'volume']), 1, 0.0, 1) self.simulateHistoricalData(reqId, 'finished', 0.0, 0.0, 0.0, 0.0, 1, 1, 0.0, 1)
def currentTime(self, tm): """ IB C++ API call back function. Return system time in datetime instance constructed from Unix timestamp using the showTimeZone from MarketManager """ serverTime = epoch_to_dt(float(tm)) currentTime = self._timeGenerator.get_current_time() # When backtester is using timeGeneratorType = 'CUSTOM' customSpotTimeList = [], user may create a list of spot time # by pd.date_range(), which returns a list of pd.Timestamp. if isinstance(currentTime, pd.Timestamp): currentTime = currentTime.to_pydatetime() localServerTimeDiff = serverTime - currentTime self._log.debug( __name__ + '::currentTime: tm=%s localServerTimeDiff=%s brokerClientId=%s' % (serverTime, localServerTimeDiff, id(self))) self._timeGenerator.set_diffBetweenLocalAndServer(localServerTimeDiff) self._activeRequests.set_all_requests_of_a_reqType_to_a_status( 'reqCurrentTime', ReqAttr.Status.COMPLETED)
def historicalData(self, reqId, timeString, price_open, price_high, price_low, price_close, volume, barCount, WAP, hasGaps): """ call back function from IB C++ API return the historical data for requested security """ self._log.notset( __name__ + '::historicalData: reqId=%s timeString=%s volume=%s barCount=%s WAP=%s hasGap=%s' % (reqId, timeString, volume, barCount, WAP, hasGaps)) # for any reason, the reqId is not in the self.activeRequests, # just ignore it. because the callback historicalData must come from the previous request. if self._activeRequests.check_valid_reqId(reqId): aRequest = self._activeRequests.get_by_reqId_otherwise_exit(reqId) else: return if 'finished' in str(timeString): aRequest.status = ReqAttr.Status.COMPLETED else: aRequest.status = ReqAttr.Status.STARTED # !!! The followings are removed because formatData is forced to 2 in # broker_client_factory::BrokerClient::_send_req_to_server after V 5.8.1 # User can still choose to formatDate = 1, which means return dt.datetime for convenience # if aRequest.param['formatDate'] == 1: # if ' ' in timeString: # dateTime = dt.datetime.strptime(timeString, '%Y%m%d %H:%M:%S') # change string to datetime # else: # dateTime = dt.datetime.strptime(timeString, '%Y%m%d') # change string to datetime # dateTime = pytz.timezone('UTC').localize(dateTime) # else: # formatDate is UTC time in seconds, str type # # The format in which the incoming bars' date should be presented. Note that for day bars, only yyyyMMdd format is available. # if len(timeString) > 9: # return datetime, not date # dateTime = epoch_to_dt(float(timeString)) # else: # return date, not datetime # dateTime = dt.datetime.strptime(timeString, '%Y%m%d') # change string to datetime # dateTime = int(dt_to_epoch(dateTime)) # change to int type if len(timeString ) > 9: # the type of returned value is datetime, not date dateTime = epoch_to_dt(float(timeString)) else: dateTime = dt.datetime.strptime( timeString, '%Y%m%d') # change string to datetime dateTime = int(dt_to_epoch(dateTime)) # change to int type if aRequest.param['formatDate'] == 1: dateTime = epoch_to_dt(dateTime) # int -> datetime if 'day' in aRequest.param['barSize']: dateTime = dateTime.date() # datetime -> date else: dateTime = dateTime.astimezone( aRequest.param['timezoneOfReturn']) # Adjust timezone newRow = pd.DataFrame( { 'open': price_open, 'high': price_high, 'low': price_low, 'close': price_close, 'volume': volume }, index=[dateTime]) aRequest.returnedResult = aRequest.returnedResult.append(newRow)
def provide_historical_data_from_local_variable_hist(self, security, endTime, goBack, barSize, whatToShow, useRTH, formatDate): """ :param security: IBridgePy::quantopian::Security :param endTime: request's ending time with format yyyyMMdd HH:mm:ss {TMZ} ---from IB api doc :param goBack: 'x S' 'x D' 'x W' 'x M' 'x Y' :param barSize: string 1 sec, 5 secs, 15 secs, 30 secs, 1 min, 2 mins, 3 mins, 5 mins, 15 mins, 30 mins, 1 hour, 1 day :param whatToShow: :param useRTH: :param formatDate: :return: """ self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: endTime=%s goBack=%s barSize=%s' % (endTime, goBack, barSize)) if not self._histIngested: self._log.error(__name__ + '::provide_historical_data_from_local_variable_hist: EXIT, hist has not been ingested.') exit() # Read in self.hist to provide data and check errors security_no_exchange_primaryExchange = stripe_exchange_primaryExchange_from_security(security) securityFullPrint = security_no_exchange_primaryExchange.full_print() barSize = barSize.lower() if securityFullPrint not in self._hist: self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: hist of %s is not ingested.' % (securityFullPrint,)) raise NotEnoughHist() if barSize not in self._hist[securityFullPrint]: self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: hist of %s exists but barSize=%s is not ingested.' % (securityFullPrint, barSize)) # print(self._hist[securityFullPrint].keys()) raise NotEnoughHist() hist = self._hist[securityFullPrint][barSize] if not isinstance(hist, pd.DataFrame): self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: hist is empty') raise NotEnoughHist() endTime = adjust_endTime(endTime, barSize) endTimePosition = _search_index_location_in_hist(hist, endTime) startTimePosition = endTimePosition - convert_goBack_barSize(goBack, barSize) if startTimePosition >= len(hist) - 2: self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: EXIT, Not enough hist security=%s barSize=%s is provided to backtest.' % (securityFullPrint, barSize)) self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: first line in hist=%s' % (epoch_to_dt(hist.index[0]))) self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: last line in hist=%s' % (epoch_to_dt(hist.index[-1]))) self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: backtest at this spot time=%s' % (endTime,)) raise NotEnoughHist() if startTimePosition < endTimePosition: return hist.iloc[startTimePosition:endTimePosition + 1] else: self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: Incorrect endTime=%s or goBack=%s when barSize=%s' % (endTime, goBack, barSize)) self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: first line in hist=%s' % (epoch_to_dt(hist.index[0]))) self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: last line in hist=%s' % (epoch_to_dt(hist.index[-1]))) self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: Hint: Based on the 1st line and last line of the hist, Are hist data too short?') raise NotEnoughHist()