def make_local_time_generator():
"""
:return: a datetime with timezone
"""
while True:
yield epoch_to_dt(int(time.time()))
def _ingest_hists(self, histIngestionPlan, funcToFetchHist):
"""
Read in ingestion plan and use funcToFetchHist to fetch hist and Then, save them to self._hist.
:param histIngestionPlan: data_provider_factor::data_loading_plan::HistIngestionPlan
:param funcToFetchHist: the passed in function is responsible to convert the format to the required format
:return: None
"""
self._log.debug(__name__ + '::_ingest_hists: loadingPlan=%s client=%s' % (histIngestionPlan, self._dataProviderClient))
if not histIngestionPlan:
return
for plan in histIngestionPlan.finalPlan:
security_no_exchange_primaryExchange = stripe_exchange_primaryExchange_from_security(plan.security)
str_security_no_exchange_primaryExchange = security_no_exchange_primaryExchange.full_print()
barSize = plan.barSize.lower()
if str_security_no_exchange_primaryExchange not in self._hist:
self._hist[str_security_no_exchange_primaryExchange] = {}
if barSize not in self._hist[str_security_no_exchange_primaryExchange]:
self._hist[str_security_no_exchange_primaryExchange][barSize] = {}
if plan.dataSourceName == DataSourceName.SIMULATED_BY_DAILY_BARS:
self._simulatedByDailyBars.add((str_security_no_exchange_primaryExchange, barSize))
continue
df_hist = funcToFetchHist(plan)
if not len(df_hist):
raise NotEnoughHist()
self._hist[str_security_no_exchange_primaryExchange][barSize] = df_hist
if histIngestionPlan.saveToFile:
PROJECT_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
targetDir = os.path.join(PROJECT_ROOT_DIR, 'Input')
if not os.path.exists(targetDir):
os.mkdir(targetDir)
targetFileFullName = os.path.join(targetDir, '%s_%s_%s.csv' % (plan.security, plan.barSize, plan.goBack))
if os.path.exists(targetFileFullName):
print(__name__ + '::_ingest_hists: %s exists !!! Please delete it or rename it and try backtest again!' % (targetFileFullName,))
exit()
df_hist.to_csv(targetFileFullName)
self._log.info('Saved hist to %s' % (targetFileFullName,))
self._log.info('ingested hist of security=%s barSize=%s from %s' % (str_security_no_exchange_primaryExchange, barSize, self.name))
try:
self._log.info('1st line=%s' % (epoch_to_dt(self._hist[str_security_no_exchange_primaryExchange][barSize].index[0], str_timezone='US/Eastern')))
self._log.info('last line=%s' % (epoch_to_dt(self._hist[str_security_no_exchange_primaryExchange][barSize].index[-1], str_timezone='US/Eastern')))
except ValueError:
self._log.error('security=%s barSize=%s index=%s is not valid. The format of index should be epoch time. Refer to https://www.epochconverter.com/' % (str_security_no_exchange_primaryExchange, barSize, self._hist[str_security_no_exchange_primaryExchange][barSize].index[0]))
self._log.error('If you need help on coding, please refer to the well known Rent-a-Coder service https://ibridgepy.com/rent-a-coder/')
exit()
def reqHistoricalDataWrapper(self, reqId, contract, endTime, goBack,
barSize, whatToShow, useRTH, formatDate):
if barSize not in ['1 day', '5 mins', '10 mins']:
self._log.error(
__name__ +
'::reqHistoricalDataWrapper: EXIT, Robinhood does not support barSize=%s'
% (barSize, ))
interval = BarSizeConverter().fromIBtoRB(barSize)
span = GoBackConverter().fromIBtoRB(goBack)
self._log.notset(
__name__ +
'::reqHistoricalDataWrapper: contract=%s interval=%s span=%s' %
(print_IBCpp_contract(contract), interval, span))
ans = self._robinhoodClient.get_historical_quotes(
contract.symbol, interval, span)
hist = None
try:
hist = ans['results'][0]['historicals']
except KeyError:
self._log.error(__name__ +
'::reqHistoricalDataWrapper: EXIT, ans=%s' %
(ans, ))
exit()
for row in hist:
t = dt.datetime.strptime(row['begins_at'], "%Y-%m-%dT%H:%M:%SZ")
epoc = dt_to_epoch(t)
if formatDate == 2:
date = epoc
else:
date = epoch_to_dt(float(epoc))
date = date.strftime(
"%Y%m%d %H:%M:%S"
) # Must be UTC because requested time was cast to UTC
self.simulateHistoricalData(
reqId,
str(date),
float(row['open_price']),
float(row['high_price']),
float(row['low_price']),
float(row['close_price']),
int(float(row['volume'])),
1, # barCount
0.0, # WAP
1) # hasGap
self.simulateHistoricalData(reqId, 'finished', 0.0, 0.0, 0.0, 0.0, 1,
1, 0.0, 1)
def _sendHistoricalData(self, hist, reqId, formatDate):
self._log.debug(__name__ + '::_sendHistoricalData: reqId=%s formatDate=%s' % (str(reqId), str(formatDate)))
for idx in hist.index:
if formatDate == 1: # asked to return datetime in string
date = epoch_to_dt(float(idx)) # int ---> datetime
date = date.strftime("%Y%m%d %H:%M:%S") # Must be UTC because requested time was cast to UTC
else: # asked to return UTC int
date = str(idx)
self.simulateHistoricalData(reqId, date,
float(hist.loc[idx, 'open']),
float(hist.loc[idx, 'high']),
float(hist.loc[idx, 'low']),
float(hist.loc[idx, 'close']),
int(hist.loc[idx, 'volume']),
1, 0.0, 1)
self.simulateHistoricalData(reqId, 'finished', 0.0, 0.0, 0.0, 0.0, 1, 1, 0.0, 1)
def currentTime(self, tm):
"""
IB C++ API call back function. Return system time in datetime instance
constructed from Unix timestamp using the showTimeZone from MarketManager
"""
serverTime = epoch_to_dt(float(tm))
currentTime = self._timeGenerator.get_current_time()
# When backtester is using timeGeneratorType = 'CUSTOM' customSpotTimeList = [], user may create a list of spot time
# by pd.date_range(), which returns a list of pd.Timestamp.
if isinstance(currentTime, pd.Timestamp):
currentTime = currentTime.to_pydatetime()
localServerTimeDiff = serverTime - currentTime
self._log.debug(
__name__ +
'::currentTime: tm=%s localServerTimeDiff=%s brokerClientId=%s' %
(serverTime, localServerTimeDiff, id(self)))
self._timeGenerator.set_diffBetweenLocalAndServer(localServerTimeDiff)
self._activeRequests.set_all_requests_of_a_reqType_to_a_status(
'reqCurrentTime', ReqAttr.Status.COMPLETED)
def historicalData(self, reqId, timeString, price_open, price_high,
price_low, price_close, volume, barCount, WAP, hasGaps):
"""
call back function from IB C++ API
return the historical data for requested security
"""
self._log.notset(
__name__ +
'::historicalData: reqId=%s timeString=%s volume=%s barCount=%s WAP=%s hasGap=%s'
% (reqId, timeString, volume, barCount, WAP, hasGaps))
# for any reason, the reqId is not in the self.activeRequests,
# just ignore it. because the callback historicalData must come from the previous request.
if self._activeRequests.check_valid_reqId(reqId):
aRequest = self._activeRequests.get_by_reqId_otherwise_exit(reqId)
else:
return
if 'finished' in str(timeString):
aRequest.status = ReqAttr.Status.COMPLETED
else:
aRequest.status = ReqAttr.Status.STARTED
# !!! The followings are removed because formatData is forced to 2 in
# broker_client_factory::BrokerClient::_send_req_to_server after V 5.8.1
# User can still choose to formatDate = 1, which means return dt.datetime for convenience
# if aRequest.param['formatDate'] == 1:
# if ' ' in timeString:
# dateTime = dt.datetime.strptime(timeString, '%Y%m%d %H:%M:%S') # change string to datetime
# else:
# dateTime = dt.datetime.strptime(timeString, '%Y%m%d') # change string to datetime
# dateTime = pytz.timezone('UTC').localize(dateTime)
# else: # formatDate is UTC time in seconds, str type
# # The format in which the incoming bars' date should be presented. Note that for day bars, only yyyyMMdd format is available.
# if len(timeString) > 9: # return datetime, not date
# dateTime = epoch_to_dt(float(timeString))
# else: # return date, not datetime
# dateTime = dt.datetime.strptime(timeString, '%Y%m%d') # change string to datetime
# dateTime = int(dt_to_epoch(dateTime)) # change to int type
if len(timeString
) > 9: # the type of returned value is datetime, not date
dateTime = epoch_to_dt(float(timeString))
else:
dateTime = dt.datetime.strptime(
timeString, '%Y%m%d') # change string to datetime
dateTime = int(dt_to_epoch(dateTime)) # change to int type
if aRequest.param['formatDate'] == 1:
dateTime = epoch_to_dt(dateTime) # int -> datetime
if 'day' in aRequest.param['barSize']:
dateTime = dateTime.date() # datetime -> date
else:
dateTime = dateTime.astimezone(
aRequest.param['timezoneOfReturn']) # Adjust timezone
newRow = pd.DataFrame(
{
'open': price_open,
'high': price_high,
'low': price_low,
'close': price_close,
'volume': volume
},
index=[dateTime])
aRequest.returnedResult = aRequest.returnedResult.append(newRow)
def provide_historical_data_from_local_variable_hist(self, security, endTime, goBack, barSize, whatToShow, useRTH, formatDate):
"""
:param security: IBridgePy::quantopian::Security
:param endTime: request's ending time with format yyyyMMdd HH:mm:ss {TMZ} ---from IB api doc
:param goBack: 'x S' 'x D' 'x W' 'x M' 'x Y'
:param barSize: string 1 sec, 5 secs, 15 secs, 30 secs, 1 min, 2 mins, 3 mins, 5 mins, 15 mins,
30 mins, 1 hour, 1 day
:param whatToShow:
:param useRTH:
:param formatDate:
:return:
"""
self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: endTime=%s goBack=%s barSize=%s' % (endTime, goBack, barSize))
if not self._histIngested:
self._log.error(__name__ + '::provide_historical_data_from_local_variable_hist: EXIT, hist has not been ingested.')
exit()
# Read in self.hist to provide data and check errors
security_no_exchange_primaryExchange = stripe_exchange_primaryExchange_from_security(security)
securityFullPrint = security_no_exchange_primaryExchange.full_print()
barSize = barSize.lower()
if securityFullPrint not in self._hist:
self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: hist of %s is not ingested.' % (securityFullPrint,))
raise NotEnoughHist()
if barSize not in self._hist[securityFullPrint]:
self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: hist of %s exists but barSize=%s is not ingested.' % (securityFullPrint, barSize))
# print(self._hist[securityFullPrint].keys())
raise NotEnoughHist()
hist = self._hist[securityFullPrint][barSize]
if not isinstance(hist, pd.DataFrame):
self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: hist is empty')
raise NotEnoughHist()
endTime = adjust_endTime(endTime, barSize)
endTimePosition = _search_index_location_in_hist(hist, endTime)
startTimePosition = endTimePosition - convert_goBack_barSize(goBack, barSize)
if startTimePosition >= len(hist) - 2:
self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: EXIT, Not enough hist security=%s barSize=%s is provided to backtest.' % (securityFullPrint, barSize))
self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: first line in hist=%s' % (epoch_to_dt(hist.index[0])))
self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: last line in hist=%s' % (epoch_to_dt(hist.index[-1])))
self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: backtest at this spot time=%s' % (endTime,))
raise NotEnoughHist()
if startTimePosition < endTimePosition:
return hist.iloc[startTimePosition:endTimePosition + 1]
else:
self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: Incorrect endTime=%s or goBack=%s when barSize=%s' % (endTime, goBack, barSize))
self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: first line in hist=%s' % (epoch_to_dt(hist.index[0])))
self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: last line in hist=%s' % (epoch_to_dt(hist.index[-1])))
self._log.debug(__name__ + '::provide_historical_data_from_local_variable_hist: Hint: Based on the 1st line and last line of the hist, Are hist data too short?')
raise NotEnoughHist()