def reqHistoricalDataWrapper(self, reqId, contract, str_endTime, goBack, barSize, whatToShow, useRTH, formatDate):
# self._log.info(__name__ + '::reqHistoricalDataWrapper: reqId=%s contract=%s endTime=%s goBack=%s barSize=%s' % (reqId, print_IBCpp_contract(contract), endTime, goBack, barSize))
# https://developer.tdameritrade.com/content/price-history-samples
self._validate_contract(contract, 'reqHistoricalDataWrapper')
freq, freqType = BarSizeConverter().fromIBtoTD(barSize)
pe, peType = GoBackConverter().fromIBtoTD(goBack, barSize)
if str_endTime:
str_endTime = dt.datetime.strptime(str_endTime, "%Y%m%d %H:%M:%S %Z") # string -> dt.datetime; strptime silently ignores timezone!!!
str_endTime = pytz.timezone('UTC').localize(str_endTime)
endDate = int(dt_to_epoch(str_endTime)) * 1000
ans = self._tdClient.history(str(contract.symbol), periodType=peType, period=pe, frequencyType=freqType, frequency=freq, endDate=endDate)
else:
ans = self._tdClient.history(str(contract.symbol), periodType=peType, period=pe, frequencyType=freqType, frequency=freq)
if 'error' in ans:
self._log.error(__name__ + '::reqHistoricalDataWrapper: EXIT, cannot handle contract=%s endTime=%s goBack=%s barSize=%s' % (print_IBCpp_contract(contract), str_endTime, goBack, barSize))
self._log.error(__name__ + '::reqHistoricalDataWrapper: EXIT, cannot handle periodType=%s period=%s frequencyType=%s frequency=%s' % (peType, pe, freqType, freq))
exit()
hist = ans['candles']
for row in hist:
epoc = int(float(str(row['datetime'])))
if epoc > 1e10:
epoc /= 1000.0
self.simulateHistoricalData(reqId,
str(epoc),
float(row['open']),
float(row['high']),
float(row['low']),
float(row['close']),
int(float(row['volume'])),
1, # barCount
0.0, # WAP
1) # hasGap
self.simulateHistoricalData(reqId, 'finished', 0.0, 0.0, 0.0, 0.0, 1, 1, 0.0, 1)
def _get_hist_from_IBPY(self, plan):
self._log.debug(__name__ + '::_get_hist_from_IBPY')
endTime = plan.endTime.astimezone(pytz.timezone('UTC'))
epoch = dt_to_epoch(endTime)
goBack = None
count, unit = plan.goBack.split(' ')
if unit == 'D':
goBack = int(count)
else:
exit(
__name__ +
'::_get_hist_from_IBPY: EXIT, cannot handle goBack=%s. Only handle goBack="xx D" right now.'
% (plan.goBack, ))
if plan.barSize != '1 day':
exit(
__name__ +
'::_get_hist_from_IBPY: EXIT, cannot handle barSize=%s. Only handle barSize="1 day" right now.'
% (plan.barSize, ))
ans = self._candleDailyGetClient.get_candle_daily(
ticker=plan.security.symbol, epochSecond=epoch, goBack=goBack)
if not len(ans):
self._log.error(
__name__ +
'::_get_hist_from_IBPY: No historical data available at IBridgePy-portal for ticker=%s'
% (plan.security.symbol, ))
raise NotEnoughHist()
return ans
def _search_index_location_in_hist(hist, aDatetime):
intDatetime = int(dt_to_epoch(aDatetime))
if intDatetime not in hist.index:
indexPosition = bisect.bisect_left(hist.index, intDatetime)
if indexPosition >= len(hist.index):
indexPosition -= 1
ans = indexPosition
# print(__name__ + '::_search_index_location_in_hist: not in hist ans=%s' % (ans,))
return ans
else:
ans = hist.index.get_loc(intDatetime)
# print(__name__ + '::_search_index_location_in_hist: ans=%s' % (ans,))
return ans
def reqHistoricalDataWrapper(self, reqId, contract, endTime, goBack,
barSize, whatToShow, useRTH, formatDate):
if barSize not in ['1 day', '5 mins', '10 mins']:
self._log.error(
__name__ +
'::reqHistoricalDataWrapper: EXIT, Robinhood does not support barSize=%s'
% (barSize, ))
interval = BarSizeConverter().fromIBtoRB(barSize)
span = GoBackConverter().fromIBtoRB(goBack)
self._log.notset(
__name__ +
'::reqHistoricalDataWrapper: contract=%s interval=%s span=%s' %
(print_IBCpp_contract(contract), interval, span))
ans = self._robinhoodClient.get_historical_quotes(
contract.symbol, interval, span)
hist = None
try:
hist = ans['results'][0]['historicals']
except KeyError:
self._log.error(__name__ +
'::reqHistoricalDataWrapper: EXIT, ans=%s' %
(ans, ))
exit()
for row in hist:
t = dt.datetime.strptime(row['begins_at'], "%Y-%m-%dT%H:%M:%SZ")
epoc = dt_to_epoch(t)
if formatDate == 2:
date = epoc
else:
date = epoch_to_dt(float(epoc))
date = date.strftime(
"%Y%m%d %H:%M:%S"
) # Must be UTC because requested time was cast to UTC
self.simulateHistoricalData(
reqId,
str(date),
float(row['open_price']),
float(row['high_price']),
float(row['low_price']),
float(row['close_price']),
int(float(row['volume'])),
1, # barCount
0.0, # WAP
1) # hasGap
self.simulateHistoricalData(reqId, 'finished', 0.0, 0.0, 0.0, 0.0, 1,
1, 0.0, 1)
def provide_hist_from_a_true_dataProvider(self, security, endTime, goBack,
barSize, whatToShow, useRTH,
formatDate):
count, unit = goBack.split(' ')
if unit == 'D':
goBack = int(count)
else:
exit(
__name__ +
'::provide_hist_from_a_true_dataProvider: EXIT, cannot handle goBack=%s. Only handle goBack="xx D" right now.'
% (goBack, ))
endTime = dt.datetime.strptime(
endTime, "%Y%m%d %H:%M:%S %Z") # string -> dt.datetime
endTime = pytz.timezone('UTC').localize(endTime)
epoch = dt_to_epoch(endTime)
ans = self._candleDailyGetClient.get_candle_daily(
ticker=security.symbol, epochSecond=epoch, goBack=goBack)
return ans
def historicalData(self, reqId, timeString, price_open, price_high,
price_low, price_close, volume, barCount, WAP, hasGaps):
"""
call back function from IB C++ API
return the historical data for requested security
"""
self._log.notset(
__name__ +
'::historicalData: reqId=%s timeString=%s volume=%s barCount=%s WAP=%s hasGap=%s'
% (reqId, timeString, volume, barCount, WAP, hasGaps))
# for any reason, the reqId is not in the self.activeRequests,
# just ignore it. because the callback historicalData must come from the previous request.
if self._activeRequests.check_valid_reqId(reqId):
aRequest = self._activeRequests.get_by_reqId_otherwise_exit(reqId)
else:
return
if 'finished' in str(timeString):
aRequest.status = ReqAttr.Status.COMPLETED
else:
aRequest.status = ReqAttr.Status.STARTED
# !!! The followings are removed because formatData is forced to 2 in
# broker_client_factory::BrokerClient::_send_req_to_server after V 5.8.1
# User can still choose to formatDate = 1, which means return dt.datetime for convenience
# if aRequest.param['formatDate'] == 1:
# if ' ' in timeString:
# dateTime = dt.datetime.strptime(timeString, '%Y%m%d %H:%M:%S') # change string to datetime
# else:
# dateTime = dt.datetime.strptime(timeString, '%Y%m%d') # change string to datetime
# dateTime = pytz.timezone('UTC').localize(dateTime)
# else: # formatDate is UTC time in seconds, str type
# # The format in which the incoming bars' date should be presented. Note that for day bars, only yyyyMMdd format is available.
# if len(timeString) > 9: # return datetime, not date
# dateTime = epoch_to_dt(float(timeString))
# else: # return date, not datetime
# dateTime = dt.datetime.strptime(timeString, '%Y%m%d') # change string to datetime
# dateTime = int(dt_to_epoch(dateTime)) # change to int type
if len(timeString
) > 9: # the type of returned value is datetime, not date
dateTime = epoch_to_dt(float(timeString))
else:
dateTime = dt.datetime.strptime(
timeString, '%Y%m%d') # change string to datetime
dateTime = int(dt_to_epoch(dateTime)) # change to int type
if aRequest.param['formatDate'] == 1:
dateTime = epoch_to_dt(dateTime) # int -> datetime
if 'day' in aRequest.param['barSize']:
dateTime = dateTime.date() # datetime -> date
else:
dateTime = dateTime.astimezone(
aRequest.param['timezoneOfReturn']) # Adjust timezone
newRow = pd.DataFrame(
{
'open': price_open,
'high': price_high,
'low': price_low,
'close': price_close,
'volume': volume
},
index=[dateTime])
aRequest.returnedResult = aRequest.returnedResult.append(newRow)
def _convert_strDatetime_to_epoch_int(aDatetime):
# !!!!strptime silently discard tzinfo!!!
aDatetime = dt.datetime.strptime(aDatetime, "%Y%m%d %H:%M:%S %Z") # string -> dt.datetime
aDatetime = pytz.timezone('UTC').localize(aDatetime)
return int(dt_to_epoch(aDatetime))
def _get_one_real_time_price_from_local_variable_hist(self, security, timeNow, tickType, freq='1 min'):
"""
Both of prices and volume will be provided.
:param freq:
:param security:
:param timeNow:
:param tickType: string ONLY
:return:
"""
if not self._histIngested:
self._log.error(__name__ + '::_get_one_real_time_price_from_local_variable_hist: EXIT, hist has not been ingested.')
exit()
fieldName = None
if tickType in [IBCpp.TickType.ASK, IBCpp.TickType.BID, IBCpp.TickType.LAST, IBCpp.TickType.OPEN]:
fieldName = 'open'
elif tickType == IBCpp.TickType.HIGH:
fieldName = 'high'
elif tickType == IBCpp.TickType.LOW:
fieldName = 'low'
elif tickType == IBCpp.TickType.CLOSE:
fieldName = 'close'
elif tickType == IBCpp.TickType.VOLUME:
fieldName = 'volume'
else:
self._log.error(__name__ + '::_get_one_real_time_price_from_local_variable_hist: EXIT, cannot handle tickType=%s' % (tickType,))
exit()
security_no_exchange_primaryExchange = stripe_exchange_primaryExchange_from_security(security)
str_security = security_no_exchange_primaryExchange.full_print()
# IB only accepts str_endTime with a timezone of UTC.
timeNow = timeNow.astimezone(pytz.utc)
# If 1 minute bar is labeled as simulatedByDailyBars, it means to get daily bar that contains the timeNow and then,
# use "close" price of the daily bar to simulate minute price.
if (str_security, freq) in self._simulatedByDailyBars:
int_timeNow = int(dt_to_epoch(timeNow))
hist = self._hist[str_security]['1 day']
if int_timeNow in hist.index:
ans = hist.loc[int_timeNow, self._useColumnNameWhenSimulatedByDailyBar]
self._log.debug(__name__ + '::_get_one_real_time_price_from_local_variable_hist: simulatedByDailyBars str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, int_timeNow, tickType, ans))
return ans
else:
# bisect_left is to find the loc and insert a row on the left of the loc.
# Therefore, "-1" is needed to find the correct daily bar
# for example, timeNow = 2020-12-23 15:59:00 -0500, then, timeNowPosition should be 2020-12-23
timeNowPosition = bisect.bisect_left(hist.index, int_timeNow) - 1
if timeNowPosition >= len(hist.index):
timeNowPosition -= 1
ans = hist.iloc[timeNowPosition][self._useColumnNameWhenSimulatedByDailyBar]
self._log.debug(__name__ + '::_get_one_real_time_price_from_local_variable_hist: simulatedByDailyBars str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, int_timeNow, tickType, ans))
return ans
if str_security not in self._hist:
str_timeNow = dt.datetime.strftime(timeNow, "%Y%m%d %H:%M:%S %Z") # datetime -> string
if (security, str_timeNow) in self._1minHist:
return self._1minHist[(security, str_timeNow)].iloc[-1][fieldName]
if self.name in [DataProviderName.IB, DataProviderName.ROBINHOOD, DataProviderName.ROBINHOOD, DataProviderName.IBRIDGEPY]:
self._log.info('Do not have hist for str_security=%s' % (str_security,))
if security.secType != 'CASH':
return self._helper(security, str_timeNow, fieldName)
else:
return self._helper(security, str_timeNow, fieldName, 'ASK')
else:
self._log.error(__name__ + '::_get_one_real_time_price_from_local_variable_hist: EXIT, not enough hist %s, %s from dataProviderName=%s' % (security, str_timeNow, self.name,))
raise NotEnoughHist()
if freq not in self._hist[str_security]:
str_timeNow = dt.datetime.strftime(timeNow, "%Y%m%d %H:%M:%S %Z") # datetime -> string
if (security, str_timeNow) in self._1minHist:
return self._1minHist[(security, str_timeNow)].iloc[-1][fieldName]
self._log.info(__name__ + '::_get_one_real_time_price_from_local_variable_hist: hist of %s does not have freq=%s' % (str_security, freq))
if security.secType != 'CASH':
return self._helper(security, str_timeNow, fieldName)
else:
return self._helper(security, str_timeNow, fieldName, 'ASK')
int_timeNow = int(dt_to_epoch(timeNow))
hist = self._hist[str_security][freq]
if int_timeNow in hist.index:
ans = hist.loc[int_timeNow, fieldName]
self._log.debug(__name__ + '::_get_one_real_time_price_from_local_variable_hist: str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, int_timeNow, tickType, ans))
return ans
else:
# Solution: if timeNow is not in hist, then raise Exception. Maybe it is not a good idea
# timeNow = epoch_to_dt(timeNow).astimezone(self.showTimeZone) # default is UTC
# time1st = epoch_to_dt(self.hist[str_security][freq].index[0]).astimezone(self.showTimeZone)
# timeLast = epoch_to_dt(self.hist[str_security][freq].index[-1]).astimezone(self.showTimeZone)
# self._log.error(__name__ + '::get_one_real_time_prices: loaded hist does not have timeNow=%s' % (str(timeNow),))
# self._log.error(__name__ + '::get_one_real_time_prices: loaded hist of security=%s from %s to %s'
# % (str_security, time1st, timeLast))
# raise AssertionError # AssertionError will be caught by broker_client_factory::BrokerClient_Local.py::processMessagesWrapper
# Solution 2: look for the timeBar immediately before timeNow, and use its value.
timeNowPosition = bisect.bisect_left(hist.index, int_timeNow)
if timeNowPosition >= len(hist.index):
timeNowPosition -= 1
ans = hist.iloc[timeNowPosition][fieldName]
self._log.debug(__name__ + '::_get_one_real_time_price_from_local_variable_hist: str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, int_timeNow, tickType, ans))
return ans
def reqCurrentTimeWrapper(self):
tmp = dt_to_epoch(self.get_datetime())
# print(__name__ + '::reqCurrentTimeWrapper: invoke self.simulateCurrentTime')
self.simulateCurrentTime(int(tmp)) # IBCpp function
def reqCurrentTimeWrapper(self):
tmp = dt_to_epoch(self.get_datetime())
self._log.debug(__name__ + '::reqCurrentTimeWrapper: tmp=%s' % (tmp, ))
self.simulateCurrentTime(int(tmp)) # IBCpp function