Ejemplo n.º 1
0
    def reqHistoricalDataWrapper(self, reqId, contract, str_endTime, goBack, barSize, whatToShow, useRTH, formatDate):
        # self._log.info(__name__ + '::reqHistoricalDataWrapper: reqId=%s contract=%s endTime=%s goBack=%s barSize=%s' % (reqId, print_IBCpp_contract(contract), endTime, goBack, barSize))
        # https://developer.tdameritrade.com/content/price-history-samples
        self._validate_contract(contract, 'reqHistoricalDataWrapper')
        freq, freqType = BarSizeConverter().fromIBtoTD(barSize)
        pe, peType = GoBackConverter().fromIBtoTD(goBack, barSize)
        if str_endTime:
            str_endTime = dt.datetime.strptime(str_endTime, "%Y%m%d %H:%M:%S %Z")  # string -> dt.datetime; strptime silently ignores timezone!!!
            str_endTime = pytz.timezone('UTC').localize(str_endTime)
            endDate = int(dt_to_epoch(str_endTime)) * 1000
            ans = self._tdClient.history(str(contract.symbol), periodType=peType, period=pe, frequencyType=freqType, frequency=freq, endDate=endDate)
        else:
            ans = self._tdClient.history(str(contract.symbol), periodType=peType, period=pe, frequencyType=freqType, frequency=freq)

        if 'error' in ans:
            self._log.error(__name__ + '::reqHistoricalDataWrapper: EXIT, cannot handle contract=%s endTime=%s goBack=%s barSize=%s' % (print_IBCpp_contract(contract), str_endTime, goBack, barSize))
            self._log.error(__name__ + '::reqHistoricalDataWrapper: EXIT, cannot handle periodType=%s period=%s frequencyType=%s frequency=%s' % (peType, pe, freqType, freq))
            exit()
        hist = ans['candles']
        for row in hist:
            epoc = int(float(str(row['datetime'])))
            if epoc > 1e10:
                epoc /= 1000.0
            self.simulateHistoricalData(reqId,
                                        str(epoc),
                                        float(row['open']),
                                        float(row['high']),
                                        float(row['low']),
                                        float(row['close']),
                                        int(float(row['volume'])),
                                        1,  # barCount
                                        0.0,  # WAP
                                        1)  # hasGap
        self.simulateHistoricalData(reqId, 'finished', 0.0, 0.0, 0.0, 0.0, 1, 1, 0.0, 1)
    def _get_hist_from_IBPY(self, plan):
        self._log.debug(__name__ + '::_get_hist_from_IBPY')
        endTime = plan.endTime.astimezone(pytz.timezone('UTC'))
        epoch = dt_to_epoch(endTime)
        goBack = None

        count, unit = plan.goBack.split(' ')
        if unit == 'D':
            goBack = int(count)
        else:
            exit(
                __name__ +
                '::_get_hist_from_IBPY: EXIT, cannot handle goBack=%s. Only handle goBack="xx D" right now.'
                % (plan.goBack, ))

        if plan.barSize != '1 day':
            exit(
                __name__ +
                '::_get_hist_from_IBPY: EXIT, cannot handle barSize=%s. Only handle barSize="1 day" right now.'
                % (plan.barSize, ))

        ans = self._candleDailyGetClient.get_candle_daily(
            ticker=plan.security.symbol, epochSecond=epoch, goBack=goBack)
        if not len(ans):
            self._log.error(
                __name__ +
                '::_get_hist_from_IBPY: No historical data available at IBridgePy-portal for ticker=%s'
                % (plan.security.symbol, ))
            raise NotEnoughHist()
        return ans
def _search_index_location_in_hist(hist, aDatetime):
    intDatetime = int(dt_to_epoch(aDatetime))
    if intDatetime not in hist.index:
        indexPosition = bisect.bisect_left(hist.index, intDatetime)
        if indexPosition >= len(hist.index):
            indexPosition -= 1
        ans = indexPosition
        # print(__name__ + '::_search_index_location_in_hist: not in hist ans=%s' % (ans,))
        return ans
    else:
        ans = hist.index.get_loc(intDatetime)
        # print(__name__ + '::_search_index_location_in_hist: ans=%s' % (ans,))
        return ans
Ejemplo n.º 4
0
    def reqHistoricalDataWrapper(self, reqId, contract, endTime, goBack,
                                 barSize, whatToShow, useRTH, formatDate):
        if barSize not in ['1 day', '5 mins', '10 mins']:
            self._log.error(
                __name__ +
                '::reqHistoricalDataWrapper: EXIT, Robinhood does not support barSize=%s'
                % (barSize, ))
        interval = BarSizeConverter().fromIBtoRB(barSize)
        span = GoBackConverter().fromIBtoRB(goBack)
        self._log.notset(
            __name__ +
            '::reqHistoricalDataWrapper: contract=%s interval=%s span=%s' %
            (print_IBCpp_contract(contract), interval, span))
        ans = self._robinhoodClient.get_historical_quotes(
            contract.symbol, interval, span)

        hist = None
        try:
            hist = ans['results'][0]['historicals']
        except KeyError:
            self._log.error(__name__ +
                            '::reqHistoricalDataWrapper: EXIT, ans=%s' %
                            (ans, ))
            exit()

        for row in hist:
            t = dt.datetime.strptime(row['begins_at'], "%Y-%m-%dT%H:%M:%SZ")
            epoc = dt_to_epoch(t)
            if formatDate == 2:
                date = epoc
            else:
                date = epoch_to_dt(float(epoc))
                date = date.strftime(
                    "%Y%m%d  %H:%M:%S"
                )  # Must be UTC because requested time was cast to UTC

            self.simulateHistoricalData(
                reqId,
                str(date),
                float(row['open_price']),
                float(row['high_price']),
                float(row['low_price']),
                float(row['close_price']),
                int(float(row['volume'])),
                1,  # barCount
                0.0,  # WAP
                1)  # hasGap
        self.simulateHistoricalData(reqId, 'finished', 0.0, 0.0, 0.0, 0.0, 1,
                                    1, 0.0, 1)
 def provide_hist_from_a_true_dataProvider(self, security, endTime, goBack,
                                           barSize, whatToShow, useRTH,
                                           formatDate):
     count, unit = goBack.split(' ')
     if unit == 'D':
         goBack = int(count)
     else:
         exit(
             __name__ +
             '::provide_hist_from_a_true_dataProvider: EXIT, cannot handle goBack=%s. Only handle goBack="xx D" right now.'
             % (goBack, ))
     endTime = dt.datetime.strptime(
         endTime, "%Y%m%d %H:%M:%S %Z")  # string -> dt.datetime
     endTime = pytz.timezone('UTC').localize(endTime)
     epoch = dt_to_epoch(endTime)
     ans = self._candleDailyGetClient.get_candle_daily(
         ticker=security.symbol, epochSecond=epoch, goBack=goBack)
     return ans
Ejemplo n.º 6
0
    def historicalData(self, reqId, timeString, price_open, price_high,
                       price_low, price_close, volume, barCount, WAP, hasGaps):
        """
        call back function from IB C++ API
        return the historical data for requested security
        """
        self._log.notset(
            __name__ +
            '::historicalData: reqId=%s timeString=%s volume=%s barCount=%s WAP=%s hasGap=%s'
            % (reqId, timeString, volume, barCount, WAP, hasGaps))

        # for any reason, the reqId is not in the self.activeRequests,
        # just ignore it. because the callback historicalData must come from the previous request.
        if self._activeRequests.check_valid_reqId(reqId):
            aRequest = self._activeRequests.get_by_reqId_otherwise_exit(reqId)
        else:
            return

        if 'finished' in str(timeString):
            aRequest.status = ReqAttr.Status.COMPLETED

        else:
            aRequest.status = ReqAttr.Status.STARTED
            # !!! The followings are removed because formatData is forced to 2 in
            # broker_client_factory::BrokerClient::_send_req_to_server after V 5.8.1
            # User can still choose to formatDate = 1, which means return dt.datetime for convenience

            # if aRequest.param['formatDate'] == 1:
            #     if '  ' in timeString:
            #         dateTime = dt.datetime.strptime(timeString, '%Y%m%d  %H:%M:%S')  # change string to datetime
            #     else:
            #         dateTime = dt.datetime.strptime(timeString, '%Y%m%d')  # change string to datetime
            #     dateTime = pytz.timezone('UTC').localize(dateTime)
            # else:  # formatDate is UTC time in seconds, str type
            #     # The format in which the incoming bars' date should be presented. Note that for day bars, only yyyyMMdd format is available.
            #     if len(timeString) > 9:  # return datetime, not date
            #         dateTime = epoch_to_dt(float(timeString))
            #     else:  # return date, not datetime
            #         dateTime = dt.datetime.strptime(timeString, '%Y%m%d')  # change string to datetime
            #     dateTime = int(dt_to_epoch(dateTime))  # change to int type

            if len(timeString
                   ) > 9:  # the type of returned value is datetime, not date
                dateTime = epoch_to_dt(float(timeString))
            else:
                dateTime = dt.datetime.strptime(
                    timeString, '%Y%m%d')  # change string to datetime
            dateTime = int(dt_to_epoch(dateTime))  # change to int type

            if aRequest.param['formatDate'] == 1:
                dateTime = epoch_to_dt(dateTime)  # int -> datetime
                if 'day' in aRequest.param['barSize']:
                    dateTime = dateTime.date()  # datetime -> date
                else:
                    dateTime = dateTime.astimezone(
                        aRequest.param['timezoneOfReturn'])  # Adjust timezone

            newRow = pd.DataFrame(
                {
                    'open': price_open,
                    'high': price_high,
                    'low': price_low,
                    'close': price_close,
                    'volume': volume
                },
                index=[dateTime])
            aRequest.returnedResult = aRequest.returnedResult.append(newRow)
def _convert_strDatetime_to_epoch_int(aDatetime):
    # !!!!strptime silently discard tzinfo!!!
    aDatetime = dt.datetime.strptime(aDatetime, "%Y%m%d %H:%M:%S %Z")  # string -> dt.datetime
    aDatetime = pytz.timezone('UTC').localize(aDatetime)
    return int(dt_to_epoch(aDatetime))
    def _get_one_real_time_price_from_local_variable_hist(self, security, timeNow, tickType, freq='1 min'):
        """
        Both of prices and volume will be provided.
        :param freq:
        :param security:
        :param timeNow:
        :param tickType: string ONLY
        :return:
        """
        if not self._histIngested:
            self._log.error(__name__ + '::_get_one_real_time_price_from_local_variable_hist: EXIT, hist has not been ingested.')
            exit()
        fieldName = None
        if tickType in [IBCpp.TickType.ASK, IBCpp.TickType.BID, IBCpp.TickType.LAST, IBCpp.TickType.OPEN]:
            fieldName = 'open'
        elif tickType == IBCpp.TickType.HIGH:
            fieldName = 'high'
        elif tickType == IBCpp.TickType.LOW:
            fieldName = 'low'
        elif tickType == IBCpp.TickType.CLOSE:
            fieldName = 'close'
        elif tickType == IBCpp.TickType.VOLUME:
            fieldName = 'volume'
        else:
            self._log.error(__name__ + '::_get_one_real_time_price_from_local_variable_hist: EXIT, cannot handle tickType=%s' % (tickType,))
            exit()

        security_no_exchange_primaryExchange = stripe_exchange_primaryExchange_from_security(security)
        str_security = security_no_exchange_primaryExchange.full_print()

        # IB only accepts str_endTime with a timezone of UTC.
        timeNow = timeNow.astimezone(pytz.utc)

        # If 1 minute bar is labeled as simulatedByDailyBars, it means to get daily bar that contains the timeNow and then,
        # use "close" price of the daily bar to simulate minute price.
        if (str_security, freq) in self._simulatedByDailyBars:
            int_timeNow = int(dt_to_epoch(timeNow))
            hist = self._hist[str_security]['1 day']
            if int_timeNow in hist.index:
                ans = hist.loc[int_timeNow, self._useColumnNameWhenSimulatedByDailyBar]
                self._log.debug(__name__ + '::_get_one_real_time_price_from_local_variable_hist: simulatedByDailyBars str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, int_timeNow, tickType, ans))
                return ans
            else:
                # bisect_left is to find the loc and insert a row on the left of the loc.
                # Therefore, "-1" is needed to find the correct daily bar
                # for example, timeNow = 2020-12-23 15:59:00 -0500, then, timeNowPosition should be 2020-12-23
                timeNowPosition = bisect.bisect_left(hist.index, int_timeNow) - 1
                if timeNowPosition >= len(hist.index):
                    timeNowPosition -= 1
                ans = hist.iloc[timeNowPosition][self._useColumnNameWhenSimulatedByDailyBar]
                self._log.debug(__name__ + '::_get_one_real_time_price_from_local_variable_hist: simulatedByDailyBars str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, int_timeNow, tickType, ans))
                return ans
        if str_security not in self._hist:
            str_timeNow = dt.datetime.strftime(timeNow, "%Y%m%d %H:%M:%S %Z")  # datetime -> string
            if (security, str_timeNow) in self._1minHist:
                return self._1minHist[(security, str_timeNow)].iloc[-1][fieldName]
            if self.name in [DataProviderName.IB, DataProviderName.ROBINHOOD, DataProviderName.ROBINHOOD, DataProviderName.IBRIDGEPY]:
                self._log.info('Do not have hist for str_security=%s' % (str_security,))
                if security.secType != 'CASH':
                    return self._helper(security, str_timeNow, fieldName)
                else:
                    return self._helper(security, str_timeNow, fieldName, 'ASK')
            else:
                self._log.error(__name__ + '::_get_one_real_time_price_from_local_variable_hist: EXIT, not enough hist %s, %s from dataProviderName=%s' % (security, str_timeNow, self.name,))
                raise NotEnoughHist()
        if freq not in self._hist[str_security]:
            str_timeNow = dt.datetime.strftime(timeNow, "%Y%m%d %H:%M:%S %Z")  # datetime -> string
            if (security, str_timeNow) in self._1minHist:
                return self._1minHist[(security, str_timeNow)].iloc[-1][fieldName]
            self._log.info(__name__ + '::_get_one_real_time_price_from_local_variable_hist: hist of %s does not have freq=%s' % (str_security, freq))
            if security.secType != 'CASH':
                return self._helper(security, str_timeNow, fieldName)
            else:
                return self._helper(security, str_timeNow, fieldName, 'ASK')

        int_timeNow = int(dt_to_epoch(timeNow))
        hist = self._hist[str_security][freq]
        if int_timeNow in hist.index:
            ans = hist.loc[int_timeNow, fieldName]
            self._log.debug(__name__ + '::_get_one_real_time_price_from_local_variable_hist: str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, int_timeNow, tickType, ans))
            return ans
        else:
            # Solution: if timeNow is not in hist, then raise Exception. Maybe it is not a good idea
            # timeNow = epoch_to_dt(timeNow).astimezone(self.showTimeZone)  # default is UTC
            # time1st = epoch_to_dt(self.hist[str_security][freq].index[0]).astimezone(self.showTimeZone)
            # timeLast = epoch_to_dt(self.hist[str_security][freq].index[-1]).astimezone(self.showTimeZone)
            # self._log.error(__name__ + '::get_one_real_time_prices: loaded hist does not have timeNow=%s' % (str(timeNow),))
            # self._log.error(__name__ + '::get_one_real_time_prices: loaded hist of security=%s from %s to %s'
            #                % (str_security,  time1st, timeLast))
            # raise AssertionError  # AssertionError will be caught by broker_client_factory::BrokerClient_Local.py::processMessagesWrapper

            # Solution 2: look for the timeBar immediately before timeNow, and use its value.
            timeNowPosition = bisect.bisect_left(hist.index, int_timeNow)
            if timeNowPosition >= len(hist.index):
                timeNowPosition -= 1
            ans = hist.iloc[timeNowPosition][fieldName]
            self._log.debug(__name__ + '::_get_one_real_time_price_from_local_variable_hist: str_security=%s timeNow=%s tickType=%s returnedValue=%s' % (str_security, int_timeNow, tickType, ans))
            return ans
 def reqCurrentTimeWrapper(self):
     tmp = dt_to_epoch(self.get_datetime())
     # print(__name__ + '::reqCurrentTimeWrapper: invoke self.simulateCurrentTime')
     self.simulateCurrentTime(int(tmp))  # IBCpp function
Ejemplo n.º 10
0
 def reqCurrentTimeWrapper(self):
     tmp = dt_to_epoch(self.get_datetime())
     self._log.debug(__name__ + '::reqCurrentTimeWrapper: tmp=%s' % (tmp, ))
     self.simulateCurrentTime(int(tmp))  # IBCpp function