def get_exchange_history_window(self, exchange_name, assets, end_dt, bar_count, frequency, field, data_frequency, ffill=True): """ Fetching price history window from the exchange bundle. Parameters ---------- exchange: Exchange assets: list[TradingPair] end_dt: datetime bar_count: int frequency: str field: str data_frequency: str ffill: bool Returns ------- DataFrame """ # TODO: verify that the exchange supports the timeframe bundle = self.exchange_bundles[exchange_name] # type: ExchangeBundle freq, candle_size, unit, adj_data_frequency = get_frequency( frequency, data_frequency) adj_bar_count = candle_size * bar_count trailing_bar_count = candle_size - 1 if data_frequency == 'minute' and adj_data_frequency == 'daily': end_dt = end_dt.floor('1D') series = bundle.get_history_window_series_and_load( assets=assets, end_dt=end_dt, bar_count=adj_bar_count, field=field, data_frequency=adj_data_frequency, algo_end_dt=self._last_available_session, trailing_bar_count=trailing_bar_count, ) df = resample_history_df(pd.DataFrame(series), freq, field) return df
def get_history_window_with_bundle(self, assets, end_dt, bar_count, frequency, field, data_frequency=None, ffill=True, force_auto_ingest=False): """ Public API method that returns a dataframe containing the requested history window. Data is fully adjusted. Parameters ---------- assets : list[TradingPair] The assets whose data is desired. end_dt: datetime The date of the last bar. bar_count: int The number of bars desired. frequency: string "1d" or "1m" field: string The desired field of the asset. data_frequency: string The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars. # TODO: fill how? ffill: boolean Forward-fill missing values. Only has effect if field is 'price'. Returns ------- DataFrame A dataframe containing the requested data. """ # TODO: this function needs some work, # we're currently using it just for benchmark data freq, candle_size, unit, data_frequency = get_frequency( frequency, data_frequency, supported_freqs=['T', 'D']) adj_bar_count = candle_size * bar_count try: series = self.bundle.get_history_window_series_and_load( assets=assets, end_dt=end_dt, bar_count=adj_bar_count, field=field, data_frequency=data_frequency, force_auto_ingest=force_auto_ingest) except (PricingDataNotLoadedError, NoDataAvailableOnExchange): series = dict() for asset in assets: if asset not in series or series[asset].index[-1] < end_dt: # Adding bars too recent to be contained in the consolidated # exchanges bundles. We go directly against the exchange # to retrieve the candles. start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) trailing_dt = \ series[asset].index[-1] + get_delta(1, data_frequency) \ if asset in series else start_dt # The get_history method supports multiple asset # Use the original frequency to let each api optimize # the size of result sets trailing_bars = get_periods(trailing_dt, end_dt, freq) candles = self.get_candles( freq=freq, assets=asset, end_dt=end_dt, bar_count=trailing_bars if trailing_bars < 500 else 500, ) last_value = series[asset].iloc(0) if asset in series \ else np.nan # Create a series with the common data_frequency, ffill # missing values candle_series = self.get_series_from_candles( candles=candles, start_dt=trailing_dt, end_dt=end_dt, data_frequency=data_frequency, field=field, previous_value=last_value) if asset in series: series[asset].append(candle_series) else: series[asset] = candle_series df = resample_history_df(pd.DataFrame(series), freq, field) # TODO: consider this more carefully df.dropna(inplace=True) return df
def get_history_window(self, assets, end_dt, bar_count, frequency, field, data_frequency=None, is_current=False): """ Public API method that returns a dataframe containing the requested history window. Data is fully adjusted. Parameters ---------- assets : list[TradingPair] The assets whose data is desired. end_dt: datetime The date of the last bar bar_count: int The number of bars desired. frequency: string "1d" or "1m" field: string The desired field of the asset. data_frequency: string The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars. is_current: bool Skip date filters when current data is requested (last few bars until now). Notes ----- Catalysts requires an end data with bar count both CCXT wants a start data with bar count. Since we have to make calculations here, we ensure that the last candle match the end_dt parameter. Returns ------- DataFrame A dataframe containing the requested data. """ freq, candle_size, unit, data_frequency = get_frequency( frequency, data_frequency, supported_freqs=['T', 'D', 'H']) # we want to avoid receiving empty candles # so we request more than needed # TODO: consider defining a const per asset # and/or some retry mechanism (in each iteration request more data) min_candles_number = get_candles_number_from_minutes( unit, candle_size, self.MIN_MINUTES_REQUESTED) requested_bar_count = bar_count if requested_bar_count < min_candles_number: requested_bar_count = min_candles_number # The get_history method supports multiple asset candles = self.get_candles( freq=freq, assets=assets, bar_count=requested_bar_count, end_dt=end_dt if not is_current else None, ) # candles sanity check - verify no empty candles were received: for asset in candles: if not candles[asset]: raise NoCandlesReceivedFromExchange( bar_count=requested_bar_count, end_dt=end_dt, asset=asset, exchange=self.name) # for avoiding unnecessary forward fill end_dt is taken back one second forward_fill_till_dt = end_dt - timedelta(seconds=1) series = get_candles_df(candles=candles, field=field, freq=frequency, bar_count=requested_bar_count, end_dt=forward_fill_till_dt) # TODO: consider how to approach this edge case # delta_candle_size = candle_size * 60 if unit == 'H' else candle_size # Checking to make sure that the dates match # delta = get_delta(delta_candle_size, data_frequency) # adj_end_dt = end_dt - delta # last_traded = asset_series.index[-1] # if last_traded < adj_end_dt: # raise LastCandleTooEarlyError( # last_traded=last_traded, # end_dt=adj_end_dt, # exchange=self.name, # ) df = pd.DataFrame(series) df.dropna(inplace=True) return df.tail(bar_count)
def get_exchange_history_window(self, exchange_name, assets, end_dt, bar_count, frequency, field, data_frequency, ffill=True): """ Fetching price history window from the exchange bundle. Parameters ---------- exchange: Exchange assets: list[TradingPair] end_dt: datetime bar_count: int frequency: str field: str data_frequency: str ffill: bool Returns ------- DataFrame """ # TODO: verify that the exchange supports the timeframe bundle = self.exchange_bundles[exchange_name] # type: ExchangeBundle freq, candle_size, unit, adj_data_frequency = get_frequency( frequency, data_frequency, supported_freqs=['T', 'D']) adj_bar_count = candle_size * bar_count if data_frequency == "minute": # for minute frequency always request data until the # current minute (do not include the current minute) last_dt_for_series = end_dt - datetime.timedelta(minutes=1) # read the minute bundles for daily frequency to # support last partial candle # TODO: optimize this by applying this logic only for the last day if adj_data_frequency == 'daily': adj_data_frequency = 'minute' adj_bar_count = adj_bar_count * 1440 else: # data_frequency == "daily": last_dt_for_series = end_dt series = bundle.get_history_window_series_and_load( assets=assets, end_dt=last_dt_for_series, bar_count=adj_bar_count, field=field, data_frequency=adj_data_frequency, algo_end_dt=self._last_available_session, ) start_dt = get_start_dt(last_dt_for_series, adj_bar_count, adj_data_frequency, False) df = resample_history_df(pd.DataFrame(series), freq, field, start_dt) return df
def get_history_window_with_bundle(self, assets, end_dt, bar_count, frequency, field, data_frequency=None, ffill=True, force_auto_ingest=False): """ Public API method that returns a dataframe containing the requested history window. Data is fully adjusted. Parameters ---------- assets : list[TradingPair] The assets whose data is desired. end_dt: datetime The date of the last bar. bar_count: int The number of bars desired. frequency: string "1d" or "1m" field: string The desired field of the asset. data_frequency: string The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars. # TODO: fill how? ffill: boolean Forward-fill missing values. Only has effect if field is 'price'. Returns ------- DataFrame A dataframe containing the requested data. """ # TODO: this function needs some work, we're currently using it just for benchmark data freq, candle_size, unit, data_frequency = get_frequency( frequency, data_frequency ) adj_bar_count = candle_size * bar_count try: series = self.bundle.get_history_window_series_and_load( assets=assets, end_dt=end_dt, bar_count=adj_bar_count, field=field, data_frequency=data_frequency, force_auto_ingest=force_auto_ingest ) except (PricingDataNotLoadedError, NoDataAvailableOnExchange): series = dict() for asset in assets: if asset not in series or series[asset].index[-1] < end_dt: # Adding bars too recent to be contained in the consolidated # exchanges bundles. We go directly against the exchange # to retrieve the candles. start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) trailing_dt = \ series[asset].index[-1] + get_delta(1, data_frequency) \ if asset in series else start_dt # The get_history method supports multiple asset # Use the original frequency to let each api optimize # the size of result sets trailing_bars = get_periods( trailing_dt, end_dt, freq ) candles = self.get_candles( freq=freq, assets=asset, end_dt=end_dt, bar_count=trailing_bars if trailing_bars < 500 else 500, ) last_value = series[asset].iloc(0) if asset in series \ else np.nan # Create a series with the common data_frequency, ffill # missing values candle_series = self.get_series_from_candles( candles=candles, start_dt=trailing_dt, end_dt=end_dt, data_frequency=data_frequency, field=field, previous_value=last_value ) if asset in series: series[asset].append(candle_series) else: series[asset] = candle_series df = resample_history_df(pd.DataFrame(series), freq, field) # TODO: consider this more carefully df.dropna(inplace=True) return df
def get_history_window(self, assets, end_dt, bar_count, frequency, field, data_frequency=None, is_current=False): """ Public API method that returns a dataframe containing the requested history window. Data is fully adjusted. Parameters ---------- assets : list[TradingPair] The assets whose data is desired. end_dt: datetime The date of the last bar bar_count: int The number of bars desired. frequency: string "1d" or "1m" field: string The desired field of the asset. data_frequency: string The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars. is_current: bool Skip date filters when current data is requested (last few bars until now). Notes ----- Catalysts requires an end data with bar count both CCXT wants a start data with bar count. Since we have to make calculations here, we ensure that the last candle match the end_dt parameter. Returns ------- DataFrame A dataframe containing the requested data. """ freq, candle_size, unit, data_frequency = get_frequency( frequency, data_frequency ) # The get_history method supports multiple asset candles = self.get_candles( freq=freq, assets=assets, bar_count=bar_count, end_dt=end_dt if not is_current else None, ) series = dict() for asset in candles: first_candle = candles[asset][0] asset_series = self.get_series_from_candles( candles=candles[asset], start_dt=first_candle['last_traded'], end_dt=end_dt, data_frequency=frequency, field=field, ) # Checking to make sure that the dates match delta = get_delta(candle_size, data_frequency) adj_end_dt = end_dt - delta last_traded = asset_series.index[-1] if last_traded < adj_end_dt: raise LastCandleTooEarlyError( last_traded=last_traded, end_dt=adj_end_dt, exchange=self.name, ) series[asset] = asset_series df = pd.DataFrame(series) df.dropna(inplace=True) return df
def get_history_window(self, assets, end_dt, bar_count, frequency, field, data_frequency=None, is_current=False): """ Public API method that returns a dataframe containing the requested history window. Data is fully adjusted. Parameters ---------- assets : list[TradingPair] The assets whose data is desired. end_dt: datetime The date of the last bar bar_count: int The number of bars desired. frequency: string "1d" or "1m" field: string The desired field of the asset. data_frequency: string The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars. is_current: bool Skip date filters when current data is requested (last few bars until now). Notes ----- Catalysts requires an end data with bar count both CCXT wants a start data with bar count. Since we have to make calculations here, we ensure that the last candle match the end_dt parameter. Returns ------- DataFrame A dataframe containing the requested data. """ freq, candle_size, unit, data_frequency = get_frequency( frequency, data_frequency) # The get_history method supports multiple asset candles = self.get_candles( freq=freq, assets=assets, bar_count=bar_count, end_dt=end_dt if not is_current else None, ) series = dict() for asset in candles: first_candle = candles[asset][0] asset_series = self.get_series_from_candles( candles=candles[asset], start_dt=first_candle['last_traded'], end_dt=end_dt, data_frequency=frequency, field=field, ) # Checking to make sure that the dates match delta = get_delta(candle_size, data_frequency) adj_end_dt = end_dt - delta last_traded = asset_series.index[-1] if last_traded < adj_end_dt: raise LastCandleTooEarlyError( last_traded=last_traded, end_dt=adj_end_dt, exchange=self.name, ) series[asset] = asset_series df = pd.DataFrame(series) df.dropna(inplace=True) return df
def get_history_window_for_asset(self, asset, end_dt, bar_count, frequency, fields, data_frequency=None, is_current=False): """ Public API method that returns a dataframe containing the requested history window. Data is fully adjusted. Parameters ---------- asset : TradingPair The asset whose data is desired. end_dt: datetime The date of the last bar bar_count: int The number of bars desired. frequency: string "1d" or "1m" fields: string The desired fields of the asset. data_frequency: string The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars. is_current: bool Skip date filters when current data is requested (last few bars until now). Notes ----- Catalysts requires an end data with bar count both CCXT wants a start data with bar count. Since we have to make calculations here, we ensure that the last candle match the end_dt parameter. Returns ------- DataFrame A dataframe containing the requested data. """ freq, candle_size, unit, data_frequency = get_frequency( frequency, data_frequency, supported_freqs=['T', 'D', 'H']) # we want to avoid receiving empty candles # so we request more than needed # TODO: consider defining a const per asset # and/or some retry mechanism (in each iteration request more data) min_candles_number = get_candles_number_from_minutes( unit, candle_size, self.MIN_MINUTES_REQUESTED) requested_bar_count = bar_count if requested_bar_count < min_candles_number: requested_bar_count = min_candles_number candles = self.get_candles( freq=freq, assets= asset, # when we pass a single asset, the response is a flat DataFrame with the fields as columns bar_count=requested_bar_count, end_dt=end_dt if not is_current else None, ) # candles sanity check - verify no empty candles were received: if len(candles) == 0: raise NoCandlesReceivedFromExchange(bar_count=requested_bar_count, end_dt=end_dt, asset=asset, exchange=self.name) # for avoiding unnecessary forward fill end_dt is taken back one second forward_fill_till_dt = end_dt - timedelta(seconds=1) candles_df = get_candles_df_for_asset(candles=candles, fields=fields, freq=frequency, bar_count=requested_bar_count, end_dt=forward_fill_till_dt) return candles_df.tail(bar_count)