def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]: """ Loads backtest data and returns the data combined with the timerange as tuple. """ self.progress.init_step(BacktestState.DATALOAD, 1) data = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, timeframe=self.timeframe, timerange=self.timerange, startup_candles=self.required_startup, fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), ) min_date, max_date = history.get_timerange(data) logger.info( f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' f'({(max_date - min_date).days} days).') # Adjust startts forward if not enough data is available self.timerange.adjust_start_if_necessary( timeframe_to_seconds(self.timeframe), self.required_startup, min_date) self.progress.set_new_value(1) return data, self.timerange
def historic_ohlcv(self, pair: str, timeframe: str = None, candle_type: str = '') -> DataFrame: """ Get stored historical candle (OHLCV) data :param pair: pair to get the data for :param timeframe: timeframe to get data for :param candle_type: '', mark, index, premiumIndex, or funding_rate """ _candle_type = CandleType.from_string( candle_type ) if candle_type != '' else self._config['candle_type_def'] saved_pair = (pair, str(timeframe), _candle_type) if saved_pair not in self.__cached_pairs_backtesting: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) # Move informative start time respecting startup_candle_count timerange.subtract_start( timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0)) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( pair=pair, timeframe=timeframe or self._config['timeframe'], datadir=self._config['datadir'], timerange=timerange, data_format=self._config.get('dataformat_ohlcv', 'json'), candle_type=_candle_type, ) return self.__cached_pairs_backtesting[saved_pair].copy()
def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]: timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, timeframe=self.timeframe, timerange=timerange, startup_candles=self.required_startup, fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), ) min_date, max_date = history.get_timerange(data) logger.info('Loading data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days) # Adjust startts forward if not enough data is available timerange.adjust_start_if_necessary( timeframe_to_seconds(self.timeframe), self.required_startup, min_date) return data, timerange
def ohlcv_load( self, pair, timeframe: str, candle_type: CandleType, timerange: Optional[TimeRange] = None, fill_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, warn_no_data: bool = True, ) -> DataFrame: """ Load cached candle (OHLCV) data for the given pair. :param pair: Pair to load data for :param timeframe: Timeframe (e.g. "5m") :param timerange: Limit data to be loaded to this timerange :param fill_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period :param warn_no_data: Log a warning message when no data is found :param candle_type: Any of the enum CandleType (must match trading mode!) :return: DataFrame with ohlcv data, or empty DataFrame """ # Fix startup period timerange_startup = deepcopy(timerange) if startup_candles > 0 and timerange_startup: timerange_startup.subtract_start( timeframe_to_seconds(timeframe) * startup_candles) pairdf = self._ohlcv_load(pair, timeframe, timerange=timerange_startup, candle_type=candle_type) if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data): return pairdf else: enddate = pairdf.iloc[-1]['date'] if timerange_startup: self._validate_pairdata(pair, pairdf, timeframe, candle_type, timerange_startup) pairdf = trim_dataframe(pairdf, timerange_startup) if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data): return pairdf # incomplete candles should only be dropped if we didn't trim the end beforehand. pairdf = clean_ohlcv_dataframe( pairdf, timeframe, pair=pair, fill_missing=fill_missing, drop_incomplete=(drop_incomplete and enddate == pairdf.iloc[-1]['date'])) self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data) return pairdf
def init_plotscript(config, markets: List, startup_candles: int = 0): """ Initialize objects needed for plotting :return: Dict with candle (OHLCV) data, trades and pairs """ if "pairs" in config: pairs = expand_pairlist(config['pairs'], markets) else: pairs = expand_pairlist(config['exchange']['pair_whitelist'], markets) # Set timerange to use timerange = TimeRange.parse_timerange(config.get('timerange')) data = load_data( datadir=config.get('datadir'), pairs=pairs, timeframe=config['timeframe'], timerange=timerange, startup_candles=startup_candles, data_format=config.get('dataformat_ohlcv', 'json'), ) if startup_candles and data: min_date, max_date = get_timerange(data) logger.info(f"Loading data from {min_date} to {max_date}") timerange.adjust_start_if_necessary( timeframe_to_seconds(config['timeframe']), startup_candles, min_date) no_trades = False filename = config.get('exportfilename') if config.get('no_trades', False): no_trades = True elif config['trade_source'] == 'file': if not filename.is_dir() and not filename.is_file(): logger.warning("Backtest file is missing skipping trades.") no_trades = True try: trades = load_trades( config['trade_source'], db_url=config.get('db_url'), exportfilename=filename, no_trades=no_trades, strategy=config.get('strategy'), ) except ValueError as e: raise OperationalException(e) from e if not trades.empty: trades = trim_dataframe(trades, timerange, 'open_date') return { "ohlcv": data, "trades": trades, "pairs": pairs, "timerange": timerange, }
def load_pair_history( pair: str, timeframe: str, datadir: Path, timerange: Optional[TimeRange] = None, refresh_pairs: bool = False, exchange: Optional[Exchange] = None, fill_up_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, ) -> DataFrame: """ Loads cached ticker history for the given pair. :param pair: Pair to load data for :param timeframe: Ticker timeframe (e.g. "5m") :param datadir: Path to the data storage location. :param timerange: Limit data to be loaded to this timerange :param refresh_pairs: Refresh pairs from exchange. (Note: Requires exchange to be passed as well.) :param exchange: Exchange object (needed when using "refresh_pairs") :param fill_up_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period :return: DataFrame with ohlcv data """ timerange_startup = deepcopy(timerange) if startup_candles > 0 and timerange_startup: timerange_startup.subtract_start( timeframe_to_seconds(timeframe) * startup_candles) # The user forced the refresh of pairs if refresh_pairs: download_pair_history(datadir=datadir, exchange=exchange, pair=pair, timeframe=timeframe, timerange=timerange) pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup) if pairdata: if timerange_startup: _validate_pairdata(pair, pairdata, timerange_startup) return parse_ticker_dataframe(pairdata, timeframe, pair=pair, fill_missing=fill_up_missing, drop_incomplete=drop_incomplete) else: logger.warning( f'No history data for pair: "{pair}", timeframe: {timeframe}. ' 'Use `freqtrade download-data` to download the data') return None
def get_signal( self, pair: str, timeframe: str, dataframe: DataFrame ) -> Tuple[bool, bool, Optional[str], Optional[str]]: """ Calculates current signal based based on the buy / sell columns of the dataframe. Used by Bot to get the signal to buy or sell :param pair: pair in format ANT/BTC :param timeframe: timeframe to use :param dataframe: Analyzed dataframe to get signal from. :return: (Buy, Sell) A bool-tuple indicating buy/sell signal """ if not isinstance(dataframe, DataFrame) or dataframe.empty: logger.warning(f'Empty candle (OHLCV) data for pair {pair}') return False, False, None, None latest_date = dataframe['date'].max() latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1] # Explicitly convert to arrow object to ensure the below comparison does not fail latest_date = arrow.get(latest_date) # Check if dataframe is out of date timeframe_minutes = timeframe_to_minutes(timeframe) offset = self.config.get('exchange', {}).get('outdated_offset', 5) if latest_date < (arrow.utcnow().shift(minutes=-(timeframe_minutes * 2 + offset))): logger.warning( 'Outdated history for pair %s. Last tick is %s minutes old', pair, int((arrow.utcnow() - latest_date).total_seconds() // 60) ) return False, False, None, None buy = latest[SignalType.BUY.value] == 1 sell = False if SignalType.SELL.value in latest: sell = latest[SignalType.SELL.value] == 1 buy_tag = latest.get(SignalTagType.BUY_TAG.value, None) exit_tag = latest.get(SignalTagType.EXIT_TAG.value, None) # Tags can be None, which does not resolve to False. buy_tag = buy_tag if isinstance(buy_tag, str) else None exit_tag = exit_tag if isinstance(exit_tag, str) else None logger.debug('trigger: %s (pair=%s) buy=%s sell=%s', latest['date'], pair, str(buy), str(sell)) timeframe_seconds = timeframe_to_seconds(timeframe) if self.ignore_expired_candle(latest_date=latest_date, current_time=datetime.now(timezone.utc), timeframe_seconds=timeframe_seconds, buy=buy): return False, sell, buy_tag, exit_tag return buy, sell, buy_tag, exit_tag
def ohlcv_load(self, pair, timeframe: str, timerange: Optional[TimeRange] = None, fill_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, warn_no_data: bool = True) -> DataFrame: """ Load cached ticker history for the given pair. :param pair: Pair to load data for :param timeframe: Ticker timeframe (e.g. "5m") :param timerange: Limit data to be loaded to this timerange :param fill_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period :param warn_no_data: Log a warning message when no data is found :return: DataFrame with ohlcv data, or empty DataFrame """ # Fix startup period timerange_startup = deepcopy(timerange) if startup_candles > 0 and timerange_startup: timerange_startup.subtract_start( timeframe_to_seconds(timeframe) * startup_candles) pairdf = self._ohlcv_load(pair, timeframe, timerange=timerange_startup) if pairdf.empty: if warn_no_data: logger.warning( f'No history data for pair: "{pair}", timeframe: {timeframe}. ' 'Use `freqtrade download-data` to download the data') return pairdf else: enddate = pairdf.iloc[-1]['date'] if timerange_startup: self._validate_pairdata(pair, pairdf, timerange_startup) pairdf = trim_dataframe(pairdf, timerange_startup) # incomplete candles should only be dropped if we didn't trim the end beforehand. return clean_ohlcv_dataframe( pairdf, timeframe, pair=pair, fill_missing=fill_missing, drop_incomplete=(drop_incomplete and enddate == pairdf.iloc[-1]['date']))
def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame: """ Get stored historical candle (OHLCV) data :param pair: pair to get the data for :param timeframe: timeframe to get data for """ saved_pair = (pair, str(timeframe)) if saved_pair not in self.__cached_pairs_backtesting: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) # Move informative start time respecting startup_candle_count timerange.subtract_start( timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0)) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( pair=pair, timeframe=timeframe or self._config['timeframe'], datadir=self._config['datadir'], timerange=timerange, data_format=self._config.get('dataformat_ohlcv', 'json')) return self.__cached_pairs_backtesting[saved_pair].copy()
def load_pair_history(pair: str, timeframe: str, datadir: Path, timerange: Optional[TimeRange] = None, fill_up_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, ) -> DataFrame: """ Load cached ticker history for the given pair. :param pair: Pair to load data for :param timeframe: Ticker timeframe (e.g. "5m") :param datadir: Path to the data storage location. :param timerange: Limit data to be loaded to this timerange :param fill_up_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period :return: DataFrame with ohlcv data, or empty DataFrame """ timerange_startup = deepcopy(timerange) if startup_candles > 0 and timerange_startup: timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup) if pairdata: if timerange_startup: _validate_pairdata(pair, pairdata, timerange_startup) return parse_ticker_dataframe(pairdata, timeframe, pair=pair, fill_missing=fill_up_missing, drop_incomplete=drop_incomplete) else: logger.warning( f'No history data for pair: "{pair}", timeframe: {timeframe}. ' 'Use `freqtrade download-data` to download the data' ) return DataFrame()
def load_bt_data(self): timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = history.load_data( datadir=Path(self.config['datadir']), pairs=self.config['exchange']['pair_whitelist'], timeframe=self.timeframe, timerange=timerange, startup_candles=self.required_startup, fail_without_data=True, ) min_date, max_date = history.get_timeframe(data) logger.info('Loading data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days) # Adjust startts forward if not enough data is available timerange.adjust_start_if_necessary( timeframe_to_seconds(self.timeframe), self.required_startup, min_date) return data, timerange
def define_index(min_date: int, max_date: int, ticker_interval: str) -> int: """ Return the index of a specific date """ interval_seconds = timeframe_to_seconds(ticker_interval) return int((max_date - min_date) / interval_seconds)