def test_json_pair_data_filename(pair, expected_result, candle_type): fn = JsonDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, '5m', CandleType.from_string(candle_type)) assert isinstance(fn, Path) assert fn == Path(expected_result) fn = JsonGzDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, '5m', candle_type=CandleType.from_string(candle_type)) assert isinstance(fn, Path) assert fn == Path(expected_result + '.gz')
def test_dp_ohlcv(mocker, default_conf, ohlcv_history, candle_type): default_conf["runmode"] = RunMode.DRY_RUN timeframe = default_conf["timeframe"] exchange = get_patched_exchange(mocker, default_conf) candletype = CandleType.from_string(candle_type) exchange._klines[("XRP/BTC", timeframe, candletype)] = ohlcv_history exchange._klines[("UNITTEST/BTC", timeframe, candletype)] = ohlcv_history dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN assert ohlcv_history.equals(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype)) assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype), DataFrame) assert dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype) is not ohlcv_history assert dp.ohlcv("UNITTEST/BTC", timeframe, copy=False, candle_type=candletype) is ohlcv_history assert not dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype).empty assert dp.ohlcv("NONESENSE/AAA", timeframe, candle_type=candletype).empty # Test with and without parameter assert dp.ohlcv( "UNITTEST/BTC", timeframe, candle_type=candletype ).equals(dp.ohlcv("UNITTEST/BTC", candle_type=candle_type)) default_conf["runmode"] = RunMode.LIVE dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.LIVE assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame) default_conf["runmode"] = RunMode.BACKTEST dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.BACKTEST assert dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type).empty
def historic_ohlcv(self, pair: str, timeframe: str = None, candle_type: str = '') -> DataFrame: """ Get stored historical candle (OHLCV) data :param pair: pair to get the data for :param timeframe: timeframe to get data for :param candle_type: '', mark, index, premiumIndex, or funding_rate """ _candle_type = CandleType.from_string( candle_type ) if candle_type != '' else self._config['candle_type_def'] saved_pair = (pair, str(timeframe), _candle_type) if saved_pair not in self.__cached_pairs_backtesting: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) # Move informative start time respecting startup_candle_count timerange.subtract_start( timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0)) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( pair=pair, timeframe=timeframe or self._config['timeframe'], datadir=self._config['datadir'], timerange=timerange, data_format=self._config.get('dataformat_ohlcv', 'json'), candle_type=_candle_type, ) return self.__cached_pairs_backtesting[saved_pair].copy()
def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True, candle_type: str = '') -> DataFrame: """ Get candle (OHLCV) data for the given pair as DataFrame Please use the `available_pairs` method to verify which pairs are currently cached. :param pair: pair to get the data for :param timeframe: Timeframe to get data for :param candle_type: '', mark, index, premiumIndex, or funding_rate :param copy: copy dataframe before returning if True. Use False only for read-only operations (where the dataframe is not modified) """ if self._exchange is None: raise OperationalException(NO_EXCHANGE_EXCEPTION) if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): _candle_type = CandleType.from_string( candle_type ) if candle_type != '' else self._config['candle_type_def'] return self._exchange.klines( (pair, timeframe or self._config['timeframe'], _candle_type), copy=copy) else: return DataFrame()
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], datadir: Path, trading_mode: str, timerange: Optional[TimeRange] = None, new_pairs_days: int = 30, erase: bool = False, data_format: str = None, ) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. :return: List of pairs that are not available. """ pairs_not_available = [] data_handler = get_datahandler(datadir, data_format) candle_type = CandleType.get_default(trading_mode) for idx, pair in enumerate(pairs, start=1): if pair not in exchange.markets: pairs_not_available.append(pair) logger.info(f"Skipping pair {pair}...") continue for timeframe in timeframes: logger.info(f'Downloading pair {pair}, interval {timeframe}.') process = f'{idx}/{len(pairs)}' _download_pair_history(pair=pair, process=process, datadir=datadir, exchange=exchange, timerange=timerange, data_handler=data_handler, timeframe=str(timeframe), new_pairs_days=new_pairs_days, candle_type=candle_type, erase=erase) if trading_mode == 'futures': # Predefined candletype (and timeframe) depending on exchange # Downloads what is necessary to backtest based on futures data. tf_mark = exchange._ft_has['mark_ohlcv_timeframe'] fr_candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price']) # All exchanges need FundingRate for futures trading. # The timeframe is aligned to the mark-price timeframe. for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type): _download_pair_history(pair=pair, process=process, datadir=datadir, exchange=exchange, timerange=timerange, data_handler=data_handler, timeframe=str(tf_mark), new_pairs_days=new_pairs_days, candle_type=funding_candle_type, erase=erase) return pairs_not_available
def _process_data_options(self, config: Dict[str, Any]) -> None: self._args_to_config(config, argname='new_pairs_days', logstring='Detected --new-pairs-days: {}') self._args_to_config(config, argname='trading_mode', logstring='Detected --trading-mode: {}') config['candle_type_def'] = CandleType.get_default( config.get('trading_mode', 'spot') or 'spot') config['trading_mode'] = TradingMode( config.get('trading_mode', 'spot') or 'spot') self._args_to_config(config, argname='candle_types', logstring='Detected --candle-types: {}')
def informative(timeframe: str, asset: str = '', fmt: Optional[Union[str, Callable[[Any], str]]] = None, *, candle_type: Optional[CandleType] = None, ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]: """ A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to define informative indicators. Example usage: @informative('1h') def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) return dataframe :param timeframe: Informative timeframe. Must always be equal or higher than strategy timeframe. :param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use current pair. :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not specified, defaults to: * {base}_{quote}_{column}_{timeframe} if asset is specified. * {column}_{timeframe} if asset is not specified. Format string supports these format variables: * {asset} - full name of the asset, for example 'BTC/USDT'. * {base} - base currency in lower case, for example 'eth'. * {BASE} - same as {base}, except in upper case. * {quote} - quote currency in lower case, for example 'usdt'. * {QUOTE} - same as {quote}, except in upper case. * {column} - name of dataframe column. * {timeframe} - timeframe of informative dataframe. :param ffill: ffill dataframe after merging informative pair. :param candle_type: '', mark, index, premiumIndex, or funding_rate """ _asset = asset _timeframe = timeframe _fmt = fmt _ffill = ffill _candle_type = CandleType.from_string(candle_type) if candle_type else None def decorator(fn: PopulateIndicators): informative_pairs = getattr(fn, '_ft_informative', []) informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill, _candle_type)) setattr(fn, '_ft_informative', informative_pairs) return fn return decorator
def ohlcv_get_available_data( cls, datadir: Path, trading_mode: TradingMode) -> ListPairsWithTimeframes: """ Returns a list of all pairs with ohlcv data available in this datadir :param datadir: Directory to search for ohlcv files :param trading_mode: trading-mode to be used :return: List of Tuples of (pair, timeframe) """ if trading_mode == TradingMode.FUTURES: datadir = datadir.joinpath('futures') _tmp = [ re.search(cls._OHLCV_REGEX, p.name) for p in datadir.glob("*.h5") ] return [(cls.rebuild_pair_from_filename(match[1]), match[2], CandleType.from_string(match[3])) for match in _tmp if match and len(match.groups()) > 1]
def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: """ Convert data from one format to another """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) if ohlcv: candle_types = [ CandleType.from_string(ct) for ct in config.get('candle_types', ['spot']) ] for candle_type in candle_types: convert_ohlcv_format(config, convert_from=args['format_from'], convert_to=args['format_to'], erase=args['erase'], candle_type=candle_type) else: convert_trades_format(config, convert_from=args['format_from'], convert_to=args['format_to'], erase=args['erase'])
def test_get_pair_dataframe(mocker, default_conf, ohlcv_history, candle_type): default_conf["runmode"] = RunMode.DRY_RUN timeframe = default_conf["timeframe"] exchange = get_patched_exchange(mocker, default_conf) candletype = CandleType.from_string(candle_type) exchange._klines[("XRP/BTC", timeframe, candletype)] = ohlcv_history exchange._klines[("UNITTEST/BTC", timeframe, candletype)] = ohlcv_history dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN assert ohlcv_history.equals(dp.get_pair_dataframe( "UNITTEST/BTC", timeframe, candle_type=candle_type)) assert ohlcv_history.equals(dp.get_pair_dataframe( "UNITTEST/BTC", timeframe, candle_type=candletype)) assert isinstance(dp.get_pair_dataframe( "UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame) assert dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type) is not ohlcv_history assert not dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type).empty assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe, candle_type=candle_type).empty # Test with and without parameter assert dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type)\ .equals(dp.get_pair_dataframe("UNITTEST/BTC", candle_type=candle_type)) default_conf["runmode"] = RunMode.LIVE dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.LIVE assert isinstance(dp.get_pair_dataframe( "UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame) assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe, candle_type=candle_type).empty historymock = MagicMock(return_value=ohlcv_history) mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock) default_conf["runmode"] = RunMode.BACKTEST dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.BACKTEST assert isinstance(dp.get_pair_dataframe( "UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame)
def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Optional[str] = None, candletype: Optional[CandleType] = None, config=Depends(get_config)): dh = get_datahandler(config['datadir'], config.get('dataformat_ohlcv', None)) trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT) pair_interval = dh.ohlcv_get_available_data(config['datadir'], trading_mode) if timeframe: pair_interval = [ pair for pair in pair_interval if pair[1] == timeframe ] if stake_currency: pair_interval = [ pair for pair in pair_interval if pair[0].endswith(stake_currency) ] if candletype: pair_interval = [ pair for pair in pair_interval if pair[2] == candletype ] else: candle_type = CandleType.get_default(trading_mode) pair_interval = [ pair for pair in pair_interval if pair[2] == candle_type ] pair_interval = sorted(pair_interval, key=lambda x: x[0]) pairs = list({x[0] for x in pair_interval}) pairs.sort() result = { 'length': len(pairs), 'pairs': pairs, 'pair_interval': pair_interval, } return result
def test_historic_ohlcv(pair, timeframe, candle_type): return data[ (pair, timeframe or strategy.timeframe, CandleType.from_string(candle_type))].copy()
def test_informative_decorator(mocker, default_conf_usdt, trading_mode): candle_def = CandleType.get_default(trading_mode) default_conf_usdt['candle_type_def'] = candle_def test_data_5m = generate_test_data('5m', 40) test_data_30m = generate_test_data('30m', 40) test_data_1h = generate_test_data('1h', 40) data = { ('XRP/USDT', '5m', candle_def): test_data_5m, ('XRP/USDT', '30m', candle_def): test_data_30m, ('XRP/USDT', '1h', candle_def): test_data_1h, ('LTC/USDT', '5m', candle_def): test_data_5m, ('LTC/USDT', '30m', candle_def): test_data_30m, ('LTC/USDT', '1h', candle_def): test_data_1h, ('NEO/USDT', '30m', candle_def): test_data_30m, ('NEO/USDT', '5m', CandleType.SPOT): test_data_5m, # Explicit request with '' as candletype ('NEO/USDT', '15m', candle_def): test_data_5m, # Explicit request with '' as candletype ('NEO/USDT', '1h', candle_def): test_data_1h, ('ETH/USDT', '1h', candle_def): test_data_1h, ('ETH/USDT', '30m', candle_def): test_data_30m, ('ETH/BTC', '1h', CandleType.SPOT): test_data_1h, # Explicitly selected as spot } default_conf_usdt['strategy'] = 'InformativeDecoratorTest' strategy = StrategyResolver.load_strategy(default_conf_usdt) exchange = get_patched_exchange(mocker, default_conf_usdt) strategy.dp = DataProvider({}, exchange, None) mocker.patch.object(strategy.dp, 'current_whitelist', return_value=[ 'XRP/USDT', 'LTC/USDT', 'NEO/USDT' ]) assert len(strategy._ft_informative) == 6 # Equal to number of decorators used informative_pairs = [ ('XRP/USDT', '1h', candle_def), ('LTC/USDT', '1h', candle_def), ('XRP/USDT', '30m', candle_def), ('LTC/USDT', '30m', candle_def), ('NEO/USDT', '1h', candle_def), ('NEO/USDT', '30m', candle_def), ('NEO/USDT', '5m', candle_def), ('NEO/USDT', '15m', candle_def), ('NEO/USDT', '2h', CandleType.FUTURES), ('ETH/BTC', '1h', CandleType.SPOT), # One candle remains as spot ('ETH/USDT', '30m', candle_def)] for inf_pair in informative_pairs: assert inf_pair in strategy.gather_informative_pairs() def test_historic_ohlcv(pair, timeframe, candle_type): return data[ (pair, timeframe or strategy.timeframe, CandleType.from_string(candle_type))].copy() mocker.patch('freqtrade.data.dataprovider.DataProvider.historic_ohlcv', side_effect=test_historic_ohlcv) analyzed = strategy.advise_all_indicators( {p: data[(p, strategy.timeframe, candle_def)] for p in ('XRP/USDT', 'LTC/USDT')}) expected_columns = [ 'rsi_1h', 'rsi_30m', # Stacked informative decorators 'neo_usdt_rsi_1h', # NEO 1h informative 'rsi_NEO_USDT_neo_usdt_NEO/USDT_30m', # Column formatting 'rsi_from_callable', # Custom column formatter 'eth_btc_rsi_1h', # Quote currency not matching stake currency 'rsi', 'rsi_less', # Non-informative columns 'rsi_5m', # Manual informative dataframe ] for _, dataframe in analyzed.items(): for col in expected_columns: assert col in dataframe.columns
def test_CandleType_get_default(input, expected): assert CandleType.get_default(input) == expected
def test_CandleType_from_string(input, expected): assert CandleType.from_string(input) == expected