def test_json_pair_data_filename(pair, expected_result, candle_type):
fn = JsonDataHandler._pair_data_filename(
Path('freqtrade/hello/world'), pair, '5m',
CandleType.from_string(candle_type))
assert isinstance(fn, Path)
assert fn == Path(expected_result)
fn = JsonGzDataHandler._pair_data_filename(
Path('freqtrade/hello/world'),
pair,
'5m',
candle_type=CandleType.from_string(candle_type))
assert isinstance(fn, Path)
assert fn == Path(expected_result + '.gz')
def test_dp_ohlcv(mocker, default_conf, ohlcv_history, candle_type):
default_conf["runmode"] = RunMode.DRY_RUN
timeframe = default_conf["timeframe"]
exchange = get_patched_exchange(mocker, default_conf)
candletype = CandleType.from_string(candle_type)
exchange._klines[("XRP/BTC", timeframe, candletype)] = ohlcv_history
exchange._klines[("UNITTEST/BTC", timeframe, candletype)] = ohlcv_history
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.DRY_RUN
assert ohlcv_history.equals(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype))
assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype), DataFrame)
assert dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype) is not ohlcv_history
assert dp.ohlcv("UNITTEST/BTC", timeframe, copy=False, candle_type=candletype) is ohlcv_history
assert not dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype).empty
assert dp.ohlcv("NONESENSE/AAA", timeframe, candle_type=candletype).empty
# Test with and without parameter
assert dp.ohlcv(
"UNITTEST/BTC",
timeframe,
candle_type=candletype
).equals(dp.ohlcv("UNITTEST/BTC", candle_type=candle_type))
default_conf["runmode"] = RunMode.LIVE
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.LIVE
assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame)
default_conf["runmode"] = RunMode.BACKTEST
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.BACKTEST
assert dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type).empty
def historic_ohlcv(self,
pair: str,
timeframe: str = None,
candle_type: str = '') -> DataFrame:
"""
Get stored historical candle (OHLCV) data
:param pair: pair to get the data for
:param timeframe: timeframe to get data for
:param candle_type: '', mark, index, premiumIndex, or funding_rate
"""
_candle_type = CandleType.from_string(
candle_type
) if candle_type != '' else self._config['candle_type_def']
saved_pair = (pair, str(timeframe), _candle_type)
if saved_pair not in self.__cached_pairs_backtesting:
timerange = TimeRange.parse_timerange(None if self._config.get(
'timerange') is None else str(self._config.get('timerange')))
# Move informative start time respecting startup_candle_count
timerange.subtract_start(
timeframe_to_seconds(str(timeframe)) *
self._config.get('startup_candle_count', 0))
self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
pair=pair,
timeframe=timeframe or self._config['timeframe'],
datadir=self._config['datadir'],
timerange=timerange,
data_format=self._config.get('dataformat_ohlcv', 'json'),
candle_type=_candle_type,
)
return self.__cached_pairs_backtesting[saved_pair].copy()
def ohlcv(self,
pair: str,
timeframe: str = None,
copy: bool = True,
candle_type: str = '') -> DataFrame:
"""
Get candle (OHLCV) data for the given pair as DataFrame
Please use the `available_pairs` method to verify which pairs are currently cached.
:param pair: pair to get the data for
:param timeframe: Timeframe to get data for
:param candle_type: '', mark, index, premiumIndex, or funding_rate
:param copy: copy dataframe before returning if True.
Use False only for read-only operations (where the dataframe is not modified)
"""
if self._exchange is None:
raise OperationalException(NO_EXCHANGE_EXCEPTION)
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
_candle_type = CandleType.from_string(
candle_type
) if candle_type != '' else self._config['candle_type_def']
return self._exchange.klines(
(pair, timeframe or self._config['timeframe'], _candle_type),
copy=copy)
else:
return DataFrame()
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
datadir: Path, trading_mode: str,
timerange: Optional[TimeRange] = None,
new_pairs_days: int = 30, erase: bool = False,
data_format: str = None,
) -> List[str]:
"""
Refresh stored ohlcv data for backtesting and hyperopt operations.
Used by freqtrade download-data subcommand.
:return: List of pairs that are not available.
"""
pairs_not_available = []
data_handler = get_datahandler(datadir, data_format)
candle_type = CandleType.get_default(trading_mode)
for idx, pair in enumerate(pairs, start=1):
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
continue
for timeframe in timeframes:
logger.info(f'Downloading pair {pair}, interval {timeframe}.')
process = f'{idx}/{len(pairs)}'
_download_pair_history(pair=pair, process=process,
datadir=datadir, exchange=exchange,
timerange=timerange, data_handler=data_handler,
timeframe=str(timeframe), new_pairs_days=new_pairs_days,
candle_type=candle_type,
erase=erase)
if trading_mode == 'futures':
# Predefined candletype (and timeframe) depending on exchange
# Downloads what is necessary to backtest based on futures data.
tf_mark = exchange._ft_has['mark_ohlcv_timeframe']
fr_candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price'])
# All exchanges need FundingRate for futures trading.
# The timeframe is aligned to the mark-price timeframe.
for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type):
_download_pair_history(pair=pair, process=process,
datadir=datadir, exchange=exchange,
timerange=timerange, data_handler=data_handler,
timeframe=str(tf_mark), new_pairs_days=new_pairs_days,
candle_type=funding_candle_type,
erase=erase)
return pairs_not_available
def _process_data_options(self, config: Dict[str, Any]) -> None:
self._args_to_config(config,
argname='new_pairs_days',
logstring='Detected --new-pairs-days: {}')
self._args_to_config(config,
argname='trading_mode',
logstring='Detected --trading-mode: {}')
config['candle_type_def'] = CandleType.get_default(
config.get('trading_mode', 'spot') or 'spot')
config['trading_mode'] = TradingMode(
config.get('trading_mode', 'spot') or 'spot')
self._args_to_config(config,
argname='candle_types',
logstring='Detected --candle-types: {}')
def informative(timeframe: str, asset: str = '',
fmt: Optional[Union[str, Callable[[Any], str]]] = None,
*,
candle_type: Optional[CandleType] = None,
ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]:
"""
A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to
define informative indicators.
Example usage:
@informative('1h')
def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14)
return dataframe
:param timeframe: Informative timeframe. Must always be equal or higher than strategy timeframe.
:param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use
current pair.
:param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not
specified, defaults to:
* {base}_{quote}_{column}_{timeframe} if asset is specified.
* {column}_{timeframe} if asset is not specified.
Format string supports these format variables:
* {asset} - full name of the asset, for example 'BTC/USDT'.
* {base} - base currency in lower case, for example 'eth'.
* {BASE} - same as {base}, except in upper case.
* {quote} - quote currency in lower case, for example 'usdt'.
* {QUOTE} - same as {quote}, except in upper case.
* {column} - name of dataframe column.
* {timeframe} - timeframe of informative dataframe.
:param ffill: ffill dataframe after merging informative pair.
:param candle_type: '', mark, index, premiumIndex, or funding_rate
"""
_asset = asset
_timeframe = timeframe
_fmt = fmt
_ffill = ffill
_candle_type = CandleType.from_string(candle_type) if candle_type else None
def decorator(fn: PopulateIndicators):
informative_pairs = getattr(fn, '_ft_informative', [])
informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill, _candle_type))
setattr(fn, '_ft_informative', informative_pairs)
return fn
return decorator
def ohlcv_get_available_data(
cls, datadir: Path,
trading_mode: TradingMode) -> ListPairsWithTimeframes:
"""
Returns a list of all pairs with ohlcv data available in this datadir
:param datadir: Directory to search for ohlcv files
:param trading_mode: trading-mode to be used
:return: List of Tuples of (pair, timeframe)
"""
if trading_mode == TradingMode.FUTURES:
datadir = datadir.joinpath('futures')
_tmp = [
re.search(cls._OHLCV_REGEX, p.name) for p in datadir.glob("*.h5")
]
return [(cls.rebuild_pair_from_filename(match[1]), match[2],
CandleType.from_string(match[3])) for match in _tmp
if match and len(match.groups()) > 1]
def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
"""
Convert data from one format to another
"""
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if ohlcv:
candle_types = [
CandleType.from_string(ct)
for ct in config.get('candle_types', ['spot'])
]
for candle_type in candle_types:
convert_ohlcv_format(config,
convert_from=args['format_from'],
convert_to=args['format_to'],
erase=args['erase'],
candle_type=candle_type)
else:
convert_trades_format(config,
convert_from=args['format_from'],
convert_to=args['format_to'],
erase=args['erase'])
def test_get_pair_dataframe(mocker, default_conf, ohlcv_history, candle_type):
default_conf["runmode"] = RunMode.DRY_RUN
timeframe = default_conf["timeframe"]
exchange = get_patched_exchange(mocker, default_conf)
candletype = CandleType.from_string(candle_type)
exchange._klines[("XRP/BTC", timeframe, candletype)] = ohlcv_history
exchange._klines[("UNITTEST/BTC", timeframe, candletype)] = ohlcv_history
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.DRY_RUN
assert ohlcv_history.equals(dp.get_pair_dataframe(
"UNITTEST/BTC", timeframe, candle_type=candle_type))
assert ohlcv_history.equals(dp.get_pair_dataframe(
"UNITTEST/BTC", timeframe, candle_type=candletype))
assert isinstance(dp.get_pair_dataframe(
"UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame)
assert dp.get_pair_dataframe("UNITTEST/BTC", timeframe,
candle_type=candle_type) is not ohlcv_history
assert not dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type).empty
assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe, candle_type=candle_type).empty
# Test with and without parameter
assert dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type)\
.equals(dp.get_pair_dataframe("UNITTEST/BTC", candle_type=candle_type))
default_conf["runmode"] = RunMode.LIVE
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.LIVE
assert isinstance(dp.get_pair_dataframe(
"UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame)
assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe, candle_type=candle_type).empty
historymock = MagicMock(return_value=ohlcv_history)
mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock)
default_conf["runmode"] = RunMode.BACKTEST
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.BACKTEST
assert isinstance(dp.get_pair_dataframe(
"UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame)
def list_available_pairs(timeframe: Optional[str] = None,
stake_currency: Optional[str] = None,
candletype: Optional[CandleType] = None,
config=Depends(get_config)):
dh = get_datahandler(config['datadir'], config.get('dataformat_ohlcv',
None))
trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT)
pair_interval = dh.ohlcv_get_available_data(config['datadir'],
trading_mode)
if timeframe:
pair_interval = [
pair for pair in pair_interval if pair[1] == timeframe
]
if stake_currency:
pair_interval = [
pair for pair in pair_interval if pair[0].endswith(stake_currency)
]
if candletype:
pair_interval = [
pair for pair in pair_interval if pair[2] == candletype
]
else:
candle_type = CandleType.get_default(trading_mode)
pair_interval = [
pair for pair in pair_interval if pair[2] == candle_type
]
pair_interval = sorted(pair_interval, key=lambda x: x[0])
pairs = list({x[0] for x in pair_interval})
pairs.sort()
result = {
'length': len(pairs),
'pairs': pairs,
'pair_interval': pair_interval,
}
return result
def test_historic_ohlcv(pair, timeframe, candle_type):
return data[
(pair, timeframe or strategy.timeframe, CandleType.from_string(candle_type))].copy()
def test_informative_decorator(mocker, default_conf_usdt, trading_mode):
candle_def = CandleType.get_default(trading_mode)
default_conf_usdt['candle_type_def'] = candle_def
test_data_5m = generate_test_data('5m', 40)
test_data_30m = generate_test_data('30m', 40)
test_data_1h = generate_test_data('1h', 40)
data = {
('XRP/USDT', '5m', candle_def): test_data_5m,
('XRP/USDT', '30m', candle_def): test_data_30m,
('XRP/USDT', '1h', candle_def): test_data_1h,
('LTC/USDT', '5m', candle_def): test_data_5m,
('LTC/USDT', '30m', candle_def): test_data_30m,
('LTC/USDT', '1h', candle_def): test_data_1h,
('NEO/USDT', '30m', candle_def): test_data_30m,
('NEO/USDT', '5m', CandleType.SPOT): test_data_5m, # Explicit request with '' as candletype
('NEO/USDT', '15m', candle_def): test_data_5m, # Explicit request with '' as candletype
('NEO/USDT', '1h', candle_def): test_data_1h,
('ETH/USDT', '1h', candle_def): test_data_1h,
('ETH/USDT', '30m', candle_def): test_data_30m,
('ETH/BTC', '1h', CandleType.SPOT): test_data_1h, # Explicitly selected as spot
}
default_conf_usdt['strategy'] = 'InformativeDecoratorTest'
strategy = StrategyResolver.load_strategy(default_conf_usdt)
exchange = get_patched_exchange(mocker, default_conf_usdt)
strategy.dp = DataProvider({}, exchange, None)
mocker.patch.object(strategy.dp, 'current_whitelist', return_value=[
'XRP/USDT', 'LTC/USDT', 'NEO/USDT'
])
assert len(strategy._ft_informative) == 6 # Equal to number of decorators used
informative_pairs = [
('XRP/USDT', '1h', candle_def),
('LTC/USDT', '1h', candle_def),
('XRP/USDT', '30m', candle_def),
('LTC/USDT', '30m', candle_def),
('NEO/USDT', '1h', candle_def),
('NEO/USDT', '30m', candle_def),
('NEO/USDT', '5m', candle_def),
('NEO/USDT', '15m', candle_def),
('NEO/USDT', '2h', CandleType.FUTURES),
('ETH/BTC', '1h', CandleType.SPOT), # One candle remains as spot
('ETH/USDT', '30m', candle_def)]
for inf_pair in informative_pairs:
assert inf_pair in strategy.gather_informative_pairs()
def test_historic_ohlcv(pair, timeframe, candle_type):
return data[
(pair, timeframe or strategy.timeframe, CandleType.from_string(candle_type))].copy()
mocker.patch('freqtrade.data.dataprovider.DataProvider.historic_ohlcv',
side_effect=test_historic_ohlcv)
analyzed = strategy.advise_all_indicators(
{p: data[(p, strategy.timeframe, candle_def)] for p in ('XRP/USDT', 'LTC/USDT')})
expected_columns = [
'rsi_1h', 'rsi_30m', # Stacked informative decorators
'neo_usdt_rsi_1h', # NEO 1h informative
'rsi_NEO_USDT_neo_usdt_NEO/USDT_30m', # Column formatting
'rsi_from_callable', # Custom column formatter
'eth_btc_rsi_1h', # Quote currency not matching stake currency
'rsi', 'rsi_less', # Non-informative columns
'rsi_5m', # Manual informative dataframe
]
for _, dataframe in analyzed.items():
for col in expected_columns:
assert col in dataframe.columns
def test_CandleType_get_default(input, expected):
assert CandleType.get_default(input) == expected
def test_CandleType_from_string(input, expected):
assert CandleType.from_string(input) == expected