def test_backtest(default_conf, fee, mocker, testdatadir) -> None:
default_conf['ask_strategy']['use_sell_signal'] = False
mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
patch_exchange(mocker)
backtesting = Backtesting(default_conf)
pair = 'UNITTEST/BTC'
timerange = TimeRange('date', None, 1517227800, 0)
data = history.load_data(datadir=testdatadir,
timeframe='5m',
pairs=['UNITTEST/BTC'],
timerange=timerange)
data_processed = backtesting.strategy.tickerdata_to_dataframe(data)
min_date, max_date = get_timerange(data_processed)
results = backtesting.backtest(
processed=data_processed,
stake_amount=default_conf['stake_amount'],
start_date=min_date,
end_date=max_date,
max_open_trades=10,
position_stacking=False,
)
assert not results.empty
assert len(results) == 2
expected = pd.DataFrame({
'pair': [pair, pair],
'profit_percent': [0.0, 0.0],
'profit_abs': [0.0, 0.0],
'open_time':
pd.to_datetime([
Arrow(2018, 1, 29, 18, 40, 0).datetime,
Arrow(2018, 1, 30, 3, 30, 0).datetime
],
utc=True),
'close_time':
pd.to_datetime([
Arrow(2018, 1, 29, 22, 35, 0).datetime,
Arrow(2018, 1, 30, 4, 10, 0).datetime
],
utc=True),
'open_index': [78, 184],
'close_index': [125, 192],
'trade_duration': [235, 40],
'open_at_end': [False, False],
'open_rate': [0.104445, 0.10302485],
'close_rate': [0.104969, 0.103541],
'sell_reason': [SellType.ROI, SellType.ROI]
})
pd.testing.assert_frame_equal(results, expected)
data_pair = data_processed[pair]
for _, t in results.iterrows():
ln = data_pair.loc[data_pair["date"] == t["open_time"]]
# Check open trade rate alignes to open rate
assert ln is not None
assert round(ln.iloc[0]["open"], 6) == round(t["open_rate"], 6)
# check close trade rate alignes to close rate or is between high and low
ln = data_pair.loc[data_pair["date"] == t["close_time"]]
assert (round(ln.iloc[0]["open"], 6) == round(t["close_rate"], 6)
or round(ln.iloc[0]["low"], 6) < round(
t["close_rate"], 6) < round(ln.iloc[0]["high"], 6))
def test_backtest_1min_ticker_interval(default_conf, fee, mocker,
testdatadir) -> None:
default_conf['ask_strategy']['use_sell_signal'] = False
mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
patch_exchange(mocker)
backtesting = Backtesting(default_conf)
# Run a backtesting for an exiting 1min timeframe
timerange = TimeRange.parse_timerange('1510688220-1510700340')
data = history.load_data(datadir=testdatadir,
timeframe='1m',
pairs=['UNITTEST/BTC'],
timerange=timerange)
processed = backtesting.strategy.tickerdata_to_dataframe(data)
min_date, max_date = get_timerange(processed)
results = backtesting.backtest(
processed=processed,
stake_amount=default_conf['stake_amount'],
start_date=min_date,
end_date=max_date,
max_open_trades=1,
position_stacking=False,
)
assert not results.empty
assert len(results) == 1
def test_sell_strategy_generator(hyperopt, testdatadir) -> None:
data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True)
dataframes = hyperopt.backtesting.strategy.ohlcvdata_to_dataframe(data)
dataframe = hyperopt.custom_hyperopt.populate_indicators(
dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'})
populate_sell_trend = hyperopt.custom_hyperopt.sell_strategy_generator({
'sell-adx-value':
20,
'sell-fastd-value':
75,
'sell-mfi-value':
80,
'sell-rsi-value':
20,
'sell-adx-enabled':
True,
'sell-fastd-enabled':
True,
'sell-mfi-enabled':
True,
'sell-rsi-enabled':
True,
'sell-trigger':
'sell-bb_upper'
})
result = populate_sell_trend(dataframe, {'pair': 'UNITTEST/BTC'})
# Check if some indicators are generated. We will not test all of them
print(result)
assert 'sell' in result
assert 1 in result['sell']
def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]:
"""
Loads backtest data and returns the data combined with the timerange
as tuple.
"""
timerange = TimeRange.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
data = history.load_data(
datadir=self.config['datadir'],
pairs=self.pairlists.whitelist,
timeframe=self.timeframe,
timerange=timerange,
startup_candles=self.required_startup,
fail_without_data=True,
data_format=self.config.get('dataformat_ohlcv', 'json'),
)
min_date, max_date = history.get_timerange(data)
logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..')
# Adjust startts forward if not enough data is available
timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
self.required_startup, min_date)
return data, timerange
def test_buy_strategy_generator(hyperopt, testdatadir) -> None:
data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True)
dataframes = hyperopt.backtesting.strategy.ohlcvdata_to_dataframe(data)
dataframe = hyperopt.custom_hyperopt.populate_indicators(
dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'})
populate_buy_trend = hyperopt.custom_hyperopt.buy_strategy_generator({
'adx-value':
20,
'fastd-value':
20,
'mfi-value':
20,
'rsi-value':
20,
'adx-enabled':
True,
'fastd-enabled':
True,
'mfi-enabled':
True,
'rsi-enabled':
True,
'trigger':
'bb_lower'
})
result = populate_buy_trend(dataframe, {'pair': 'UNITTEST/BTC'})
# Check if some indicators are generated. We will not test all of them
assert 'buy' in result
assert 1 in result['buy']
def test_generate_profit_graph(testdatadir):
filename = testdatadir / "backtest-result_test.json"
trades = load_backtest_data(filename)
timerange = TimeRange.parse_timerange("20180110-20180112")
pairs = ["TRX/BTC", "ADA/BTC"]
tickers = history.load_data(datadir=testdatadir,
pairs=pairs,
timeframe='5m',
timerange=timerange)
trades = trades[trades['pair'].isin(pairs)]
fig = generate_profit_graph(pairs, tickers, trades, timeframe="5m")
assert isinstance(fig, go.Figure)
assert fig.layout.title.text == "Freqtrade Profit plot"
assert fig.layout.yaxis.title.text == "Price"
assert fig.layout.yaxis2.title.text == "Profit"
assert fig.layout.yaxis3.title.text == "Profit"
figure = fig.layout.figure
assert len(figure.data) == 4
avgclose = find_trace_in_fig_data(figure.data, "Avg close price")
assert isinstance(avgclose, go.Scatter)
profit = find_trace_in_fig_data(figure.data, "Profit")
assert isinstance(profit, go.Scatter)
for pair in pairs:
profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}")
assert isinstance(profit_pair, go.Scatter)
def init_plotscript(config):
"""
Initialize objects needed for plotting
:return: Dict with tickers, trades and pairs
"""
if "pairs" in config:
pairs = config["pairs"]
else:
pairs = config["exchange"]["pair_whitelist"]
# Set timerange to use
timerange = TimeRange.parse_timerange(config.get("timerange"))
tickers = history.load_data(
datadir=Path(str(config.get("datadir"))),
pairs=pairs,
timeframe=config.get('ticker_interval', '5m'),
timerange=timerange,
)
trades = load_trades(
config['trade_source'],
db_url=config.get('db_url'),
exportfilename=config.get('exportfilename'),
)
trades = history.trim_dataframe(trades, timerange, 'open_time')
return {
"tickers": tickers,
"trades": trades,
"pairs": pairs,
}
def load_bt_data(self):
timerange = TimeRange.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
data = history.load_data(
datadir=self.config['datadir'],
pairs=self.config['exchange']['pair_whitelist'],
timeframe=self.timeframe,
timerange=timerange,
startup_candles=self.required_startup,
fail_without_data=True,
data_format=self.config.get('dataformat_ohlcv', 'json'),
)
min_date, max_date = history.get_timerange(data)
logger.info('Loading data from %s up to %s (%s days)..',
min_date.isoformat(), max_date.isoformat(),
(max_date - min_date).days)
# Adjust startts forward if not enough data is available
timerange.adjust_start_if_necessary(
timeframe_to_seconds(self.timeframe), self.required_startup,
min_date)
return data, timerange
def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair):
def _trend_alternate_hold(dataframe=None, metadata=None):
"""
Buy every xth candle - sell every other xth -2 (hold on to pairs a bit)
"""
if metadata['pair'] in ('ETH/BTC', 'LTC/BTC'):
multi = 20
else:
multi = 18
dataframe['buy'] = np.where(dataframe.index % multi == 0, 1, 0)
dataframe['sell'] = np.where(
(dataframe.index + multi - 2) % multi == 0, 1, 0)
return dataframe
mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
patch_exchange(mocker)
pairs = ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC', 'NXT/BTC']
data = history.load_data(datadir=None, ticker_interval='5m', pairs=pairs)
# Only use 500 lines to increase performance
data = trim_dictlist(data, -500)
# Remove data for one pair from the beginning of the data
data[pair] = data[pair][tres:].reset_index()
# We need to enable sell-signal - otherwise it sells on ROI!!
default_conf['experimental'] = {"use_sell_signal": True}
default_conf['ticker_interval'] = '5m'
backtesting = Backtesting(default_conf)
backtesting.advise_buy = _trend_alternate_hold # Override
backtesting.advise_sell = _trend_alternate_hold # Override
data_processed = backtesting.strategy.tickerdata_to_dataframe(data)
min_date, max_date = get_timeframe(data_processed)
backtest_conf = {
'stake_amount': default_conf['stake_amount'],
'processed': data_processed,
'max_open_trades': 3,
'position_stacking': False,
'start_date': min_date,
'end_date': max_date,
}
results = backtesting.backtest(backtest_conf)
# Make sure we have parallel trades
assert len(evaluate_result_multi(results, '5min', 2)) > 0
# make sure we don't have trades with more than configured max_open_trades
assert len(evaluate_result_multi(results, '5min', 3)) == 0
backtest_conf = {
'stake_amount': default_conf['stake_amount'],
'processed': data_processed,
'max_open_trades': 1,
'position_stacking': False,
'start_date': min_date,
'end_date': max_date,
}
results = backtesting.backtest(backtest_conf)
assert len(evaluate_result_multi(results, '5min', 1)) == 0
def calculate(self) -> bool:
pairs = self.config['exchange']['pair_whitelist']
heartbeat = self.edge_config.get('process_throttle_secs')
if (self._last_updated > 0) and (
self._last_updated + heartbeat > arrow.utcnow().timestamp):
return False
data: Dict[str, Any] = {}
logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
logger.info('Using local backtesting data (using whitelist in given config) ...')
data = history.load_data(
datadir=Path(self.config['datadir']) if self.config.get('datadir') else None,
pairs=pairs,
ticker_interval=self.ticker_interval,
refresh_pairs=self._refresh_pairs,
exchange=self.exchange,
timerange=self._timerange
)
if not data:
# Reinitializing cached pairs
self._cached_pairs = {}
logger.critical("No data found. Edge is stopped ...")
return False
preprocessed = self.tickerdata_to_dataframe(data)
# Print timeframe
min_date, max_date = self.get_timeframe(preprocessed)
logger.info(
'Measuring data from %s up to %s (%s days) ...',
min_date.isoformat(),
max_date.isoformat(),
(max_date - min_date).days
)
headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']
trades: list = []
for pair, pair_data in preprocessed.items():
# Sorting dataframe by date and reset index
pair_data = pair_data.sort_values(by=['date'])
pair_data = pair_data.reset_index(drop=True)
ticker_data = self.advise_sell(
self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range)
# If no trade found then exit
if len(trades) == 0:
return False
# Fill missing, calculable columns, profit, duration , abs etc.
trades_df = self._fill_calculable_fields(DataFrame(trades))
self._cached_pairs = self._process_expectancy(trades_df)
self._last_updated = arrow.utcnow().timestamp
return True
def test_generate_profit_graph():
filename = history.make_testdata_path(None) / "backtest-result_test.json"
trades = load_backtest_data(filename)
timerange = TimeRange.parse_timerange("20180110-20180112")
pairs = ["POWR/BTC", "XLM/BTC"]
tickers = history.load_data(datadir=None,
pairs=pairs,
ticker_interval='5m',
timerange=timerange
)
trades = trades[trades['pair'].isin(pairs)]
fig = generate_profit_graph(pairs, tickers, trades)
assert isinstance(fig, go.Figure)
assert fig.layout.title.text == "Profit plot"
figure = fig.layout.figure
assert len(figure.data) == 4
avgclose = find_trace_in_fig_data(figure.data, "Avg close price")
assert isinstance(avgclose, go.Scattergl)
profit = find_trace_in_fig_data(figure.data, "Profit")
assert isinstance(profit, go.Scattergl)
for pair in pairs:
profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}")
assert isinstance(profit_pair, go.Scattergl)
def analyse_and_plot_pairs(config: Dict[str, Any]):
"""
From arguments provided in cli:
-Initialise backtest env
-Get tickers data
-Generate Dafaframes populated with indicators and signals
-Load trades excecuted on same periods
-Generate Plotly plot objects
-Generate plot files
:return: None
"""
exchange = ExchangeResolver(config.get('exchange', {}).get('name'), config).exchange
strategy = StrategyResolver(config).strategy
if "pairs" in config:
pairs = config["pairs"].split(',')
else:
pairs = config["exchange"]["pair_whitelist"]
# Set timerange to use
timerange = Arguments.parse_timerange(config["timerange"])
ticker_interval = strategy.ticker_interval
tickers = history.load_data(
datadir=Path(str(config.get("datadir"))),
pairs=pairs,
ticker_interval=config['ticker_interval'],
refresh_pairs=config.get('refresh_pairs', False),
timerange=timerange,
exchange=exchange,
live=config.get("live", False),
)
pair_counter = 0
for pair, data in tickers.items():
pair_counter += 1
logger.info("analyse pair %s", pair)
tickers = {}
tickers[pair] = data
dataframe = generate_dataframe(strategy, tickers, pair)
if config["trade_source"] == "DB":
trades = load_trades_from_db(config["db_url"])
elif config["trade_source"] == "file":
trades = load_backtest_data(Path(config["exportfilename"]))
trades = trades.loc[trades['pair'] == pair]
trades = extract_trades_of_period(dataframe, trades)
fig = generate_graph(
pair=pair,
data=dataframe,
trades=trades,
indicators1=config["indicators1"].split(","),
indicators2=config["indicators2"].split(",")
)
generate_plot_file(fig, pair, ticker_interval)
logger.info('End of ploting process %s plots generated', pair_counter)
def test_combine_tickers_with_mean(testdatadir):
pairs = ["ETH/BTC", "XLM/BTC"]
tickers = load_data(datadir=testdatadir, pairs=pairs, ticker_interval='5m')
df = combine_tickers_with_mean(tickers)
assert isinstance(df, DataFrame)
assert "ETH/BTC" in df.columns
assert "XLM/BTC" in df.columns
assert "mean" in df.columns
def test_combine_dataframes_with_mean(testdatadir):
pairs = ["ETH/BTC", "ADA/BTC"]
data = load_data(datadir=testdatadir, pairs=pairs, timeframe='5m')
df = combine_dataframes_with_mean(data)
assert isinstance(df, DataFrame)
assert "ETH/BTC" in df.columns
assert "ADA/BTC" in df.columns
assert "mean" in df.columns
def test_init(default_conf, mocker) -> None:
exchange = get_patched_exchange(mocker, default_conf)
assert {} == history.load_data(
datadir='',
exchange=exchange,
pairs=[],
refresh_pairs=True,
ticker_interval=default_conf['ticker_interval'])
def test_get_timerange(default_conf, mocker, testdatadir) -> None:
patch_exchange(mocker)
strategy = DefaultStrategy(default_conf)
data = strategy.tickerdata_to_dataframe(
load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC']))
min_date, max_date = get_timerange(data)
assert min_date.isoformat() == '2017-11-04T23:02:00+00:00'
assert max_date.isoformat() == '2017-11-14T22:58:00+00:00'
def test_ohlcvdata_to_dataframe(default_conf, testdatadir) -> None:
default_conf.update({'strategy': 'DefaultStrategy'})
strategy = StrategyResolver.load_strategy(default_conf)
timerange = TimeRange.parse_timerange('1510694220-1510700340')
data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange,
fill_up_missing=True)
processed = strategy.ohlcvdata_to_dataframe(data)
assert len(processed['UNITTEST/BTC']) == 102 # partial candle was removed
def init_plotscript(config, markets: List, startup_candles: int = 0):
"""
Initialize objects needed for plotting
:return: Dict with candle (OHLCV) data, trades and pairs
"""
if "pairs" in config:
pairs = expand_pairlist(config['pairs'], markets)
else:
pairs = expand_pairlist(config['exchange']['pair_whitelist'], markets)
# Set timerange to use
timerange = TimeRange.parse_timerange(config.get('timerange'))
data = load_data(
datadir=config.get('datadir'),
pairs=pairs,
timeframe=config['timeframe'],
timerange=timerange,
startup_candles=startup_candles,
data_format=config.get('dataformat_ohlcv', 'json'),
)
if startup_candles and data:
min_date, max_date = get_timerange(data)
logger.info(f"Loading data from {min_date} to {max_date}")
timerange.adjust_start_if_necessary(
timeframe_to_seconds(config['timeframe']), startup_candles,
min_date)
no_trades = False
filename = config.get('exportfilename')
if config.get('no_trades', False):
no_trades = True
elif config['trade_source'] == 'file':
if not filename.is_dir() and not filename.is_file():
logger.warning("Backtest file is missing skipping trades.")
no_trades = True
try:
trades = load_trades(
config['trade_source'],
db_url=config.get('db_url'),
exportfilename=filename,
no_trades=no_trades,
strategy=config.get('strategy'),
)
except ValueError as e:
raise OperationalException(e) from e
if not trades.empty:
trades = trim_dataframe(trades, timerange, 'open_date')
return {
"ohlcv": data,
"trades": trades,
"pairs": pairs,
"timerange": timerange,
}
def test_populate_indicators(hyperopt, testdatadir) -> None:
data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True)
dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data)
dataframe = dataframes['UNITTEST/BTC']
# Check if some indicators are generated. We will not test all of them
assert 'adx' in dataframe
assert 'macd' in dataframe
assert 'rsi' in dataframe
def test_init_with_refresh(default_conf, mocker) -> None:
exchange = get_patched_exchange(mocker, default_conf)
refresh_data(datadir='',
pairs=[],
timeframe=default_conf['ticker_interval'],
exchange=exchange)
assert {} == load_data(datadir='',
pairs=[],
timeframe=default_conf['ticker_interval'])
def test_advise_all_indicators(default_conf, testdatadir) -> None:
strategy = StrategyResolver.load_strategy(default_conf)
timerange = TimeRange.parse_timerange('1510694220-1510700340')
data = load_data(testdatadir,
'1m', ['UNITTEST/BTC'],
timerange=timerange,
fill_up_missing=True)
processed = strategy.advise_all_indicators(data)
assert len(processed['UNITTEST/BTC']) == 102 # partial candle was removed
def test_populate_indicators(hyperopt, testdatadir) -> None:
data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True)
dataframes = hyperopt.backtesting.strategy.ohlcvdata_to_dataframe(data)
dataframe = hyperopt.custom_hyperopt.populate_indicators(
dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'})
# Check if some indicators are generated. We will not test all of them
assert 'adx' in dataframe
assert 'mfi' in dataframe
assert 'rsi' in dataframe
def test_generate_profit_graph(testdatadir):
filename = testdatadir / "backtest_results/backtest-result_new.json"
trades = load_backtest_data(filename)
timerange = TimeRange.parse_timerange("20180110-20180112")
pairs = ["TRX/BTC", "XLM/BTC"]
trades = trades[
trades['close_date'] < pd.Timestamp('2018-01-12', tz='UTC')]
data = history.load_data(datadir=testdatadir,
pairs=pairs,
timeframe='5m',
timerange=timerange)
trades = trades[trades['pair'].isin(pairs)]
fig = generate_profit_graph(pairs,
data,
trades,
timeframe="5m",
stake_currency='BTC')
assert isinstance(fig, go.Figure)
assert fig.layout.title.text == "Freqtrade Profit plot"
assert fig.layout.yaxis.title.text == "Price"
assert fig.layout.yaxis2.title.text == "Profit BTC"
assert fig.layout.yaxis3.title.text == "Profit BTC"
figure = fig.layout.figure
assert len(figure.data) == 7
avgclose = find_trace_in_fig_data(figure.data, "Avg close price")
assert isinstance(avgclose, go.Scatter)
profit = find_trace_in_fig_data(figure.data, "Profit")
assert isinstance(profit, go.Scatter)
drawdown = find_trace_in_fig_data(figure.data, "Max drawdown 35.69%")
assert isinstance(drawdown, go.Scatter)
parallel = find_trace_in_fig_data(figure.data, "Parallel trades")
assert isinstance(parallel, go.Scatter)
underwater = find_trace_in_fig_data(figure.data, "Underwater Plot")
assert isinstance(underwater, go.Scatter)
for pair in pairs:
profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}")
assert isinstance(profit_pair, go.Scatter)
with pytest.raises(OperationalException, match=r"No trades found.*"):
# Pair cannot be empty - so it's an empty dataframe.
generate_profit_graph(pairs,
data,
trades.loc[trades['pair'].isnull()],
timeframe="5m",
stake_currency='BTC')
def test_ohlcvdata_to_dataframe_copy(mocker, default_conf, testdatadir) -> None:
default_conf.update({'strategy': 'DefaultStrategy'})
strategy = StrategyResolver.load_strategy(default_conf)
aimock = mocker.patch('freqtrade.strategy.interface.IStrategy.advise_indicators')
timerange = TimeRange.parse_timerange('1510694220-1510700340')
data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange,
fill_up_missing=True)
strategy.ohlcvdata_to_dataframe(data)
assert aimock.call_count == 1
# Ensure that a copy of the dataframe is passed to advice_indicators
assert aimock.call_args_list[0][0][0] is not data
def test_load_partial_missing(caplog) -> None:
# Make sure we start fresh - test missing data at start
start = arrow.get('2018-01-01T00:00:00')
end = arrow.get('2018-01-11T00:00:00')
tickerdata = history.load_data(None,
'5m', ['UNITTEST/BTC'],
refresh_pairs=False,
timerange=TimeRange('date', 'date',
start.timestamp,
end.timestamp))
# timedifference in 5 minutes
td = ((end - start).total_seconds() // 60 // 5) + 1
assert td != len(tickerdata['UNITTEST/BTC'])
start_real = tickerdata['UNITTEST/BTC'].iloc[0, 0]
assert log_has(
f'Missing data at start for pair '
f'UNITTEST/BTC, data starts at {start_real.strftime("%Y-%m-%d %H:%M:%S")}',
caplog.record_tuples)
# Make sure we start fresh - test missing data at end
caplog.clear()
start = arrow.get('2018-01-10T00:00:00')
end = arrow.get('2018-02-20T00:00:00')
tickerdata = history.load_data(datadir=None,
ticker_interval='5m',
pairs=['UNITTEST/BTC'],
refresh_pairs=False,
timerange=TimeRange('date', 'date',
start.timestamp,
end.timestamp))
# timedifference in 5 minutes
td = ((end - start).total_seconds() // 60 // 5) + 1
assert td != len(tickerdata['UNITTEST/BTC'])
# Shift endtime with +5 - as last candle is dropped (partial candle)
end_real = arrow.get(tickerdata['UNITTEST/BTC'].iloc[-1,
0]).shift(minutes=5)
assert log_has(
f'Missing data at end for pair '
f'UNITTEST/BTC, data ends at {end_real.strftime("%Y-%m-%d %H:%M:%S")}',
caplog.record_tuples)
def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC'):
data = history.load_data(datadir=datadir, timeframe='1m', pairs=[pair])
data = trim_dictlist(data, -201)
patch_exchange(mocker)
backtesting = Backtesting(conf)
processed = backtesting.strategy.ohlcvdata_to_dataframe(data)
min_date, max_date = get_timerange(processed)
return {
'processed': processed,
'start_date': min_date,
'end_date': max_date,
'max_open_trades': 10,
'position_stacking': False,
}
def test_data_to_dataframe_bt(default_conf, mocker, testdatadir) -> None:
patch_exchange(mocker)
timerange = TimeRange.parse_timerange('1510694220-1510700340')
data = history.load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange,
fill_up_missing=True)
backtesting = Backtesting(default_conf)
processed = backtesting.strategy.ohlcvdata_to_dataframe(data)
assert len(processed['UNITTEST/BTC']) == 102
# Load strategy to compare the result between Backtesting function and strategy are the same
default_conf.update({'strategy': 'DefaultStrategy'})
strategy = StrategyResolver.load_strategy(default_conf)
processed2 = strategy.ohlcvdata_to_dataframe(data)
assert processed['UNITTEST/BTC'].equals(processed2['UNITTEST/BTC'])
def test_trim_dataframe(testdatadir) -> None:
data = load_data(
datadir=testdatadir,
timeframe='1m',
pairs=['UNITTEST/BTC']
)['UNITTEST/BTC']
min_date = int(data.iloc[0]['date'].timestamp())
max_date = int(data.iloc[-1]['date'].timestamp())
data_modify = data.copy()
# Remove first 30 minutes (1800 s)
tr = TimeRange('date', None, min_date + 1800, 0)
data_modify = trim_dataframe(data_modify, tr)
assert not data_modify.equals(data)
assert len(data_modify) < len(data)
assert len(data_modify) == len(data) - 30
assert all(data_modify.iloc[-1] == data.iloc[-1])
assert all(data_modify.iloc[0] == data.iloc[30])
data_modify = data.copy()
tr = TimeRange('date', None, min_date + 1800, 0)
# Remove first 20 candles - ignores min date
data_modify = trim_dataframe(data_modify, tr, startup_candles=20)
assert not data_modify.equals(data)
assert len(data_modify) < len(data)
assert len(data_modify) == len(data) - 20
assert all(data_modify.iloc[-1] == data.iloc[-1])
assert all(data_modify.iloc[0] == data.iloc[20])
data_modify = data.copy()
# Remove last 30 minutes (1800 s)
tr = TimeRange(None, 'date', 0, max_date - 1800)
data_modify = trim_dataframe(data_modify, tr)
assert not data_modify.equals(data)
assert len(data_modify) < len(data)
assert len(data_modify) == len(data) - 30
assert all(data_modify.iloc[0] == data.iloc[0])
assert all(data_modify.iloc[-1] == data.iloc[-31])
data_modify = data.copy()
# Remove first 25 and last 30 minutes (1800 s)
tr = TimeRange('date', 'date', min_date + 1500, max_date - 1800)
data_modify = trim_dataframe(data_modify, tr)
assert not data_modify.equals(data)
assert len(data_modify) < len(data)
assert len(data_modify) == len(data) - 55
# first row matches 25th original row
assert all(data_modify.iloc[0] == data.iloc[25])
def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None):
data = history.load_data(datadir=None, ticker_interval='1m', pairs=[pair])
data = trim_dictlist(data, -201)
patch_exchange(mocker)
backtesting = Backtesting(conf)
processed = backtesting.strategy.tickerdata_to_dataframe(data)
min_date, max_date = get_timeframe(processed)
return {
'stake_amount': conf['stake_amount'],
'processed': processed,
'max_open_trades': 10,
'position_stacking': False,
'record': record,
'start_date': min_date,
'end_date': max_date,
}
def load_bt_data_detail(self) -> None:
"""
Loads backtest detail data (smaller timeframe) if necessary.
"""
if self.timeframe_detail:
self.detail_data = history.load_data(
datadir=self.config['datadir'],
pairs=self.pairlists.whitelist,
timeframe=self.timeframe_detail,
timerange=self.timerange,
startup_candles=0,
fail_without_data=True,
data_format=self.config.get('dataformat_ohlcv', 'json'),
)
else:
self.detail_data = {}
