def ohlcv_fill_up_missing_data(dataframe: DataFrame, ticker_interval: str) -> DataFrame:
"""
Fills up missing data with 0 volume rows,
using the previous close as price for "open", "high" "low" and "close", volume is set to 0
"""
ohlc_dict = {
'open': 'first',
'high': 'max',
'low': 'min',
'close': 'last',
'volume': 'sum'
}
ticker_minutes = timeframe_to_minutes(ticker_interval)
# Resample to create "NAN" values
df = dataframe.resample(f'{ticker_minutes}min', on='date').agg(ohlc_dict)
# Forwardfill close for missing columns
df['close'] = df['close'].fillna(method='ffill')
# Use close for "open, high, low"
df.loc[:, ['open', 'high', 'low']] = df[['open', 'high', 'low']].fillna(
value={'open': df['close'],
'high': df['close'],
'low': df['close'],
})
df.reset_index(inplace=True)
logger.debug(f"Missing data fillup: before: {len(dataframe)} - after: {len(df)}")
return df
def __init__(self, config: Dict[str, Any]) -> None:
self.config = config
# Reset keys for backtesting
self.config['exchange']['key'] = ''
self.config['exchange']['secret'] = ''
self.config['exchange']['password'] = ''
self.config['exchange']['uid'] = ''
self.config['dry_run'] = True
self.strategylist: List[IStrategy] = []
exchange_name = self.config.get('exchange', {}).get('name',
'bittrex').title()
self.exchange = ExchangeResolver(exchange_name, self.config).exchange
self.fee = self.exchange.get_fee()
if self.config.get('runmode') != RunMode.HYPEROPT:
self.dataprovider = DataProvider(self.config, self.exchange)
IStrategy.dp = self.dataprovider
if self.config.get('strategy_list', None):
# Force one interval
self.ticker_interval = str(self.config.get('ticker_interval'))
self.ticker_interval_mins = timeframe_to_minutes(
self.ticker_interval)
for strat in list(self.config['strategy_list']):
stratconf = deepcopy(self.config)
stratconf['strategy'] = strat
self.strategylist.append(StrategyResolver(stratconf).strategy)
else:
# only one strategy
self.strategylist.append(StrategyResolver(self.config).strategy)
# Load one strategy
self._set_strategy(self.strategylist[0])
def get_signal(self, pair: str, interval: str,
dataframe: DataFrame) -> Tuple[bool, bool]:
"""
Calculates current signal based several technical analysis indicators
:param pair: pair in format ANT/BTC
:param interval: Interval to use (in min)
:param dataframe: Dataframe to analyze
:return: (Buy, Sell) A bool-tuple indicating buy/sell signal
"""
if not isinstance(dataframe, DataFrame) or dataframe.empty:
logger.warning('Empty ticker history for pair %s', pair)
return False, False
try:
dataframe = self.analyze_ticker(dataframe, {'pair': pair})
except ValueError as error:
logger.warning(
'Unable to analyze ticker for pair %s: %s',
pair,
str(error)
)
return False, False
except Exception as error:
logger.exception(
'Unexpected error when analyzing ticker for pair %s: %s',
pair,
str(error)
)
return False, False
if dataframe.empty:
logger.warning('Empty dataframe for pair %s', pair)
return False, False
latest = dataframe.iloc[-1]
# Check if dataframe is out of date
signal_date = arrow.get(latest['date'])
interval_minutes = timeframe_to_minutes(interval)
offset = self.config.get('exchange', {}).get('outdated_offset', 5)
if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + offset))):
logger.warning(
'Outdated history for pair %s. Last tick is %s minutes old',
pair,
(arrow.utcnow() - signal_date).seconds // 60
)
return False, False
(buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1
logger.debug(
'trigger: %s (pair=%s) buy=%s sell=%s',
latest['date'],
pair,
str(buy),
str(sell)
)
return buy, sell
def _set_strategy(self, strategy):
"""
Load strategy into backtesting
"""
self.strategy = strategy
self.ticker_interval = self.config.get('ticker_interval')
self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval)
self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe
self.advise_buy = strategy.advise_buy
self.advise_sell = strategy.advise_sell
# Set stoploss_on_exchange to false for backtesting,
# since a "perfect" stoploss-sell is assumed anyway
# And the regular "stoploss" function would not apply to that case
self.strategy.order_types['stoploss_on_exchange'] = False
def test_validate_backtest_data(default_conf, mocker, caplog) -> None:
patch_exchange(mocker)
strategy = DefaultStrategy(default_conf)
timerange = TimeRange('index', 'index', 200, 250)
data = strategy.tickerdata_to_dataframe(
history.load_data(datadir=None,
ticker_interval='5m',
pairs=['UNITTEST/BTC'],
timerange=timerange))
min_date, max_date = optimize.get_timeframe(data)
caplog.clear()
assert not optimize.validate_backtest_data(data, min_date, max_date,
timeframe_to_minutes('5m'))
assert len(caplog.record_tuples) == 0
def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None:
patch_exchange(mocker)
strategy = DefaultStrategy(default_conf)
data = strategy.tickerdata_to_dataframe(
history.load_data(datadir=None,
ticker_interval='1m',
pairs=['UNITTEST/BTC'],
fill_up_missing=False))
min_date, max_date = optimize.get_timeframe(data)
caplog.clear()
assert optimize.validate_backtest_data(data, min_date, max_date,
timeframe_to_minutes('1m'))
assert len(caplog.record_tuples) == 1
assert log_has(
"UNITTEST/BTC has missing frames: expected 14396, got 13680, that's 716 missing values",
caplog.record_tuples)
def load_cached_data_for_updating(
filename: Path, ticker_interval: str,
timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]:
"""
Load cached data and choose what part of the data should be updated
"""
since_ms = None
# user sets timerange, so find the start time
if timerange:
if timerange.starttype == 'date':
since_ms = timerange.startts * 1000
elif timerange.stoptype == 'line':
num_minutes = timerange.stopts * timeframe_to_minutes(
ticker_interval)
since_ms = arrow.utcnow().shift(
minutes=num_minutes).timestamp * 1000
# read the cached file
if filename.is_file():
with open(filename, "rt") as file:
data = misc.json_load(file)
# remove the last item, could be incomplete candle
if data:
data.pop()
else:
data = []
if data:
if since_ms and since_ms < data[0][0]:
# Earlier data than existing data requested, redownload all
data = []
else:
# a part of the data was already downloaded, so download unexist data only
since_ms = data[-1][0] + 1
return (data, since_ms)
def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None) -> bool:
"""
Executes a limit buy for the given pair
:param pair: pair for which we want to create a LIMIT_BUY
:return: None
"""
pair_s = pair.replace('_', '/')
stake_currency = self.config['stake_currency']
fiat_currency = self.config.get('fiat_display_currency', None)
time_in_force = self.strategy.order_time_in_force['buy']
if price:
buy_limit_requested = price
else:
# Calculate amount
buy_limit_requested = self.get_target_bid(pair)
min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit_requested)
if min_stake_amount is not None and min_stake_amount > stake_amount:
logger.warning(
f'Can\'t open a new trade for {pair_s}: stake amount '
f'is too small ({stake_amount} < {min_stake_amount})'
)
return False
amount = stake_amount / buy_limit_requested
order = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'],
amount=amount, rate=buy_limit_requested,
time_in_force=time_in_force)
order_id = order['id']
order_status = order.get('status', None)
# we assume the order is executed at the price requested
buy_limit_filled_price = buy_limit_requested
if order_status == 'expired' or order_status == 'rejected':
order_type = self.strategy.order_types['buy']
order_tif = self.strategy.order_time_in_force['buy']
# return false if the order is not filled
if float(order['filled']) == 0:
logger.warning('Buy %s order with time in force %s for %s is %s by %s.'
' zero amount is fulfilled.',
order_tif, order_type, pair_s, order_status, self.exchange.name)
return False
else:
# the order is partially fulfilled
# in case of IOC orders we can check immediately
# if the order is fulfilled fully or partially
logger.warning('Buy %s order with time in force %s for %s is %s by %s.'
' %s amount fulfilled out of %s (%s remaining which is canceled).',
order_tif, order_type, pair_s, order_status, self.exchange.name,
order['filled'], order['amount'], order['remaining']
)
stake_amount = order['cost']
amount = order['amount']
buy_limit_filled_price = order['price']
order_id = None
# in case of FOK the order may be filled immediately and fully
elif order_status == 'closed':
stake_amount = order['cost']
amount = order['amount']
buy_limit_filled_price = order['price']
self.rpc.send_msg({
'type': RPCMessageType.BUY_NOTIFICATION,
'exchange': self.exchange.name.capitalize(),
'pair': pair_s,
'limit': buy_limit_filled_price,
'stake_amount': stake_amount,
'stake_currency': stake_currency,
'fiat_currency': fiat_currency
})
# Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL
fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker')
trade = Trade(
pair=pair,
stake_amount=stake_amount,
amount=amount,
fee_open=fee,
fee_close=fee,
open_rate=buy_limit_filled_price,
open_rate_requested=buy_limit_requested,
open_date=datetime.utcnow(),
exchange=self.exchange.id,
open_order_id=order_id,
strategy=self.strategy.get_strategy_name(),
ticker_interval=timeframe_to_minutes(self.config['ticker_interval'])
)
# Update fees if order is closed
if order_status == 'closed':
self.update_trade_state(trade, order)
Trade.session.add(trade)
Trade.session.flush()
# Updating wallets
self.wallets.update()
return True
def start(self) -> None:
"""
Run a backtesting end-to-end
:return: None
"""
data: Dict[str, Any] = {}
pairs = self.config['exchange']['pair_whitelist']
logger.info('Using stake_currency: %s ...',
self.config['stake_currency'])
logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
if self.config.get('live'):
logger.info('Downloading data for all pairs in whitelist ...')
self.exchange.refresh_latest_ohlcv([(pair, self.ticker_interval)
for pair in pairs])
data = {
key[0]: value
for key, value in self.exchange._klines.items()
}
else:
logger.info(
'Using local backtesting data (using whitelist in given config) ...'
)
timerange = Arguments.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
data = history.load_data(datadir=Path(self.config['datadir'])
if self.config.get('datadir') else None,
pairs=pairs,
ticker_interval=self.ticker_interval,
refresh_pairs=self.config.get(
'refresh_pairs', False),
exchange=self.exchange,
timerange=timerange)
if not data:
logger.critical("No data found. Terminating.")
return
# Use max_open_trades in backtesting, except --disable-max-market-positions is set
if self.config.get('use_max_market_positions', True):
max_open_trades = self.config['max_open_trades']
else:
logger.info(
'Ignoring max_open_trades (--disable-max-market-positions was used) ...'
)
max_open_trades = 0
all_results = {}
for strat in self.strategylist:
logger.info("Running backtesting for Strategy %s",
strat.get_strategy_name())
self._set_strategy(strat)
min_date, max_date = optimize.get_timeframe(data)
# Validate dataframe for missing values (mainly at start and end, as fillup is called)
optimize.validate_backtest_data(
data, min_date, max_date,
timeframe_to_minutes(self.ticker_interval))
logger.info('Measuring data from %s up to %s (%s days)..',
min_date.isoformat(), max_date.isoformat(),
(max_date - min_date).days)
# need to reprocess data every time to populate signals
preprocessed = self.strategy.tickerdata_to_dataframe(data)
# Execute backtest and print results
all_results[self.strategy.get_strategy_name()] = self.backtest({
'stake_amount':
self.config.get('stake_amount'),
'processed':
preprocessed,
'max_open_trades':
max_open_trades,
'position_stacking':
self.config.get('position_stacking', False),
'start_date':
min_date,
'end_date':
max_date,
})
for strategy, results in all_results.items():
if self.config.get('export', False):
self._store_backtest_result(
self.config['exportfilename'], results,
strategy if len(self.strategylist) > 1 else None)
print(f"Result for strategy {strategy}")
print(' BACKTESTING REPORT '.center(133, '='))
print(self._generate_text_table(data, results))
print(' SELL REASON STATS '.center(133, '='))
print(self._generate_text_table_sell_reason(data, results))
print(' LEFT OPEN TRADES REPORT '.center(133, '='))
print(
self._generate_text_table(data,
results.loc[results.open_at_end],
True))
print()
if len(all_results) > 1:
# Print Strategy summary table
print(' Strategy Summary '.center(133, '='))
print(self._generate_text_table_strategy(all_results))
print('\nFor more details, please look at the detail tables above')
def _get_frame_time_from_offset(offset):
return ticker_start_time.shift(
minutes=(offset *
timeframe_to_minutes(tests_ticker_interval))).datetime