def ohlcv_fill_up_missing_data(dataframe: DataFrame, ticker_interval: str) -> DataFrame: """ Fills up missing data with 0 volume rows, using the previous close as price for "open", "high" "low" and "close", volume is set to 0 """ ohlc_dict = { 'open': 'first', 'high': 'max', 'low': 'min', 'close': 'last', 'volume': 'sum' } ticker_minutes = timeframe_to_minutes(ticker_interval) # Resample to create "NAN" values df = dataframe.resample(f'{ticker_minutes}min', on='date').agg(ohlc_dict) # Forwardfill close for missing columns df['close'] = df['close'].fillna(method='ffill') # Use close for "open, high, low" df.loc[:, ['open', 'high', 'low']] = df[['open', 'high', 'low']].fillna( value={'open': df['close'], 'high': df['close'], 'low': df['close'], }) df.reset_index(inplace=True) logger.debug(f"Missing data fillup: before: {len(dataframe)} - after: {len(df)}") return df
def __init__(self, config: Dict[str, Any]) -> None: self.config = config # Reset keys for backtesting self.config['exchange']['key'] = '' self.config['exchange']['secret'] = '' self.config['exchange']['password'] = '' self.config['exchange']['uid'] = '' self.config['dry_run'] = True self.strategylist: List[IStrategy] = [] exchange_name = self.config.get('exchange', {}).get('name', 'bittrex').title() self.exchange = ExchangeResolver(exchange_name, self.config).exchange self.fee = self.exchange.get_fee() if self.config.get('runmode') != RunMode.HYPEROPT: self.dataprovider = DataProvider(self.config, self.exchange) IStrategy.dp = self.dataprovider if self.config.get('strategy_list', None): # Force one interval self.ticker_interval = str(self.config.get('ticker_interval')) self.ticker_interval_mins = timeframe_to_minutes( self.ticker_interval) for strat in list(self.config['strategy_list']): stratconf = deepcopy(self.config) stratconf['strategy'] = strat self.strategylist.append(StrategyResolver(stratconf).strategy) else: # only one strategy self.strategylist.append(StrategyResolver(self.config).strategy) # Load one strategy self._set_strategy(self.strategylist[0])
def get_signal(self, pair: str, interval: str, dataframe: DataFrame) -> Tuple[bool, bool]: """ Calculates current signal based several technical analysis indicators :param pair: pair in format ANT/BTC :param interval: Interval to use (in min) :param dataframe: Dataframe to analyze :return: (Buy, Sell) A bool-tuple indicating buy/sell signal """ if not isinstance(dataframe, DataFrame) or dataframe.empty: logger.warning('Empty ticker history for pair %s', pair) return False, False try: dataframe = self.analyze_ticker(dataframe, {'pair': pair}) except ValueError as error: logger.warning( 'Unable to analyze ticker for pair %s: %s', pair, str(error) ) return False, False except Exception as error: logger.exception( 'Unexpected error when analyzing ticker for pair %s: %s', pair, str(error) ) return False, False if dataframe.empty: logger.warning('Empty dataframe for pair %s', pair) return False, False latest = dataframe.iloc[-1] # Check if dataframe is out of date signal_date = arrow.get(latest['date']) interval_minutes = timeframe_to_minutes(interval) offset = self.config.get('exchange', {}).get('outdated_offset', 5) if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + offset))): logger.warning( 'Outdated history for pair %s. Last tick is %s minutes old', pair, (arrow.utcnow() - signal_date).seconds // 60 ) return False, False (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 logger.debug( 'trigger: %s (pair=%s) buy=%s sell=%s', latest['date'], pair, str(buy), str(sell) ) return buy, sell
def _set_strategy(self, strategy): """ Load strategy into backtesting """ self.strategy = strategy self.ticker_interval = self.config.get('ticker_interval') self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval) self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe self.advise_buy = strategy.advise_buy self.advise_sell = strategy.advise_sell # Set stoploss_on_exchange to false for backtesting, # since a "perfect" stoploss-sell is assumed anyway # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False
def test_validate_backtest_data(default_conf, mocker, caplog) -> None: patch_exchange(mocker) strategy = DefaultStrategy(default_conf) timerange = TimeRange('index', 'index', 200, 250) data = strategy.tickerdata_to_dataframe( history.load_data(datadir=None, ticker_interval='5m', pairs=['UNITTEST/BTC'], timerange=timerange)) min_date, max_date = optimize.get_timeframe(data) caplog.clear() assert not optimize.validate_backtest_data(data, min_date, max_date, timeframe_to_minutes('5m')) assert len(caplog.record_tuples) == 0
def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: patch_exchange(mocker) strategy = DefaultStrategy(default_conf) data = strategy.tickerdata_to_dataframe( history.load_data(datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC'], fill_up_missing=False)) min_date, max_date = optimize.get_timeframe(data) caplog.clear() assert optimize.validate_backtest_data(data, min_date, max_date, timeframe_to_minutes('1m')) assert len(caplog.record_tuples) == 1 assert log_has( "UNITTEST/BTC has missing frames: expected 14396, got 13680, that's 716 missing values", caplog.record_tuples)
def load_cached_data_for_updating( filename: Path, ticker_interval: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: """ Load cached data and choose what part of the data should be updated """ since_ms = None # user sets timerange, so find the start time if timerange: if timerange.starttype == 'date': since_ms = timerange.startts * 1000 elif timerange.stoptype == 'line': num_minutes = timerange.stopts * timeframe_to_minutes( ticker_interval) since_ms = arrow.utcnow().shift( minutes=num_minutes).timestamp * 1000 # read the cached file if filename.is_file(): with open(filename, "rt") as file: data = misc.json_load(file) # remove the last item, could be incomplete candle if data: data.pop() else: data = [] if data: if since_ms and since_ms < data[0][0]: # Earlier data than existing data requested, redownload all data = [] else: # a part of the data was already downloaded, so download unexist data only since_ms = data[-1][0] + 1 return (data, since_ms)
def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None) -> bool: """ Executes a limit buy for the given pair :param pair: pair for which we want to create a LIMIT_BUY :return: None """ pair_s = pair.replace('_', '/') stake_currency = self.config['stake_currency'] fiat_currency = self.config.get('fiat_display_currency', None) time_in_force = self.strategy.order_time_in_force['buy'] if price: buy_limit_requested = price else: # Calculate amount buy_limit_requested = self.get_target_bid(pair) min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit_requested) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( f'Can\'t open a new trade for {pair_s}: stake amount ' f'is too small ({stake_amount} < {min_stake_amount})' ) return False amount = stake_amount / buy_limit_requested order = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], amount=amount, rate=buy_limit_requested, time_in_force=time_in_force) order_id = order['id'] order_status = order.get('status', None) # we assume the order is executed at the price requested buy_limit_filled_price = buy_limit_requested if order_status == 'expired' or order_status == 'rejected': order_type = self.strategy.order_types['buy'] order_tif = self.strategy.order_time_in_force['buy'] # return false if the order is not filled if float(order['filled']) == 0: logger.warning('Buy %s order with time in force %s for %s is %s by %s.' ' zero amount is fulfilled.', order_tif, order_type, pair_s, order_status, self.exchange.name) return False else: # the order is partially fulfilled # in case of IOC orders we can check immediately # if the order is fulfilled fully or partially logger.warning('Buy %s order with time in force %s for %s is %s by %s.' ' %s amount fulfilled out of %s (%s remaining which is canceled).', order_tif, order_type, pair_s, order_status, self.exchange.name, order['filled'], order['amount'], order['remaining'] ) stake_amount = order['cost'] amount = order['amount'] buy_limit_filled_price = order['price'] order_id = None # in case of FOK the order may be filled immediately and fully elif order_status == 'closed': stake_amount = order['cost'] amount = order['amount'] buy_limit_filled_price = order['price'] self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': self.exchange.name.capitalize(), 'pair': pair_s, 'limit': buy_limit_filled_price, 'stake_amount': stake_amount, 'stake_currency': stake_currency, 'fiat_currency': fiat_currency }) # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') trade = Trade( pair=pair, stake_amount=stake_amount, amount=amount, fee_open=fee, fee_close=fee, open_rate=buy_limit_filled_price, open_rate_requested=buy_limit_requested, open_date=datetime.utcnow(), exchange=self.exchange.id, open_order_id=order_id, strategy=self.strategy.get_strategy_name(), ticker_interval=timeframe_to_minutes(self.config['ticker_interval']) ) # Update fees if order is closed if order_status == 'closed': self.update_trade_state(trade, order) Trade.session.add(trade) Trade.session.flush() # Updating wallets self.wallets.update() return True
def start(self) -> None: """ Run a backtesting end-to-end :return: None """ data: Dict[str, Any] = {} pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') self.exchange.refresh_latest_ohlcv([(pair, self.ticker_interval) for pair in pairs]) data = { key[0]: value for key, value in self.exchange._klines.items() } else: logger.info( 'Using local backtesting data (using whitelist in given config) ...' ) timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = history.load_data(datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self.config.get( 'refresh_pairs', False), exchange=self.exchange, timerange=timerange) if not data: logger.critical("No data found. Terminating.") return # Use max_open_trades in backtesting, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): max_open_trades = self.config['max_open_trades'] else: logger.info( 'Ignoring max_open_trades (--disable-max-market-positions was used) ...' ) max_open_trades = 0 all_results = {} for strat in self.strategylist: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) self._set_strategy(strat) min_date, max_date = optimize.get_timeframe(data) # Validate dataframe for missing values (mainly at start and end, as fillup is called) optimize.validate_backtest_data( data, min_date, max_date, timeframe_to_minutes(self.ticker_interval)) logger.info('Measuring data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days) # need to reprocess data every time to populate signals preprocessed = self.strategy.tickerdata_to_dataframe(data) # Execute backtest and print results all_results[self.strategy.get_strategy_name()] = self.backtest({ 'stake_amount': self.config.get('stake_amount'), 'processed': preprocessed, 'max_open_trades': max_open_trades, 'position_stacking': self.config.get('position_stacking', False), 'start_date': min_date, 'end_date': max_date, }) for strategy, results in all_results.items(): if self.config.get('export', False): self._store_backtest_result( self.config['exportfilename'], results, strategy if len(self.strategylist) > 1 else None) print(f"Result for strategy {strategy}") print(' BACKTESTING REPORT '.center(133, '=')) print(self._generate_text_table(data, results)) print(' SELL REASON STATS '.center(133, '=')) print(self._generate_text_table_sell_reason(data, results)) print(' LEFT OPEN TRADES REPORT '.center(133, '=')) print( self._generate_text_table(data, results.loc[results.open_at_end], True)) print() if len(all_results) > 1: # Print Strategy summary table print(' Strategy Summary '.center(133, '=')) print(self._generate_text_table_strategy(all_results)) print('\nFor more details, please look at the detail tables above')
def _get_frame_time_from_offset(offset): return ticker_start_time.shift( minutes=(offset * timeframe_to_minutes(tests_ticker_interval))).datetime