def analyse_and_plot_pairs(config: Dict[str, Any]): """ From arguments provided in cli: -Initialise backtest env -Get tickers data -Generate Dafaframes populated with indicators and signals -Load trades excecuted on same periods -Generate Plotly plot objects -Generate plot files :return: None """ exchange = ExchangeResolver(config.get('exchange', {}).get('name'), config).exchange strategy = StrategyResolver(config).strategy if "pairs" in config: pairs = config["pairs"].split(',') else: pairs = config["exchange"]["pair_whitelist"] # Set timerange to use timerange = Arguments.parse_timerange(config["timerange"]) ticker_interval = strategy.ticker_interval tickers = history.load_data( datadir=Path(str(config.get("datadir"))), pairs=pairs, ticker_interval=config['ticker_interval'], refresh_pairs=config.get('refresh_pairs', False), timerange=timerange, exchange=exchange, live=config.get("live", False), ) pair_counter = 0 for pair, data in tickers.items(): pair_counter += 1 logger.info("analyse pair %s", pair) tickers = {} tickers[pair] = data dataframe = generate_dataframe(strategy, tickers, pair) if config["trade_source"] == "DB": trades = load_trades_from_db(config["db_url"]) elif config["trade_source"] == "file": trades = load_backtest_data(Path(config["exportfilename"])) trades = trades.loc[trades['pair'] == pair] trades = extract_trades_of_period(dataframe, trades) fig = generate_graph( pair=pair, data=dataframe, trades=trades, indicators1=config["indicators1"].split(","), indicators2=config["indicators2"].split(",") ) generate_plot_file(fig, pair, ticker_interval) logger.info('End of ploting process %s plots generated', pair_counter)
def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = load_data(datadir=str(self.config.get('datadir')), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, timerange=timerange) if self.has_space('buy'): self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore self.processed = self.tickerdata_to_dataframe(data) logger.info('Preparing Trials..') signal.signal(signal.SIGINT, self.signal_handler) # read trials file if we have one if os.path.exists( self.trials_file) and os.path.getsize(self.trials_file) > 0: self.trials = self.read_trials() self.current_tries = len(self.trials.results) self.total_tries += self.current_tries logger.info('Continuing with trials. Current: %d, Total: %d', self.current_tries, self.total_tries) try: best_parameters = fmin(fn=self.generate_optimizer, space=self.hyperopt_space(), algo=tpe.suggest, max_evals=self.total_tries, trials=self.trials) results = sorted(self.trials.results, key=itemgetter('loss')) best_result = results[0]['result'] except ValueError: best_parameters = {} best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ 'try with more epochs (param: -e).' # Improve best parameter logging display if best_parameters: best_parameters = space_eval(self.hyperopt_space(), best_parameters) logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) if 'roi_t1' in best_parameters: logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) logger.info('Best Result:\n%s', best_result) # Store trials result to file to resume next time self.save_trials()
def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: self.config = config self.exchange = exchange self.strategy = strategy self.ticker_interval = self.strategy.ticker_interval self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe self.get_timeframe = get_timeframe self.advise_sell = self.strategy.advise_sell self.advise_buy = self.strategy.advise_buy self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs self._final_pairs: list = [] # checking max_open_trades. it should be -1 as with Edge # the number of trades is determined by position size if self.config['max_open_trades'] != float('inf'): logger.critical('max_open_trades should be -1 in config !') if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT: raise OperationalException( 'Edge works only with unlimited stake amount') self._capital_percentage: float = self.edge_config.get( 'capital_available_percentage') self._allowed_risk: float = self.edge_config.get('allowed_risk') self._since_number_of_days: int = self.edge_config.get( 'calculate_since_number_of_days', 14) self._last_updated: int = 0 # Timestamp of pairs last updated time self._refresh_pairs = True self._stoploss_range_min = float( self.edge_config.get('stoploss_range_min', -0.01)) self._stoploss_range_max = float( self.edge_config.get('stoploss_range_max', -0.05)) self._stoploss_range_step = float( self.edge_config.get('stoploss_range_step', -0.001)) # calculating stoploss range self._stoploss_range = np.arange(self._stoploss_range_min, self._stoploss_range_max, self._stoploss_range_step) self._timerange: TimeRange = Arguments.parse_timerange( "%s-" % arrow.now().shift( days=-1 * self._since_number_of_days).format('YYYYMMDD')) self.fee = self.exchange.get_fee()
def __init__(self, config: Dict[str, Any]) -> None: self.config = config # Reset keys for edge self.config['exchange']['key'] = '' self.config['exchange']['secret'] = '' self.config['exchange']['password'] = '' self.config['exchange']['uid'] = '' self.config['dry_run'] = True self.exchange = Exchange(self.config) self.strategy = StrategyResolver(self.config).strategy self.edge = Edge(config, self.exchange, self.strategy) self.edge._refresh_pairs = self.config.get('refresh_pairs', False) self.timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) self.edge._timerange = self.timerange
def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = load_data( datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, timerange=timerange ) if self.has_space('buy') or self.has_space('sell'): self.strategy.advise_indicators = \ self.custom_hyperopt.populate_indicators # type: ignore dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() cpus = multiprocessing.cpu_count() logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') opt = self.get_optimizer(cpus) EVALS = max(self.total_tries // cpus, 1) try: with Parallel(n_jobs=cpus) as parallel: for i in range(EVALS): asked = opt.ask(n_points=cpus) f_val = self.run_optimizer_parallel(parallel, asked) opt.tell(asked, [i['loss'] for i in f_val]) self.trials += f_val for j in range(cpus): self.log_results({ 'loss': f_val[j]['loss'], 'current_tries': i * cpus + j, 'total_tries': self.total_tries, 'result': f_val[j]['result'], }) except KeyboardInterrupt: print('User interrupted..') self.save_trials() self.log_trials_result()
def start(self) -> None: """ Run a backtesting end-to-end :return: None """ data = {} pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) timerange = Arguments.parse_timerange(self.config.get('timerange')) data = optimize.load_data(self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=False, timerange=timerange) preprocessed = self.ML_tickerdata_to_dataframe(data) out_coin = 'BTC_XMR' test_ratio = 0.2 data = ml_utils.run_pipeline(preprocessed[out_coin], out_coin, test_ratio) initial_cash = 100 # in USD transaction_cost = 0.01 # as ratio for fee alpha = 0.05 # ratio of portfolio value that we trade each transaction (trading_portfolio, trading_stats) = self.run_buy_sell_strategy(data, initial_cash, transaction_cost, alpha) print(trading_portfolio) #print(trading_portfolio.head(),"Head of trading series.") print("Stats from trading: ", trading_stats) # plots trading portfolio value in time next to out_coin price trading_portfolio.plot(subplots=True, figsize=(6, 6)) plt.legend(loc='best') plt.show()
def analyse_and_plot_pairs(args: Namespace): """ From arguments provided in cli: -Initialise backtest env -Get tickers data -Generate Dafaframes populated with indicators and signals -Load trades excecuted on same periods -Generate Plotly plot objects -Generate plot files :return: None """ strategy, exchange, pairs = get_trading_env(args) # Set timerange to use timerange = Arguments.parse_timerange(args.timerange) tick_interval = strategy.ticker_interval tickers = get_tickers_data(strategy, exchange, pairs, args) pair_counter = 0 for pair, data in tickers.items(): pair_counter += 1 logger.info("analyse pair %s", pair) tickers = {} tickers[pair] = data dataframe = generate_dataframe(strategy, tickers, pair) trades = load_trades(args, pair, timerange) trades = extract_trades_of_period(dataframe, trades) fig = generate_graph( pair=pair, trades=trades, data=dataframe, indicators1=args.indicators1, indicators2=args.indicators2 ) is_last = (False, True)[pair_counter == len(tickers)] generate_plot_file(fig, pair, tick_interval, is_last) logger.info('End of ploting process %s plots generated', pair_counter)
def get_tickers_data(strategy, exchange, pairs: List[str], args): """ Get tickers data for each pairs on live or local, option defined in args :return: dictinnary of tickers. output format: {'pair': tickersdata} """ tick_interval = strategy.ticker_interval timerange = Arguments.parse_timerange(args.timerange) tickers = {} if args.live: logger.info('Downloading pairs.') exchange.refresh_latest_ohlcv([(pair, tick_interval) for pair in pairs]) for pair in pairs: tickers[pair] = exchange.klines((pair, tick_interval)) else: tickers = history.load_data( datadir=Path(_CONF.get("datadir")), pairs=pairs, ticker_interval=tick_interval, refresh_pairs=_CONF.get('refresh_pairs', False), timerange=timerange, exchange=Exchange(_CONF) ) # No ticker found, impossible to download, len mismatch for pair, data in tickers.copy().items(): logger.debug("checking tickers data of pair: %s", pair) logger.debug("data.empty: %s", data.empty) logger.debug("len(data): %s", len(data)) if data.empty: del tickers[pair] logger.info( 'An issue occured while retreiving datas of %s pair, please retry ' 'using -l option for live or --refresh-pairs-cached', pair) return tickers
def test_parse_timerange_incorrect() -> None: assert TimeRange(None, 'line', 0, -200) == Arguments.parse_timerange('-200') assert TimeRange('line', None, 200, 0) == Arguments.parse_timerange('200-') assert TimeRange('index', 'index', 200, 500) == Arguments.parse_timerange('200-500') assert TimeRange('date', None, 1274486400, 0) == Arguments.parse_timerange('20100522-') assert TimeRange(None, 'date', 0, 1274486400) == Arguments.parse_timerange('-20100522') timerange = Arguments.parse_timerange('20100522-20150730') assert timerange == TimeRange('date', 'date', 1274486400, 1438214400) # Added test for unix timestamp - BTC genesis date assert TimeRange('date', None, 1231006505, 0) == Arguments.parse_timerange('1231006505-') assert TimeRange(None, 'date', 0, 1233360000) == Arguments.parse_timerange('-1233360000') timerange = Arguments.parse_timerange('1231006505-1233360000') assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange # TODO: Find solution for the following case (passing timestamp in ms) timerange = Arguments.parse_timerange('1231006505000-1233360000000') assert TimeRange('date', 'date', 1231006505, 1233360000) != timerange with pytest.raises(Exception, match=r'Incorrect syntax.*'): Arguments.parse_timerange('-')
def plot_profit(args: Namespace) -> None: """ Plots the total profit for all pairs. Note, the profit calculation isn't realistic. But should be somewhat proportional, and therefor useful in helping out to find a good algorithm. """ # We need to use the same pairs, same tick_interval # and same timeperiod as used in backtesting # to match the tickerdata against the profits-results timerange = Arguments.parse_timerange(args.timerange) config = Configuration(args).get_config() # Init strategy try: strategy = StrategyResolver({ 'strategy': config.get('strategy') }).strategy except AttributeError: logger.critical( 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', config.get('strategy')) exit(1) # Load the profits results try: filename = args.exportfilename with open(filename) as file: data = json.load(file) except FileNotFoundError: logger.critical( 'File "backtest-result.json" not found. This script require backtesting ' 'results to run.\nPlease run a backtesting with the parameter --export.' ) exit(1) # Take pairs from the cli otherwise switch to the pair in the config file if args.pair: filter_pairs = args.pair filter_pairs = filter_pairs.split(',') else: filter_pairs = config['exchange']['pair_whitelist'] tick_interval = strategy.ticker_interval pairs = config['exchange']['pair_whitelist'] if filter_pairs: pairs = list(set(pairs) & set(filter_pairs)) logger.info('Filter, keep pairs %s' % pairs) tickers = history.load_data(datadir=Path(config.get('datadir')), pairs=pairs, ticker_interval=tick_interval, refresh_pairs=False, timerange=timerange) dataframes = strategy.tickerdata_to_dataframe(tickers) # NOTE: the dataframes are of unequal length, # 'dates' is an merged date array of them all. dates = misc.common_datearray(dataframes) min_date = int(min(dates).timestamp()) max_date = int(max(dates).timestamp()) num_iterations = define_index(min_date, max_date, tick_interval) + 1 # Make an average close price of all the pairs that was involved. # this could be useful to gauge the overall market trend # We are essentially saying: # array <- sum dataframes[*]['close'] / num_items dataframes # FIX: there should be some onliner numpy/panda for this avgclose = np.zeros(num_iterations) num = 0 for pair, pair_data in dataframes.items(): close = pair_data['close'] maxprice = max(close) # Normalize price to [0,1] logger.info('Pair %s has length %s' % (pair, len(close))) for x in range(0, len(close)): avgclose[x] += close[x] / maxprice # avgclose += close num += 1 avgclose /= num # make an profits-growth array pg = make_profit_array(data, num_iterations, min_date, tick_interval, filter_pairs) # # Plot the pairs average close prices, and total profit growth # avgclose = go.Scattergl( x=dates, y=avgclose, name='Avg close price', ) profit = go.Scattergl( x=dates, y=pg, name='Profit', ) fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1]) fig.append_trace(avgclose, 1, 1) fig.append_trace(profit, 2, 1) for pair in pairs: pg = make_profit_array(data, num_iterations, min_date, tick_interval, pair) pair_profit = go.Scattergl( x=dates, y=pg, name=pair, ) fig.append_trace(pair_profit, 3, 1) plot(fig, filename=str( Path('user_data').joinpath('freqtrade-profit-plot.html')))
def start(self) -> None: """ Run a backtesting end-to-end :return: None """ data = {} pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') for pair in pairs: data[pair] = self.exchange.get_ticker_history( pair, self.ticker_interval) else: logger.info( 'Using local backtesting data (using whitelist in given config) ...' ) timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = optimize.load_data(self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self.config.get( 'refresh_pairs', False), exchange=self.exchange, timerange=timerange) if not data: logger.critical("No data found. Terminating.") return # Ignore max_open_trades in backtesting, except realistic flag was passed if self.config.get('realistic_simulation', False): max_open_trades = self.config['max_open_trades'] else: logger.info( 'Ignoring max_open_trades (realistic_simulation not set) ...') max_open_trades = 0 preprocessed = self.tickerdata_to_dataframe(data) # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) logger.info('Measuring data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days) # Execute backtest and print results results = self.backtest({ 'stake_amount': self.config.get('stake_amount'), 'processed': preprocessed, 'max_open_trades': max_open_trades, 'realistic': self.config.get('realistic_simulation', False), }) if self.config.get('export', False): self._store_backtest_result(self.config.get('exportfilename'), results) logger.info( '\n======================================== ' 'BACKTESTING REPORT' ' =========================================\n' '%s', self._generate_text_table(data, results)) logger.info( '\n====================================== ' 'LEFT OPEN TRADES REPORT' ' ======================================\n' '%s', self._generate_text_table(data, results.loc[results.open_at_end]))
def run_backtest(self, exchange, strategy_name, pairs, ticker_interval, is_local_upload): print("run_backtest") asyncio.set_event_loop(asyncio.new_event_loop()) pairs_str = ','.join(pairs) # To Do: remove unused parameters args = [ '--config', 'config.json', '--strategy', strategy_name, '--datadir', 'freqtrade/tests/testdata', '--pairs', pairs_str, '--ticker-interval', ticker_interval, '--indicators1', 'ema50', '--indicators2', 'macd,macdsignal', '--live', '--timerange', '-2000', '--enable-position-stacking', '--disable-max-market-positions' ] args = plot_parse_args(args) pp = pprint.PrettyPrinter(indent=4) pp.pprint(args) strategy, exchange, pairs = get_trading_env(args) timerange = Arguments.parse_timerange(args.timerange) if is_local_upload: tickers = self.tickers else: tickers = get_tickers_data(strategy, exchange, pairs, args) dataframes = {} plots = [] pair_counter = 0 for pair, data in tickers.items(): pair_counter += 1 logger.info("analyse pair %s", pair) tickers = {} tickers[pair] = data dataframe = generate_dataframe(strategy, tickers, pair) dataframes[pair] = dataframe.reset_index() trades = load_trades(args, pair, timerange) # ungly, to refactor if not trades.empty: trades = extract_trades_of_period(dataframe, trades) figure = generate_graph( pair=pair, trades=trades, data=dataframe.reset_index(), indicators1=args.indicators1, indicators2=args.indicators2 ) # Get plot ID corresponding to this pair plot_id = self.plot_ids[pair] print(f'plot id for {pair}: {plot_id} ') plots.append(get_plot_component(dataframe, figure, plot_id)) # Save dataframes in memory self.dataframes = dataframes logger.info('End of ploting process %s plots generated', pair_counter) return html.Div(plots)
def start(self) -> None: """ Run a backtesting end-to-end :return: None """ data = {} pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') for pair in pairs: data[pair] = self.exchange.get_candle_history(pair, self.ticker_interval) else: logger.info('Using local backtesting data (using whitelist in given config) ...') timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = optimize.load_data( self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self.config.get('refresh_pairs', False), exchange=self.exchange, timerange=timerange ) if not data: logger.critical("No data found. Terminating.") return # Use max_open_trades in backtesting, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): max_open_trades = self.config['max_open_trades'] else: logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 all_results = {} for strat in self.strategylist: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) self._set_strategy(strat) # need to reprocess data every time to populate signals preprocessed = self.tickerdata_to_dataframe(data) # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) logger.info( 'Measuring data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days ) # Execute backtest and print results all_results[self.strategy.get_strategy_name()] = self.backtest( { 'stake_amount': self.config.get('stake_amount'), 'processed': preprocessed, 'max_open_trades': max_open_trades, 'position_stacking': self.config.get('position_stacking', False), } ) for strategy, results in all_results.items(): if self.config.get('export', False): self._store_backtest_result(self.config['exportfilename'], results, strategy if len(self.strategylist) > 1 else None) print(f"Result for strategy {strategy}") print(' BACKTESTING REPORT '.center(119, '=')) print(self._generate_text_table(data, results)) print(' SELL REASON STATS '.center(119, '=')) print(self._generate_text_table_sell_reason(data, results)) print(' LEFT OPEN TRADES REPORT '.center(119, '=')) print(self._generate_text_table(data, results.loc[results.open_at_end])) print() if len(all_results) > 1: # Print Strategy summary table print(' Strategy Summary '.center(119, '=')) print(self._generate_text_table_strategy(all_results)) print('\nFor more details, please look at the detail tables above')
def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe :return: None """ global _CONF # Load the configuration _CONF.update(setup_configuration(args)) # Set the pair to audit pair = args.pair if pair is None: logger.critical('Parameter --pair mandatory;. E.g --pair ETH/BTC') exit() if '/' not in pair: logger.critical('--pair format must be XXX/YYY') exit() # Set timerange to use timerange = Arguments.parse_timerange(args.timerange) # Load the strategy try: analyze = Analyze(_CONF) exchange = Exchange(_CONF) except AttributeError: logger.critical( 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', args.strategy) exit() # Set the ticker to use tick_interval = analyze.get_ticker_interval() # Load pair tickers tickers = {} if args.live: logger.info('Downloading pair.') tickers[pair] = exchange.get_ticker_history(pair, tick_interval) else: tickers = optimize.load_data(datadir=_CONF.get("datadir"), pairs=[pair], ticker_interval=tick_interval, refresh_pairs=_CONF.get( 'refresh_pairs', False), timerange=timerange) # No ticker found, or impossible to download if tickers == {}: exit() # Get trades already made from the DB trades: List[Trade] = [] if args.db_url: persistence.init(_CONF) trades = Trade.query.filter(Trade.pair.is_(pair)).all() dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] dataframe = analyze.populate_buy_trend(dataframe) dataframe = analyze.populate_sell_trend(dataframe) if len(dataframe.index) > 750: logger.warning('Ticker contained more than 750 candles, clipping.') fig = generate_graph(pair=pair, trades=trades, data=dataframe.tail(750), args=args) plot(fig, filename=os.path.join('user_data', 'freqtrade-plot.html'))
if not pairs or args.pairs_file: logger.info(f'Reading pairs file "{pairs_file}".') # Download pairs from the pairs file if no config is specified # or if pairs file is specified explicitely if not pairs_file.exists(): sys.exit(f'No pairs file found with path "{pairs_file}".') with pairs_file.open() as file: pairs = list(set(json.load(file))) pairs.sort() timerange = TimeRange() if args.days: time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d") timerange = arguments.parse_timerange(f'{time_since}-') logger.info( f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}') pairs_not_available = [] try: # Init exchange exchange = Exchange(config) for pair in pairs: if pair not in exchange._api.markets: pairs_not_available.append(pair) logger.info(f"Skipping pair {pair}...") continue
def start(self) -> None: """ Run a backtesting end-to-end :return: None """ data = {} pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') for pair in pairs: data[pair] = exchange.get_ticker_history(pair, self.ticker_interval) else: logger.info('Using local backtesting data (using whitelist in given config) ...') timerange = Arguments.parse_timerange(self.config.get('timerange')) data = optimize.load_data( self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self.config.get('refresh_pairs', False), timerange=timerange ) # Ignore max_open_trades in backtesting, except realistic flag was passed if self.config.get('realistic_simulation', False): max_open_trades = self.config['max_open_trades'] else: logger.info('Ignoring max_open_trades (realistic_simulation not set) ...') max_open_trades = 0 preprocessed = self.tickerdata_to_dataframe(data) print(preprocessed) # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) logger.info( 'Measuring data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days ) # Execute backtest and print results sell_profit_only = self.config.get('experimental', {}).get('sell_profit_only', False) use_sell_signal = self.config.get('experimental', {}).get('use_sell_signal', False) results = self.backtest( { 'stake_amount': self.config.get('stake_amount'), 'processed': preprocessed, 'max_open_trades': max_open_trades, 'realistic': self.config.get('realistic_simulation', False), 'sell_profit_only': sell_profit_only, 'use_sell_signal': use_sell_signal, 'record': self.config.get('export') } ) logger.info( '\n==================================== ' 'BACKTESTING REPORT' ' ====================================\n' '%s', self._generate_text_table( data, results ) )
def start(self) -> None: timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = load_data( datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, refresh_pairs=self.config.get('refresh_pairs', False), exchange=self.exchange, timerange=timerange ) if not data: logger.critical("No data found. Terminating.") return min_date, max_date = get_timeframe(data) logger.info( 'Hyperopting with data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days ) if self.has_space('buy') or self.has_space('sell'): self.strategy.advise_indicators = \ self.custom_hyperopt.populate_indicators # type: ignore preprocessed = self.strategy.tickerdata_to_dataframe(data) dump(preprocessed, TICKERDATA_PICKLE) # We don't need exchange instance anymore while running hyperopt self.exchange = None # type: ignore self.load_previous_results() cpus = cpu_count() logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') config_jobs = self.config.get('hyperopt_jobs', -1) logger.info(f'Number of parallel jobs set as: {config_jobs}') opt = self.get_optimizer(config_jobs) try: with Parallel(n_jobs=config_jobs) as parallel: jobs = parallel._effective_n_jobs() logger.info(f'Effective number of parallel workers used: {jobs}') EVALS = max(self.total_tries // jobs, 1) for i in range(EVALS): asked = opt.ask(n_points=jobs) f_val = self.run_optimizer_parallel(parallel, asked) opt.tell(asked, [i['loss'] for i in f_val]) self.trials += f_val for j in range(jobs): current = i * jobs + j self.log_results({ 'loss': f_val[j]['loss'], 'current_tries': current, 'initial_point': current < INITIAL_POINTS, 'total_tries': self.total_tries, 'result': f_val[j]['result'], }) logger.debug(f"Optimizer params: {f_val[j]['params']}") for j in range(jobs): logger.debug(f"Optimizer state: Xi: {opt.Xi[-j-1]}, yi: {opt.yi[-j-1]}") except KeyboardInterrupt: print('User interrupted..') self.save_trials() self.log_trials_result()
def test_parse_timerange_incorrect() -> None: assert ((None, 'line'), None, -200) == Arguments.parse_timerange('-200') assert (('line', None), 200, None) == Arguments.parse_timerange('200-') with pytest.raises(Exception, match=r'Incorrect syntax.*'): Arguments.parse_timerange('-')
def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe :return: None """ global _CONF # Load the configuration _CONF.update(setup_configuration(args)) print(_CONF) # Set the pair to audit pair = args.pair if pair is None: logger.critical('Parameter --pair mandatory;. E.g --pair ETH/BTC') exit() if '/' not in pair: logger.critical('--pair format must be XXX/YYY') exit() # Set timerange to use timerange = Arguments.parse_timerange(args.timerange) # Load the strategy try: strategy = StrategyResolver(_CONF).strategy exchange = Exchange(_CONF) except AttributeError: logger.critical( 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', args.strategy ) exit() # Set the ticker to use tick_interval = strategy.ticker_interval # Load pair tickers tickers = {} if args.live: logger.info('Downloading pair.') tickers[pair] = exchange.get_candle_history(pair, tick_interval) else: tickers = optimize.load_data( datadir=_CONF.get("datadir"), pairs=[pair], ticker_interval=tick_interval, refresh_pairs=_CONF.get('refresh_pairs', False), timerange=timerange, exchange=Exchange(_CONF) ) # No ticker found, or impossible to download if tickers == {}: exit() # Get trades already made from the DB trades = load_trades(args, pair, timerange) dataframes = strategy.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] dataframe = strategy.advise_buy(dataframe, {'pair': pair}) dataframe = strategy.advise_sell(dataframe, {'pair': pair}) if len(dataframe.index) > args.plot_limit: logger.warning('Ticker contained more than %s candles as defined ' 'with --plot-limit, clipping.', args.plot_limit) dataframe = dataframe.tail(args.plot_limit) trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']] fig = generate_graph( pair=pair, trades=trades, data=dataframe, args=args ) plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')))
def plot_analyzed_dataframe(args: Namespace) -> None: """ Calls analyze() and plots the returned dataframe :return: None """ pair = args.pair.replace('-', '_') timerange = Arguments.parse_timerange(args.timerange) # Init strategy try: analyze = Analyze({'strategy': args.strategy}) except AttributeError: logger.critical( 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', args.strategy) exit() tick_interval = analyze.strategy.ticker_interval tickers = {} if args.live: logger.info('Downloading pair.') # Init Bittrex to use public API exchange._API = exchange.Bittrex({'key': '', 'secret': ''}) tickers[pair] = exchange.get_ticker_history(pair, tick_interval) else: tickers = optimize.load_data(datadir=args.datadir, pairs=[pair], ticker_interval=tick_interval, refresh_pairs=False, timerange=timerange) dataframes = analyze.tickerdata_to_dataframe(tickers) dataframe = dataframes[pair] dataframe = analyze.populate_buy_trend(dataframe) dataframe = analyze.populate_sell_trend(dataframe) if len(dataframe.index) > 750: logger.warning('Ticker contained more than 750 candles, clipping.') data = dataframe.tail(750) candles = go.Candlestick(x=data.date, open=data.open, high=data.high, low=data.low, close=data.close, name='Price') df_buy = data[data['buy'] == 1] buys = go.Scattergl(x=df_buy.date, y=df_buy.close, mode='markers', name='buy', marker=dict( symbol='triangle-up-dot', size=9, line=dict(width=1), color='green', )) df_sell = data[data['sell'] == 1] sells = go.Scattergl(x=df_sell.date, y=df_sell.close, mode='markers', name='sell', marker=dict( symbol='triangle-down-dot', size=9, line=dict(width=1), color='red', )) bb_lower = go.Scatter( x=data.date, y=data.bb_lowerband, name='BB lower', line={'color': "transparent"}, ) bb_upper = go.Scatter( x=data.date, y=data.bb_upperband, name='BB upper', fill="tonexty", fillcolor="rgba(0,176,246,0.2)", line={'color': "transparent"}, ) macd = go.Scattergl(x=data['date'], y=data['macd'], name='MACD') macdsignal = go.Scattergl(x=data['date'], y=data['macdsignal'], name='MACD signal') volume = go.Bar(x=data['date'], y=data['volume'], name='Volume') fig = tools.make_subplots( rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 4], vertical_spacing=0.0001, ) fig.append_trace(candles, 1, 1) fig.append_trace(bb_lower, 1, 1) fig.append_trace(bb_upper, 1, 1) fig.append_trace(buys, 1, 1) fig.append_trace(sells, 1, 1) fig.append_trace(volume, 2, 1) fig.append_trace(macd, 3, 1) fig.append_trace(macdsignal, 3, 1) fig['layout'].update(title=args.pair) fig['layout']['yaxis1'].update(title='Price') fig['layout']['yaxis2'].update(title='Volume') fig['layout']['yaxis3'].update(title='MACD') plot(fig, filename='freqtrade-plot.html')