def start_download_data(args: Dict[str, Any]) -> None: """ Download data (former download_backtest_data.py script) """ config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) timerange = TimeRange() if 'days' in config: time_since = arrow.utcnow().shift( days=-config['days']).strftime("%Y%m%d") timerange = TimeRange.parse_timerange(f'{time_since}-') if 'pairs' not in config: raise OperationalException( "Downloading data requires a list of pairs. " "Please check the documentation on how to configure this.") dl_path = Path(config['datadir']) logger.info(f'About to download pairs: {config["pairs"]}, ' f'intervals: {config["timeframes"]} to {dl_path}') pairs_not_available: List[str] = [] # Init exchange exchange = ExchangeResolver(config['exchange']['name'], config).exchange try: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( exchange, pairs=config["pairs"], datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv(pairs=config["pairs"], timeframes=config["timeframes"], datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=config["pairs"], timeframes=config["timeframes"], dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") finally: if pairs_not_available: logger.info( f"Pairs [{','.join(pairs_not_available)}] not available " f"on exchange {exchange.name}.")
def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir): dl_mock = mocker.patch('freqtrade.data.history.download_trades_history', MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) mocker.patch.object(Path, "exists", MagicMock(return_value=True)) mocker.patch.object(Path, "unlink", MagicMock()) ex = get_patched_exchange(mocker, default_conf) timerange = TimeRange.parse_timerange("20190101-20190102") unavailable_pairs = refresh_backtest_trades_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC", "XRP/ETH"], datadir=testdatadir, timerange=timerange, erase=True ) assert dl_mock.call_count == 2 assert dl_mock.call_args[1]['timerange'].starttype == 'date' assert log_has("Downloading trades for pair ETH/BTC.", caplog) assert unavailable_pairs == ["XRP/ETH"] assert log_has("Skipping pair XRP/ETH...", caplog)
def start_download_data(args: Dict[str, Any]) -> None: """ Download data (former download_backtest_data.py script) """ config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) if 'days' in config and 'timerange' in config: raise OperationalException( "--days and --timerange are mutually exclusive. " "You can only specify one or the other.") timerange = TimeRange() if 'days' in config: time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d") timerange = TimeRange.parse_timerange(f'{time_since}-') if 'timerange' in config: timerange = timerange.parse_timerange(config['timerange']) # Remove stake-currency to skip checks which are not relevant for datadownload config['stake_currency'] = '' if 'pairs' not in config: raise OperationalException( "Downloading data requires a list of pairs. " "Please check the documentation on how to configure this.") pairs_not_available: List[str] = [] # Init exchange exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) markets = [ p for p, m in exchange.markets.items() if market_is_active(m) or config.get('include_inactive') ] expanded_pairs = expand_pairlist(config['pairs'], markets) # Manual validations of relevant settings if not config['exchange'].get('skip_pair_validation', False): exchange.validate_pairs(expanded_pairs) logger.info(f"About to download pairs: {expanded_pairs}, " f"intervals: {config['timeframes']} to {config['datadir']}") for timeframe in config['timeframes']: exchange.validate_timeframes(timeframe) try: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( exchange, pairs=expanded_pairs, datadir=config['datadir'], timerange=timerange, new_pairs_days=config['new_pairs_days'], erase=bool(config.get('erase')), data_format=config['dataformat_trades']) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], ) else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, new_pairs_days=config['new_pairs_days'], erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv']) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") finally: if pairs_not_available: logger.info( f"Pairs [{','.join(pairs_not_available)}] not available " f"on exchange {exchange.name}.")
def start_download_data(args: Dict[str, Any]) -> None: """ Download data (former download_backtest_data.py script) """ config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) if 'days' in config and 'timerange' in config: raise OperationalException( "--days and --timerange are mutually exclusive. " "You can only specify one or the other.") timerange = TimeRange() if 'days' in config: time_since = arrow.utcnow().shift( days=-config['days']).strftime("%Y%m%d") timerange = TimeRange.parse_timerange(f'{time_since}-') if 'timerange' in config: timerange = timerange.parse_timerange(config['timerange']) if 'pairs' not in config: raise OperationalException( "Downloading data requires a list of pairs. " "Please check the documentation on how to configure this.") logger.info(f"About to download pairs: {config['pairs']}, " f"intervals: {config['timeframes']} to {config['datadir']}") pairs_not_available: List[str] = [] # Init exchange exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) # Manual validations of relevant settings exchange.validate_pairs(config['pairs']) for timeframe in config['timeframes']: exchange.validate_timeframes(timeframe) try: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( exchange, pairs=config['pairs'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format=config['dataformat_trades']) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( pairs=config['pairs'], timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], ) else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=config['pairs'], timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv']) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") finally: if pairs_not_available: logger.info( f"Pairs [{','.join(pairs_not_available)}] not available " f"on exchange {exchange.name}.")