def start_download_data(args: Namespace) -> None: """ Download data (former download_backtest_data.py script) """ config = setup_utils_configuration(args, RunMode.OTHER) timerange = TimeRange() if 'days' in config: time_since = arrow.utcnow().shift( days=-config['days']).strftime("%Y%m%d") timerange = TimeRange.parse_timerange(f'{time_since}-') dl_path = Path(config['datadir']) logger.info(f'About to download pairs: {config["pairs"]}, ' f'intervals: {config["timeframes"]} to {dl_path}') pairs_not_available = [] try: # Init exchange exchange = ExchangeResolver(config['exchange']['name'], config).exchange for pair in config["pairs"]: if pair not in exchange.markets: pairs_not_available.append(pair) logger.info(f"Skipping pair {pair}...") continue for ticker_interval in config["timeframes"]: pair_print = pair.replace('/', '_') filename = f'{pair_print}-{ticker_interval}.json' dl_file = dl_path.joinpath(filename) if config.get("erase") and dl_file.exists(): logger.info( f'Deleting existing data for pair {pair}, interval {ticker_interval}.' ) dl_file.unlink() logger.info( f'Downloading pair {pair}, interval {ticker_interval}.') download_pair_history(datadir=dl_path, exchange=exchange, pair=pair, ticker_interval=str(ticker_interval), timerange=timerange) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") finally: if pairs_not_available: logger.info( f"Pairs [{','.join(pairs_not_available)}] not available " f"on exchange {config['exchange']['name']}.") # configuration.resolve_pairs_list() print(config)
def test_download_pair_history(ticker_history_list, mocker, default_conf, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history_list) exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') file2_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-1m.json') file2_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-5m.json') _backup_file(file1_1) _backup_file(file1_5) _backup_file(file2_1) _backup_file(file2_5) assert os.path.isfile(file1_1) is False assert os.path.isfile(file2_1) is False assert download_pair_history(datadir=testdatadir, exchange=exchange, pair='MEME/BTC', ticker_interval='1m') assert download_pair_history(datadir=testdatadir, exchange=exchange, pair='CFI/BTC', ticker_interval='1m') assert not exchange._pairs_last_refresh_time assert os.path.isfile(file1_1) is True assert os.path.isfile(file2_1) is True # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file2_1) assert os.path.isfile(file1_5) is False assert os.path.isfile(file2_5) is False assert download_pair_history(datadir=testdatadir, exchange=exchange, pair='MEME/BTC', ticker_interval='5m') assert download_pair_history(datadir=testdatadir, exchange=exchange, pair='CFI/BTC', ticker_interval='5m') assert not exchange._pairs_last_refresh_time assert os.path.isfile(file1_5) is True assert os.path.isfile(file2_5) is True # clean files freshly downloaded _clean_test_file(file1_5) _clean_test_file(file2_5)
def test_download_pair_history2(mocker, default_conf) -> None: tick = [ [1509836520000, 0.00162008, 0.00162008, 0.00162008, 0.00162008, 108.14853839], [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) download_pair_history(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') download_pair_history(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') assert json_dump_mock.call_count == 2
def test_download_pair_history(ticker_history_list, mocker, default_conf, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history_list) exchange = get_patched_exchange(mocker, default_conf) file1_1 = testdatadir / 'MEME_BTC-1m.json' file1_5 = testdatadir / 'MEME_BTC-5m.json' file2_1 = testdatadir / 'CFI_BTC-1m.json' file2_5 = testdatadir / 'CFI_BTC-5m.json' _backup_file(file1_1) _backup_file(file1_5) _backup_file(file2_1) _backup_file(file2_5) assert not file1_1.is_file() assert not file2_1.is_file() assert download_pair_history(datadir=testdatadir, exchange=exchange, pair='MEME/BTC', ticker_interval='1m') assert download_pair_history(datadir=testdatadir, exchange=exchange, pair='CFI/BTC', ticker_interval='1m') assert not exchange._pairs_last_refresh_time assert file1_1.is_file() assert file2_1.is_file() # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file2_1) assert not file1_5.is_file() assert not file2_5.is_file() assert download_pair_history(datadir=testdatadir, exchange=exchange, pair='MEME/BTC', ticker_interval='5m') assert download_pair_history(datadir=testdatadir, exchange=exchange, pair='CFI/BTC', ticker_interval='5m') assert not exchange._pairs_last_refresh_time assert file1_5.is_file() assert file2_5.is_file() # clean files freshly downloaded _clean_test_file(file1_5) _clean_test_file(file2_5)
def test_download_backtesting_data_exception(ticker_history, mocker, caplog, default_conf, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', side_effect=Exception('File Error')) exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') _backup_file(file1_1) _backup_file(file1_5) assert not download_pair_history(datadir=testdatadir, exchange=exchange, pair='MEME/BTC', ticker_interval='1m') # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file1_5) assert log_has( 'Failed to download history data for pair: "MEME/BTC", interval: 1m. ' 'Error: File Error', caplog)
print(f'About to download pairs: {PAIRS} to {dl_path}') # Init exchange exchange = Exchange(config) pairs_not_available = [] for pair in PAIRS: if pair not in exchange._api.markets: pairs_not_available.append(pair) print(f"skipping pair {pair}") continue for tick_interval in timeframes: pair_print = pair.replace('/', '_') filename = f'{pair_print}-{tick_interval}.json' dl_file = dl_path.joinpath(filename) if args.erase and dl_file.exists(): print( f'Deleting existing data for pair {pair}, interval {tick_interval}' ) dl_file.unlink() print(f'downloading pair {pair}, interval {tick_interval}') download_pair_history(datadir=dl_path, exchange=exchange, pair=pair, tick_interval=tick_interval, timerange=timerange) if pairs_not_available: print(f"Pairs [{','.join(pairs_not_available)}] not availble.")