def start_download_data(args: Namespace) -> None:
"""
Download data (former download_backtest_data.py script)
"""
config = setup_utils_configuration(args, RunMode.OTHER)
timerange = TimeRange()
if 'days' in config:
time_since = arrow.utcnow().shift(
days=-config['days']).strftime("%Y%m%d")
timerange = TimeRange.parse_timerange(f'{time_since}-')
dl_path = Path(config['datadir'])
logger.info(f'About to download pairs: {config["pairs"]}, '
f'intervals: {config["timeframes"]} to {dl_path}')
pairs_not_available = []
try:
# Init exchange
exchange = ExchangeResolver(config['exchange']['name'],
config).exchange
for pair in config["pairs"]:
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
continue
for ticker_interval in config["timeframes"]:
pair_print = pair.replace('/', '_')
filename = f'{pair_print}-{ticker_interval}.json'
dl_file = dl_path.joinpath(filename)
if config.get("erase") and dl_file.exists():
logger.info(
f'Deleting existing data for pair {pair}, interval {ticker_interval}.'
)
dl_file.unlink()
logger.info(
f'Downloading pair {pair}, interval {ticker_interval}.')
download_pair_history(datadir=dl_path,
exchange=exchange,
pair=pair,
ticker_interval=str(ticker_interval),
timerange=timerange)
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")
finally:
if pairs_not_available:
logger.info(
f"Pairs [{','.join(pairs_not_available)}] not available "
f"on exchange {config['exchange']['name']}.")
# configuration.resolve_pairs_list()
print(config)
def test_download_pair_history(ticker_history_list, mocker, default_conf,
testdatadir) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv',
return_value=ticker_history_list)
exchange = get_patched_exchange(mocker, default_conf)
file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata',
'MEME_BTC-1m.json')
file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata',
'MEME_BTC-5m.json')
file2_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata',
'CFI_BTC-1m.json')
file2_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata',
'CFI_BTC-5m.json')
_backup_file(file1_1)
_backup_file(file1_5)
_backup_file(file2_1)
_backup_file(file2_5)
assert os.path.isfile(file1_1) is False
assert os.path.isfile(file2_1) is False
assert download_pair_history(datadir=testdatadir,
exchange=exchange,
pair='MEME/BTC',
ticker_interval='1m')
assert download_pair_history(datadir=testdatadir,
exchange=exchange,
pair='CFI/BTC',
ticker_interval='1m')
assert not exchange._pairs_last_refresh_time
assert os.path.isfile(file1_1) is True
assert os.path.isfile(file2_1) is True
# clean files freshly downloaded
_clean_test_file(file1_1)
_clean_test_file(file2_1)
assert os.path.isfile(file1_5) is False
assert os.path.isfile(file2_5) is False
assert download_pair_history(datadir=testdatadir,
exchange=exchange,
pair='MEME/BTC',
ticker_interval='5m')
assert download_pair_history(datadir=testdatadir,
exchange=exchange,
pair='CFI/BTC',
ticker_interval='5m')
assert not exchange._pairs_last_refresh_time
assert os.path.isfile(file1_5) is True
assert os.path.isfile(file2_5) is True
# clean files freshly downloaded
_clean_test_file(file1_5)
_clean_test_file(file2_5)
def test_download_pair_history2(mocker, default_conf) -> None:
tick = [
[1509836520000, 0.00162008, 0.00162008, 0.00162008, 0.00162008, 108.14853839],
[1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199]
]
json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None)
mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick)
exchange = get_patched_exchange(mocker, default_conf)
download_pair_history(None, exchange, pair="UNITTEST/BTC", tick_interval='1m')
download_pair_history(None, exchange, pair="UNITTEST/BTC", tick_interval='3m')
assert json_dump_mock.call_count == 2
def test_download_pair_history(ticker_history_list, mocker, default_conf,
testdatadir) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv',
return_value=ticker_history_list)
exchange = get_patched_exchange(mocker, default_conf)
file1_1 = testdatadir / 'MEME_BTC-1m.json'
file1_5 = testdatadir / 'MEME_BTC-5m.json'
file2_1 = testdatadir / 'CFI_BTC-1m.json'
file2_5 = testdatadir / 'CFI_BTC-5m.json'
_backup_file(file1_1)
_backup_file(file1_5)
_backup_file(file2_1)
_backup_file(file2_5)
assert not file1_1.is_file()
assert not file2_1.is_file()
assert download_pair_history(datadir=testdatadir,
exchange=exchange,
pair='MEME/BTC',
ticker_interval='1m')
assert download_pair_history(datadir=testdatadir,
exchange=exchange,
pair='CFI/BTC',
ticker_interval='1m')
assert not exchange._pairs_last_refresh_time
assert file1_1.is_file()
assert file2_1.is_file()
# clean files freshly downloaded
_clean_test_file(file1_1)
_clean_test_file(file2_1)
assert not file1_5.is_file()
assert not file2_5.is_file()
assert download_pair_history(datadir=testdatadir,
exchange=exchange,
pair='MEME/BTC',
ticker_interval='5m')
assert download_pair_history(datadir=testdatadir,
exchange=exchange,
pair='CFI/BTC',
ticker_interval='5m')
assert not exchange._pairs_last_refresh_time
assert file1_5.is_file()
assert file2_5.is_file()
# clean files freshly downloaded
_clean_test_file(file1_5)
_clean_test_file(file2_5)
def test_download_backtesting_data_exception(ticker_history, mocker, caplog,
default_conf,
testdatadir) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv',
side_effect=Exception('File Error'))
exchange = get_patched_exchange(mocker, default_conf)
file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata',
'MEME_BTC-1m.json')
file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata',
'MEME_BTC-5m.json')
_backup_file(file1_1)
_backup_file(file1_5)
assert not download_pair_history(datadir=testdatadir,
exchange=exchange,
pair='MEME/BTC',
ticker_interval='1m')
# clean files freshly downloaded
_clean_test_file(file1_1)
_clean_test_file(file1_5)
assert log_has(
'Failed to download history data for pair: "MEME/BTC", interval: 1m. '
'Error: File Error', caplog)
print(f'About to download pairs: {PAIRS} to {dl_path}')
# Init exchange
exchange = Exchange(config)
pairs_not_available = []
for pair in PAIRS:
if pair not in exchange._api.markets:
pairs_not_available.append(pair)
print(f"skipping pair {pair}")
continue
for tick_interval in timeframes:
pair_print = pair.replace('/', '_')
filename = f'{pair_print}-{tick_interval}.json'
dl_file = dl_path.joinpath(filename)
if args.erase and dl_file.exists():
print(
f'Deleting existing data for pair {pair}, interval {tick_interval}'
)
dl_file.unlink()
print(f'downloading pair {pair}, interval {tick_interval}')
download_pair_history(datadir=dl_path,
exchange=exchange,
pair=pair,
tick_interval=tick_interval,
timerange=timerange)
if pairs_not_available:
print(f"Pairs [{','.join(pairs_not_available)}] not availble.")
