def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog):
pair = 'XRP/ETH'
file1 = testdatadir / 'XRP_ETH-1m.json'
file5 = testdatadir / 'XRP_ETH-5m.json'
# Compare downloaded dataset with converted dataset
dfbak_1m = load_pair_history(datadir=testdatadir,
timeframe="1m",
pair=pair)
dfbak_5m = load_pair_history(datadir=testdatadir,
timeframe="5m",
pair=pair)
_backup_file(file1, copy_file=True)
_backup_file(file5)
tr = TimeRange.parse_timerange('20191011-20191012')
convert_trades_to_ohlcv([pair],
timeframes=['1m', '5m'],
datadir=testdatadir,
timerange=tr,
erase=True)
assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.",
caplog)
# Load new data
df_1m = load_pair_history(datadir=testdatadir, timeframe="1m", pair=pair)
df_5m = load_pair_history(datadir=testdatadir, timeframe="5m", pair=pair)
assert df_1m.equals(dfbak_1m)
assert df_5m.equals(dfbak_5m)
_clean_test_file(file1)
_clean_test_file(file5)
def start_download_data(args: Dict[str, Any]) -> None:
"""
Download data (former download_backtest_data.py script)
"""
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
timerange = TimeRange()
if 'days' in config:
time_since = arrow.utcnow().shift(
days=-config['days']).strftime("%Y%m%d")
timerange = TimeRange.parse_timerange(f'{time_since}-')
if 'pairs' not in config:
raise OperationalException(
"Downloading data requires a list of pairs. "
"Please check the documentation on how to configure this.")
dl_path = Path(config['datadir'])
logger.info(f'About to download pairs: {config["pairs"]}, '
f'intervals: {config["timeframes"]} to {dl_path}')
pairs_not_available: List[str] = []
# Init exchange
exchange = ExchangeResolver(config['exchange']['name'], config).exchange
try:
if config.get('download_trades'):
pairs_not_available = refresh_backtest_trades_data(
exchange,
pairs=config["pairs"],
datadir=Path(config['datadir']),
timerange=timerange,
erase=config.get("erase"))
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(pairs=config["pairs"],
timeframes=config["timeframes"],
datadir=Path(config['datadir']),
timerange=timerange,
erase=config.get("erase"))
else:
pairs_not_available = refresh_backtest_ohlcv_data(
exchange,
pairs=config["pairs"],
timeframes=config["timeframes"],
dl_path=Path(config['datadir']),
timerange=timerange,
erase=config.get("erase"))
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")
finally:
if pairs_not_available:
logger.info(
f"Pairs [{','.join(pairs_not_available)}] not available "
f"on exchange {exchange.name}.")
def start_convert_trades(args: Dict[str, Any]) -> None:
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
timerange = TimeRange()
# Remove stake-currency to skip checks which are not relevant for datadownload
config['stake_currency'] = ''
if 'pairs' not in config:
raise OperationalException(
"Downloading data requires a list of pairs. "
"Please check the documentation on how to configure this.")
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'],
config,
validate=False)
# Manual validations of relevant settings
if not config['exchange'].get('skip_pair_validation', False):
exchange.validate_pairs(config['pairs'])
expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
logger.info(f"About to Convert pairs: {expanded_pairs}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
for timeframe in config['timeframes']:
exchange.validate_timeframes(timeframe)
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
pairs=expanded_pairs,
timeframes=config['timeframes'],
datadir=config['datadir'],
timerange=timerange,
erase=bool(config.get('erase')),
data_format_ohlcv=config['dataformat_ohlcv'],
data_format_trades=config['dataformat_trades'],
)
def start_download_data(args: Dict[str, Any]) -> None:
"""
Download data (former download_backtest_data.py script)
"""
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
if 'days' in config and 'timerange' in config:
raise OperationalException(
"--days and --timerange are mutually exclusive. "
"You can only specify one or the other.")
timerange = TimeRange()
if 'days' in config:
time_since = (datetime.now() -
timedelta(days=config['days'])).strftime("%Y%m%d")
timerange = TimeRange.parse_timerange(f'{time_since}-')
if 'timerange' in config:
timerange = timerange.parse_timerange(config['timerange'])
# Remove stake-currency to skip checks which are not relevant for datadownload
config['stake_currency'] = ''
if 'pairs' not in config:
raise OperationalException(
"Downloading data requires a list of pairs. "
"Please check the documentation on how to configure this.")
pairs_not_available: List[str] = []
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'],
config,
validate=False)
markets = [
p for p, m in exchange.markets.items()
if market_is_active(m) or config.get('include_inactive')
]
expanded_pairs = expand_pairlist(config['pairs'], markets)
# Manual validations of relevant settings
if not config['exchange'].get('skip_pair_validation', False):
exchange.validate_pairs(expanded_pairs)
logger.info(f"About to download pairs: {expanded_pairs}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
for timeframe in config['timeframes']:
exchange.validate_timeframes(timeframe)
try:
if config.get('download_trades'):
pairs_not_available = refresh_backtest_trades_data(
exchange,
pairs=expanded_pairs,
datadir=config['datadir'],
timerange=timerange,
new_pairs_days=config['new_pairs_days'],
erase=bool(config.get('erase')),
data_format=config['dataformat_trades'])
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
pairs=expanded_pairs,
timeframes=config['timeframes'],
datadir=config['datadir'],
timerange=timerange,
erase=bool(config.get('erase')),
data_format_ohlcv=config['dataformat_ohlcv'],
data_format_trades=config['dataformat_trades'],
)
else:
pairs_not_available = refresh_backtest_ohlcv_data(
exchange,
pairs=expanded_pairs,
timeframes=config['timeframes'],
datadir=config['datadir'],
timerange=timerange,
new_pairs_days=config['new_pairs_days'],
erase=bool(config.get('erase')),
data_format=config['dataformat_ohlcv'])
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")
finally:
if pairs_not_available:
logger.info(
f"Pairs [{','.join(pairs_not_available)}] not available "
f"on exchange {exchange.name}.")
def start_download_data(args: Dict[str, Any]) -> None:
"""
Download data (former download_backtest_data.py script)
"""
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
if 'days' in config and 'timerange' in config:
raise OperationalException(
"--days and --timerange are mutually exclusive. "
"You can only specify one or the other.")
timerange = TimeRange()
if 'days' in config:
time_since = arrow.utcnow().shift(
days=-config['days']).strftime("%Y%m%d")
timerange = TimeRange.parse_timerange(f'{time_since}-')
if 'timerange' in config:
timerange = timerange.parse_timerange(config['timerange'])
if 'pairs' not in config:
raise OperationalException(
"Downloading data requires a list of pairs. "
"Please check the documentation on how to configure this.")
logger.info(f"About to download pairs: {config['pairs']}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
pairs_not_available: List[str] = []
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'],
config,
validate=False)
# Manual validations of relevant settings
exchange.validate_pairs(config['pairs'])
for timeframe in config['timeframes']:
exchange.validate_timeframes(timeframe)
try:
if config.get('download_trades'):
pairs_not_available = refresh_backtest_trades_data(
exchange,
pairs=config['pairs'],
datadir=config['datadir'],
timerange=timerange,
erase=bool(config.get('erase')),
data_format=config['dataformat_trades'])
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
pairs=config['pairs'],
timeframes=config['timeframes'],
datadir=config['datadir'],
timerange=timerange,
erase=bool(config.get('erase')),
data_format_ohlcv=config['dataformat_ohlcv'],
data_format_trades=config['dataformat_trades'],
)
else:
pairs_not_available = refresh_backtest_ohlcv_data(
exchange,
pairs=config['pairs'],
timeframes=config['timeframes'],
datadir=config['datadir'],
timerange=timerange,
erase=bool(config.get('erase')),
data_format=config['dataformat_ohlcv'])
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")
finally:
if pairs_not_available:
logger.info(
f"Pairs [{','.join(pairs_not_available)}] not available "
f"on exchange {exchange.name}.")
