def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
"""
Convert data from one format to another
"""
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if ohlcv:
convert_ohlcv_format(config,
convert_from=args['format_from'], convert_to=args['format_to'],
erase=args['erase'])
else:
convert_trades_format(config,
convert_from=args['format_from'], convert_to=args['format_to'],
erase=args['erase'])
def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base,
candletype):
tmpdir1 = Path(tmpdir)
prependix = '' if candletype == CandleType.SPOT else 'futures/'
files_orig = []
files_temp = []
files_new = []
for file in file_base:
file_orig = testdatadir / f"{prependix}{file}.json"
file_temp = tmpdir1 / f"{prependix}{file}.json"
file_new = tmpdir1 / f"{prependix}{file}.json.gz"
IDataHandler.create_dir_if_needed(file_temp)
copyfile(file_orig, file_temp)
files_orig.append(file_orig)
files_temp.append(file_temp)
files_new.append(file_new)
default_conf['datadir'] = tmpdir1
default_conf['pairs'] = ['XRP_ETH', 'XRP_USDT', 'UNITTEST_USDT']
default_conf['timeframes'] = ['1m', '5m', '1h']
assert not file_new.exists()
convert_ohlcv_format(default_conf,
convert_from='json',
convert_to='jsongz',
erase=False,
candle_type=candletype)
for file in (files_temp + files_new):
assert file.exists()
# Remove original files
for file in (files_temp):
file.unlink()
# Convert back
convert_ohlcv_format(default_conf,
convert_from='jsongz',
convert_to='json',
erase=True,
candle_type=candletype)
for file in (files_temp):
assert file.exists()
for file in (files_new):
assert not file.exists()
def test_convert_ohlcv_format(mocker, default_conf, testdatadir):
file1 = testdatadir / "XRP_ETH-5m.json"
file1_new = testdatadir / "XRP_ETH-5m.json.gz"
file2 = testdatadir / "XRP_ETH-1m.json"
file2_new = testdatadir / "XRP_ETH-1m.json.gz"
_backup_file(file1, copy_file=True)
_backup_file(file2, copy_file=True)
default_conf['datadir'] = testdatadir
default_conf['pairs'] = ['XRP_ETH']
default_conf['timeframes'] = ['1m', '5m']
assert not file1_new.exists()
assert not file2_new.exists()
convert_ohlcv_format(default_conf,
convert_from='json',
convert_to='jsongz',
erase=False)
assert file1_new.exists()
assert file2_new.exists()
assert file1.exists()
assert file2.exists()
# Remove original files
file1.unlink()
file2.unlink()
# Convert back
convert_ohlcv_format(default_conf,
convert_from='jsongz',
convert_to='json',
erase=True)
assert file1.exists()
assert file2.exists()
assert not file1_new.exists()
assert not file2_new.exists()
_clean_test_file(file1)
_clean_test_file(file2)
if file1_new.exists():
file1_new.unlink()
if file2_new.exists():
file2_new.unlink()
def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir):
tmpdir1 = Path(tmpdir)
file1_orig = testdatadir / "XRP_ETH-5m.json"
file1 = tmpdir1 / "XRP_ETH-5m.json"
file1_new = tmpdir1 / "XRP_ETH-5m.json.gz"
file2_orig = testdatadir / "XRP_ETH-1m.json"
file2 = tmpdir1 / "XRP_ETH-1m.json"
file2_new = tmpdir1 / "XRP_ETH-1m.json.gz"
copyfile(file1_orig, file1)
copyfile(file2_orig, file2)
default_conf['datadir'] = tmpdir1
default_conf['pairs'] = ['XRP_ETH']
default_conf['timeframes'] = ['1m', '5m']
assert not file1_new.exists()
assert not file2_new.exists()
convert_ohlcv_format(default_conf,
convert_from='json',
convert_to='jsongz',
erase=False)
assert file1_new.exists()
assert file2_new.exists()
assert file1.exists()
assert file2.exists()
# Remove original files
file1.unlink()
file2.unlink()
# Convert back
convert_ohlcv_format(default_conf,
convert_from='jsongz',
convert_to='json',
erase=True)
assert file1.exists()
assert file2.exists()
assert not file1_new.exists()
assert not file2_new.exists()
def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
"""
Convert data from one format to another
"""
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if ohlcv:
candle_types = [
CandleType.from_string(ct)
for ct in config.get('candle_types', ['spot'])
]
for candle_type in candle_types:
convert_ohlcv_format(config,
convert_from=args['format_from'],
convert_to=args['format_to'],
erase=args['erase'],
candle_type=candle_type)
else:
convert_trades_format(config,
convert_from=args['format_from'],
convert_to=args['format_to'],
erase=args['erase'])
