def _download_trades_history(exchange: Exchange,
pair: str, *,
new_pairs_days: int = 30,
timerange: Optional[TimeRange] = None,
data_handler: IDataHandler
) -> bool:
"""
Download trade history from the exchange.
Appends to previously downloaded trades data.
"""
try:
since = timerange.startts * 1000 if \
(timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift(
days=-new_pairs_days).float_timestamp) * 1000
trades = data_handler.trades_load(pair)
# TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS
# DEFAULT_TRADES_COLUMNS: 0 -> timestamp
# DEFAULT_TRADES_COLUMNS: 1 -> id
if trades and since < trades[0][0]:
# since is before the first trade
logger.info(f"Start earlier than available data. Redownloading trades for {pair}...")
trades = []
from_id = trades[-1][1] if trades else None
if trades and since < trades[-1][0]:
# Reset since to the last available point
# - 5 seconds (to ensure we're getting all trades)
since = trades[-1][0] - (5 * 1000)
logger.info(f"Using last trade date -5s - Downloading trades for {pair} "
f"since: {format_ms_time(since)}.")
logger.debug(f"Current Start: {format_ms_time(trades[0][0]) if trades else 'None'}")
logger.debug(f"Current End: {format_ms_time(trades[-1][0]) if trades else 'None'}")
logger.info(f"Current Amount of trades: {len(trades)}")
# Default since_ms to 30 days if nothing is given
new_trades = exchange.get_historic_trades(pair=pair,
since=since,
from_id=from_id,
)
trades.extend(new_trades[1])
# Remove duplicates to make sure we're not storing data we don't need
trades = trades_remove_duplicates(trades)
data_handler.trades_store(pair, data=trades)
logger.debug(f"New Start: {format_ms_time(trades[0][0])}")
logger.debug(f"New End: {format_ms_time(trades[-1][0])}")
logger.info(f"New Amount of trades: {len(trades)}")
return True
except Exception:
logger.exception(
f'Failed to download historic trades for pair: "{pair}". '
)
return False
def load_pair_history(pair: str,
timeframe: str,
datadir: Path, *,
timerange: Optional[TimeRange] = None,
fill_up_missing: bool = True,
drop_incomplete: bool = True,
startup_candles: int = 0,
data_format: str = None,
data_handler: IDataHandler = None,
) -> DataFrame:
"""
Load cached ohlcv history for the given pair.
:param pair: Pair to load data for
:param timeframe: Timeframe (e.g. "5m")
:param datadir: Path to the data storage location.
:param data_format: Format of the data. Ignored if data_handler is set.
:param timerange: Limit data to be loaded to this timerange
:param fill_up_missing: Fill missing values with "No action"-candles
:param drop_incomplete: Drop last candle assuming it may be incomplete.
:param startup_candles: Additional candles to load at the start of the period
:param data_handler: Initialized data-handler to use.
Will be initialized from data_format if not set
:return: DataFrame with ohlcv data, or empty DataFrame
"""
data_handler = get_datahandler(datadir, data_format, data_handler)
return data_handler.ohlcv_load(pair=pair,
timeframe=timeframe,
timerange=timerange,
fill_missing=fill_up_missing,
drop_incomplete=drop_incomplete,
startup_candles=startup_candles,
)
def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange],
data_handler: IDataHandler) -> Tuple[DataFrame, Optional[int]]:
"""
Load cached data to download more data.
If timerange is passed in, checks whether data from an before the stored data will be
downloaded.
If that's the case then what's available should be completely overwritten.
Otherwise downloads always start at the end of the available data to avoid data gaps.
Note: Only used by download_pair_history().
"""
start = None
if timerange:
if timerange.starttype == 'date':
# TODO: convert to date for conversion
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
# Intentionally don't pass timerange in - since we need to load the full dataset.
data = data_handler.ohlcv_load(pair, timeframe=timeframe,
timerange=None, fill_missing=False,
drop_incomplete=True, warn_no_data=False)
if not data.empty:
if start and start < data.iloc[0]['date']:
# Earlier data than existing data requested, redownload all
data = DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS)
else:
start = data.iloc[-1]['date']
start_ms = int(start.timestamp() * 1000) if start else None
return data, start_ms
def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base,
candletype):
tmpdir1 = Path(tmpdir)
prependix = '' if candletype == CandleType.SPOT else 'futures/'
files_orig = []
files_temp = []
files_new = []
for file in file_base:
file_orig = testdatadir / f"{prependix}{file}.json"
file_temp = tmpdir1 / f"{prependix}{file}.json"
file_new = tmpdir1 / f"{prependix}{file}.json.gz"
IDataHandler.create_dir_if_needed(file_temp)
copyfile(file_orig, file_temp)
files_orig.append(file_orig)
files_temp.append(file_temp)
files_new.append(file_new)
default_conf['datadir'] = tmpdir1
default_conf['pairs'] = ['XRP_ETH', 'XRP_USDT', 'UNITTEST_USDT']
default_conf['timeframes'] = ['1m', '5m', '1h']
assert not file_new.exists()
convert_ohlcv_format(default_conf,
convert_from='json',
convert_to='jsongz',
erase=False,
candle_type=candletype)
for file in (files_temp + files_new):
assert file.exists()
# Remove original files
for file in (files_temp):
file.unlink()
# Convert back
convert_ohlcv_format(default_conf,
convert_from='jsongz',
convert_to='json',
erase=True,
candle_type=candletype)
for file in (files_temp):
assert file.exists()
for file in (files_new):
assert not file.exists()
def _download_trades_history(exchange: Exchange,
pair: str, *,
timerange: Optional[TimeRange] = None,
data_handler: IDataHandler
) -> bool:
"""
Download trade history from the exchange.
Appends to previously downloaded trades data.
"""
try:
since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None
trades = data_handler.trades_load(pair)
from_id = trades[-1]['id'] if trades else None
logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None')
logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None')
# Default since_ms to 30 days if nothing is given
new_trades = exchange.get_historic_trades(pair=pair,
since=since if since else
int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000,
from_id=from_id,
)
trades.extend(new_trades[1])
data_handler.trades_store(pair, data=trades)
logger.debug("New Start: %s", trades[0]['datetime'])
logger.debug("New End: %s", trades[-1]['datetime'])
logger.info(f"New Amount of trades: {len(trades)}")
return True
except Exception as e:
logger.error(
f'Failed to download historic trades for pair: "{pair}". '
f'Error: {e}'
)
return False
def _download_pair_history(datadir: Path,
exchange: Exchange,
pair: str, *,
timeframe: str = '5m',
timerange: Optional[TimeRange] = None,
data_handler: IDataHandler = None) -> bool:
"""
Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that
exists in a cache. If timerange starts earlier than the data in the cache,
the full data will be redownloaded
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pair: pair to download
:param timeframe: Timeframe (e.g "5m")
:param timerange: range of time to download
:return: bool with success state
"""
data_handler = get_datahandler(datadir, data_handler=data_handler)
try:
logger.info(
f'Download history data for pair: "{pair}", timeframe: {timeframe} '
f'and store in {datadir}.'
)
# data, since_ms = _load_cached_data_for_updating_old(datadir, pair, timeframe, timerange)
data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange,
data_handler=data_handler)
logger.debug("Current Start: %s",
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
logger.debug("Current End: %s",
f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
# Default since_ms to 30 days if nothing is given
new_data = exchange.get_historic_ohlcv(pair=pair,
timeframe=timeframe,
since_ms=since_ms if since_ms else
int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000
)
# TODO: Maybe move parsing to exchange class (?)
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
fill_missing=False, drop_incomplete=True)
if data.empty:
data = new_dataframe
else:
data = data.append(new_dataframe)
logger.debug("New Start: %s",
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
logger.debug("New End: %s",
f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
data_handler.ohlcv_store(pair, timeframe, data=data)
return True
except Exception as e:
logger.error(
f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}. '
f'Error: {e}'
)
return False
def _download_pair_history(pair: str, *,
datadir: Path,
exchange: Exchange,
timeframe: str = '5m',
process: str = '',
new_pairs_days: int = 30,
data_handler: IDataHandler = None,
timerange: Optional[TimeRange] = None,
candle_type: CandleType,
erase: bool = False,
) -> bool:
"""
Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that
exists in a cache. If timerange starts earlier than the data in the cache,
the full data will be redownloaded
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pair: pair to download
:param timeframe: Timeframe (e.g "5m")
:param timerange: range of time to download
:param candle_type: Any of the enum CandleType (must match trading mode!)
:param erase: Erase existing data
:return: bool with success state
"""
data_handler = get_datahandler(datadir, data_handler=data_handler)
try:
if erase:
if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type):
logger.info(f'Deleting existing data for pair {pair}, {timeframe}, {candle_type}.')
logger.info(
f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe}, '
f'candle type: {candle_type} and store in {datadir}.'
)
data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange,
data_handler=data_handler,
candle_type=candle_type)
logger.debug("Current Start: %s",
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
logger.debug("Current End: %s",
f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
# Default since_ms to 30 days if nothing is given
new_data = exchange.get_historic_ohlcv(pair=pair,
timeframe=timeframe,
since_ms=since_ms if since_ms else
arrow.utcnow().shift(
days=-new_pairs_days).int_timestamp * 1000,
is_new_pair=data.empty,
candle_type=candle_type,
)
# TODO: Maybe move parsing to exchange class (?)
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
fill_missing=False, drop_incomplete=True)
if data.empty:
data = new_dataframe
else:
# Run cleaning again to ensure there were no duplicate candles
# Especially between existing and new data.
data = clean_ohlcv_dataframe(concat([data, new_dataframe], axis=0), timeframe, pair,
fill_missing=False, drop_incomplete=False)
logger.debug("New Start: %s",
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
logger.debug("New End: %s",
f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
data_handler.ohlcv_store(pair, timeframe, data=data, candle_type=candle_type)
return True
except Exception:
logger.exception(
f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}.'
)
return False
def test_rebuild_pair_from_filename(input, expected):
assert IDataHandler.rebuild_pair_from_filename(input) == expected
