def load_data_test(what, testdatadir):
timerange = TimeRange.parse_timerange('1510694220-1510700340')
data = history.load_pair_history(pair='UNITTEST/BTC', datadir=testdatadir,
timeframe='1m', timerange=timerange,
drop_incomplete=False,
fill_up_missing=False)
base = 0.001
if what == 'raise':
data.loc[:, 'open'] = data.index * base
data.loc[:, 'high'] = data.index * base + 0.0001
data.loc[:, 'low'] = data.index * base - 0.0001
data.loc[:, 'close'] = data.index * base
if what == 'lower':
data.loc[:, 'open'] = 1 - data.index * base
data.loc[:, 'high'] = 1 - data.index * base + 0.0001
data.loc[:, 'low'] = 1 - data.index * base - 0.0001
data.loc[:, 'close'] = 1 - data.index * base
if what == 'sine':
hz = 0.1 # frequency
data.loc[:, 'open'] = np.sin(data.index * hz) / 1000 + base
data.loc[:, 'high'] = np.sin(data.index * hz) / 1000 + base + 0.0001
data.loc[:, 'low'] = np.sin(data.index * hz) / 1000 + base - 0.0001
data.loc[:, 'close'] = np.sin(data.index * hz) / 1000 + base
return {'UNITTEST/BTC': clean_ohlcv_dataframe(data, timeframe='1m', pair='UNITTEST/BTC',
fill_missing=True)}
def ohlcv_load(
self,
pair,
timeframe: str,
candle_type: CandleType,
timerange: Optional[TimeRange] = None,
fill_missing: bool = True,
drop_incomplete: bool = True,
startup_candles: int = 0,
warn_no_data: bool = True,
) -> DataFrame:
"""
Load cached candle (OHLCV) data for the given pair.
:param pair: Pair to load data for
:param timeframe: Timeframe (e.g. "5m")
:param timerange: Limit data to be loaded to this timerange
:param fill_missing: Fill missing values with "No action"-candles
:param drop_incomplete: Drop last candle assuming it may be incomplete.
:param startup_candles: Additional candles to load at the start of the period
:param warn_no_data: Log a warning message when no data is found
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: DataFrame with ohlcv data, or empty DataFrame
"""
# Fix startup period
timerange_startup = deepcopy(timerange)
if startup_candles > 0 and timerange_startup:
timerange_startup.subtract_start(
timeframe_to_seconds(timeframe) * startup_candles)
pairdf = self._ohlcv_load(pair,
timeframe,
timerange=timerange_startup,
candle_type=candle_type)
if self._check_empty_df(pairdf, pair, timeframe, candle_type,
warn_no_data):
return pairdf
else:
enddate = pairdf.iloc[-1]['date']
if timerange_startup:
self._validate_pairdata(pair, pairdf, timeframe, candle_type,
timerange_startup)
pairdf = trim_dataframe(pairdf, timerange_startup)
if self._check_empty_df(pairdf, pair, timeframe, candle_type,
warn_no_data):
return pairdf
# incomplete candles should only be dropped if we didn't trim the end beforehand.
pairdf = clean_ohlcv_dataframe(
pairdf,
timeframe,
pair=pair,
fill_missing=fill_missing,
drop_incomplete=(drop_incomplete
and enddate == pairdf.iloc[-1]['date']))
self._check_empty_df(pairdf, pair, timeframe, candle_type,
warn_no_data)
return pairdf
def ohlcv_load(self,
pair,
timeframe: str,
timerange: Optional[TimeRange] = None,
fill_missing: bool = True,
drop_incomplete: bool = True,
startup_candles: int = 0,
warn_no_data: bool = True) -> DataFrame:
"""
Load cached ticker history for the given pair.
:param pair: Pair to load data for
:param timeframe: Ticker timeframe (e.g. "5m")
:param timerange: Limit data to be loaded to this timerange
:param fill_missing: Fill missing values with "No action"-candles
:param drop_incomplete: Drop last candle assuming it may be incomplete.
:param startup_candles: Additional candles to load at the start of the period
:param warn_no_data: Log a warning message when no data is found
:return: DataFrame with ohlcv data, or empty DataFrame
"""
# Fix startup period
timerange_startup = deepcopy(timerange)
if startup_candles > 0 and timerange_startup:
timerange_startup.subtract_start(
timeframe_to_seconds(timeframe) * startup_candles)
pairdf = self._ohlcv_load(pair, timeframe, timerange=timerange_startup)
if pairdf.empty:
if warn_no_data:
logger.warning(
f'No history data for pair: "{pair}", timeframe: {timeframe}. '
'Use `freqtrade download-data` to download the data')
return pairdf
else:
enddate = pairdf.iloc[-1]['date']
if timerange_startup:
self._validate_pairdata(pair, pairdf, timerange_startup)
pairdf = trim_dataframe(pairdf, timerange_startup)
# incomplete candles should only be dropped if we didn't trim the end beforehand.
return clean_ohlcv_dataframe(
pairdf,
timeframe,
pair=pair,
fill_missing=fill_missing,
drop_incomplete=(drop_incomplete
and enddate == pairdf.iloc[-1]['date']))
def _download_pair_history(pair: str, *,
datadir: Path,
exchange: Exchange,
timeframe: str = '5m',
process: str = '',
new_pairs_days: int = 30,
data_handler: IDataHandler = None,
timerange: Optional[TimeRange] = None,
candle_type: CandleType,
erase: bool = False,
) -> bool:
"""
Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that
exists in a cache. If timerange starts earlier than the data in the cache,
the full data will be redownloaded
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pair: pair to download
:param timeframe: Timeframe (e.g "5m")
:param timerange: range of time to download
:param candle_type: Any of the enum CandleType (must match trading mode!)
:param erase: Erase existing data
:return: bool with success state
"""
data_handler = get_datahandler(datadir, data_handler=data_handler)
try:
if erase:
if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type):
logger.info(f'Deleting existing data for pair {pair}, {timeframe}, {candle_type}.')
logger.info(
f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe}, '
f'candle type: {candle_type} and store in {datadir}.'
)
data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange,
data_handler=data_handler,
candle_type=candle_type)
logger.debug("Current Start: %s",
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
logger.debug("Current End: %s",
f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
# Default since_ms to 30 days if nothing is given
new_data = exchange.get_historic_ohlcv(pair=pair,
timeframe=timeframe,
since_ms=since_ms if since_ms else
arrow.utcnow().shift(
days=-new_pairs_days).int_timestamp * 1000,
is_new_pair=data.empty,
candle_type=candle_type,
)
# TODO: Maybe move parsing to exchange class (?)
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
fill_missing=False, drop_incomplete=True)
if data.empty:
data = new_dataframe
else:
# Run cleaning again to ensure there were no duplicate candles
# Especially between existing and new data.
data = clean_ohlcv_dataframe(concat([data, new_dataframe], axis=0), timeframe, pair,
fill_missing=False, drop_incomplete=False)
logger.debug("New Start: %s",
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
logger.debug("New End: %s",
f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
data_handler.ohlcv_store(pair, timeframe, data=data, candle_type=candle_type)
return True
except Exception:
logger.exception(
f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}.'
)
return False
def _download_pair_history(datadir: Path,
exchange: Exchange,
pair: str,
*,
timeframe: str = '5m',
timerange: Optional[TimeRange] = None,
data_handler: IDataHandler = None) -> bool:
"""
Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that
exists in a cache. If timerange starts earlier than the data in the cache,
the full data will be redownloaded
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pair: pair to download
:param timeframe: Timeframe (e.g "5m")
:param timerange: range of time to download
:return: bool with success state
"""
data_handler = get_datahandler(datadir, data_handler=data_handler)
try:
logger.info(
f'Download history data for pair: "{pair}", timeframe: {timeframe} '
f'and store in {datadir}.')
# data, since_ms = _load_cached_data_for_updating_old(datadir, pair, timeframe, timerange)
data, since_ms = _load_cached_data_for_updating(
pair, timeframe, timerange, data_handler=data_handler)
logger.debug(
"Current Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}"
if not data.empty else 'None')
logger.debug(
"Current End: %s", f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}"
if not data.empty else 'None')
# Default since_ms to 30 days if nothing is given
new_data = exchange.get_historic_ohlcv(
pair=pair,
timeframe=timeframe,
since_ms=since_ms if since_ms else
int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000)
# TODO: Maybe move parsing to exchange class (?)
new_dataframe = ohlcv_to_dataframe(new_data,
timeframe,
pair,
fill_missing=False,
drop_incomplete=True)
if data.empty:
data = new_dataframe
else:
# Run cleaning again to ensure there were no duplicate candles
# Especially between existing and new data.
data = clean_ohlcv_dataframe(data.append(new_dataframe),
timeframe,
pair,
fill_missing=False,
drop_incomplete=False)
logger.debug(
"New Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}"
if not data.empty else 'None')
logger.debug(
"New End: %s", f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}"
if not data.empty else 'None')
data_handler.ohlcv_store(pair, timeframe, data=data)
return True
except Exception as e:
logger.error(
f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}. '
f'Error: {e}')
return False
