def load_data_test(what, testdatadir): timerange = TimeRange.parse_timerange('1510694220-1510700340') data = history.load_pair_history(pair='UNITTEST/BTC', datadir=testdatadir, timeframe='1m', timerange=timerange, drop_incomplete=False, fill_up_missing=False) base = 0.001 if what == 'raise': data.loc[:, 'open'] = data.index * base data.loc[:, 'high'] = data.index * base + 0.0001 data.loc[:, 'low'] = data.index * base - 0.0001 data.loc[:, 'close'] = data.index * base if what == 'lower': data.loc[:, 'open'] = 1 - data.index * base data.loc[:, 'high'] = 1 - data.index * base + 0.0001 data.loc[:, 'low'] = 1 - data.index * base - 0.0001 data.loc[:, 'close'] = 1 - data.index * base if what == 'sine': hz = 0.1 # frequency data.loc[:, 'open'] = np.sin(data.index * hz) / 1000 + base data.loc[:, 'high'] = np.sin(data.index * hz) / 1000 + base + 0.0001 data.loc[:, 'low'] = np.sin(data.index * hz) / 1000 + base - 0.0001 data.loc[:, 'close'] = np.sin(data.index * hz) / 1000 + base return {'UNITTEST/BTC': clean_ohlcv_dataframe(data, timeframe='1m', pair='UNITTEST/BTC', fill_missing=True)}
def ohlcv_load( self, pair, timeframe: str, candle_type: CandleType, timerange: Optional[TimeRange] = None, fill_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, warn_no_data: bool = True, ) -> DataFrame: """ Load cached candle (OHLCV) data for the given pair. :param pair: Pair to load data for :param timeframe: Timeframe (e.g. "5m") :param timerange: Limit data to be loaded to this timerange :param fill_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period :param warn_no_data: Log a warning message when no data is found :param candle_type: Any of the enum CandleType (must match trading mode!) :return: DataFrame with ohlcv data, or empty DataFrame """ # Fix startup period timerange_startup = deepcopy(timerange) if startup_candles > 0 and timerange_startup: timerange_startup.subtract_start( timeframe_to_seconds(timeframe) * startup_candles) pairdf = self._ohlcv_load(pair, timeframe, timerange=timerange_startup, candle_type=candle_type) if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data): return pairdf else: enddate = pairdf.iloc[-1]['date'] if timerange_startup: self._validate_pairdata(pair, pairdf, timeframe, candle_type, timerange_startup) pairdf = trim_dataframe(pairdf, timerange_startup) if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data): return pairdf # incomplete candles should only be dropped if we didn't trim the end beforehand. pairdf = clean_ohlcv_dataframe( pairdf, timeframe, pair=pair, fill_missing=fill_missing, drop_incomplete=(drop_incomplete and enddate == pairdf.iloc[-1]['date'])) self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data) return pairdf
def ohlcv_load(self, pair, timeframe: str, timerange: Optional[TimeRange] = None, fill_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, warn_no_data: bool = True) -> DataFrame: """ Load cached ticker history for the given pair. :param pair: Pair to load data for :param timeframe: Ticker timeframe (e.g. "5m") :param timerange: Limit data to be loaded to this timerange :param fill_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period :param warn_no_data: Log a warning message when no data is found :return: DataFrame with ohlcv data, or empty DataFrame """ # Fix startup period timerange_startup = deepcopy(timerange) if startup_candles > 0 and timerange_startup: timerange_startup.subtract_start( timeframe_to_seconds(timeframe) * startup_candles) pairdf = self._ohlcv_load(pair, timeframe, timerange=timerange_startup) if pairdf.empty: if warn_no_data: logger.warning( f'No history data for pair: "{pair}", timeframe: {timeframe}. ' 'Use `freqtrade download-data` to download the data') return pairdf else: enddate = pairdf.iloc[-1]['date'] if timerange_startup: self._validate_pairdata(pair, pairdf, timerange_startup) pairdf = trim_dataframe(pairdf, timerange_startup) # incomplete candles should only be dropped if we didn't trim the end beforehand. return clean_ohlcv_dataframe( pairdf, timeframe, pair=pair, fill_missing=fill_missing, drop_incomplete=(drop_incomplete and enddate == pairdf.iloc[-1]['date']))
def _download_pair_history(pair: str, *, datadir: Path, exchange: Exchange, timeframe: str = '5m', process: str = '', new_pairs_days: int = 30, data_handler: IDataHandler = None, timerange: Optional[TimeRange] = None, candle_type: CandleType, erase: bool = False, ) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data :param pair: pair to download :param timeframe: Timeframe (e.g "5m") :param timerange: range of time to download :param candle_type: Any of the enum CandleType (must match trading mode!) :param erase: Erase existing data :return: bool with success state """ data_handler = get_datahandler(datadir, data_handler=data_handler) try: if erase: if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): logger.info(f'Deleting existing data for pair {pair}, {timeframe}, {candle_type}.') logger.info( f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe}, ' f'candle type: {candle_type} and store in {datadir}.' ) data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange, data_handler=data_handler, candle_type=candle_type) logger.debug("Current Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') logger.debug("Current End: %s", f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') # Default since_ms to 30 days if nothing is given new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms if since_ms else arrow.utcnow().shift( days=-new_pairs_days).int_timestamp * 1000, is_new_pair=data.empty, candle_type=candle_type, ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, fill_missing=False, drop_incomplete=True) if data.empty: data = new_dataframe else: # Run cleaning again to ensure there were no duplicate candles # Especially between existing and new data. data = clean_ohlcv_dataframe(concat([data, new_dataframe], axis=0), timeframe, pair, fill_missing=False, drop_incomplete=False) logger.debug("New Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') logger.debug("New End: %s", f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') data_handler.ohlcv_store(pair, timeframe, data=data, candle_type=candle_type) return True except Exception: logger.exception( f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}.' ) return False
def _download_pair_history(datadir: Path, exchange: Exchange, pair: str, *, timeframe: str = '5m', timerange: Optional[TimeRange] = None, data_handler: IDataHandler = None) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data :param pair: pair to download :param timeframe: Timeframe (e.g "5m") :param timerange: range of time to download :return: bool with success state """ data_handler = get_datahandler(datadir, data_handler=data_handler) try: logger.info( f'Download history data for pair: "{pair}", timeframe: {timeframe} ' f'and store in {datadir}.') # data, since_ms = _load_cached_data_for_updating_old(datadir, pair, timeframe, timerange) data, since_ms = _load_cached_data_for_updating( pair, timeframe, timerange, data_handler=data_handler) logger.debug( "Current Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') logger.debug( "Current End: %s", f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') # Default since_ms to 30 days if nothing is given new_data = exchange.get_historic_ohlcv( pair=pair, timeframe=timeframe, since_ms=since_ms if since_ms else int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, fill_missing=False, drop_incomplete=True) if data.empty: data = new_dataframe else: # Run cleaning again to ensure there were no duplicate candles # Especially between existing and new data. data = clean_ohlcv_dataframe(data.append(new_dataframe), timeframe, pair, fill_missing=False, drop_incomplete=False) logger.debug( "New Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') logger.debug( "New End: %s", f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') data_handler.ohlcv_store(pair, timeframe, data=data) return True except Exception as e: logger.error( f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}. ' f'Error: {e}') return False