def download_pair_history(datadir: Optional[Path], exchange: Optional[Exchange], pair: str, ticker_interval: str = '5m', timerange: Optional[TimeRange] = None) -> bool: """ Download the latest ticker intervals from the exchange for the pair passed in parameters The data is downloaded starting from the last correct ticker interval data that exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data :param pair: pair to download :param ticker_interval: ticker interval :param timerange: range of time to download :return: bool with success state """ if not exchange: raise OperationalException( "Exchange needs to be initialized when downloading pair history data" ) try: filename = pair_data_filename(datadir, pair, ticker_interval) logger.info( f'Download history data for pair: "{pair}", interval: {ticker_interval} ' f'and store in {datadir}.') data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange) logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') # Default since_ms to 30 days if nothing is given new_data = exchange.get_history( pair=pair, ticker_interval=ticker_interval, since_ms=since_ms if since_ms else int(arrow.utcnow().shift(days=-3 * 365).float_timestamp) * 1000) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) misc.file_dump_json(filename, data) return True except Exception as e: logger.error( f'Failed to download history data for pair: "{pair}", interval: {ticker_interval}. ' f'Error: {e}') return False
def test_format_ms_time() -> None: # Date 2018-04-10 18:02:01 date_in_epoch_ms = 1523383321000 date = format_ms_time(date_in_epoch_ms) assert type(date) is str res = datetime.datetime(2018, 4, 10, 18, 2, 1, tzinfo=datetime.timezone.utc) assert date == res.astimezone(None).strftime('%Y-%m-%dT%H:%M:%S') res = datetime.datetime(2017, 12, 13, 8, 2, 1, tzinfo=datetime.timezone.utc) # Date 2017-12-13 08:02:01 date_in_epoch_ms = 1513152121000 assert format_ms_time(date_in_epoch_ms) == res.astimezone(None).strftime('%Y-%m-%dT%H:%M:%S')
def download_pair_history(datadir: Optional[Path], exchange: Exchange, pair: str, ticker_interval: str = '5m', timerange: Optional[TimeRange] = None) -> bool: """ Download the latest ticker intervals from the exchange for the pair passed in parameters The data is downloaded starting from the last correct ticker interval data that exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data :param pair: pair to download :param ticker_interval: ticker interval :param timerange: range of time to download :return: bool with success state """ try: path = make_testdata_path(datadir) filepair = pair.replace("/", "_") filename = path.joinpath(f'{filepair}-{ticker_interval}.json') logger.info('Download the pair: "%s", Interval: %s', pair, ticker_interval) data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange) logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') # Default since_ms to 30 days if nothing is given new_data = exchange.get_history( pair=pair, ticker_interval=ticker_interval, since_ms=since_ms if since_ms else int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) misc.file_dump_json(filename, data) return True except BaseException: logger.info('Failed to download the pair: "%s", Interval: %s', pair, ticker_interval) return False
def _download_pair_history(datadir: Path, exchange: Exchange, pair: str, timeframe: str = '5m', timerange: Optional[TimeRange] = None) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data :param pair: pair to download :param timeframe: Ticker Timeframe (e.g 5m) :param timerange: range of time to download :return: bool with success state """ try: logger.info( f'Download history data for pair: "{pair}", timeframe: {timeframe} ' f'and store in {datadir}.' ) data, since_ms = _load_cached_data_for_updating(datadir, pair, timeframe, timerange) logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') # Default since_ms to 30 days if nothing is given new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms if since_ms else int(arrow.utcnow().shift( days=-30).float_timestamp) * 1000 ) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) store_tickerdata_file(datadir, pair, timeframe, data=data) return True except Exception as e: logger.error( f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}. ' f'Error: {e}' ) return False
def download_backtesting_testdata(datadir: str, exchange: Exchange, pair: str, tick_interval: str = '5m', timerange: Optional[TimeRange] = None) -> None: """ Download the latest ticker intervals from the exchange for the pairs passed in parameters The data is downloaded starting from the last correct ticker interval data that esists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data :param pairs: list of pairs to download :param tick_interval: ticker interval :param timerange: range of time to download :return: None """ path = make_testdata_path(datadir) filepair = pair.replace("/", "_") filename = os.path.join(path, f'{filepair}-{tick_interval}.json') logger.info( 'Download the pair: "%s", Interval: %s', pair, tick_interval ) data, since_ms = load_cached_data_for_updating(filename, tick_interval, timerange) logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') new_data = exchange.get_ticker_history(pair=pair, tick_interval=tick_interval, since_ms=since_ms) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) misc.file_dump_json(filename, data)