def mocked_load_data(datadir, pairs=[], timeframe='0m', timerange=None, *args, **kwargs): hz = 0.1 base = 0.001 NEOBTC = [ [ tests_start_time.shift(minutes=(x * timeframe_in_minute)).int_timestamp * 1000, math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, math.sin(x * hz) / 1000 + base, 123.45 ] for x in range(0, 500)] hz = 0.2 base = 0.002 LTCBTC = [ [ tests_start_time.shift(minutes=(x * timeframe_in_minute)).int_timestamp * 1000, math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, math.sin(x * hz) / 1000 + base, 123.45 ] for x in range(0, 500)] pairdata = {'NEO/BTC': ohlcv_to_dataframe(NEOBTC, '1h', pair="NEO/BTC", fill_missing=True), 'LTC/BTC': ohlcv_to_dataframe(LTCBTC, '1h', pair="LTC/BTC", fill_missing=True)} return pairdata
def test_ohlcv_drop_incomplete(caplog): timeframe = '1d' ticks = [ [ 1559750400000, # 2019-06-04 8.794e-05, # open 8.948e-05, # high 8.794e-05, # low 8.88e-05, # close 2255, # volume (in quote currency) ], [ 1559836800000, # 2019-06-05 8.88e-05, 8.942e-05, 8.88e-05, 8.893e-05, 9911, ], [ 1559923200000, # 2019-06-06 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, 2251 ], [ 1560009600000, # 2019-06-07 8.877e-05, 8.883e-05, 8.895e-05, 8.817e-05, 123551 ] ] caplog.set_level(logging.DEBUG) data = ohlcv_to_dataframe(ticks, timeframe, pair="UNITTEST/BTC", fill_missing=False, drop_incomplete=False) assert len(data) == 4 assert not log_has("Dropping last candle", caplog) # Drop last candle data = ohlcv_to_dataframe(ticks, timeframe, pair="UNITTEST/BTC", fill_missing=False, drop_incomplete=True) assert len(data) == 3 assert log_has("Dropping last candle", caplog)
def test_ohlcv_fill_up_missing_data2(caplog): timeframe = '5m' ticks = [ [ 1511686200000, # 8:50:00 8.794e-05, # open 8.948e-05, # high 8.794e-05, # low 8.88e-05, # close 2255, # volume (in quote currency) ], [ 1511686500000, # 8:55:00 8.88e-05, 8.942e-05, 8.88e-05, 8.893e-05, 9911, ], [ 1511687100000, # 9:05:00 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, 2251 ], [ 1511687400000, # 9:10:00 8.877e-05, 8.883e-05, 8.895e-05, 8.817e-05, 123551 ] ] # Generate test-data without filling missing data = ohlcv_to_dataframe(ticks, timeframe, pair="UNITTEST/BTC", fill_missing=False) assert len(data) == 3 caplog.set_level(logging.DEBUG) data2 = ohlcv_fill_up_missing_data(data, timeframe, "UNITTEST/BTC") assert len(data2) == 4 # 3rd candle has been filled row = data2.loc[2, :] assert row['volume'] == 0 # close should match close of previous candle assert row['close'] == data.loc[1, 'close'] assert row['open'] == row['close'] assert row['high'] == row['close'] assert row['low'] == row['close'] # Column names should not change assert (data.columns == data2.columns).all() assert log_has_re( f"Missing data fillup for UNITTEST/BTC: before: " f"{len(data)} - after: {len(data2)}.*", caplog)
def test_ohlcv_to_dataframe(ohlcv_history_list, caplog): columns = ['date', 'open', 'high', 'low', 'close', 'volume'] caplog.set_level(logging.DEBUG) # Test file with BV data dataframe = ohlcv_to_dataframe(ohlcv_history_list, '5m', pair="UNITTEST/BTC", fill_missing=True) assert dataframe.columns.tolist() == columns assert log_has('Converting candle (OHLCV) data to dataframe for pair UNITTEST/BTC.', caplog)
def test_load_cached_data_for_updating(mocker, testdatadir) -> None: data_handler = get_datahandler(testdatadir, 'json') test_data = None test_filename = testdatadir.joinpath('UNITTEST_BTC-1m.json') with open(test_filename, "rt") as file: test_data = json.load(file) test_data_df = ohlcv_to_dataframe(test_data, '1m', 'UNITTEST/BTC', fill_missing=False, drop_incomplete=False) # now = last cached item + 1 hour now_ts = test_data[-1][0] / 1000 + 60 * 60 mocker.patch('arrow.utcnow', return_value=arrow.get(now_ts)) # timeframe starts earlier than the cached data # should fully update data timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler) assert data.empty assert start_ts == test_data[0][0] - 1000 # timeframe starts in the center of the cached data # should return the chached data w/o the last item timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler) assert_frame_equal(data, test_data_df.iloc[:-1]) assert test_data[-2][0] <= start_ts < test_data[-1][0] # timeframe starts after the chached data # should return the chached data w/o the last item timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 100, 0) data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler) assert_frame_equal(data, test_data_df.iloc[:-1]) assert test_data[-2][0] <= start_ts < test_data[-1][0] # no datafile exist # should return timestamp start time timerange = TimeRange('date', None, now_ts - 10000, 0) data, start_ts = _load_cached_data_for_updating('NONEXIST/BTC', '1m', timerange, data_handler) assert data.empty assert start_ts == (now_ts - 10000) * 1000 # no datafile exist, no timeframe is set # should return an empty array and None data, start_ts = _load_cached_data_for_updating('NONEXIST/BTC', '1m', None, data_handler) assert data.empty assert start_ts is None
def test_datesarray_to_datetimearray(ohlcv_history_list): dataframes = ohlcv_to_dataframe(ohlcv_history_list, "5m", pair="UNITTEST/BTC", fill_missing=True) dates = datesarray_to_datetimearray(dataframes['date']) assert isinstance(dates[0], datetime.datetime) assert dates[0].year == 2017 assert dates[0].month == 11 assert dates[0].day == 26 assert dates[0].hour == 8 assert dates[0].minute == 50 date_len = len(dates) assert date_len == 2
def refresh_latest_ohlcv(self, pair_list: List[Tuple[str, str]]) -> List[Tuple[str, List]]: """ Refresh in-memory OHLCV asynchronously and set `_klines` with the result Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). Only used in the dataprovider.refresh() method. :param pair_list: List of 2 element tuples containing pair, interval to refresh :return: TODO: return value is only used in the tests, get rid of it """ logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) input_coroutines = [] # Gather coroutines to run for pair, timeframe in set(pair_list): if (not ((pair, timeframe) in self._klines) or self._now_is_time_to_refresh(pair, timeframe)): input_coroutines.append(self._async_get_candle_history(pair, timeframe)) else: logger.debug( "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", pair, timeframe ) results = asyncio.get_event_loop().run_until_complete( asyncio.gather(*input_coroutines, return_exceptions=True)) # handle caching for res in results: if isinstance(res, Exception): logger.warning("Async code raised an exception: %s", res.__class__.__name__) continue pair = res[0] timeframe = res[1] ticks = res[2] # keeping last candle time as last refreshed time of the pair if ticks: self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache self._klines[(pair, timeframe)] = ohlcv_to_dataframe( ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) return results
def _download_pair_history(datadir: Path, exchange: Exchange, pair: str, *, timeframe: str = '5m', timerange: Optional[TimeRange] = None, data_handler: IDataHandler = None) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data :param pair: pair to download :param timeframe: Timeframe (e.g "5m") :param timerange: range of time to download :return: bool with success state """ data_handler = get_datahandler(datadir, data_handler=data_handler) try: logger.info( f'Download history data for pair: "{pair}", timeframe: {timeframe} ' f'and store in {datadir}.' ) # data, since_ms = _load_cached_data_for_updating_old(datadir, pair, timeframe, timerange) data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange, data_handler=data_handler) logger.debug("Current Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') logger.debug("Current End: %s", f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') # Default since_ms to 30 days if nothing is given new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms if since_ms else int(arrow.utcnow().shift( days=-30).float_timestamp) * 1000 ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, fill_missing=False, drop_incomplete=True) if data.empty: data = new_dataframe else: data = data.append(new_dataframe) logger.debug("New Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') logger.debug("New End: %s", f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') data_handler.ohlcv_store(pair, timeframe, data=data) return True except Exception as e: logger.error( f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}. ' f'Error: {e}' ) return False
def _download_pair_history(pair: str, *, datadir: Path, exchange: Exchange, timeframe: str = '5m', process: str = '', new_pairs_days: int = 30, data_handler: IDataHandler = None, timerange: Optional[TimeRange] = None, candle_type: CandleType, erase: bool = False, ) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded Based on @Rybolov work: https://github.com/rybolov/freqtrade-data :param pair: pair to download :param timeframe: Timeframe (e.g "5m") :param timerange: range of time to download :param candle_type: Any of the enum CandleType (must match trading mode!) :param erase: Erase existing data :return: bool with success state """ data_handler = get_datahandler(datadir, data_handler=data_handler) try: if erase: if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): logger.info(f'Deleting existing data for pair {pair}, {timeframe}, {candle_type}.') logger.info( f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe}, ' f'candle type: {candle_type} and store in {datadir}.' ) data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange, data_handler=data_handler, candle_type=candle_type) logger.debug("Current Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') logger.debug("Current End: %s", f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') # Default since_ms to 30 days if nothing is given new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms if since_ms else arrow.utcnow().shift( days=-new_pairs_days).int_timestamp * 1000, is_new_pair=data.empty, candle_type=candle_type, ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, fill_missing=False, drop_incomplete=True) if data.empty: data = new_dataframe else: # Run cleaning again to ensure there were no duplicate candles # Especially between existing and new data. data = clean_ohlcv_dataframe(concat([data, new_dataframe], axis=0), timeframe, pair, fill_missing=False, drop_incomplete=False) logger.debug("New Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') logger.debug("New End: %s", f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') data_handler.ohlcv_store(pair, timeframe, data=data, candle_type=candle_type) return True except Exception: logger.exception( f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}.' ) return False