def test_trades_remove_duplicates(trades_history): trades_history1 = trades_history * 3 assert len(trades_history1) == len(trades_history) * 3 res = trades_remove_duplicates(trades_history1) assert len(res) == len(trades_history) for i, t in enumerate(res): assert t == trades_history[i]
def _download_trades_history(exchange: Exchange, pair: str, *, new_pairs_days: int = 30, timerange: Optional[TimeRange] = None, data_handler: IDataHandler ) -> bool: """ Download trade history from the exchange. Appends to previously downloaded trades data. """ try: since = timerange.startts * 1000 if \ (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift( days=-new_pairs_days).float_timestamp) * 1000 trades = data_handler.trades_load(pair) # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS # DEFAULT_TRADES_COLUMNS: 0 -> timestamp # DEFAULT_TRADES_COLUMNS: 1 -> id if trades and since < trades[0][0]: # since is before the first trade logger.info(f"Start earlier than available data. Redownloading trades for {pair}...") trades = [] from_id = trades[-1][1] if trades else None if trades and since < trades[-1][0]: # Reset since to the last available point # - 5 seconds (to ensure we're getting all trades) since = trades[-1][0] - (5 * 1000) logger.info(f"Using last trade date -5s - Downloading trades for {pair} " f"since: {format_ms_time(since)}.") logger.debug(f"Current Start: {format_ms_time(trades[0][0]) if trades else 'None'}") logger.debug(f"Current End: {format_ms_time(trades[-1][0]) if trades else 'None'}") logger.info(f"Current Amount of trades: {len(trades)}") # Default since_ms to 30 days if nothing is given new_trades = exchange.get_historic_trades(pair=pair, since=since, from_id=from_id, ) trades.extend(new_trades[1]) # Remove duplicates to make sure we're not storing data we don't need trades = trades_remove_duplicates(trades) data_handler.trades_store(pair, data=trades) logger.debug(f"New Start: {format_ms_time(trades[0][0])}") logger.debug(f"New End: {format_ms_time(trades[-1][0])}") logger.info(f"New Amount of trades: {len(trades)}") return True except Exception: logger.exception( f'Failed to download historic trades for pair: "{pair}". ' ) return False
def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: """ Load a pair from file, either .json.gz or .json Removes duplicates in the process. :param pair: Load trades for this pair :param timerange: Timerange to load trades for - currently not implemented :return: List of trades """ return trades_remove_duplicates(self._trades_load(pair, timerange=timerange))