def test_set_more_than_max_count_orders(self): test_inst = ExchangePersonalData() test_inst.orders = {} test_inst._MAX_ORDERS_COUNT = 500 nb_max_stored_orders = test_inst._MAX_ORDERS_COUNT max_timestamp = 0 for i in range(nb_max_stored_orders): time_stamp = time.time() test_inst.upsert_order( i, { "id": i, "timestamp": time_stamp, "status": OrderStatus.CLOSED.value }) if i == nb_max_stored_orders / 2: max_timestamp = time_stamp time.sleep(0.000000001) assert len(test_inst.orders) == nb_max_stored_orders test_inst.upsert_order( nb_max_stored_orders + 1, { "id": nb_max_stored_orders + 1, "timestamp": time.time(), "status": OrderStatus.CLOSED.value }) assert not [ order for order in test_inst.orders.values() if order["timestamp"] < max_timestamp ] assert len(test_inst.orders) == nb_max_stored_orders / 2 + 1
def test_upsert_order(self): test_inst = ExchangePersonalData() # without orders test_inst.orders = {} assert not test_inst.has_order(10) test_inst.upsert_order(10, None) assert test_inst.has_order(10)
def __init__(self, config, exchange_class_string): super().__init__() self.id = str(uuid.uuid4()) self.config = config self.tentacles_setup_config = None self.exchange_class_string = exchange_class_string self.exchange_name = exchange_class_string self._logger = get_logger(self.__class__.__name__) self.is_ready = False self.is_simulated: bool = False self.is_backtesting: bool = False self.rest_only: bool = False self.ignore_config: bool = False self.is_collecting: bool = False self.is_spot_only: bool = False self.is_margin: bool = False self.is_future: bool = False self.is_sandboxed: bool = False self.is_trading: bool = True self.without_auth: bool = False # exchange_only is True when exchange channels are not required (therefore not created) self.exchange_only: bool = False self.backtesting = None self.is_trader_simulated = is_trader_simulator_enabled(self.config) self.has_websocket = False self.trader = None self.exchange = None self.trading_modes = [] self.exchange_web_socket = None self.exchange_type = None self.client_symbols = [] self.client_time_frames = [] self.exchange_config = ExchangeConfig(self) self.exchange_personal_data = ExchangePersonalData(self) self.exchange_symbols_data = ExchangeSymbolsData(self)
def test_has_order(self): test_inst = ExchangePersonalData() # without orders test_inst.orders = {} assert not test_inst.has_order(10) test_inst.orders[10] = self.create_fake_order(10, None, None, None) assert test_inst.has_order(10) assert not test_inst.has_order(153)
def test_set_orders(self): test_inst = ExchangePersonalData() # without orders test_inst.orders = {} assert not test_inst.has_order(10) assert not test_inst.has_order(20) test_inst.upsert_orders([{ "id": 15 }, { "id": 10 }, { "id": 20 }, { "id": 15 }]) assert test_inst.get_order(15) assert test_inst.has_order(15) assert test_inst.has_order(20) assert test_inst.has_order(10) assert not test_inst.has_order(12) assert not test_inst.has_order(30)
def test_update_portfolio(self): test_inst = ExchangePersonalData() # without pf test_inst.portfolio = {} test_inst.update_portfolio("BTC", 10, 7, 3) assert test_inst.portfolio["BTC"][CONFIG_PORTFOLIO_FREE] == 7 assert test_inst.portfolio["BTC"][CONFIG_PORTFOLIO_USED] == 3 assert test_inst.portfolio["BTC"][CONFIG_PORTFOLIO_TOTAL] == 10 # with pf test_inst.update_portfolio("ETH", 100, 60, 40) assert test_inst.portfolio["BTC"][CONFIG_PORTFOLIO_FREE] == 7 assert test_inst.portfolio["BTC"][CONFIG_PORTFOLIO_USED] == 3 assert test_inst.portfolio["BTC"][CONFIG_PORTFOLIO_TOTAL] == 10 assert test_inst.portfolio["ETH"][CONFIG_PORTFOLIO_FREE] == 60 assert test_inst.portfolio["ETH"][CONFIG_PORTFOLIO_USED] == 40 assert test_inst.portfolio["ETH"][CONFIG_PORTFOLIO_TOTAL] == 100
def __init__(self, config, exchange_class_string, is_simulated=False, is_backtesting=False, rest_only=False, ignore_config=False, is_collecting=False, exchange_only=False, backtesting_files=None): super().__init__() self.id = str(uuid.uuid4()) self.config = config self.exchange_class_string = exchange_class_string self.rest_only = rest_only self.ignore_config = ignore_config self.backtesting_files = backtesting_files self._logger = get_logger(self.__class__.__name__) self.is_ready = False self.is_backtesting = is_backtesting self.is_simulated = is_simulated self.is_collecting = is_collecting self.exchange_only = exchange_only self.is_trader_simulated = is_trader_simulator_enabled(self.config) self.has_websocket = False self.trader = None self.exchange = None self.exchange_web_socket = None self.exchange_type = None self.client_symbols = [] self.client_time_frames = {} self.exchange_config = ExchangeConfig(self) self.exchange_personal_data = ExchangePersonalData(self) self.exchange_symbols_data = ExchangeSymbolsData(self)
class ExchangeManager(Initializable): def __init__(self, config, exchange_class_string): super().__init__() self.id = str(uuid.uuid4()) self.config = config self.exchange_class_string = exchange_class_string self.exchange_name = exchange_class_string self._logger = get_logger(self.__class__.__name__) self.is_ready = False self.is_simulated: bool = False self.is_backtesting: bool = False self.rest_only: bool = False self.ignore_config: bool = False self.is_collecting: bool = False self.is_spot_only: bool = False self.is_margin: bool = False self.is_future: bool = False self.is_sandboxed: bool = False self.without_auth: bool = False # exchange_only is True when exchange channels are not required (therefore not created) self.exchange_only: bool = False self.backtesting = None self.is_trader_simulated = is_trader_simulator_enabled(self.config) self.has_websocket = False self.trader = None self.exchange = None self.trading_modes = [] self.exchange_web_socket = None self.exchange_type = None self.client_symbols = [] self.client_time_frames = [] self.exchange_config = ExchangeConfig(self) self.exchange_personal_data = ExchangePersonalData(self) self.exchange_symbols_data = ExchangeSymbolsData(self) async def initialize_impl(self): await self.create_exchanges() async def stop(self): for trading_mode in self.trading_modes: await trading_mode.stop() if self.exchange is not None: if not self.exchange_only: await self.stop_exchange_channels() await self.exchange.stop() Exchanges.instance().del_exchange(self.exchange.name, self.id) self.exchange.exchange_manager = None if self.exchange_personal_data is not None: self.exchange_personal_data.clear() self.exchange_config = None self.exchange_personal_data = None self.exchange_symbols_data = None self.trader = None self.trading_modes = [] self.backtesting = None async def stop_exchange_channels(self): try: chan_names = list(get_exchange_channels(self.id).keys()) for channel_name in chan_names: channel = get_chan(channel_name, self.id) await channel.stop() for consumer in channel.consumers: await channel.remove_consumer(consumer) get_chan(channel_name, self.id).flush() del_chan(channel_name, self.id) del_exchange_channel_container(self.id) except KeyError: self._logger.error( f"No exchange channel for this exchange (id: {self.id})") async def register_trader(self, trader): self.trader = trader await self.exchange_personal_data.initialize() await self.exchange_config.initialize() def _load_constants(self): self._load_config_symbols_and_time_frames() self.exchange_config.set_config_time_frame() self.exchange_config.set_config_traded_pairs() def need_user_stream(self): return self.config[CONFIG_TRADER][CONFIG_ENABLED_OPTION] def reset_exchange_symbols_data(self): self.exchange_symbols_data = ExchangeSymbolsData(self) def reset_exchange_personal_data(self): self.exchange_personal_data = ExchangePersonalData(self) async def create_exchanges(self): if self.is_sandboxed: self._logger.info( f"Using sandbox exchange for {self.exchange_name}") self.exchange_type = RestExchange.create_exchange_type( self.exchange_class_string) if not self.is_backtesting: # real : create a rest or websocket exchange instance await self._create_real_exchange() else: # simulated : create exchange simulator instance await self._create_simulated_exchange() if not self.exchange_only: # create exchange producers if necessary await self._create_exchange_producers() if self.is_backtesting: await self._init_simulated_exchange() self.exchange_name = self.exchange.name self.is_ready = True """ Real exchange """ async def _create_real_exchange(self): # create REST based on ccxt exchange if self.is_spot_only: await self._search_and_create_spot_exchange() elif self.is_future: await self._search_and_create_future_exchange() elif self.is_margin: await self._search_and_create_margin_exchange() if not self.exchange: await self._search_and_create_rest_exchange() try: await self.exchange.initialize() except AuthenticationError: self._logger.error( "Authentication error, retrying without authentication...") self.without_auth = True await self._create_real_exchange() return self._load_constants() if not self.exchange_only: await self._create_exchange_channels() # create Websocket exchange if possible if not self.rest_only: # search for websocket if check_web_socket_config(self.config, self.exchange.name): await self._search_and_create_websocket() """ Simulated Exchange """ async def _create_simulated_exchange(self): self.exchange = ExchangeSimulator(self.config, self.exchange_type, self, self.backtesting) await self.exchange.initialize() self._initialize_simulator_time_frames() self.exchange_config.set_config_time_frame() self.exchange_config.set_config_traded_pairs() await self._create_exchange_channels() async def _init_simulated_exchange(self): await self.exchange.create_backtesting_exchange_producers() """ Rest exchange """ async def _search_and_create_rest_exchange(self): rest_exchange_class = get_rest_exchange_class(self.exchange_type) if rest_exchange_class: self.exchange = rest_exchange_class( config=self.config, exchange_type=self.exchange_type, exchange_manager=self, is_sandboxed=self.is_sandboxed) """ Spot exchange """ async def _search_and_create_spot_exchange(self): spot_exchange_class = get_spot_exchange_class(self.exchange_type) if spot_exchange_class: self.exchange = spot_exchange_class( config=self.config, exchange_type=self.exchange_type, exchange_manager=self, is_sandboxed=self.is_sandboxed) """ Margin exchange """ async def _search_and_create_margin_exchange(self): margin_exchange_class = get_margin_exchange_class(self.exchange_type) if margin_exchange_class: self.exchange = margin_exchange_class( config=self.config, exchange_type=self.exchange_type, exchange_manager=self, is_sandboxed=self.is_sandboxed) """ Future exchange """ async def _search_and_create_future_exchange(self): future_exchange_class = get_future_exchange_class(self.exchange_type) if future_exchange_class: self.exchange = future_exchange_class( config=self.config, exchange_type=self.exchange_type, exchange_manager=self, is_sandboxed=self.is_sandboxed) """ Exchange channels """ async def _create_exchange_channels( self): # TODO filter creation --> not required if pause is managed for exchange_channel_class_type in [ ExchangeChannel, TimeFrameExchangeChannel ]: await create_all_subclasses_channel( exchange_channel_class_type, set_chan, is_synchronized=self.is_backtesting, exchange_manager=self) async def _create_exchange_producers(self): # Real data producers if not self.is_backtesting: for updater in UNAUTHENTICATED_UPDATER_PRODUCERS: if not self._is_managed_by_websocket(updater.CHANNEL_NAME): await updater(get_chan(updater.CHANNEL_NAME, self.id)).run() if self.exchange.is_authenticated \ and self.trader \ and not (self.is_simulated or self.is_backtesting or self.is_collecting): for updater in AUTHENTICATED_UPDATER_PRODUCERS: if not self._is_managed_by_websocket(updater.CHANNEL_NAME): await updater(get_chan(updater.CHANNEL_NAME, self.id)).run() # Simulated producers if (not self.exchange.is_authenticated or self.is_simulated or self.is_backtesting) \ and self.trader \ and not self.is_collecting: for updater in AUTHENTICATED_UPDATER_SIMULATOR_PRODUCERS: await updater(get_chan(updater.CHANNEL_NAME, self.id)).run() """ Websocket """ def _is_managed_by_websocket(self, channel): # TODO improve checker return not self.rest_only and self.has_websocket and \ any([self.exchange_web_socket.is_feed_available(feed) for feed in WEBSOCKET_FEEDS_TO_TRADING_CHANNELS[channel]]) async def _search_and_create_websocket(self): socket_manager = search_websocket_class(AbstractWebsocket, self.exchange_name) if socket_manager is not None: await self._create_websocket(AbstractWebsocket.__name__, socket_manager) async def _create_websocket(self, websocket_class_name, socket_manager): try: self.exchange_web_socket = socket_manager.get_websocket_client( self.config, self) await self._init_websocket() self._logger.info( f"{socket_manager.get_name()} connected to {self.exchange.name}" ) except Exception as e: self._logger.error( f"Fail to init websocket for {websocket_class_name} : {e}") self.exchange_web_socket = None self.has_websocket = False raise e async def _init_websocket(self): await self.exchange_web_socket.init_websocket( self.exchange_config.traded_time_frames, self.exchange_config.traded_symbol_pairs) await self.exchange_web_socket.start_sockets() self.has_websocket = self.exchange_web_socket.is_websocket_running """ Exchange Configuration """ def check_config(self, exchange_name): if CONFIG_EXCHANGE_KEY not in self.config[CONFIG_EXCHANGES][exchange_name] \ or CONFIG_EXCHANGE_SECRET not in self.config[CONFIG_EXCHANGES][exchange_name]: return False else: return True def enabled(self): # if we can get candlestick data if self.is_simulated or self.exchange.name in self.config[ CONFIG_EXCHANGES]: return True else: self._logger.warning( f"Exchange {self.exchange.name} is currently disabled") return False def get_exchange_symbol_id(self, symbol): return self.exchange.get_exchange_pair(symbol) def get_exchange_symbol(self, symbol): return self.exchange.get_pair_from_exchange(symbol) def get_exchange_quote_and_base(self, symbol): return self.exchange.get_split_pair_from_exchange(symbol) def get_rest_pairs_refresh_threshold( self) -> RestExchangePairsRefreshMaxThresholds: traded_pairs_count = len(self.exchange_config.traded_symbol_pairs) if traded_pairs_count < RestExchangePairsRefreshMaxThresholds.FAST.value: return RestExchangePairsRefreshMaxThresholds.FAST if traded_pairs_count < RestExchangePairsRefreshMaxThresholds.MEDIUM.value: return RestExchangePairsRefreshMaxThresholds.MEDIUM return RestExchangePairsRefreshMaxThresholds.SLOW def _load_config_symbols_and_time_frames(self): client = self.exchange.client if client: self.client_symbols = client.symbols self.client_time_frames = list(client.timeframes) if hasattr( client, "timeframes") else [] else: self._logger.error("Failed to load client from REST exchange") self._raise_exchange_load_error() def _initialize_simulator_time_frames(self): self.client_time_frames = self.exchange.get_available_time_frames() # SYMBOLS def symbol_exists(self, symbol): if self.client_symbols is None: self._logger.error( f"Failed to load available symbols from REST exchange, impossible to check if " f"{symbol} exists on {self.exchange.name}") return False return symbol in self.client_symbols # TIME FRAMES def time_frame_exists(self, time_frame): if not self.client_time_frames: return False return time_frame in self.client_time_frames def get_rate_limit(self): return self.exchange_type.rateLimit / 1000 @staticmethod def need_to_uniformize_timestamp(timestamp): return not is_valid_timestamp(timestamp) def get_uniformized_timestamp(self, timestamp): if ExchangeManager.need_to_uniformize_timestamp(timestamp): return self.exchange.get_uniform_timestamp(timestamp) return timestamp def uniformize_candles_if_necessary(self, candle_or_candles): if candle_or_candles: # TODO improve if isinstance(candle_or_candles[0], list): if self.need_to_uniformize_timestamp(candle_or_candles[0][ PriceIndexes.IND_PRICE_TIME.value]): self._uniformize_candles_timestamps(candle_or_candles) else: if self.need_to_uniformize_timestamp( candle_or_candles[PriceIndexes.IND_PRICE_TIME.value]): self._uniformize_candle_timestamps(candle_or_candles) return candle_or_candles def _uniformize_candles_timestamps(self, candles): for candle in candles: self._uniformize_candle_timestamps(candle) def _uniformize_candle_timestamps(self, candle): candle[PriceIndexes.IND_PRICE_TIME.value] = \ self.exchange.get_uniform_timestamp(candle[PriceIndexes.IND_PRICE_TIME.value]) # Exceptions def _raise_exchange_load_error(self): raise Exception(f"{self.exchange} - Failed to load exchange instances") def get_exchange_name(self): return self.exchange_type.__name__ def should_decrypt_token(self, logger): if has_invalid_default_config_value( self.config[CONFIG_EXCHANGES][self.get_exchange_name()].get( CONFIG_EXCHANGE_KEY, ''), self.config[CONFIG_EXCHANGES][self.get_exchange_name()].get( CONFIG_EXCHANGE_SECRET, '')): logger.warning( "Exchange configuration tokens are not set yet, to use OctoBot's real trader's features, " "please enter your api tokens in exchange configuration") return False return True def get_exchange_credentials(self, logger, exchange_name): if self.ignore_config or not self.should_decrypt_token( logger) or self.without_auth: return "", "", "" config_exchange = self.config[CONFIG_EXCHANGES][exchange_name] return (decrypt_element_if_possible(CONFIG_EXCHANGE_KEY, config_exchange, None), decrypt_element_if_possible(CONFIG_EXCHANGE_SECRET, config_exchange, None), decrypt_element_if_possible(CONFIG_EXCHANGE_PASSWORD, config_exchange, None)) @staticmethod def handle_token_error(error, logger): logger.error( f"Exchange configuration tokens are invalid : please check your configuration ! " f"({error.__class__.__name__})") def get_symbol_data(self, symbol): return self.exchange_symbols_data.get_exchange_symbol_data(symbol)
def reset_exchange_personal_data(self): self.exchange_personal_data = ExchangePersonalData(self)
def test_select_orders(self): test_inst = ExchangePersonalData() symbol_1 = "BTC/USDT" symbol_2 = "ETH/BTC" symbol_3 = "ETH/USDT" # without orders test_inst.orders = {} assert test_inst.get_all_orders(symbol_1, None, None) == [] assert test_inst.get_open_orders(symbol_2, None, None) == [] assert test_inst.get_closed_orders(symbol_3, None, None) == [] order_1 = self.create_fake_order(10, OrderStatus.CLOSED.value, symbol_1, 0) test_inst.upsert_order(10, order_1) assert test_inst.get_all_orders(symbol_1, None, None) == [order_1] assert test_inst.get_open_orders(symbol_1, None, None) == [] assert test_inst.get_closed_orders(symbol_1, None, None) == [order_1] order_2 = self.create_fake_order(100, OrderStatus.OPEN.value, symbol_3, 0) test_inst.upsert_order(100, order_2) assert test_inst.get_all_orders(symbol_3, None, None) == [order_2] assert test_inst.get_open_orders(symbol_3, None, None) == [order_2] assert test_inst.get_closed_orders(symbol_3, None, None) == [] assert test_inst.get_all_orders(symbol_2, None, None) == [] assert test_inst.get_open_orders(symbol_2, None, None) == [] assert test_inst.get_closed_orders(symbol_2, None, None) == [] assert test_inst.get_all_orders(None, None, None) == [order_1, order_2] assert test_inst.get_open_orders(None, None, None) == [order_2] assert test_inst.get_closed_orders(None, None, None) == [order_1] order_3 = self.create_fake_order(10, OrderStatus.OPEN.value, symbol_2, 0) test_inst.upsert_order(10, order_3) assert test_inst.get_all_orders(symbol_2, None, None) == [order_3] assert test_inst.get_open_orders(symbol_2, None, None) == [order_3] assert test_inst.get_closed_orders(symbol_2, None, None) == [] assert test_inst.get_all_orders(None, None, None) == [order_3, order_2] assert test_inst.get_open_orders(None, None, None) == [order_3, order_2] assert test_inst.get_closed_orders(None, None, None) == [] assert test_inst.get_all_orders(symbol_1, None, None) == [] assert test_inst.get_open_orders(symbol_1, None, None) == [] assert test_inst.get_closed_orders(symbol_1, None, None) == [] assert test_inst.get_all_orders(symbol_3, None, None) == [order_2] assert test_inst.get_open_orders(symbol_3, None, None) == [order_2] assert test_inst.get_closed_orders(symbol_3, None, None) == [] order_4 = self.create_fake_order(11, OrderStatus.OPEN.value, symbol_1, 100) order_5 = self.create_fake_order(12, OrderStatus.CLOSED.value, symbol_2, 1000) order_6 = self.create_fake_order(13, OrderStatus.CLOSED.value, symbol_3, 50) test_inst.upsert_orders([order_4, order_5, order_6]) assert test_inst.get_all_orders(symbol_1, None, None) == [order_4] assert test_inst.get_open_orders(symbol_1, None, None) == [order_4] assert test_inst.get_closed_orders(symbol_1, None, None) == [] assert test_inst.get_all_orders(symbol_2, None, None) == [order_3, order_5] assert test_inst.get_open_orders(symbol_2, None, None) == [order_3] assert test_inst.get_closed_orders(symbol_2, None, None) == [order_5] assert test_inst.get_all_orders(symbol_3, None, None) == [order_2, order_6] assert test_inst.get_open_orders(symbol_3, None, None) == [order_2] assert test_inst.get_closed_orders(symbol_3, None, None) == [order_6] assert test_inst.get_all_orders( None, None, None) == [order_3, order_2, order_4, order_5, order_6] assert test_inst.get_open_orders(None, None, None) == [order_3, order_2, order_4] assert test_inst.get_closed_orders(None, None, None) == [order_5, order_6] # test limit assert test_inst.get_all_orders(None, None, 3) == [order_3, order_2, order_4] assert test_inst.get_open_orders(None, None, 3) == [order_3, order_2, order_4] assert test_inst.get_closed_orders(None, None, 3) == [order_5, order_6] # test timestamps assert test_inst.get_all_orders(None, 30, 3) == [order_3, order_2] assert test_inst.get_open_orders(None, 1000, None) == [order_3, order_2, order_4] assert test_inst.get_closed_orders(None, 60, 1) == [order_6]