def __init__(self, watcher_service, trader_service, monitor_service, options): super().__init__("strategy", options) self._strategies = {} self._indicators = {} self._appliances = {} self._tradeops = {} self._regions = {} self._watcher_service = watcher_service self._trader_service = trader_service self._monitor_service = monitor_service self._identity = options.get('identity', 'demo') self._report_path = options.get('reports-path', './') self._watcher_only = options.get('watcher-only', False) self._profile = options.get('profile', 'default') self._indicators_config = utils.load_config(options, 'indicators') self._tradeops_config = utils.load_config(options, 'tradeops') self._regions_config = utils.load_config(options, 'regions') self._strategies_config = utils.load_config(options, 'strategies') self._profile_config = utils.load_config(options, "profiles/%s" % self._profile) # backtesting options self._backtesting = options.get('backtesting', False) self._from_date = options.get('from') # UTC tz self._to_date = options.get('to') # UTC tz self._timestep = options.get('timestep', 60.0) self._timestamp = 0 # in backtesting current processed timestamp # cannot be more recent than now from common.utils import UTC today = datetime.now().astimezone(UTC()) if self._from_date and self._from_date > today: self._from_date = today if self._to_date and self._to_date > today: self._to_date = today self._backtest = False self._start_ts = self._from_date.timestamp() if self._from_date else 0 self._end_ts = self._to_date.timestamp() if self._to_date else 0 self._timestep_thread = None self._time_factor = 0.0 if self._backtesting: # can use the time factor in backtesting only self._time_factor = options.get('time-factor', 0.0) # paper mode options self._paper_mode = options.get('paper-mode', False) self._next_key = 1 # worker pool of jobs for running data analysis self._worker_pool = WorkerPool()
class StrategyService(Service): def __init__(self, watcher_service, trader_service, monitor_service, options): super().__init__("strategy", options) self._strategies = {} self._indicators = {} self._appliances = {} self._tradeops = {} self._regions = {} self._watcher_service = watcher_service self._trader_service = trader_service self._monitor_service = monitor_service self._identity = options.get('identity', 'demo') self._report_path = options.get('reports-path', './') self._watcher_only = options.get('watcher-only', False) self._profile = options.get('profile', 'default') self._indicators_config = config.INDICATORS or {} self._tradeops_config = config.TRADEOPS or {} self._regions_config = config.REGIONS or {} self._strategies_config = config.STRATEGIES or {} self._appliances_config = utils.appliances( options.get('config-path')) or {} self._profile_config = utils.profiles(options.get('config-path')) or {} # backtesting options self._backtesting = options.get('backtesting', False) self._from_date = options.get('from') # UTC tz self._to_date = options.get('to') # UTC tz self._timestep = options.get('timestep', 60.0) self._timestamp = 0 # in backtesting current processed timestamp # cannot be more recent than now from common.utils import UTC today = datetime.now().astimezone(UTC()) if self._from_date and self._from_date > today: self._from_date = today if self._to_date and self._to_date > today: self._to_date = today self._backtest = False self._start_ts = self._from_date.timestamp() if self._from_date else 0 self._end_ts = self._to_date.timestamp() if self._to_date else 0 self._timestep_thread = None self._time_factor = 0.0 if self._backtesting: # can use the time factor in backtesting only self._time_factor = options.get('time-factor', 0.0) # paper mode options self._paper_mode = options.get('paper-mode', False) self._next_key = 1 # worker pool of jobs for running data analysis self._worker_pool = WorkerPool() @property def watcher_service(self): return self._watcher_service @property def trader_service(self): return self._trader_service @property def monitor_service(self): return self._monitor_service @property def worker_pool(self): return self._worker_pool @property def tradeops(self): return self._tradeops @property def regions(self): return self._regions def set_activity(self, status): """ Enable/disable execution of orders for all appliances. """ for k, appliance in self._appliances.items(): appliance.set_activity(status) def start(self): # indicators for k, indicators in self._indicators_config.items(): if indicators.get("status") is not None and indicators.get( "status") == "load": # retrieve the classname and instanciate it parts = indicators.get('classpath').split('.') module = import_module('.'.join(parts[:-1])) Clazz = getattr(module, parts[-1]) if not Clazz: raise Exception("Cannot load indicator %s" % k) self._indicators[k] = Clazz # tradeops for k, tradeops in self._tradeops_config.items(): if tradeops.get("status") is not None and tradeops.get( "status") == "load": # retrieve the classname and instanciate it parts = tradeops.get('classpath').split('.') module = import_module('.'.join(parts[:-1])) Clazz = getattr(module, parts[-1]) if not Clazz: raise Exception("Cannot load tradeop %s" % k) self._tradeops[k] = Clazz # regions for k, regions in self._regions_config.items(): if regions.get("status") is not None and regions.get( "status") == "load": # retrieve the classname and instanciate it parts = regions.get('classpath').split('.') module = import_module('.'.join(parts[:-1])) Clazz = getattr(module, parts[-1]) if not Clazz: raise Exception("Cannot load region %s" % k) self._regions[k] = Clazz # strategies for k, strategy in self._strategies_config.items(): if k == "default": continue if strategy.get("status") is not None and strategy.get( "status") == "load": # retrieve the classname and instanciate it parts = strategy.get('classpath').split('.') module = import_module('.'.join(parts[:-1])) Clazz = getattr(module, parts[-1]) if not Clazz: # @todo subclass of... raise Exception("Cannot load strategy %s" % k) self._strategies[k] = Clazz # no appliance in watcher only if self._watcher_only: return # and finally appliances for k, appl in self._appliances_config.items(): if k == "default": continue if not self.profile_has_appliance(k): # ignore if not in the current profile continue if self._appliances.get(k) is not None: logger.error("Strategy appliance %s already started" % k) continue if appl.get("status") is not None and appl.get( "status") == "enabled": # retrieve the classname and instanciate it strategy = appl.get('strategy') # overrided strategy parameters parameters = strategy.get('parameters', {}) if not strategy or not strategy.get('name'): logger.error( "Invalid strategy configuration for appliance %s !" % k) Clazz = self._strategies.get(strategy['name']) if not Clazz: logger.error("Unknown strategy name for appliance %s !" % k) appl_inst = Clazz(self, self.watcher_service, self.trader_service, appl, parameters) appl_inst.set_identifier(k) if appl_inst.start(): self._appliances[k] = appl_inst # start the worker pool self._worker_pool.start() def terminate(self): if self._timestep_thread and self._timestep_thread.is_alive(): # abort backtesting self._timestep_thread.abort = True self._timestep_thread.join() self._timestep_thread = None for k, appl in self._appliances.items(): if not appl: continue # stop all workers if appl.running: appl.stop() for k, appl in self._appliances.items(): if not appl: continue # join them if appl.thread.is_alive(): appl.thread.join() # and save state to database if not self.backtesting and (appl.trader() and not appl.trader().paper_mode): appl.save() # terminate the worker pool self._worker_pool.stop() self._appliances = {} self._strategies = {} self._indicators = {} def sync(self): # start backtesting if self._backtesting and not self._backtest: go_ready = True self._mutex.acquire() self._backtest_progress = 0 for k, appl, in self._appliances.items(): self._mutex.release() if not appl.running or not appl.ready(): go_ready = False self._mutex.acquire() break else: self._mutex.acquire() self._mutex.release() if go_ready: # start the time thread once all appliance get theirs data and are ready class TimeStepThread(threading.Thread): def __init__(self, service, s, e, ts, tf=0.0): super().__init__(name="backtest") self.service = service self.abort = False self.s = s self.e = e self.c = s self.ts = ts self.ppc = 0 self.tf = tf def run(self): prev = self.c min_limit = 0.0001 limit = min_limit # starts with min limit last_saturation = 0 last_sleep = time.time() Terminal.inst().info("Backtesting started...", view='status') appliances = self.service._appliances.values() traders = [] wait = False appl = None # get the list of used traders, to sync them after each pass for appl in appliances: if appl.trader() and appl.trader() not in traders: traders.append(appl.trader()) if len(appliances) == 1: # a signe appliance, don't need to parellelize, and to sync, python sync suxx a lot, avoid the overload in most of the # backtesting usage while self.c < self.e + self.ts: # now sync the trader base time for trader in traders: trader.set_timestamp(self.c) appl.backtest_update(self.c, self.e) if self.tf > 0: # wait factor of time step, so 1 mean realtime simulation, 0 mean as fast as possible time.sleep((1 / self.tf) * self.ts) self.c += self.ts # add one time step self.service._timestamp = self.c # one more step then we can update traders (limits orders, P/L update...) for trader in traders: trader.update() time.sleep(0) # yield if self.abort: break else: # multiple appliances, parralelise them while self.c < self.e + self.ts: if not wait: # now sync the trader base time for trader in traders: # @todo it could be better if we add two step, one update the market and then the strategy computation # to avoid to update multiple time the same market and potentially with different ut... but not really an issue trader.set_timestamp(self.c) # query async update per appliance for appl in appliances: appl.query_backtest_update( self.c, self.e) # @todo could use a semaphore or condition counter wait = False for appl in appliances: # appl.backtest_update(self.c, self.e) # wait all appliance did theirs jobs if appl._last_done_ts < self.c: wait = True break if not wait: if self.tf > 0: # wait factor of time step, so 1 mean realtime simulation, 0 mean as fast as possible time.sleep((1 / self.tf) * self.ts) self.c += self.ts # add one time step self.service._timestamp = self.c # one more step then we can update traders (limits orders, P/L update...) for trader in traders: trader.update() time.sleep(0) # yield if self.abort: break self._timestep_thread = TimeStepThread(self, self._start_ts, self._end_ts, self._timestep, self._time_factor) self._timestep_thread.setDaemon(True) self._timestep_thread.start() # backtesting started, avoid re-enter self._backtest = True if self._backtesting and self._backtest and self._backtest_progress < 100.0: progress = 0 self._mutex.acquire() for k, appl, in self._appliances.items(): if appl.running: progress += appl.progress() if self._appliances: progress /= float(len(self._appliances)) self._mutex.release() total = self._end_ts - self._start_ts remaining = self._end_ts - progress pc = 100.0 - (remaining / (total + 0.001) * 100) if pc - self._backtest_progress >= 1.0 and pc < 100.0: self._backtest_progress = pc Terminal.inst().info("Backtesting %s%%..." % round(pc), view='status') if self._end_ts - progress <= 0.0: # finished ! self._backtest_progress = 100.0 # backtesting done => waiting user Terminal.inst().info("Backtesting 100% finished !", view='status') def notify(self, signal_type, source_name, signal_data): if signal_data is None: return signal = Signal(Signal.SOURCE_STRATEGY, source_name, signal_type, signal_data) self._mutex.acquire() self._notifier.notify(signal) self._mutex.release() def command(self, command_type, data): if Strategy.COMMAND_SHOW_STATS <= command_type <= Strategy.COMMAND_INFO: # any or specific commands appliance_identifier = data.get('appliance') if appliance_identifier: # for a specific appliance appliance = self._appliances.get(appliance_identifier) if appliance: appliance.command(command_type, data) else: # or any for k, appliance in self._appliances.items(): appliance.command(command_type, data) else: # specific commands appliance_identifier = data.get('appliance') appliance = None if appliance_identifier: appliance = self._appliances.get(appliance_identifier) if appliance: appliance.command(command_type, data) def __gen_command_key(self): self._mutex.acquire() next_key = self._next_key self._next_key += 1 self._mutex.release() return next_key def receiver(self, signal): self._mutex.acquire() now = time.time() self._mutex.release() def indicator(self, name): return self._indicators.get(name) def strategy(self, name): return self._strategies.get(name) def appliance(self, name): return self._appliances.get(name) def get_appliances(self): return list(self._appliances.values()) def appliances_identifiers(self): return [app.identifier for k, app in self._appliances.items()] @property def timestamp(self): return self._timestamp if self._backtesting else time.time() @property def backtesting(self): return self._backtesting @property def from_date(self): return self._from_date @property def to_date(self): return self._to_date def appliance_config(self, name): """ Get the configurations for an appliance as dict. """ return self._appliances_config.get(name, {}) def profile_has_appliance(self, name): """ Check if an appliance is allowed for a the current loaded profile. """ profile = self._profile_config.get(self._profile, {'appliances': []}) if 'appliances' not in profile: profile['appliances'] = [] for app_name in profile['appliances']: if app_name.startswith('!'): if app_name[1:] == name: # ignored return False if app_name == '*': # any except ignored return True if app_name == name: return True return False def strategy_config(self, name): """ Get the configurations for a strategy as dict. """ return self._strategies_config.get(name, {}) def indicator_config(self, name): """ Get the configurations for an indicator as dict. """ return self._indicators_config.get(name, {}) def tradeop_config(self, name): """ Get the configurations for a tradeop as dict. """ return self._tradeops_config.get(name, {}) def ping(self): self._mutex.acquire() for k, appl, in self._appliances.items(): appl.ping() self._worker_pool.ping() self._mutex.release()