def __init__(self, options):
super().__init__("watcher", options)
self._watchers = {}
self._identity = options.get('identity', 'demo')
self._backtesting = options.get('backtesting', False)
# fetchers config
self._fetchers_config = utils.load_config(options, 'fetchers')
# user identities
self._identities_config = utils.identities(options.get('config-path'))
self._profile = options.get('profile', 'default')
self._profile_config = utils.load_config(options, "profiles/%s" % self._profile)
# watchers config
self._watchers_config = self._init_watchers_config(options)
# backtesting options
self._backtesting = options.get('backtesting', False)
self._start_timestamp = options['from'].timestamp() if options.get('from') else 0
self._end_timestamp = options['to'].timestamp() if options.get('to') else 0
# read-only, means to do not write to the market DB
self._read_only = options.get('read-only', False)
def __init__(self, watcher_service, trader_service, monitor_service, options):
super().__init__("strategy", options)
self._strategies = {}
self._indicators = {}
self._appliances = {}
self._tradeops = {}
self._regions = {}
self._watcher_service = watcher_service
self._trader_service = trader_service
self._monitor_service = monitor_service
self._identity = options.get('identity', 'demo')
self._report_path = options.get('reports-path', './')
self._watcher_only = options.get('watcher-only', False)
self._profile = options.get('profile', 'default')
self._indicators_config = utils.load_config(options, 'indicators')
self._tradeops_config = utils.load_config(options, 'tradeops')
self._regions_config = utils.load_config(options, 'regions')
self._strategies_config = utils.load_config(options, 'strategies')
self._profile_config = utils.load_config(options, "profiles/%s" % self._profile)
# backtesting options
self._backtesting = options.get('backtesting', False)
self._from_date = options.get('from') # UTC tz
self._to_date = options.get('to') # UTC tz
self._timestep = options.get('timestep', 60.0)
self._timestamp = 0 # in backtesting current processed timestamp
# cannot be more recent than now
from common.utils import UTC
today = datetime.now().astimezone(UTC())
if self._from_date and self._from_date > today:
self._from_date = today
if self._to_date and self._to_date > today:
self._to_date = today
self._backtest = False
self._start_ts = self._from_date.timestamp() if self._from_date else 0
self._end_ts = self._to_date.timestamp() if self._to_date else 0
self._timestep_thread = None
self._time_factor = 0.0
if self._backtesting:
# can use the time factor in backtesting only
self._time_factor = options.get('time-factor', 0.0)
# paper mode options
self._paper_mode = options.get('paper-mode', False)
self._next_key = 1
# worker pool of jobs for running data analysis
self._worker_pool = WorkerPool()
def _init_traders_config(self, options):
"""
Get the profile configuration for a specific trader name.
"""
traders_profile = self._profile_config.get('traders', {})
# @todo could rebuild the list of symbols according to what is found in appliances
traders_config = {}
for k, profile_trader_config in traders_profile.items():
user_trader_config = utils.load_config(options, 'traders/' + k)
if user_trader_config:
if 'symbols' not in user_trader_config:
# at least an empty list of symbols
user_trader_config['symbols'] = []
if 'symbols' in profile_trader_config:
# profile overrides any symbols
user_trader_config['symbols'] = profile_trader_config['symbols']
if 'paper-mode' in profile_trader_config:
# paper-mode from profile overrides
user_trader_config['paper-mode'] = profile_trader_config['paper-mode']
# keep overrided
traders_config[k] = user_trader_config
return traders_config
def __init__(self, options):
super().__init__("notifier", options)
self._notifiers = {}
self._notifiers_insts = {}
self._profile = options.get('profile', 'default')
self._profile_config = utils.load_config(options,
"profiles/%s" % self._profile)
if 'notifiers' not in self._profile_config:
self._profile_config['notifiers'] = {}
if "desktop" not in self._profile_config['notifiers']:
# always default add a desktop notifier, but could be disable opt-in in the profile
self._profile_config['notifiers']["desktop"] = {
'status': "enabled",
'name': "desktop"
}
# notifiers config
self._notifiers_config = self._init_notifier_config(options)
# use to post formatted or raw tables of active and historical trades
self._strategy_service = None
self._trader_service = None
def create(cls, options):
config = utils.load_config(options, 'databases')
if config['siis'].get('type', 'mysql') == 'mysql':
from .mysql import MySql
Database.__instance = MySql()
elif config['siis'].get('type', 'pgsql') == 'pgsql':
from .pgsql import PgSql
Database.__instance = PgSql()
else:
raise ValueError("Unknown DB type")
def setup(self, options):
# load database
config = utils.load_config(options, 'databases')
self.connect(config)
# optionnal tables creation
self.setup_market_sql()
self.setup_userdata_sql()
self.setup_ohlc_sql()
# keep data path for usage in per market DB location
self._markets_path = pathlib.Path(options['markets-path'])
# start the thread
self._running = True
self._thread.start()
def create_watcher(self, options, watcher_name, markets):
watcher_config = utils.load_config(options, 'watchers/' + watcher_name)
if not watcher_config:
logger.error("Watcher %s not found !" % watcher_name)
return None
if markets:
watcher_config['symbols'] = markets
# force watcher config
self._watchers_config[watcher_name] = watcher_config
# retrieve the classname and instanciate it
parts = watcher_config.get('classpath').split('.')
module = import_module('.'.join(parts[:-1]))
Clazz = getattr(module, parts[-1])
return Clazz(self)
def create_fetcher(self, options, watcher_name):
fetcher = self._fetchers_config.get(watcher_name)
if not fetcher:
logger.error("Fetcher %s not found !" % watcher_name)
return None
# additional configuration and user specifications
specific_config = utils.load_config(options, 'fetchers/' + watcher_name)
fetcher = merge_parameters(fetcher, specific_config)
# force fetcher config
self._fetchers_config[watcher_name] = fetcher
# retrieve the classname and instanciate it
parts = fetcher.get('classpath').split('.')
module = import_module('.'.join(parts[:-1]))
Clazz = getattr(module, parts[-1])
return Clazz(self)
def _init_traders_config(self, options):
"""
Get the profile configuration for a specific trader name.
"""
traders_profile = self._profile_config.get('traders', {})
traders_config = {}
for k, profile_trader_config in traders_profile.items():
user_trader_config = utils.load_config(options, 'traders/' + k)
if user_trader_config:
if 'paper-mode' in profile_trader_config:
# paper-mode from profile overrides
user_trader_config['paper-mode'] = profile_trader_config[
'paper-mode']
# keep overrided
traders_config[k] = user_trader_config
return traders_config
def _init_notifier_config(self, options):
"""
Get the profile configuration for a specific notifier name.
"""
notifiers_profile = self._profile_config.get('notifiers', {})
notifier_config = {}
for k, profile_notifer_config in notifiers_profile.items():
if not profile_notifer_config.get('name'):
error_logger.error(
"Invalid configuration for notifier %s. Ignored !" % k)
user_notifier_config = utils.load_config(
options, 'notifiers/' + profile_notifer_config.get('name'))
if user_notifier_config:
# keep overrided
notifier_config[k] = user_notifier_config
return notifier_config
def __init__(self, watcher_service, monitor_service, options):
super().__init__("trader", options)
self._traders = {}
self._keys_map_to_trader = {}
self._next_trader_key = 1
self._next_trader_uid = 1
self._keys_map_to_order = {}
self._next_key = 1
self._next_order_uid = 1
self._watcher_service = watcher_service
self._monitor_service = monitor_service
self._identity = options.get('identity', 'demo')
self._report_path = options.get('reports-path', './')
self._watcher_only = options.get('watcher-only', False)
# user identities
self._identities_config = utils.identities(options.get('config-path'))
self._profile = options.get('profile', 'default')
self._profile_config = utils.load_config(options,
"profiles/%s" % self._profile)
# traders config
self._traders_config = self._init_traders_config(options)
# backtesting options
self._backtesting = options.get('backtesting', False)
self._start_timestamp = options['from'].timestamp() if options.get(
'from') else 0
self._end_timestamp = options['to'].timestamp() if options.get(
'to') else 0
# enable/disable execution of order/position (play/pause)
self._activity = True
# paper mode options
self._paper_mode = options.get('paper-mode', False)
def __init__(self, options):
super().__init__("monitor", options)
# monitoring config
self._monitoring_config = utils.load_config(options, 'monitoring')
if options.get('monitored', True):
self._monitoring = True
else:
self._monitoring = False
self._content = collections.deque()
self._thread = None
self._running = False
self._strategy_service = None
self._trader_service = None
self._watcher_service = None
self._server = None
self._loop = None
# host, port, allowed host, order, deny... from config
self._mode = MonitorService.MODE_FIFO # MODE_HTTP_WEBSOCKET
self._host = self._monitoring_config.get('host', '127.0.0.1')
self._port = self._monitoring_config.get('port', '8080')
# @todo allowdeny...
# fifo
self._tmpdir = None
self._filename_read = None
self._filename = None
self._fifo = -1
self._fifo_read = None
def start(self, options):
# indicators
for k, indicators in self._indicators_config.items():
if indicators.get("status") is not None and indicators.get("status") == "load":
# retrieve the classname and instanciate it
parts = indicators.get('classpath').split('.')
module = import_module('.'.join(parts[:-1]))
Clazz = getattr(module, parts[-1])
if not Clazz:
raise Exception("Cannot load indicator %s" % k)
self._indicators[k] = Clazz
# tradeops
for k, tradeops in self._tradeops_config.items():
if tradeops.get("status") is not None and tradeops.get("status") == "load":
# retrieve the classname and instanciate it
parts = tradeops.get('classpath').split('.')
module = import_module('.'.join(parts[:-1]))
Clazz = getattr(module, parts[-1])
if not Clazz:
raise Exception("Cannot load tradeop %s" % k)
self._tradeops[k] = Clazz
# regions
for k, regions in self._regions_config.items():
if regions.get("status") is not None and regions.get("status") == "load":
# retrieve the classname and instanciate it
parts = regions.get('classpath').split('.')
module = import_module('.'.join(parts[:-1]))
Clazz = getattr(module, parts[-1])
if not Clazz:
raise Exception("Cannot load region %s" % k)
self._regions[k] = Clazz
# strategies
for k, strategy in self._strategies_config.items():
if k == "default":
continue
if strategy.get("status") is not None and strategy.get("status") == "load":
# retrieve the classname and instanciate it
parts = strategy.get('classpath').split('.')
module = import_module('.'.join(parts[:-1]))
Clazz = getattr(module, parts[-1])
if not Clazz:
# @todo subclass of...
raise Exception("Cannot load strategy %s" % k)
self._strategies[k] = Clazz
# no appliance in watcher only
if self._watcher_only:
return
# and finally appliances
for k in self._profile_config.get('appliances', []):
appl = utils.load_config(options, "appliances/%s" % k)
if k == "default":
continue
if self._appliances.get(k) is not None:
logger.error("Strategy appliance %s already started" % k)
continue
if appl.get("status") is not None and appl.get("status") == "enabled":
# retrieve the classname and instanciate it
strategy = appl.get('strategy')
# overrided strategy parameters
parameters = strategy.get('parameters', {})
if not strategy or not strategy.get('name'):
logger.error("Invalid strategy configuration for appliance %s !" % k)
Clazz = self._strategies.get(strategy['name'])
if not Clazz:
logger.error("Unknown strategy name %s for appliance %s !" % (strategy['name'], k))
appl_inst = Clazz(self, self.watcher_service, self.trader_service, appl, parameters)
appl_inst.set_identifier(k)
if appl_inst.start():
self._appliances[k] = appl_inst
# start the worker pool
self._worker_pool.start()
