class Manager(): """class for running asynchronous services """ # ------------------------------------------------------------------ def __init__(self): self.class_name = self.__class__.__name__ self.app_name = 'data_manager' settings = parse_args(app_name=self.app_name) # southern or northen CTA sites have different telescope configurations site_type = settings['site_type'] # the address for the site app_host = settings['app_host'] # local log file location log_file = settings['log_file'] # logging level log_level = settings['log_level'] # the port for the site app_port = settings['app_port'] # the redis port use for this site self.redis_port = settings['redis_port'] # define the prefix to all urls (must be non-empy string) app_prefix = settings['app_prefix'] # is this a simulation is_simulation = settings['is_simulation'] # development mode debug_opts = settings['debug_opts'] # do we flush redis on startup self.do_flush_redis = settings['do_flush_redis'] # instantiate the general settings class (must come first!) self.base_config = BaseConfig( site_type=site_type, redis_port=self.redis_port, app_port=app_port, app_prefix=app_prefix, app_host=app_host, log_level=log_level, websocket_route=None, allow_panel_sync=None, debug_opts=debug_opts, is_simulation=is_simulation, ) self.log = LogParser( base_config=self.base_config, title=__name__, log_level=log_level, log_file=log_file, ) self.redis = RedisManager( name=self.class_name, base_config=self.base_config, log=self.log ) return # ------------------------------------------------------------------ def cleanup_services(self, service_name, is_verb=False): """graceful exit of services """ # is_verb = True if is_verb: self.log.info([ ['c', ' - Manager.service_cleanup for '], ['y', service_name], ['b', ' ...'], ]) # service_manager = ServiceManager() # service_manager.unset_active_instance(parent=self, class_prefix=service_name) ServiceManager.unset_active_instance(parent=self, service_name=service_name) # if service_name == 'clock_sim_service': # pass return # ------------------------------------------------------------------ def run_service(self, service_name, interrupt_sig): """run services in individual processes """ # ------------------------------------------------------------------ # simaple locker test (should be moved to unit tests...) check_lock = False # check_lock = True if check_lock: lock_prefix = 'utils;lock;' + 'test' + ';' # dynamic lock names, based on the current properties lock_namespace = { 'loop': lambda: 'service_loop', } # initialise the lock manager locker = LockManager( log=self.log, redis=self.redis, base_config=self.base_config, lock_namespace=lock_namespace, lock_prefix=lock_prefix, is_passive=True, ) self.log.info([['o', ' -- trying to acquire locks for '], ['c', service_name]]) with locker.locks.acquire(names='loop', debug=True, can_exist=False): self.log.info([['y', ' -- now i am locked 0 ... '], ['c', service_name]]) sleep(0.3) with locker.locks.acquire(names='loop', debug=True, can_exist=True): self.log.info([['y', ' -- now i am locked 1 ... '], ['c', service_name]]) sleep(0.2) self.log.info([['o', ' -- released locks for '], ['c', service_name]]) # ------------------------------------------------------------------ # in case of backlog from a previous reload, call the cleanup for good measure self.cleanup_services(service_name) self.log.info([ ['g', ' - starting services for '], ['y', service_name], ['g', ' \t(pid: '], ['p', os.getpid()], ['g', ') ...'], ]) # set the list of telescopes for this particular site and attach it to base_config InstData(base_config=self.base_config) # set passive instance of the clock class, to add access functions base_config ClockSim( base_config=self.base_config, interrupt_sig=interrupt_sig, is_passive=True, ) # for debugging, override the global flag # self.do_flush_redis = True if service_name == 'redis_flush': if self.do_flush_redis: self.log.warn([['bb', ' --- flusing redis --- ']]) self.redis.flush() elif service_name == 'redis_services': # prefix for all lock names in redis lock_prefix = 'utils;lock;' + service_name + ';' # dynamic lock names, based on the current properties lock_namespace = { 'loop': lambda: 'service_loop', } LockManager( log=self.log, redis=self.redis, base_config=self.base_config, lock_namespace=lock_namespace, lock_prefix=lock_prefix, is_passive=False, interrupt_sig=interrupt_sig, service_name=service_name, ) # ------------------------------------------------------------------ # ------------------------------------------------------------------ elif service_name == 'time_of_night_service': # start the time_of_night clock (to be phased out....) utils.time_of_night( base_config=self.base_config, service_name=service_name, interrupt_sig=interrupt_sig, ) # ------------------------------------------------------------------ # ------------------------------------------------------------------ elif service_name == 'clock_sim_service': ClockSim( base_config=self.base_config, interrupt_sig=interrupt_sig, is_passive=False, ) elif service_name == 'inst_health_service': InstHealth( base_config=self.base_config, service_name=service_name, interrupt_sig=interrupt_sig, ) elif service_name == 'inst_pos_service': InstPos( base_config=self.base_config, service_name=service_name, interrupt_sig=interrupt_sig, ) elif service_name == 'scheduler_service': if has_acs: raise Exception( 'threading has not been properly updated for the acs version....' ) SchedulerACS( base_config=self.base_config, service_name=service_name, interrupt_sig=interrupt_sig, ) else: MockTarget(base_config=self.base_config) SchedulerStandalone( base_config=self.base_config, service_name=service_name, interrupt_sig=interrupt_sig, ) else: raise Exception('unknown service_name ?!?', service_name) # all service classes inherit from ServiceManager, which keeps track of # all thread. after initialising all classes, start the threads (blocking action) ServiceManager.run_threads() # after interrupt_sig has released the block from outside # of this process, do some cleanup self.cleanup_services(service_name) return
class ClockSim(ServiceManager): """clock simulation class, simulating the procession of a night Only a single active instance is allowed to exist. Multiple passive instances are allowed. A passive instance only serves as an interface for the clock via redis """ lock = Lock() # ------------------------------------------------------------------ def __init__( self, base_config, is_passive, interrupt_sig=None, *args, **kwargs, ): self.class_name = self.__class__.__name__ super().__init__(service_name=self.class_name) self.log = LogParser(base_config=base_config, title=__name__) self.base_config = base_config self.is_passive = is_passive if self.is_passive: self.base_config.clock_sim = self self.redis = RedisManager(name=self.class_name, base_config=base_config, log=self.log) self.interrupt_sig = interrupt_sig if self.interrupt_sig is None: self.interrupt_sig = multiprocessing.Event() if not self.is_passive: with ClockSim.lock: self.setup_active_instance() return # ------------------------------------------------------------------ def setup_active_instance(self): """setup the active instance of the class """ self.rnd_gen = Random(11) self.debug_datetime_now = False # sleep duration for thread loops self.loop_sleep_sec = 1 self.pubsub_sleep_sec = 0.1 # self.is_skip_daytime = False self.is_skip_daytime = True self.is_short_night = False # self.is_short_night = True # safety measure self.min_speed_factor = 1 self.max_speed_factor = 10 * 60 * self.loop_sleep_sec # speedup simulation e.g.,: # 60*10 --> every 1 real sec goes to 10 simulated min self.speed_factor = 30 # self.speed_factor = 10 self.datetime_epoch = self.base_config.datetime_epoch self.init_sim_params_from_redis = True # self.init_sim_params_from_redis = False self.sim_params = { 'speed_factor': self.speed_factor, 'min_speed_factor': self.min_speed_factor, 'max_speed_factor': self.max_speed_factor, 'is_skip_daytime': self.is_skip_daytime, 'is_short_night': self.is_short_night, } self.set_sim_speed( data_in={ 'speed_factor': self.speed_factor, 'is_skip_daytime': self.is_skip_daytime, 'is_short_night': self.is_short_night, }, from_redis=self.init_sim_params_from_redis, ) # make sure this is the only active instance self.init_active_instance() self.init_night_times() self.setup_threads() return # ------------------------------------------------------------------ def setup_threads(self): """register threads to be run after this and all other services have been initialised """ self.add_thread(target=self.loop_main) self.add_thread(target=self.pubsub_sim_params) return # ------------------------------------------------------------------ def check_passive(self): """check if this is an active or passive instance if this is a passive instance, make sure that an active instance has been initialised by some other proccess. after n_sec_try of waiting, raise an exception Returns ------- bool is this a passive instance """ need_check = (self.can_loop() and self.is_passive and not self.has_active_instance()) # print('xxxxxxxx', self.can_loop() , self.is_passive , self.has_active_instance(),'---',need_check) if not need_check: return self.is_passive n_sec_sleep, n_sec_try = 0.01, 10 n_loops = 1 + int(n_sec_try / n_sec_sleep) # check that the active instance has finished the initialisation stage for n_loop in range(n_loops + 1): sleep(n_sec_sleep) active_state = self.has_active_instance() or (not self.can_loop()) if active_state: break if n_loop >= n_loops: raise Exception( ' - ClockSim active instance can not initialise ?!?!') if n_loop > 0 and (n_loop % int(1 / n_sec_sleep) == 0): self.log.warn([ [ 'r', ' - ClockSim blocking ( service_name = ', self.class_name ], [ 'r', ' ) --> waiting for the active instance to init ...' ], ]) return self.is_passive # ------------------------------------------------------------------ def get_time_now_sec(self): datetime_now = self.get_datetime_now() time_now_sec = int(datetime_to_secs(datetime_now)) return time_now_sec # ------------------------------------------------------------------ def get_is_night_now(self): if self.check_passive(): return self.redis.get('clock_sim_is_night_now') return self.is_night_now # ------------------------------------------------------------------ def get_n_nights(self): if self.check_passive(): return self.redis.get('clock_sim_n_nights') return self.n_nights # ------------------------------------------------------------------ def get_night_start_sec(self): if self.check_passive(): return self.redis.get('clock_sim_night_start_sec') return self.night_start_sec # ------------------------------------------------------------------ def get_night_end_sec(self): if self.check_passive(): return self.redis.get('clock_sim_night_end_sec') return self.night_end_sec # ------------------------------------------------------------------ def get_time_series_start_time_sec(self): if self.check_passive(): start_time_sec = self.redis.get( 'clock_sim_time_series_start_time_sec') else: start_time_sec = self.time_series_start_time_sec return int(start_time_sec) # ------------------------------------------------------------------ def get_datetime_now(self): if self.check_passive(): time_now_sec = self.redis.get('clock_sim_time_now_sec') return secs_to_datetime(time_now_sec) return self.datetime_now # ------------------------------------------------------------------ def is_night_time_now(self): time_now_sec = self.get_time_now_sec() is_night = (time_now_sec > self.get_night_start_sec() and time_now_sec <= self.get_night_end_sec()) return is_night # ------------------------------------------------------------------ def get_night_duration_sec(self): return (self.get_night_end_sec() - self.get_night_start_sec()) # ------------------------------------------------------------------ def get_astro_night_start_sec(self): # beginig of the astronomical night return int(self.get_night_start_sec()) # ------------------------------------------------------------------ def get_sim_params(self): if self.check_passive(): sim_params = self.redis.get(name='clock_sim_sim_params') else: sim_params = self.sim_params return sim_params # ------------------------------------------------------------------ def get_speed_factor(self): sim_params = self.get_sim_params() return sim_params['speed_factor'] # ------------------------------------------------------------------ def get_sec_since_midnight(self): days_since_epoch = (self.datetime_now - self.datetime_epoch).days sec_since_midnight = ( (self.datetime_now - self.datetime_epoch).seconds + timedelta(days=days_since_epoch).total_seconds()) return sec_since_midnight # ------------------------------------------------------------------ def init_night_times(self): """reset the night """ self.n_nights = 0 self.datetime_now = None self.night_start_sec = datetime_to_secs(self.datetime_epoch) self.night_end_sec = datetime_to_secs(self.datetime_epoch) self.time_series_start_time_sec = self.night_start_sec self.set_night_times() self.update_once() return # ------------------------------------------------------------------ def update_once(self): """single update, to be run as part of a loop """ self.datetime_now += timedelta(seconds=self.loop_sleep_sec * self.speed_factor) if self.debug_datetime_now: self.log.info([ ['g', ' --- Now (night:', self.n_nights, '/', ''], ['p', self.is_night_time_now()], ['g', ') '], ['y', self.datetime_now], [ 'c', ' (' + str(datetime_to_secs(self.datetime_now)) + ' sec)' ], ]) self.update_n_night() time_now_sec = datetime_to_secs(self.datetime_now) is_night_now = self.is_night_time_now() self.redis.set( name='clock_sim_time_now_sec', data=time_now_sec, ) self.redis.set( name='clock_sim_is_night_now', data=is_night_now, ) self.redis.set( name='clock_sim_n_nights', data=self.n_nights, ) self.redis.set( name='clock_sim_night_start_sec', data=self.night_start_sec, ) self.redis.set( name='clock_sim_night_end_sec', data=self.night_end_sec, ) self.redis.set( name='clock_sim_time_series_start_time_sec', data=self.time_series_start_time_sec, ) return # ------------------------------------------------------------------ def set_night_times(self): """reset the night """ night_start_hours = self.rnd_gen.randint(18, 19) night_start_minutes = self.rnd_gen.randint(0, 59) night_end_hours = self.rnd_gen.randint(4, 5) night_end_minutes = self.rnd_gen.randint(0, 59) # short night for debugging if self.is_short_night: night_start_hours = 23 night_start_minutes = 0 night_end_hours = 2 night_end_minutes = 0 if self.datetime_now is None: self.datetime_now = self.datetime_epoch.replace( hour=(night_start_hours - 1), ) self.time_series_start_time_sec = self.night_start_sec n_days = (self.datetime_now - self.datetime_epoch).days self.night_start_sec = timedelta( days=n_days, hours=night_start_hours, minutes=night_start_minutes, seconds=0, ).total_seconds() # e.g., night ends at 06:40 self.night_end_sec = timedelta( days=(n_days + 1), hours=night_end_hours, minutes=night_end_minutes, seconds=0, ).total_seconds() if self.is_skip_daytime or self.is_short_night: self.datetime_now = (secs_to_datetime(self.night_start_sec) - timedelta(seconds=10)) night_start = date_to_string( secs_to_datetime(self.night_start_sec), date_string=None, ) night_end = date_to_string( secs_to_datetime(self.night_end_sec), date_string=None, ) self.log.info([ ['b', ' - setting new night: ['], ['g', night_start], ['b', ' --> '], ['g', night_end], ['b', ']'], ]) return # ------------------------------------------------------------------ def update_n_night(self): sec_since_midnight = self.get_sec_since_midnight() days_since_epoch = (self.datetime_now - self.datetime_epoch).days is_new_day = days_since_epoch > self.n_nights is_past_night_time = sec_since_midnight > self.night_end_sec if is_new_day and is_past_night_time: self.n_nights = days_since_epoch self.set_night_times() return # ------------------------------------------------------------------ def need_data_update(self, update_opts): """check if a service needs to run an update, where updates only happen after min_wait of simulation time """ time_now = self.get_time_now_sec() set_prev_update = (('prev_update' not in update_opts.keys()) or (update_opts['prev_update'] is None)) if set_prev_update: update_opts['prev_update'] = time_now - 2 * update_opts['min_wait'] time_diff = time_now - update_opts['prev_update'] can_update = (time_diff > update_opts['min_wait']) # updates only happen during the astronimical night is_night_time = self.is_night_time_now() need_update = (is_night_time and can_update) if need_update: update_opts['prev_update'] = time_now return need_update # ------------------------------------------------------------------ def set_sim_speed(self, data_in, from_redis=False): """set parameters which determine the lenght of the night, the real-time duration, given a speed factor, the delay between nights, etc. """ speed_factor = data_in['speed_factor'] is_skip_daytime = data_in['is_skip_daytime'] is_short_night = data_in['is_short_night'] if from_redis: red_data = self.redis.get(name='clock_sim_sim_params') if red_data is not None: speed_factor = red_data['speed_factor'] is_skip_daytime = red_data['is_skip_daytime'] is_short_night = red_data['is_short_night'] if speed_factor is not None: speed_factor = float(speed_factor) is_ok = (speed_factor >= self.min_speed_factor and speed_factor <= self.max_speed_factor) if not is_ok: raise ValueError( 'trying to set speed_factor out of bounds ...', speed_factor) self.speed_factor = float(speed_factor) if is_skip_daytime is not None: self.is_skip_daytime = is_skip_daytime if is_short_night is not None: self.is_short_night = is_short_night self.log.info([ ['b', ' - updating clock_sim_sim_params: '], ['c', ' speed_factor: '], ['p', self.speed_factor], ['c', ' , is_skip_daytime: '], ['p', self.is_skip_daytime], ['c', ' , is_short_night: '], ['p', self.is_short_night], ]) self.sim_params = { 'speed_factor': self.speed_factor, 'min_speed_factor': self.min_speed_factor, 'max_speed_factor': self.max_speed_factor, 'is_skip_daytime': self.is_skip_daytime, 'is_short_night': self.is_short_night, } self.redis.set( name='clock_sim_sim_params', data=self.sim_params, ) self.redis.publish(channel='clock_sim_updated_sim_params') return # ------------------------------------------------------------------ def loop_main(self): """main loop running in its own thread, updating the night """ self.log.info([['g', ' - starting ClockSim.loop_main ...']]) while self.can_loop(): sleep(self.loop_sleep_sec) with ClockSim.lock: self.update_once() self.log.info([['c', ' - ending ClockSim.loop_main ...']]) return # ------------------------------------------------------------------ def pubsub_sim_params(self): """loop running in its own thread, reacting to pubsub events """ self.log.info([['g', ' - starting ClockSim.pubsub_sim_params ...']]) # setup the channel once pubsub_tag = 'clock_sim_set_sim_params' pubsub = self.redis.pubsub_subscribe(pubsub_tag) # listen to changes on the channel and do stuff while self.can_loop(): sleep(self.pubsub_sleep_sec) msg = self.redis.pubsub_get_message(pubsub=pubsub) if msg is None: continue with ClockSim.lock: keys = ['speed_factor', 'is_skip_daytime', 'is_short_night'] data_out = dict() for key in keys: data_out[ key] = msg['data'][key] if key in msg['data'] else None self.set_sim_speed(data_in=data_out) self.log.info([['c', ' - ending ClockSim.pubsub_sim_params ...']]) return