class ClockSim(ServiceManager):
"""clock simulation class, simulating the procession of a night
Only a single active instance is allowed to exist. Multiple passive instances
are allowed. A passive instance only serves as an interface for the clock via redis
"""
lock = Lock()
# ------------------------------------------------------------------
def __init__(
self,
base_config,
is_passive,
interrupt_sig=None,
*args,
**kwargs,
):
self.class_name = self.__class__.__name__
super().__init__(service_name=self.class_name)
self.log = LogParser(base_config=base_config, title=__name__)
self.base_config = base_config
self.is_passive = is_passive
if self.is_passive:
self.base_config.clock_sim = self
self.redis = RedisManager(name=self.class_name,
base_config=base_config,
log=self.log)
self.interrupt_sig = interrupt_sig
if self.interrupt_sig is None:
self.interrupt_sig = multiprocessing.Event()
if not self.is_passive:
with ClockSim.lock:
self.setup_active_instance()
return
# ------------------------------------------------------------------
def setup_active_instance(self):
"""setup the active instance of the class
"""
self.rnd_gen = Random(11)
self.debug_datetime_now = False
# sleep duration for thread loops
self.loop_sleep_sec = 1
self.pubsub_sleep_sec = 0.1
# self.is_skip_daytime = False
self.is_skip_daytime = True
self.is_short_night = False
# self.is_short_night = True
# safety measure
self.min_speed_factor = 1
self.max_speed_factor = 10 * 60 * self.loop_sleep_sec
# speedup simulation e.g.,:
# 60*10 --> every 1 real sec goes to 10 simulated min
self.speed_factor = 30
# self.speed_factor = 10
self.datetime_epoch = self.base_config.datetime_epoch
self.init_sim_params_from_redis = True
# self.init_sim_params_from_redis = False
self.sim_params = {
'speed_factor': self.speed_factor,
'min_speed_factor': self.min_speed_factor,
'max_speed_factor': self.max_speed_factor,
'is_skip_daytime': self.is_skip_daytime,
'is_short_night': self.is_short_night,
}
self.set_sim_speed(
data_in={
'speed_factor': self.speed_factor,
'is_skip_daytime': self.is_skip_daytime,
'is_short_night': self.is_short_night,
},
from_redis=self.init_sim_params_from_redis,
)
# make sure this is the only active instance
self.init_active_instance()
self.init_night_times()
self.setup_threads()
return
# ------------------------------------------------------------------
def setup_threads(self):
"""register threads to be run after this and all other services have
been initialised
"""
self.add_thread(target=self.loop_main)
self.add_thread(target=self.pubsub_sim_params)
return
# ------------------------------------------------------------------
def check_passive(self):
"""check if this is an active or passive instance
if this is a passive instance, make sure that an active instance
has been initialised by some other proccess. after n_sec_try of
waiting, raise an exception
Returns
-------
bool
is this a passive instance
"""
need_check = (self.can_loop() and self.is_passive
and not self.has_active_instance())
# print('xxxxxxxx', self.can_loop() , self.is_passive , self.has_active_instance(),'---',need_check)
if not need_check:
return self.is_passive
n_sec_sleep, n_sec_try = 0.01, 10
n_loops = 1 + int(n_sec_try / n_sec_sleep)
# check that the active instance has finished the initialisation stage
for n_loop in range(n_loops + 1):
sleep(n_sec_sleep)
active_state = self.has_active_instance() or (not self.can_loop())
if active_state:
break
if n_loop >= n_loops:
raise Exception(
' - ClockSim active instance can not initialise ?!?!')
if n_loop > 0 and (n_loop % int(1 / n_sec_sleep) == 0):
self.log.warn([
[
'r', ' - ClockSim blocking ( service_name = ',
self.class_name
],
[
'r',
' ) --> waiting for the active instance to init ...'
],
])
return self.is_passive
# ------------------------------------------------------------------
def get_time_now_sec(self):
datetime_now = self.get_datetime_now()
time_now_sec = int(datetime_to_secs(datetime_now))
return time_now_sec
# ------------------------------------------------------------------
def get_is_night_now(self):
if self.check_passive():
return self.redis.get('clock_sim_is_night_now')
return self.is_night_now
# ------------------------------------------------------------------
def get_n_nights(self):
if self.check_passive():
return self.redis.get('clock_sim_n_nights')
return self.n_nights
# ------------------------------------------------------------------
def get_night_start_sec(self):
if self.check_passive():
return self.redis.get('clock_sim_night_start_sec')
return self.night_start_sec
# ------------------------------------------------------------------
def get_night_end_sec(self):
if self.check_passive():
return self.redis.get('clock_sim_night_end_sec')
return self.night_end_sec
# ------------------------------------------------------------------
def get_time_series_start_time_sec(self):
if self.check_passive():
start_time_sec = self.redis.get(
'clock_sim_time_series_start_time_sec')
else:
start_time_sec = self.time_series_start_time_sec
return int(start_time_sec)
# ------------------------------------------------------------------
def get_datetime_now(self):
if self.check_passive():
time_now_sec = self.redis.get('clock_sim_time_now_sec')
return secs_to_datetime(time_now_sec)
return self.datetime_now
# ------------------------------------------------------------------
def is_night_time_now(self):
time_now_sec = self.get_time_now_sec()
is_night = (time_now_sec > self.get_night_start_sec()
and time_now_sec <= self.get_night_end_sec())
return is_night
# ------------------------------------------------------------------
def get_night_duration_sec(self):
return (self.get_night_end_sec() - self.get_night_start_sec())
# ------------------------------------------------------------------
def get_astro_night_start_sec(self):
# beginig of the astronomical night
return int(self.get_night_start_sec())
# ------------------------------------------------------------------
def get_sim_params(self):
if self.check_passive():
sim_params = self.redis.get(name='clock_sim_sim_params')
else:
sim_params = self.sim_params
return sim_params
# ------------------------------------------------------------------
def get_speed_factor(self):
sim_params = self.get_sim_params()
return sim_params['speed_factor']
# ------------------------------------------------------------------
def get_sec_since_midnight(self):
days_since_epoch = (self.datetime_now - self.datetime_epoch).days
sec_since_midnight = (
(self.datetime_now - self.datetime_epoch).seconds +
timedelta(days=days_since_epoch).total_seconds())
return sec_since_midnight
# ------------------------------------------------------------------
def init_night_times(self):
"""reset the night
"""
self.n_nights = 0
self.datetime_now = None
self.night_start_sec = datetime_to_secs(self.datetime_epoch)
self.night_end_sec = datetime_to_secs(self.datetime_epoch)
self.time_series_start_time_sec = self.night_start_sec
self.set_night_times()
self.update_once()
return
# ------------------------------------------------------------------
def update_once(self):
"""single update, to be run as part of a loop
"""
self.datetime_now += timedelta(seconds=self.loop_sleep_sec *
self.speed_factor)
if self.debug_datetime_now:
self.log.info([
['g', ' --- Now (night:', self.n_nights, '/', ''],
['p', self.is_night_time_now()],
['g', ') '],
['y', self.datetime_now],
[
'c',
' (' + str(datetime_to_secs(self.datetime_now)) + ' sec)'
],
])
self.update_n_night()
time_now_sec = datetime_to_secs(self.datetime_now)
is_night_now = self.is_night_time_now()
self.redis.set(
name='clock_sim_time_now_sec',
data=time_now_sec,
)
self.redis.set(
name='clock_sim_is_night_now',
data=is_night_now,
)
self.redis.set(
name='clock_sim_n_nights',
data=self.n_nights,
)
self.redis.set(
name='clock_sim_night_start_sec',
data=self.night_start_sec,
)
self.redis.set(
name='clock_sim_night_end_sec',
data=self.night_end_sec,
)
self.redis.set(
name='clock_sim_time_series_start_time_sec',
data=self.time_series_start_time_sec,
)
return
# ------------------------------------------------------------------
def set_night_times(self):
"""reset the night
"""
night_start_hours = self.rnd_gen.randint(18, 19)
night_start_minutes = self.rnd_gen.randint(0, 59)
night_end_hours = self.rnd_gen.randint(4, 5)
night_end_minutes = self.rnd_gen.randint(0, 59)
# short night for debugging
if self.is_short_night:
night_start_hours = 23
night_start_minutes = 0
night_end_hours = 2
night_end_minutes = 0
if self.datetime_now is None:
self.datetime_now = self.datetime_epoch.replace(
hour=(night_start_hours - 1), )
self.time_series_start_time_sec = self.night_start_sec
n_days = (self.datetime_now - self.datetime_epoch).days
self.night_start_sec = timedelta(
days=n_days,
hours=night_start_hours,
minutes=night_start_minutes,
seconds=0,
).total_seconds()
# e.g., night ends at 06:40
self.night_end_sec = timedelta(
days=(n_days + 1),
hours=night_end_hours,
minutes=night_end_minutes,
seconds=0,
).total_seconds()
if self.is_skip_daytime or self.is_short_night:
self.datetime_now = (secs_to_datetime(self.night_start_sec) -
timedelta(seconds=10))
night_start = date_to_string(
secs_to_datetime(self.night_start_sec),
date_string=None,
)
night_end = date_to_string(
secs_to_datetime(self.night_end_sec),
date_string=None,
)
self.log.info([
['b', ' - setting new night: ['],
['g', night_start],
['b', ' --> '],
['g', night_end],
['b', ']'],
])
return
# ------------------------------------------------------------------
def update_n_night(self):
sec_since_midnight = self.get_sec_since_midnight()
days_since_epoch = (self.datetime_now - self.datetime_epoch).days
is_new_day = days_since_epoch > self.n_nights
is_past_night_time = sec_since_midnight > self.night_end_sec
if is_new_day and is_past_night_time:
self.n_nights = days_since_epoch
self.set_night_times()
return
# ------------------------------------------------------------------
def need_data_update(self, update_opts):
"""check if a service needs to run an update, where
updates only happen after min_wait of simulation time
"""
time_now = self.get_time_now_sec()
set_prev_update = (('prev_update' not in update_opts.keys())
or (update_opts['prev_update'] is None))
if set_prev_update:
update_opts['prev_update'] = time_now - 2 * update_opts['min_wait']
time_diff = time_now - update_opts['prev_update']
can_update = (time_diff > update_opts['min_wait'])
# updates only happen during the astronimical night
is_night_time = self.is_night_time_now()
need_update = (is_night_time and can_update)
if need_update:
update_opts['prev_update'] = time_now
return need_update
# ------------------------------------------------------------------
def set_sim_speed(self, data_in, from_redis=False):
"""set parameters which determine the lenght of the night, the
real-time duration, given a speed factor, the delay between nights, etc.
"""
speed_factor = data_in['speed_factor']
is_skip_daytime = data_in['is_skip_daytime']
is_short_night = data_in['is_short_night']
if from_redis:
red_data = self.redis.get(name='clock_sim_sim_params')
if red_data is not None:
speed_factor = red_data['speed_factor']
is_skip_daytime = red_data['is_skip_daytime']
is_short_night = red_data['is_short_night']
if speed_factor is not None:
speed_factor = float(speed_factor)
is_ok = (speed_factor >= self.min_speed_factor
and speed_factor <= self.max_speed_factor)
if not is_ok:
raise ValueError(
'trying to set speed_factor out of bounds ...',
speed_factor)
self.speed_factor = float(speed_factor)
if is_skip_daytime is not None:
self.is_skip_daytime = is_skip_daytime
if is_short_night is not None:
self.is_short_night = is_short_night
self.log.info([
['b', ' - updating clock_sim_sim_params: '],
['c', ' speed_factor: '],
['p', self.speed_factor],
['c', ' , is_skip_daytime: '],
['p', self.is_skip_daytime],
['c', ' , is_short_night: '],
['p', self.is_short_night],
])
self.sim_params = {
'speed_factor': self.speed_factor,
'min_speed_factor': self.min_speed_factor,
'max_speed_factor': self.max_speed_factor,
'is_skip_daytime': self.is_skip_daytime,
'is_short_night': self.is_short_night,
}
self.redis.set(
name='clock_sim_sim_params',
data=self.sim_params,
)
self.redis.publish(channel='clock_sim_updated_sim_params')
return
# ------------------------------------------------------------------
def loop_main(self):
"""main loop running in its own thread, updating the night
"""
self.log.info([['g', ' - starting ClockSim.loop_main ...']])
while self.can_loop():
sleep(self.loop_sleep_sec)
with ClockSim.lock:
self.update_once()
self.log.info([['c', ' - ending ClockSim.loop_main ...']])
return
# ------------------------------------------------------------------
def pubsub_sim_params(self):
"""loop running in its own thread, reacting to pubsub events
"""
self.log.info([['g', ' - starting ClockSim.pubsub_sim_params ...']])
# setup the channel once
pubsub_tag = 'clock_sim_set_sim_params'
pubsub = self.redis.pubsub_subscribe(pubsub_tag)
# listen to changes on the channel and do stuff
while self.can_loop():
sleep(self.pubsub_sleep_sec)
msg = self.redis.pubsub_get_message(pubsub=pubsub)
if msg is None:
continue
with ClockSim.lock:
keys = ['speed_factor', 'is_skip_daytime', 'is_short_night']
data_out = dict()
for key in keys:
data_out[
key] = msg['data'][key] if key in msg['data'] else None
self.set_sim_speed(data_in=data_out)
self.log.info([['c', ' - ending ClockSim.pubsub_sim_params ...']])
return
class InstPos(ServiceManager):
"""telescope pointing-position simulation class, simulating changes of pointing
Only a single active instance is allowed to exist
"""
lock = Lock()
# ------------------------------------------------------------------
def __init__(self, base_config, service_name, interrupt_sig):
self.class_name = self.__class__.__name__
service_name = (service_name
if service_name is not None else self.class_name)
super().__init__(service_name=service_name)
self.log = LogParser(base_config=base_config, title=__name__)
self.base_config = base_config
self.site_type = self.base_config.site_type
self.clock_sim = self.base_config.clock_sim
self.inst_data = self.base_config.inst_data
self.service_name = service_name
self.interrupt_sig = interrupt_sig
self.tel_ids = self.inst_data.get_inst_ids()
self.inst_pos_0 = self.base_config.inst_pos_0
self.redis = RedisManager(name=self.class_name,
base_config=self.base_config,
log=self.log)
# ------------------------------------------------------------------
rnd_seed = 10989152934
self.rnd_gen = Random(rnd_seed)
# minimum interval of simulation-time to wait before randomising values
min_wait_update_sec = 10
self.check_update_opts = {
'prev_update': None,
'min_wait': min_wait_update_sec,
}
# sleep duration for thread loops
self.loop_sleep_sec = 1
# minimal real-time delay between randomisations (once every self.loop_act_rate sec)
self.loop_act_rate = max(int(5 / self.loop_sleep_sec), 1)
self.init()
# make sure this is the only active instance
self.init_active_instance()
self.setup_threads()
return
# ------------------------------------------------------------------
def setup_threads(self):
self.add_thread(target=self.loop_main)
return
# ------------------------------------------------------------------
def init(self):
# self.log.info([['g', ' - InstPos.init() ...']])
with InstPos.lock:
self.update_inst_pos()
return
# ------------------------------------------------------------------
def update_inst_pos(self):
min_delta_pos_sqr = pow(0.05, 2)
frac_delta_pos = 0.25
inst_pos_in = dict()
if self.redis.exists('inst_pos'):
inst_pos_in = self.redis.h_get_all(name='inst_pos')
obs_block_ids = self.redis.get(name=('obs_block_ids_' + 'run'),
default_val=[])
pipe = self.redis.get_pipe()
for obs_block_id in obs_block_ids:
pipe.get(obs_block_id)
blocks = pipe.execute()
tel_point_pos = dict()
for n_block in range(len(blocks)):
if not isinstance(blocks[n_block], dict):
continue
if len(blocks[n_block]['pointings']) == 0:
continue
tel_ids = (blocks[n_block]['telescopes']['large']['ids'] +
blocks[n_block]['telescopes']['medium']['ids'] +
blocks[n_block]['telescopes']['small']['ids'])
point_pos = blocks[n_block]['pointings'][0]['pos']
for id_now in tel_ids:
tel_point_pos[id_now] = point_pos
for id_now in self.tel_ids:
inst_pos_now = inst_pos_in[
id_now] if id_now in inst_pos_in else self.inst_pos_0
if inst_pos_now is None:
inst_pos_now = self.inst_pos_0
inst_pos_new = inst_pos_now
if id_now in tel_point_pos:
point_pos = tel_point_pos[id_now]
pos_dif = [(point_pos[0] - inst_pos_now[0]),
(point_pos[1] - inst_pos_now[1])]
if (pos_dif[0] > 360):
pos_dif[0] -= 360
elif (pos_dif[0] < -360):
pos_dif[0] += 360
# if(pos_dif[0] > 180):
# pos_dif[0] -= 360
# elif(pos_dif[0] < -180):
# pos_dif[0] += 360
if (pos_dif[1] >= 90):
pos_dif[1] -= 90
rnd_scale = 1
if (pos_dif[0] * pos_dif[0] +
pos_dif[1] * pos_dif[1]) < min_delta_pos_sqr:
rnd_scale = -1.5 if (self.rnd_gen.random() < 0.5) else 1.5
inst_pos_new = [
inst_pos_now[0] + pos_dif[0] * rnd_scale *
self.rnd_gen.random() * frac_delta_pos,
inst_pos_now[1] + pos_dif[1] * rnd_scale *
self.rnd_gen.random() * frac_delta_pos
]
pipe.h_set(name='inst_pos', key=id_now, data=inst_pos_new)
pipe.execute()
return
# ------------------------------------------------------------------
def loop_main(self):
self.log.info([['g', ' - starting InstPos.loop_main ...']])
sleep(0.1)
n_loop = 0
while self.can_loop():
n_loop += 1
sleep(self.loop_sleep_sec)
if n_loop % self.loop_act_rate != 0:
continue
need_update = self.clock_sim.need_data_update(
update_opts=self.check_update_opts, )
if not need_update:
continue
with InstPos.lock:
self.update_inst_pos()
self.log.info([['c', ' - ending InstPos.loop_main ...']])
return
class SchedulerStandalone(ServiceManager):
"""scheduler simulation class, simulating the execution of scheduling blocks
Only a single active instance is allowed to exist
"""
lock = Lock()
# ------------------------------------------------------------------
def __init__(self, base_config, service_name, interrupt_sig):
self.class_name = self.__class__.__name__
service_name = (service_name if service_name is not None else self.class_name)
super().__init__(service_name=service_name)
self.log = LogParser(base_config=base_config, title=__name__)
self.log.info([['g', ' - starting SchedulerStandalone ...']])
self.base_config = base_config
self.site_type = self.base_config.site_type
self.clock_sim = self.base_config.clock_sim
self.inst_data = self.base_config.inst_data
self.service_name = service_name
self.interrupt_sig = interrupt_sig
self.tel_ids = self.inst_data.get_inst_ids(inst_types=['LST', 'MST', 'SST'])
self.sub_array_insts = self.inst_data.get_sub_array_insts()
self.no_sub_arr_name = self.base_config.no_sub_arr_name
self.redis = RedisManager(
name=self.class_name, base_config=self.base_config, log=self.log
)
self.debug = not True
self.expire_sec = 86400 * 2 # two days
# self.expire_sec = 5
# self.max_n_obs_block = 4 if self.site_type == 'N' else 7
# self.max_n_obs_block = min(self.max_n_obs_block, floor(len(self.tel_ids) / 4))
# sleep duration for thread loops
self.loop_sleep_sec = 1
# minimal real-time delay between randomisations (once every self.loop_act_rate sec)
self.loop_act_rate = max(int(2 / self.loop_sleep_sec), 1)
self.max_n_cycles = 100
self.min_n_sched_block = 2 # 2
self.max_n_sched_block = 5 # 5
self.min_n_obs_block = 1
self.max_n_obs_block = 5
self.min_n_tel_block = 4
self.max_n_free_tels = 5
self.name_prefix = get_rnd(n_digits=5, out_type=str)
self.az_min_max = [-180, 180]
self.zen_min_max_tel = [0, 70]
self.zen_min_max_pnt = [0, 20]
self.phases_exe = {
'start': [
'run_config_mount', 'run_config_camera', 'run_config_DAQ',
'run_config_mirror'
],
'during': ['run_take_data'],
'finish': ['run_finish_mount', 'run_finish_camera', 'run_finish_cleanup'],
}
self.error_rnd_frac = {
'E1': 0.3,
'E2': 0.4,
'E3': 0.5,
'E4': 0.6,
'E5': 0.7,
'E6': 0.8,
'E7': 0.9,
'E8': 1,
}
self.phase_rnd_frac = {
'start': 0.29,
'finish': 0.1,
'cancel': 0.06,
'fail': 0.1,
}
# 1800 = 30 minutes
self.obs_block_sec = 1800
self.n_init_cycle = -1
self.n_nights = -1
self.update_name = 'obs_block_update'
self.sched_block_prefix = 'sched_block_'
self.obs_block_prefix = 'obs_block_'
rnd_seed = get_rnd_seed()
self.rnd_gen = Random(rnd_seed)
self.external_clock_events = []
external_generate_clock_events(self)
self.redis.delete(self.update_name)
self.init()
# make sure this is the only active instance
self.init_active_instance()
self.setup_threads()
return
# ------------------------------------------------------------------
def setup_threads(self):
self.add_thread(target=self.loop_main)
return
# ------------------------------------------------------------------
def init(self):
debug_tmp = False
# debug_tmp = True
self.exe_phase = dict()
self.all_obs_blocks = []
self.external_events = []
self.n_nights = self.clock_sim.get_n_nights()
night_start_sec = self.clock_sim.get_night_start_sec()
night_end_sec = self.clock_sim.get_night_end_sec()
night_duration_sec = self.clock_sim.get_night_duration_sec()
self.n_init_cycle += 1
is_cycle_done = False
n_cycle_now = 0
n_sched_block = -1
overhead_sec = self.obs_block_sec * 0.05
tot_sched_duration_sec = night_start_sec
max_block_duration_sec = night_end_sec - self.obs_block_sec
pipe = self.redis.get_pipe()
while True:
can_break = not ((tot_sched_duration_sec < max_block_duration_sec) and
(n_cycle_now < self.max_n_cycles) and (not is_cycle_done))
if can_break:
break
base_cycle_name = (
self.name_prefix + '_' + str(self.n_init_cycle) + '_' + str(n_cycle_now)
+ '_'
)
n_cycle_now += 1
# derive a random combination of sub-arrays which do not
# conflict with each other
sub_array_ids = list(self.sub_array_insts.keys())
n_sa_0 = self.rnd_gen.randint(0, len(sub_array_ids) - 1)
sa_id_0 = sub_array_ids[n_sa_0]
allowed_sa_ids = self.inst_data.get_allowed_sub_arrays(sa_id_0)
sa_ids = [sa_id_0]
while len(allowed_sa_ids) > 0:
# select a random id from the allowed list of the initial sa
check_n_sa = self.rnd_gen.randint(0, len(allowed_sa_ids) - 1)
sa_id_add = allowed_sa_ids[check_n_sa]
allowed_sa_ids.remove(sa_id_add)
# check if this id is allowed by all included sas
check_sa_ids = []
for sa_id in sa_ids:
check_sa_ids_now = self.inst_data.get_allowed_sub_arrays(sa_id)
check_sa_ids += [int(sa_id_add in check_sa_ids_now)]
# add the new sa if it is allowed by all
if sum(check_sa_ids) == len(check_sa_ids):
sa_ids += [sa_id_add]
if debug_tmp:
precent = (tot_sched_duration_sec - night_start_sec) / night_duration_sec
print()
print('-' * 100)
print(
' - n_nights/n_cycle_now',
[self.n_nights, n_cycle_now],
'tot_sched_duration_sec / percentage:',
[tot_sched_duration_sec, int(100 * precent)],
)
sched_block_duration_sec = []
# for n_sched_block_now in range(n_cycle_sched_blocks):
for n_sched_block_now in range(len(sa_ids)):
sched_block_id = (
self.sched_block_prefix + base_cycle_name + str(n_sched_block_now)
)
n_sched_block += 1
sa_id = sa_ids[n_sched_block_now]
tel_ids = self.sub_array_insts[sa_id]
n_tel_now = len(tel_ids)
if debug_tmp:
print(' -- sub-array:', sa_id, '\n', ' ' * 15, tel_ids)
# choose the number of obs blocks inside these blocks
n_obs_blocks = self.rnd_gen.randint(
self.min_n_obs_block, self.max_n_obs_block
)
if debug_tmp:
print(
' --- n_sched_block:', n_sched_block,
' --- n_sched_block_now / n_tel_now:', n_sched_block_now,
n_tel_now, '-------', sched_block_id
)
tot_obs_block_duration_sec = 0
block_duration_sec = tot_sched_duration_sec
targets = get_rnd_targets(
self=self,
night_duration_sec=night_duration_sec,
block_duration_sec=block_duration_sec,
)
for n_obs_now in range(n_obs_blocks):
obs_block_id = (
self.obs_block_prefix + base_cycle_name + str(n_sched_block_now)
+ '_' + str(n_obs_now)
)
obs_block_name = (str(n_sched_block) + ' (' + str(n_obs_now) + ')')
self.exe_phase[obs_block_id] = ''
rnd = self.rnd_gen.random()
obs_block_sec = self.obs_block_sec
if rnd < 0.05:
obs_block_sec /= 1.8
elif rnd < 0.3:
obs_block_sec /= 1.5
elif rnd < 0.5:
obs_block_sec /= 1.1
obs_block_sec = int(floor(obs_block_sec))
planed_block_end_sec = block_duration_sec + obs_block_sec
is_cycle_done = (planed_block_end_sec > night_end_sec)
if is_cycle_done:
if debug_tmp:
print(
' - is_cycle_done - ',
'n_obs_now / start_time_sec / duration:', n_obs_now,
block_duration_sec, obs_block_sec
)
break
# integrated time for all obs blocks within this sched block
tot_obs_block_duration_sec += obs_block_sec
pointings = get_rnd_pointings(
self=self,
tel_ids=tel_ids,
targets=targets,
sched_block_id=sched_block_id,
obs_block_id=obs_block_id,
n_obs_now=n_obs_now,
)
if debug_tmp:
print(
' ---- n_obs_now / start_time_sec / duration:',
n_obs_now,
block_duration_sec,
obs_block_sec,
'-------',
obs_block_id,
)
time = {
'start': block_duration_sec,
'duration': obs_block_sec - overhead_sec,
}
time['end'] = time['start'] + time['duration']
exe_state = {'state': 'wait', 'can_run': True}
metadata = {
'n_sched': n_sched_block,
'n_obs': n_obs_now,
'block_name': obs_block_name
}
telescopes = {
'large': {
'min':
int(len(list(filter(lambda x: 'L' in x, tel_ids))) / 2),
'max': 4,
'ids': list(filter(lambda x: 'L' in x, tel_ids))
},
'medium': {
'min':
int(len(list(filter(lambda x: 'M' in x, tel_ids))) / 2),
'max': 25,
'ids': list(filter(lambda x: 'M' in x, tel_ids))
},
'small': {
'min':
int(len(list(filter(lambda x: 'S' in x, tel_ids))) / 2),
'max': 70,
'ids': list(filter(lambda x: 'S' in x, tel_ids))
}
}
block = {
'sched_block_id': sched_block_id,
'obs_block_id': obs_block_id,
'time': time,
'metadata': metadata,
'timestamp': get_time('msec'),
'telescopes': telescopes,
'exe_state': exe_state,
'run_phase': [],
'targets': targets,
'pointings': pointings,
'tel_ids': tel_ids,
}
pipe.set(
name=block['obs_block_id'],
data=block,
expire_sec=self.expire_sec
)
self.all_obs_blocks.append(block)
block_duration_sec += obs_block_sec
# list of duration of all sched blocks within this cycle
if tot_obs_block_duration_sec > 0: # timedelta(seconds = 0):
sched_block_duration_sec += [tot_obs_block_duration_sec]
# the maximal duration of all blocks within this cycle
tot_sched_duration_sec += max(sched_block_duration_sec)
pipe.set(name='external_events', data=self.external_events)
pipe.set(name='external_clock_events', data=self.external_clock_events)
pipe.execute()
self.update_exe_statuses()
return
# ------------------------------------------------------------------
def get_obs_block_template(self):
"""temporary hardcoded dict......
"""
# generated with:
# print jsonAcs.encode(jsonAcs.classFactory.defaultValues[sb.ObservationBlock])
template = {
'py/object': 'sb.ObservationBlock',
'src': {
'py/object': 'sb.Source',
'proposal_priority': {
'py/object': 'sb.High'
},
'proposal_type': {
'py/object': 'sb.placeholder'
},
'region_of_interest': {
'py/object': 'sb.RegionOfInterest',
'circle_radius': 100
},
'coords': {
'py/object': 'sb.Coordinates',
'equatorial': {
'py/object': 'sb.EquatorialCoordinates',
'dec': 4,
'ra': 2
}
},
'id': 'source',
'observing_mode': {
'py/object': 'sb.ObservingMode',
'slewing_': {
'py/object': 'sb.Slewing',
'take_data': 1
},
'observing_type': {
'py/object': 'sb.ObservingType',
'wobble_': {
'py/object': 'sb.Wobble',
'angle': 1,
'offset': 1
}
}
}
},
'observing_conditions': {
'py/object': 'sb.ObservingConditions',
'quality_': {
'py/object': 'sb.Quality',
'illumination': 1,
'min_nsb_range': 1,
'max_nsb_range': 1
},
'start_time_sec': {
'py/object': 'sb.DateTime',
'placeholder': 1
},
'weather_': {
'py/object': 'sb.Weather',
'wind_speed': 1,
'precision_pointing': 1,
'cloudiness': 1,
'humidity': 1
},
'duration': 0,
'tolerance': 1
},
'max_script_duration': 0,
'script_id': 'script_id',
'id': 'ob_id'
}
return template
# ------------------------------------------------------------------
def wait_to_run(self):
"""move one from wait to run
"""
# time_now_sec = self.time_of_night.get_current_time()
time_now_sec = self.clock_sim.get_time_now_sec()
# move to run state
wait_blocks = [
x for x in self.all_obs_blocks if (x['exe_state']['state'] == 'wait')
]
pipe = self.redis.get_pipe()
has_change = False
for block in wait_blocks:
time_comp = (
block['time']['start'] - (self.loop_sleep_sec * self.loop_act_rate)
)
if time_now_sec < time_comp:
# datetime.strptime(block['start_time_sec'], '%Y-%m-%d %H:%M:%S'):
# - deltatime((self.loop_sleep_sec * self.loop_act_rate))
continue
block['exe_state']['state'] = 'run'
self.exe_phase[block['obs_block_id']] = 'start'
block['run_phase'] = copy.deepcopy(self.phases_exe['start'])
has_change = True
pipe.set(name=block['obs_block_id'], data=block, expire_sec=self.expire_sec)
if has_change:
pipe.execute()
# check for blocks which cant begin as their time is already past
wait_blocks = [
x for x in self.all_obs_blocks if x['exe_state']['state'] == 'wait'
]
has_change = False
for block in wait_blocks:
# # adjust the starting/ending time
# block['end_time_sec'] = block['start_time_sec'] + block['duration']
is_over_time = time_now_sec >= block['time']['end']
is_rnd_stop = (
self.rnd_gen.random() < self.phase_rnd_frac['cancel'] * 0.1
)
if is_over_time or is_rnd_stop:
block['exe_state']['state'] = 'cancel'
if self.rnd_gen.random() < self.error_rnd_frac['E1']:
block['exe_state']['error'] = 'E1'
elif self.rnd_gen.random() < self.error_rnd_frac['E2']:
block['exe_state']['error'] = 'E2'
elif self.rnd_gen.random() < self.error_rnd_frac['E3']:
block['exe_state']['error'] = 'E3'
elif self.rnd_gen.random() < self.error_rnd_frac['E4']:
block['exe_state']['error'] = 'E4'
elif self.rnd_gen.random() < self.error_rnd_frac['E8']:
block['exe_state']['error'] = 'E8'
block['exe_state']['can_run'] = False
block['run_phase'] = []
self.exe_phase[block['obs_block_id']] = ''
has_change = True
pipe.set(
name=block['obs_block_id'],
data=block,
expire_sec=self.expire_sec,
)
if has_change:
pipe.execute()
return
# ------------------------------------------------------------------
def run_phases(self):
"""progress run phases
"""
time_now_sec = self.clock_sim.get_time_now_sec()
runs = [x for x in self.all_obs_blocks if (x['exe_state']['state'] == 'run')]
pipe = self.redis.get_pipe()
has_change = False
for block in runs:
phase = self.exe_phase[block['obs_block_id']]
if phase == '':
continue
for phase_now in self.phases_exe[phase]:
if phase_now in block['run_phase']:
if phase_now in self.phases_exe['start']:
is_done = (self.rnd_gen.random() < self.phase_rnd_frac['start'])
# if is_done:
# block['end_time_sec'] = block['start_time_sec'] + block['duration']
elif phase_now in self.phases_exe['during']:
is_done = (
time_now_sec >= (
block['time']['end'] -
block['time']['duration'] * self.phase_rnd_frac['finish']
)
) # (datetime.strptime(block['end_time_sec'], '%Y-%m-%d %H:%M:%S') - timedelta(seconds = int(block['duration']) * self.phase_rnd_frac['finish'])))
else:
is_done = (
time_now_sec >= block['time']['end']
) # is_done = (time_now_sec >= datetime.strptime(block['end_time_sec'], '%Y-%m-%d %H:%M:%S'))
if is_done:
block['run_phase'].remove(phase_now)
# print is_done,block['run_phase']
if len(block['run_phase']) == 0:
next_phase = ''
if phase == 'start':
next_phase = 'during'
elif phase == 'during':
next_phase = 'finish'
if next_phase in self.phases_exe:
block['run_phase'] = copy.deepcopy(self.phases_exe[next_phase])
self.exe_phase[block['obs_block_id']] = next_phase
has_change = True
pipe.set(name=block['obs_block_id'], data=block, expire_sec=self.expire_sec)
if has_change:
pipe.execute()
return
# ------------------------------------------------------------------
def run_to_done(self):
"""move one from run to done
"""
# time_now_sec = self.time_of_night.get_current_time()
time_now_sec = self.clock_sim.get_time_now_sec()
runs = [x for x in self.all_obs_blocks if x['exe_state']['state'] == 'run']
pipe = self.redis.get_pipe()
has_change = False
for block in runs:
if time_now_sec < block['time']['end']:
continue
if self.rnd_gen.random() < self.phase_rnd_frac['cancel']:
block['exe_state']['state'] = 'cancel'
if self.rnd_gen.random() < self.error_rnd_frac['E1']:
block['exe_state']['error'] = 'E1'
elif self.rnd_gen.random() < self.error_rnd_frac['E2']:
block['exe_state']['error'] = 'E2'
elif self.rnd_gen.random() < self.error_rnd_frac['E3']:
block['exe_state']['error'] = 'E3'
elif self.rnd_gen.random() < self.error_rnd_frac['E4']:
block['exe_state']['error'] = 'E4'
elif self.rnd_gen.random() < self.error_rnd_frac['E8']:
block['exe_state']['error'] = 'E8'
elif self.rnd_gen.random() < self.phase_rnd_frac['fail']:
block['exe_state']['state'] = 'fail'
if self.rnd_gen.random() < self.error_rnd_frac['E1']:
block['exe_state']['error'] = 'E1'
elif self.rnd_gen.random() < self.error_rnd_frac['E2']:
block['exe_state']['error'] = 'E2'
elif self.rnd_gen.random() < self.error_rnd_frac['E3']:
block['exe_state']['error'] = 'E3'
elif self.rnd_gen.random() < self.error_rnd_frac['E4']:
block['exe_state']['error'] = 'E4'
elif self.rnd_gen.random() < self.error_rnd_frac['E8']:
block['exe_state']['error'] = 'E8'
else:
block['exe_state']['state'] = 'done'
block['run_phase'] = []
has_change = True
pipe.set(name=block['obs_block_id'], data=block, expire_sec=self.expire_sec)
self.exe_phase[block['obs_block_id']] = ''
if has_change:
pipe.execute()
return
# ------------------------------------------------------------------
def update_exe_statuses(self):
"""update the exeStatus lists in redis
"""
blocks_run = []
obs_block_ids = {'wait': [], 'run': [], 'done': [], 'cancel': [], 'fail': []}
pipe = self.redis.get_pipe()
for block in self.all_obs_blocks:
obs_block_id = block['obs_block_id']
exe_state = block['exe_state']['state']
if self.redis.exists(obs_block_id):
obs_block_ids[exe_state].append(obs_block_id)
if exe_state == 'run':
blocks_run += [block]
for key, val in obs_block_ids.items():
pipe.set(name='obs_block_ids_' + key, data=val)
pipe.execute()
update_sub_arrs(self=self, blocks=blocks_run)
return
# # ------------------------------------------------------------------
# def update_sub_arrs(self, blocks=None):
# pipe = self.redis.get_pipe()
# if blocks is None:
# obs_block_ids = self.redis.get(
# name=('obs_block_ids_' + 'run'), default_val=[]
# )
# for obs_block_id in obs_block_ids:
# pipe.get(obs_block_id)
# blocks = pipe.execute()
# #
# sub_arrs = []
# all_tel_ids = copy.deepcopy(self.tel_ids)
# for n_block in range(len(blocks)):
# block_tel_ids = (
# blocks[n_block]['telescopes']['large']['ids']
# + blocks[n_block]['telescopes']['medium']['ids']
# + blocks[n_block]['telescopes']['small']['ids']
# )
# pnt_id = blocks[n_block]['pointings'][0]['id']
# pointing_name = blocks[n_block]['pointings'][0]['name']
# # compile the telescope list for this block
# tels = []
# for id_now in block_tel_ids:
# tels.append({'id': id_now})
# if id_now in all_tel_ids:
# all_tel_ids.remove(id_now)
# # add the telescope list for this block
# sub_arrs.append({'id': pnt_id, 'N': pointing_name, 'children': tels})
# # ------------------------------------------------------------------
# # now take care of all free telescopes
# # ------------------------------------------------------------------
# tels = []
# for id_now in all_tel_ids:
# tels.append({'id': id_now})
# sub_arrs.append({'id': self.no_sub_arr_name, 'children': tels})
# # ------------------------------------------------------------------
# # for now - a simple/stupid solution, where we write the sub-arrays and publish each
# # time, even if the content is actually the same ...
# # ------------------------------------------------------------------
# self.redis.set(name='sub_arrs', data=sub_arrs)
# self.redis.publish(channel='sub_arrs')
# return
# ------------------------------------------------------------------
def external_add_new_redis_blocks(self):
obs_block_update = self.redis.get(self.update_name, default_val=None)
if obs_block_update is None:
return
pipe = self.redis.get_pipe()
# for key in self.all_obs_blocks[0]:
# self.log.info([['g', key, self.all_obs_blocks[0][key]]])
# self.log.info([['g', obs_block_update]])
self.log.info([['g', len(obs_block_update), len(self.all_obs_blocks)]])
total = 0
for n_block in range(len(obs_block_update)):
if self.redis.exists(obs_block_update[n_block]['obs_block_id']):
# for x in self.all_obs_blocks:
# if x['obs_block_id'] == obs_block_update[n_block]['obs_block_id']:
# current = [x][0]
current = [
x for x in self.all_obs_blocks
if x['obs_block_id'] == obs_block_update[n_block]['obs_block_id']
]
if len(current) == 0:
current = obs_block_update[n_block]
self.all_obs_blocks.append(current)
# for key in obs_block_update[n_block]:
# self.log.info([['g', key, obs_block_update[n_block][key]]])
else:
current = current[0]
if current['exe_state']['state'] not in ['wait', 'run']:
continue
total += 1
pipe.set(
name=obs_block_update[n_block]['obs_block_id'],
data=obs_block_update[n_block],
expire_sec=self.expire_sec,
)
current = obs_block_update[n_block]
else:
self.all_obs_blocks.append(obs_block_update[n_block])
pipe.set(
name=obs_block_update[n_block]['obs_block_id'],
data=obs_block_update[n_block],
expire_sec=self.expire_sec,
)
self.update_exe_statuses()
# for block in self.all_obs_blocks:
# exe_state = block['exe_state']['state']
# self.log.info([['g', block['metadata']['block_name'] + ' ' + exe_state]])
pipe.delete(self.update_name)
pipe.execute()
self.log.info([['g', total, len(obs_block_update), len(self.all_obs_blocks)]])
return
# ------------------------------------------------------------------
def loop_main(self):
self.log.info([['g', ' - starting SchedulerStandalone.loop_main ...']])
sleep(0.1)
n_loop = 0
while self.can_loop():
n_loop += 1
sleep(self.loop_sleep_sec)
if n_loop % self.loop_act_rate != 0:
continue
with SchedulerStandalone.lock:
if self.n_nights < self.clock_sim.get_n_nights():
self.init()
else:
self.external_add_new_redis_blocks()
wait_blocks = [
x for x in self.all_obs_blocks
if (x['exe_state']['state'] == 'wait')
]
runs = [
x for x in self.all_obs_blocks
if (x['exe_state']['state'] == 'run')
]
if len(wait_blocks) + len(runs) == 0:
self.init()
else:
self.wait_to_run()
self.run_phases()
self.run_to_done()
external_generate_events(self)
self.update_exe_statuses()
self.log.info([['c', ' - ending SchedulerStandalone.loop_main ...']])
return