class SchedulerACS(ThreadManager):
has_active = False
lock = Lock()
def __init__(self, base_config, interrupt_sig):
self.log = LogParser(base_config=base_config, title=__name__)
# self.log.info([['y', ' - SchedulerACS - '], ['g', base_config.site_type]])
if SchedulerACS.has_active:
raise Exception('Can not instantiate SchedulerACS more than once...')
else:
SchedulerACS.has_active = True
self.base_config = base_config
self.site_type = self.base_config.site_type
self.clock_sim = self.base_config.clock_sim
self.inst_data = self.base_config.inst_data
self.interrupt_sig = interrupt_sig
self.tel_ids = self.inst_data.get_inst_ids(inst_types=['LST', 'MST', 'SST'])
self.no_sub_arr_name = self.base_config.no_sub_arr_name
self.class_name = self.__class__.__name__
self.redis = RedisManager(
name=self.class_name, base_config=self.base_config, log=self.log
)
self.debug = not True
self.expire_sec = 86400 # one day
# self.expire_sec = 5
self.MockSched = None
# self.client = PySimpleClient()
# self.supervisor = self.client.getComponent('ArraySupervisor')
# self.log.info([['y',' - SchedulerACS - '],['p','got supervisor!']])
self.phases_exe = dict()
self.phases_exe['start'] = [
'run_config_mount',
'run_config_camera',
'run_config_daq',
'run_config_mirror',
]
self.phases_exe['during'] = [
'run_take_data',
]
self.phases_exe['finish'] = [
'run_finish_mount',
'run_finish_camera',
'run_finish_daq',
]
self.az_min_max = [-180, 180]
self.loop_sleep_sec = 3
rnd_seed = get_rnd_seed()
# rnd_seed = 10987268332
self.rnd_gen = Random(rnd_seed)
self.setup_threads()
self.MockSched = MockSched(
base_config=self.base_config, interrupt_sig=self.interrupt_sig
)
# ------------------------------------------------------------------
# temporary hack to be consistent with SchedulerStandalone
# ------------------------------------------------------------------
self.external_events = []
self.external_clock_events = []
external_generate_clock_events(self)
external_generate_events(self)
return
# ---------------------------------------------------------------------------
def setup_threads(self):
self.add_thread(target=self.loop_main)
return
# ------------------------------------------------------------------
def reset_blocks(self):
debug_tmp = False
# debug_tmp = True
if debug_tmp:
self.log.info([['p', ' - SchedulerACS.reset_blocks() ...']])
if self.MockSched is None:
sleep(0.1)
self.log.debug([[
'r', ' - no MockSched ... will try to reset_blocks() again ...'
]])
self.reset_blocks()
return
night_duration_sec = self.clock_sim.get_night_duration_sec()
sched_blocks = self.MockSched.get_blocks()
obs_block_ids = dict()
obs_block_ids['wait'] = []
obs_block_ids['run'] = []
obs_block_ids['done'] = []
obs_block_ids['cancel'] = []
obs_block_ids['fail'] = []
blocks_run = []
active = sched_blocks['active']
pipe = self.redis.get_pipe()
for sched_blk_id, schBlock in sched_blocks['blocks'].items():
sub_array_tels = (
schBlock['sched_block'].config.instrument.sub_array.telescopes
)
tel_ids = [x.id for x in sub_array_tels]
obs_blocks = schBlock['sched_block'].observation_blocks
# get the total duration of all obs blocks
block_duration_sec = 0
for n_obs_block_now in range(len(obs_blocks)):
obs_block_id = obs_blocks[n_obs_block_now].id
block_duration_sec += (sched_blocks['metadata'][obs_block_id]['duration'])
targets = get_rnd_targets(
self=self,
night_duration_sec=night_duration_sec,
block_duration_sec=block_duration_sec,
)
for n_obs_block_now in range(len(obs_blocks)):
obs_block_now = obs_blocks[n_obs_block_now]
obs_block_id = obs_block_now.id
# trg_id = obs_block_now.src.id
# coords = obs_block_now.src.coords.horizontal
sched_metadata = sched_blocks['metadata'][obs_block_id]
timestamp = sched_metadata['timestamp']
metadata = sched_metadata['metadata']
status = sched_metadata['status']
phases = sched_metadata['phases']
duration = sched_metadata['duration']
start_time_sec_plan = sched_metadata['start_time_sec_plan']
start_time_sec_exe = sched_metadata['start_time_sec_exe']
start_time_sec = (
start_time_sec_plan
if start_time_sec_exe is None else start_time_sec_exe
)
# state of ob
if status == sb.OB_PENDING:
state = 'wait'
elif status == sb.OB_RUNNING:
state = 'run'
elif status == sb.OB_CANCELED:
state = 'cancel'
elif status == sb.OB_FAILED:
state = 'fail'
else:
state = 'done'
# final sanity check
if state == 'run' and sched_blk_id not in active:
state = 'done'
run_phase = []
if state == 'run':
for p in phases:
if p.status == sb.OB_RUNNING:
phase_name = 'run_' + p.name
for phases_exe in self.phases_exe:
if phase_name in self.phases_exe[phases_exe]:
run_phase.append(phase_name)
if debug_tmp:
self.log.debug([
['b', ' -- run_phase - '],
['y', run_phase, ' '],
['b', tel_ids],
])
can_run = True
if state == 'cancel' or state == 'fail':
can_run = (self.clock_sim.get_time_now_sec() >= start_time_sec)
exe_state = {'state': state, 'can_run': can_run}
time = {
'start': start_time_sec,
'duration': duration,
}
time['end'] = time['start'] + time['duration']
telescopes = {
'large': {
'min': int(len(list(filter(lambda x: 'L' in x, tel_ids))) / 2),
'max': 4,
'ids': list(filter(lambda x: 'L' in x, tel_ids))
},
'medium': {
'min': int(len(list(filter(lambda x: 'M' in x, tel_ids))) / 2),
'max': 25,
'ids': list(filter(lambda x: 'M' in x, tel_ids))
},
'small': {
'min': int(len(list(filter(lambda x: 'S' in x, tel_ids))) / 2),
'max': 70,
'ids': list(filter(lambda x: 'S' in x, tel_ids))
}
}
pointings = get_rnd_pointings(
self=self,
tel_ids=tel_ids,
targets=targets,
sched_block_id=sched_blk_id,
obs_block_id=obs_block_id,
n_obs_now=n_obs_block_now,
)
block = dict()
block['sched_block_id'] = sched_blk_id
block['obs_block_id'] = obs_block_id
block['time'] = time
block['metadata'] = metadata
block['timestamp'] = timestamp
block['telescopes'] = telescopes
block['exe_state'] = exe_state
block['run_phase'] = run_phase
block['targets'] = targets
block['pointings'] = pointings
block['tel_ids'] = tel_ids
if state == 'run':
blocks_run += [block]
obs_block_ids[state].append(obs_block_id)
pipe.set(
name=obs_block_id,
data=block,
expire_sec=self.expire_sec,
packed=True
)
pipe.set(name='obs_block_ids_' + 'wait', data=obs_block_ids['wait'], packed=True)
pipe.set(name='obs_block_ids_' + 'run', data=obs_block_ids['run'], packed=True)
pipe.set(name='obs_block_ids_' + 'done', data=obs_block_ids['done'], packed=True)
pipe.set(
name='obs_block_ids_' + 'cancel', data=obs_block_ids['cancel'], packed=True
)
pipe.set(name='obs_block_ids_' + 'fail', data=obs_block_ids['fail'], packed=True)
pipe.execute()
update_sub_arrs(self=self, blocks=blocks_run)
return
# # ------------------------------------------------------------------
# #
# # ------------------------------------------------------------------
# def update_sub_arrs(self, blocks=None):
# # inst_pos = self.redis.h_get_all(name='inst_pos')
# pipe = self.redis.get_pipe()
# if blocks is None:
# obs_block_ids = self.redis.get(
# name=('obs_block_ids_' + 'run'), packed=True, default_val=[]
# )
# for obs_block_id in obs_block_ids:
# pipe.get(obs_block_id)
# blocks = pipe.execute(packed=True)
# # sort so last is first in the list (latest sub-array defined gets the telescope)
# blocks = sorted(
# blocks, cmp=lambda a, b: int(b['timestamp']) - int(a['timestamp'])
# )
# # print [a['timestamp'] for a in blocks]
# sub_arrs = []
# all_tel_ids_in = []
# for n_block in range(len(blocks)):
# block_tel_ids = blocks[n_block]['tel_ids']
# pnt_id = blocks[n_block]['point_id']
# pointing_name = blocks[n_block]['pointing_name']
# # compile the telescope list for this block
# tels = []
# for id_now in block_tel_ids:
# if id_now not in all_tel_ids_in:
# all_tel_ids_in.append(id_now)
# tels.append({'id': id_now})
# # add the telescope list for this block
# sub_arrs.append({'id': pnt_id, 'N': pointing_name, 'children': tels})
# # ------------------------------------------------------------------
# # now take care of all free telescopes
# # ------------------------------------------------------------------
# tels = []
# all_tel_ids = [x for x in self.tel_ids if x not in all_tel_ids_in]
# for id_now in all_tel_ids:
# tels.append({'id': id_now})
# sub_arrs.append({'id': self.no_sub_arr_name, 'children': tels})
# # ------------------------------------------------------------------
# # for now - a simple/stupid solution, where we write the sub-arrays and publish each
# # time, even if the content is actually the same ...
# # ------------------------------------------------------------------
# self.redis.set(name='sub_arrs', data=sub_arrs, packed=True)
# self.redis.publish(channel='sub_arrs')
# return
# ------------------------------------------------------------------
def loop_main(self):
self.log.info([['g', ' - starting SchedulerACS.loop_main ...']])
pipe = self.redis.get_pipe()
pipe.set(name='obs_block_ids_' + 'wait', data='')
pipe.set(name='obs_block_ids_' + 'run', data='')
pipe.set(name='obs_block_ids_' + 'done', data='')
pipe.set(name='obs_block_ids_' + 'cancel', data='')
pipe.set(name='obs_block_ids_' + 'fail', data='')
pipe.execute()
update_sub_arrs(self=self, blocks=[])
while self.can_loop():
sleep(self.loop_sleep_sec)
with SchedulerACS.lock:
self.reset_blocks()
self.log.info([['c', ' - ending SchedulerACS.loop_main ...']])
return
class BaseWidget():
# all session ids for this user/widget
widget_group_sess = dict()
# ------------------------------------------------------------------
def __init__(self, widget_id=None, sm=None, *args, **kwargs):
self.log = LogParser(base_config=sm.base_config, title=__name__)
# the parent of this widget
self.sm = sm
# the shared basic configuration class
self.base_config = self.sm.base_config
self.sess_id = self.sm.sess_id
self.user_id = self.sm.user_id
# the id of this instance
self.widget_id = widget_id
# widget-class and widget group names
self.widget_type = self.__class__.__name__
# for common threading
self.widget_group = self.sm.user_group_id + '_' + self.widget_type
# redis interface
self.redis = RedisManager(
name=self.widget_type, base_config=self.base_config, log=self.log
)
# turn on periodic data updates
self.do_data_updates = True
# some etra logging messages for this module
self.log_send_packet = False
# fixed or dynamic icon
self.n_icon = None
# self.n_icon = -1
# self.icon_id = -1
# list of utility classes to loop over
self.my_utils = dict()
# arguments given to the setup function, to later be
# passed to utils if needed
self.setup_args = None
return
# ------------------------------------------------------------------
async def setup(self, *args):
self.setup_args = args
widget_info = await self.sm.get_lazy_widget_info(
sess_id=self.sess_id,
widget_id=self.widget_id,
)
if widget_info is None:
return
if self.n_icon is None:
self.n_icon = widget_info['n_icon']
# self.icon_id = widget_info['icon_id']
# override the global logging variable with a
# name corresponding to the current session id
self.log = LogParser(
base_config=self.base_config,
title=(
str(self.user_id) + '/' + str(self.sm.sess_id) + '/' + __name__ + '/'
+ self.widget_id
),
)
return
# ------------------------------------------------------------------
async def util_setup(self, data):
"""load a utility class and set it up
"""
util_id = data['util_id']
util_type = data['util_type']
self.log.debug([
['b', ' - util_setup: '],
['y', util_type],
['b', ' with '],
['y', util_id],
['b', ' to '],
['o', self.widget_id],
])
widget_info = await self.sm.get_lazy_widget_info(
sess_id=self.sess_id,
widget_id=self.widget_id,
)
if widget_info is None:
return
if util_id not in widget_info['util_ids']:
widget_info['util_ids'] += [util_id]
# dynamic loading of the module
util_source = self.sm.util_module_dir + '.' + util_type
util_module = importlib.import_module(util_source, package=None)
util_cls = getattr(util_module, util_type)
# instantiate the util class
self.my_utils[util_id] = util_cls(util_id=util_id, parent=self)
# add a lock for initialisation. needed in case of restoration
# of sessions, in order to make sure the initialisation
# method is called before any other
expire_sec = self.my_utils[util_id].sm.get_expite_sec(name='widget_init_expire')
self.my_utils[util_id].locker.semaphores.add(
name=self.get_util_lock_name(util_id),
key=util_id,
expire_sec=expire_sec,
)
# run the setup function of the util
await self.my_utils[util_id].setup(self.setup_args)
self.redis.h_set(name='ws;widget_info', key=self.widget_id, data=widget_info)
return
# ------------------------------------------------------------------
async def util_func(self, data):
"""execute util methods following client events
"""
util_id = data['util_id']
method_name = data['method_name']
method_args = data['method_args'] if 'method_args' in data else dict()
# is this the initialisation method
is_init_func = (method_name == 'util_init')
# in case this is the first call, the util should be loaded
if util_id not in self.my_utils:
await self.util_setup(data)
# if the first call is not the initialisation method, this is probably
# a restored session. in this case we send the client a request to
# send the initialisation method request, and the current method call will
# be blocked by the initialisation lock
if not is_init_func:
opt_in = {
'widget': self,
'event_name': ('ask_init_util;' + util_id),
}
await self.sm.emit_widget_event(opt_in=opt_in)
# block non-initialisation calls if initialisation has not finished yet
if not is_init_func:
max_lock_sec = self.sm.get_expite_sec(
name='widget_init_expire',
is_lock_check=True,
)
await self.my_utils[util_id].locker.semaphores.async_block(
is_locked=await self.is_util_init_locked(util_id),
max_lock_sec=max_lock_sec,
)
# execute the requested method
init_func = getattr(self.my_utils[util_id], method_name)
await init_func(method_args)
# remove the initialisation lock
if is_init_func:
self.my_utils[util_id].locker.semaphores.remove(
name=self.get_util_lock_name(util_id),
key=util_id,
)
return
# ------------------------------------------------------------------
def get_util_lock_name(self, util_id):
"""a unique name for initialisation locking
"""
lock_name = (
'ws;base_widget;util_func;' + self.my_utils[util_id].class_name + ';'
+ self.my_utils[util_id].util_id
)
return lock_name
# ------------------------------------------------------------------
async def is_util_init_locked(self, util_id):
"""a function which checks if the initialisation is locked
"""
async def is_locked():
locked = self.my_utils[util_id].locker.semaphores.check(
name=self.get_util_lock_name(util_id),
key=util_id,
)
return locked
return is_locked
# ------------------------------------------------------------------
async def back_from_offline(self, *args):
"""interface function for back-from-offline events
"""
# check if any util is missing (e.g., in case we are back
# after a session recovery) and ask the client to respond
widget_info = self.redis.h_get(name='ws;widget_info', key=self.widget_id)
util_ids = widget_info['util_ids']
util_ids = [u for u in util_ids if u not in self.my_utils.keys()]
for util_id in util_ids:
opt_in = {
'widget': self,
'event_name': ('ask_init_util;' + util_id),
}
await self.sm.emit_widget_event(opt_in=opt_in)
# loop over utils
for util_now in self.my_utils.values():
await util_now.back_from_offline(args)
return