def __init__(self,
source_queue,
*args,
config_file=CONFIG_FILE,
control_queue=None,
**kwargs):
"""
:param Queue source_queue: Input queue
:param str config_file: Path to config file
:param Queue control_queue: Control queue of parent Process
"""
super(OfflineProcessing, self).__init__()
self.observation_queue = source_queue
self.control_queue = control_queue
self.stop_event = mp.Event()
self.threads = {}
# load config
self.config_file = config_file
self.load_config()
# setup logger
self.logger = get_logger(__name__, self.log_file)
self.logger.info('OfflineProcessing initialized')
def __init__(self, source_queue, *args, config_file=CONFIG_FILE, control_queue=None, **kwargs):
"""
:param Queue source_queue: Input queue for controlling this service
:param str config_file: Path to config file
:param Queue control_queue: Control queue of parent Process
"""
super(LOFARTrigger, self).__init__()
self.stop_event = mp.Event()
self.input_queue = source_queue
self.control_queue = control_queue
self.trigger_server = None
with open(config_file, 'r') as f:
config = yaml.load(f, Loader=yaml.SafeLoader)['lofar_trigger']
# set config, expanding strings
kwargs = {'home': os.path.expanduser('~'), 'hostname': socket.gethostname()}
for key, value in config.items():
if isinstance(value, str):
value = value.format(**kwargs)
setattr(self, key, value)
# setup logger
self.logger = get_logger(__name__, self.log_file)
# Initalize the queue server
trigger_queue = mp.Queue()
LOFARTriggerQueueServer.register('get_queue', callable=lambda: trigger_queue)
self.trigger_queue = trigger_queue
self.logger.info("LOFAR Trigger initialized")
def _load_config(self):
"""
Load configuration file
"""
with open(self.config_file, 'r') as f:
config = yaml.load(f, Loader=yaml.SafeLoader)['darc_master']
# set config, expanding strings
kwargs = {'home': os.path.expanduser('~'), 'hostname': self.hostname}
for key, value in config.items():
if isinstance(value, str):
value = value.format(**kwargs)
setattr(self, key, value)
# create main log dir
log_dir = os.path.dirname(self.log_file)
try:
util.makedirs(log_dir)
except Exception as e:
raise DARCMasterException(
"Cannot create log directory: {}".format(e))
# setup logger
self.logger = get_logger(__name__, self.log_file)
# store services
if self.hostname == MASTER:
if self.mode == 'real-time':
self.services = self.services_master_rt
elif self.mode == 'mixed':
self.services = self.services_master_mix
else:
self.services = self.services_master_off
elif self.hostname in WORKERS:
if self.mode == 'real-time':
self.services = self.services_worker_rt
elif self.mode == 'mixed':
self.services = self.services_worker_mix
else:
self.services = self.services_worker_off
else:
self.services = []
# terminate any existing threads in case this is a reload
for service, thread in self.threads.items():
if thread is not None:
self.logger.warning(
f"Config reload, terminating existing {service}")
thread.terminate()
# start with empty thread for each service
for service in self.services:
self.threads[service] = None
# print config file path
self.logger.info("Loaded config from {}".format(self.config_file))
def __init__(self, source_queue, *args, config_file=CONFIG_FILE, control_queue=None, **kwargs):
"""
:param Queue source_queue: Input queue
:param str config_file: Path to config file
:param Queue control_queue: Control queue of parent Process
"""
super(StatusWebsite, self).__init__()
self.stop_event = mp.Event()
self.source_queue = source_queue
self.control_queue = control_queue
# load config, including master for list of services
with open(config_file, 'r') as f:
config_all = yaml.load(f, Loader=yaml.SafeLoader)
config = config_all['status_website']
config_master = config_all['darc_master']
# set config, expanding strings
kwargs = {'home': os.path.expanduser('~'), 'hostname': socket.gethostname()}
for key, value in config.items():
if isinstance(value, str):
value = value.format(**kwargs)
setattr(self, key, value)
# set services
if config_master['mode'] == 'real-time':
self.services_master = config_master['services_master_rt']
self.services_worker = config_master['services_worker_rt']
elif config_master['mode'] == 'mixed':
self.services_master = config_master['services_master_mix']
self.services_worker = config_master['services_worker_mix']
else:
self.services_master = config_master['services_master_off']
self.services_worker = config_master['services_worker_off']
# store all node names
self.all_nodes = [MASTER] + WORKERS
# setup logger
self.logger = get_logger(__name__, self.log_file)
self.command_checker = None
# reduce logging from status check commands
logging.getLogger('darc.control').setLevel(logging.ERROR)
# create website directory
try:
util.makedirs(self.web_dir)
except Exception as e:
self.logger.error("Failed to create website directory: {}".format(e))
raise StatusWebsiteException("Failed to create website directory: {}".format(e))
self.logger.info('Status website initialized')
def __init__(self,
source_queue,
target_queue=None,
second_target_queue=None,
control_queue=None,
config_file=CONFIG_FILE,
no_logger=False):
"""
:param Queue source_queue: Input queue
:param Queue target_queue: Output queue
:param Queue second_target_queue: second output queue
:param Queue control_queue: Control queue
:param str config_file: Path to config file
:param bool no_logger: Skip logger initialization
"""
super(DARCBase, self).__init__()
self.stop_event = mp.Event()
self.source_queue = source_queue
self.target_queue = target_queue
self.second_target_queue = second_target_queue
self.control_queue = control_queue
# set names for config and logger
self.module_name = type(self).__module__.split('.')[-1]
self.log_name = type(self).__name__
# load config
self.config_file = config_file
self.load_config()
if not no_logger:
# setup logger
self.logger = get_logger(self.module_name, self.log_file)
self.logger.info("{} initialized".format(self.log_name))
# set host type
hostname = socket.gethostname()
if hostname == MASTER:
self.host_type = 'master'
elif hostname in WORKERS:
self.host_type = 'worker'
else:
if not no_logger:
self.logger.warning("Running on unknown host")
self.host_type = None
def __init__(self, source_queue, *args, config_file=CONFIG_FILE, control_queue=None, **kwargs):
"""
:param Queue source_queue: Input queue for controlling this service
:param str config_file: Path to config file
:param Queue control_queue: Control queue of parent Process
"""
super(VOEventGenerator, self).__init__()
self.stop_event = mp.Event()
self.input_queue = source_queue
self.control_queue = control_queue
self.voevent_server = None
with open(config_file, 'r') as f:
config = yaml.load(f, Loader=yaml.SafeLoader)['voevent_generator']
# set config, expanding strings
kwargs = {'home': os.path.expanduser('~'), 'hostname': socket.gethostname()}
for key, value in config.items():
if isinstance(value, str):
value = value.format(**kwargs)
setattr(self, key, value)
# setup logger
self.logger = get_logger(__name__, self.log_file)
# create and cd to voevent directory
try:
util.makedirs(self.voevent_dir)
except Exception as e:
self.logger.error("Cannot create voevent directory: {}".format(e))
raise VOEventGeneratorException("Cannot create voevent directory")
os.chdir(self.voevent_dir)
# Initalize the queue server
voevent_queue = mp.Queue()
VOEventQueueServer.register('get_queue', callable=lambda: voevent_queue)
self.voevent_queue = voevent_queue
self.logger.info("VOEvent Generator initialized")
