def get_logger(name, log_file, level=logging.INFO):
"""
Create logger
:param str name: name to use in log prints
:param str log_file: Path to log file
:param int level: log level (default: logging.INFO)
:return: logger
"""
# create log dir
log_dir = os.path.dirname(os.path.realpath(log_file))
try:
util.makedirs(log_dir)
except Exception as e:
sys.stderr.write("Could not create log directory: {}\n".format(e))
# setup logger
logger = logging.getLogger(name)
handler = logging.handlers.WatchedFileHandler(log_file)
formatter = logging.Formatter(
'%(asctime)s.%(levelname)s.%(name)s: %(message)s')
handler.setFormatter(formatter)
logger.setLevel(level)
# remove any old handlers
for h in copy(logger.handlers):
h.close()
logger.removeHandler(h)
logger.addHandler(handler)
logger.propagate = False
return logger
def _run_calibration_tools(self, source, obs_config):
"""
:param str source: Source name (like 3C296, *not* 3C286drift0107)
:param dict obs_config: Observation config
"""
# init settings for calibration tools script
kwargs = {
'source': source,
'calibration_tools': self.calibration_tools
}
# get number of dishes
# stored as string in parset, using a comma-separated format
dishes = obs_config['parset']['task.telescopes']
kwargs['ndish'] = dishes.count(',') + 1
# create output directories
prefix = '{}_{}'.format(obs_config['date'], source)
output_dir = os.path.join(self.calibration_dir, prefix)
for sub_dir in ['plots', 'data']:
util.makedirs(os.path.join(output_dir, sub_dir))
kwargs['output_dir'] = output_dir
# find full path to cp, to avoid using user alias like "cp -i"
kwargs['cp'] = which('cp')
# run in filterbank dir
os.chdir("{output_dir}/filterbank".format(**obs_config))
# define command (python2!)
cmd = "(nice python2 {calibration_tools} --Ndish {ndish} --src {source} CB??_??.fil;" \
" {cp} *.pdf {output_dir}/plots; {cp} *.npy {output_dir}/data) &".format(**kwargs)
self.logger.info("Running {}".format(cmd))
os.system(cmd)
def _load_config(self):
"""
Load configuration file
"""
with open(self.config_file, 'r') as f:
config = yaml.load(f, Loader=yaml.SafeLoader)['darc_master']
# set config, expanding strings
kwargs = {'home': os.path.expanduser('~'), 'hostname': self.hostname}
for key, value in config.items():
if isinstance(value, str):
value = value.format(**kwargs)
setattr(self, key, value)
# create main log dir
log_dir = os.path.dirname(self.log_file)
try:
util.makedirs(log_dir)
except Exception as e:
raise DARCMasterException(
"Cannot create log directory: {}".format(e))
# setup logger
self.logger = get_logger(__name__, self.log_file)
# store services
if self.hostname == MASTER:
if self.mode == 'real-time':
self.services = self.services_master_rt
elif self.mode == 'mixed':
self.services = self.services_master_mix
else:
self.services = self.services_master_off
elif self.hostname in WORKERS:
if self.mode == 'real-time':
self.services = self.services_worker_rt
elif self.mode == 'mixed':
self.services = self.services_worker_mix
else:
self.services = self.services_worker_off
else:
self.services = []
# terminate any existing threads in case this is a reload
for service, thread in self.threads.items():
if thread is not None:
self.logger.warning(
f"Config reload, terminating existing {service}")
thread.terminate()
# start with empty thread for each service
for service in self.services:
self.threads[service] = None
# print config file path
self.logger.info("Loaded config from {}".format(self.config_file))
def processing_status_generator(self):
"""
At regular interval, create status file for processing website
"""
self.logger.info("Starting processing status file generator")
# create the output directory if it does not exist
util.makedirs(self.processing_status_path)
hostname = socket.gethostname()
out_file = os.path.join(self.processing_status_path, f"{hostname}.js")
while not self.stop_event.is_set():
# get list of taskids that are being processed
taskids = sorted(self.observations.keys())
times = []
if not taskids:
# nothing is running
status = "idle"
else:
status = "running"
now = Time.now()
for taskid in taskids:
# check elapsed time
processing_time = now - self.observation_end_times[taskid]
# if negative, the observation is still running
if processing_time.sec < 0:
times.append('observing')
else:
# format as hh:mm:ss
full_min, seconds = divmod(processing_time.sec, 60)
hours, minutes = divmod(full_min, 60)
times.append(
f"{hours:02.0f}h{minutes:02.0f}m{seconds:02.0f}s")
content = dedent(f"""
var {hostname} = {{
"node_name": "{hostname}",
"node_status": "{status}",
"node_process": "{','.join(taskids)}",
"time": "{','.join(times)}"
}};
""")
with open(out_file, 'w') as f:
f.write(content)
self.stop_event.wait(self.processing_status_generator_interval)
# upon exit, create file to indicate node is offline
content = dedent(f"""
var {hostname} = {{
"node_name": "{hostname}",
"node_status": "offline",
"node_process": "",
"time": ""
}};
""")
with open(out_file, 'w') as f:
f.write(content)
f.flush()
def __init__(self, source_queue, *args, config_file=CONFIG_FILE, control_queue=None, **kwargs):
"""
:param Queue source_queue: Input queue
:param str config_file: Path to config file
:param Queue control_queue: Control queue of parent Process
"""
super(StatusWebsite, self).__init__()
self.stop_event = mp.Event()
self.source_queue = source_queue
self.control_queue = control_queue
# load config, including master for list of services
with open(config_file, 'r') as f:
config_all = yaml.load(f, Loader=yaml.SafeLoader)
config = config_all['status_website']
config_master = config_all['darc_master']
# set config, expanding strings
kwargs = {'home': os.path.expanduser('~'), 'hostname': socket.gethostname()}
for key, value in config.items():
if isinstance(value, str):
value = value.format(**kwargs)
setattr(self, key, value)
# set services
if config_master['mode'] == 'real-time':
self.services_master = config_master['services_master_rt']
self.services_worker = config_master['services_worker_rt']
elif config_master['mode'] == 'mixed':
self.services_master = config_master['services_master_mix']
self.services_worker = config_master['services_worker_mix']
else:
self.services_master = config_master['services_master_off']
self.services_worker = config_master['services_worker_off']
# store all node names
self.all_nodes = [MASTER] + WORKERS
# setup logger
self.logger = get_logger(__name__, self.log_file)
self.command_checker = None
# reduce logging from status check commands
logging.getLogger('darc.control').setLevel(logging.ERROR)
# create website directory
try:
util.makedirs(self.web_dir)
except Exception as e:
self.logger.error("Failed to create website directory: {}".format(e))
raise StatusWebsiteException("Failed to create website directory: {}".format(e))
self.logger.info('Status website initialized')
def _start_observation_master(self, obs_config, reload=True):
"""
Start observation on master node
Generated observation summary and send email once all workers are done
:param dict obs_config: Observation config
:param bool reload: reload service settings (default: True)
"""
# reload config
if reload:
self.load_config()
if not self.full_processing_enabled:
self.logger.info("Full processing disabled - nothing to run on master node")
return
self.logger.info("Starting observation on master node")
# create result dir
try:
util.makedirs(obs_config['result_dir'])
except Exception as e:
self.logger.error("Failed to create results directory")
raise OfflineProcessingException("Failed to create result directory: {}".format(e))
# create general info file
self._get_overview(obs_config)
# create coordinates file
self._get_coordinates(obs_config)
# wait until end time + delay
start_processing_time = Time(obs_config['parset']['task.stopTime']) + TimeDelta(self.delay, format='sec')
self.logger.info("Sleeping until {}".format(start_processing_time.iso))
util.sleepuntil_utc(start_processing_time, event=self.stop_event)
# start emailer
cmd = "python2 {emailer} {result_dir} '{beams}' {ntabs}".format(**obs_config)
self.logger.info("Running {}".format(cmd))
os.system(cmd)
# fetch known FRB candidates
# do this _after_ email is sent, as then we are sure the grouped_pulses file exists for all beams
# NOTE: disabled because:
# 1) the (external) plotting script does not work without an active X session (should be fixable)
# 2) the script re-uses the same temporary directory, and can run only one copy at a time
# While these issues are not fixed, the plotting is not run automatically
# if coord_cb00 is not None:
# self._plot_known_frb_cands(obs_config, coord_cb00)
# else:
# self.logger.warning("Skipping plotting of known FRB candidates: CB00 coordinates not available")
self.logger.info("Finished processing of observation {output_dir}".format(**obs_config))
def _start_observation_master(self, obs_config, reload=True):
"""
Start observation on master node
Generated observation summary and send email once all workers are done
:param dict obs_config: Observation config
:param bool reload: reload service settings (default: True)
"""
self.logger.info("Starting observation on master node")
# reload config
if reload:
self.load_config()
# create result dir
try:
util.makedirs(obs_config['result_dir'])
except Exception as e:
self.logger.error("Failed to create results directory")
raise OfflineProcessingException(
"Failed to create result directory: {}".format(e))
# create general info file
self._get_overview(obs_config)
# create coordinates file
coord_cb00 = self._get_coordinates(obs_config)
# wait until end time + delay
start_processing_time = Time(
obs_config['parset']['task.stopTime']) + TimeDelta(self.delay,
format='sec')
self.logger.info("Sleeping until {}".format(start_processing_time.iso))
util.sleepuntil_utc(start_processing_time, event=self.stop_event)
# start emailer
cmd = "python2 {emailer} {result_dir} '{beams}' {ntabs}".format(
**obs_config)
self.logger.info("Running {}".format(cmd))
os.system(cmd)
# fetch known FRB candidates
# do this _after_ email is sent, as then we are sure the grouped_pulses file exists for all beams
if coord_cb00 is not None:
self._plot_known_frb_cands(obs_config, coord_cb00)
else:
self.logger.warning(
"Skipping plotting of known FRB candidates: CB00 coordinates not available"
)
self.logger.info(
"Finished processing of observation {output_dir}".format(
**obs_config))
def setUp(self):
self.output_dir = '/data/arts/darc/output/triggers_realtime'
self.result_dir = '/data/arts/darc/output/central'
# ensure we start clean
try:
rmtree(self.result_dir)
except FileNotFoundError:
pass
util.makedirs(self.result_dir)
for fname in glob.glob(os.path.join(self.output_dir, '*.pdf')):
os.remove(fname)
def __init__(self,
obs_config,
output_dir,
log_queue,
input_queue,
output_queue,
ncand_above_threshold,
config_file=CONFIG_FILE,
obs_name=''):
"""
:param dict obs_config: Observation settings
:param str output_dir: Output directory for data products
:param Queue log_queue: Queue to use for logging
:param Queue input_queue: Input queue for clusters
:param Queue output_queue: Output queue for classifier
:param mp.Value ncand_above_threshold: 0
:param str config_file: Path to config file
:param str obs_name: Observation name to use in log messages
"""
super(Extractor, self).__init__()
module_name = type(self).__module__.split('.')[-1]
self.logger = get_queue_logger(module_name, log_queue)
self.output_dir = os.path.join(output_dir, 'data')
self.obs_config = obs_config
self.input_queue = input_queue
self.output_queue = output_queue
self.obs_name = obs_name
# load config
self.config_file = config_file
self.config = self._load_config()
# create directory for output data
util.makedirs(self.output_dir)
# create stop event
self.stop_event = mp.Event()
self.input_empty = False
self.filterbank_reader = None
self.rfi_mask = np.array([], dtype=int)
self.data = None
self.data_dm_time = None
self.ncand_above_threshold = ncand_above_threshold
def _publish_results(self, body, files):
"""
Publish email content as local website
"""
# create output folder
web_folder = '{home}/public_html/darc/{webdir}/{date}/{datetimesource}'.format(
webdir=self.webdir, **self.obs_config)
util.makedirs(web_folder)
# save the email body, ensuring it is at the top of the list in a browser
with open(os.path.join(web_folder, 'A_info.html'), 'w') as f:
f.write(body)
# create symlinks to PDFs. These are outside the public_html folder, but they are readable as long as they
# are owned by the same user
for src in files:
dest = os.path.join(web_folder, os.path.basename(src['path']))
os.symlink(src['path'], dest)
self.logger.info(
f"Published results of {self.obs_config['datetimesource']}")
def get_queue_logger_listener(queue, log_file):
"""
Create thread that logs message from a queue
:param Queue queue: Log queue
:param str log_file: Path to log file
:return: QueueListener thread
"""
# create log dir
log_dir = os.path.dirname(os.path.realpath(log_file))
try:
util.makedirs(log_dir)
except Exception as e:
sys.stderr.write("Could not create log directory: {}\n".format(e))
# setup handler
handler = logging.handlers.WatchedFileHandler(log_file)
formatter = logging.Formatter('%(asctime)s.%(levelname)s.%(name)s: %(message)s')
handler.setFormatter(formatter)
listener = logging.handlers.QueueListener(queue, handler)
return listener
def setUp(self):
self.output_dir = '/data/arts/darc/output/triggers_realtime'
self.result_dir = '/data/arts/darc/output/central'
# ensure we start clean
try:
rmtree(self.result_dir)
except FileNotFoundError:
pass
util.makedirs(self.result_dir)
for fname in glob.glob(os.path.join(self.output_dir, '*.pdf')):
os.remove(fname)
# create log handler
log_queue = mp.Queue()
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s.%(levelname)s.%(name)s: %(message)s')
handler.setFormatter(formatter)
self.ql = logging.handlers.QueueListener(log_queue, handler)
self.ql.start()
self.log_queue = log_queue
def __init__(self,
obs_config,
output_dir,
logger,
input_queue,
output_queue,
ncand_above_threshold,
config_file=CONFIG_FILE):
"""
:param dict obs_config: Observation settings
:param str output_dir: Output directory for data products
:param Logger logger: Processor logger object
:param Queue input_queue: Input queue for clusters
:param Queue output_queue: Output queue for classifier
:param mp.Value ncand_above_threshold: 0
:param str config_file: Path to config file
"""
super(Extractor, self).__init__()
self.logger = logger
self.output_dir = os.path.join(output_dir, 'data')
self.obs_config = obs_config
self.input_queue = input_queue
self.output_queue = output_queue
# load config
self.config_file = config_file
self.config = self._load_config()
# create directory for output data
util.makedirs(self.output_dir)
# create stop event
self.stop_event = mp.Event()
self.input_empty = False
self.filterbank_reader = None
self.rfi_mask = np.array([], dtype=int)
self.data = None
self.data_dm_time = None
self.ncand_above_threshold = ncand_above_threshold
def __init__(self, source_queue, *args, config_file=CONFIG_FILE, control_queue=None, **kwargs):
"""
:param Queue source_queue: Input queue for controlling this service
:param str config_file: Path to config file
:param Queue control_queue: Control queue of parent Process
"""
super(VOEventGenerator, self).__init__()
self.stop_event = mp.Event()
self.input_queue = source_queue
self.control_queue = control_queue
self.voevent_server = None
with open(config_file, 'r') as f:
config = yaml.load(f, Loader=yaml.SafeLoader)['voevent_generator']
# set config, expanding strings
kwargs = {'home': os.path.expanduser('~'), 'hostname': socket.gethostname()}
for key, value in config.items():
if isinstance(value, str):
value = value.format(**kwargs)
setattr(self, key, value)
# setup logger
self.logger = get_logger(__name__, self.log_file)
# create and cd to voevent directory
try:
util.makedirs(self.voevent_dir)
except Exception as e:
self.logger.error("Cannot create voevent directory: {}".format(e))
raise VOEventGeneratorException("Cannot create voevent directory")
os.chdir(self.voevent_dir)
# Initalize the queue server
voevent_queue = mp.Queue()
VOEventQueueServer.register('get_queue', callable=lambda: voevent_queue)
self.voevent_queue = voevent_queue
self.logger.info("VOEvent Generator initialized")
def start_observation(self, obs_config, reload=True):
"""
Parse obs config and start listening for amber triggers on queue
:param dict obs_config: Observation configuration
:param bool reload: reload service settings (default: True)
"""
# reload config
if reload:
self.load_config()
# add observation-specific path to result_dir
self.central_result_dir = os.path.join(self.result_dir,
obs_config['date'],
obs_config['datetimesource'])
util.makedirs(self.central_result_dir)
self.obs_config = obs_config
# process the observation in a separate Process
self.process = mp.Process(target=self._process_observation)
self.process.start()
def start_observation(self, obs_config, reload=True):
"""
Parse obs config and start observation processing after end time has passed
:param dict obs_config: Observation configuration
:param bool reload: reload service settings (default: True)
"""
# reload config
if reload:
self.load_config()
# add observation-specific path to result_dir
self.central_result_dir = os.path.join(self.result_dir,
obs_config['date'],
obs_config['datetimesource'])
util.makedirs(self.central_result_dir)
self.obs_config = obs_config
# process the observation in a separate thread (not a process, as then we can't stop it directly
# through the stop event of ProcessorMaster; and no other processing happens anyway)
self.process = threading.Thread(target=self._process_observation)
self.process.start()
def start_observation(self, obs_config, reload=True):
"""
Parse obs config and start listening for amber triggers on queue
:param dict obs_config: Observation configuration
:param bool reload: reload service settings (default: True)
"""
# reload config
if reload:
self.load_config()
# store obs name for logging
self.obs_name = f"{obs_config['parset']['task.taskID']} - {obs_config['datetimesource']}: "
# clean any old triggers
self.amber_triggers = []
# set config
self.obs_config = obs_config
# add observation-specific path to result_dir
self.central_result_dir = os.path.join(self.result_dir,
obs_config['date'],
obs_config['datetimesource'])
# create output dir
output_dir = os.path.join('{output_dir}'.format(**obs_config),
self.output_subdir)
for path in (output_dir, self.central_result_dir):
try:
util.makedirs(path)
except Exception as e:
self.logger.error(
f"{self.obs_name}Failed to create directory {path}: {e}")
raise ProcessorException(
f"Failed to create directory {path}: {e}")
self.output_dir = output_dir
self.observation_running = True # this must be set before starting the processing thread
# start processing
thread = threading.Thread(target=self._read_and_process_data,
name='processing')
self.threads['processing'] = thread
# start clustering
thread = Clustering(obs_config, output_dir, self.log_queue,
self.clustering_queue, self.extractor_queue,
self.ncluster, self.config_file, self.obs_name)
thread.name = 'clustering'
self.threads['clustering'] = thread
# start extractor(s)
for i in range(self.num_extractor):
thread = Extractor(obs_config, output_dir, self.log_queue,
self.extractor_queue, self.classifier_queue,
self.ncand_above_threshold, self.config_file,
self.obs_name)
thread.name = f'extractor_{i}'
self.threads[f'extractor_{i}'] = thread
# start classifier
thread = Classifier(self.log_queue, self.classifier_queue,
self.classifier_child_conn, self.config_file,
self.obs_name)
thread.name = 'classifier'
self.threads['classifier'] = thread
# start all threads/processes
for thread in self.threads.values():
thread.start()
# If this is reprocessing instead of a new observation, read the AMBER triggers
# Reprocessing is assumed if the end time is in the past
utc_start = Time(obs_config['startpacket'] / TIME_UNIT, format='unix')
utc_end = utc_start + TimeDelta(obs_config['duration'], format='sec')
if utc_end < Time.now():
self.logger.info(
f"{self.obs_name}End time is in the past, reading AMBER triggers for reprocessing"
)
thread = threading.Thread(target=self._read_amber_triggers,
name='read_amber_triggers')
thread.daemon = True
thread.start()
self.reprocessing = True
else:
self.reprocessing = False
self.logger.info(f"{self.obs_name}Observation started")
def start_observation(self, config_file, service=None):
"""
Start an observation
:param str config_file: Path to observation config file
:param str service: Which service to send start_observation to (default: all)
:return: status, reply
"""
self.logger.info(
"Received start_observation command with config file {}".format(
config_file))
# check if config file exists
if not os.path.isfile(config_file):
self.logger.error("File not found: {}".format(config_file))
return "Error", "Failed: config file not found"
# load config
if config_file.endswith('.yaml'):
config = self._load_yaml(config_file)
elif config_file.endswith('.parset'):
config = self._load_parset(config_file)
else:
self.logger.error(
"Failed to determine config file type from {}".format(
config_file))
return "Error", "Failed: unknown config file type"
# check if process_triggers is enabled
if not config['proctrigger']:
self.logger.info(
"Process triggers is disabled; not starting observation")
return "Success", "Process triggers disabled - not starting"
# store the config for future reference
config_output_dir = os.path.join(self.parset_dir,
config['datetimesource'])
try:
util.makedirs(config_output_dir)
except Exception as e:
raise DARCMasterException(
"Cannot create config output directory: {}".format(e))
try:
copy2(config_file, config_output_dir)
except Exception as e:
self.logger.error("Could not store config file: {}".format(e))
# initialize observation
# ensure services are running
if service is None:
for s in self.services:
self.start_service(s)
else:
self.start_service(service)
# check host type
if self.hostname == MASTER:
host_type = 'master'
elif self.hostname in WORKERS:
host_type = 'worker'
else:
self.logger.error("Running on unknown host: {}".format(
self.hostname))
return "Error", "Failed: running on unknown host"
# create command
command = {
'command': 'start_observation',
'obs_config': config,
'host_type': host_type
}
# wait until start time
utc_start = Time(config['startpacket'] / TIME_UNIT, format='unix')
utc_end = utc_start + TimeDelta(config['duration'], format='sec')
# if end time is in the past, only start offline processing and processor
if utc_end < Time.now():
self.logger.warning(
"End time in past! Only starting offline processing and/or processor"
)
if service is None:
self.offline_queue.put(command)
self.processor_queue.put(command)
return "Warning", "Only offline processing and processor started"
elif service == 'offline_processing':
self.offline_queue.put(command)
return "Warning", "Only offline processing started"
elif service == 'processor':
self.processor_queue.put(command)
return "Warning", "Only processor started"
else:
return "Error", "Can only start offline processing and processor when end time is in past"
t_setup = utc_start - TimeDelta(self.setup_time, format='sec')
self.logger.info("Starting observation at {}".format(t_setup.isot))
util.sleepuntil_utc(t_setup)
if service is None:
# clear queues, then send command
for queue in self.all_queues:
util.clear_queue(queue)
for queue in self.all_queues:
queue.put(command)
return "Success", "Observation started"
else:
# only start specified service. As this is only used in case e.g. something fails during
# an observation, do not clear the queues first
queue = self.get_queue(service)
queue.put(command)
return "Warning", "Only observation for {} started".format(service)
def _start_observation_worker(self, obs_config, reload=True):
"""
Start observation on worker node:
#. Candidate clustering
#. Extraction of filterbank data
#. ML classification
:param dict obs_config: Observation config
:param bool reload: reload service settings (default: True)
"""
self.logger.info("Starting observation on worker node")
# reload config
if reload:
self.load_config()
# create result dir
try:
util.makedirs(obs_config['result_dir'])
except Exception as e:
self.logger.error("Failed to create results directory")
raise OfflineProcessingException(
"Failed to create result directory: {}".format(e))
# TAB or IAB mode
if obs_config['ntabs'] == 1:
obs_config['mode'] = 'IAB'
trigger_output_file = "{output_dir}/triggers/data/data_00_full.hdf5".format(
**obs_config)
else:
obs_config['mode'] = 'TAB'
trigger_output_file = "{output_dir}/triggers/data/data_full.hdf5".format(
**obs_config)
# wait until end time + delay
start_processing_time = Time(
obs_config['parset']['task.stopTime']) + TimeDelta(self.delay,
format='sec')
self.logger.info("Sleeping until {}".format(start_processing_time.iso))
util.sleepuntil_utc(start_processing_time, event=self.stop_event)
# fold pulsar if this is beam 0 and a test pulsar is being observed
try:
source = obs_config['parset']['task.source.name']
ref_beam = int(obs_config['parset']['task.source.beam'])
# Requires access to parset
if source in self.test_pulsars and (obs_config['beam']
== ref_beam):
self.logger.info("Test pulsar detected: {}".format(source))
self._fold_pulsar(source, obs_config)
except Exception as e:
self.logger.error("Pulsar folding failed: {}".format(e))
# run calibration tools if this is a calibrator scan
# these have "drift" in the source name
try:
source = obs_config['parset']['task.source.name']
if 'drift' in source:
# split into actual source name and which beams were scanned
name, beam_range = source.split('drift')
# parse beam range, can be one beam or start/end beam
if len(beam_range) == 2:
# one beam
drift_beams = [int(beam_range)]
elif len(beam_range) == 4:
# start and end beam
sbeam = int(beam_range[:2])
ebeam = int(beam_range[2:])
drift_beams = range(sbeam, ebeam + 1)
else:
self.logger.error(
"Failed to parse beam range for calibrator scan: {}".
format(source))
drift_beams = []
# run calibration tools if this is a calibrator scan of this beam
if name in self.calibrators and (obs_config['beam']
in drift_beams):
self.logger.info(
"Calibrator scan through this beam detected: {}".
format(source))
self._run_calibration_tools(name, obs_config)
except Exception as e:
self.logger.error("Calibration tools failed: {}".format(e))
# create trigger directory
trigger_dir = "{output_dir}/triggers".format(**obs_config)
try:
util.makedirs(trigger_dir)
except Exception as e:
self.logger.error("Failed to create triggers directory")
raise OfflineProcessingException(
"Failed to create triggers directory: {}".format(e))
# change to trigger directory
try:
os.chdir(trigger_dir)
except Exception as e:
self.logger.error(
"Failed to cd to triggers directory {}: {}".format(
trigger_dir, e))
# create subdir
try:
util.makedirs('data')
except Exception as e:
self.logger.error("Failed to create triggers/data directory")
raise OfflineProcessingException(
"Failed to create triggers/data directory: {}".format(e))
# merge the trigger files
self.logger.info("Merging raw trigger files")
numcand_raw = self._merge_triggers(obs_config)
# Run clustering for each SB in TAB mode if enabled
if self.process_sb and obs_config['mode'] == 'TAB':
self.logger.info("Clustering candidates in SB mode")
tstart = Time.now()
# spread the SBs over the allowed number of threads
# output grouped pulses file will only be generated by thread including SB00
chunks = np.array_split(range(obs_config['nsynbeams']),
self.numthread)
numcand_all = np.zeros(self.numthread)
filterbank_prefix = "{output_dir}/filterbank/CB{beam:02d}".format(
**obs_config)
threads = []
self.logger.info(
"Starting trigger clustering with {} threads".format(
self.numthread))
for ind, chunk in enumerate(chunks):
# pick the SB range
sbmin, sbmax = min(chunk), max(chunk)
# create thread
thread = threading.Thread(target=self._cluster,
args=[obs_config, filterbank_prefix],
kwargs={
'out': numcand_all,
'sbmin': sbmin,
'sbmax': sbmax,
'ind': ind
})
thread.daemon = True
threads.append(thread)
thread.start()
# wait until all are done
for thread in threads:
thread.join()
# gather results
if self.process_sb:
# each element equal
numcand_grouped = int(numcand_all[0])
else:
# each element is one TAB
numcand_grouped = int(np.sum(numcand_all[numcand_all != -1]))
# Run clustering for IAB / each TAB
else:
self.logger.info("Clustering candidates in {} mode".format(
obs_config['mode']))
tstart = Time.now()
if obs_config['mode'] == 'IAB':
filterbank_file = "{output_dir}/filterbank/CB{beam:02d}.fil".format(
**obs_config)
numcand_grouped = self._cluster(obs_config, 0, filterbank_file)
elif obs_config['mode'] == 'TAB':
numcand_all = np.zeros(obs_config['ntabs'])
# max numtread tabs per run; so ntabs / numthread chunks
n_chunk = int(
np.ceil(obs_config['ntabs'] / float(self.numthread)))
chunks = np.array_split(range(obs_config['ntabs']), n_chunk)
self.logger.info(
"Starting trigger clustering with {} chunks of {} threads".
format(n_chunk, self.numthread))
# start the threads
for tab_set in chunks:
threads = []
for tab in tab_set:
filterbank_file = "{output_dir}/filterbank/CB{beam:02d}_{tab:02d}.fil".format(
tab=tab, **obs_config)
thread = threading.Thread(
target=self._cluster,
args=[obs_config, filterbank_file],
kwargs={
'out': numcand_all,
'tab': tab
})
thread.daemon = True
threads.append(thread)
thread.start()
# wait until all are done
for thread in threads:
thread.join()
# gather results
numcand_grouped = int(np.sum(numcand_all[numcand_all != -1]))
tend = Time.now()
self.logger.info("Trigger clustering took {}s".format(
(tend - tstart).sec))
# Create one hdf5 file for entire CB
if obs_config['mode'] == 'TAB':
self.logger.info("Merging output HDF5 files")
numcand_merged = self._merge_hdf5(obs_config, trigger_output_file)
# TEMP so that IAB still works
else:
numcand_merged = 9999
# Run classifier
if numcand_merged != 0:
self.logger.info("Classifying candidates")
output_prefix = self._classify(obs_config, trigger_output_file)
else:
self.logger.info(
"No candidates post-merge. Not running classifier")
output_prefix = ''
# Merge PDFs
if numcand_merged > 0:
self.logger.info("Merging classifier output files")
self._merge_plots(obs_config)
else:
self.logger.info(
"No candidates found post-classifier, not creating merged PDF")
# Centralize results
self.logger.info("Gathering results")
kwargs = {
'output_prefix': output_prefix,
'data_file': trigger_output_file,
'numcand_raw': numcand_raw,
'numcand_grouped': numcand_grouped
}
self._gather_results(obs_config, **kwargs)
self.logger.info(
"Finished processing of observation {output_dir}".format(
**obs_config))
return
def setUp(self):
"""
Set up the pipeline and observation software
"""
print("Setup")
# observation settings
files = glob.glob('/tank/data/sky/B1933+16/20200211_dump/dada/*.dada')
self.assertTrue(len(files) > 0)
output_dir = '/tank/users/oostrum/test_full_run'
amber_dir = os.path.join(output_dir, 'amber')
amber_conf_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'amber.conf')
amber_conf_dir = '/home/arts/.controller/amber_conf'
# ensure we start clean
rmtree(amber_dir)
util.makedirs(amber_dir)
# load the encoded parset
with open('/tank/data/sky/B1933+16/20200211_dump/parset', 'r') as f:
parset = f.read().strip()
# nreader: one for each programme reading from the buffer, i.e. 3x AMBER
self.settings = {'resolution': 1536 * 12500 * 12, 'nbuf': 5, 'key_i': 'aaaa',
'hdr_size': 40960, 'dada_files': files, 'nreader': 3,
'freq': 1370, 'amber_dir': amber_dir, 'nbatch': len(files) * 10,
'beam': 0, 'amber_config': amber_conf_file, 'amber_conf_dir': amber_conf_dir,
'min_freq': 1219.70092773, 'parset': parset, 'datetimesource': '2019-01-01-00:00:00.FAKE'}
# open custom config file
self.config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.yaml')
with open(self.config_file, 'r') as f:
self.master_config = yaml.load(f, Loader=yaml.SafeLoader)['darc_master']
# create buffers
# stokes I
print("Creating buffers")
os.system('dada_db -d -k {key_i} 2>/dev/null ; dada_db -k {key_i} -a {hdr_size} '
'-b {resolution} -n {nbuf} -r {nreader}'.format(**self.settings))
# init GPU pipeline thread
self.t_amber = threading.Thread(target=self.amber, name='AMBER', daemon=True)
# init amber listener thread
self.listener = AMBERListener()
self.listener.set_source_queue(mp.Queue())
self.listener.set_target_queue(mp.Queue())
self.listener.start()
# init amber clustering thread
# do not connect to VOEvent server nor LOFAR trigger system
self.clustering = AMBERClustering(connect_vo=False, connect_lofar=False)
self.clustering.set_source_queue(self.listener.target_queue)
self.clustering.set_target_queue(mp.Queue())
self.clustering.start()
# init DADA trigger thread
self.dadatrigger = DADATrigger()
self.dadatrigger.set_source_queue(self.clustering.target_queue)
self.dadatrigger.start()
# init writer thread
self.t_disk_to_db = threading.Thread(target=self.writer, name='disk_to_db', daemon=True)
# init output listener thread
self.event_queue = mp.Queue()
self.t_dbevent = threading.Thread(target=self.dbevent, name='dbevent', daemon=True,
args=(self.event_queue,))
def setUp(self):
if socket.gethostname() == 'zeus':
self.dada_files = glob.glob('/data/arts/data/dada/*.dada')[:1]
self.dada_files.sort()
output_dir = '/data/arts/darc/output'
log_dir = f'{output_dir}/log'
filterbank_dir = f'{output_dir}/filterbank'
amber_dir = f'{output_dir}/amber'
amber_conf_dir = '/data/arts/darc/amber_conf'
amber_conf_file = '/data/arts/darc/amber.conf'
sb_table = '/data/arts/darc/sbtable-sc4-12tabs-71sbs.txt'
else:
self.dada_files = glob.glob('/tank/data/sky/B1933+16/20200211_dump/dada/*.dada')[:1]
self.dada_files.sort()
user = os.getlogin()
if user == 'oostrum':
main_dir = '/tank/users/oostrum/darc/automated_testing'
elif user == 'arts':
main_dir = '/tank/users/arts/darc_automated_testing'
else:
self.skipTest(f"Cannot run test as user {user}")
output_dir = f'{main_dir}/output'
log_dir = f'{output_dir}/log'
filterbank_dir = f'{output_dir}/filterbank'
amber_dir = f'{output_dir}/amber'
amber_conf_dir = f'{main_dir}/amber/amber_conf'
amber_conf_file = f'{main_dir}/amber/amber.conf'
sb_table = '/home/arts/.controller/synthesized_beam_tables/sbtable-sc4-12tabs-71sbs.txt'
# ensure we start clean
try:
rmtree(output_dir)
except FileNotFoundError:
pass
for d in (output_dir, log_dir, amber_dir, filterbank_dir):
util.makedirs(d)
# create log handler
log_queue = mp.Queue()
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s.%(levelname)s.%(name)s: %(message)s')
handler.setFormatter(formatter)
self.ql = logging.handlers.QueueListener(log_queue, handler)
self.ql.start()
self.processes = {}
# extract PSRDADA header
self.header = self.get_psrdada_header(self.dada_files[0])
self.tstart = Time.now() + TimeDelta(5, format='sec')
# add general settings
self.header['nreader'] = 2
self.header['nbuffer'] = 5
self.header['key_i'] = '5000'
self.header['beam'] = 0
self.header['ntab'] = 12
self.header['nsb'] = 71
# self.header['nbatch'] = int(float(self.header['SCANLEN']) / 1.024)
self.header['nbatch'] = 10
self.header['duration'] = float(self.header['SCANLEN'])
self.header['log_dir'] = log_dir
self.header['output_dir'] = output_dir
self.header['filterbank_dir'] = filterbank_dir
self.header['amber_dir'] = amber_dir
self.header['amber_conf_dir'] = amber_conf_dir
self.header['amber_config'] = amber_conf_file
self.header['sb_table'] = sb_table
self.header['date'] = '20200101'
self.header['datetimesource'] = '2020-01-01-00:00:00.FAKE'
self.header['freq'] = int(np.round(float(self.header['FREQ'])))
self.header['snrmin'] = 8
self.header['min_freq'] = 1220.7
self.header['startpacket'] = int(self.tstart.unix * TIME_UNIT)
# add parset
parset = {'task.duration': str(self.header['SCANLEN']),
'task.startTime': self.tstart.isot,
'task.taskID': '001122',
'task.beamSet.0.compoundBeam.0.phaseCenter': '[293.94876deg, 16.27778deg]',
'task.directionReferenceFrame': 'HADEC'}
self.header['parset'] = parset
# create ringbuffer
self.create_ringbuffer()
# create processes for the different pipeline steps
self.diskdb_proc = self.diskdb_command()
self.amber_proc = self.amber_command()
self.dadafilterbank_proc = self.dadafilterbank_command()
# start all except data reader
self.dadafilterbank_proc.start()
self.amber_proc.start()
# initialize AMBERListener, used for feeding triggers to Processor
self.amber_queue = mp.Queue()
self.processor_queue = mp.Queue()
self.amber_listener = AMBERListener(self.amber_queue, target_queue=self.processor_queue)
self.amber_listener.start()
# initialize Processor, connect input queue to output of AMBERListener
self.processor = Processor(log_queue=log_queue, source_queue=self.processor_queue)
def setUp(self):
if socket.gethostname() == 'zeus':
self.result_dir = '/data/arts/darc/central_dir_processor_master'
else:
self.result_dir = '/tank/users/arts/darc_automated_testing/processor_master'
self.processor_queue = mp.Queue()
self.processor = ProcessorMaster(self.processor_queue)
tstart = Time.now()
duration = TimeDelta(5, format='sec')
parset = {
'task.duration': duration.sec,
'task.startTime': tstart.isot,
'task.stopTime': (tstart + duration).isot,
'task.source.name': 'FAKE',
'task.taskID': '001122',
'task.beamSet.0.compoundBeam.0.phaseCenter':
'[293.94876deg, 16.27778deg]',
'task.directionReferenceFrame': 'J2000',
'task.telescopes': '[RT2, RT3, RT4, RT5, RT6, RT7, RT8, RT9]'
}
self.obs_config = {
'beams': [0],
'freq': 1370,
'startpacket': int(tstart.unix * TIME_UNIT),
'date': '20200101',
'datetimesource': '2020-01-01-00:00:00.FAKE',
'result_dir': self.result_dir,
'parset': parset
}
# output directory
output_dir = os.path.join(self.result_dir, self.obs_config['date'],
self.obs_config['datetimesource'])
# web directory
webdir = '{home}/public_html/darc/{webdir}/{date}/{datetimesource}'.format(
home=os.path.expanduser('~'),
webdir=self.processor.webdir,
**self.obs_config)
# remove existing output
for d in (output_dir, webdir):
try:
shutil.rmtree(d)
except FileNotFoundError:
pass
util.makedirs(output_dir)
# create empty PDF
open(os.path.join(output_dir, 'CB00.pdf'), 'w').close()
# create trigger overview
# classifier cands must be > 0 to test processing of PDF attachments
trigger_results = {
'ncand_raw': 1000,
'ncand_post_clustering': 100,
'ncand_post_thresholds': 10,
'ncand_post_classifier': 1
}
with open(os.path.join(output_dir, 'CB00_summary.yaml'), 'w') as f:
yaml.dump(trigger_results, f, default_flow_style=False)
# create file with trigger list
with open(os.path.join(output_dir, 'CB00_triggers.txt'), 'w') as f:
f.write('#cb snr dm time downsamp sb p\n')
f.write('00 10.00 20.00 30.0000 50 35 1.00\n')
def start_observation(self, obs_config, reload=True):
"""
Parse obs config and start listening for amber triggers on queue
:param dict obs_config: Observation configuration
:param bool reload: reload service settings (default: True)
"""
# reload config
if reload:
self.load_config()
# clean any old triggers
self.amber_triggers = []
# set config
self.obs_config = obs_config
# add observation-specific path to result_dir
self.central_result_dir = os.path.join(self.result_dir,
obs_config['date'],
obs_config['datetimesource'])
# create output dir
output_dir = os.path.join('{output_dir}'.format(**obs_config),
self.output_subdir)
for path in (output_dir, self.central_result_dir):
try:
util.makedirs(path)
except Exception as e:
self.logger.error(f"Failed to create directory {path}: {e}")
raise ProcessorException(
f"Failed to create directory {path}: {e}")
self.output_dir = output_dir
self.observation_running = True # this must be set before starting the processing thread
# start processing
thread = threading.Thread(target=self._read_and_process_data,
name='processing')
thread.start()
self.threads['processing'] = thread
# start clustering
thread = Clustering(obs_config, output_dir, self.logger,
self.clustering_queue, self.extractor_queue,
self.ncluster, self.config_file)
thread.name = 'clustering'
thread.start()
self.threads['clustering'] = thread
# start extractor(s)
for i in range(self.num_extractor):
thread = Extractor(obs_config, output_dir, self.logger,
self.extractor_queue, self.classifier_queue,
self.ncand_above_threshold, self.config_file)
thread.name = f'extractor_{i}'
thread.start()
self.threads[f'extractor_{i}'] = thread
# start classifier
thread = Classifier(self.logger, self.classifier_queue,
self.classifier_child_conn, self.config_file)
thread.name = 'classifier'
thread.start()
self.threads['classifier'] = thread
self.logger.info("Observation started")
def test_worker_processing_tab(self):
# config for worker should contain:
# ntabs
# nsynbeams
# mode (IAB/TAB)
# output_dir (/data2/output/<date>/<datetimesource)
# startpacket
# beam (CB)
# amber_dir
# duration
# result_dir
# config
config = self.gen_config(sb=False)
logger = logging.getLogger('test_worker_processing_tab')
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s.%(levelname)s.%(name)s: %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
# output file names
fname_yaml = "CB00_summary.yaml"
fname_txt = "CB00_triggers.txt"
fname_txt_in = "CB00_triggers_tab.txt"
fname_pdf = "CB00_candidates_summary.pdf"
# set expected output for yaml
expected_output_yaml = {
'ncand_abovethresh': 75,
'ncand_classifier': 75,
'ncand_raw': 5923,
'ncand_skipped': 1339,
'ncand_trigger': 1414
}
# read expected output for txt
with open(os.path.join(TESTDIR, fname_txt_in)) as f:
expected_output_txt = f.read().strip().split()
# Create result dir (normally done in run method, but that is skipped)
try:
util.makedirs(config['result_dir'])
except Exception as e:
self.fail('Cannot create result dir {}: {}'.format(
config['result_dir'], e))
else:
proc = OfflineProcessing()
# override logger (first initalized message still goes to normal logger)
proc.logger = logger
# override sb processing mode
proc.process_sb = False
# override config
proc.host_type = 'worker'
proc.config['nfreq_plot'] = 32
proc.config['snrmin_processing'] = 45
proc.config['snrmin_processing_local'] = 5
proc.config['dmmin'] = 20
proc.config['dmmax'] = 5000
proc.config['pthresh_dmtime'] = 0
# Add offline processing config to obs config (normally done in run method, but that is skipped)
fullconfig = proc.config.copy()
fullconfig.update(config)
# run worker observation
try:
proc._start_observation_worker(fullconfig, reload=False)
except Exception as e:
self.fail(
"Unhandled exception in offline processing: {}".format(e))
# check if output files exist
for fname in [fname_yaml, fname_txt, fname_pdf]:
self.assertTrue(
os.path.isfile(os.path.join(config['result_dir'], fname)))
# for the yaml and txt, verify the content
with open(os.path.join(config['result_dir'], fname_yaml)) as f:
output_yaml = yaml.load(f, Loader=yaml.SafeLoader)
self.assertDictEqual(expected_output_yaml, output_yaml)
with open(os.path.join(config['result_dir'], fname_txt)) as f:
output_txt = f.read().strip().split()
self.assertListEqual(expected_output_txt, output_txt)
# remove the results dir if it exists
shutil.rmtree(config['result_dir'], ignore_errors=True)
