def get_logger(name, log_file, level=logging.INFO): """ Create logger :param str name: name to use in log prints :param str log_file: Path to log file :param int level: log level (default: logging.INFO) :return: logger """ # create log dir log_dir = os.path.dirname(os.path.realpath(log_file)) try: util.makedirs(log_dir) except Exception as e: sys.stderr.write("Could not create log directory: {}\n".format(e)) # setup logger logger = logging.getLogger(name) handler = logging.handlers.WatchedFileHandler(log_file) formatter = logging.Formatter( '%(asctime)s.%(levelname)s.%(name)s: %(message)s') handler.setFormatter(formatter) logger.setLevel(level) # remove any old handlers for h in copy(logger.handlers): h.close() logger.removeHandler(h) logger.addHandler(handler) logger.propagate = False return logger
def _run_calibration_tools(self, source, obs_config): """ :param str source: Source name (like 3C296, *not* 3C286drift0107) :param dict obs_config: Observation config """ # init settings for calibration tools script kwargs = { 'source': source, 'calibration_tools': self.calibration_tools } # get number of dishes # stored as string in parset, using a comma-separated format dishes = obs_config['parset']['task.telescopes'] kwargs['ndish'] = dishes.count(',') + 1 # create output directories prefix = '{}_{}'.format(obs_config['date'], source) output_dir = os.path.join(self.calibration_dir, prefix) for sub_dir in ['plots', 'data']: util.makedirs(os.path.join(output_dir, sub_dir)) kwargs['output_dir'] = output_dir # find full path to cp, to avoid using user alias like "cp -i" kwargs['cp'] = which('cp') # run in filterbank dir os.chdir("{output_dir}/filterbank".format(**obs_config)) # define command (python2!) cmd = "(nice python2 {calibration_tools} --Ndish {ndish} --src {source} CB??_??.fil;" \ " {cp} *.pdf {output_dir}/plots; {cp} *.npy {output_dir}/data) &".format(**kwargs) self.logger.info("Running {}".format(cmd)) os.system(cmd)
def _load_config(self): """ Load configuration file """ with open(self.config_file, 'r') as f: config = yaml.load(f, Loader=yaml.SafeLoader)['darc_master'] # set config, expanding strings kwargs = {'home': os.path.expanduser('~'), 'hostname': self.hostname} for key, value in config.items(): if isinstance(value, str): value = value.format(**kwargs) setattr(self, key, value) # create main log dir log_dir = os.path.dirname(self.log_file) try: util.makedirs(log_dir) except Exception as e: raise DARCMasterException( "Cannot create log directory: {}".format(e)) # setup logger self.logger = get_logger(__name__, self.log_file) # store services if self.hostname == MASTER: if self.mode == 'real-time': self.services = self.services_master_rt elif self.mode == 'mixed': self.services = self.services_master_mix else: self.services = self.services_master_off elif self.hostname in WORKERS: if self.mode == 'real-time': self.services = self.services_worker_rt elif self.mode == 'mixed': self.services = self.services_worker_mix else: self.services = self.services_worker_off else: self.services = [] # terminate any existing threads in case this is a reload for service, thread in self.threads.items(): if thread is not None: self.logger.warning( f"Config reload, terminating existing {service}") thread.terminate() # start with empty thread for each service for service in self.services: self.threads[service] = None # print config file path self.logger.info("Loaded config from {}".format(self.config_file))
def processing_status_generator(self): """ At regular interval, create status file for processing website """ self.logger.info("Starting processing status file generator") # create the output directory if it does not exist util.makedirs(self.processing_status_path) hostname = socket.gethostname() out_file = os.path.join(self.processing_status_path, f"{hostname}.js") while not self.stop_event.is_set(): # get list of taskids that are being processed taskids = sorted(self.observations.keys()) times = [] if not taskids: # nothing is running status = "idle" else: status = "running" now = Time.now() for taskid in taskids: # check elapsed time processing_time = now - self.observation_end_times[taskid] # if negative, the observation is still running if processing_time.sec < 0: times.append('observing') else: # format as hh:mm:ss full_min, seconds = divmod(processing_time.sec, 60) hours, minutes = divmod(full_min, 60) times.append( f"{hours:02.0f}h{minutes:02.0f}m{seconds:02.0f}s") content = dedent(f""" var {hostname} = {{ "node_name": "{hostname}", "node_status": "{status}", "node_process": "{','.join(taskids)}", "time": "{','.join(times)}" }}; """) with open(out_file, 'w') as f: f.write(content) self.stop_event.wait(self.processing_status_generator_interval) # upon exit, create file to indicate node is offline content = dedent(f""" var {hostname} = {{ "node_name": "{hostname}", "node_status": "offline", "node_process": "", "time": "" }}; """) with open(out_file, 'w') as f: f.write(content) f.flush()
def __init__(self, source_queue, *args, config_file=CONFIG_FILE, control_queue=None, **kwargs): """ :param Queue source_queue: Input queue :param str config_file: Path to config file :param Queue control_queue: Control queue of parent Process """ super(StatusWebsite, self).__init__() self.stop_event = mp.Event() self.source_queue = source_queue self.control_queue = control_queue # load config, including master for list of services with open(config_file, 'r') as f: config_all = yaml.load(f, Loader=yaml.SafeLoader) config = config_all['status_website'] config_master = config_all['darc_master'] # set config, expanding strings kwargs = {'home': os.path.expanduser('~'), 'hostname': socket.gethostname()} for key, value in config.items(): if isinstance(value, str): value = value.format(**kwargs) setattr(self, key, value) # set services if config_master['mode'] == 'real-time': self.services_master = config_master['services_master_rt'] self.services_worker = config_master['services_worker_rt'] elif config_master['mode'] == 'mixed': self.services_master = config_master['services_master_mix'] self.services_worker = config_master['services_worker_mix'] else: self.services_master = config_master['services_master_off'] self.services_worker = config_master['services_worker_off'] # store all node names self.all_nodes = [MASTER] + WORKERS # setup logger self.logger = get_logger(__name__, self.log_file) self.command_checker = None # reduce logging from status check commands logging.getLogger('darc.control').setLevel(logging.ERROR) # create website directory try: util.makedirs(self.web_dir) except Exception as e: self.logger.error("Failed to create website directory: {}".format(e)) raise StatusWebsiteException("Failed to create website directory: {}".format(e)) self.logger.info('Status website initialized')
def _start_observation_master(self, obs_config, reload=True): """ Start observation on master node Generated observation summary and send email once all workers are done :param dict obs_config: Observation config :param bool reload: reload service settings (default: True) """ # reload config if reload: self.load_config() if not self.full_processing_enabled: self.logger.info("Full processing disabled - nothing to run on master node") return self.logger.info("Starting observation on master node") # create result dir try: util.makedirs(obs_config['result_dir']) except Exception as e: self.logger.error("Failed to create results directory") raise OfflineProcessingException("Failed to create result directory: {}".format(e)) # create general info file self._get_overview(obs_config) # create coordinates file self._get_coordinates(obs_config) # wait until end time + delay start_processing_time = Time(obs_config['parset']['task.stopTime']) + TimeDelta(self.delay, format='sec') self.logger.info("Sleeping until {}".format(start_processing_time.iso)) util.sleepuntil_utc(start_processing_time, event=self.stop_event) # start emailer cmd = "python2 {emailer} {result_dir} '{beams}' {ntabs}".format(**obs_config) self.logger.info("Running {}".format(cmd)) os.system(cmd) # fetch known FRB candidates # do this _after_ email is sent, as then we are sure the grouped_pulses file exists for all beams # NOTE: disabled because: # 1) the (external) plotting script does not work without an active X session (should be fixable) # 2) the script re-uses the same temporary directory, and can run only one copy at a time # While these issues are not fixed, the plotting is not run automatically # if coord_cb00 is not None: # self._plot_known_frb_cands(obs_config, coord_cb00) # else: # self.logger.warning("Skipping plotting of known FRB candidates: CB00 coordinates not available") self.logger.info("Finished processing of observation {output_dir}".format(**obs_config))
def _start_observation_master(self, obs_config, reload=True): """ Start observation on master node Generated observation summary and send email once all workers are done :param dict obs_config: Observation config :param bool reload: reload service settings (default: True) """ self.logger.info("Starting observation on master node") # reload config if reload: self.load_config() # create result dir try: util.makedirs(obs_config['result_dir']) except Exception as e: self.logger.error("Failed to create results directory") raise OfflineProcessingException( "Failed to create result directory: {}".format(e)) # create general info file self._get_overview(obs_config) # create coordinates file coord_cb00 = self._get_coordinates(obs_config) # wait until end time + delay start_processing_time = Time( obs_config['parset']['task.stopTime']) + TimeDelta(self.delay, format='sec') self.logger.info("Sleeping until {}".format(start_processing_time.iso)) util.sleepuntil_utc(start_processing_time, event=self.stop_event) # start emailer cmd = "python2 {emailer} {result_dir} '{beams}' {ntabs}".format( **obs_config) self.logger.info("Running {}".format(cmd)) os.system(cmd) # fetch known FRB candidates # do this _after_ email is sent, as then we are sure the grouped_pulses file exists for all beams if coord_cb00 is not None: self._plot_known_frb_cands(obs_config, coord_cb00) else: self.logger.warning( "Skipping plotting of known FRB candidates: CB00 coordinates not available" ) self.logger.info( "Finished processing of observation {output_dir}".format( **obs_config))
def setUp(self): self.output_dir = '/data/arts/darc/output/triggers_realtime' self.result_dir = '/data/arts/darc/output/central' # ensure we start clean try: rmtree(self.result_dir) except FileNotFoundError: pass util.makedirs(self.result_dir) for fname in glob.glob(os.path.join(self.output_dir, '*.pdf')): os.remove(fname)
def __init__(self, obs_config, output_dir, log_queue, input_queue, output_queue, ncand_above_threshold, config_file=CONFIG_FILE, obs_name=''): """ :param dict obs_config: Observation settings :param str output_dir: Output directory for data products :param Queue log_queue: Queue to use for logging :param Queue input_queue: Input queue for clusters :param Queue output_queue: Output queue for classifier :param mp.Value ncand_above_threshold: 0 :param str config_file: Path to config file :param str obs_name: Observation name to use in log messages """ super(Extractor, self).__init__() module_name = type(self).__module__.split('.')[-1] self.logger = get_queue_logger(module_name, log_queue) self.output_dir = os.path.join(output_dir, 'data') self.obs_config = obs_config self.input_queue = input_queue self.output_queue = output_queue self.obs_name = obs_name # load config self.config_file = config_file self.config = self._load_config() # create directory for output data util.makedirs(self.output_dir) # create stop event self.stop_event = mp.Event() self.input_empty = False self.filterbank_reader = None self.rfi_mask = np.array([], dtype=int) self.data = None self.data_dm_time = None self.ncand_above_threshold = ncand_above_threshold
def _publish_results(self, body, files): """ Publish email content as local website """ # create output folder web_folder = '{home}/public_html/darc/{webdir}/{date}/{datetimesource}'.format( webdir=self.webdir, **self.obs_config) util.makedirs(web_folder) # save the email body, ensuring it is at the top of the list in a browser with open(os.path.join(web_folder, 'A_info.html'), 'w') as f: f.write(body) # create symlinks to PDFs. These are outside the public_html folder, but they are readable as long as they # are owned by the same user for src in files: dest = os.path.join(web_folder, os.path.basename(src['path'])) os.symlink(src['path'], dest) self.logger.info( f"Published results of {self.obs_config['datetimesource']}")
def get_queue_logger_listener(queue, log_file): """ Create thread that logs message from a queue :param Queue queue: Log queue :param str log_file: Path to log file :return: QueueListener thread """ # create log dir log_dir = os.path.dirname(os.path.realpath(log_file)) try: util.makedirs(log_dir) except Exception as e: sys.stderr.write("Could not create log directory: {}\n".format(e)) # setup handler handler = logging.handlers.WatchedFileHandler(log_file) formatter = logging.Formatter('%(asctime)s.%(levelname)s.%(name)s: %(message)s') handler.setFormatter(formatter) listener = logging.handlers.QueueListener(queue, handler) return listener
def setUp(self): self.output_dir = '/data/arts/darc/output/triggers_realtime' self.result_dir = '/data/arts/darc/output/central' # ensure we start clean try: rmtree(self.result_dir) except FileNotFoundError: pass util.makedirs(self.result_dir) for fname in glob.glob(os.path.join(self.output_dir, '*.pdf')): os.remove(fname) # create log handler log_queue = mp.Queue() handler = logging.StreamHandler() formatter = logging.Formatter('%(asctime)s.%(levelname)s.%(name)s: %(message)s') handler.setFormatter(formatter) self.ql = logging.handlers.QueueListener(log_queue, handler) self.ql.start() self.log_queue = log_queue
def __init__(self, obs_config, output_dir, logger, input_queue, output_queue, ncand_above_threshold, config_file=CONFIG_FILE): """ :param dict obs_config: Observation settings :param str output_dir: Output directory for data products :param Logger logger: Processor logger object :param Queue input_queue: Input queue for clusters :param Queue output_queue: Output queue for classifier :param mp.Value ncand_above_threshold: 0 :param str config_file: Path to config file """ super(Extractor, self).__init__() self.logger = logger self.output_dir = os.path.join(output_dir, 'data') self.obs_config = obs_config self.input_queue = input_queue self.output_queue = output_queue # load config self.config_file = config_file self.config = self._load_config() # create directory for output data util.makedirs(self.output_dir) # create stop event self.stop_event = mp.Event() self.input_empty = False self.filterbank_reader = None self.rfi_mask = np.array([], dtype=int) self.data = None self.data_dm_time = None self.ncand_above_threshold = ncand_above_threshold
def __init__(self, source_queue, *args, config_file=CONFIG_FILE, control_queue=None, **kwargs): """ :param Queue source_queue: Input queue for controlling this service :param str config_file: Path to config file :param Queue control_queue: Control queue of parent Process """ super(VOEventGenerator, self).__init__() self.stop_event = mp.Event() self.input_queue = source_queue self.control_queue = control_queue self.voevent_server = None with open(config_file, 'r') as f: config = yaml.load(f, Loader=yaml.SafeLoader)['voevent_generator'] # set config, expanding strings kwargs = {'home': os.path.expanduser('~'), 'hostname': socket.gethostname()} for key, value in config.items(): if isinstance(value, str): value = value.format(**kwargs) setattr(self, key, value) # setup logger self.logger = get_logger(__name__, self.log_file) # create and cd to voevent directory try: util.makedirs(self.voevent_dir) except Exception as e: self.logger.error("Cannot create voevent directory: {}".format(e)) raise VOEventGeneratorException("Cannot create voevent directory") os.chdir(self.voevent_dir) # Initalize the queue server voevent_queue = mp.Queue() VOEventQueueServer.register('get_queue', callable=lambda: voevent_queue) self.voevent_queue = voevent_queue self.logger.info("VOEvent Generator initialized")
def start_observation(self, obs_config, reload=True): """ Parse obs config and start listening for amber triggers on queue :param dict obs_config: Observation configuration :param bool reload: reload service settings (default: True) """ # reload config if reload: self.load_config() # add observation-specific path to result_dir self.central_result_dir = os.path.join(self.result_dir, obs_config['date'], obs_config['datetimesource']) util.makedirs(self.central_result_dir) self.obs_config = obs_config # process the observation in a separate Process self.process = mp.Process(target=self._process_observation) self.process.start()
def start_observation(self, obs_config, reload=True): """ Parse obs config and start observation processing after end time has passed :param dict obs_config: Observation configuration :param bool reload: reload service settings (default: True) """ # reload config if reload: self.load_config() # add observation-specific path to result_dir self.central_result_dir = os.path.join(self.result_dir, obs_config['date'], obs_config['datetimesource']) util.makedirs(self.central_result_dir) self.obs_config = obs_config # process the observation in a separate thread (not a process, as then we can't stop it directly # through the stop event of ProcessorMaster; and no other processing happens anyway) self.process = threading.Thread(target=self._process_observation) self.process.start()
def start_observation(self, obs_config, reload=True): """ Parse obs config and start listening for amber triggers on queue :param dict obs_config: Observation configuration :param bool reload: reload service settings (default: True) """ # reload config if reload: self.load_config() # store obs name for logging self.obs_name = f"{obs_config['parset']['task.taskID']} - {obs_config['datetimesource']}: " # clean any old triggers self.amber_triggers = [] # set config self.obs_config = obs_config # add observation-specific path to result_dir self.central_result_dir = os.path.join(self.result_dir, obs_config['date'], obs_config['datetimesource']) # create output dir output_dir = os.path.join('{output_dir}'.format(**obs_config), self.output_subdir) for path in (output_dir, self.central_result_dir): try: util.makedirs(path) except Exception as e: self.logger.error( f"{self.obs_name}Failed to create directory {path}: {e}") raise ProcessorException( f"Failed to create directory {path}: {e}") self.output_dir = output_dir self.observation_running = True # this must be set before starting the processing thread # start processing thread = threading.Thread(target=self._read_and_process_data, name='processing') self.threads['processing'] = thread # start clustering thread = Clustering(obs_config, output_dir, self.log_queue, self.clustering_queue, self.extractor_queue, self.ncluster, self.config_file, self.obs_name) thread.name = 'clustering' self.threads['clustering'] = thread # start extractor(s) for i in range(self.num_extractor): thread = Extractor(obs_config, output_dir, self.log_queue, self.extractor_queue, self.classifier_queue, self.ncand_above_threshold, self.config_file, self.obs_name) thread.name = f'extractor_{i}' self.threads[f'extractor_{i}'] = thread # start classifier thread = Classifier(self.log_queue, self.classifier_queue, self.classifier_child_conn, self.config_file, self.obs_name) thread.name = 'classifier' self.threads['classifier'] = thread # start all threads/processes for thread in self.threads.values(): thread.start() # If this is reprocessing instead of a new observation, read the AMBER triggers # Reprocessing is assumed if the end time is in the past utc_start = Time(obs_config['startpacket'] / TIME_UNIT, format='unix') utc_end = utc_start + TimeDelta(obs_config['duration'], format='sec') if utc_end < Time.now(): self.logger.info( f"{self.obs_name}End time is in the past, reading AMBER triggers for reprocessing" ) thread = threading.Thread(target=self._read_amber_triggers, name='read_amber_triggers') thread.daemon = True thread.start() self.reprocessing = True else: self.reprocessing = False self.logger.info(f"{self.obs_name}Observation started")
def start_observation(self, config_file, service=None): """ Start an observation :param str config_file: Path to observation config file :param str service: Which service to send start_observation to (default: all) :return: status, reply """ self.logger.info( "Received start_observation command with config file {}".format( config_file)) # check if config file exists if not os.path.isfile(config_file): self.logger.error("File not found: {}".format(config_file)) return "Error", "Failed: config file not found" # load config if config_file.endswith('.yaml'): config = self._load_yaml(config_file) elif config_file.endswith('.parset'): config = self._load_parset(config_file) else: self.logger.error( "Failed to determine config file type from {}".format( config_file)) return "Error", "Failed: unknown config file type" # check if process_triggers is enabled if not config['proctrigger']: self.logger.info( "Process triggers is disabled; not starting observation") return "Success", "Process triggers disabled - not starting" # store the config for future reference config_output_dir = os.path.join(self.parset_dir, config['datetimesource']) try: util.makedirs(config_output_dir) except Exception as e: raise DARCMasterException( "Cannot create config output directory: {}".format(e)) try: copy2(config_file, config_output_dir) except Exception as e: self.logger.error("Could not store config file: {}".format(e)) # initialize observation # ensure services are running if service is None: for s in self.services: self.start_service(s) else: self.start_service(service) # check host type if self.hostname == MASTER: host_type = 'master' elif self.hostname in WORKERS: host_type = 'worker' else: self.logger.error("Running on unknown host: {}".format( self.hostname)) return "Error", "Failed: running on unknown host" # create command command = { 'command': 'start_observation', 'obs_config': config, 'host_type': host_type } # wait until start time utc_start = Time(config['startpacket'] / TIME_UNIT, format='unix') utc_end = utc_start + TimeDelta(config['duration'], format='sec') # if end time is in the past, only start offline processing and processor if utc_end < Time.now(): self.logger.warning( "End time in past! Only starting offline processing and/or processor" ) if service is None: self.offline_queue.put(command) self.processor_queue.put(command) return "Warning", "Only offline processing and processor started" elif service == 'offline_processing': self.offline_queue.put(command) return "Warning", "Only offline processing started" elif service == 'processor': self.processor_queue.put(command) return "Warning", "Only processor started" else: return "Error", "Can only start offline processing and processor when end time is in past" t_setup = utc_start - TimeDelta(self.setup_time, format='sec') self.logger.info("Starting observation at {}".format(t_setup.isot)) util.sleepuntil_utc(t_setup) if service is None: # clear queues, then send command for queue in self.all_queues: util.clear_queue(queue) for queue in self.all_queues: queue.put(command) return "Success", "Observation started" else: # only start specified service. As this is only used in case e.g. something fails during # an observation, do not clear the queues first queue = self.get_queue(service) queue.put(command) return "Warning", "Only observation for {} started".format(service)
def _start_observation_worker(self, obs_config, reload=True): """ Start observation on worker node: #. Candidate clustering #. Extraction of filterbank data #. ML classification :param dict obs_config: Observation config :param bool reload: reload service settings (default: True) """ self.logger.info("Starting observation on worker node") # reload config if reload: self.load_config() # create result dir try: util.makedirs(obs_config['result_dir']) except Exception as e: self.logger.error("Failed to create results directory") raise OfflineProcessingException( "Failed to create result directory: {}".format(e)) # TAB or IAB mode if obs_config['ntabs'] == 1: obs_config['mode'] = 'IAB' trigger_output_file = "{output_dir}/triggers/data/data_00_full.hdf5".format( **obs_config) else: obs_config['mode'] = 'TAB' trigger_output_file = "{output_dir}/triggers/data/data_full.hdf5".format( **obs_config) # wait until end time + delay start_processing_time = Time( obs_config['parset']['task.stopTime']) + TimeDelta(self.delay, format='sec') self.logger.info("Sleeping until {}".format(start_processing_time.iso)) util.sleepuntil_utc(start_processing_time, event=self.stop_event) # fold pulsar if this is beam 0 and a test pulsar is being observed try: source = obs_config['parset']['task.source.name'] ref_beam = int(obs_config['parset']['task.source.beam']) # Requires access to parset if source in self.test_pulsars and (obs_config['beam'] == ref_beam): self.logger.info("Test pulsar detected: {}".format(source)) self._fold_pulsar(source, obs_config) except Exception as e: self.logger.error("Pulsar folding failed: {}".format(e)) # run calibration tools if this is a calibrator scan # these have "drift" in the source name try: source = obs_config['parset']['task.source.name'] if 'drift' in source: # split into actual source name and which beams were scanned name, beam_range = source.split('drift') # parse beam range, can be one beam or start/end beam if len(beam_range) == 2: # one beam drift_beams = [int(beam_range)] elif len(beam_range) == 4: # start and end beam sbeam = int(beam_range[:2]) ebeam = int(beam_range[2:]) drift_beams = range(sbeam, ebeam + 1) else: self.logger.error( "Failed to parse beam range for calibrator scan: {}". format(source)) drift_beams = [] # run calibration tools if this is a calibrator scan of this beam if name in self.calibrators and (obs_config['beam'] in drift_beams): self.logger.info( "Calibrator scan through this beam detected: {}". format(source)) self._run_calibration_tools(name, obs_config) except Exception as e: self.logger.error("Calibration tools failed: {}".format(e)) # create trigger directory trigger_dir = "{output_dir}/triggers".format(**obs_config) try: util.makedirs(trigger_dir) except Exception as e: self.logger.error("Failed to create triggers directory") raise OfflineProcessingException( "Failed to create triggers directory: {}".format(e)) # change to trigger directory try: os.chdir(trigger_dir) except Exception as e: self.logger.error( "Failed to cd to triggers directory {}: {}".format( trigger_dir, e)) # create subdir try: util.makedirs('data') except Exception as e: self.logger.error("Failed to create triggers/data directory") raise OfflineProcessingException( "Failed to create triggers/data directory: {}".format(e)) # merge the trigger files self.logger.info("Merging raw trigger files") numcand_raw = self._merge_triggers(obs_config) # Run clustering for each SB in TAB mode if enabled if self.process_sb and obs_config['mode'] == 'TAB': self.logger.info("Clustering candidates in SB mode") tstart = Time.now() # spread the SBs over the allowed number of threads # output grouped pulses file will only be generated by thread including SB00 chunks = np.array_split(range(obs_config['nsynbeams']), self.numthread) numcand_all = np.zeros(self.numthread) filterbank_prefix = "{output_dir}/filterbank/CB{beam:02d}".format( **obs_config) threads = [] self.logger.info( "Starting trigger clustering with {} threads".format( self.numthread)) for ind, chunk in enumerate(chunks): # pick the SB range sbmin, sbmax = min(chunk), max(chunk) # create thread thread = threading.Thread(target=self._cluster, args=[obs_config, filterbank_prefix], kwargs={ 'out': numcand_all, 'sbmin': sbmin, 'sbmax': sbmax, 'ind': ind }) thread.daemon = True threads.append(thread) thread.start() # wait until all are done for thread in threads: thread.join() # gather results if self.process_sb: # each element equal numcand_grouped = int(numcand_all[0]) else: # each element is one TAB numcand_grouped = int(np.sum(numcand_all[numcand_all != -1])) # Run clustering for IAB / each TAB else: self.logger.info("Clustering candidates in {} mode".format( obs_config['mode'])) tstart = Time.now() if obs_config['mode'] == 'IAB': filterbank_file = "{output_dir}/filterbank/CB{beam:02d}.fil".format( **obs_config) numcand_grouped = self._cluster(obs_config, 0, filterbank_file) elif obs_config['mode'] == 'TAB': numcand_all = np.zeros(obs_config['ntabs']) # max numtread tabs per run; so ntabs / numthread chunks n_chunk = int( np.ceil(obs_config['ntabs'] / float(self.numthread))) chunks = np.array_split(range(obs_config['ntabs']), n_chunk) self.logger.info( "Starting trigger clustering with {} chunks of {} threads". format(n_chunk, self.numthread)) # start the threads for tab_set in chunks: threads = [] for tab in tab_set: filterbank_file = "{output_dir}/filterbank/CB{beam:02d}_{tab:02d}.fil".format( tab=tab, **obs_config) thread = threading.Thread( target=self._cluster, args=[obs_config, filterbank_file], kwargs={ 'out': numcand_all, 'tab': tab }) thread.daemon = True threads.append(thread) thread.start() # wait until all are done for thread in threads: thread.join() # gather results numcand_grouped = int(np.sum(numcand_all[numcand_all != -1])) tend = Time.now() self.logger.info("Trigger clustering took {}s".format( (tend - tstart).sec)) # Create one hdf5 file for entire CB if obs_config['mode'] == 'TAB': self.logger.info("Merging output HDF5 files") numcand_merged = self._merge_hdf5(obs_config, trigger_output_file) # TEMP so that IAB still works else: numcand_merged = 9999 # Run classifier if numcand_merged != 0: self.logger.info("Classifying candidates") output_prefix = self._classify(obs_config, trigger_output_file) else: self.logger.info( "No candidates post-merge. Not running classifier") output_prefix = '' # Merge PDFs if numcand_merged > 0: self.logger.info("Merging classifier output files") self._merge_plots(obs_config) else: self.logger.info( "No candidates found post-classifier, not creating merged PDF") # Centralize results self.logger.info("Gathering results") kwargs = { 'output_prefix': output_prefix, 'data_file': trigger_output_file, 'numcand_raw': numcand_raw, 'numcand_grouped': numcand_grouped } self._gather_results(obs_config, **kwargs) self.logger.info( "Finished processing of observation {output_dir}".format( **obs_config)) return
def setUp(self): """ Set up the pipeline and observation software """ print("Setup") # observation settings files = glob.glob('/tank/data/sky/B1933+16/20200211_dump/dada/*.dada') self.assertTrue(len(files) > 0) output_dir = '/tank/users/oostrum/test_full_run' amber_dir = os.path.join(output_dir, 'amber') amber_conf_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'amber.conf') amber_conf_dir = '/home/arts/.controller/amber_conf' # ensure we start clean rmtree(amber_dir) util.makedirs(amber_dir) # load the encoded parset with open('/tank/data/sky/B1933+16/20200211_dump/parset', 'r') as f: parset = f.read().strip() # nreader: one for each programme reading from the buffer, i.e. 3x AMBER self.settings = {'resolution': 1536 * 12500 * 12, 'nbuf': 5, 'key_i': 'aaaa', 'hdr_size': 40960, 'dada_files': files, 'nreader': 3, 'freq': 1370, 'amber_dir': amber_dir, 'nbatch': len(files) * 10, 'beam': 0, 'amber_config': amber_conf_file, 'amber_conf_dir': amber_conf_dir, 'min_freq': 1219.70092773, 'parset': parset, 'datetimesource': '2019-01-01-00:00:00.FAKE'} # open custom config file self.config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.yaml') with open(self.config_file, 'r') as f: self.master_config = yaml.load(f, Loader=yaml.SafeLoader)['darc_master'] # create buffers # stokes I print("Creating buffers") os.system('dada_db -d -k {key_i} 2>/dev/null ; dada_db -k {key_i} -a {hdr_size} ' '-b {resolution} -n {nbuf} -r {nreader}'.format(**self.settings)) # init GPU pipeline thread self.t_amber = threading.Thread(target=self.amber, name='AMBER', daemon=True) # init amber listener thread self.listener = AMBERListener() self.listener.set_source_queue(mp.Queue()) self.listener.set_target_queue(mp.Queue()) self.listener.start() # init amber clustering thread # do not connect to VOEvent server nor LOFAR trigger system self.clustering = AMBERClustering(connect_vo=False, connect_lofar=False) self.clustering.set_source_queue(self.listener.target_queue) self.clustering.set_target_queue(mp.Queue()) self.clustering.start() # init DADA trigger thread self.dadatrigger = DADATrigger() self.dadatrigger.set_source_queue(self.clustering.target_queue) self.dadatrigger.start() # init writer thread self.t_disk_to_db = threading.Thread(target=self.writer, name='disk_to_db', daemon=True) # init output listener thread self.event_queue = mp.Queue() self.t_dbevent = threading.Thread(target=self.dbevent, name='dbevent', daemon=True, args=(self.event_queue,))
def setUp(self): if socket.gethostname() == 'zeus': self.dada_files = glob.glob('/data/arts/data/dada/*.dada')[:1] self.dada_files.sort() output_dir = '/data/arts/darc/output' log_dir = f'{output_dir}/log' filterbank_dir = f'{output_dir}/filterbank' amber_dir = f'{output_dir}/amber' amber_conf_dir = '/data/arts/darc/amber_conf' amber_conf_file = '/data/arts/darc/amber.conf' sb_table = '/data/arts/darc/sbtable-sc4-12tabs-71sbs.txt' else: self.dada_files = glob.glob('/tank/data/sky/B1933+16/20200211_dump/dada/*.dada')[:1] self.dada_files.sort() user = os.getlogin() if user == 'oostrum': main_dir = '/tank/users/oostrum/darc/automated_testing' elif user == 'arts': main_dir = '/tank/users/arts/darc_automated_testing' else: self.skipTest(f"Cannot run test as user {user}") output_dir = f'{main_dir}/output' log_dir = f'{output_dir}/log' filterbank_dir = f'{output_dir}/filterbank' amber_dir = f'{output_dir}/amber' amber_conf_dir = f'{main_dir}/amber/amber_conf' amber_conf_file = f'{main_dir}/amber/amber.conf' sb_table = '/home/arts/.controller/synthesized_beam_tables/sbtable-sc4-12tabs-71sbs.txt' # ensure we start clean try: rmtree(output_dir) except FileNotFoundError: pass for d in (output_dir, log_dir, amber_dir, filterbank_dir): util.makedirs(d) # create log handler log_queue = mp.Queue() handler = logging.StreamHandler() formatter = logging.Formatter('%(asctime)s.%(levelname)s.%(name)s: %(message)s') handler.setFormatter(formatter) self.ql = logging.handlers.QueueListener(log_queue, handler) self.ql.start() self.processes = {} # extract PSRDADA header self.header = self.get_psrdada_header(self.dada_files[0]) self.tstart = Time.now() + TimeDelta(5, format='sec') # add general settings self.header['nreader'] = 2 self.header['nbuffer'] = 5 self.header['key_i'] = '5000' self.header['beam'] = 0 self.header['ntab'] = 12 self.header['nsb'] = 71 # self.header['nbatch'] = int(float(self.header['SCANLEN']) / 1.024) self.header['nbatch'] = 10 self.header['duration'] = float(self.header['SCANLEN']) self.header['log_dir'] = log_dir self.header['output_dir'] = output_dir self.header['filterbank_dir'] = filterbank_dir self.header['amber_dir'] = amber_dir self.header['amber_conf_dir'] = amber_conf_dir self.header['amber_config'] = amber_conf_file self.header['sb_table'] = sb_table self.header['date'] = '20200101' self.header['datetimesource'] = '2020-01-01-00:00:00.FAKE' self.header['freq'] = int(np.round(float(self.header['FREQ']))) self.header['snrmin'] = 8 self.header['min_freq'] = 1220.7 self.header['startpacket'] = int(self.tstart.unix * TIME_UNIT) # add parset parset = {'task.duration': str(self.header['SCANLEN']), 'task.startTime': self.tstart.isot, 'task.taskID': '001122', 'task.beamSet.0.compoundBeam.0.phaseCenter': '[293.94876deg, 16.27778deg]', 'task.directionReferenceFrame': 'HADEC'} self.header['parset'] = parset # create ringbuffer self.create_ringbuffer() # create processes for the different pipeline steps self.diskdb_proc = self.diskdb_command() self.amber_proc = self.amber_command() self.dadafilterbank_proc = self.dadafilterbank_command() # start all except data reader self.dadafilterbank_proc.start() self.amber_proc.start() # initialize AMBERListener, used for feeding triggers to Processor self.amber_queue = mp.Queue() self.processor_queue = mp.Queue() self.amber_listener = AMBERListener(self.amber_queue, target_queue=self.processor_queue) self.amber_listener.start() # initialize Processor, connect input queue to output of AMBERListener self.processor = Processor(log_queue=log_queue, source_queue=self.processor_queue)
def setUp(self): if socket.gethostname() == 'zeus': self.result_dir = '/data/arts/darc/central_dir_processor_master' else: self.result_dir = '/tank/users/arts/darc_automated_testing/processor_master' self.processor_queue = mp.Queue() self.processor = ProcessorMaster(self.processor_queue) tstart = Time.now() duration = TimeDelta(5, format='sec') parset = { 'task.duration': duration.sec, 'task.startTime': tstart.isot, 'task.stopTime': (tstart + duration).isot, 'task.source.name': 'FAKE', 'task.taskID': '001122', 'task.beamSet.0.compoundBeam.0.phaseCenter': '[293.94876deg, 16.27778deg]', 'task.directionReferenceFrame': 'J2000', 'task.telescopes': '[RT2, RT3, RT4, RT5, RT6, RT7, RT8, RT9]' } self.obs_config = { 'beams': [0], 'freq': 1370, 'startpacket': int(tstart.unix * TIME_UNIT), 'date': '20200101', 'datetimesource': '2020-01-01-00:00:00.FAKE', 'result_dir': self.result_dir, 'parset': parset } # output directory output_dir = os.path.join(self.result_dir, self.obs_config['date'], self.obs_config['datetimesource']) # web directory webdir = '{home}/public_html/darc/{webdir}/{date}/{datetimesource}'.format( home=os.path.expanduser('~'), webdir=self.processor.webdir, **self.obs_config) # remove existing output for d in (output_dir, webdir): try: shutil.rmtree(d) except FileNotFoundError: pass util.makedirs(output_dir) # create empty PDF open(os.path.join(output_dir, 'CB00.pdf'), 'w').close() # create trigger overview # classifier cands must be > 0 to test processing of PDF attachments trigger_results = { 'ncand_raw': 1000, 'ncand_post_clustering': 100, 'ncand_post_thresholds': 10, 'ncand_post_classifier': 1 } with open(os.path.join(output_dir, 'CB00_summary.yaml'), 'w') as f: yaml.dump(trigger_results, f, default_flow_style=False) # create file with trigger list with open(os.path.join(output_dir, 'CB00_triggers.txt'), 'w') as f: f.write('#cb snr dm time downsamp sb p\n') f.write('00 10.00 20.00 30.0000 50 35 1.00\n')
def start_observation(self, obs_config, reload=True): """ Parse obs config and start listening for amber triggers on queue :param dict obs_config: Observation configuration :param bool reload: reload service settings (default: True) """ # reload config if reload: self.load_config() # clean any old triggers self.amber_triggers = [] # set config self.obs_config = obs_config # add observation-specific path to result_dir self.central_result_dir = os.path.join(self.result_dir, obs_config['date'], obs_config['datetimesource']) # create output dir output_dir = os.path.join('{output_dir}'.format(**obs_config), self.output_subdir) for path in (output_dir, self.central_result_dir): try: util.makedirs(path) except Exception as e: self.logger.error(f"Failed to create directory {path}: {e}") raise ProcessorException( f"Failed to create directory {path}: {e}") self.output_dir = output_dir self.observation_running = True # this must be set before starting the processing thread # start processing thread = threading.Thread(target=self._read_and_process_data, name='processing') thread.start() self.threads['processing'] = thread # start clustering thread = Clustering(obs_config, output_dir, self.logger, self.clustering_queue, self.extractor_queue, self.ncluster, self.config_file) thread.name = 'clustering' thread.start() self.threads['clustering'] = thread # start extractor(s) for i in range(self.num_extractor): thread = Extractor(obs_config, output_dir, self.logger, self.extractor_queue, self.classifier_queue, self.ncand_above_threshold, self.config_file) thread.name = f'extractor_{i}' thread.start() self.threads[f'extractor_{i}'] = thread # start classifier thread = Classifier(self.logger, self.classifier_queue, self.classifier_child_conn, self.config_file) thread.name = 'classifier' thread.start() self.threads['classifier'] = thread self.logger.info("Observation started")
def test_worker_processing_tab(self): # config for worker should contain: # ntabs # nsynbeams # mode (IAB/TAB) # output_dir (/data2/output/<date>/<datetimesource) # startpacket # beam (CB) # amber_dir # duration # result_dir # config config = self.gen_config(sb=False) logger = logging.getLogger('test_worker_processing_tab') handler = logging.StreamHandler() formatter = logging.Formatter( '%(asctime)s.%(levelname)s.%(name)s: %(message)s') handler.setFormatter(formatter) logger.addHandler(handler) logger.setLevel(logging.DEBUG) # output file names fname_yaml = "CB00_summary.yaml" fname_txt = "CB00_triggers.txt" fname_txt_in = "CB00_triggers_tab.txt" fname_pdf = "CB00_candidates_summary.pdf" # set expected output for yaml expected_output_yaml = { 'ncand_abovethresh': 75, 'ncand_classifier': 75, 'ncand_raw': 5923, 'ncand_skipped': 1339, 'ncand_trigger': 1414 } # read expected output for txt with open(os.path.join(TESTDIR, fname_txt_in)) as f: expected_output_txt = f.read().strip().split() # Create result dir (normally done in run method, but that is skipped) try: util.makedirs(config['result_dir']) except Exception as e: self.fail('Cannot create result dir {}: {}'.format( config['result_dir'], e)) else: proc = OfflineProcessing() # override logger (first initalized message still goes to normal logger) proc.logger = logger # override sb processing mode proc.process_sb = False # override config proc.host_type = 'worker' proc.config['nfreq_plot'] = 32 proc.config['snrmin_processing'] = 45 proc.config['snrmin_processing_local'] = 5 proc.config['dmmin'] = 20 proc.config['dmmax'] = 5000 proc.config['pthresh_dmtime'] = 0 # Add offline processing config to obs config (normally done in run method, but that is skipped) fullconfig = proc.config.copy() fullconfig.update(config) # run worker observation try: proc._start_observation_worker(fullconfig, reload=False) except Exception as e: self.fail( "Unhandled exception in offline processing: {}".format(e)) # check if output files exist for fname in [fname_yaml, fname_txt, fname_pdf]: self.assertTrue( os.path.isfile(os.path.join(config['result_dir'], fname))) # for the yaml and txt, verify the content with open(os.path.join(config['result_dir'], fname_yaml)) as f: output_yaml = yaml.load(f, Loader=yaml.SafeLoader) self.assertDictEqual(expected_output_yaml, output_yaml) with open(os.path.join(config['result_dir'], fname_txt)) as f: output_txt = f.read().strip().split() self.assertListEqual(expected_output_txt, output_txt) # remove the results dir if it exists shutil.rmtree(config['result_dir'], ignore_errors=True)