def _process_observation(self): """ Process observation """ # wait until the observation finishes start_processing_time = Time( self.obs_config['parset']['task.stopTime']) self.logger.info("Sleeping until {}".format(start_processing_time.iso)) self.status = 'Observation in progress' util.sleepuntil_utc(start_processing_time, event=self.stop_event) try: # generate observation info files self.status = 'Generating observation info files' info, coordinates = self._generate_info_file() # wait for all result files to be present self.status = 'Waiting for nodes to finish processing' self._wait_for_workers() # combine results, copy to website and generate email self.status = 'Combining node results' email, attachments = self._process_results(info, coordinates) # publish results on web link and send email self.status = 'Sending results to website' self._publish_results(email, attachments) self.status = 'Sending results to email' self._send_email(email, attachments) self.status = 'Done' except Exception as e: self.logger.error( f"Failed to process observation. Status = {self.status}: {type(e)}: {e}" )
def test_processor_obs(self): # start processor self.processor.start() # start amber listener and processor cmd = {'command': 'start_observation', 'obs_config': self.header, 'reload': False} self.amber_queue.put(cmd) self.processor_queue.put(cmd) # at start time, read data into buffer, other processes are already set up and waiting for data util.sleepuntil_utc(self.tstart) self.diskdb_proc.start() # wait until processes are done for proc in (self.diskdb_proc, self.amber_proc, self.dadafilterbank_proc): proc.join() # stop observation self.amber_queue.put({'command': 'stop_observation'}) self.processor.source_queue.put({'command': 'stop_observation'}) # wait for processor to exit self.processor.join() # stop services self.amber_listener.source_queue.put('stop') self.amber_listener.join()
def _start_observation_master(self, obs_config, reload=True): """ Start observation on master node Generated observation summary and send email once all workers are done :param dict obs_config: Observation config :param bool reload: reload service settings (default: True) """ self.logger.info("Starting observation on master node") # reload config if reload: self.load_config() # create result dir try: util.makedirs(obs_config['result_dir']) except Exception as e: self.logger.error("Failed to create results directory") raise OfflineProcessingException( "Failed to create result directory: {}".format(e)) # create general info file self._get_overview(obs_config) # create coordinates file coord_cb00 = self._get_coordinates(obs_config) # wait until end time + delay start_processing_time = Time( obs_config['parset']['task.stopTime']) + TimeDelta(self.delay, format='sec') self.logger.info("Sleeping until {}".format(start_processing_time.iso)) util.sleepuntil_utc(start_processing_time, event=self.stop_event) # start emailer cmd = "python2 {emailer} {result_dir} '{beams}' {ntabs}".format( **obs_config) self.logger.info("Running {}".format(cmd)) os.system(cmd) # fetch known FRB candidates # do this _after_ email is sent, as then we are sure the grouped_pulses file exists for all beams if coord_cb00 is not None: self._plot_known_frb_cands(obs_config, coord_cb00) else: self.logger.warning( "Skipping plotting of known FRB candidates: CB00 coordinates not available" ) self.logger.info( "Finished processing of observation {output_dir}".format( **obs_config))
def _start_observation_master(self, obs_config, reload=True): """ Start observation on master node Generated observation summary and send email once all workers are done :param dict obs_config: Observation config :param bool reload: reload service settings (default: True) """ # reload config if reload: self.load_config() if not self.full_processing_enabled: self.logger.info("Full processing disabled - nothing to run on master node") return self.logger.info("Starting observation on master node") # create result dir try: util.makedirs(obs_config['result_dir']) except Exception as e: self.logger.error("Failed to create results directory") raise OfflineProcessingException("Failed to create result directory: {}".format(e)) # create general info file self._get_overview(obs_config) # create coordinates file self._get_coordinates(obs_config) # wait until end time + delay start_processing_time = Time(obs_config['parset']['task.stopTime']) + TimeDelta(self.delay, format='sec') self.logger.info("Sleeping until {}".format(start_processing_time.iso)) util.sleepuntil_utc(start_processing_time, event=self.stop_event) # start emailer cmd = "python2 {emailer} {result_dir} '{beams}' {ntabs}".format(**obs_config) self.logger.info("Running {}".format(cmd)) os.system(cmd) # fetch known FRB candidates # do this _after_ email is sent, as then we are sure the grouped_pulses file exists for all beams # NOTE: disabled because: # 1) the (external) plotting script does not work without an active X session (should be fixable) # 2) the script re-uses the same temporary directory, and can run only one copy at a time # While these issues are not fixed, the plotting is not run automatically # if coord_cb00 is not None: # self._plot_known_frb_cands(obs_config, coord_cb00) # else: # self.logger.warning("Skipping plotting of known FRB candidates: CB00 coordinates not available") self.logger.info("Finished processing of observation {output_dir}".format(**obs_config))
def test_full_run(self): """ Do a full pipeline run """ # Buffers are already set up # start AMBER self.t_amber.start() # Give it some time to start sleep(1) # start DARC observation 5s in the future tstart = Time.now() + TimeDelta(5, format='sec') # add start time to config self.settings['startpacket'] = tstart.unix * TIME_UNIT # Todo: ensure DARC waits for start time # start observation cmd = {'command': 'start_observation', 'obs_config': self.settings, 'reload_conf': False} self.listener.source_queue.put(cmd) self.clustering.source_queue.put(cmd) # start writer at start time util.sleepuntil_utc(tstart) self.t_disk_to_db.start() # Now AMBER should start producing triggers # Listen for any IQUV triggers in thread self.t_dbevent.start() # wait for the writer to finish self.t_disk_to_db.join() # AMBER may still be processing, wait for that to finish too self.t_amber.join() # stop the observation self.listener.source_queue.put({'command': 'stop_observation'}) self.clustering.source_queue.put({'command': 'stop_observation'}) # verify an IQUV event was received try: event = self.event_queue.get(timeout=1) except Empty: event = None # event looks like (depending on start time): # N_EVENTS 2 # 2019-10-14-13:14:26.016 # 2019-10-14-13:14:37 137 2019-10-14-13:14:39 185 12.3749 26.4 50.0 21 # 2019-10-14-13:14:37 849 2019-10-14-13:14:39 897 14.1344 31.8 50.0 21 self.assertTrue(event is not None) print("Received event(s):\n{}".format(event))
def polcal_dumps(self, obs_config): """ Automatically dump IQUV data at regular intervals for polcal calibrator observations :param dict obs_config: Observation config """ tstart = Time(obs_config['startpacket'] / TIME_UNIT, format='unix') duration = TimeDelta(obs_config['duration'], format='sec') tend = tstart + duration # round up polcal dump size to nearest 1.024 s dump_size = TimeDelta(np.ceil(self.polcal_dump_size / 1.024) * 1.024, format='sec') dump_interval = TimeDelta(self.polcal_interval, format='sec') # sleep until first trigger time util.sleepuntil_utc(tstart + dump_interval, event=self.stop_event) # run until trigger would be end past end time # add a second to avoid trigger running a little bit over end time # also stay below global limit on number of dumps during one obs # TODO: also set a total length limit ndump = 0 while Time.now() + dump_interval - TimeDelta( 1.0, format='sec') < tend and ndump < self.polcal_max_dumps: # generate an IQUV trigger params = {'utc_start': tstart.iso.replace(' ', '-')} # trigger start time: now, rounded to nearest 1.024s since utc start dt = TimeDelta(np.round((Time.now() - tstart).sec / 1.024) * 1.024, format='sec') # trigger start/end times event_start_full = tstart + dt event_end_full = tstart + dt + dump_size # convert to dada_dbevent format event_start, event_start_frac = event_start_full.iso.split('.') event_end, event_end_frac = event_end_full.iso.split('.') # store to params params['event_start'] = event_start.replace(' ', '-') params['event_start_frac'] = event_start_frac params['event_end'] = event_end.replace(' ', '-') params['event_end_frac'] = event_end_frac event = "N_EVENTS 1\n{utc_start}\n{event_start} {event_start_frac} {event_end} {event_end_frac} " \ "0 0 0 0".format(**params) # dm, snr, width, beam are 0 self.logger.info( "Sending automated polcal IQUV dump event: {}".format(params)) self.send_events(event, 'IQUV') # keep track of number of performed dumps ndump += 1 # sleep self.stop_event.wait(dump_interval.sec)
def _start_observation_worker(self, obs_config, reload=True): """ Start observation on worker node: #. Candidate clustering #. Extraction of filterbank data #. ML classification :param dict obs_config: Observation config :param bool reload: reload service settings (default: True) """ self.logger.info("Starting observation on worker node") # reload config if reload: self.load_config() # create result dir try: util.makedirs(obs_config['result_dir']) except Exception as e: self.logger.error("Failed to create results directory") raise OfflineProcessingException( "Failed to create result directory: {}".format(e)) # TAB or IAB mode if obs_config['ntabs'] == 1: obs_config['mode'] = 'IAB' trigger_output_file = "{output_dir}/triggers/data/data_00_full.hdf5".format( **obs_config) else: obs_config['mode'] = 'TAB' trigger_output_file = "{output_dir}/triggers/data/data_full.hdf5".format( **obs_config) # wait until end time + delay start_processing_time = Time( obs_config['parset']['task.stopTime']) + TimeDelta(self.delay, format='sec') self.logger.info("Sleeping until {}".format(start_processing_time.iso)) util.sleepuntil_utc(start_processing_time, event=self.stop_event) # fold pulsar if this is beam 0 and a test pulsar is being observed try: source = obs_config['parset']['task.source.name'] ref_beam = int(obs_config['parset']['task.source.beam']) # Requires access to parset if source in self.test_pulsars and (obs_config['beam'] == ref_beam): self.logger.info("Test pulsar detected: {}".format(source)) self._fold_pulsar(source, obs_config) except Exception as e: self.logger.error("Pulsar folding failed: {}".format(e)) # run calibration tools if this is a calibrator scan # these have "drift" in the source name try: source = obs_config['parset']['task.source.name'] if 'drift' in source: # split into actual source name and which beams were scanned name, beam_range = source.split('drift') # parse beam range, can be one beam or start/end beam if len(beam_range) == 2: # one beam drift_beams = [int(beam_range)] elif len(beam_range) == 4: # start and end beam sbeam = int(beam_range[:2]) ebeam = int(beam_range[2:]) drift_beams = range(sbeam, ebeam + 1) else: self.logger.error( "Failed to parse beam range for calibrator scan: {}". format(source)) drift_beams = [] # run calibration tools if this is a calibrator scan of this beam if name in self.calibrators and (obs_config['beam'] in drift_beams): self.logger.info( "Calibrator scan through this beam detected: {}". format(source)) self._run_calibration_tools(name, obs_config) except Exception as e: self.logger.error("Calibration tools failed: {}".format(e)) # create trigger directory trigger_dir = "{output_dir}/triggers".format(**obs_config) try: util.makedirs(trigger_dir) except Exception as e: self.logger.error("Failed to create triggers directory") raise OfflineProcessingException( "Failed to create triggers directory: {}".format(e)) # change to trigger directory try: os.chdir(trigger_dir) except Exception as e: self.logger.error( "Failed to cd to triggers directory {}: {}".format( trigger_dir, e)) # create subdir try: util.makedirs('data') except Exception as e: self.logger.error("Failed to create triggers/data directory") raise OfflineProcessingException( "Failed to create triggers/data directory: {}".format(e)) # merge the trigger files self.logger.info("Merging raw trigger files") numcand_raw = self._merge_triggers(obs_config) # Run clustering for each SB in TAB mode if enabled if self.process_sb and obs_config['mode'] == 'TAB': self.logger.info("Clustering candidates in SB mode") tstart = Time.now() # spread the SBs over the allowed number of threads # output grouped pulses file will only be generated by thread including SB00 chunks = np.array_split(range(obs_config['nsynbeams']), self.numthread) numcand_all = np.zeros(self.numthread) filterbank_prefix = "{output_dir}/filterbank/CB{beam:02d}".format( **obs_config) threads = [] self.logger.info( "Starting trigger clustering with {} threads".format( self.numthread)) for ind, chunk in enumerate(chunks): # pick the SB range sbmin, sbmax = min(chunk), max(chunk) # create thread thread = threading.Thread(target=self._cluster, args=[obs_config, filterbank_prefix], kwargs={ 'out': numcand_all, 'sbmin': sbmin, 'sbmax': sbmax, 'ind': ind }) thread.daemon = True threads.append(thread) thread.start() # wait until all are done for thread in threads: thread.join() # gather results if self.process_sb: # each element equal numcand_grouped = int(numcand_all[0]) else: # each element is one TAB numcand_grouped = int(np.sum(numcand_all[numcand_all != -1])) # Run clustering for IAB / each TAB else: self.logger.info("Clustering candidates in {} mode".format( obs_config['mode'])) tstart = Time.now() if obs_config['mode'] == 'IAB': filterbank_file = "{output_dir}/filterbank/CB{beam:02d}.fil".format( **obs_config) numcand_grouped = self._cluster(obs_config, 0, filterbank_file) elif obs_config['mode'] == 'TAB': numcand_all = np.zeros(obs_config['ntabs']) # max numtread tabs per run; so ntabs / numthread chunks n_chunk = int( np.ceil(obs_config['ntabs'] / float(self.numthread))) chunks = np.array_split(range(obs_config['ntabs']), n_chunk) self.logger.info( "Starting trigger clustering with {} chunks of {} threads". format(n_chunk, self.numthread)) # start the threads for tab_set in chunks: threads = [] for tab in tab_set: filterbank_file = "{output_dir}/filterbank/CB{beam:02d}_{tab:02d}.fil".format( tab=tab, **obs_config) thread = threading.Thread( target=self._cluster, args=[obs_config, filterbank_file], kwargs={ 'out': numcand_all, 'tab': tab }) thread.daemon = True threads.append(thread) thread.start() # wait until all are done for thread in threads: thread.join() # gather results numcand_grouped = int(np.sum(numcand_all[numcand_all != -1])) tend = Time.now() self.logger.info("Trigger clustering took {}s".format( (tend - tstart).sec)) # Create one hdf5 file for entire CB if obs_config['mode'] == 'TAB': self.logger.info("Merging output HDF5 files") numcand_merged = self._merge_hdf5(obs_config, trigger_output_file) # TEMP so that IAB still works else: numcand_merged = 9999 # Run classifier if numcand_merged != 0: self.logger.info("Classifying candidates") output_prefix = self._classify(obs_config, trigger_output_file) else: self.logger.info( "No candidates post-merge. Not running classifier") output_prefix = '' # Merge PDFs if numcand_merged > 0: self.logger.info("Merging classifier output files") self._merge_plots(obs_config) else: self.logger.info( "No candidates found post-classifier, not creating merged PDF") # Centralize results self.logger.info("Gathering results") kwargs = { 'output_prefix': output_prefix, 'data_file': trigger_output_file, 'numcand_raw': numcand_raw, 'numcand_grouped': numcand_grouped } self._gather_results(obs_config, **kwargs) self.logger.info( "Finished processing of observation {output_dir}".format( **obs_config)) return
def _extract(self, dm, snr, toa, downsamp, sb): """ Execute data extraction :param float dm: Dispersion Measure (pc/cc) :param float snr: AMBER S/N :param float toa: Arrival time at top of band (s) :param int downsamp: Downsampling factor :param int sb: Synthesized beam index """ # keep track of the time taken to process this candidate timer_start = Time.now() # add units dm = dm * u.pc / u.cm**3 toa = toa * u.s # If the intra-channel smearing timescale is several samples, downsample # by at most 1/4th of that factor chan_width = BANDWIDTH / float(NCHAN) t_smear = util.dm_to_smearing(dm, self.obs_config['freq'] * u.MHz, chan_width) predownsamp = max( 1, int( t_smear.to(u.s).value / self.filterbank_reader.header.tsamp / 4)) # calculate remaining downsampling to do after dedispersion postdownsamp = max(1, downsamp // predownsamp) # total downsampling factor (can be slightly different from original downsampling) self.logger.debug( f"Original dowsamp: {downsamp}, new downsamp: {predownsamp * postdownsamp} " f"for ToA={toa.value:.4f}, DM={dm.value:.2f}") downsamp_effective = predownsamp * postdownsamp tsamp_effective = self.filterbank_reader.header.tsamp * downsamp_effective # calculate start/end bin we need to load from filterbank # DM delay in units of samples dm_delay = util.dm_to_delay( dm, self.obs_config['min_freq'] * u.MHz, self.obs_config['min_freq'] * u.MHz + BANDWIDTH) sample_delay = dm_delay.to( u.s).value // self.filterbank_reader.header.tsamp # number of input bins to store in final data ntime = self.config.ntime * downsamp_effective start_bin = int( toa.to(u.s).value // self.filterbank_reader.header.tsamp - .5 * ntime) # ensure start bin is not before start of observation if start_bin < 0: self.logger.warning( f"Start bin before start of file, shifting for ToA={toa.value:.4f}, DM={dm.value:.2f}" ) start_bin = 0 # number of bins to load is number to store plus dm delay nbin = int(ntime + sample_delay) end_bin = start_bin + nbin # Wait if the filterbank does not have enough samples yet # first update filterbank parameters to have correct nsamp self.filterbank_reader.get_header() tstart = Time(self.obs_config['startpacket'] / TIME_UNIT, format='unix') # wait until this much of the filterbank should be present on disk # start time, plus last bin to load, plus extra delay # (because filterbank is not immediately flushed to disk) twait = tstart + end_bin * self.filterbank_reader.header.tsamp * u.s + self.config.delay * u.s first_loop = True while end_bin >= self.filterbank_reader.header.nsamples: if first_loop: # wait until data should be present on disk self.logger.debug( f"Waiting until at least {twait.isot} for filterbank data to be present on disk " f"for ToA={toa.value:.4f}, DM={dm.value:.2f}") util.sleepuntil_utc(twait, event=self.stop_event) first_loop = False else: # data should be there, but is not. Wait just a short time, then check again self.stop_event.wait(.5) # re-read the number of samples to check if the data are available now self.filterbank_reader.get_header() # if we are past the end time of the observation, we give up and try shifting the end bin instead tend = tstart + self.obs_config[ 'duration'] * u.s + self.config.delay * u.s if Time.now() > tend: break # if start time before start of file, or end time beyond end of file, shift the start/end time # first update filterbank parameters to have correct nsamp self.filterbank_reader.get_header() if end_bin >= self.filterbank_reader.header.nsamples: # if start time is also beyond the end of the file, we cannot process this candidate and give an error if start_bin >= self.filterbank_reader.header.nsamples: self.logger.error( f"Start bin beyond end of file, error in filterbank data? Skipping" f" ToA={toa.value:.4f}, DM={dm.value:.2f}") # log time taken timer_end = Time.now() self.logger.info( f"Processed ToA={toa.value:.4f}, DM={dm.value:.2f} " f"in {(timer_end - timer_start).to(u.s):.0f}") return self.logger.warning( f"End bin beyond end of file, shifting for ToA={toa.value:.4f}, DM={dm.value:.2f}" ) diff = end_bin - self.filterbank_reader.header.nsamples + 1 start_bin -= diff # load the data. Store as attribute so it can be accessed from other methods (ok as we only run # one extraction at a time) try: self.data = self.filterbank_reader.load_single_sb( sb, start_bin, nbin) except ValueError as e: self.logger.error( f"Failed to load filterbank data for ToA={toa.value:.4f}, DM={dm.value:.2f}: {e}" ) # log time taken timer_end = Time.now() self.logger.info( f"Extracted ToA={toa.value:.4f}, DM={dm.value:.2f} " f"in {(timer_end - timer_start).to(u.s):.0f}") return # apply AMBER RFI mask if self.config.rfi_apply_mask: self.data[self.rfi_mask, :] = 0. # run rfi cleaning if self.config.rfi_clean_type is not None: self._rficlean() # apply predownsampling self.data.downsample(predownsamp) # subtract median self.data.data -= np.median(self.data.data, axis=1, keepdims=True) # get S/N and width at AMBER DM self.data.dedisperse(dm.to(u.pc / u.cm**3).value) # create timeseries timeseries = self.data.data.sum(axis=0)[:ntime] # get S/N and width # range of widths for S/N determination. Never go above 250 samples, # which is typically RFI even without pre-downsampling widths = np.arange(max(1, postdownsamp // 2), min(250, postdownsamp * 2)) snrmax, width_best = util.calc_snr_matched_filter(timeseries, widths=widths) # correct for already applied downsampling width_best *= predownsamp # if max S/N is below local threshold, skip this trigger if snrmax < self.config.snr_min_local: self.logger.warning( f"Skipping trigger with S/N ({snrmax:.2f}) below local threshold, " f"ToA={toa.value:.4f}, DM={dm.value:.2f}") # log time taken timer_end = Time.now() self.logger.info( f"Extracted ToA={toa.value:.4f}, DM={dm.value:.2f} in " f"{(timer_end - timer_start).to(u.s):.0f}") return # calculate DM range to try # increase dm half range by 5 units for each ms of pulse width # add another unit for each 100 units of DM dm_halfrange = self.config.dm_halfrange * u.pc / u.cm ** 3 + \ 5 * tsamp_effective / 1000. * u.pc / u.cm ** 3 + \ dm / 100 # get dms, ensure detection DM is in the list if self.config.ndm % 2 == 0: dms = np.linspace(dm - dm_halfrange, dm + dm_halfrange, self.config.ndm + 1)[:-1] else: dms = np.linspace(dm - dm_halfrange, dm + dm_halfrange, self.config.ndm) # ensure DM does not go below zero by shifting the entire range if this happens mindm = min(dms) if mindm < 0: dms += mindm # Dedisperse # initialize DM-time array self.data_dm_time = np.zeros((self.config.ndm, self.config.ntime)) for dm_ind, dm_val in enumerate(dms): # copy the data and dedisperse data = copy.deepcopy(self.data) data.dedisperse(dm_val.to(u.pc / u.cm**3).value) # apply any remaining downsampling data.downsample(postdownsamp) # cut of excess bins and store self.data_dm_time[dm_ind] = data.data.sum( axis=0)[:self.config.ntime] # apply downsampling in time and freq to global freq-time data self.data.downsample(postdownsamp) self.data.subband(nsub=self.config.nfreq) # cut off extra bins self.data.data = self.data.data[:, :self.config.ntime] # roll data to put brightest pixel in the centre brightest_pixel = np.argmax(self.data.data.sum(axis=0)) shift = self.config.ntime // 2 - brightest_pixel self.data.data = np.roll(self.data.data, shift, axis=1) # apply the same roll to the DM-time data for row in range(self.config.ndm): self.data_dm_time[row] = np.roll(self.data_dm_time[row], shift) # calculate effective toa after applying shift # ToDo: only apply this shift if not caused by candidate near start/end of observation toa_effective = toa + shift * tsamp_effective * u.s # create output file output_file = os.path.join( self.output_dir, f'TOA{toa_effective.value:.4f}_DM{dm.value:.2f}_' f'DS{width_best:.0f}_SNR{snrmax:.0f}.hdf5') params_amber = (dm.value, snr, toa.value, downsamp) params_opt = (snrmax, toa_effective.value, width_best) self._store_data(output_file, sb, tsamp_effective, dms, params_amber, params_opt) # put path to file on output queue to be picked up by classifier self.output_queue.put(output_file) self.ncand_above_threshold.value += 1 # log time taken timer_end = Time.now() self.logger.info( f"Successfully extracted ToA={toa.value:.4f}, DM={dm.value:.2f} in " f"{(timer_end - timer_start).to(u.s):.0f}")
def start_observation(self, config_file, service=None): """ Start an observation :param str config_file: Path to observation config file :param str service: Which service to send start_observation to (default: all) :return: status, reply """ self.logger.info( "Received start_observation command with config file {}".format( config_file)) # check if config file exists if not os.path.isfile(config_file): self.logger.error("File not found: {}".format(config_file)) return "Error", "Failed: config file not found" # load config if config_file.endswith('.yaml'): config = self._load_yaml(config_file) elif config_file.endswith('.parset'): config = self._load_parset(config_file) else: self.logger.error( "Failed to determine config file type from {}".format( config_file)) return "Error", "Failed: unknown config file type" # check if process_triggers is enabled if not config['proctrigger']: self.logger.info( "Process triggers is disabled; not starting observation") return "Success", "Process triggers disabled - not starting" # store the config for future reference config_output_dir = os.path.join(self.parset_dir, config['datetimesource']) try: util.makedirs(config_output_dir) except Exception as e: raise DARCMasterException( "Cannot create config output directory: {}".format(e)) try: copy2(config_file, config_output_dir) except Exception as e: self.logger.error("Could not store config file: {}".format(e)) # initialize observation # ensure services are running if service is None: for s in self.services: self.start_service(s) else: self.start_service(service) # check host type if self.hostname == MASTER: host_type = 'master' elif self.hostname in WORKERS: host_type = 'worker' else: self.logger.error("Running on unknown host: {}".format( self.hostname)) return "Error", "Failed: running on unknown host" # create command command = { 'command': 'start_observation', 'obs_config': config, 'host_type': host_type } # wait until start time utc_start = Time(config['startpacket'] / TIME_UNIT, format='unix') utc_end = utc_start + TimeDelta(config['duration'], format='sec') # if end time is in the past, only start offline processing and processor if utc_end < Time.now(): self.logger.warning( "End time in past! Only starting offline processing and/or processor" ) if service is None: self.offline_queue.put(command) self.processor_queue.put(command) return "Warning", "Only offline processing and processor started" elif service == 'offline_processing': self.offline_queue.put(command) return "Warning", "Only offline processing started" elif service == 'processor': self.processor_queue.put(command) return "Warning", "Only processor started" else: return "Error", "Can only start offline processing and processor when end time is in past" t_setup = utc_start - TimeDelta(self.setup_time, format='sec') self.logger.info("Starting observation at {}".format(t_setup.isot)) util.sleepuntil_utc(t_setup) if service is None: # clear queues, then send command for queue in self.all_queues: util.clear_queue(queue) for queue in self.all_queues: queue.put(command) return "Success", "Observation started" else: # only start specified service. As this is only used in case e.g. something fails during # an observation, do not clear the queues first queue = self.get_queue(service) queue.put(command) return "Warning", "Only observation for {} started".format(service)
def test_polcal(self): """ Test automated IQUV dumps during polcal observation """ # create input queue queue = mp.Queue() # init DADA Trigger dadatrigger = DADATrigger(queue) # set IQUV dump size, interval, max number of dumps dadatrigger.polcal_dump_size = 1 dadatrigger.polcal_interval = 2 dadatrigger.polcal_max_dumps = 5 # start dadatrigger dadatrigger.start() # timeout for receiving first dump event # must be larger than polcal_interval timeout = 5 # open a listening socket for stokes IQUV events try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(("", dadatrigger.port_iquv)) sock.listen(5) sock.settimeout(timeout) except socket.error as e: self.fail( "Failed to set up listening socket for events: {}".format(e)) # observation settings tstart = Time.now() + TimeDelta(2, format='sec') # set duration. dada_trigger first sleeps for dump_interval seconds, then continues # dumping until dump end time is past obs end time, or max dumps is reached # if max dumps is not reached, the total number of dumps is duration/interval - 1 ndump = 3 duration = (ndump + 1) * dadatrigger.polcal_interval # create parset # source name must contain polcal parset = {'task.source.name': 'testpolcal', 'task.source.beam': 0} parset_str = '' for k, v in parset.items(): parset_str += '{} = {}\n'.format(k, v) # create full configuration obs_config = { 'startpacket': tstart.unix * TIME_UNIT, 'duration': duration, 'beam': 0, 'parset': util.encode_parset(parset_str) } # start observation queue.put({ 'command': 'start_observation', 'obs_config': obs_config, 'reload_conf': False }) # sleep until observation is over util.sleepuntil_utc(tstart + TimeDelta(duration, format='sec')) # receive events received_events = [] while True: try: client, adr = sock.accept() except socket.timeout: break # receive event. Work around bug on MAC if sys.platform == 'Darwin': received = False while not received: try: event = client.recv(1024).decode() received = True except socket.error as e: if e.errno == errno.EAGAIN: sleep(.1) else: raise else: event = client.recv(1024).decode() client.close() received_events.append(event) # close the socket sock.close() # stop dada trigger queue.put('stop') self.assertTrue(ndump == len(received_events))