def _process_observation(self):
"""
Process observation
"""
# wait until the observation finishes
start_processing_time = Time(
self.obs_config['parset']['task.stopTime'])
self.logger.info("Sleeping until {}".format(start_processing_time.iso))
self.status = 'Observation in progress'
util.sleepuntil_utc(start_processing_time, event=self.stop_event)
try:
# generate observation info files
self.status = 'Generating observation info files'
info, coordinates = self._generate_info_file()
# wait for all result files to be present
self.status = 'Waiting for nodes to finish processing'
self._wait_for_workers()
# combine results, copy to website and generate email
self.status = 'Combining node results'
email, attachments = self._process_results(info, coordinates)
# publish results on web link and send email
self.status = 'Sending results to website'
self._publish_results(email, attachments)
self.status = 'Sending results to email'
self._send_email(email, attachments)
self.status = 'Done'
except Exception as e:
self.logger.error(
f"Failed to process observation. Status = {self.status}: {type(e)}: {e}"
)
def test_processor_obs(self):
# start processor
self.processor.start()
# start amber listener and processor
cmd = {'command': 'start_observation', 'obs_config': self.header, 'reload': False}
self.amber_queue.put(cmd)
self.processor_queue.put(cmd)
# at start time, read data into buffer, other processes are already set up and waiting for data
util.sleepuntil_utc(self.tstart)
self.diskdb_proc.start()
# wait until processes are done
for proc in (self.diskdb_proc, self.amber_proc, self.dadafilterbank_proc):
proc.join()
# stop observation
self.amber_queue.put({'command': 'stop_observation'})
self.processor.source_queue.put({'command': 'stop_observation'})
# wait for processor to exit
self.processor.join()
# stop services
self.amber_listener.source_queue.put('stop')
self.amber_listener.join()
def _start_observation_master(self, obs_config, reload=True):
"""
Start observation on master node
Generated observation summary and send email once all workers are done
:param dict obs_config: Observation config
:param bool reload: reload service settings (default: True)
"""
self.logger.info("Starting observation on master node")
# reload config
if reload:
self.load_config()
# create result dir
try:
util.makedirs(obs_config['result_dir'])
except Exception as e:
self.logger.error("Failed to create results directory")
raise OfflineProcessingException(
"Failed to create result directory: {}".format(e))
# create general info file
self._get_overview(obs_config)
# create coordinates file
coord_cb00 = self._get_coordinates(obs_config)
# wait until end time + delay
start_processing_time = Time(
obs_config['parset']['task.stopTime']) + TimeDelta(self.delay,
format='sec')
self.logger.info("Sleeping until {}".format(start_processing_time.iso))
util.sleepuntil_utc(start_processing_time, event=self.stop_event)
# start emailer
cmd = "python2 {emailer} {result_dir} '{beams}' {ntabs}".format(
**obs_config)
self.logger.info("Running {}".format(cmd))
os.system(cmd)
# fetch known FRB candidates
# do this _after_ email is sent, as then we are sure the grouped_pulses file exists for all beams
if coord_cb00 is not None:
self._plot_known_frb_cands(obs_config, coord_cb00)
else:
self.logger.warning(
"Skipping plotting of known FRB candidates: CB00 coordinates not available"
)
self.logger.info(
"Finished processing of observation {output_dir}".format(
**obs_config))
def _start_observation_master(self, obs_config, reload=True):
"""
Start observation on master node
Generated observation summary and send email once all workers are done
:param dict obs_config: Observation config
:param bool reload: reload service settings (default: True)
"""
# reload config
if reload:
self.load_config()
if not self.full_processing_enabled:
self.logger.info("Full processing disabled - nothing to run on master node")
return
self.logger.info("Starting observation on master node")
# create result dir
try:
util.makedirs(obs_config['result_dir'])
except Exception as e:
self.logger.error("Failed to create results directory")
raise OfflineProcessingException("Failed to create result directory: {}".format(e))
# create general info file
self._get_overview(obs_config)
# create coordinates file
self._get_coordinates(obs_config)
# wait until end time + delay
start_processing_time = Time(obs_config['parset']['task.stopTime']) + TimeDelta(self.delay, format='sec')
self.logger.info("Sleeping until {}".format(start_processing_time.iso))
util.sleepuntil_utc(start_processing_time, event=self.stop_event)
# start emailer
cmd = "python2 {emailer} {result_dir} '{beams}' {ntabs}".format(**obs_config)
self.logger.info("Running {}".format(cmd))
os.system(cmd)
# fetch known FRB candidates
# do this _after_ email is sent, as then we are sure the grouped_pulses file exists for all beams
# NOTE: disabled because:
# 1) the (external) plotting script does not work without an active X session (should be fixable)
# 2) the script re-uses the same temporary directory, and can run only one copy at a time
# While these issues are not fixed, the plotting is not run automatically
# if coord_cb00 is not None:
# self._plot_known_frb_cands(obs_config, coord_cb00)
# else:
# self.logger.warning("Skipping plotting of known FRB candidates: CB00 coordinates not available")
self.logger.info("Finished processing of observation {output_dir}".format(**obs_config))
def test_full_run(self):
"""
Do a full pipeline run
"""
# Buffers are already set up
# start AMBER
self.t_amber.start()
# Give it some time to start
sleep(1)
# start DARC observation 5s in the future
tstart = Time.now() + TimeDelta(5, format='sec')
# add start time to config
self.settings['startpacket'] = tstart.unix * TIME_UNIT
# Todo: ensure DARC waits for start time
# start observation
cmd = {'command': 'start_observation', 'obs_config': self.settings, 'reload_conf': False}
self.listener.source_queue.put(cmd)
self.clustering.source_queue.put(cmd)
# start writer at start time
util.sleepuntil_utc(tstart)
self.t_disk_to_db.start()
# Now AMBER should start producing triggers
# Listen for any IQUV triggers in thread
self.t_dbevent.start()
# wait for the writer to finish
self.t_disk_to_db.join()
# AMBER may still be processing, wait for that to finish too
self.t_amber.join()
# stop the observation
self.listener.source_queue.put({'command': 'stop_observation'})
self.clustering.source_queue.put({'command': 'stop_observation'})
# verify an IQUV event was received
try:
event = self.event_queue.get(timeout=1)
except Empty:
event = None
# event looks like (depending on start time):
# N_EVENTS 2
# 2019-10-14-13:14:26.016
# 2019-10-14-13:14:37 137 2019-10-14-13:14:39 185 12.3749 26.4 50.0 21
# 2019-10-14-13:14:37 849 2019-10-14-13:14:39 897 14.1344 31.8 50.0 21
self.assertTrue(event is not None)
print("Received event(s):\n{}".format(event))
def polcal_dumps(self, obs_config):
"""
Automatically dump IQUV data at regular intervals for polcal calibrator observations
:param dict obs_config: Observation config
"""
tstart = Time(obs_config['startpacket'] / TIME_UNIT, format='unix')
duration = TimeDelta(obs_config['duration'], format='sec')
tend = tstart + duration
# round up polcal dump size to nearest 1.024 s
dump_size = TimeDelta(np.ceil(self.polcal_dump_size / 1.024) * 1.024,
format='sec')
dump_interval = TimeDelta(self.polcal_interval, format='sec')
# sleep until first trigger time
util.sleepuntil_utc(tstart + dump_interval, event=self.stop_event)
# run until trigger would be end past end time
# add a second to avoid trigger running a little bit over end time
# also stay below global limit on number of dumps during one obs
# TODO: also set a total length limit
ndump = 0
while Time.now() + dump_interval - TimeDelta(
1.0, format='sec') < tend and ndump < self.polcal_max_dumps:
# generate an IQUV trigger
params = {'utc_start': tstart.iso.replace(' ', '-')}
# trigger start time: now, rounded to nearest 1.024s since utc start
dt = TimeDelta(np.round((Time.now() - tstart).sec / 1.024) * 1.024,
format='sec')
# trigger start/end times
event_start_full = tstart + dt
event_end_full = tstart + dt + dump_size
# convert to dada_dbevent format
event_start, event_start_frac = event_start_full.iso.split('.')
event_end, event_end_frac = event_end_full.iso.split('.')
# store to params
params['event_start'] = event_start.replace(' ', '-')
params['event_start_frac'] = event_start_frac
params['event_end'] = event_end.replace(' ', '-')
params['event_end_frac'] = event_end_frac
event = "N_EVENTS 1\n{utc_start}\n{event_start} {event_start_frac} {event_end} {event_end_frac} " \
"0 0 0 0".format(**params) # dm, snr, width, beam are 0
self.logger.info(
"Sending automated polcal IQUV dump event: {}".format(params))
self.send_events(event, 'IQUV')
# keep track of number of performed dumps
ndump += 1
# sleep
self.stop_event.wait(dump_interval.sec)
def _start_observation_worker(self, obs_config, reload=True):
"""
Start observation on worker node:
#. Candidate clustering
#. Extraction of filterbank data
#. ML classification
:param dict obs_config: Observation config
:param bool reload: reload service settings (default: True)
"""
self.logger.info("Starting observation on worker node")
# reload config
if reload:
self.load_config()
# create result dir
try:
util.makedirs(obs_config['result_dir'])
except Exception as e:
self.logger.error("Failed to create results directory")
raise OfflineProcessingException(
"Failed to create result directory: {}".format(e))
# TAB or IAB mode
if obs_config['ntabs'] == 1:
obs_config['mode'] = 'IAB'
trigger_output_file = "{output_dir}/triggers/data/data_00_full.hdf5".format(
**obs_config)
else:
obs_config['mode'] = 'TAB'
trigger_output_file = "{output_dir}/triggers/data/data_full.hdf5".format(
**obs_config)
# wait until end time + delay
start_processing_time = Time(
obs_config['parset']['task.stopTime']) + TimeDelta(self.delay,
format='sec')
self.logger.info("Sleeping until {}".format(start_processing_time.iso))
util.sleepuntil_utc(start_processing_time, event=self.stop_event)
# fold pulsar if this is beam 0 and a test pulsar is being observed
try:
source = obs_config['parset']['task.source.name']
ref_beam = int(obs_config['parset']['task.source.beam'])
# Requires access to parset
if source in self.test_pulsars and (obs_config['beam']
== ref_beam):
self.logger.info("Test pulsar detected: {}".format(source))
self._fold_pulsar(source, obs_config)
except Exception as e:
self.logger.error("Pulsar folding failed: {}".format(e))
# run calibration tools if this is a calibrator scan
# these have "drift" in the source name
try:
source = obs_config['parset']['task.source.name']
if 'drift' in source:
# split into actual source name and which beams were scanned
name, beam_range = source.split('drift')
# parse beam range, can be one beam or start/end beam
if len(beam_range) == 2:
# one beam
drift_beams = [int(beam_range)]
elif len(beam_range) == 4:
# start and end beam
sbeam = int(beam_range[:2])
ebeam = int(beam_range[2:])
drift_beams = range(sbeam, ebeam + 1)
else:
self.logger.error(
"Failed to parse beam range for calibrator scan: {}".
format(source))
drift_beams = []
# run calibration tools if this is a calibrator scan of this beam
if name in self.calibrators and (obs_config['beam']
in drift_beams):
self.logger.info(
"Calibrator scan through this beam detected: {}".
format(source))
self._run_calibration_tools(name, obs_config)
except Exception as e:
self.logger.error("Calibration tools failed: {}".format(e))
# create trigger directory
trigger_dir = "{output_dir}/triggers".format(**obs_config)
try:
util.makedirs(trigger_dir)
except Exception as e:
self.logger.error("Failed to create triggers directory")
raise OfflineProcessingException(
"Failed to create triggers directory: {}".format(e))
# change to trigger directory
try:
os.chdir(trigger_dir)
except Exception as e:
self.logger.error(
"Failed to cd to triggers directory {}: {}".format(
trigger_dir, e))
# create subdir
try:
util.makedirs('data')
except Exception as e:
self.logger.error("Failed to create triggers/data directory")
raise OfflineProcessingException(
"Failed to create triggers/data directory: {}".format(e))
# merge the trigger files
self.logger.info("Merging raw trigger files")
numcand_raw = self._merge_triggers(obs_config)
# Run clustering for each SB in TAB mode if enabled
if self.process_sb and obs_config['mode'] == 'TAB':
self.logger.info("Clustering candidates in SB mode")
tstart = Time.now()
# spread the SBs over the allowed number of threads
# output grouped pulses file will only be generated by thread including SB00
chunks = np.array_split(range(obs_config['nsynbeams']),
self.numthread)
numcand_all = np.zeros(self.numthread)
filterbank_prefix = "{output_dir}/filterbank/CB{beam:02d}".format(
**obs_config)
threads = []
self.logger.info(
"Starting trigger clustering with {} threads".format(
self.numthread))
for ind, chunk in enumerate(chunks):
# pick the SB range
sbmin, sbmax = min(chunk), max(chunk)
# create thread
thread = threading.Thread(target=self._cluster,
args=[obs_config, filterbank_prefix],
kwargs={
'out': numcand_all,
'sbmin': sbmin,
'sbmax': sbmax,
'ind': ind
})
thread.daemon = True
threads.append(thread)
thread.start()
# wait until all are done
for thread in threads:
thread.join()
# gather results
if self.process_sb:
# each element equal
numcand_grouped = int(numcand_all[0])
else:
# each element is one TAB
numcand_grouped = int(np.sum(numcand_all[numcand_all != -1]))
# Run clustering for IAB / each TAB
else:
self.logger.info("Clustering candidates in {} mode".format(
obs_config['mode']))
tstart = Time.now()
if obs_config['mode'] == 'IAB':
filterbank_file = "{output_dir}/filterbank/CB{beam:02d}.fil".format(
**obs_config)
numcand_grouped = self._cluster(obs_config, 0, filterbank_file)
elif obs_config['mode'] == 'TAB':
numcand_all = np.zeros(obs_config['ntabs'])
# max numtread tabs per run; so ntabs / numthread chunks
n_chunk = int(
np.ceil(obs_config['ntabs'] / float(self.numthread)))
chunks = np.array_split(range(obs_config['ntabs']), n_chunk)
self.logger.info(
"Starting trigger clustering with {} chunks of {} threads".
format(n_chunk, self.numthread))
# start the threads
for tab_set in chunks:
threads = []
for tab in tab_set:
filterbank_file = "{output_dir}/filterbank/CB{beam:02d}_{tab:02d}.fil".format(
tab=tab, **obs_config)
thread = threading.Thread(
target=self._cluster,
args=[obs_config, filterbank_file],
kwargs={
'out': numcand_all,
'tab': tab
})
thread.daemon = True
threads.append(thread)
thread.start()
# wait until all are done
for thread in threads:
thread.join()
# gather results
numcand_grouped = int(np.sum(numcand_all[numcand_all != -1]))
tend = Time.now()
self.logger.info("Trigger clustering took {}s".format(
(tend - tstart).sec))
# Create one hdf5 file for entire CB
if obs_config['mode'] == 'TAB':
self.logger.info("Merging output HDF5 files")
numcand_merged = self._merge_hdf5(obs_config, trigger_output_file)
# TEMP so that IAB still works
else:
numcand_merged = 9999
# Run classifier
if numcand_merged != 0:
self.logger.info("Classifying candidates")
output_prefix = self._classify(obs_config, trigger_output_file)
else:
self.logger.info(
"No candidates post-merge. Not running classifier")
output_prefix = ''
# Merge PDFs
if numcand_merged > 0:
self.logger.info("Merging classifier output files")
self._merge_plots(obs_config)
else:
self.logger.info(
"No candidates found post-classifier, not creating merged PDF")
# Centralize results
self.logger.info("Gathering results")
kwargs = {
'output_prefix': output_prefix,
'data_file': trigger_output_file,
'numcand_raw': numcand_raw,
'numcand_grouped': numcand_grouped
}
self._gather_results(obs_config, **kwargs)
self.logger.info(
"Finished processing of observation {output_dir}".format(
**obs_config))
return
def _extract(self, dm, snr, toa, downsamp, sb):
"""
Execute data extraction
:param float dm: Dispersion Measure (pc/cc)
:param float snr: AMBER S/N
:param float toa: Arrival time at top of band (s)
:param int downsamp: Downsampling factor
:param int sb: Synthesized beam index
"""
# keep track of the time taken to process this candidate
timer_start = Time.now()
# add units
dm = dm * u.pc / u.cm**3
toa = toa * u.s
# If the intra-channel smearing timescale is several samples, downsample
# by at most 1/4th of that factor
chan_width = BANDWIDTH / float(NCHAN)
t_smear = util.dm_to_smearing(dm, self.obs_config['freq'] * u.MHz,
chan_width)
predownsamp = max(
1,
int(
t_smear.to(u.s).value / self.filterbank_reader.header.tsamp /
4))
# calculate remaining downsampling to do after dedispersion
postdownsamp = max(1, downsamp // predownsamp)
# total downsampling factor (can be slightly different from original downsampling)
self.logger.debug(
f"Original dowsamp: {downsamp}, new downsamp: {predownsamp * postdownsamp} "
f"for ToA={toa.value:.4f}, DM={dm.value:.2f}")
downsamp_effective = predownsamp * postdownsamp
tsamp_effective = self.filterbank_reader.header.tsamp * downsamp_effective
# calculate start/end bin we need to load from filterbank
# DM delay in units of samples
dm_delay = util.dm_to_delay(
dm, self.obs_config['min_freq'] * u.MHz,
self.obs_config['min_freq'] * u.MHz + BANDWIDTH)
sample_delay = dm_delay.to(
u.s).value // self.filterbank_reader.header.tsamp
# number of input bins to store in final data
ntime = self.config.ntime * downsamp_effective
start_bin = int(
toa.to(u.s).value // self.filterbank_reader.header.tsamp -
.5 * ntime)
# ensure start bin is not before start of observation
if start_bin < 0:
self.logger.warning(
f"Start bin before start of file, shifting for ToA={toa.value:.4f}, DM={dm.value:.2f}"
)
start_bin = 0
# number of bins to load is number to store plus dm delay
nbin = int(ntime + sample_delay)
end_bin = start_bin + nbin
# Wait if the filterbank does not have enough samples yet
# first update filterbank parameters to have correct nsamp
self.filterbank_reader.get_header()
tstart = Time(self.obs_config['startpacket'] / TIME_UNIT,
format='unix')
# wait until this much of the filterbank should be present on disk
# start time, plus last bin to load, plus extra delay
# (because filterbank is not immediately flushed to disk)
twait = tstart + end_bin * self.filterbank_reader.header.tsamp * u.s + self.config.delay * u.s
first_loop = True
while end_bin >= self.filterbank_reader.header.nsamples:
if first_loop:
# wait until data should be present on disk
self.logger.debug(
f"Waiting until at least {twait.isot} for filterbank data to be present on disk "
f"for ToA={toa.value:.4f}, DM={dm.value:.2f}")
util.sleepuntil_utc(twait, event=self.stop_event)
first_loop = False
else:
# data should be there, but is not. Wait just a short time, then check again
self.stop_event.wait(.5)
# re-read the number of samples to check if the data are available now
self.filterbank_reader.get_header()
# if we are past the end time of the observation, we give up and try shifting the end bin instead
tend = tstart + self.obs_config[
'duration'] * u.s + self.config.delay * u.s
if Time.now() > tend:
break
# if start time before start of file, or end time beyond end of file, shift the start/end time
# first update filterbank parameters to have correct nsamp
self.filterbank_reader.get_header()
if end_bin >= self.filterbank_reader.header.nsamples:
# if start time is also beyond the end of the file, we cannot process this candidate and give an error
if start_bin >= self.filterbank_reader.header.nsamples:
self.logger.error(
f"Start bin beyond end of file, error in filterbank data? Skipping"
f" ToA={toa.value:.4f}, DM={dm.value:.2f}")
# log time taken
timer_end = Time.now()
self.logger.info(
f"Processed ToA={toa.value:.4f}, DM={dm.value:.2f} "
f"in {(timer_end - timer_start).to(u.s):.0f}")
return
self.logger.warning(
f"End bin beyond end of file, shifting for ToA={toa.value:.4f}, DM={dm.value:.2f}"
)
diff = end_bin - self.filterbank_reader.header.nsamples + 1
start_bin -= diff
# load the data. Store as attribute so it can be accessed from other methods (ok as we only run
# one extraction at a time)
try:
self.data = self.filterbank_reader.load_single_sb(
sb, start_bin, nbin)
except ValueError as e:
self.logger.error(
f"Failed to load filterbank data for ToA={toa.value:.4f}, DM={dm.value:.2f}: {e}"
)
# log time taken
timer_end = Time.now()
self.logger.info(
f"Extracted ToA={toa.value:.4f}, DM={dm.value:.2f} "
f"in {(timer_end - timer_start).to(u.s):.0f}")
return
# apply AMBER RFI mask
if self.config.rfi_apply_mask:
self.data[self.rfi_mask, :] = 0.
# run rfi cleaning
if self.config.rfi_clean_type is not None:
self._rficlean()
# apply predownsampling
self.data.downsample(predownsamp)
# subtract median
self.data.data -= np.median(self.data.data, axis=1, keepdims=True)
# get S/N and width at AMBER DM
self.data.dedisperse(dm.to(u.pc / u.cm**3).value)
# create timeseries
timeseries = self.data.data.sum(axis=0)[:ntime]
# get S/N and width
# range of widths for S/N determination. Never go above 250 samples,
# which is typically RFI even without pre-downsampling
widths = np.arange(max(1, postdownsamp // 2),
min(250, postdownsamp * 2))
snrmax, width_best = util.calc_snr_matched_filter(timeseries,
widths=widths)
# correct for already applied downsampling
width_best *= predownsamp
# if max S/N is below local threshold, skip this trigger
if snrmax < self.config.snr_min_local:
self.logger.warning(
f"Skipping trigger with S/N ({snrmax:.2f}) below local threshold, "
f"ToA={toa.value:.4f}, DM={dm.value:.2f}")
# log time taken
timer_end = Time.now()
self.logger.info(
f"Extracted ToA={toa.value:.4f}, DM={dm.value:.2f} in "
f"{(timer_end - timer_start).to(u.s):.0f}")
return
# calculate DM range to try
# increase dm half range by 5 units for each ms of pulse width
# add another unit for each 100 units of DM
dm_halfrange = self.config.dm_halfrange * u.pc / u.cm ** 3 + \
5 * tsamp_effective / 1000. * u.pc / u.cm ** 3 + \
dm / 100
# get dms, ensure detection DM is in the list
if self.config.ndm % 2 == 0:
dms = np.linspace(dm - dm_halfrange, dm + dm_halfrange,
self.config.ndm + 1)[:-1]
else:
dms = np.linspace(dm - dm_halfrange, dm + dm_halfrange,
self.config.ndm)
# ensure DM does not go below zero by shifting the entire range if this happens
mindm = min(dms)
if mindm < 0:
dms += mindm
# Dedisperse
# initialize DM-time array
self.data_dm_time = np.zeros((self.config.ndm, self.config.ntime))
for dm_ind, dm_val in enumerate(dms):
# copy the data and dedisperse
data = copy.deepcopy(self.data)
data.dedisperse(dm_val.to(u.pc / u.cm**3).value)
# apply any remaining downsampling
data.downsample(postdownsamp)
# cut of excess bins and store
self.data_dm_time[dm_ind] = data.data.sum(
axis=0)[:self.config.ntime]
# apply downsampling in time and freq to global freq-time data
self.data.downsample(postdownsamp)
self.data.subband(nsub=self.config.nfreq)
# cut off extra bins
self.data.data = self.data.data[:, :self.config.ntime]
# roll data to put brightest pixel in the centre
brightest_pixel = np.argmax(self.data.data.sum(axis=0))
shift = self.config.ntime // 2 - brightest_pixel
self.data.data = np.roll(self.data.data, shift, axis=1)
# apply the same roll to the DM-time data
for row in range(self.config.ndm):
self.data_dm_time[row] = np.roll(self.data_dm_time[row], shift)
# calculate effective toa after applying shift
# ToDo: only apply this shift if not caused by candidate near start/end of observation
toa_effective = toa + shift * tsamp_effective * u.s
# create output file
output_file = os.path.join(
self.output_dir, f'TOA{toa_effective.value:.4f}_DM{dm.value:.2f}_'
f'DS{width_best:.0f}_SNR{snrmax:.0f}.hdf5')
params_amber = (dm.value, snr, toa.value, downsamp)
params_opt = (snrmax, toa_effective.value, width_best)
self._store_data(output_file, sb, tsamp_effective, dms, params_amber,
params_opt)
# put path to file on output queue to be picked up by classifier
self.output_queue.put(output_file)
self.ncand_above_threshold.value += 1
# log time taken
timer_end = Time.now()
self.logger.info(
f"Successfully extracted ToA={toa.value:.4f}, DM={dm.value:.2f} in "
f"{(timer_end - timer_start).to(u.s):.0f}")
def start_observation(self, config_file, service=None):
"""
Start an observation
:param str config_file: Path to observation config file
:param str service: Which service to send start_observation to (default: all)
:return: status, reply
"""
self.logger.info(
"Received start_observation command with config file {}".format(
config_file))
# check if config file exists
if not os.path.isfile(config_file):
self.logger.error("File not found: {}".format(config_file))
return "Error", "Failed: config file not found"
# load config
if config_file.endswith('.yaml'):
config = self._load_yaml(config_file)
elif config_file.endswith('.parset'):
config = self._load_parset(config_file)
else:
self.logger.error(
"Failed to determine config file type from {}".format(
config_file))
return "Error", "Failed: unknown config file type"
# check if process_triggers is enabled
if not config['proctrigger']:
self.logger.info(
"Process triggers is disabled; not starting observation")
return "Success", "Process triggers disabled - not starting"
# store the config for future reference
config_output_dir = os.path.join(self.parset_dir,
config['datetimesource'])
try:
util.makedirs(config_output_dir)
except Exception as e:
raise DARCMasterException(
"Cannot create config output directory: {}".format(e))
try:
copy2(config_file, config_output_dir)
except Exception as e:
self.logger.error("Could not store config file: {}".format(e))
# initialize observation
# ensure services are running
if service is None:
for s in self.services:
self.start_service(s)
else:
self.start_service(service)
# check host type
if self.hostname == MASTER:
host_type = 'master'
elif self.hostname in WORKERS:
host_type = 'worker'
else:
self.logger.error("Running on unknown host: {}".format(
self.hostname))
return "Error", "Failed: running on unknown host"
# create command
command = {
'command': 'start_observation',
'obs_config': config,
'host_type': host_type
}
# wait until start time
utc_start = Time(config['startpacket'] / TIME_UNIT, format='unix')
utc_end = utc_start + TimeDelta(config['duration'], format='sec')
# if end time is in the past, only start offline processing and processor
if utc_end < Time.now():
self.logger.warning(
"End time in past! Only starting offline processing and/or processor"
)
if service is None:
self.offline_queue.put(command)
self.processor_queue.put(command)
return "Warning", "Only offline processing and processor started"
elif service == 'offline_processing':
self.offline_queue.put(command)
return "Warning", "Only offline processing started"
elif service == 'processor':
self.processor_queue.put(command)
return "Warning", "Only processor started"
else:
return "Error", "Can only start offline processing and processor when end time is in past"
t_setup = utc_start - TimeDelta(self.setup_time, format='sec')
self.logger.info("Starting observation at {}".format(t_setup.isot))
util.sleepuntil_utc(t_setup)
if service is None:
# clear queues, then send command
for queue in self.all_queues:
util.clear_queue(queue)
for queue in self.all_queues:
queue.put(command)
return "Success", "Observation started"
else:
# only start specified service. As this is only used in case e.g. something fails during
# an observation, do not clear the queues first
queue = self.get_queue(service)
queue.put(command)
return "Warning", "Only observation for {} started".format(service)
def test_polcal(self):
"""
Test automated IQUV dumps during polcal observation
"""
# create input queue
queue = mp.Queue()
# init DADA Trigger
dadatrigger = DADATrigger(queue)
# set IQUV dump size, interval, max number of dumps
dadatrigger.polcal_dump_size = 1
dadatrigger.polcal_interval = 2
dadatrigger.polcal_max_dumps = 5
# start dadatrigger
dadatrigger.start()
# timeout for receiving first dump event
# must be larger than polcal_interval
timeout = 5
# open a listening socket for stokes IQUV events
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(("", dadatrigger.port_iquv))
sock.listen(5)
sock.settimeout(timeout)
except socket.error as e:
self.fail(
"Failed to set up listening socket for events: {}".format(e))
# observation settings
tstart = Time.now() + TimeDelta(2, format='sec')
# set duration. dada_trigger first sleeps for dump_interval seconds, then continues
# dumping until dump end time is past obs end time, or max dumps is reached
# if max dumps is not reached, the total number of dumps is duration/interval - 1
ndump = 3
duration = (ndump + 1) * dadatrigger.polcal_interval
# create parset
# source name must contain polcal
parset = {'task.source.name': 'testpolcal', 'task.source.beam': 0}
parset_str = ''
for k, v in parset.items():
parset_str += '{} = {}\n'.format(k, v)
# create full configuration
obs_config = {
'startpacket': tstart.unix * TIME_UNIT,
'duration': duration,
'beam': 0,
'parset': util.encode_parset(parset_str)
}
# start observation
queue.put({
'command': 'start_observation',
'obs_config': obs_config,
'reload_conf': False
})
# sleep until observation is over
util.sleepuntil_utc(tstart + TimeDelta(duration, format='sec'))
# receive events
received_events = []
while True:
try:
client, adr = sock.accept()
except socket.timeout:
break
# receive event. Work around bug on MAC
if sys.platform == 'Darwin':
received = False
while not received:
try:
event = client.recv(1024).decode()
received = True
except socket.error as e:
if e.errno == errno.EAGAIN:
sleep(.1)
else:
raise
else:
event = client.recv(1024).decode()
client.close()
received_events.append(event)
# close the socket
sock.close()
# stop dada trigger
queue.put('stop')
self.assertTrue(ndump == len(received_events))
