def _create_ring_buffer(self, bufferSize, blocks, key, numa_node):
"""
@brief Create a ring buffer of given size with given key on specified numa node.
Adds and register an appropriate sensor to thw list
"""
# always clear buffer first. Allow fail here
yield command_watcher("dada_db -d -k {key}".format(key=key), allow_fail=True)
cmd = "numactl --cpubind={numa_node} --membind={numa_node} dada_db -k {key} -n {blocks} -b {bufferSize} -p -l".format(key=key, blocks=blocks, bufferSize=bufferSize, numa_node=numa_node)
log.debug("Running command: {0}".format(cmd))
yield command_watcher(cmd)
M = DbMonitor(key, self._buffer_status_handle)
M.start()
self._dada_buffers.append({'key': key, 'monitor': M})
def __ansible_subplay_executioner(self, play, additional_args=""):
"""
Uses ansible-playbook to execute thegiven play by writing it in a tempfile.
Args:
play The play to be executed
additional_args Additional args added to the ansible execution, e.g. --tags=stop
"""
playfile = tempfile.NamedTemporaryFile(delete=False)
yaml.dump([play], playfile)
playfile.close()
try:
yield command_watcher(
"ansible-playbook -i {} {} {}".format(self.__inventory,
playfile.name,
additional_args),
env={
"ANSIBLE_ROLES_PATH":
os.path.join(self.__edd_ansible_git_repository_folder,
"roles")
},
timeout=300)
except Exception as E:
playfile.unlink()
raise RuntimeError("Error {} processing play:\n {}".format(
E, yaml.dump(play)))
def deconfigure(self):
"""
Deconfigure the gated spectrometer pipeline.
Clears all dada buffers.
"""
log.info("Deconfiguring EDD backend")
if self.previous_state == 'streaming':
yield self.capture_stop()
if self._subprocessMonitor is not None:
yield self._subprocessMonitor.stop()
for proc in self._subprocesses:
yield proc.terminate()
self.mkrec_cmd = []
log.debug("Destroying dada buffers")
for k in self._dada_buffers:
k['monitor'].stop()
cmd = "dada_db -d -k {0}".format(k['key'])
log.debug("Running command: {0}".format(cmd))
yield command_watcher(cmd, allow_fail=True)
self._dada_buffers = []
def deconfigure(self):
"""@brief deconfigure the pipeline."""
log.debug("Destroying dada buffers")
for k in self._dada_buffers_monitor:
cmd = "dada_db -d -k {0}".format(k)
log.debug("Running command: {0}".format(cmd))
yield command_watcher(cmd, allow_fail=True)
self._dada_buffers_monitor = []
def _reset_ring_buffer(self, key, numa_node):
"""
@brief Create a ring buffer of given size with given key on specified numa node.
Adds and register an appropriate sensor to thw list
"""
# always clear buffer first. Allow fail here
cmd = "numactl --cpubind={numa_node} --membind={numa_node} dbreset -k {key} --log_level debug".format(
numa_node=numa_node, key=key)
log.debug("Running command: {0}".format(cmd))
yield command_watcher(cmd, allow_fail=True)
def deconfigure(self):
"""@brief deconfigure the dspsr pipeline."""
log.info("Deconfiguring EDD backend")
if self.state == 'runnning':
yield self.capture_stop()
self.state = "deconfiguring"
if self._subprocessMonitor is not None:
self._subprocessMonitor.stop()
for proc in self._subprocesses:
proc.terminate()
self.mkrec_cmd = []
log.debug("Destroying dada buffers")
for k in self._dada_buffers:
k['monitor'].stop()
cmd = "dada_db -d -k {0}".format(k['key'])
log.debug("Running command: {0}".format(cmd))
yield command_watcher(cmd)
self._dada_buffers = []
self.state = "idle"
def measurement_start(self):
"""
@brief Create output directory, create mkrecv header, calculate tempo2 predictor file if needed, start data capture, processing, and monitoring
"""
if self._config["active"] == 0:
log.info("Pipeline is not active")
raise StateChange("ready")
self._timer = Time.now()
self._subprocessMonitor = SubprocessMonitor()
try:
self._png_monitor_callback.stop()
except Exception as E:
log.error("Png monitor already stopped.")
log.exception(E)
try:
self._folder_size_monitor_callback.stop()
except Exception as E:
log.error("File size monitor already stopped.")
log.exception(E)
self._source_name = self.__eddDataStore.getTelescopeDataItem(
"source-name")
ra = self.__eddDataStore.getTelescopeDataItem("ra")
decl = self.__eddDataStore.getTelescopeDataItem("dec")
scannum = self.__eddDataStore.getTelescopeDataItem("scannum")
subscannum = self.__eddDataStore.getTelescopeDataItem("subscannum")
receiver_name = self.__eddDataStore.getTelescopeDataItem("receiver")
project_id = self.__eddDataStore.getTelescopeDataItem("project")
log.info(
"Retrieved data from telescope:\n Source name: {}\n RA = {}, decl = {}, receiver = {}"
.format(self._source_name, ra, decl, receiver_name))
if self._config["mode"] == "Timing":
self.tzpar_file = os.path.join(
self.tzpar_dir,
'{}.par'.format(self._source_name.split("_")[0][1:]))
log.debug("Checking parfile file {}".format(self.tzpar_file))
self.pulsar_flag = is_accessible(self.tzpar_file)
if (parse_tag(self._source_name) !=
"R") and (not self.pulsar_flag):
log.warning(
"source {} is neither pulsar nor calibrator. Will not react until next measurement start"
.format(self._source_name))
raise StateChange("ready")
log.debug("Setting blank image")
self._fscrunch.set_value(BLANK_IMAGE)
self._tscrunch.set_value(BLANK_IMAGE)
self._profile.set_value(BLANK_IMAGE)
log.debug("writing mkrecv header")
self.cuda_number = numa.getInfo()[self.numa_number]['gpus'][0]
log.info(" - Running on cuda core: {}".format(self.cuda_number))
header = self._config["dada_header_params"]
central_freq = header["frequency_mhz"]
self._central_freq.set_value(str(header["frequency_mhz"]))
self._source_name_sensor.set_value(self._source_name)
self._nchannels.set_value(self._config["nchannels"])
self._nbins.set_value(self._config["nbins"])
header["telescope"] = self._config["tempo2_telescope_name"]
log.info(" - Tempo2 telescope name: {}".format(header['telescope']))
c = SkyCoord("{} {}".format(ra, decl), unit=(u.deg, u.deg))
header["ra"] = c.to_string("hmsdms").split(" ")[0].replace(
"h", ":").replace("m", ":").replace("s", "")
header["dec"] = c.to_string("hmsdms").split(" ")[1].replace(
"d", ":").replace("m", ":").replace("s", "")
header["key"] = self._dada_buffers[0]
log.debug(" - Dada key: {}".format(header['key']))
header["mc_source"] = ""
for i in self._config['input_data_streams']:
header["mc_source"] += i["ip"] + ","
header["mc_source"] = header["mc_source"][:-1]
log.info(" - mc source: {}".format(header['mc_source']))
header["mc_streaming_port"] = self._config['input_data_streams'][0][
"port"]
log.info(" - mc streaming port: {}".format(
header['mc_streaming_port']))
header["interface"] = numa.getFastestNic(self.numa_number)[1]['ip']
log.info(" - mc interface: {}".format(header['interface']))
header["sync_time"] = self.sync_epoch
log.info(" - sync time: {}".format(header['sync_time']))
if header['sample_clock'] == "unset":
header["sample_clock"] = float(
self._config['input_data_streams'][0]["sample_rate"]
) # adjsutment for the predecimation factor is done in the amster controller
log.info(" - sample_clock: {}".format(header['sample_clock']))
header["source_name"] = self._source_name
header["obs_id"] = "{0}_{1}".format(scannum, subscannum)
header["filesize"] = int(float(self._config["dada"]["size"]))
log.info(" - obs_id: {}".format(header['obs_id']))
header["receiver_name"] = receiver_name
log.info(" - receiver_name: {}".format(header['receiver_name']))
tstr = Time.now().isot.replace(":", "-")
tdate = tstr.split("T")[0]
log.debug("Setting up the input and scrunch data directories")
if self._config["mode"] == "Timing":
try:
self.in_path = os.path.join("/mnt/", project_id,
tdate, self._source_name,
str(central_freq), tstr,
"raw_data")
self.out_path = os.path.join("/mnt/", project_id, tdate,
self._source_name,
str(central_freq), tstr,
"combined_data")
log.debug("Creating directories")
log.info("Data will be written to {}".format(self.in_path))
log.debug("out path {}".format(self.out_path))
if not os.path.isdir(self.in_path):
os.makedirs(self.in_path)
if not os.path.isdir(self.out_path):
os.makedirs(self.out_path)
os.chdir(self.in_path)
log.debug("Change to workdir: {}".format(os.getcwd()))
log.debug("Current working directory: {}".format(os.getcwd()))
except Exception as error:
raise EddPulsarPipelineError(str(error))
else:
try:
self.in_path = os.path.join("/mnt/", project_id,
tdate, self._source_name,
str(central_freq), tstr)
log.debug("Creating directories")
log.info("Data will be written to {}".format(self.in_path))
if not os.path.isdir(self.in_path):
os.makedirs(self.in_path)
os.chdir(self.in_path)
log.debug("Change to workdir: {}".format(os.getcwd()))
log.debug("Current working directory: {}".format(os.getcwd()))
except Exception as error:
raise EddPulsarPipelineError(str(error))
os.chdir("/tmp/")
log.debug("Creating the predictor with tempo2")
if self._config["mode"] == "Timing":
if (parse_tag(self._source_name) != "R") & is_accessible(
self.tzpar_file):
cmd = 'numactl -m {} taskset -c {} tempo2 -f {} -pred'.format(
self.numa_number, self.__coreManager.get_coresstr('dspsr'),
self.tzpar_file).split()
cmd.append("{} {} {} {} {} {} {} 3599.999999999".format(
self._config["tempo2_telescope_name"],
Time.now().mjd - 0.5,
Time.now().mjd + 0.5,
float(central_freq) - (float(header["bandwidth"]) / 2),
float(central_freq) + (float(header["bandwidth"]) / 2),
self._config["tempo2_ntimecoeff"],
self._config["tempo2_nfreqcoeff"]))
log.info("Command to run: {}".format(cmd))
yield command_watcher(cmd, allow_fail=True)
attempts = 0
retries = 5
while True:
if attempts >= retries:
error = "Could not read t2pred.dat"
log.warning(
"{}. Will not react until next measurement start".
format(error))
raise StateChange("ready")
else:
yield sleep(1)
if is_accessible('{}/t2pred.dat'.format(os.getcwd())):
log.debug('found {}/t2pred.dat'.format(
os.getcwd()))
break
else:
attempts += 1
self.dada_header_file = tempfile.NamedTemporaryFile(
mode="w",
prefix="edd_dada_header_",
suffix=".txt",
dir="/tmp/",
delete=False)
log.debug("Writing dada header file to {0}".format(
self.dada_header_file.name))
header_string = render_dada_header(header)
self.dada_header_file.write(header_string)
self.dada_key_file = tempfile.NamedTemporaryFile(
mode="w",
prefix="dada_keyfile_",
suffix=".key",
dir="/tmp/",
delete=False)
log.debug("Writing dada key file to {0}".format(
self.dada_key_file.name))
key_string = make_dada_key_string(self._dada_buffers[1])
self.dada_key_file.write(make_dada_key_string(self._dada_buffers[1]))
log.debug("Dada key file contains:\n{0}".format(key_string))
self.dada_header_file.close()
self.dada_key_file.close()
attempts = 0
retries = 5
while True:
if attempts >= retries:
error = "could not read dada_key_file"
log.warning(
"{}. Will not react until next measurement start".format(
error))
raise StateChange("ready")
else:
yield sleep(1)
if is_accessible('{}'.format(self.dada_key_file.name)):
log.debug('found {}'.format(self.dada_key_file.name))
break
else:
attempts += 1
## Setting DM value for filterbank recording
self.par_dict = {}
self.dm = self._config["cdd_dm"]
try:
with open(
os.path.join(
self.tzpar_dir, '{}.par'.format(
self._source_name.split("_")[0][1:]))) as fh:
for line in fh:
if len(line.strip().split()) == 2:
key, value = line.strip().split()
elif len(line.strip().split()) == 3:
key, value, error = line.strip().split()
elif len(line.strip().split()) == 4:
key, value, lock, error = line.strip().split()
self.par_dict[key] = value.strip()
except IOError as error:
log.error(error)
try:
self.dm = float(self.par_dict["DM"])
except KeyError as error:
log.info("Key {} not found, will use default value of {}".format(
error, self.dm))
if parse_tag(self._source_name) == "R":
log.info("This is a calibrator scan, will set dm to zero")
self.dm = 0
self._dm_sensor.set_value(self.dm)
self._par_dict_sensor.set_value(
json.dumps(self.par_dict).strip("{").strip("}").replace(",", "\n"))
os.chdir(self.in_path)
log.debug("source_name = {}".format(self._source_name))
if self._config["mode"] == "Timing":
if (parse_tag(self._source_name) != "R") and self.pulsar_flag:
cmd = "numactl -m {numa} dspsr {args} {intergration_time} {nchan} {nbin} -fft-bench -x {fft_length} -cpu {cpus} -cuda {cuda_number} -P {predictor} -N {name} -E {parfile} {keyfile}".format(
numa=self.numa_number,
fft_length=self._config["fft_length"],
args=self._config["dspsr_params"]["args"],
intergration_time="-L {}".format(
self._config["intergration_time"]),
nchan="-F {}:D".format(self._config["nchannels"]),
nbin="-b {}".format(self._config["nbins"]),
name=self._source_name.split("_")[0],
predictor="/tmp/t2pred.dat",
parfile=self.tzpar_file,
cpus=self.__coreManager.get_coresstr('dspsr'),
cuda_number=self.cuda_number,
keyfile=self.dada_key_file.name)
elif parse_tag(self._source_name) == "R":
cmd = "numactl -m {numa} dspsr {args} {intergration_time} -c {period} -D 0.0001 -fft-bench -x {fft_length} {nchan} -cpu {cpus} -N {name} -cuda {cuda_number} {keyfile}".format(
numa=self.numa_number,
args=self._config["dspsr_params"]["args"],
intergration_time="-L {}".format(
self._config["intergration_time"]),
period=self._config["cal_period"],
fft_length=self._config["fft_length"],
nchan="-F {}:D".format(self._config["nchannels"]),
name=self._source_name,
cpus=self.__coreManager.get_coresstr('dspsr'),
cuda_number=self.cuda_number,
keyfile=self.dada_key_file.name)
else:
error = "source is unknown"
raise EddPulsarPipelineError(error)
if self._config["mode"] == "Searching":
if self._config["file_length"] == "":
file_length = ""
filename = "-o {}_{}_{}.fits".format(self._source_name,
self.dm,
self._config["npol"])
else:
file_length = "-L {}".format(self._config["file_length"])
filename = ""
cmd = "numactl -m {numa} digifits {args} -b 8 -F {nchan}:D -D {DM} -t {tsamp} -nsblk {nsblk} {file_length} -p {npol} -f {decimation} -do_dedisp -x {fft_length} -cpu {cpus} -cuda {cuda_number} {filename} {keyfile}".format(
numa=self.numa_number,
npol=self._config["npol"],
args=self._config["digifits_params"]["args"],
DM=self.dm,
nchan=self._config["filterbank_nchannels"],
fft_length=self._config["fft_length"],
decimation=self._config["decimation"],
nsblk=self._config["nsblk"],
tsamp=self._config["tsamp"],
file_length=file_length,
filename=filename,
cpus=self.__coreManager.get_coresstr('dspsr'),
cuda_number=self.cuda_number,
keyfile=self.dada_key_file.name)
if self._config["mode"] == "Baseband":
cmd = "numactl -m {numa} taskset -c {cpus} dbdisk_multithread -n {thread} -k dadc".format(
numa=self.numa_number,
in_path=self.in_path,
thread=self._config["dbdisk_writing_threads"],
cpus=self.__coreManager.get_coresstr('dspsr'))
if self._config["mode"] == "Leap_baseband":
cmd = "numactl -m {numa} taskset -c {cpus} dbdiskleap -n 8".format(
numa=self.numa_number,
cpus=self.__coreManager.get_coresstr('dspsr'))
log.debug("Running command: {0}".format(cmd))
if self._config["mode"] == "Timing":
log.info("Staring dspsr")
if self._config["mode"] == "Searching":
log.info("Staring digifits")
if self._config["mode"] == "Baseband":
log.info("Staring dbdisk_multithread")
if self._config["mode"] == "Leap_baseband":
log.info("Staring dbdiskleap")
self._data_processing_proc = ManagedProcess(cmd)
self._subprocessMonitor.add(self._data_processing_proc,
self._subprocess_error)
####################################################
#STARTING merging code #
####################################################
if self._config["mode"] not in "Leap_baseband":
cmd = "numactl -m {numa} taskset -c {cpu} {merge_application} -p {npart} -n {nthreads} --log_level=info".format(
numa=self.numa_number,
cpu=self.__coreManager.get_coresstr('merge'),
nthreads=self._config["merge_threads"],
merge_application=self._config["merge_application"],
npart=self._config["npart"])
log.debug("Running command: {0}".format(cmd))
log.info("Staring EDDPolnMerge")
self._merge_proc = ManagedProcess(cmd)
self._subprocessMonitor.add(self._merge_proc,
self._subprocess_error)
####################################################
#STARTING MKRECV #
####################################################
cmd = "numactl -m {numa} taskset -c {cpu} mkrecv_v4 --header {dada_header} --nthreads {threads} --lst --quiet".format(
numa=self.numa_number,
cpu=self.__coreManager.get_coresstr('mkrecv'),
threads=self._config["nstreams"],
dada_header=self.dada_header_file.name)
log.debug("Running command: {0}".format(cmd))
log.info("Staring MKRECV")
self._mkrecv_ingest_proc = ManagedProcess(
cmd,
stdout_handler=self._polarization_sensors[""]
["mkrecv_sensors"].stdout_handler)
self._subprocessMonitor.add(self._mkrecv_ingest_proc,
self._subprocess_error)
####################################################
#STARTING ARCHIVE MONITOR #
####################################################
if self._config["mode"] == "Timing":
log.info("Staring archive monitor")
self._archive_observer = Observer()
self._archive_observer.daemon = False
log.info("Input directory: {}".format(self.in_path))
log.info("Output directory: {}".format(self.out_path))
log.info("Setting up ArchiveAdder handler")
self._handler = ArchiveAdder(self.out_path,
self._config["zaplist"])
self._archive_observer.schedule(self._handler,
str(self.in_path),
recursive=False)
log.info("Starting directory monitor")
self._archive_observer.start()
self._png_monitor_callback = tornado.ioloop.PeriodicCallback(
self._png_monitor, 5000)
self._png_monitor_callback.start()
else:
self._folder_size_monitor_callback = tornado.ioloop.PeriodicCallback(
self._folder_size_monitor, 5000)
self._folder_size_monitor_callback.start()
self._subprocessMonitor.start()
self._timer = Time.now() - self._timer
log.info("Took {} s to start".format(self._timer * 86400))