def _create_ring_buffer(self, bufferSize, blocks, key, numa_node): """ @brief Create a ring buffer of given size with given key on specified numa node. Adds and register an appropriate sensor to thw list """ # always clear buffer first. Allow fail here yield command_watcher("dada_db -d -k {key}".format(key=key), allow_fail=True) cmd = "numactl --cpubind={numa_node} --membind={numa_node} dada_db -k {key} -n {blocks} -b {bufferSize} -p -l".format(key=key, blocks=blocks, bufferSize=bufferSize, numa_node=numa_node) log.debug("Running command: {0}".format(cmd)) yield command_watcher(cmd) M = DbMonitor(key, self._buffer_status_handle) M.start() self._dada_buffers.append({'key': key, 'monitor': M})
def __ansible_subplay_executioner(self, play, additional_args=""): """ Uses ansible-playbook to execute thegiven play by writing it in a tempfile. Args: play The play to be executed additional_args Additional args added to the ansible execution, e.g. --tags=stop """ playfile = tempfile.NamedTemporaryFile(delete=False) yaml.dump([play], playfile) playfile.close() try: yield command_watcher( "ansible-playbook -i {} {} {}".format(self.__inventory, playfile.name, additional_args), env={ "ANSIBLE_ROLES_PATH": os.path.join(self.__edd_ansible_git_repository_folder, "roles") }, timeout=300) except Exception as E: playfile.unlink() raise RuntimeError("Error {} processing play:\n {}".format( E, yaml.dump(play)))
def deconfigure(self): """ Deconfigure the gated spectrometer pipeline. Clears all dada buffers. """ log.info("Deconfiguring EDD backend") if self.previous_state == 'streaming': yield self.capture_stop() if self._subprocessMonitor is not None: yield self._subprocessMonitor.stop() for proc in self._subprocesses: yield proc.terminate() self.mkrec_cmd = [] log.debug("Destroying dada buffers") for k in self._dada_buffers: k['monitor'].stop() cmd = "dada_db -d -k {0}".format(k['key']) log.debug("Running command: {0}".format(cmd)) yield command_watcher(cmd, allow_fail=True) self._dada_buffers = []
def deconfigure(self): """@brief deconfigure the pipeline.""" log.debug("Destroying dada buffers") for k in self._dada_buffers_monitor: cmd = "dada_db -d -k {0}".format(k) log.debug("Running command: {0}".format(cmd)) yield command_watcher(cmd, allow_fail=True) self._dada_buffers_monitor = []
def _reset_ring_buffer(self, key, numa_node): """ @brief Create a ring buffer of given size with given key on specified numa node. Adds and register an appropriate sensor to thw list """ # always clear buffer first. Allow fail here cmd = "numactl --cpubind={numa_node} --membind={numa_node} dbreset -k {key} --log_level debug".format( numa_node=numa_node, key=key) log.debug("Running command: {0}".format(cmd)) yield command_watcher(cmd, allow_fail=True)
def deconfigure(self): """@brief deconfigure the dspsr pipeline.""" log.info("Deconfiguring EDD backend") if self.state == 'runnning': yield self.capture_stop() self.state = "deconfiguring" if self._subprocessMonitor is not None: self._subprocessMonitor.stop() for proc in self._subprocesses: proc.terminate() self.mkrec_cmd = [] log.debug("Destroying dada buffers") for k in self._dada_buffers: k['monitor'].stop() cmd = "dada_db -d -k {0}".format(k['key']) log.debug("Running command: {0}".format(cmd)) yield command_watcher(cmd) self._dada_buffers = [] self.state = "idle"
def measurement_start(self): """ @brief Create output directory, create mkrecv header, calculate tempo2 predictor file if needed, start data capture, processing, and monitoring """ if self._config["active"] == 0: log.info("Pipeline is not active") raise StateChange("ready") self._timer = Time.now() self._subprocessMonitor = SubprocessMonitor() try: self._png_monitor_callback.stop() except Exception as E: log.error("Png monitor already stopped.") log.exception(E) try: self._folder_size_monitor_callback.stop() except Exception as E: log.error("File size monitor already stopped.") log.exception(E) self._source_name = self.__eddDataStore.getTelescopeDataItem( "source-name") ra = self.__eddDataStore.getTelescopeDataItem("ra") decl = self.__eddDataStore.getTelescopeDataItem("dec") scannum = self.__eddDataStore.getTelescopeDataItem("scannum") subscannum = self.__eddDataStore.getTelescopeDataItem("subscannum") receiver_name = self.__eddDataStore.getTelescopeDataItem("receiver") project_id = self.__eddDataStore.getTelescopeDataItem("project") log.info( "Retrieved data from telescope:\n Source name: {}\n RA = {}, decl = {}, receiver = {}" .format(self._source_name, ra, decl, receiver_name)) if self._config["mode"] == "Timing": self.tzpar_file = os.path.join( self.tzpar_dir, '{}.par'.format(self._source_name.split("_")[0][1:])) log.debug("Checking parfile file {}".format(self.tzpar_file)) self.pulsar_flag = is_accessible(self.tzpar_file) if (parse_tag(self._source_name) != "R") and (not self.pulsar_flag): log.warning( "source {} is neither pulsar nor calibrator. Will not react until next measurement start" .format(self._source_name)) raise StateChange("ready") log.debug("Setting blank image") self._fscrunch.set_value(BLANK_IMAGE) self._tscrunch.set_value(BLANK_IMAGE) self._profile.set_value(BLANK_IMAGE) log.debug("writing mkrecv header") self.cuda_number = numa.getInfo()[self.numa_number]['gpus'][0] log.info(" - Running on cuda core: {}".format(self.cuda_number)) header = self._config["dada_header_params"] central_freq = header["frequency_mhz"] self._central_freq.set_value(str(header["frequency_mhz"])) self._source_name_sensor.set_value(self._source_name) self._nchannels.set_value(self._config["nchannels"]) self._nbins.set_value(self._config["nbins"]) header["telescope"] = self._config["tempo2_telescope_name"] log.info(" - Tempo2 telescope name: {}".format(header['telescope'])) c = SkyCoord("{} {}".format(ra, decl), unit=(u.deg, u.deg)) header["ra"] = c.to_string("hmsdms").split(" ")[0].replace( "h", ":").replace("m", ":").replace("s", "") header["dec"] = c.to_string("hmsdms").split(" ")[1].replace( "d", ":").replace("m", ":").replace("s", "") header["key"] = self._dada_buffers[0] log.debug(" - Dada key: {}".format(header['key'])) header["mc_source"] = "" for i in self._config['input_data_streams']: header["mc_source"] += i["ip"] + "," header["mc_source"] = header["mc_source"][:-1] log.info(" - mc source: {}".format(header['mc_source'])) header["mc_streaming_port"] = self._config['input_data_streams'][0][ "port"] log.info(" - mc streaming port: {}".format( header['mc_streaming_port'])) header["interface"] = numa.getFastestNic(self.numa_number)[1]['ip'] log.info(" - mc interface: {}".format(header['interface'])) header["sync_time"] = self.sync_epoch log.info(" - sync time: {}".format(header['sync_time'])) if header['sample_clock'] == "unset": header["sample_clock"] = float( self._config['input_data_streams'][0]["sample_rate"] ) # adjsutment for the predecimation factor is done in the amster controller log.info(" - sample_clock: {}".format(header['sample_clock'])) header["source_name"] = self._source_name header["obs_id"] = "{0}_{1}".format(scannum, subscannum) header["filesize"] = int(float(self._config["dada"]["size"])) log.info(" - obs_id: {}".format(header['obs_id'])) header["receiver_name"] = receiver_name log.info(" - receiver_name: {}".format(header['receiver_name'])) tstr = Time.now().isot.replace(":", "-") tdate = tstr.split("T")[0] log.debug("Setting up the input and scrunch data directories") if self._config["mode"] == "Timing": try: self.in_path = os.path.join("/mnt/", project_id, tdate, self._source_name, str(central_freq), tstr, "raw_data") self.out_path = os.path.join("/mnt/", project_id, tdate, self._source_name, str(central_freq), tstr, "combined_data") log.debug("Creating directories") log.info("Data will be written to {}".format(self.in_path)) log.debug("out path {}".format(self.out_path)) if not os.path.isdir(self.in_path): os.makedirs(self.in_path) if not os.path.isdir(self.out_path): os.makedirs(self.out_path) os.chdir(self.in_path) log.debug("Change to workdir: {}".format(os.getcwd())) log.debug("Current working directory: {}".format(os.getcwd())) except Exception as error: raise EddPulsarPipelineError(str(error)) else: try: self.in_path = os.path.join("/mnt/", project_id, tdate, self._source_name, str(central_freq), tstr) log.debug("Creating directories") log.info("Data will be written to {}".format(self.in_path)) if not os.path.isdir(self.in_path): os.makedirs(self.in_path) os.chdir(self.in_path) log.debug("Change to workdir: {}".format(os.getcwd())) log.debug("Current working directory: {}".format(os.getcwd())) except Exception as error: raise EddPulsarPipelineError(str(error)) os.chdir("/tmp/") log.debug("Creating the predictor with tempo2") if self._config["mode"] == "Timing": if (parse_tag(self._source_name) != "R") & is_accessible( self.tzpar_file): cmd = 'numactl -m {} taskset -c {} tempo2 -f {} -pred'.format( self.numa_number, self.__coreManager.get_coresstr('dspsr'), self.tzpar_file).split() cmd.append("{} {} {} {} {} {} {} 3599.999999999".format( self._config["tempo2_telescope_name"], Time.now().mjd - 0.5, Time.now().mjd + 0.5, float(central_freq) - (float(header["bandwidth"]) / 2), float(central_freq) + (float(header["bandwidth"]) / 2), self._config["tempo2_ntimecoeff"], self._config["tempo2_nfreqcoeff"])) log.info("Command to run: {}".format(cmd)) yield command_watcher(cmd, allow_fail=True) attempts = 0 retries = 5 while True: if attempts >= retries: error = "Could not read t2pred.dat" log.warning( "{}. Will not react until next measurement start". format(error)) raise StateChange("ready") else: yield sleep(1) if is_accessible('{}/t2pred.dat'.format(os.getcwd())): log.debug('found {}/t2pred.dat'.format( os.getcwd())) break else: attempts += 1 self.dada_header_file = tempfile.NamedTemporaryFile( mode="w", prefix="edd_dada_header_", suffix=".txt", dir="/tmp/", delete=False) log.debug("Writing dada header file to {0}".format( self.dada_header_file.name)) header_string = render_dada_header(header) self.dada_header_file.write(header_string) self.dada_key_file = tempfile.NamedTemporaryFile( mode="w", prefix="dada_keyfile_", suffix=".key", dir="/tmp/", delete=False) log.debug("Writing dada key file to {0}".format( self.dada_key_file.name)) key_string = make_dada_key_string(self._dada_buffers[1]) self.dada_key_file.write(make_dada_key_string(self._dada_buffers[1])) log.debug("Dada key file contains:\n{0}".format(key_string)) self.dada_header_file.close() self.dada_key_file.close() attempts = 0 retries = 5 while True: if attempts >= retries: error = "could not read dada_key_file" log.warning( "{}. Will not react until next measurement start".format( error)) raise StateChange("ready") else: yield sleep(1) if is_accessible('{}'.format(self.dada_key_file.name)): log.debug('found {}'.format(self.dada_key_file.name)) break else: attempts += 1 ## Setting DM value for filterbank recording self.par_dict = {} self.dm = self._config["cdd_dm"] try: with open( os.path.join( self.tzpar_dir, '{}.par'.format( self._source_name.split("_")[0][1:]))) as fh: for line in fh: if len(line.strip().split()) == 2: key, value = line.strip().split() elif len(line.strip().split()) == 3: key, value, error = line.strip().split() elif len(line.strip().split()) == 4: key, value, lock, error = line.strip().split() self.par_dict[key] = value.strip() except IOError as error: log.error(error) try: self.dm = float(self.par_dict["DM"]) except KeyError as error: log.info("Key {} not found, will use default value of {}".format( error, self.dm)) if parse_tag(self._source_name) == "R": log.info("This is a calibrator scan, will set dm to zero") self.dm = 0 self._dm_sensor.set_value(self.dm) self._par_dict_sensor.set_value( json.dumps(self.par_dict).strip("{").strip("}").replace(",", "\n")) os.chdir(self.in_path) log.debug("source_name = {}".format(self._source_name)) if self._config["mode"] == "Timing": if (parse_tag(self._source_name) != "R") and self.pulsar_flag: cmd = "numactl -m {numa} dspsr {args} {intergration_time} {nchan} {nbin} -fft-bench -x {fft_length} -cpu {cpus} -cuda {cuda_number} -P {predictor} -N {name} -E {parfile} {keyfile}".format( numa=self.numa_number, fft_length=self._config["fft_length"], args=self._config["dspsr_params"]["args"], intergration_time="-L {}".format( self._config["intergration_time"]), nchan="-F {}:D".format(self._config["nchannels"]), nbin="-b {}".format(self._config["nbins"]), name=self._source_name.split("_")[0], predictor="/tmp/t2pred.dat", parfile=self.tzpar_file, cpus=self.__coreManager.get_coresstr('dspsr'), cuda_number=self.cuda_number, keyfile=self.dada_key_file.name) elif parse_tag(self._source_name) == "R": cmd = "numactl -m {numa} dspsr {args} {intergration_time} -c {period} -D 0.0001 -fft-bench -x {fft_length} {nchan} -cpu {cpus} -N {name} -cuda {cuda_number} {keyfile}".format( numa=self.numa_number, args=self._config["dspsr_params"]["args"], intergration_time="-L {}".format( self._config["intergration_time"]), period=self._config["cal_period"], fft_length=self._config["fft_length"], nchan="-F {}:D".format(self._config["nchannels"]), name=self._source_name, cpus=self.__coreManager.get_coresstr('dspsr'), cuda_number=self.cuda_number, keyfile=self.dada_key_file.name) else: error = "source is unknown" raise EddPulsarPipelineError(error) if self._config["mode"] == "Searching": if self._config["file_length"] == "": file_length = "" filename = "-o {}_{}_{}.fits".format(self._source_name, self.dm, self._config["npol"]) else: file_length = "-L {}".format(self._config["file_length"]) filename = "" cmd = "numactl -m {numa} digifits {args} -b 8 -F {nchan}:D -D {DM} -t {tsamp} -nsblk {nsblk} {file_length} -p {npol} -f {decimation} -do_dedisp -x {fft_length} -cpu {cpus} -cuda {cuda_number} {filename} {keyfile}".format( numa=self.numa_number, npol=self._config["npol"], args=self._config["digifits_params"]["args"], DM=self.dm, nchan=self._config["filterbank_nchannels"], fft_length=self._config["fft_length"], decimation=self._config["decimation"], nsblk=self._config["nsblk"], tsamp=self._config["tsamp"], file_length=file_length, filename=filename, cpus=self.__coreManager.get_coresstr('dspsr'), cuda_number=self.cuda_number, keyfile=self.dada_key_file.name) if self._config["mode"] == "Baseband": cmd = "numactl -m {numa} taskset -c {cpus} dbdisk_multithread -n {thread} -k dadc".format( numa=self.numa_number, in_path=self.in_path, thread=self._config["dbdisk_writing_threads"], cpus=self.__coreManager.get_coresstr('dspsr')) if self._config["mode"] == "Leap_baseband": cmd = "numactl -m {numa} taskset -c {cpus} dbdiskleap -n 8".format( numa=self.numa_number, cpus=self.__coreManager.get_coresstr('dspsr')) log.debug("Running command: {0}".format(cmd)) if self._config["mode"] == "Timing": log.info("Staring dspsr") if self._config["mode"] == "Searching": log.info("Staring digifits") if self._config["mode"] == "Baseband": log.info("Staring dbdisk_multithread") if self._config["mode"] == "Leap_baseband": log.info("Staring dbdiskleap") self._data_processing_proc = ManagedProcess(cmd) self._subprocessMonitor.add(self._data_processing_proc, self._subprocess_error) #################################################### #STARTING merging code # #################################################### if self._config["mode"] not in "Leap_baseband": cmd = "numactl -m {numa} taskset -c {cpu} {merge_application} -p {npart} -n {nthreads} --log_level=info".format( numa=self.numa_number, cpu=self.__coreManager.get_coresstr('merge'), nthreads=self._config["merge_threads"], merge_application=self._config["merge_application"], npart=self._config["npart"]) log.debug("Running command: {0}".format(cmd)) log.info("Staring EDDPolnMerge") self._merge_proc = ManagedProcess(cmd) self._subprocessMonitor.add(self._merge_proc, self._subprocess_error) #################################################### #STARTING MKRECV # #################################################### cmd = "numactl -m {numa} taskset -c {cpu} mkrecv_v4 --header {dada_header} --nthreads {threads} --lst --quiet".format( numa=self.numa_number, cpu=self.__coreManager.get_coresstr('mkrecv'), threads=self._config["nstreams"], dada_header=self.dada_header_file.name) log.debug("Running command: {0}".format(cmd)) log.info("Staring MKRECV") self._mkrecv_ingest_proc = ManagedProcess( cmd, stdout_handler=self._polarization_sensors[""] ["mkrecv_sensors"].stdout_handler) self._subprocessMonitor.add(self._mkrecv_ingest_proc, self._subprocess_error) #################################################### #STARTING ARCHIVE MONITOR # #################################################### if self._config["mode"] == "Timing": log.info("Staring archive monitor") self._archive_observer = Observer() self._archive_observer.daemon = False log.info("Input directory: {}".format(self.in_path)) log.info("Output directory: {}".format(self.out_path)) log.info("Setting up ArchiveAdder handler") self._handler = ArchiveAdder(self.out_path, self._config["zaplist"]) self._archive_observer.schedule(self._handler, str(self.in_path), recursive=False) log.info("Starting directory monitor") self._archive_observer.start() self._png_monitor_callback = tornado.ioloop.PeriodicCallback( self._png_monitor, 5000) self._png_monitor_callback.start() else: self._folder_size_monitor_callback = tornado.ioloop.PeriodicCallback( self._folder_size_monitor, 5000) self._folder_size_monitor_callback.start() self._subprocessMonitor.start() self._timer = Time.now() - self._timer log.info("Took {} s to start".format(self._timer * 86400))