def setup_sensors(self):
"""
@brief Set up monitoring sensors.
"""
self._target_sensors = []
for beam in self._beam_manager.get_beams():
sensor = Sensor.string("{}-target".format(beam.idx),
description="Target for beam {}".format(
beam.idx),
default=beam.target.format_katcp(),
initial_status=Sensor.UNKNOWN)
self.add_sensor(sensor)
beam.register_observer(lambda beam, sensor=sensor: sensor.
set_value(beam.target.format_katcp()))
antenna_map = {
a.name: a.format_katcp()
for a in self._beam_manager.antennas
}
self._antennas_sensor = Sensor.string(
"antennas",
description=
"JSON breakdown of the antennas (in KATPOINT format) associated with this delay engine",
default=json.dumps(antenna_map),
initial_status=Sensor.NOMINAL)
self.add_sensor(self._antennas_sensor)
self._phase_reference_sensor = Sensor.string(
"phase-reference",
description=
"A KATPOINT target string denoting the F-engine phasing centre",
default="unset,radec,0,0",
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._phase_reference_sensor)
reference_antenna = Antenna(
"reference,{ref.lat},{ref.lon},{ref.elev}".format(
ref=self._beam_manager.antennas[0].ref_observer))
self._reference_antenna_sensor = Sensor.string(
"reference-antenna",
description=
"A KATPOINT antenna string denoting the reference antenna",
default=reference_antenna.format_katcp(),
initial_status=Sensor.NOMINAL)
self.add_sensor(self._reference_antenna_sensor)
class DelayBufferController(object):
def __init__(self, delay_client, ordered_beams, ordered_antennas, nreaders):
"""
@brief Controls shared memory delay buffers that are accessed by one or more
beamformer instances.
@params delay_client A KATCPResourceClient connected to an FBFUSE delay engine server
@params ordered_beams A list of beam IDs in the order that they should be generated by the beamformer
@params orderded_antennas A list of antenna IDs in the order which they should be captured by the beamformer
@params nreaders The number of posix shared memory readers that will access the memory
buffers that are managed by this instance.
"""
self._nreaders = nreaders
self._delay_client = delay_client
self._ordered_antennas = ordered_antennas
self._ordered_beams = ordered_beams
self.shared_buffer_key = "delay_buffer"
self.mutex_semaphore_key = "delay_buffer_mutex"
self.counting_semaphore_key = "delay_buffer_count"
self._nbeams = len(self._ordered_beams)
self._nantennas = len(self._ordered_antennas)
self._delays_array = self._delays = np.rec.recarray((self._nbeams, self._nantennas),
dtype=[("delay_rate","float32"),("delay_offset","float32")])
self._targets = OrderedDict()
for beam in self._ordered_beams:
self._targets[beam] = Target(DEFAULT_TARGET)
self._phase_reference = Target(DEFAULT_TARGET)
self._update_rate = DEFAULT_UPDATE_RATE
self._delay_span = DEFAULT_DELAY_SPAN
self._update_callback = None
self._beam_callbacks = {}
def unlink_all(self):
"""
@brief Unlink (remove) all posix shared memory sections and semaphores.
"""
log.debug("Unlinking all relevant posix shared memory segments and semaphores")
try:
posix_ipc.unlink_semaphore(self.counting_semaphore_key)
except posix_ipc.ExistentialError:
pass
try:
posix_ipc.unlink_semaphore(self.mutex_semaphore_key)
except posix_ipc.ExistentialError:
pass
try:
posix_ipc.unlink_shared_memory(self.shared_buffer_key)
except posix_ipc.ExistentialError:
pass
@coroutine
def fetch_config_info(self):
"""
@brief Retrieve configuration information from the delay configuration server
"""
log.debug("Fetching configuration information from the delay configuration server")
yield self._delay_client.until_synced()
antennas_json = yield self._delay_client.sensor.antennas.get_value()
try:
antennas = json.loads(antennas_json)
except Exception as error:
log.exception("Failed to parse antennas")
raise error
self._antennas = [Antenna(antennas[antenna]) for antenna in self._ordered_antennas]
log.debug("Ordered the antenna capture list to:\n {}".format("\n".join([i.format_katcp() for i in self._antennas])))
reference_antenna = yield self._delay_client.sensor.reference_antenna.get_value()
self._reference_antenna = Antenna(reference_antenna)
log.debug("Reference antenna: {}".format(self._reference_antenna.format_katcp()))
@coroutine
def start(self):
"""
@brief Start the delay buffer controller
@detail This method will create all necessary posix shared memory segments
and semaphores, retreive necessary information from the delay
configuration server and start the delay update callback loop.
"""
yield self.fetch_config_info()
self.register_callbacks()
self.unlink_all()
# This semaphore is required to protect access to the shared_buffer
# so that it is not read and written simultaneously
# The value is set to two such that two processes can read simultaneously
log.info("Creating mutex semaphore, key='{}'".format(self.mutex_semaphore_key))
self._mutex_semaphore = posix_ipc.Semaphore(
self.mutex_semaphore_key,
flags=posix_ipc.O_CREX,
initial_value=self._nreaders)
# This semaphore is used to notify beamformer instances of a change to the
# delay models. Upon any change its value is simply incremented by one.
# Note: There sem_getvalue does not work on Mac OS X so the value of this
# semaphore cannot be tested on OS X (this is only a problem for local testing).
log.info("Creating counting semaphore, key='{}'".format(self.counting_semaphore_key))
self._counting_semaphore = posix_ipc.Semaphore(self.counting_semaphore_key,
flags=posix_ipc.O_CREX,
initial_value=0)
# This is the share memory buffer that contains the delay models for the
log.info("Creating shared memory, key='{}'".format(self.shared_buffer_key))
self._shared_buffer = posix_ipc.SharedMemory(
self.shared_buffer_key,
flags=posix_ipc.O_CREX,
size=len(self._delays_array.tobytes()))
# For reference one can access this memory from another python process using:
# shm = posix_ipc.SharedMemory("delay_buffer")
# data_map = mmap.mmap(shm.fd, shm.size)
# data = np.frombuffer(data_map, dtype=[("delay_rate","float32"),("delay_offset","float32")])
# data = data.reshape(nbeams, nantennas)
self._shared_buffer_mmap = mmap(self._shared_buffer.fd, self._shared_buffer.size)
self._update_callback = PeriodicCallback(self._safe_update_delays, self._update_rate*1000)
self._update_callback.start()
def stop(self):
"""
@brief Stop the delay buffer controller
@detail This method will stop the delay update callback loop, deregister
any sensor callbacks and trigger the closing and unlinking of
posix IPC objects.
"""
self._update_callback.stop()
self.deregister_callbacks()
log.debug("Closing shared memory mmap and file descriptor")
self._shared_buffer_mmap.close()
self._shared_buffer.close_fd()
self.unlink_all()
def _update_phase_reference(self, rt, t, status, value):
if status != "nominal":
return
log.debug("Received update to phase-reference: {}, {}, {}, {}".format(rt, t, status, value))
self._phase_reference = Target(value)
def register_callbacks(self):
"""
@brief Register callbacks on the phase-reference and target positions for each beam
@detail The delay configuration server provides information about antennas, reference
antennas, phase centres and beam targets. It is currently assumed that the
antennas and reference antenna will not (can not) change during an observation
as such we here only register callbacks on the phase-reference (a KATPOINT target
string specifying the bore sight pointing position) and the individial beam targets.
"""
log.debug("Registering phase-reference update callback")
self._delay_client.sensor.phase_reference.set_sampling_strategy('event')
self._delay_client.sensor.phase_reference.register_listener(self._update_phase_reference)
for beam in self._ordered_beams:
sensor_name = "{}_target".format(beam)
def callback(rt, t, status, value, beam):
log.debug("Received target update for beam {}: {}".format(beam, value))
if status == 'nominal':
try:
self._targets[beam] = Target(value)
except Exception as error:
log.exception("Error when updating target for beam {}".format(beam))
self._delay_client.sensor[sensor_name].set_sampling_strategy('event')
self._delay_client.sensor[sensor_name].register_listener(
lambda rt, r, status, value, beam=beam: callback(rt, r, status, value, beam))
self._beam_callbacks[beam] = callback
def deregister_callbacks(self):
"""
@brief Deregister any callbacks started with the register callbacks method
"""
log.debug("Deregistering phase-reference update callback")
self._delay_client.sensor.phase_reference.set_sampling_strategy('none')
self._delay_client.sensor.phase_reference.unregister_listener(self._update_phase_reference)
log.debug("Deregistering targets update callbacks")
for beam in self._ordered_beams:
sensor_name = "{}_target".format(beam)
self._delay_client.sensor[sensor_name].set_sampling_strategy('none')
self._delay_client.sensor[sensor_name].unregister_listener(self._beam_callbacks[beam])
self._beam_callbacks = {}
def _safe_update_delays(self):
# This is just a wrapper around update delays that
# stops it throwing an exception
try:
self.update_delays()
except Exception as error:
log.exception("Failure while updating delays")
def update_delays(self):
"""
@brief Calculate updated delays based on the currently set targets and
phase reference.
@detail The delays will be calculated in the order specified in the constructor
of the class and the delays will be written to the shared memory segment.
Two semaphores are used here:
- mutex: This is required to stop clients reading the shared
memory segment while it is being written to.
- counting: This semaphore is incremented after a succesful write
to inform clients that there is fresh data to read
from the shared memory segment. It is the responsibility
of client applications to track the value of this semaphore.
"""
log.info("Updating delays")
timer = Timer()
delay_calc = DelayPolynomial(self._antennas, self._phase_reference,
self._targets.values(), self._reference_antenna)
poly = delay_calc.get_delay_polynomials(time.time(), duration=self._delay_span)
poly_calc_time = timer.elapsed()
log.debug("Poly calculation took {} seconds".format(poly_calc_time))
if poly_calc_time >= self._update_rate:
log.warning("The time required for polynomial calculation >= delay update rate, "
"this may result in degredation of beamforming quality")
timer.reset()
# Acquire the semaphore for each possible reader
log.debug("Acquiring semaphore for each reader")
for ii in range(self._nreaders):
self._mutex_semaphore.acquire()
self._shared_buffer_mmap.seek(0)
self._shared_buffer_mmap.write(poly.astype('float32').tobytes())
# Increment the counting semaphore to notify the readers
# that a new model is available
log.debug("Incrementing counting semaphore")
self._counting_semaphore.release()
# Release the semaphore for each reader
log.debug("Releasing semaphore for each reader")
for ii in range(self._nreaders):
self._mutex_semaphore.release()
log.debug("Delay model writing took {} seconds on worker side".format(timer.elapsed()))
def setup_sensors(self):
"""
@brief Setup the default KATCP sensors.
@note As this call is made only upon an FBFUSE configure call a mass inform
is required to let connected clients know that the proxy interface has
changed.
"""
self._state_sensor = LoggingSensor.discrete(
"state",
description = "Denotes the state of this FBF instance",
params = self.STATES,
default = self.IDLE,
initial_status = Sensor.NOMINAL)
self._state_sensor.set_logger(self.log)
self.add_sensor(self._state_sensor)
self._ca_address_sensor = Sensor.string(
"configuration-authority",
description = "The address of the server that will be deferred to for configurations",
default = "",
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._ca_address_sensor)
self._available_antennas_sensor = Sensor.string(
"available-antennas",
description = "The antennas that are currently available for beamforming",
default = json.dumps({antenna.name:antenna.format_katcp() for antenna in self._katpoint_antennas}),
initial_status = Sensor.NOMINAL)
self.add_sensor(self._available_antennas_sensor)
self._phase_reference_sensor = Sensor.string(
"phase-reference",
description="A KATPOINT target string denoting the F-engine phasing centre",
default="unset,radec,0,0",
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._phase_reference_sensor)
reference_antenna = Antenna("reference,{ref.lat},{ref.lon},{ref.elev}".format(
ref=self._katpoint_antennas[0].ref_observer))
self._reference_antenna_sensor = Sensor.string(
"reference-antenna",
description="A KATPOINT antenna string denoting the reference antenna",
default=reference_antenna.format_katcp(),
initial_status=Sensor.NOMINAL)
self.add_sensor(self._reference_antenna_sensor)
self._bandwidth_sensor = Sensor.float(
"bandwidth",
description = "The bandwidth this product is configured to process",
default = self._default_sb_config['bandwidth'],
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._bandwidth_sensor)
self._nchans_sensor = Sensor.integer(
"nchannels",
description = "The number of channels to be processesed",
default = self._n_channels,
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._nchans_sensor)
self._cfreq_sensor = Sensor.float(
"centre-frequency",
description = "The centre frequency of the band this product configured to process",
default = self._default_sb_config['centre-frequency'],
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._cfreq_sensor)
self._cbc_nbeams_sensor = Sensor.integer(
"coherent-beam-count",
description = "The number of coherent beams that this FBF instance can currently produce",
default = self._default_sb_config['coherent-beams-nbeams'],
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._cbc_nbeams_sensor)
self._cbc_nbeams_per_group = Sensor.integer(
"coherent-beam-count-per-group",
description = "The number of coherent beams packed into a multicast group",
default = 1,
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._cbc_nbeams_per_group)
self._cbc_ngroups = Sensor.integer(
"coherent-beam-ngroups",
description = "The number of multicast groups used for coherent beam transmission",
default = 1,
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._cbc_ngroups)
self._cbc_nbeams_per_server_set = Sensor.integer(
"coherent-beam-nbeams-per-server-set",
description = "The number of beams produced by each server set",
default = 1,
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._cbc_nbeams_per_server_set)
self._cbc_tscrunch_sensor = Sensor.integer(
"coherent-beam-tscrunch",
description = "The number time samples that will be integrated when producing coherent beams",
default = self._default_sb_config['coherent-beams-tscrunch'],
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._cbc_tscrunch_sensor)
self._cbc_fscrunch_sensor = Sensor.integer(
"coherent-beam-fscrunch",
description = "The number frequency channels that will be integrated when producing coherent beams",
default = self._default_sb_config['coherent-beams-fscrunch'],
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._cbc_fscrunch_sensor)
self._cbc_antennas_sensor = Sensor.string(
"coherent-beam-antennas",
description = "The antennas that will be used when producing coherent beams",
default = self._default_sb_config['coherent-beams-antennas'],
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._cbc_antennas_sensor)
self._cbc_mcast_groups_sensor = Sensor.string(
"coherent-beam-multicast-groups",
description = "Multicast groups used by this instance for sending coherent beam data",
default = "",
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._cbc_mcast_groups_sensor)
self._cbc_mcast_groups_mapping_sensor = Sensor.string(
"coherent-beam-multicast-group-mapping",
description = "Mapping of mutlicast group address to the coherent beams in that group",
default= "",
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._cbc_mcast_groups_mapping_sensor)
self._ibc_nbeams_sensor = Sensor.integer(
"incoherent-beam-count",
description = "The number of incoherent beams that this FBF instance can currently produce",
default = 1,
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._ibc_nbeams_sensor)
self._ibc_tscrunch_sensor = Sensor.integer(
"incoherent-beam-tscrunch",
description = "The number time samples that will be integrated when producing incoherent beams",
default = self._default_sb_config['incoherent-beam-tscrunch'],
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._ibc_tscrunch_sensor)
self._ibc_fscrunch_sensor = Sensor.integer(
"incoherent-beam-fscrunch",
description = "The number frequency channels that will be integrated when producing incoherent beams",
default = self._default_sb_config['incoherent-beam-fscrunch'],
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._ibc_fscrunch_sensor)
self._ibc_antennas_sensor = Sensor.string(
"incoherent-beam-antennas",
description = "The antennas that will be used when producing incoherent beams",
default = self._default_sb_config['incoherent-beam-antennas'],
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._ibc_antennas_sensor)
self._ibc_mcast_group_sensor = Sensor.string(
"incoherent-beam-multicast-group",
description = "Multicast group used by this instance for sending incoherent beam data",
default = "",
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._ibc_mcast_group_sensor)
self._servers_sensor = Sensor.string(
"servers",
description = "The worker server instances currently allocated to this product",
default = ",".join(["{s.hostname}:{s.port}".format(s=server) for server in self._servers]),
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._servers_sensor)
self._nserver_sets_sensor = Sensor.integer(
"nserver-sets",
description = "The number of server sets (independent subscriptions to the F-engines)",
default = 1,
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._nserver_sets_sensor)
self._nservers_per_set_sensor = Sensor.integer(
"nservers-per-set",
description = "The number of servers per server set",
default = 1,
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._nservers_per_set_sensor)
self._delay_config_server_sensor = Sensor.string(
"delay-config-server",
description = "The address of the delay configuration server for this product",
default = "",
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._delay_config_server_sensor)