def _determine_feng_capture_order(self, antenna_to_feng_id_map, coherent_beam_config, incoherent_beam_config):
# Need to sort the f-engine IDs into 4 states
# 1. Incoherent but not coherent
# 2. Incoherent and coherent
# 3. Coherent but not incoherent
# 4. Neither coherent nor incoherent
#
# We must catch all antennas as even in case 4 the data is required for the
# transient buffer.
#
# To make this split, we first create the three sets, coherent, incoherent and all.
mapping = antenna_to_feng_id_map
all_feng_ids = set(mapping.values())
coherent_feng_ids = set(mapping[antenna] for antenna in parse_csv_antennas(coherent_beam_config['antennas']))
incoherent_feng_ids = set(mapping[antenna] for antenna in parse_csv_antennas(incoherent_beam_config['antennas']))
incoh_not_coh = incoherent_feng_ids.difference(coherent_feng_ids)
incoh_and_coh = incoherent_feng_ids.intersection(coherent_feng_ids)
coh_not_incoh = coherent_feng_ids.difference(incoherent_feng_ids)
used_fengs = incoh_not_coh.union(incoh_and_coh).union(coh_not_incoh)
unused_fengs = all_feng_ids.difference(used_fengs)
# Output final order
final_order = list(incoh_not_coh) + list(incoh_and_coh) + list(coh_not_incoh) + list(unused_fengs)
start_of_incoherent_fengs = 0
end_of_incoherent_fengs = len(incoh_not_coh) + len(incoh_and_coh)
start_of_coherent_fengs = len(incoh_not_coh)
end_of_coherent_fengs = len(incoh_not_coh) + len(incoh_and_coh) + len(coh_not_incoh)
start_of_unused_fengs = end_of_coherent_fengs
end_of_unused_fengs = len(all_feng_ids)
info = {
"order": final_order,
"incoherent_span":(start_of_incoherent_fengs, end_of_incoherent_fengs),
"coherent_span":(start_of_coherent_fengs, end_of_coherent_fengs),
"unused_span":(start_of_unused_fengs, end_of_unused_fengs)
}
return info
def request_configure(self, req, product_id, antennas_csv, n_channels, streams_json, proxy_name):
"""
@brief Configure FBFUSE to receive and process data from a subarray
@detail REQUEST ?configure product_id antennas_csv n_channels streams_json proxy_name
Configure FBFUSE for the particular data products
@param req A katcp request object
@param product_id This is a name for the data product, which is a useful tag to include
in the data, but should not be analysed further. For example "array_1_bc856M4k".
@param antennas_csv A comma separated list of physical antenna names used in particular sub-array
to which the data products belongs (e.g. m007,m008,m009).
@param n_channels The integer number of frequency channels provided by the CBF.
@param streams_json a JSON struct containing config keys and values describing the streams.
For example:
@code
{'stream_type1': {
'stream_name1': 'stream_address1',
'stream_name2': 'stream_address2',
...},
'stream_type2': {
'stream_name1': 'stream_address1',
'stream_name2': 'stream_address2',
...},
...}
@endcode
The steam type keys indicate the source of the data and the type, e.g. cam.http.
stream_address will be a URI. For SPEAD streams, the format will be spead://<ip>[+<count>]:<port>,
representing SPEAD stream multicast groups. When a single logical stream requires too much bandwidth
to accommodate as a single multicast group, the count parameter indicates the number of additional
consecutively numbered multicast group ip addresses, and sharing the same UDP port number.
stream_name is the name used to identify the stream in CAM.
A Python example is shown below, for five streams:
One CAM stream, with type cam.http. The camdata stream provides the connection string for katportalclient
(for the subarray that this FBFUSE instance is being configured on).
One F-engine stream, with type: cbf.antenna_channelised_voltage.
One X-engine stream, with type: cbf.baseline_correlation_products.
Two beam streams, with type: cbf.tied_array_channelised_voltage. The stream names ending in x are
horizontally polarised, and those ending in y are vertically polarised.
@code
pprint(streams_dict)
{'cam.http':
{'camdata':'http://10.8.67.235/api/client/1'},
'cbf.antenna_channelised_voltage':
{'i0.antenna-channelised-voltage':'spead://239.2.1.150+15:7148'},
...}
@endcode
If using katportalclient to get information from CAM, then reconnect and re-subscribe to all sensors
of interest at this time.
@param proxy_name The CAM name for the instance of the FBFUSE data proxy that is being configured.
For example, "FBFUSE_3". This can be used to query sensors on the correct proxy,
in the event that there are multiple instances in the same subarray.
@note A configure call will result in the generation of a new subarray instance in FBFUSE that will be added to the clients list.
@return katcp reply object [[[ !configure ok | (fail [error description]) ]]]
"""
msg = ("Configuring new FBFUSE product",
"Product ID: {}".format(product_id),
"Antennas: {}".format(antennas_csv),
"Nchannels: {}".format(n_channels),
"Streams: {}".format(streams_json),
"Proxy name: {}".format(proxy_name))
log.info("\n".join(msg))
# Test if product_id already exists
if product_id in self._products:
return ("fail", "FBF already has a configured product with ID: {}".format(product_id))
# Determine number of nodes required based on number of antennas in subarray
# Note this is a poor way of handling this that may be updated later. In theory
# there is a throughput measure as a function of bandwidth, polarisations and number
# of antennas that allows one to determine the number of nodes to run. Currently we
# just assume one antennas worth of data per NIC on our servers, so two antennas per
# node.
try:
antennas = parse_csv_antennas(antennas_csv)
except AntennaValidationError as error:
return ("fail", str(error))
valid_n_channels = [1024, 4096, 32768]
if not n_channels in valid_n_channels:
return ("fail", "The provided number of channels ({}) is not valid. Valid options are {}".format(n_channels, valid_n_channels))
streams = json.loads(streams_json)
try:
streams['cam.http']['camdata']
# Need to check for endswith('.antenna-channelised-voltage') as the i0 is not
# guaranteed to stay the same.
# i0 = instrument name
# Need to keep this for future sensor lookups
streams['cbf.antenna_channelised_voltage']
except KeyError as error:
return ("fail", "JSON streams object does not contain required key: {}".format(str(error)))
for key, value in streams['cbf.antenna_channelised_voltage'].items():
if key.endswith('.antenna-channelised-voltage'):
instrument_name, _ = key.split('.')
feng_stream_name = key
feng_groups = value
log.debug("Parsed instrument name from streams: {}".format(instrument_name))
break
else:
return ("fail", "Could not determine instrument name (e.g. 'i0') from streams")
# TODO: change this request to @async_reply and make the whole thing a coroutine
@coroutine
def configure():
kpc = self._katportal_wrapper_type(streams['cam.http']['camdata'])
# Get all antenna observer strings
futures, observers = [],[]
for antenna in antennas:
log.debug("Fetching katpoint string for antenna {}".format(antenna))
futures.append(kpc.get_observer_string(antenna))
for ii,future in enumerate(futures):
try:
observer = yield future
except Exception as error:
log.error("Error on katportalclient call: {}".format(str(error)))
req.reply("fail", "Error retrieving katpoint string for antenna {}".format(antennas[ii]))
return
else:
log.debug("Fetched katpoint antenna: {}".format(observer))
observers.append(Antenna(observer))
# Get bandwidth, cfreq, sideband, f-eng mapping
#TODO: Also get sync-epoch
log.debug("Fetching F-engine and subarray configuration information")
bandwidth_future = kpc.get_bandwidth(feng_stream_name)
cfreq_future = kpc.get_cfreq(feng_stream_name)
sideband_future = kpc.get_sideband(feng_stream_name)
feng_antenna_map_future = kpc.get_antenna_feng_id_map(instrument_name, antennas)
sync_epoch_future = kpc.get_sync_epoch()
bandwidth = yield bandwidth_future
cfreq = yield cfreq_future
sideband = yield sideband_future
feng_antenna_map = yield feng_antenna_map_future
sync_epoch = yield sync_epoch_future
feng_config = {
'bandwidth': bandwidth,
'centre-frequency': cfreq,
'sideband': sideband,
'feng-antenna-map': feng_antenna_map,
'sync-epoch': sync_epoch,
'nchans': n_channels
}
for key, value in feng_config.items():
log.debug("{}: {}".format(key, value))
product = FbfProductController(self, product_id, observers, n_channels,
feng_groups, proxy_name, feng_config)
self._products[product_id] = product
self._update_products_sensor()
req.reply("ok",)
log.debug("Configured FBFUSE instance with ID: {}".format(product_id))
self.ioloop.add_callback(configure)
raise AsyncReply
def prepare(self, sb_id):
"""
@brief Prepare the beamformer for streaming
@detail This method evaluates the current configuration creates a new DelayEngine
and passes a prepare call to all allocated servers.
"""
if not self.idle:
raise FbfProductStateError([self.IDLE], self.state)
self.log.info("Preparing FBFUSE product")
self._state_sensor.set_value(self.PREPARING)
self.log.debug("Product moved to 'preparing' state")
# Here we need to parse the streams and assign beams to streams:
#mcast_addrs, mcast_port = parse_stream(self._streams['cbf.antenna_channelised_voltage']['i0.antenna-channelised-voltage'])
if not self._ca_client:
self.log.warning("No configuration authority found, using default configuration parameters")
cm = self.set_sb_configuration(self._default_sb_config)
else:
#TODO: get the schedule block ID into this call from somewhere (configure?)
try:
config = yield self.get_ca_sb_configuration(sb_id)
cm = self.set_sb_configuration(config)
except Exception as error:
self.log.error("Configuring from CA failed with error: {}".format(str(error)))
self.log.warning("Reverting to default configuration")
cm = self.set_sb_configuration(self._default_sb_config)
cbc_antennas_names = parse_csv_antennas(self._cbc_antennas_sensor.value())
cbc_antennas = [self._antenna_map[name] for name in cbc_antennas_names]
self._beam_manager = BeamManager(self._cbc_nbeams_sensor.value(), cbc_antennas)
self._delay_config_server = DelayConfigurationServer("127.0.0.1", 0, self._beam_manager)
self._delay_config_server.start()
self.log.info("Started delay engine at: {}".format(self._delay_config_server.bind_address))
de_ip, de_port = self._delay_config_server.bind_address
self._delay_config_server_sensor.set_value((de_ip, de_port))
# Need to tear down the beam sensors here
# Here calculate the beam to multicast map
self._beam_sensors = []
mcast_to_beam_map = {}
groups = [ip for ip in self._cbc_mcast_groups]
idxs = [beam.idx for beam in self._beam_manager.get_beams()]
for group in groups:
self.log.debug("Allocating beams to {}".format(str(group)))
key = str(group)
for _ in range(self._cbc_nbeams_per_group.value()):
if not key in mcast_to_beam_map:
mcast_to_beam_map[str(group)] = []
value = idxs.pop(0)
self.log.debug("--> Allocated {} to {}".format(value, str(group)))
mcast_to_beam_map[str(group)].append(value)
self._cbc_mcast_groups_mapping_sensor.set_value(json.dumps(mcast_to_beam_map))
for beam in self._beam_manager.get_beams():
sensor = Sensor.string(
"coherent-beam-{}".format(beam.idx),
description="R.A. (deg), declination (deg) and source name for coherent beam with ID {}".format(beam.idx),
default=self._beam_to_sensor_string(beam),
initial_status=Sensor.UNKNOWN)
beam.register_observer(lambda beam, sensor=sensor:
sensor.set_value(self._beam_to_sensor_string(beam)))
self._beam_sensors.append(sensor)
self.add_sensor(sensor)
self._parent.mass_inform(Message.inform('interface-changed'))
#Here we actually start to prepare the remote workers
ip_splits = self._streams.split(N_FENG_STREAMS_PER_WORKER)
# This is assuming lower sideband and bandwidth is always +ve
fbottom = self._feng_config['centre-frequency'] - self._feng_config['bandwidth']/2.
coherent_beam_config = {
'tscrunch':self._cbc_tscrunch_sensor.value(),
'fscrunch':self._cbc_fscrunch_sensor.value(),
'antennas':self._cbc_antennas_sensor.value()
}
incoherent_beam_config = {
'tscrunch':self._ibc_tscrunch_sensor.value(),
'fscrunch':self._ibc_fscrunch_sensor.value(),
'antennas':self._ibc_antennas_sensor.value()
}
prepare_futures = []
for ii, (server, ip_range) in enumerate(zip(self._servers, ip_splits)):
chan0_idx = cm.nchans_per_worker * ii
chan0_freq = fbottom + chan0_idx * cm.channel_bandwidth
future = server.prepare(ip_range.format_katcp(), cm.nchans_per_group,
chan0_idx, chan0_freq, cm.channel_bandwidth, mcast_to_beam_map,
self._feng_config['feng-antenna-map'], coherent_beam_config,
incoherent_beam_config, de_ip, de_port)
prepare_futures.append(future)
failure_count = 0
for future in prepare_futures:
try:
yield future
except Exception as error:
log.error("Failed to configure server with error: {}".format(str(error)))
failure_count += 1
if failure_count > 0:
self._state_sensor.set_value(self.ERROR)
self.log.info("Failed to prepare FBFUSE product")
else:
self._state_sensor.set_value(self.READY)
self.log.info("Successfully prepared FBFUSE product")
def set_sb_configuration(self, config_dict):
"""
@brief Set the schedule block configuration for this product
@param config_dict A dictionary specifying configuation parameters, e.g.
@code
{
u'coherent-beams-nbeams':100,
u'coherent-beams-tscrunch':22,
u'coherent-beams-fscrunch':2,
u'coherent-beams-antennas':'m007',
u'coherent-beams-granularity':6,
u'incoherent-beam-tscrunch':16,
u'incoherent-beam-fscrunch':1,
u'incoherent-beam-antennas':'m008'
}
@endcode
@detail Valid parameters for the configuration dictionary are as follows:
coherent-beams-nbeams - The desired number of coherent beams to produce
coherent-beams-tscrunch - The number of spectra to integrate in the coherent beamformer
coherent-beams-tscrunch - The number of spectra to integrate in the coherent beamformer
coherent-beams-fscrunch - The number of channels to integrate in the coherent beamformer
coherent-beams-antennas - The specific antennas to use for the coherent beamformer
coherent-beams-granularity - The number of beams per output mutlicast group
(an integer divisor or multiplier of this number will be used)
incoherent-beam-tscrunch - The number of spectra to integrate in the incoherent beamformer
incoherent-beam-fscrunch - The number of channels to integrate in the incoherent beamformer
incoherent-beam-antennas - The specific antennas to use for the incoherent beamformer
centre-frequency - The desired centre frequency in Hz
bandwidth - The desired bandwidth in Hz
@note FBFUSE reasonably assumes that the user does not know the possible configurations at
any given time. As such it tries to satisfy the users request but will not throw an
error if the requested configuration is not acheivable, instead opting to provide a
reduced configuration. For example the user may request 1000 beams and 6 beams per
multicast group but FBFUSE may configure to produce 860 beams and 24 beams per multicast
group. If the user can only use 6 beams per multcast group, then in the 24-beam case
they must subscribe to the same multicast group 4 times on different nodes.
"""
if self._previous_sb_config == config_dict:
self.log.info("Configuration is unchanged, proceeding with existing configuration")
return
else:
self._previous_sb_config = config_dict
self.reset_sb_configuration()
self.log.info("Setting schedule block configuration")
config = deepcopy(self._default_sb_config)
config.update(config_dict)
self.log.info("Configuring using: {}".format(config))
requested_cbc_antenna = parse_csv_antennas(config['coherent-beams-antennas'])
if not self._verify_antennas(requested_cbc_antenna):
raise Exception("Requested coherent beam antennas are not a subset of the available antennas")
requested_ibc_antenna = parse_csv_antennas(config['incoherent-beam-antennas'])
if not self._verify_antennas(requested_ibc_antenna):
raise Exception("Requested incoherent beam antennas are not a subset of the available antennas")
# first we need to get one ip address for the incoherent beam
self._ibc_mcast_group = self._parent._ip_pool.allocate(1)
self._ibc_mcast_group_sensor.set_value(self._ibc_mcast_group.format_katcp())
largest_ip_range = self._parent._ip_pool.largest_free_range()
nworkers_available = self._parent._server_pool.navailable()
cm = FbfConfigurationManager(len(self._katpoint_antennas),
self._feng_config['bandwidth'], self._n_channels,
nworkers_available, largest_ip_range)
requested_nantennas = len(parse_csv_antennas(config['coherent-beams-antennas']))
mcast_config = cm.get_configuration(
config['coherent-beams-tscrunch'],
config['coherent-beams-fscrunch'],
config['coherent-beams-nbeams'],
requested_nantennas,
config['bandwidth'],
config['coherent-beams-granularity'])
self._bandwidth_sensor.set_value(config['bandwidth'])
self._cfreq_sensor.set_value(config['centre-frequency'])
self._nchans_sensor.set_value(mcast_config['num_chans'])
self._cbc_nbeams_sensor.set_value(mcast_config['num_beams'])
self._cbc_nbeams_per_group.set_value(mcast_config['num_beams_per_mcast_group'])
self._cbc_ngroups.set_value(mcast_config['num_mcast_groups'])
self._cbc_nbeams_per_server_set.set_value(mcast_config['num_beams_per_worker_set'])
self._cbc_tscrunch_sensor.set_value(config['coherent-beams-tscrunch'])
self._cbc_fscrunch_sensor.set_value(config['coherent-beams-fscrunch'])
self._cbc_antennas_sensor.set_value(config['coherent-beams-antennas'])
self._ibc_tscrunch_sensor.set_value(config['incoherent-beam-tscrunch'])
self._ibc_fscrunch_sensor.set_value(config['incoherent-beam-fscrunch'])
self._ibc_antennas_sensor.set_value(config['incoherent-beam-antennas'])
self._servers = self._parent._server_pool.allocate(mcast_config['num_workers_total'])
server_str = ",".join(["{s.hostname}:{s.port}".format(s=server) for server in self._servers])
self._servers_sensor.set_value(server_str)
self._nserver_sets_sensor.set_value(mcast_config['num_worker_sets'])
self._nservers_per_set_sensor.set_value(mcast_config['num_workers_per_set'])
self._cbc_mcast_groups = self._parent._ip_pool.allocate(mcast_config['num_mcast_groups'])
self._cbc_mcast_groups_sensor.set_value(self._cbc_mcast_groups.format_katcp())
return cm
def configure():
self._state_sensor.set_value(self.PREPARING)
log.debug("Starting delay configuration server client")
self._delay_client = KATCPClientResource(dict(
name="delay-configuration-client",
address=(dc_ip, dc_port),
controlled=True))
self._delay_client.start()
log.debug("Determining F-engine capture order")
feng_capture_order_info = self._determine_feng_capture_order(feng_config['feng-antenna-map'], coherent_beam_config,
incoherent_beam_config)
log.debug("Capture order info: {}".format(feng_capture_order_info))
feng_to_antenna_map = {value:key for key,value in feng_config['feng-antenna-map'].items()}
antenna_capture_order_csv = ",".join([feng_to_antenna_map[feng_id] for feng_id in feng_capture_order_info['order']])
self._antenna_capture_order_sensor.set_value(antenna_capture_order_csv)
log.debug("Parsing F-engines to capture: {}".format(feng_groups))
capture_range = ip_range_from_stream(feng_groups)
ngroups = capture_range.count
partition_nchans = nchans_per_group * ngroups
partition_bandwidth = partition_nchans * chan_bw
npol = 2
ndim = 2
nbits = 8
tsamp = 1.0 / (feng_config['bandwidth'] / feng_config['nchans'])
sample_clock = feng_config['bandwidth'] * 2
timestamp_step = feng_config['nchans'] * 2 * 256 # WARNING: This is only valid in 4k mode
frequency_ids = [chan0_idx+nchans_per_group*ii for ii in range(ngroups)] #WARNING: Assumes contigous groups
mkrecv_config = {
'frequency_mhz': (chan0_freq + feng_config['nchans']/2.0 * chan_bw) / 1e6,
'bandwidth': partition_bandwidth,
'tsamp_us': tsamp * 1e6,
'bytes_per_second': partition_bandwidth * npol * ndim * nbits,
'nchan': partition_nchans,
'dada_key': self._dada_input_key,
'nantennas': len(feng_capture_order_info['order']),
'antennas_csv': antenna_capture_order_csv,
'sync_epoch': feng_config['sync-epoch'],
'sample_clock': sample_clock,
'mcast_sources': ",".join([str(group) for group in capture_range]),
'mcast_port': capture_range.port,
'interface': "192.168.0.1",
'timestamp_step': timestamp_step,
'ordered_feng_ids_csv': ",".join(map(str, feng_capture_order_info['order'])),
'frequency_partition_ids_csv': ",".join(map(str,frequency_ids))
}
mkrecv_header = make_mkrecv_header(mkrecv_config)
self._mkrecv_header_sensor.set_value(mkrecv_header)
log.info("Determined MKRECV configuration:\n{}".format(mkrecv_header))
log.debug("Parsing beam to multicast mapping")
incoherent_beam = None
incoherent_beam_group = None
coherent_beam_to_group_map = {}
for group, beams in mcast_to_beam_map.items():
for beam in beams.split(","):
if beam.startswith("cfbf"):
coherent_beam_to_group_map[beam] = group
if beam.startswith("ifbf"):
incoherent_beam = beam
incoherent_beam_group = group
log.debug("Determined coherent beam to multicast mapping: {}".format(coherent_beam_to_group_map))
if incoherent_beam:
log.debug("Incoherent beam will be sent to: {}".format(incoherent_beam_group))
else:
log.debug("No incoherent beam specified")
"""
Tasks:
- compile kernels
- create shared memory banks
"""
# Compile beamformer
# TBD
# Need to come up with a good way to allocate keys for dada buffers
# Create input DADA buffer
log.debug("Creating dada buffer for input with key '{}'".format("%x"%self._dada_input_key))
#self._system_call_wrapper(["dada_db","-k",self._dada_input_key,"-n","64","-l","-p"])
# Create coherent beam output DADA buffer
log.debug("Creating dada buffer for coherent beam output with key '{}'".format("%x"%self._dada_coh_output_key))
#self._system_call_wrapper(["dada_db","-k",self._dada_coh_output_key,"-n","64","-l","-p"])
# Create incoherent beam output DADA buffer
log.debug("Creating dada buffer for incoherent beam output with key '{}'".format("%x"%self._dada_incoh_output_key))
#self._system_call_wrapper(["dada_db","-k",self._dada_incoh_output_key,"-n","64","-l","-p"])
# Create SPEAD transmitter for coherent beams
# Call to MKSEND
# Create SPEAD transmitter for incoherent beam
# Call to MKSEND
# Need to pass the delay buffer controller the F-engine capture order but only for the coherent beams
cstart, cend = feng_capture_order_info['coherent_span']
coherent_beam_feng_capture_order = feng_capture_order_info['order'][cstart:cend]
coherent_beam_antenna_capture_order = [feng_to_antenna_map[idx] for idx in coherent_beam_feng_capture_order]
# Start DelayBufferController instance
# Here we are going to make the assumption that the server and processing all run in
# one docker container that will be preallocated with the right CPU set, GPUs, memory
# etc. This means that the configurations need to be unique by NUMA node... [Note: no
# they don't, we can use the container IPC channel which isolates the IPC namespaces.]
if not self._dummy:
n_coherent_beams = len(coherent_beam_to_group_map)
coherent_beam_antennas = parse_csv_antennas(coherent_beam_config['antennas'])
self._delay_buffer_controller = DelayBufferController(self._delay_client,
coherent_beam_to_group_map.keys(),
coherent_beam_antenna_capture_order, 1)
yield self._delay_buffer_controller.start()
# Start beamformer instance
# TBD
# Define MKRECV configuration file
# SPEAD receiver does not get started until a capture init call
self._state_sensor.set_value(self.READY)
req.reply("ok",)