class FxCorrelator(Instrument):
"""
A generic FxCorrelator composed of fengines that channelise antenna inputs
and xengines that each produce cross products from a continuous portion of
the channels and accumulate the result. SPEAD data products are produced.
"""
def __init__(self, descriptor, identifier=-1, config_source=None,
logger=LOGGER):
"""
An abstract base class for instruments.
:param descriptor: A text description of the instrument. Required.
:param identifier: An optional integer identifier.
:param config_source: The instrument configuration source. Can be a text
file, hostname, whatever.
:param logger: Use the module logger by default, unless something else
is given.
:return: <nothing>
"""
self.logger = logger
self.loghandler = None
# we know about f and x hosts and engines, not just engines and hosts
self.fhosts = []
self.xhosts = []
self.filthosts = None
self.found_beamformer = False
self.fops = None
self.xops = None
self.bops = None
self.filtops = None
# attributes
self.katcp_port = None
self.f_per_fpga = None
self.x_per_fpga = None
self.accumulation_len = None
self.xeng_accumulation_len = None
self.xeng_tx_destination = None
self.meta_destination = None
self.fengine_sources = None
self.spead_tx = None
self.spead_meta_ig = None
self._sensors = {}
# parent constructor
Instrument.__init__(self, descriptor, identifier, config_source)
self._initialised = False
def standard_log_config(self, log_level=logging.INFO, silence_spead=True):
"""Convenience method for setting up logging in scripts etc.
:param logger: The loglevel to use by default, logging.INFO.
:param silence_spead: Set 'spead' logger to level WARNING if True
"""
# Set the root logging handler - submodules loggers will automatically
# use it
self.loghandler = log.Corr2LogHandler()
logging.root.addHandler(self.loghandler)
logging.root.setLevel(log_level)
self.logger.setLevel(log_level)
self.logger.addHandler(self.loghandler)
if silence_spead:
# set the SPEAD logger to warning only
spead_logger = logging.getLogger('spead')
spead_logger.setLevel(logging.WARNING)
def initialise(self, program=True, qdr_cal=True):
"""
Set up the correlator using the information in the config file.
:return:
"""
# set up the F, X, B and filter handlers
self.fops = FEngineOperations(self)
self.xops = XEngineOperations(self)
# self.bops = BEngineOperations(self)
self.filtops = FilterOperations(self)
# set up the filter boards if we need to
if 'filter' in self.configd:
try:
self.filtops.initialise(program=program)
except Exception as e:
raise e
# connect to the other hosts that make up this correlator
THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=5,
target_function='connect')
igmp_version = self.configd['FxCorrelator'].get('igmp_version')
if igmp_version is not None:
self.logger.info('Setting FPGA hosts IGMP version to '
'%s' % igmp_version)
THREADED_FPGA_FUNC(
self.fhosts + self.xhosts, timeout=5, target_function=(
'set_igmp_version', (igmp_version, ), {}))
# if we need to program the FPGAs, do so
if program:
self.logger.info('Programming FPGA hosts')
fpgautils.program_fpgas([(host, host.boffile) for host in
(self.fhosts + self.xhosts)],
progfile=None, timeout=15)
else:
self.logger.info('Loading design information')
THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=7,
target_function='get_system_information')
# remove test hardware from designs
utils.disable_test_gbes(self)
utils.remove_test_objects(self)
if program:
# cal the qdr on all boards
if qdr_cal:
self.qdr_calibrate()
else:
self.logger.info('Skipping QDR cal - are you sure you'
' want to do this?')
# init the engines
self.fops.initialise()
self.xops.initialise()
# are there beamformers?
if self.found_beamformer:
bengops.beng_initialise(self)
# for fpga_ in self.fhosts:
# fpga_.tap_arp_reload()
# for fpga_ in self.xhosts:
# fpga_.tap_arp_reload()
# subscribe all the engines to the multicast groups
self.fops.subscribe_to_multicast()
self.xops.subscribe_to_multicast()
post_mess_delay = 10
self.logger.info('post mess-with-the-switch delay of '
'%is' % post_mess_delay)
time.sleep(post_mess_delay)
# force a reset on the f-engines
self.fops.sys_reset(sleeptime=1)
# reset all counters on fhosts and xhosts
self.fops.clear_status_all()
self.xops.clear_status_all()
# check to see if the f engines are receiving all their data
if not self.fops.check_rx():
raise RuntimeError('The f-engines RX have a problem.')
# start f-engine TX
self.logger.info('Starting f-engine datastream')
self.fops.tx_enable()
# check that the F-engines are transmitting data correctly
if not self.fops.check_tx():
raise RuntimeError('The f-engines TX have a problem.')
# check that the X-engines are receiving data
if not self.xops.check_rx():
raise RuntimeError('The x-engines RX have a problem.')
# arm the vaccs on the x-engines
self.xops.vacc_sync()
# reset all counters on fhosts and xhosts
self.fops.clear_status_all()
self.xops.clear_status_all()
# set an initialised flag
self._initialised = True
def initialised(self):
"""
Has initialise successfully passed?
:return:
"""
return self._initialised
def est_sync_epoch(self):
"""
Estimates the synchronisation epoch based on current F-engine timestamp,
and the system time.
"""
self.logger.warn("Estimating synchronisation epoch...")
# get current time from an f-engine
mcnt_before = self.fhosts[0].get_local_time()
self.synchronisation_epoch = (time.time() - mcnt_before /
float(self.sample_rate_hz))
self.logger.info('Current f engine timetamp: %i' % mcnt_before)
assert mcnt_before & 0xfff == 0, 'Bottom 12 bits of timestamp from ' \
'f-engine are not zero?!'
def time_from_mcnt(self, mcnt):
"""
Returns the unix time UTC equivalent to the board timestamp. Does
NOT account for wrapping timestamps.
"""
if self.synchronisation_epoch < 0:
self.est_sync_epoch()
return (self.synchronisation_epoch + (float(mcnt) /
float(self.sample_rate_hz)))
def mcnt_from_time(self, time_seconds):
"""
Returns the board timestamp from a given UTC system time
(seconds since Unix Epoch). Accounts for wrapping timestamps.
"""
if self.synchronisation_epoch < 0:
self.est_sync_epoch()
time_diff_from_synch_epoch = time_seconds - self.synchronisation_epoch
time_diff_in_samples = int(time_diff_from_synch_epoch *
self.sample_rate_hz)
_tmp = 2**int(self.configd['FxCorrelator']['timestamp_bits'])
return time_diff_in_samples % _tmp
def qdr_calibrate(self):
"""
Run a software calibration routine on all the FPGA hosts.
Do it on F- and X-hosts in parallel.
:return:
"""
def _qdr_cal(_fpga):
_tic = time.time()
_results = {}
for _qdr in _fpga.qdrs:
_results[_qdr.name] = _qdr.qdr_cal(fail_hard=False)
_toc = time.time()
return {'results': _results, 'tic': _tic, 'toc': _toc}
self.logger.info('Calibrating QDR on F- and X-engines, this takes a '
'while.')
qdr_calfail = False
results = THREADED_FPGA_OP(self.fhosts + self.xhosts, timeout=30,
target_function=(_qdr_cal,))
for fpga, result in results.items():
self.logger.info('FPGA %s QDR cal results: start(%.3f) end(%.3f) '
'took(%.3f)' % (fpga, result['tic'], result['toc'],
result['toc'] - result['tic']))
for qdr, qdrres in result['results'].items():
if not qdrres:
qdr_calfail = True
break
self.logger.info('\t%s: cal okay: '
'%s' % (qdr, 'True' if qdrres else 'False'))
if qdr_calfail:
break
if qdr_calfail:
raise RuntimeError('QDR calibration failure.')
# for host in self.fhosts:
# for qdr in host.qdrs:
# qdr.qdr_delay_in_step(0b111111111111111111111111111111111111,
# -1)
# for host in self.xhosts:
# for qdr in host.qdrs:
# qdr.qdr_delay_in_step(0b111111111111111111111111111111111111,
# -1)
def set_labels(self, newlist):
"""
Apply new source labels to the configured fengine sources.
:param newlist:
:return:
"""
if len(newlist) != len(self.fengine_sources):
errstr = 'Number of supplied source labels (%i) does not match ' \
'number of configured sources (%i).' % \
(len(newlist), len(self.fengine_sources))
self.logger.error(errstr)
raise ValueError(errstr)
for ctr, source in enumerate(self.fengine_sources):
_source = source['source']
# update the source name
old_name = _source.name
_source.name = newlist[ctr]
# update the eq associated with that name
found_eq = False
for fhost in self.fhosts:
if old_name in fhost.eqs.keys():
fhost.eqs[_source.name] = fhost.eqs.pop(old_name)
found_eq = True
break
if not found_eq:
raise ValueError('Could not find the old EQ value, %s, to '
'update to new name, %s.' % (old_name,
_source.name))
if self.spead_meta_ig is not None:
metalist = numpy.array(self._spead_meta_get_labelling())
self.spead_meta_ig['input_labelling'] = metalist
self.spead_tx.send_heap(self.spead_meta_ig.get_heap())
def get_labels(self):
"""
Get the current fengine source labels as a string.
:return:
"""
source_names = ''
for fsrc in self.fengine_sources:
source_names += fsrc['source'].name + ' '
source_names = source_names.strip()
return source_names
def _read_config(self):
"""
Read the instrument configuration from self.config_source.
:return: True if the instrument read a config successfully, raise
an error if not?
"""
Instrument._read_config(self)
_d = self.configd
# check that the bitstream names are present
try:
open(_d['fengine']['bitstream'], 'r').close()
open(_d['xengine']['bitstream'], 'r').close()
except IOError:
self.logger.error('xengine bitstream: '
'%s' % _d['xengine']['bitstream'])
self.logger.error('fengine bitstream: '
'%s' % _d['fengine']['bitstream'])
self.logger.error('One or more bitstream files not found.')
raise IOError('One or more bitstream files not found.')
# TODO: Load config values from the bitstream meta information -
# f per fpga, x per fpga, etc
self.arp_wait_time = int(_d['FxCorrelator']['arp_wait_time'])
self.sensor_poll_time = int(_d['FxCorrelator']['sensor_poll_time'])
self.katcp_port = int(_d['FxCorrelator']['katcp_port'])
self.f_per_fpga = int(_d['fengine']['f_per_fpga'])
self.x_per_fpga = int(_d['xengine']['x_per_fpga'])
self.sample_rate_hz = int(_d['FxCorrelator']['sample_rate_hz'])
self.adc_demux_factor = int(_d['fengine']['adc_demux_factor'])
self.accumulation_len = int(_d['xengine']['accumulation_len'])
self.xeng_accumulation_len = int(_d['xengine']['xeng_accumulation_len'])
self.n_chans = int(_d['fengine']['n_chans'])
self.n_antennas = int(_d['fengine']['n_antennas'])
self.min_load_time = float(_d['fengine']['min_load_time'])
self.set_stream_destination(
_d['xengine']['output_destination_ip'],
int(_d['xengine']['output_destination_port']))
self.set_meta_destination(_d['xengine']['output_destination_ip'],
int(_d['xengine']['output_destination_port']))
# get this from the running x-engines?
self.xeng_clk = int(_d['xengine']['x_fpga_clock'])
self.xeng_outbits = int(_d['xengine']['xeng_outbits'])
# the f-engines have this many 10Gbe ports per f-engine
# unit of operation
self.ports_per_fengine = int(_d['fengine']['ports_per_fengine'])
# check if beamformer exists with x-engines
self.found_beamformer = False
if 'bengine' in self.configd.keys():
self.found_beamformer = True
# set up the hosts and engines based on the configuration in the ini file
self.fhosts = []
for host in _d['fengine']['hosts'].split(','):
host = host.strip()
fpgahost = fhost_fpga.FpgaFHost.from_config_source(
host, self.katcp_port, config_source=_d['fengine'])
self.fhosts.append(fpgahost)
# choose class (b-engine inherits x-engine functionality)
if self.found_beamformer:
_targetClass = bhost_fpga.FpgaBHost
else:
_targetClass = xhost_fpga.FpgaXHost
self.xhosts = []
for host in _d['xengine']['hosts'].split(','):
host = host.strip()
fpgahost = xhost_fpga.FpgaXHost.from_config_source(
host, self.katcp_port, config_source=_d['xengine'])
self.xhosts.append(fpgahost)
self.bhosts = []
# x-eng host b-eng
for host in _d['xengine']['hosts'].split(','):
host = host.strip()
fpgahost = bhost_fpga.FpgaBHost.from_config_source(
host, self.katcp_port, config_source=_d['xengine'])
self.bhosts.append(fpgahost)
# check that no hosts overlap
for _fh in self.fhosts:
for _xh in self.xhosts:
if _fh.host == _xh.host:
self.logger.error('Host %s is assigned to both X- and '
'F-engines' % _fh.host)
raise RuntimeError
# what data sources have we been allocated?
self._handle_sources()
# turn the product names into a list
prodlist = _d['xengine']['output_products'].replace('[', '').replace(']', '').split(',')
_d['xengine']['output_products'] = []
for prod in prodlist:
_d['xengine']['output_products'].append(prod.strip(''))
def _handle_sources(self):
"""
Sort out sources and eqs for them
:return:
"""
assert len(self.fhosts) > 0
_feng_cfg = self.configd['fengine']
source_names = _feng_cfg['source_names'].strip().split(',')
source_mcast = _feng_cfg['source_mcast_ips'].strip().split(',')
assert len(source_mcast) == len(source_names), (
'Source names (%d) must be paired with multicast source '
'addresses (%d)' % (len(source_names), len(source_mcast)))
# match eq polys to source names
eq_polys = {}
for src_name in source_names:
eq_polys[src_name] = utils.process_new_eq(
_feng_cfg['eq_poly_%s' % src_name])
# assemble the sources given into a list
_fengine_sources = []
for source_ctr, address in enumerate(source_mcast):
new_source = DataSource.from_mcast_string(address)
new_source.name = source_names[source_ctr]
assert new_source.ip_range == self.ports_per_fengine, (
'F-engines should be receiving from %d streams.' %
self.ports_per_fengine)
_fengine_sources.append(new_source)
# assign sources and eqs to fhosts
self.logger.info('Assigning DataSources, EQs and DelayTrackers to '
'f-engines...')
source_ctr = 0
self.fengine_sources = []
for fhost in self.fhosts:
self.logger.info('\t%s:' % fhost.host)
_eq_dict = {}
for fengnum in range(0, self.f_per_fpga):
_source = _fengine_sources[source_ctr]
_eq_dict[_source.name] = {'eq': eq_polys[_source.name],
'bram_num': fengnum}
assert _source.ip_range == _fengine_sources[0].ip_range, (
'All f-engines should be receiving from %d streams.' %
self.ports_per_fengine)
self.fengine_sources.append({'source': _source,
'source_num': source_ctr,
'host': fhost,
'numonhost': fengnum})
fhost.add_source(_source)
self.logger.info('\t\t%s' % _source)
source_ctr += 1
fhost.eqs = _eq_dict
if source_ctr != len(self.fhosts) * self.f_per_fpga:
raise RuntimeError('We have different numbers of sources (%d) and '
'f-engines (%d). Problem.', source_ctr,
len(self.fhosts) * self.f_per_fpga)
self.logger.info('done.')
def _read_config_file(self):
"""
Read the instrument configuration from self.config_source.
:return: True if we read the file successfully, False if not
"""
self.configd = utils.parse_ini_file(self.config_source)
def _read_config_server(self):
"""
Get instance-specific setup information from a given server. Via KATCP?
:return:
"""
raise NotImplementedError('_read_config_server not implemented')
def set_stream_destination(self, txip_str=None, txport=None):
"""
Set destination for output of fxcorrelator.
:param txip_str: A dotted-decimal string representation of the IP address. e.g. '1.2.3.4'
:param txport: An integer port number.
:return: <nothing>
"""
if txip_str is None:
txip = tengbe.IpAddress.str2ip(self.xeng_tx_destination[0])
else:
txip = tengbe.IpAddress.str2ip(txip_str)
if txport is None:
txport = self.xeng_tx_destination[1]
else:
txport = int(txport)
self.logger.info('Setting stream destination to %s:%d' %
(tengbe.IpAddress.ip2str(txip), txport))
try:
THREADED_FPGA_OP(self.xhosts, timeout=10,
target_function=(lambda fpga_:
fpga_.registers.gbe_iptx.write(reg=txip),))
THREADED_FPGA_OP(self.xhosts, timeout=10,
target_function=(lambda fpga_:
fpga_.registers.gbe_porttx.write(reg=txport),))
except AttributeError:
self.logger.warning('Set SPEAD stream destination called, but '
'devices NOT written! Have they been created?')
self.xeng_tx_destination = (tengbe.IpAddress.ip2str(txip), txport)
def set_meta_destination(self, txip_str=None, txport=None):
"""
Set destination for meta info output of fxcorrelator.
:param txip_str: A dotted-decimal string representation of the IP address. e.g. '1.2.3.4'
:param txport: An integer port number.
:return: <nothing>
"""
if txip_str is None:
txip_str = self.meta_destination[0]
if txport is None:
txport = self.meta_destination[1]
else:
txport = int(txport)
if txport is None or txip_str is None:
self.logger.error('Cannot set part of meta destination to None - %s:%d' %
(txip_str, txport))
raise RuntimeError('Cannot set part of meta destination to None - %s:%d' %
(txip_str, txport))
self.meta_destination = (txip_str, txport)
self.logger.info('Setting meta destination to %s:%d' % (txip_str, txport))
def tx_start(self, issue_spead=True):
"""
Turns on xengine output pipes needed to start data flow from xengines
"""
self.logger.info('Starting transmission')
if issue_spead:
self.spead_issue_meta()
# start tx on the x-engines
for f in self.xhosts:
f.registers.control.write(gbe_txen=True)
def tx_stop(self, stop_f=False):
"""
Turns off output pipes to start data flow from xengines
:param stop_f: stop output of fengines as well
"""
self.logger.info('Stopping X transmission')
THREADED_FPGA_OP(
self.xhosts, timeout=10, target_function=(
lambda fpga_: fpga_.registers.control.write(gbe_txen=False),))
if stop_f:
self.logger.info('Stopping F transmission')
THREADED_FPGA_OP(
self.fhosts, timeout=10, target_function=(
lambda fpga_:
fpga_.registers.control.write(comms_en=False),))
def _spead_meta_get_labelling(self):
"""
Get a list of the source labelling for SPEAD metadata
:return:
"""
metalist = []
for fsrc in self.fengine_sources:
metalist.append((fsrc['source'].name, fsrc['source_num'],
fsrc['host'].host, fsrc['numonhost']))
return metalist
def spead_issue_meta(self):
"""
All FxCorrelators issued SPEAD in the same way, with tweakings that are
implemented by the child class.
:return: <nothing>
"""
if self.meta_destination is None:
logging.info('SPEAD meta destination is still unset, NOT sending '
'metadata at this time.')
return
# make a new SPEAD transmitter
del self.spead_tx, self.spead_meta_ig
self.spead_tx = spead.Transmitter(spead.TransportUDPtx(*self.meta_destination))
# update the multicast socket option to use a TTL of 2,
# in order to traverse the L3 network on site.
ttl_bin = struct.pack('@i', 2)
self.spead_tx.t._udp_out.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl_bin)
#mcast_interface = self.configd['xengine']['multicast_interface_address']
#self.spead_tx.t._udp_out.setsockopt(socket.SOL_IP, socket.IP_MULTICAST_IF,
# socket.inet_aton(mcast_interface))
# self.spead_tx.t._udp_out.setsockopt(socket.SOL_IP, socket.IP_ADD_MEMBERSHIP,
# socket.inet_aton(txip_str) + socket.inet_aton(mcast_interface))
# make the item group we're going to use
self.spead_meta_ig = spead.ItemGroup()
self.spead_meta_ig.add_item(name='adc_sample_rate', id=0x1007,
description='The expected ADC sample rate (samples '
'per second) of incoming data.',
shape=[], fmt=spead.mkfmt(('u', 64)),
init_val=self.sample_rate_hz)
self.spead_meta_ig.add_item(name='n_bls', id=0x1008,
description='Number of baselines in the data product.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=len(self.xops.get_baseline_order()))
self.spead_meta_ig.add_item(name='n_chans', id=0x1009,
description='Number of frequency channels in '
'an integration.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=self.n_chans)
self.spead_meta_ig.add_item(name='n_ants', id=0x100A,
description='The number of antennas in the system.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=self.n_antennas)
self.spead_meta_ig.add_item(name='n_xengs', id=0x100B,
description='The number of x-engines in the system.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=(len(self.xhosts) * self.x_per_fpga))
self.spead_meta_ig.add_item(name='bls_ordering', id=0x100C,
description='The baseline ordering in the output '
'data product.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=numpy.array(
[baseline for baseline in self.xops.get_baseline_order()]))
# spead_ig.add_item(name='crosspol_ordering', id=0x100D,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
metalist = numpy.array(self._spead_meta_get_labelling())
self.spead_meta_ig.add_item(name='input_labelling', id=0x100E,
description='input labels and numbers',
init_val=metalist)
# spead_ig.add_item(name='n_bengs', id=0x100F,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
self.spead_meta_ig.add_item(name='center_freq', id=0x1011,
description='The on-sky centre-frequency.',
shape=[], fmt=spead.mkfmt(('f', 64)),
init_val=int(self.configd['fengine']['true_cf']))
self.spead_meta_ig.add_item(name='bandwidth', id=0x1013,
description='The input (analogue) bandwidth of '
'the system.',
shape=[], fmt=spead.mkfmt(('f', 64)),
init_val=int(self.configd['fengine']['bandwidth']))
self.spead_meta_ig.add_item(name='n_accs', id=0x1015,
description='The number of spectra that are '
'accumulated per X-engine dump.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=self.accumulation_len * self.xeng_accumulation_len)
self.spead_meta_ig.add_item(name='int_time', id=0x1016,
description='The time per integration, in seconds.',
shape=[], fmt=spead.mkfmt(('f', 64)),
init_val=self.xops.get_acc_time())
# spead_ig.add_item(name='coarse_chans', id=0x1017,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
#
# spead_ig.add_item(name='current_coarse_chan', id=0x1018,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
#
# spead_ig.add_item(name='fft_shift_fine', id=0x101C,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
#
# spead_ig.add_item(name='fft_shift_coarse', id=0x101D,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
self.spead_meta_ig.add_item(name='fft_shift', id=0x101E,
description='The FFT bitshift pattern. F-engine correlator internals.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=int(self.configd['fengine']['fft_shift']))
self.spead_meta_ig.add_item(name='xeng_acc_len', id=0x101F,
description='Number of spectra accumulated inside X engine. '
'Determines minimum integration time and '
'user-configurable integration time stepsize. '
'X-engine correlator internals.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=self.xeng_accumulation_len)
quant_bits = int(self.configd['fengine']['quant_format'].split('.')[0])
self.spead_meta_ig.add_item(name='requant_bits', id=0x1020,
description='Number of bits after requantisation in the '
'F engines (post FFT and any '
'phasing stages).',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=quant_bits)
pkt_len = int(self.configd['fengine']['10gbe_pkt_len'])
self.spead_meta_ig.add_item(name='feng_pkt_len', id=0x1021,
description='Payload size of 10GbE packet exchange between '
'F and X engines in 64 bit words. Usually equal '
'to the number of spectra accumulated inside X '
'engine. F-engine correlator internals.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=pkt_len)
# port = int(self.configd['xengine']['output_destination_port'])
# spead_ig.add_item(name='rx_udp_port', id=0x1022,
# description='Destination UDP port for X engine output.',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=port)
port = int(self.configd['fengine']['10gbe_port'])
self.spead_meta_ig.add_item(name='feng_udp_port', id=0x1023,
description='Port for F-engines 10Gbe links in the system.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=port)
# ip = self.configd['xengine']['output_destination_ip']
# spead_ig.add_item(name='rx_udp_ip_str', id=0x1024,
# description='Destination UDP IP for X engine output.',
# shape=[-1], fmt=spead.STR_FMT,
# init_val=ip)
ip = struct.unpack('>I', socket.inet_aton(self.configd['fengine']['10gbe_start_ip']))[0]
self.spead_meta_ig.add_item(name='feng_start_ip', id=0x1025,
description='Start IP address for F-engines in the system.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=ip)
self.spead_meta_ig.add_item(name='xeng_rate', id=0x1026,
description='Target clock rate of processing engines (xeng).',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=self.xeng_clk)
self.spead_meta_ig.add_item(name='sync_time', id=0x1027,
description='The time at which the digitisers were synchronised. '
'Seconds since the Unix Epoch.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=self.synchronisation_epoch)
# spead_ig.add_item(name='n_stokes', id=0x1040,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
x_per_fpga = int(self.configd['xengine']['x_per_fpga'])
self.spead_meta_ig.add_item(name='x_per_fpga', id=0x1041,
description='Number of X engines per FPGA host.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=x_per_fpga)
# n_ants_per_xaui = 1
# spead_ig.add_item(name='n_ants_per_xaui', id=0x1042,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=n_ants_per_xaui)
# spead_ig.add_item(name='ddc_mix_freq', id=0x1043,
# description='',
# shape=[],fmt=spead.mkfmt(('f', 64)),
# init_val=)
# spead_ig.add_item(name='ddc_bandwidth', id=0x1044,
# description='',
# shape=[],fmt=spead.mkfmt(('f', 64)),
# init_val=)
sample_bits = int(self.configd['fengine']['sample_bits'])
self.spead_meta_ig.add_item(name='adc_bits', id=0x1045,
description='How many bits per ADC sample.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=sample_bits)
self.spead_meta_ig.add_item(name='scale_factor_timestamp', id=0x1046,
description='Timestamp scaling factor. Divide the SPEAD '
'data packet timestamp by this number to get '
'back to seconds since last sync.',
shape=[], fmt=spead.mkfmt(('f', 64)),
init_val=self.sample_rate_hz)
self.spead_meta_ig.add_item(name='xeng_out_bits_per_sample', id=0x1048,
description='The number of bits per value of the xeng '
'accumulator output. Note this is for a '
'single value, not the combined complex size.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=self.xeng_outbits)
self.spead_meta_ig.add_item(name='f_per_fpga', id=0x1049,
description='Number of F engines per FPGA host.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
init_val=self.f_per_fpga)
# spead_ig.add_item(name='rf_gain_MyAntStr ', id=0x1200+inputN,
# description='',
# shape=[], fmt=spead.mkfmt(('f', 64)),
# init_val=)
# 0x1400 +++
self.fops.eq_update_metadata()
# spead_ig.add_item(name='eq_coef_MyAntStr', id=0x1400+inputN,
# description='',
# shape=[], fmt=spead.mkfmt(('u', 32)),
# init_val=)
# ndarray = numpy.dtype(numpy.int64), (4096 * 40 * 1, 1, 1)
self.spead_meta_ig.add_item(name='timestamp', id=0x1600,
description='Timestamp of start of this integration. uint counting multiples '
'of ADC samples since last sync (sync_time, id=0x1027). Divide this '
'number by timestamp_scale (id=0x1046) to get back to seconds since '
'last sync when this integration was actually started.',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)))
self.spead_meta_ig.add_item(name='flags_xeng_raw', id=0x1601,
description='Flags associated with xeng_raw data output.'
'bit 34 - corruption or data missing during integration '
'bit 33 - overrange in data path '
'bit 32 - noise diode on during integration '
'bits 0 - 31 reserved for internal debugging',
shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)))
ndarray = numpy.dtype(numpy.int32), (self.n_chans, len(self.xops.get_baseline_order()), 2)
self.spead_meta_ig.add_item(name='xeng_raw', id=0x1800,
description='Raw data for %i xengines in the system. This item represents a '
'full spectrum (all frequency channels) assembled from lowest '
'frequency to highest frequency. Each frequency channel contains '
'the data for all baselines (n_bls given by SPEAD ID 0x100b). '
'Each value is a complex number -- two (real and imaginary) '
'unsigned integers.' % len(self.xhosts * self.x_per_fpga),
ndarray=ndarray)
# TODO hard-coded !!!!!!! :(
# self.spead_ig.add_item(name=("xeng_raw"),id=0x1800,
# description="Raw data for %i xengines in the system. This
# item represents a full spectrum (all frequency channels) assembled from lowest frequency
# to highest frequency. Each frequency channel contains the data for all baselines
# (n_bls given by SPEAD ID 0x100B). Each value is a complex number -- two
# (real and imaginary) unsigned integers."%(32),
# ndarray=(numpy.dtype(numpy.int32),(4096,((4*(4+1))/2)*4,2)))
# spead_ig.add_item(name='beamweight_MyAntStr', id=0x2000+inputN,
# description='',
# shape=[], fmt=spead.mkfmt(('u', 32)),
# init_val=)
# spead_ig.add_item(name='incoherent_sum', id=0x3000,
# description='',
# shape=[], fmt=spead.mkfmt(('u', 32)),
# init_val=)
# spead_ig.add_item(name='n_inputs', id=0x3100,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
# spead_ig.add_item(name='digitiser_id', id=0x3101,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
# spead_ig.add_item(name='digitiser_status', id=0x3102,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
# spead_ig.add_item(name='pld_len', id=0x3103,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
# spead_ig.add_item(name='raw_data_MyAntStr', id=0x3300+inputN,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
# spead_ig.add_item(name='Reserved for SP-CAM meta-data', id=0x7000-0x7fff,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
# spead_ig.add_item(name='feng_id', id=0xf101,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
# spead_ig.add_item(name='feng_status', id=0xf102,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
# spead_ig.add_item(name='frequency', id=0xf103,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
# spead_ig.add_item(name='raw_freq_MyAntStr', id=0xf300+inputN,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
# spead_ig.add_item(name='bf_MyBeamName', id=0xb000+beamN,
# description='',
# shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
# init_val=)
self.spead_tx.send_heap(self.spead_meta_ig.get_heap())
self.logger.info('Issued SPEAD data descriptor to %s:%i.' % (self.meta_destination[0],
self.meta_destination[1]))
def initialise(self, program=True, qdr_cal=True, require_epoch=False):
"""
Set up the correlator using the information in the config file.
:param program: program the FPGA boards if True
:param qdr_cal: perform QDR cal if True
:param require_epoch: the synch epoch MUST be set before init if True
:return:
"""
# check that the instrument's synch epoch has been set
if require_epoch:
if self.get_synch_time() == -1:
raise RuntimeError('System synch epoch has not been set prior'
' to initialisation!')
# set up the F, X, B and filter handlers
self.fops = FEngineOperations(self)
self.xops = XEngineOperations(self)
self.bops = BEngineOperations(self)
self.filtops = FilterOperations(self)
self.speadops = SpeadOperations(self)
# set up the filter boards if we need to
if 'filter' in self.configd:
try:
self.filtops.initialise(program=program)
except Exception as e:
raise e
# connect to the other hosts that make up this correlator
THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=5,
target_function='connect')
igmp_version = self.configd['FxCorrelator'].get('igmp_version')
if igmp_version is not None:
self.logger.info('Setting FPGA hosts IGMP version '
'to %s' % igmp_version)
THREADED_FPGA_FUNC(
self.fhosts + self.xhosts, timeout=5,
target_function=('set_igmp_version', (igmp_version, ), {}))
# if we need to program the FPGAs, do so
if program:
self.logger.info('Programming FPGA hosts')
fpgautils.program_fpgas([(host, host.boffile) for host in
(self.fhosts + self.xhosts)],
progfile=None, timeout=15)
else:
self.logger.info('Loading design information')
THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=10,
target_function='get_system_information')
# remove test hardware from designs
utils.disable_test_gbes(self)
utils.remove_test_objects(self)
# run configuration on the parts of the instrument
self.configure()
# run post-programming initialisation
self._initialise(program, qdr_cal)
# reset all counters on fhosts and xhosts
self.fops.clear_status_all()
self.xops.clear_status_all()
# set an initialised flag
self._initialised = True
def initialise(self, program=True, qdr_cal=True):
"""
Set up the correlator using the information in the config file.
:return:
"""
# set up the F, X, B and filter handlers
self.fops = FEngineOperations(self)
self.xops = XEngineOperations(self)
# self.bops = BEngineOperations(self)
self.filtops = FilterOperations(self)
# set up the filter boards if we need to
if 'filter' in self.configd:
try:
self.filtops.initialise(program=program)
except Exception as e:
raise e
# connect to the other hosts that make up this correlator
THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=5,
target_function='connect')
igmp_version = self.configd['FxCorrelator'].get('igmp_version')
if igmp_version is not None:
self.logger.info('Setting FPGA hosts IGMP version to '
'%s' % igmp_version)
THREADED_FPGA_FUNC(
self.fhosts + self.xhosts, timeout=5, target_function=(
'set_igmp_version', (igmp_version, ), {}))
# if we need to program the FPGAs, do so
if program:
self.logger.info('Programming FPGA hosts')
fpgautils.program_fpgas([(host, host.boffile) for host in
(self.fhosts + self.xhosts)],
progfile=None, timeout=15)
else:
self.logger.info('Loading design information')
THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=7,
target_function='get_system_information')
# remove test hardware from designs
utils.disable_test_gbes(self)
utils.remove_test_objects(self)
if program:
# cal the qdr on all boards
if qdr_cal:
self.qdr_calibrate()
else:
self.logger.info('Skipping QDR cal - are you sure you'
' want to do this?')
# init the engines
self.fops.initialise()
self.xops.initialise()
# are there beamformers?
if self.found_beamformer:
bengops.beng_initialise(self)
# for fpga_ in self.fhosts:
# fpga_.tap_arp_reload()
# for fpga_ in self.xhosts:
# fpga_.tap_arp_reload()
# subscribe all the engines to the multicast groups
self.fops.subscribe_to_multicast()
self.xops.subscribe_to_multicast()
post_mess_delay = 10
self.logger.info('post mess-with-the-switch delay of '
'%is' % post_mess_delay)
time.sleep(post_mess_delay)
# force a reset on the f-engines
self.fops.sys_reset(sleeptime=1)
# reset all counters on fhosts and xhosts
self.fops.clear_status_all()
self.xops.clear_status_all()
# check to see if the f engines are receiving all their data
if not self.fops.check_rx():
raise RuntimeError('The f-engines RX have a problem.')
# start f-engine TX
self.logger.info('Starting f-engine datastream')
self.fops.tx_enable()
# check that the F-engines are transmitting data correctly
if not self.fops.check_tx():
raise RuntimeError('The f-engines TX have a problem.')
# check that the X-engines are receiving data
if not self.xops.check_rx():
raise RuntimeError('The x-engines RX have a problem.')
# arm the vaccs on the x-engines
self.xops.vacc_sync()
# reset all counters on fhosts and xhosts
self.fops.clear_status_all()
self.xops.clear_status_all()
# set an initialised flag
self._initialised = True
class FxCorrelator(Instrument):
"""
A generic FxCorrelator composed of fengines that channelise antenna inputs
and xengines that each produce cross products from a continuous portion
of the channels and accumulate the result.
SPEAD data streams are produced.
"""
def __init__(self, descriptor, identifier=-1, config_source=None,
logger=LOGGER):
"""
An abstract base class for instruments.
:param descriptor: A text description of the instrument. Required.
:param identifier: An optional integer identifier.
:param config_source: The instrument configuration source. Can be a
text file, hostname, whatever.
:param logger: Use the module logger by default, unless something else
is given.
:return: <nothing>
"""
self.logger = logger
# we know about f and x hosts and engines, not just engines and hosts
self.fhosts = []
self.xhosts = []
self.filthosts = None
self.found_beamformer = False
self.fops = None
self.xops = None
self.bops = None
self.filtops = None
self.speadops = None
# attributes
self.katcp_port = None
self.f_per_fpga = None
self.x_per_fpga = None
self.accumulation_len = None
self.xeng_accumulation_len = None
self.fengine_sources = None
self.baselines = None
self.sensor_manager = None
# parent constructor
Instrument.__init__(self, descriptor, identifier, config_source, logger)
def initialise(self, program=True, qdr_cal=True, require_epoch=False):
"""
Set up the correlator using the information in the config file.
:param program: program the FPGA boards if True
:param qdr_cal: perform QDR cal if True
:param require_epoch: the synch epoch MUST be set before init if True
:return:
"""
# check that the instrument's synch epoch has been set
if require_epoch:
if self.get_synch_time() == -1:
raise RuntimeError('System synch epoch has not been set prior'
' to initialisation!')
# set up the F, X, B and filter handlers
self.fops = FEngineOperations(self)
self.xops = XEngineOperations(self)
self.bops = BEngineOperations(self)
self.filtops = FilterOperations(self)
self.speadops = SpeadOperations(self)
# set up the filter boards if we need to
if 'filter' in self.configd:
try:
self.filtops.initialise(program=program)
except Exception as e:
raise e
# connect to the other hosts that make up this correlator
THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=5,
target_function='connect')
igmp_version = self.configd['FxCorrelator'].get('igmp_version')
if igmp_version is not None:
self.logger.info('Setting FPGA hosts IGMP version '
'to %s' % igmp_version)
THREADED_FPGA_FUNC(
self.fhosts + self.xhosts, timeout=5,
target_function=('set_igmp_version', (igmp_version, ), {}))
# if we need to program the FPGAs, do so
if program:
self.logger.info('Programming FPGA hosts')
fpgautils.program_fpgas([(host, host.boffile) for host in
(self.fhosts + self.xhosts)],
progfile=None, timeout=15)
else:
self.logger.info('Loading design information')
THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=10,
target_function='get_system_information')
# remove test hardware from designs
utils.disable_test_gbes(self)
utils.remove_test_objects(self)
# run configuration on the parts of the instrument
self.configure()
# run post-programming initialisation
self._initialise(program, qdr_cal)
# reset all counters on fhosts and xhosts
self.fops.clear_status_all()
self.xops.clear_status_all()
# set an initialised flag
self._initialised = True
def _gbe_setup(self):
"""
Set up the 10gbe ports on the hosts
:return:
"""
feng_port = int(self.configd['fengine']['10gbe_port'])
xeng_port = int(self.configd['xengine']['10gbe_port'])
info_dict = {host.host: (feng_port, 'fhost') for host in self.fhosts}
info_dict.update(
{host.host: (xeng_port, 'xhost') for host in self.xhosts})
timeout = len(self.fhosts[0].tengbes) * 30 * 1.1
THREADED_FPGA_FUNC(
self.fhosts + self.xhosts, timeout=timeout,
target_function=('setup_host_gbes',
(self.logger, info_dict), {}))
def _initialise(self, program, qdr_cal):
"""
Run this if boards in the system have been programmed. Basic setup
of devices.
:return:
"""
if not program:
# only run the contents of this function after programming.
return
# cal the qdr on all boards
# logging.getLogger('casperfpga.qdr').setLevel(logging.INFO + 7)
if qdr_cal:
self.qdr_calibrate()
else:
self.logger.info('Skipping QDR cal - are you sure you '
'want to do this?')
# init the engines
self.fops.initialise_pre_gbe()
self.xops.initialise_pre_gbe()
# set up the tengbe ports in parallel
self._gbe_setup()
# continue with init
self.fops.initialise_post_gbe()
self.xops.initialise_post_gbe()
if self.found_beamformer:
self.bops.initialise()
# subscribe all the engines to the multicast groups
self.fops.subscribe_to_multicast()
self.xops.subscribe_to_multicast()
# start f-engine TX
self.logger.info('Starting f-engine datastream')
self.fops.tx_enable()
# jason's hack to force a reset on the f-engines
time.sleep(1)
self.fops.sys_reset()
# wait for switches to learn, um, stuff
self.logger.info('post mess-with-the-switch delay of %is' %
self.post_switch_delay)
time.sleep(self.post_switch_delay)
# jason's hack to force a reset on the f-engines
self.fops.sys_reset()
time.sleep(1)
# reset all counters on fhosts and xhosts
self.fops.clear_status_all()
self.xops.clear_status_all()
# check to see if the f engines are receiving all their data
if not self.fops.check_rx():
raise RuntimeError('The f-engines RX have a problem.')
# check that the timestamps received on the f-engines make sense
result, times, times_unix = self.fops.check_rx_timestamps()
if not result:
raise RuntimeError('The timestamps received by the f-engines '
'are not okay. Check the logs')
# check the f-engine QDR uses for parity errors
if ((not self.fops.check_ct_parity()) or
(not self.fops.check_cd_parity())):
raise RuntimeError('The f-engine QDRs are reporting errors.')
# check that the F-engines are transmitting data correctly
if not self.fops.check_tx():
raise RuntimeError('The f-engines TX have a problem.')
# check that the X-engines are receiving data
if not self.xops.check_rx():
raise RuntimeError('The x-engines RX have a problem.')
# arm the vaccs on the x-engines
self.xops.vacc_sync()
def configure(self):
"""
Operations to run to configure the instrument, after programming.
:return:
"""
self.fops.configure()
self.xops.configure()
if self.found_beamformer:
self.bops.configure()
def get_scale_factor(self):
"""
By what number do we divide timestamps to get seconds?
:return:
"""
return self.sample_rate_hz
def set_synch_time(self, new_synch_time):
"""
Set the last sync time for this system
:param new_synch_time: UNIX time
:return: <nothing>
"""
time_now = time.time()
# future?
if new_synch_time > time_now:
_err = 'Synch time in the future makes no sense? %.3f > %.3f' % (
new_synch_time, time_now)
self.logger.error(_err)
raise RuntimeError(_err)
# too far in the past?
if new_synch_time < time_now - (2**self.timestamp_bits):
_err = 'Synch epoch supplied is too far in the past: now(%.3f) ' \
'epoch(%.3f)' % (time_now, new_synch_time)
self.logger.error(_err)
raise RuntimeError(_err)
self._synchronisation_epoch = new_synch_time
if self.sensor_manager:
sensor = self.sensor_manager.sensor_get('synch-epoch')
sensor.set_value(self._synchronisation_epoch)
self.logger.info('Set synch epoch to %.5f' % new_synch_time)
def est_synch_epoch(self):
"""
Estimates the synchronisation epoch based on current F-engine
timestamp, and the system time.
"""
self.logger.info('Estimating synchronisation epoch:')
# get current time from an f-engine
feng_mcnt = self.fhosts[0].get_local_time()
self.logger.info('\tcurrent f-engine mcnt: %i' % feng_mcnt)
if feng_mcnt & 0xfff != 0:
_err = 'Bottom 12 bits of timestamp from f-engine are not ' \
'zero?! feng_mcnt(%i)' % feng_mcnt
self.logger.error(_err)
raise RuntimeError(_err)
t_now = time.time()
self.set_synch_time(t_now - feng_mcnt / float(self.sample_rate_hz))
self.logger.info('\tnew epoch: %.3f' % self.get_synch_time())
def time_from_mcnt(self, mcnt):
"""
Returns the unix time UTC equivalent to the board timestamp. Does
NOT account for wrapping timestamps.
:param mcnt: the time from system boards (ADC ticks since startup)
"""
if self.get_synch_time() < 0:
self.logger.info('time_from_mcnt: synch epoch unset, estimating')
self.est_synch_epoch()
return self.get_synch_time() + (
float(mcnt) / float(self.sample_rate_hz))
def mcnt_from_time(self, time_seconds):
"""
Returns the board timestamp from a given UTC system time
(seconds since Unix Epoch). Accounts for wrapping timestamps.
:param time_seconds: UNIX time
"""
if self.get_synch_time() < 0:
self.logger.info('mcnt_from_time: synch epoch unset, estimating')
self.est_synch_epoch()
time_diff_from_synch_epoch = time_seconds - self.get_synch_time()
time_diff_in_samples = int(time_diff_from_synch_epoch *
self.sample_rate_hz)
_tmp = 2**self.timestamp_bits
return time_diff_in_samples % _tmp
def qdr_calibrate(self, timeout=120 * MAX_QDR_ATTEMPTS):
"""
Run a software calibration routine on all the FPGA hosts.
Do it on F- and X-hosts in parallel.
:param timeout: how many seconds to wait for it to complete
:return:
"""
def _qdr_cal(_fpga):
"""
Calibrate the QDRs found on a given FPGA.
:param _fpga:
:return:
"""
_tic = time.time()
attempts = 0
_results = {_qdr.name: False for _qdr in _fpga.qdrs}
while True:
all_passed = True
for _qdr in _fpga.qdrs:
if not _results[_qdr.name]:
try:
_res = _qdr.qdr_cal(fail_hard=False)
except RuntimeError:
_res = False
try:
_resval = _res[0]
except TypeError:
_resval = _res
if not _resval:
all_passed = False
_results[_qdr.name] = _resval
attempts += 1
if all_passed or (attempts >= MAX_QDR_ATTEMPTS):
break
_toc = time.time()
return {'results': _results, 'tic': _tic, 'toc': _toc,
'attempts': attempts}
self.logger.info('Calibrating QDR on F- and X-engines, this may '
'take a while.')
qdr_calfail = False
results = THREADED_FPGA_OP(
self.fhosts + self.xhosts, timeout, (_qdr_cal,))
for fpga, result in results.items():
_time_taken = result['toc'] - result['tic']
self.logger.info('FPGA %s QDR cal: %.3fs, %i attempts' %
(fpga, _time_taken, result['attempts']))
for qdr, qdrres in result['results'].items():
if not qdrres:
qdr_calfail = True
break
self.logger.info('\t%s: cal okay: %s' %
(qdr, 'True' if qdrres else 'False'))
if qdr_calfail:
raise RuntimeError('QDR calibration failure.')
# for host in self.fhosts:
# for qdr in host.qdrs:
# qdr.qdr_delay_in_step(0b111111111111111111111111111111111111,
# -1)
# for host in self.xhosts:
# for qdr in host.qdrs:
# qdr.qdr_delay_in_step(0b111111111111111111111111111111111111,
# -1)
def set_labels(self, newlist):
"""
Apply new source labels to the configured fengine sources.
:param newlist:
:return:
"""
old_labels = self.get_labels()
if len(newlist) != len(old_labels):
errstr = 'Number of supplied source labels (%i) does not match ' \
'number of configured sources (%i).' % \
(len(newlist), len(old_labels))
self.logger.error(errstr)
raise ValueError(errstr)
new_dict = {}
for ctr, source_name in enumerate(old_labels):
_source = self.fengine_sources[source_name]
_source.update_name(newlist[ctr])
new_dict[newlist[ctr]] = _source
self.fengine_sources = new_dict
# update the list of baselines
self.baselines = utils.baselines_from_source_list(newlist)
# update the hostname and baseline sensors
if self.sensor_manager:
sm = self.sensor_manager
try:
for fhost in self.fhosts:
sensor = sm.sensor_get('%s-input-mapping' % fhost.host)
rv = [dsrc.name for dsrc in fhost.data_sources]
sensor.set_value(str(rv))
except Exception as ve:
self.logger.warning('Could not update input_mapping '
'sensors!\n%s' % ve.message)
sm.sensor_get('baseline-ordering').set_value(str(self.baselines))
# update the beam input labels
if self.found_beamformer:
self.bops.update_labels(old_labels, self.get_labels())
self.speadops.update_metadata([0x100e])
self.logger.info('Source labels updated from %s to %s' % (
old_labels, self.get_labels()))
def get_labels(self):
"""
Get the current fengine source labels as a list of label names.
:return:
"""
return sorted(self.fengine_sources,
key=lambda k: self.fengine_sources[k].source_number)
def _read_config(self):
"""
Read the instrument configuration from self.config_source.
:return:
"""
Instrument._read_config(self)
_d = self.configd
# check that the bitstream names are present
try:
open(_d['fengine']['bitstream'], 'r').close()
open(_d['xengine']['bitstream'], 'r').close()
except IOError:
self.logger.error('xengine bitstream: '
'%s' % _d['xengine']['bitstream'])
self.logger.error('fengine bitstream: '
'%s' % _d['fengine']['bitstream'])
self.logger.error('One or more bitstream files not found.')
raise IOError('One or more bitstream files not found.')
# TODO: Load config values from the bitstream meta information -
# f per fpga, x per fpga, etc
_fxcorr_d = self.configd['FxCorrelator']
self.arp_wait_time = int(_fxcorr_d['arp_wait_time'])
self.sensor_poll_time = int(_fxcorr_d['sensor_poll_time'])
self.katcp_port = int(_fxcorr_d['katcp_port'])
self.sample_rate_hz = int(_fxcorr_d['sample_rate_hz'])
self.timestamp_bits = int(_fxcorr_d['timestamp_bits'])
self.time_jitter_allowed_ms = int(_fxcorr_d['time_jitter_allowed_ms'])
self.time_offset_allowed_s = int(_fxcorr_d['time_offset_allowed_s'])
try:
self.post_switch_delay = int(_fxcorr_d['switch_delay'])
except KeyError:
self.post_switch_delay = 10
_feng_d = self.configd['fengine']
self.adc_demux_factor = int(_feng_d['adc_demux_factor'])
self.n_chans = int(_feng_d['n_chans'])
self.n_antennas = int(_feng_d['n_antennas'])
self.min_load_time = float(_feng_d['min_load_time'])
self.f_per_fpga = int(_feng_d['f_per_fpga'])
self.ports_per_fengine = int(_feng_d['ports_per_fengine'])
self.analogue_bandwidth = int(_feng_d['bandwidth'])
self.true_cf = float(_feng_d['true_cf'])
self.quant_format = _feng_d['quant_format']
self.adc_bitwidth = int(_feng_d['sample_bits'])
self.fft_shift = int(_feng_d['fft_shift'])
try:
self.qdr_ct_error_threshold = int(_feng_d['qdr_ct_error_threshold'])
except KeyError:
self.qdr_ct_error_threshold = 100
try:
self.qdr_cd_error_threshold = int(_feng_d['qdr_cd_error_threshold'])
except KeyError:
self.qdr_cd_error_threshold = 100
_xeng_d = self.configd['xengine']
self.x_per_fpga = int(_xeng_d['x_per_fpga'])
self.accumulation_len = int(_xeng_d['accumulation_len'])
self.xeng_accumulation_len = int(_xeng_d['xeng_accumulation_len'])
try:
self.qdr_vacc_error_threshold = int(
_xeng_d['qdr_vacc_error_threshold'])
except KeyError:
self.qdr_vacc_error_threshold = 100
# get this from the running x-engines?
self.xeng_clk = int(_xeng_d['x_fpga_clock'])
self.xeng_outbits = int(_xeng_d['xeng_outbits'])
# check if beamformer exists with x-engines
self.found_beamformer = False
if 'bengine' in self.configd.keys():
self.found_beamformer = True
self.beng_outbits = 8
# set up hosts and engines based on the configuration in the ini file
_targetClass = fhost_fpga.FpgaFHost
self.fhosts = []
for host in _feng_d['hosts'].split(','):
host = host.strip()
fpgahost = _targetClass.from_config_source(host, self.katcp_port,
config_source=_feng_d)
self.fhosts.append(fpgahost)
# choose class (b-engine inherits x-engine functionality)
if self.found_beamformer:
_targetClass = bhost_fpga.FpgaBHost
else:
_targetClass = xhost_fpga.FpgaXHost
self.xhosts = []
hostlist = _xeng_d['hosts'].split(',')
for hostindex, host in enumerate(hostlist):
host = host.strip()
fpgahost = _targetClass.from_config_source(
host, hostindex, self.katcp_port, config_source=_xeng_d)
self.xhosts.append(fpgahost)
# check that no hosts overlap
for _fh in self.fhosts:
for _xh in self.xhosts:
if _fh.host == _xh.host:
self.logger.error('Host %s is assigned to '
'both X- and F-engines' % _fh.host)
raise RuntimeError
# update the list of baselines on this system
self.baselines = utils.baselines_from_config(config=self.configd)
# what data sources have we been allocated?
self._handle_sources()
# turn the stream names into a list
prodlist = _xeng_d['output_products'].replace('[', '')
prodlist = prodlist.replace(']', '').split(',')
_xeng_d['output_products'] = [prod.strip(' ') for prod in prodlist]
def _handle_sources(self):
"""
Sort out sources and eqs for them
:return:
"""
assert len(self.fhosts) > 0
_feng_cfg = self.configd['fengine']
source_names = utils.sources_from_config(config=self.configd)
source_mcast = _feng_cfg['source_mcast_ips'].strip().split(',')
assert len(source_mcast) == len(source_names), (
'Source names (%d) must be paired with multicast source '
'addresses (%d)' % (len(source_names), len(source_mcast)))
# match eq polys to source names
eq_polys = {}
for src_name in source_names:
eq_polys[src_name] = utils.process_new_eq(
_feng_cfg['eq_poly_%s' % src_name])
# assemble the sources given into a list
_feng_src_temp = []
for source_ctr, address in enumerate(source_mcast):
new_source = FengineSource.from_mcast_string(address)
new_source.name = source_names[source_ctr]
new_source.source_number = source_ctr
new_source.offset = source_ctr % self.f_per_fpga
new_source.eq_poly = eq_polys[new_source.name]
new_source.eq_bram_name = 'eq%i' % new_source.offset
assert new_source.ip_range == self.ports_per_fengine, (
'F-engines should be receiving from %d streams.' %
self.ports_per_fengine)
_feng_src_temp.append(new_source)
# check that the sources all have the same IP ranges
for _source in _feng_src_temp:
assert _source.ip_range == _feng_src_temp[0].ip_range, (
'All f-engines should be receiving from %d streams.' %
self.ports_per_fengine)
# assign sources to fhosts
self.logger.info('Assigning FengineSources to f-hosts')
_src_ctr = 0
self.fengine_sources = {}
for fhost in self.fhosts:
self.logger.info('\t%s:' % fhost.host)
for fengnum in range(0, self.f_per_fpga):
_src = _feng_src_temp[_src_ctr]
_src.host = fhost
self.fengine_sources[_src.name] = _src
fhost.add_source(_src)
self.logger.info('\t\t%s' % _src)
_src_ctr += 1
if _src_ctr != len(self.fhosts) * self.f_per_fpga:
raise RuntimeError('We have different numbers of sources (%d) and '
'f-engines (%d). Problem.', _src_ctr,
len(self.fhosts) * self.f_per_fpga)
self.logger.info('done.')
def _read_config_file(self):
"""
Read the instrument configuration from self.config_source.
:return: True if we read the file successfully, False if not
"""
self.configd = utils.parse_ini_file(self.config_source)
def _read_config_server(self):
"""
Get instance-specific setup information from a given server. Via KATCP?
:return:
"""
raise NotImplementedError('_read_config_server not implemented')
def stream_set_destination(self, stream_name, txip_str=None, txport=None):
"""
Set the destination for a data stream.
:param stream_name:
:param txip_str: A dotted-decimal string representation of the
IP address. e.g. '1.2.3.4'
:param txport: An integer port number.
:return: <nothing>
"""
stream = self.get_data_stream(stream_name)
stream.set_destination(txip_str, txport)
stream.set_meta_destination(txip_str, txport)
if self.sensor_manager:
sensor_name = '%s-destination' % stream.name
sensor = self.sensor_manager.sensor_get(sensor_name)
sensor.set_value(str(stream.destination))
sensor_name = '%s-meta-destination' % stream.name
sensor = self.sensor_manager.sensor_get(sensor_name)
sensor.set_value(str(stream.meta_destination))
