Example #1
0
class FxCorrelator(Instrument):
    """
    A generic FxCorrelator composed of fengines that channelise antenna inputs
    and xengines that each produce cross products from a continuous portion of
    the channels and accumulate the result. SPEAD data products are produced.
    """
    def __init__(self, descriptor, identifier=-1, config_source=None,
                 logger=LOGGER):
        """
        An abstract base class for instruments.
        :param descriptor: A text description of the instrument. Required.
        :param identifier: An optional integer identifier.
        :param config_source: The instrument configuration source. Can be a text
                              file, hostname, whatever.
        :param logger: Use the module logger by default, unless something else
        is given.
        :return: <nothing>

        """
        self.logger = logger
        self.loghandler = None

        # we know about f and x hosts and engines, not just engines and hosts
        self.fhosts = []
        self.xhosts = []
        self.filthosts = None
        self.found_beamformer = False

        self.fops = None
        self.xops = None
        self.bops = None
        self.filtops = None

        # attributes
        self.katcp_port = None
        self.f_per_fpga = None
        self.x_per_fpga = None
        self.accumulation_len = None
        self.xeng_accumulation_len = None
        self.xeng_tx_destination = None
        self.meta_destination = None
        self.fengine_sources = None

        self.spead_tx = None
        self.spead_meta_ig = None

        self._sensors = {}

        # parent constructor
        Instrument.__init__(self, descriptor, identifier, config_source)

        self._initialised = False

    def standard_log_config(self, log_level=logging.INFO, silence_spead=True):
        """Convenience method for setting up logging in scripts etc.
        :param logger: The loglevel to use by default, logging.INFO.
        :param silence_spead: Set 'spead' logger to level WARNING if True

        """
        # Set the root logging handler - submodules loggers will automatically
        # use it
        self.loghandler = log.Corr2LogHandler()
        logging.root.addHandler(self.loghandler)
        logging.root.setLevel(log_level)
        self.logger.setLevel(log_level)
        self.logger.addHandler(self.loghandler)

        if silence_spead:
            # set the SPEAD logger to warning only
            spead_logger = logging.getLogger('spead')
            spead_logger.setLevel(logging.WARNING)

    def initialise(self, program=True, qdr_cal=True):
        """
        Set up the correlator using the information in the config file.
        :return:
        """
        # set up the F, X, B and filter handlers
        self.fops = FEngineOperations(self)
        self.xops = XEngineOperations(self)
        # self.bops = BEngineOperations(self)
        self.filtops = FilterOperations(self)

        # set up the filter boards if we need to
        if 'filter' in self.configd:
            try:
                self.filtops.initialise(program=program)
            except Exception as e:
                raise e

        # connect to the other hosts that make up this correlator
        THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=5,
                           target_function='connect')

        igmp_version = self.configd['FxCorrelator'].get('igmp_version')
        if igmp_version is not None:
            self.logger.info('Setting FPGA hosts IGMP version to '
                             '%s' % igmp_version)
            THREADED_FPGA_FUNC(
                self.fhosts + self.xhosts, timeout=5, target_function=(
                    'set_igmp_version', (igmp_version, ), {}))

        # if we need to program the FPGAs, do so
        if program:
            self.logger.info('Programming FPGA hosts')
            fpgautils.program_fpgas([(host, host.boffile) for host in
                                     (self.fhosts + self.xhosts)],
                                    progfile=None, timeout=15)
        else:
            self.logger.info('Loading design information')
            THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=7,
                               target_function='get_system_information')

        # remove test hardware from designs
        utils.disable_test_gbes(self)
        utils.remove_test_objects(self)
        if program:
            # cal the qdr on all boards
            if qdr_cal:
                self.qdr_calibrate()
            else:
                self.logger.info('Skipping QDR cal - are you sure you'
                                 ' want to do this?')

            # init the engines
            self.fops.initialise()
            self.xops.initialise()

            # are there beamformers?
            if self.found_beamformer:
                bengops.beng_initialise(self)

            # for fpga_ in self.fhosts:
            #     fpga_.tap_arp_reload()
            # for fpga_ in self.xhosts:
            #     fpga_.tap_arp_reload()

            # subscribe all the engines to the multicast groups
            self.fops.subscribe_to_multicast()
            self.xops.subscribe_to_multicast()

            post_mess_delay = 10
            self.logger.info('post mess-with-the-switch delay of '
                             '%is' % post_mess_delay)
            time.sleep(post_mess_delay)

            # force a reset on the f-engines
            self.fops.sys_reset(sleeptime=1)

            # reset all counters on fhosts and xhosts
            self.fops.clear_status_all()
            self.xops.clear_status_all()

            # check to see if the f engines are receiving all their data
            if not self.fops.check_rx():
                raise RuntimeError('The f-engines RX have a problem.')

            # start f-engine TX
            self.logger.info('Starting f-engine datastream')
            self.fops.tx_enable()

            # check that the F-engines are transmitting data correctly
            if not self.fops.check_tx():
                raise RuntimeError('The f-engines TX have a problem.')

            # check that the X-engines are receiving data
            if not self.xops.check_rx():
                raise RuntimeError('The x-engines RX have a problem.')

            # arm the vaccs on the x-engines
            self.xops.vacc_sync()

        # reset all counters on fhosts and xhosts
        self.fops.clear_status_all()
        self.xops.clear_status_all()

        # set an initialised flag
        self._initialised = True


    def initialised(self):
        """
        Has initialise successfully passed?
        :return:
        """
        return self._initialised

    def est_sync_epoch(self):
        """
        Estimates the synchronisation epoch based on current F-engine timestamp,
        and the system time.
        """
        self.logger.warn("Estimating synchronisation epoch...")
        # get current time from an f-engine
        mcnt_before = self.fhosts[0].get_local_time()
        self.synchronisation_epoch = (time.time() - mcnt_before /
                                      float(self.sample_rate_hz))
        self.logger.info('Current f engine timetamp: %i' % mcnt_before)
        assert mcnt_before & 0xfff == 0, 'Bottom 12 bits of timestamp from ' \
                                         'f-engine are not zero?!'

    def time_from_mcnt(self, mcnt):
        """
        Returns the unix time UTC equivalent to the board timestamp. Does
        NOT account for wrapping timestamps.
        """
        if self.synchronisation_epoch < 0:
            self.est_sync_epoch()
        return (self.synchronisation_epoch + (float(mcnt) /
                                              float(self.sample_rate_hz)))

    def mcnt_from_time(self, time_seconds):
        """
        Returns the board timestamp from a given UTC system time
        (seconds since Unix Epoch). Accounts for wrapping timestamps.
        """
        if self.synchronisation_epoch < 0:
            self.est_sync_epoch()
        time_diff_from_synch_epoch = time_seconds - self.synchronisation_epoch
        time_diff_in_samples = int(time_diff_from_synch_epoch *
                                   self.sample_rate_hz)
        _tmp = 2**int(self.configd['FxCorrelator']['timestamp_bits'])
        return time_diff_in_samples % _tmp

    def qdr_calibrate(self):
        """
        Run a software calibration routine on all the FPGA hosts.
        Do it on F- and X-hosts in parallel.
        :return:
        """
        def _qdr_cal(_fpga):
            _tic = time.time()
            _results = {}
            for _qdr in _fpga.qdrs:
                _results[_qdr.name] = _qdr.qdr_cal(fail_hard=False)
            _toc = time.time()
            return {'results': _results, 'tic': _tic, 'toc': _toc}
        self.logger.info('Calibrating QDR on F- and X-engines, this takes a '
                         'while.')
        qdr_calfail = False
        results = THREADED_FPGA_OP(self.fhosts + self.xhosts, timeout=30,
                                   target_function=(_qdr_cal,))
        for fpga, result in results.items():
            self.logger.info('FPGA %s QDR cal results: start(%.3f) end(%.3f) '
                             'took(%.3f)' % (fpga, result['tic'], result['toc'],
                                             result['toc'] - result['tic']))
            for qdr, qdrres in result['results'].items():
                if not qdrres:
                    qdr_calfail = True
                    break
                self.logger.info('\t%s: cal okay: '
                                 '%s' % (qdr, 'True' if qdrres else 'False'))
            if qdr_calfail:
                break
        if qdr_calfail:
            raise RuntimeError('QDR calibration failure.')
        # for host in self.fhosts:
        #     for qdr in host.qdrs:
        #         qdr.qdr_delay_in_step(0b111111111111111111111111111111111111,
        # -1)
        # for host in self.xhosts:
        #     for qdr in host.qdrs:
        #         qdr.qdr_delay_in_step(0b111111111111111111111111111111111111,
        # -1)

    def set_labels(self, newlist):
        """
        Apply new source labels to the configured fengine sources.
        :param newlist:
        :return:
        """
        if len(newlist) != len(self.fengine_sources):
            errstr = 'Number of supplied source labels (%i) does not match ' \
                     'number of configured sources (%i).' % \
                     (len(newlist), len(self.fengine_sources))
            self.logger.error(errstr)
            raise ValueError(errstr)
        for ctr, source in enumerate(self.fengine_sources):
            _source = source['source']
            # update the source name
            old_name = _source.name
            _source.name = newlist[ctr]
            # update the eq associated with that name
            found_eq = False
            for fhost in self.fhosts:
                if old_name in fhost.eqs.keys():
                    fhost.eqs[_source.name] = fhost.eqs.pop(old_name)
                    found_eq = True
                    break
            if not found_eq:
                raise ValueError('Could not find the old EQ value, %s, to '
                                 'update to new name, %s.' % (old_name,
                                                              _source.name))
        if self.spead_meta_ig is not None:
            metalist = numpy.array(self._spead_meta_get_labelling())
            self.spead_meta_ig['input_labelling'] = metalist
            self.spead_tx.send_heap(self.spead_meta_ig.get_heap())

    def get_labels(self):
        """
        Get the current fengine source labels as a string.
        :return:
        """
        source_names = ''
        for fsrc in self.fengine_sources:
            source_names += fsrc['source'].name + ' '
        source_names = source_names.strip()
        return source_names

    def _read_config(self):
        """
        Read the instrument configuration from self.config_source.
        :return: True if the instrument read a config successfully, raise
        an error if not?
        """
        Instrument._read_config(self)
        _d = self.configd

        # check that the bitstream names are present
        try:
            open(_d['fengine']['bitstream'], 'r').close()
            open(_d['xengine']['bitstream'], 'r').close()
        except IOError:
            self.logger.error('xengine bitstream: '
                              '%s' % _d['xengine']['bitstream'])
            self.logger.error('fengine bitstream: '
                              '%s' % _d['fengine']['bitstream'])
            self.logger.error('One or more bitstream files not found.')
            raise IOError('One or more bitstream files not found.')

        # TODO: Load config values from the bitstream meta information -
        # f per fpga, x per fpga, etc
        self.arp_wait_time = int(_d['FxCorrelator']['arp_wait_time'])
        self.sensor_poll_time = int(_d['FxCorrelator']['sensor_poll_time'])
        self.katcp_port = int(_d['FxCorrelator']['katcp_port'])
        self.f_per_fpga = int(_d['fengine']['f_per_fpga'])
        self.x_per_fpga = int(_d['xengine']['x_per_fpga'])
        self.sample_rate_hz = int(_d['FxCorrelator']['sample_rate_hz'])
        self.adc_demux_factor = int(_d['fengine']['adc_demux_factor'])
        self.accumulation_len = int(_d['xengine']['accumulation_len'])
        self.xeng_accumulation_len = int(_d['xengine']['xeng_accumulation_len'])
        self.n_chans = int(_d['fengine']['n_chans'])
        self.n_antennas = int(_d['fengine']['n_antennas'])
        self.min_load_time = float(_d['fengine']['min_load_time'])

        self.set_stream_destination(
            _d['xengine']['output_destination_ip'],
            int(_d['xengine']['output_destination_port']))
        self.set_meta_destination(_d['xengine']['output_destination_ip'],
                                  int(_d['xengine']['output_destination_port']))
        # get this from the running x-engines?
        self.xeng_clk = int(_d['xengine']['x_fpga_clock'])
        self.xeng_outbits = int(_d['xengine']['xeng_outbits'])

        # the f-engines have this many 10Gbe ports per f-engine
        # unit of operation
        self.ports_per_fengine = int(_d['fengine']['ports_per_fengine'])

        # check if beamformer exists with x-engines
        self.found_beamformer = False
        if 'bengine' in self.configd.keys():
            self.found_beamformer = True

        # set up the hosts and engines based on the configuration in the ini file
        self.fhosts = []
        for host in _d['fengine']['hosts'].split(','):
            host = host.strip()
            fpgahost = fhost_fpga.FpgaFHost.from_config_source(
                host, self.katcp_port, config_source=_d['fengine'])
            self.fhosts.append(fpgahost)

        # choose class (b-engine inherits x-engine functionality)
        if self.found_beamformer:
            _targetClass = bhost_fpga.FpgaBHost
        else:
            _targetClass = xhost_fpga.FpgaXHost
        self.xhosts = []
        for host in _d['xengine']['hosts'].split(','):
            host = host.strip()
            fpgahost = xhost_fpga.FpgaXHost.from_config_source(
                host, self.katcp_port, config_source=_d['xengine'])
            self.xhosts.append(fpgahost)
        self.bhosts = []
        # x-eng host b-eng
        for host in _d['xengine']['hosts'].split(','):
            host = host.strip()
            fpgahost = bhost_fpga.FpgaBHost.from_config_source(
                host, self.katcp_port, config_source=_d['xengine'])
            self.bhosts.append(fpgahost)

        # check that no hosts overlap
        for _fh in self.fhosts:
            for _xh in self.xhosts:
                if _fh.host == _xh.host:
                    self.logger.error('Host %s is assigned to both X- and '
                                      'F-engines' % _fh.host)
                    raise RuntimeError

        # what data sources have we been allocated?
        self._handle_sources()

        # turn the product names into a list
        prodlist = _d['xengine']['output_products'].replace('[', '').replace(']', '').split(',')
        _d['xengine']['output_products'] = []
        for prod in prodlist:
            _d['xengine']['output_products'].append(prod.strip(''))

    def _handle_sources(self):
        """
        Sort out sources and eqs for them
        :return:
        """
        assert len(self.fhosts) > 0
        _feng_cfg = self.configd['fengine']
        source_names = _feng_cfg['source_names'].strip().split(',')
        source_mcast = _feng_cfg['source_mcast_ips'].strip().split(',')
        assert len(source_mcast) == len(source_names), (
            'Source names (%d) must be paired with multicast source '
            'addresses (%d)' % (len(source_names), len(source_mcast)))

        # match eq polys to source names
        eq_polys = {}
        for src_name in source_names:
            eq_polys[src_name] = utils.process_new_eq(
                _feng_cfg['eq_poly_%s' % src_name])

        # assemble the sources given into a list
        _fengine_sources = []
        for source_ctr, address in enumerate(source_mcast):
            new_source = DataSource.from_mcast_string(address)
            new_source.name = source_names[source_ctr]
            assert new_source.ip_range == self.ports_per_fengine, (
                'F-engines should be receiving from %d streams.' %
                self.ports_per_fengine)
            _fengine_sources.append(new_source)

        # assign sources and eqs to fhosts
        self.logger.info('Assigning DataSources, EQs and DelayTrackers to '
                         'f-engines...')
        source_ctr = 0
        self.fengine_sources = []
        for fhost in self.fhosts:
            self.logger.info('\t%s:' % fhost.host)
            _eq_dict = {}
            for fengnum in range(0, self.f_per_fpga):
                _source = _fengine_sources[source_ctr]
                _eq_dict[_source.name] = {'eq': eq_polys[_source.name],
                                          'bram_num': fengnum}
                assert _source.ip_range == _fengine_sources[0].ip_range, (
                    'All f-engines should be receiving from %d streams.' %
                    self.ports_per_fengine)
                self.fengine_sources.append({'source': _source,
                                             'source_num': source_ctr,
                                             'host': fhost,
                                             'numonhost': fengnum})
                fhost.add_source(_source)
                self.logger.info('\t\t%s' % _source)
                source_ctr += 1
            fhost.eqs = _eq_dict
        if source_ctr != len(self.fhosts) * self.f_per_fpga:
            raise RuntimeError('We have different numbers of sources (%d) and '
                               'f-engines (%d). Problem.', source_ctr,
                               len(self.fhosts) * self.f_per_fpga)
        self.logger.info('done.')

    def _read_config_file(self):
        """
        Read the instrument configuration from self.config_source.
        :return: True if we read the file successfully, False if not
        """
        self.configd = utils.parse_ini_file(self.config_source)

    def _read_config_server(self):
        """
        Get instance-specific setup information from a given server. Via KATCP?
        :return:
        """
        raise NotImplementedError('_read_config_server not implemented')

    def set_stream_destination(self, txip_str=None, txport=None):
        """
        Set destination for output of fxcorrelator.
        :param txip_str: A dotted-decimal string representation of the IP address. e.g. '1.2.3.4'
        :param txport: An integer port number.
        :return: <nothing>
        """
        if txip_str is None:
            txip = tengbe.IpAddress.str2ip(self.xeng_tx_destination[0])
        else:
            txip = tengbe.IpAddress.str2ip(txip_str)
        if txport is None:
            txport = self.xeng_tx_destination[1]
        else:
            txport = int(txport)
        self.logger.info('Setting stream destination to %s:%d' %
                         (tengbe.IpAddress.ip2str(txip), txport))
        try:
            THREADED_FPGA_OP(self.xhosts, timeout=10,
                             target_function=(lambda fpga_:
                                              fpga_.registers.gbe_iptx.write(reg=txip),))
            THREADED_FPGA_OP(self.xhosts, timeout=10,
                             target_function=(lambda fpga_:
                                              fpga_.registers.gbe_porttx.write(reg=txport),))
        except AttributeError:
            self.logger.warning('Set SPEAD stream destination called, but '
                                'devices NOT written! Have they been created?')
        self.xeng_tx_destination = (tengbe.IpAddress.ip2str(txip), txport)

    def set_meta_destination(self, txip_str=None, txport=None):
        """
        Set destination for meta info output of fxcorrelator.
        :param txip_str: A dotted-decimal string representation of the IP address. e.g. '1.2.3.4'
        :param txport: An integer port number.
        :return: <nothing>
        """
        if txip_str is None:
            txip_str = self.meta_destination[0]
        if txport is None:
            txport = self.meta_destination[1]
        else:
            txport = int(txport)
        if txport is None or txip_str is None:
            self.logger.error('Cannot set part of meta destination to None - %s:%d' %
                              (txip_str, txport))
            raise RuntimeError('Cannot set part of meta destination to None - %s:%d' %
                               (txip_str, txport))
        self.meta_destination = (txip_str, txport)
        self.logger.info('Setting meta destination to %s:%d' % (txip_str, txport))

    def tx_start(self, issue_spead=True):
        """
        Turns on xengine output pipes needed to start data flow from xengines
        """
        self.logger.info('Starting transmission')
        if issue_spead:
            self.spead_issue_meta()
        # start tx on the x-engines
        for f in self.xhosts:
            f.registers.control.write(gbe_txen=True)

    def tx_stop(self, stop_f=False):
        """
        Turns off output pipes to start data flow from xengines
        :param stop_f: stop output of fengines as well
        """
        self.logger.info('Stopping X transmission')

        THREADED_FPGA_OP(
            self.xhosts, timeout=10, target_function=(
                lambda fpga_: fpga_.registers.control.write(gbe_txen=False),))
        if stop_f:
            self.logger.info('Stopping F transmission')
            THREADED_FPGA_OP(
                self.fhosts, timeout=10, target_function=(
                    lambda fpga_:
                    fpga_.registers.control.write(comms_en=False),))

    def _spead_meta_get_labelling(self):
        """
        Get a list of the source labelling for SPEAD metadata
        :return:
        """
        metalist = []
        for fsrc in self.fengine_sources:
            metalist.append((fsrc['source'].name, fsrc['source_num'],
                             fsrc['host'].host, fsrc['numonhost']))
        return metalist

    def spead_issue_meta(self):
        """
        All FxCorrelators issued SPEAD in the same way, with tweakings that are
        implemented by the child class.
        :return: <nothing>
        """
        if self.meta_destination is None:
            logging.info('SPEAD meta destination is still unset, NOT sending '
                         'metadata at this time.')
            return
        # make a new SPEAD transmitter
        del self.spead_tx, self.spead_meta_ig
        self.spead_tx = spead.Transmitter(spead.TransportUDPtx(*self.meta_destination))
        # update the multicast socket option to use a TTL of 2,
        # in order to traverse the L3 network on site.
        ttl_bin = struct.pack('@i', 2)
        self.spead_tx.t._udp_out.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl_bin)
        #mcast_interface = self.configd['xengine']['multicast_interface_address']
        #self.spead_tx.t._udp_out.setsockopt(socket.SOL_IP, socket.IP_MULTICAST_IF,
        #                                    socket.inet_aton(mcast_interface))
        # self.spead_tx.t._udp_out.setsockopt(socket.SOL_IP, socket.IP_ADD_MEMBERSHIP,
        #                                     socket.inet_aton(txip_str) + socket.inet_aton(mcast_interface))
        # make the item group we're going to use
        self.spead_meta_ig = spead.ItemGroup()

        self.spead_meta_ig.add_item(name='adc_sample_rate', id=0x1007,
                                    description='The expected ADC sample rate (samples '
                                                'per second) of incoming data.',
                                    shape=[], fmt=spead.mkfmt(('u', 64)),
                                    init_val=self.sample_rate_hz)

        self.spead_meta_ig.add_item(name='n_bls', id=0x1008,
                                    description='Number of baselines in the data product.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=len(self.xops.get_baseline_order()))

        self.spead_meta_ig.add_item(name='n_chans', id=0x1009,
                                    description='Number of frequency channels in '
                                                'an integration.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=self.n_chans)

        self.spead_meta_ig.add_item(name='n_ants', id=0x100A,
                                    description='The number of antennas in the system.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=self.n_antennas)

        self.spead_meta_ig.add_item(name='n_xengs', id=0x100B,
                                    description='The number of x-engines in the system.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=(len(self.xhosts) * self.x_per_fpga))

        self.spead_meta_ig.add_item(name='bls_ordering', id=0x100C,
                                    description='The baseline ordering in the output '
                                                'data product.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=numpy.array(
                                        [baseline for baseline in self.xops.get_baseline_order()]))

        # spead_ig.add_item(name='crosspol_ordering', id=0x100D,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        metalist = numpy.array(self._spead_meta_get_labelling())
        self.spead_meta_ig.add_item(name='input_labelling', id=0x100E,
                                    description='input labels and numbers',
                                    init_val=metalist)

        # spead_ig.add_item(name='n_bengs', id=0x100F,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        self.spead_meta_ig.add_item(name='center_freq', id=0x1011,
                                    description='The on-sky centre-frequency.',
                                    shape=[], fmt=spead.mkfmt(('f', 64)),
                                    init_val=int(self.configd['fengine']['true_cf']))

        self.spead_meta_ig.add_item(name='bandwidth', id=0x1013,
                                    description='The input (analogue) bandwidth of '
                                                'the system.',
                                    shape=[], fmt=spead.mkfmt(('f', 64)),
                                    init_val=int(self.configd['fengine']['bandwidth']))

        self.spead_meta_ig.add_item(name='n_accs', id=0x1015,
                                    description='The number of spectra that are '
                                                'accumulated per X-engine dump.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=self.accumulation_len * self.xeng_accumulation_len)

        self.spead_meta_ig.add_item(name='int_time', id=0x1016,
                                    description='The time per integration, in seconds.',
                                    shape=[], fmt=spead.mkfmt(('f', 64)),
                                    init_val=self.xops.get_acc_time())

        # spead_ig.add_item(name='coarse_chans', id=0x1017,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)
        #
        # spead_ig.add_item(name='current_coarse_chan', id=0x1018,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)
        #
        # spead_ig.add_item(name='fft_shift_fine', id=0x101C,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)
        #
        # spead_ig.add_item(name='fft_shift_coarse', id=0x101D,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        self.spead_meta_ig.add_item(name='fft_shift', id=0x101E,
                                    description='The FFT bitshift pattern. F-engine correlator internals.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=int(self.configd['fengine']['fft_shift']))

        self.spead_meta_ig.add_item(name='xeng_acc_len', id=0x101F,
                                    description='Number of spectra accumulated inside X engine. '
                                                'Determines minimum integration time and '
                                                'user-configurable integration time stepsize. '
                                                'X-engine correlator internals.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=self.xeng_accumulation_len)

        quant_bits = int(self.configd['fengine']['quant_format'].split('.')[0])
        self.spead_meta_ig.add_item(name='requant_bits', id=0x1020,
                                    description='Number of bits after requantisation in the '
                                                'F engines (post FFT and any '
                                                'phasing stages).',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=quant_bits)

        pkt_len = int(self.configd['fengine']['10gbe_pkt_len'])
        self.spead_meta_ig.add_item(name='feng_pkt_len', id=0x1021,
                                    description='Payload size of 10GbE packet exchange between '
                                                'F and X engines in 64 bit words. Usually equal '
                                                'to the number of spectra accumulated inside X '
                                                'engine. F-engine correlator internals.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=pkt_len)

        # port = int(self.configd['xengine']['output_destination_port'])
        # spead_ig.add_item(name='rx_udp_port', id=0x1022,
        #                   description='Destination UDP port for X engine output.',
        #                   shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                   init_val=port)

        port = int(self.configd['fengine']['10gbe_port'])
        self.spead_meta_ig.add_item(name='feng_udp_port', id=0x1023,
                                    description='Port for F-engines 10Gbe links in the system.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=port)

        # ip = self.configd['xengine']['output_destination_ip']
        # spead_ig.add_item(name='rx_udp_ip_str', id=0x1024,
        #                   description='Destination UDP IP for X engine output.',
        #                   shape=[-1], fmt=spead.STR_FMT,
        #                   init_val=ip)

        ip = struct.unpack('>I', socket.inet_aton(self.configd['fengine']['10gbe_start_ip']))[0]
        self.spead_meta_ig.add_item(name='feng_start_ip', id=0x1025,
                                    description='Start IP address for F-engines in the system.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=ip)

        self.spead_meta_ig.add_item(name='xeng_rate', id=0x1026,
                                    description='Target clock rate of processing engines (xeng).',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=self.xeng_clk)

        self.spead_meta_ig.add_item(name='sync_time', id=0x1027,
                                    description='The time at which the digitisers were synchronised. '
                                                'Seconds since the Unix Epoch.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=self.synchronisation_epoch)

        # spead_ig.add_item(name='n_stokes', id=0x1040,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        x_per_fpga = int(self.configd['xengine']['x_per_fpga'])
        self.spead_meta_ig.add_item(name='x_per_fpga', id=0x1041,
                                    description='Number of X engines per FPGA host.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=x_per_fpga)

        # n_ants_per_xaui = 1
        # spead_ig.add_item(name='n_ants_per_xaui', id=0x1042,
        #                   description='',
        #                   shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                   init_val=n_ants_per_xaui)

        # spead_ig.add_item(name='ddc_mix_freq', id=0x1043,
        #                        description='',
        #                        shape=[],fmt=spead.mkfmt(('f', 64)),
        #                        init_val=)

        # spead_ig.add_item(name='ddc_bandwidth', id=0x1044,
        #                        description='',
        #                        shape=[],fmt=spead.mkfmt(('f', 64)),
        #                        init_val=)

        sample_bits = int(self.configd['fengine']['sample_bits'])
        self.spead_meta_ig.add_item(name='adc_bits', id=0x1045,
                                    description='How many bits per ADC sample.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=sample_bits)

        self.spead_meta_ig.add_item(name='scale_factor_timestamp', id=0x1046,
                                    description='Timestamp scaling factor. Divide the SPEAD '
                                                'data packet timestamp by this number to get '
                                                'back to seconds since last sync.',
                                    shape=[], fmt=spead.mkfmt(('f', 64)),
                                    init_val=self.sample_rate_hz)
        self.spead_meta_ig.add_item(name='xeng_out_bits_per_sample', id=0x1048,
                                    description='The number of bits per value of the xeng '
                                                'accumulator output. Note this is for a '
                                                'single value, not the combined complex size.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=self.xeng_outbits)

        self.spead_meta_ig.add_item(name='f_per_fpga', id=0x1049,
                                    description='Number of F engines per FPGA host.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
                                    init_val=self.f_per_fpga)
        # spead_ig.add_item(name='rf_gain_MyAntStr ', id=0x1200+inputN,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('f', 64)),
        #                        init_val=)

        # 0x1400 +++
        self.fops.eq_update_metadata()

        # spead_ig.add_item(name='eq_coef_MyAntStr', id=0x1400+inputN,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', 32)),
        #                        init_val=)

        # ndarray = numpy.dtype(numpy.int64), (4096 * 40 * 1, 1, 1)

        self.spead_meta_ig.add_item(name='timestamp', id=0x1600,
                                    description='Timestamp of start of this integration. uint counting multiples '
                                                'of ADC samples since last sync (sync_time, id=0x1027). Divide this '
                                                'number by timestamp_scale (id=0x1046) to get back to seconds since '
                                                'last sync when this integration was actually started.',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)))

        self.spead_meta_ig.add_item(name='flags_xeng_raw', id=0x1601,
                                    description='Flags associated with xeng_raw data output.'
                                                'bit 34 - corruption or data missing during integration '
                                                'bit 33 - overrange in data path '
                                                'bit 32 - noise diode on during integration '
                                                'bits 0 - 31 reserved for internal debugging',
                                    shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)))

        ndarray = numpy.dtype(numpy.int32), (self.n_chans, len(self.xops.get_baseline_order()), 2)
        self.spead_meta_ig.add_item(name='xeng_raw', id=0x1800,
                                    description='Raw data for %i xengines in the system. This item represents a '
                                                'full spectrum (all frequency channels) assembled from lowest '
                                                'frequency to highest frequency. Each frequency channel contains '
                                                'the data for all baselines (n_bls given by SPEAD ID 0x100b). '
                                                'Each value is a complex number -- two (real and imaginary) '
                                                'unsigned integers.' % len(self.xhosts * self.x_per_fpga),
                                    ndarray=ndarray)

        # TODO hard-coded !!!!!!! :(
#        self.spead_ig.add_item(name=("xeng_raw"),id=0x1800,
#                               description="Raw data for %i xengines in the system. This
# item represents a full spectrum (all frequency channels) assembled from lowest frequency
# to highest frequency. Each frequency channel contains the data for all baselines
# (n_bls given by SPEAD ID 0x100B). Each value is a complex number -- two
# (real and imaginary) unsigned integers."%(32),
#                               ndarray=(numpy.dtype(numpy.int32),(4096,((4*(4+1))/2)*4,2)))

        # spead_ig.add_item(name='beamweight_MyAntStr', id=0x2000+inputN,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', 32)),
        #                        init_val=)

        # spead_ig.add_item(name='incoherent_sum', id=0x3000,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', 32)),
        #                        init_val=)

        # spead_ig.add_item(name='n_inputs', id=0x3100,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        # spead_ig.add_item(name='digitiser_id', id=0x3101,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        # spead_ig.add_item(name='digitiser_status', id=0x3102,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        # spead_ig.add_item(name='pld_len', id=0x3103,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        # spead_ig.add_item(name='raw_data_MyAntStr', id=0x3300+inputN,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        # spead_ig.add_item(name='Reserved for SP-CAM meta-data', id=0x7000-0x7fff,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        # spead_ig.add_item(name='feng_id', id=0xf101,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        # spead_ig.add_item(name='feng_status', id=0xf102,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        # spead_ig.add_item(name='frequency', id=0xf103,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        # spead_ig.add_item(name='raw_freq_MyAntStr', id=0xf300+inputN,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        # spead_ig.add_item(name='bf_MyBeamName', id=0xb000+beamN,
        #                        description='',
        #                        shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)),
        #                        init_val=)

        self.spead_tx.send_heap(self.spead_meta_ig.get_heap())
        self.logger.info('Issued SPEAD data descriptor to %s:%i.' % (self.meta_destination[0],
                                                                     self.meta_destination[1]))
Example #2
0
class FxCorrelator(Instrument):
    """
    A generic FxCorrelator composed of fengines that channelise antenna inputs
    and xengines that each produce cross products from a continuous portion
    of the channels and accumulate the result.
    SPEAD data streams are produced.
    """
    def __init__(self, descriptor, identifier=-1, config_source=None,
                 logger=LOGGER):
        """
        An abstract base class for instruments.
        :param descriptor: A text description of the instrument. Required.
        :param identifier: An optional integer identifier.
        :param config_source: The instrument configuration source. Can be a
        text file, hostname, whatever.
        :param logger: Use the module logger by default, unless something else
        is given.
        :return: <nothing>

        """
        self.logger = logger

        # we know about f and x hosts and engines, not just engines and hosts
        self.fhosts = []
        self.xhosts = []
        self.filthosts = None
        self.found_beamformer = False

        self.fops = None
        self.xops = None
        self.bops = None
        self.filtops = None
        self.speadops = None

        # attributes
        self.katcp_port = None
        self.f_per_fpga = None
        self.x_per_fpga = None
        self.accumulation_len = None
        self.xeng_accumulation_len = None
        self.fengine_sources = None
        self.baselines = None

        self.sensor_manager = None

        # parent constructor
        Instrument.__init__(self, descriptor, identifier, config_source, logger)

    def initialise(self, program=True, qdr_cal=True, require_epoch=False):
        """
        Set up the correlator using the information in the config file.
        :param program: program the FPGA boards if True
        :param qdr_cal: perform QDR cal if True
        :param require_epoch: the synch epoch MUST be set before init if True
        :return:
        """
        # check that the instrument's synch epoch has been set
        if require_epoch:
            if self.get_synch_time() == -1:
                raise RuntimeError('System synch epoch has not been set prior'
                                   ' to initialisation!')

        # set up the F, X, B and filter handlers
        self.fops = FEngineOperations(self)
        self.xops = XEngineOperations(self)
        self.bops = BEngineOperations(self)
        self.filtops = FilterOperations(self)
        self.speadops = SpeadOperations(self)

        # set up the filter boards if we need to
        if 'filter' in self.configd:
            try:
                self.filtops.initialise(program=program)
            except Exception as e:
                raise e

        # connect to the other hosts that make up this correlator
        THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=5,
                           target_function='connect')

        igmp_version = self.configd['FxCorrelator'].get('igmp_version')
        if igmp_version is not None:
            self.logger.info('Setting FPGA hosts IGMP version '
                             'to %s' % igmp_version)
            THREADED_FPGA_FUNC(
                self.fhosts + self.xhosts, timeout=5,
                target_function=('set_igmp_version', (igmp_version, ), {}))

        # if we need to program the FPGAs, do so
        if program:
            self.logger.info('Programming FPGA hosts')
            fpgautils.program_fpgas([(host, host.boffile) for host in
                                     (self.fhosts + self.xhosts)],
                                    progfile=None, timeout=15)
        else:
            self.logger.info('Loading design information')
            THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=10,
                               target_function='get_system_information')

        # remove test hardware from designs
        utils.disable_test_gbes(self)
        utils.remove_test_objects(self)

        # run configuration on the parts of the instrument
        self.configure()

        # run post-programming initialisation
        self._initialise(program, qdr_cal)

        # reset all counters on fhosts and xhosts
        self.fops.clear_status_all()
        self.xops.clear_status_all()

        # set an initialised flag
        self._initialised = True

    def _gbe_setup(self):
        """
        Set up the 10gbe ports on the hosts
        :return:
        """
        feng_port = int(self.configd['fengine']['10gbe_port'])
        xeng_port = int(self.configd['xengine']['10gbe_port'])
        info_dict = {host.host: (feng_port, 'fhost') for host in self.fhosts}
        info_dict.update(
            {host.host: (xeng_port, 'xhost') for host in self.xhosts})
        timeout = len(self.fhosts[0].tengbes) * 30 * 1.1
        THREADED_FPGA_FUNC(
                self.fhosts + self.xhosts, timeout=timeout,
                target_function=('setup_host_gbes',
                                 (self.logger, info_dict), {}))

    def _initialise(self, program, qdr_cal):
        """
        Run this if boards in the system have been programmed. Basic setup
        of devices.
        :return:
        """
        if not program:
            # only run the contents of this function after programming.
            return

        # cal the qdr on all boards
        # logging.getLogger('casperfpga.qdr').setLevel(logging.INFO + 7)
        if qdr_cal:
            self.qdr_calibrate()
        else:
            self.logger.info('Skipping QDR cal - are you sure you '
                             'want to do this?')

        # init the engines
        self.fops.initialise_pre_gbe()
        self.xops.initialise_pre_gbe()

        # set up the tengbe ports in parallel
        self._gbe_setup()

        # continue with init
        self.fops.initialise_post_gbe()
        self.xops.initialise_post_gbe()
        if self.found_beamformer:
            self.bops.initialise()

        # subscribe all the engines to the multicast groups
        self.fops.subscribe_to_multicast()
        self.xops.subscribe_to_multicast()

        # start f-engine TX
        self.logger.info('Starting f-engine datastream')
        self.fops.tx_enable()

        # jason's hack to force a reset on the f-engines
        time.sleep(1)
        self.fops.sys_reset()

        # wait for switches to learn, um, stuff
        self.logger.info('post mess-with-the-switch delay of %is' %
                         self.post_switch_delay)
        time.sleep(self.post_switch_delay)

        # jason's hack to force a reset on the f-engines
        self.fops.sys_reset()
        time.sleep(1)

        # reset all counters on fhosts and xhosts
        self.fops.clear_status_all()
        self.xops.clear_status_all()

        # check to see if the f engines are receiving all their data
        if not self.fops.check_rx():
            raise RuntimeError('The f-engines RX have a problem.')

        # check that the timestamps received on the f-engines make sense
        result, times, times_unix = self.fops.check_rx_timestamps()
        if not result:
            raise RuntimeError('The timestamps received by the f-engines '
                               'are not okay. Check the logs')

        # check the f-engine QDR uses for parity errors
        if ((not self.fops.check_ct_parity()) or
                (not self.fops.check_cd_parity())):
            raise RuntimeError('The f-engine QDRs are reporting errors.')

        # check that the F-engines are transmitting data correctly
        if not self.fops.check_tx():
            raise RuntimeError('The f-engines TX have a problem.')

        # check that the X-engines are receiving data
        if not self.xops.check_rx():
            raise RuntimeError('The x-engines RX have a problem.')

        # arm the vaccs on the x-engines
        self.xops.vacc_sync()

    def configure(self):
        """
        Operations to run to configure the instrument, after programming.
        :return:
        """
        self.fops.configure()
        self.xops.configure()
        if self.found_beamformer:
            self.bops.configure()

    def get_scale_factor(self):
        """
        By what number do we divide timestamps to get seconds?
        :return:
        """
        return self.sample_rate_hz

    def set_synch_time(self, new_synch_time):
        """
        Set the last sync time for this system
        :param new_synch_time: UNIX time
        :return: <nothing>
        """
        time_now = time.time()
        # future?
        if new_synch_time > time_now:
            _err = 'Synch time in the future makes no sense? %.3f > %.3f' % (
                new_synch_time, time_now)
            self.logger.error(_err)
            raise RuntimeError(_err)
        # too far in the past?
        if new_synch_time < time_now - (2**self.timestamp_bits):
            _err = 'Synch epoch supplied is too far in the past: now(%.3f) ' \
                   'epoch(%.3f)' % (time_now, new_synch_time)
            self.logger.error(_err)
            raise RuntimeError(_err)
        self._synchronisation_epoch = new_synch_time
        if self.sensor_manager:
            sensor = self.sensor_manager.sensor_get('synch-epoch')
            sensor.set_value(self._synchronisation_epoch)
        self.logger.info('Set synch epoch to %.5f' % new_synch_time)

    def est_synch_epoch(self):
        """
        Estimates the synchronisation epoch based on current F-engine
        timestamp, and the system time.
        """
        self.logger.info('Estimating synchronisation epoch:')
        # get current time from an f-engine
        feng_mcnt = self.fhosts[0].get_local_time()
        self.logger.info('\tcurrent f-engine mcnt: %i' % feng_mcnt)
        if feng_mcnt & 0xfff != 0:
            _err = 'Bottom 12 bits of timestamp from f-engine are not ' \
                   'zero?! feng_mcnt(%i)' % feng_mcnt
            self.logger.error(_err)
            raise RuntimeError(_err)
        t_now = time.time()
        self.set_synch_time(t_now - feng_mcnt / float(self.sample_rate_hz))
        self.logger.info('\tnew epoch: %.3f' % self.get_synch_time())

    def time_from_mcnt(self, mcnt):
        """
        Returns the unix time UTC equivalent to the board timestamp. Does
        NOT account for wrapping timestamps.
        :param mcnt: the time from system boards (ADC ticks since startup)
        """
        if self.get_synch_time() < 0:
            self.logger.info('time_from_mcnt: synch epoch unset, estimating')
            self.est_synch_epoch()
        return self.get_synch_time() + (
            float(mcnt) / float(self.sample_rate_hz))

    def mcnt_from_time(self, time_seconds):
        """
        Returns the board timestamp from a given UTC system time
        (seconds since Unix Epoch). Accounts for wrapping timestamps.
        :param time_seconds: UNIX time
        """
        if self.get_synch_time() < 0:
            self.logger.info('mcnt_from_time: synch epoch unset, estimating')
            self.est_synch_epoch()
        time_diff_from_synch_epoch = time_seconds - self.get_synch_time()
        time_diff_in_samples = int(time_diff_from_synch_epoch *
                                   self.sample_rate_hz)
        _tmp = 2**self.timestamp_bits
        return time_diff_in_samples % _tmp

    def qdr_calibrate(self, timeout=120 * MAX_QDR_ATTEMPTS):
        """
        Run a software calibration routine on all the FPGA hosts.
        Do it on F- and X-hosts in parallel.
        :param timeout: how many seconds to wait for it to complete
        :return:
        """
        def _qdr_cal(_fpga):
            """
            Calibrate the QDRs found on a given FPGA.
            :param _fpga:
            :return:
            """
            _tic = time.time()
            attempts = 0
            _results = {_qdr.name: False for _qdr in _fpga.qdrs}
            while True:
                all_passed = True
                for _qdr in _fpga.qdrs:
                    if not _results[_qdr.name]:
                        try:
                            _res = _qdr.qdr_cal(fail_hard=False)
                        except RuntimeError:
                            _res = False
                        try:
                            _resval = _res[0]
                        except TypeError:
                            _resval = _res
                        if not _resval:
                            all_passed = False
                        _results[_qdr.name] = _resval
                attempts += 1
                if all_passed or (attempts >= MAX_QDR_ATTEMPTS):
                    break
            _toc = time.time()
            return {'results': _results, 'tic': _tic, 'toc': _toc,
                    'attempts': attempts}

        self.logger.info('Calibrating QDR on F- and X-engines, this may '
                         'take a while.')
        qdr_calfail = False
        results = THREADED_FPGA_OP(
            self.fhosts + self.xhosts, timeout, (_qdr_cal,))
        for fpga, result in results.items():
            _time_taken = result['toc'] - result['tic']
            self.logger.info('FPGA %s QDR cal: %.3fs, %i attempts' %
                             (fpga, _time_taken, result['attempts']))
            for qdr, qdrres in result['results'].items():
                if not qdrres:
                    qdr_calfail = True
                    break
                self.logger.info('\t%s: cal okay: %s' %
                                 (qdr, 'True' if qdrres else 'False'))
        if qdr_calfail:
            raise RuntimeError('QDR calibration failure.')
        # for host in self.fhosts:
        #     for qdr in host.qdrs:
        #         qdr.qdr_delay_in_step(0b111111111111111111111111111111111111,
        # -1)
        # for host in self.xhosts:
        #     for qdr in host.qdrs:
        #         qdr.qdr_delay_in_step(0b111111111111111111111111111111111111,
        # -1)

    def set_labels(self, newlist):
        """
        Apply new source labels to the configured fengine sources.
        :param newlist:
        :return:
        """
        old_labels = self.get_labels()
        if len(newlist) != len(old_labels):
            errstr = 'Number of supplied source labels (%i) does not match ' \
                     'number of configured sources (%i).' % \
                     (len(newlist), len(old_labels))
            self.logger.error(errstr)
            raise ValueError(errstr)
        new_dict = {}
        for ctr, source_name in enumerate(old_labels):
            _source = self.fengine_sources[source_name]
            _source.update_name(newlist[ctr])
            new_dict[newlist[ctr]] = _source
        self.fengine_sources = new_dict

        # update the list of baselines
        self.baselines = utils.baselines_from_source_list(newlist)

        # update the hostname and baseline sensors
        if self.sensor_manager:
            sm = self.sensor_manager
            try:
                for fhost in self.fhosts:
                    sensor = sm.sensor_get('%s-input-mapping' % fhost.host)
                    rv = [dsrc.name for dsrc in fhost.data_sources]
                    sensor.set_value(str(rv))
            except Exception as ve:
                self.logger.warning('Could not update input_mapping '
                                    'sensors!\n%s' % ve.message)
            sm.sensor_get('baseline-ordering').set_value(str(self.baselines))

        # update the beam input labels
        if self.found_beamformer:
            self.bops.update_labels(old_labels, self.get_labels())
        self.speadops.update_metadata([0x100e])

        self.logger.info('Source labels updated from %s to %s' % (
            old_labels, self.get_labels()))

    def get_labels(self):
        """
        Get the current fengine source labels as a list of label names.
        :return:
        """
        return sorted(self.fengine_sources,
                      key=lambda k: self.fengine_sources[k].source_number)

    def _read_config(self):
        """
        Read the instrument configuration from self.config_source.
        :return:
        """
        Instrument._read_config(self)
        _d = self.configd

        # check that the bitstream names are present
        try:
            open(_d['fengine']['bitstream'], 'r').close()
            open(_d['xengine']['bitstream'], 'r').close()
        except IOError:
            self.logger.error('xengine bitstream: '
                              '%s' % _d['xengine']['bitstream'])
            self.logger.error('fengine bitstream: '
                              '%s' % _d['fengine']['bitstream'])
            self.logger.error('One or more bitstream files not found.')
            raise IOError('One or more bitstream files not found.')

        # TODO: Load config values from the bitstream meta information -
        # f per fpga, x per fpga, etc
        _fxcorr_d = self.configd['FxCorrelator']
        self.arp_wait_time = int(_fxcorr_d['arp_wait_time'])
        self.sensor_poll_time = int(_fxcorr_d['sensor_poll_time'])
        self.katcp_port = int(_fxcorr_d['katcp_port'])
        self.sample_rate_hz = int(_fxcorr_d['sample_rate_hz'])
        self.timestamp_bits = int(_fxcorr_d['timestamp_bits'])
        self.time_jitter_allowed_ms = int(_fxcorr_d['time_jitter_allowed_ms'])
        self.time_offset_allowed_s = int(_fxcorr_d['time_offset_allowed_s'])
        try:
            self.post_switch_delay = int(_fxcorr_d['switch_delay'])
        except KeyError:
            self.post_switch_delay = 10

        _feng_d = self.configd['fengine']
        self.adc_demux_factor = int(_feng_d['adc_demux_factor'])
        self.n_chans = int(_feng_d['n_chans'])
        self.n_antennas = int(_feng_d['n_antennas'])
        self.min_load_time = float(_feng_d['min_load_time'])
        self.f_per_fpga = int(_feng_d['f_per_fpga'])
        self.ports_per_fengine = int(_feng_d['ports_per_fengine'])
        self.analogue_bandwidth = int(_feng_d['bandwidth'])
        self.true_cf = float(_feng_d['true_cf'])
        self.quant_format = _feng_d['quant_format']
        self.adc_bitwidth = int(_feng_d['sample_bits'])
        self.fft_shift = int(_feng_d['fft_shift'])

        try:
            self.qdr_ct_error_threshold = int(_feng_d['qdr_ct_error_threshold'])
        except KeyError:
            self.qdr_ct_error_threshold = 100
        try:
            self.qdr_cd_error_threshold = int(_feng_d['qdr_cd_error_threshold'])
        except KeyError:
            self.qdr_cd_error_threshold = 100

        _xeng_d = self.configd['xengine']
        self.x_per_fpga = int(_xeng_d['x_per_fpga'])
        self.accumulation_len = int(_xeng_d['accumulation_len'])
        self.xeng_accumulation_len = int(_xeng_d['xeng_accumulation_len'])
        try:
            self.qdr_vacc_error_threshold = int(
                _xeng_d['qdr_vacc_error_threshold'])
        except KeyError:
            self.qdr_vacc_error_threshold = 100

        # get this from the running x-engines?
        self.xeng_clk = int(_xeng_d['x_fpga_clock'])
        self.xeng_outbits = int(_xeng_d['xeng_outbits'])

        # check if beamformer exists with x-engines
        self.found_beamformer = False
        if 'bengine' in self.configd.keys():
            self.found_beamformer = True
            self.beng_outbits = 8

        # set up hosts and engines based on the configuration in the ini file
        _targetClass = fhost_fpga.FpgaFHost
        self.fhosts = []
        for host in _feng_d['hosts'].split(','):
            host = host.strip()
            fpgahost = _targetClass.from_config_source(host, self.katcp_port,
                                                       config_source=_feng_d)
            self.fhosts.append(fpgahost)

        # choose class (b-engine inherits x-engine functionality)
        if self.found_beamformer:
            _targetClass = bhost_fpga.FpgaBHost
        else:
            _targetClass = xhost_fpga.FpgaXHost
        self.xhosts = []
        hostlist = _xeng_d['hosts'].split(',')
        for hostindex, host in enumerate(hostlist):
            host = host.strip()
            fpgahost = _targetClass.from_config_source(
                host, hostindex, self.katcp_port, config_source=_xeng_d)
            self.xhosts.append(fpgahost)

        # check that no hosts overlap
        for _fh in self.fhosts:
            for _xh in self.xhosts:
                if _fh.host == _xh.host:
                    self.logger.error('Host %s is assigned to '
                                      'both X- and F-engines' % _fh.host)
                    raise RuntimeError

        # update the list of baselines on this system
        self.baselines = utils.baselines_from_config(config=self.configd)

        # what data sources have we been allocated?
        self._handle_sources()

        # turn the stream names into a list
        prodlist = _xeng_d['output_products'].replace('[', '')
        prodlist = prodlist.replace(']', '').split(',')
        _xeng_d['output_products'] = [prod.strip(' ') for prod in prodlist]

    def _handle_sources(self):
        """
        Sort out sources and eqs for them
        :return:
        """
        assert len(self.fhosts) > 0
        _feng_cfg = self.configd['fengine']
        source_names = utils.sources_from_config(config=self.configd)
        source_mcast = _feng_cfg['source_mcast_ips'].strip().split(',')
        assert len(source_mcast) == len(source_names), (
            'Source names (%d) must be paired with multicast source '
            'addresses (%d)' % (len(source_names), len(source_mcast)))

        # match eq polys to source names
        eq_polys = {}
        for src_name in source_names:
            eq_polys[src_name] = utils.process_new_eq(
                _feng_cfg['eq_poly_%s' % src_name])

        # assemble the sources given into a list
        _feng_src_temp = []
        for source_ctr, address in enumerate(source_mcast):
            new_source = FengineSource.from_mcast_string(address)
            new_source.name = source_names[source_ctr]
            new_source.source_number = source_ctr
            new_source.offset = source_ctr % self.f_per_fpga
            new_source.eq_poly = eq_polys[new_source.name]
            new_source.eq_bram_name = 'eq%i' % new_source.offset
            assert new_source.ip_range == self.ports_per_fengine, (
                'F-engines should be receiving from %d streams.' %
                self.ports_per_fengine)
            _feng_src_temp.append(new_source)

        # check that the sources all have the same IP ranges
        for _source in _feng_src_temp:
            assert _source.ip_range == _feng_src_temp[0].ip_range, (
                'All f-engines should be receiving from %d streams.' %
                self.ports_per_fengine)

        # assign sources to fhosts
        self.logger.info('Assigning FengineSources to f-hosts')
        _src_ctr = 0
        self.fengine_sources = {}
        for fhost in self.fhosts:
            self.logger.info('\t%s:' % fhost.host)
            for fengnum in range(0, self.f_per_fpga):
                _src = _feng_src_temp[_src_ctr]
                _src.host = fhost
                self.fengine_sources[_src.name] = _src
                fhost.add_source(_src)
                self.logger.info('\t\t%s' % _src)
                _src_ctr += 1
        if _src_ctr != len(self.fhosts) * self.f_per_fpga:
            raise RuntimeError('We have different numbers of sources (%d) and '
                               'f-engines (%d). Problem.', _src_ctr,
                               len(self.fhosts) * self.f_per_fpga)
        self.logger.info('done.')

    def _read_config_file(self):
        """
        Read the instrument configuration from self.config_source.
        :return: True if we read the file successfully, False if not
        """
        self.configd = utils.parse_ini_file(self.config_source)

    def _read_config_server(self):
        """
        Get instance-specific setup information from a given server. Via KATCP?
        :return:
        """
        raise NotImplementedError('_read_config_server not implemented')

    def stream_set_destination(self, stream_name, txip_str=None, txport=None):
        """
        Set the destination for a data stream.
        :param stream_name:
        :param txip_str: A dotted-decimal string representation of the
        IP address. e.g. '1.2.3.4'
        :param txport: An integer port number.
        :return: <nothing>
        """
        stream = self.get_data_stream(stream_name)
        stream.set_destination(txip_str, txport)
        stream.set_meta_destination(txip_str, txport)
        if self.sensor_manager:
            sensor_name = '%s-destination' % stream.name
            sensor = self.sensor_manager.sensor_get(sensor_name)
            sensor.set_value(str(stream.destination))
            sensor_name = '%s-meta-destination' % stream.name
            sensor = self.sensor_manager.sensor_get(sensor_name)
            sensor.set_value(str(stream.meta_destination))