Example #1
0
    def initialise(self, program=True, qdr_cal=True, require_epoch=False):
        """
        Set up the correlator using the information in the config file.
        :param program: program the FPGA boards if True
        :param qdr_cal: perform QDR cal if True
        :param require_epoch: the synch epoch MUST be set before init if True
        :return:
        """
        # check that the instrument's synch epoch has been set
        if require_epoch:
            if self.get_synch_time() == -1:
                raise RuntimeError('System synch epoch has not been set prior'
                                   ' to initialisation!')

        # set up the F, X, B and filter handlers
        self.fops = FEngineOperations(self)
        self.xops = XEngineOperations(self)
        self.bops = BEngineOperations(self)
        self.filtops = FilterOperations(self)
        self.speadops = SpeadOperations(self)

        # set up the filter boards if we need to
        if 'filter' in self.configd:
            try:
                self.filtops.initialise(program=program)
            except Exception as e:
                raise e

        # connect to the other hosts that make up this correlator
        THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=5,
                           target_function='connect')

        igmp_version = self.configd['FxCorrelator'].get('igmp_version')
        if igmp_version is not None:
            self.logger.info('Setting FPGA hosts IGMP version '
                             'to %s' % igmp_version)
            THREADED_FPGA_FUNC(
                self.fhosts + self.xhosts, timeout=5,
                target_function=('set_igmp_version', (igmp_version, ), {}))

        # if we need to program the FPGAs, do so
        if program:
            self.logger.info('Programming FPGA hosts')
            fpgautils.program_fpgas([(host, host.boffile) for host in
                                     (self.fhosts + self.xhosts)],
                                    progfile=None, timeout=15)
        else:
            self.logger.info('Loading design information')
            THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=10,
                               target_function='get_system_information')

        # remove test hardware from designs
        utils.disable_test_gbes(self)
        utils.remove_test_objects(self)

        # run configuration on the parts of the instrument
        self.configure()

        # run post-programming initialisation
        self._initialise(program, qdr_cal)

        # reset all counters on fhosts and xhosts
        self.fops.clear_status_all()
        self.xops.clear_status_all()

        # set an initialised flag
        self._initialised = True
Example #2
0
class FxCorrelator(Instrument):
    """
    A generic FxCorrelator composed of fengines that channelise antenna inputs
    and xengines that each produce cross products from a continuous portion
    of the channels and accumulate the result.
    SPEAD data streams are produced.
    """
    def __init__(self, descriptor, identifier=-1, config_source=None,
                 logger=LOGGER):
        """
        An abstract base class for instruments.
        :param descriptor: A text description of the instrument. Required.
        :param identifier: An optional integer identifier.
        :param config_source: The instrument configuration source. Can be a
        text file, hostname, whatever.
        :param logger: Use the module logger by default, unless something else
        is given.
        :return: <nothing>

        """
        self.logger = logger

        # we know about f and x hosts and engines, not just engines and hosts
        self.fhosts = []
        self.xhosts = []
        self.filthosts = None
        self.found_beamformer = False

        self.fops = None
        self.xops = None
        self.bops = None
        self.filtops = None
        self.speadops = None

        # attributes
        self.katcp_port = None
        self.f_per_fpga = None
        self.x_per_fpga = None
        self.accumulation_len = None
        self.xeng_accumulation_len = None
        self.fengine_sources = None
        self.baselines = None

        self.sensor_manager = None

        # parent constructor
        Instrument.__init__(self, descriptor, identifier, config_source, logger)

    def initialise(self, program=True, qdr_cal=True, require_epoch=False):
        """
        Set up the correlator using the information in the config file.
        :param program: program the FPGA boards if True
        :param qdr_cal: perform QDR cal if True
        :param require_epoch: the synch epoch MUST be set before init if True
        :return:
        """
        # check that the instrument's synch epoch has been set
        if require_epoch:
            if self.get_synch_time() == -1:
                raise RuntimeError('System synch epoch has not been set prior'
                                   ' to initialisation!')

        # set up the F, X, B and filter handlers
        self.fops = FEngineOperations(self)
        self.xops = XEngineOperations(self)
        self.bops = BEngineOperations(self)
        self.filtops = FilterOperations(self)
        self.speadops = SpeadOperations(self)

        # set up the filter boards if we need to
        if 'filter' in self.configd:
            try:
                self.filtops.initialise(program=program)
            except Exception as e:
                raise e

        # connect to the other hosts that make up this correlator
        THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=5,
                           target_function='connect')

        igmp_version = self.configd['FxCorrelator'].get('igmp_version')
        if igmp_version is not None:
            self.logger.info('Setting FPGA hosts IGMP version '
                             'to %s' % igmp_version)
            THREADED_FPGA_FUNC(
                self.fhosts + self.xhosts, timeout=5,
                target_function=('set_igmp_version', (igmp_version, ), {}))

        # if we need to program the FPGAs, do so
        if program:
            self.logger.info('Programming FPGA hosts')
            fpgautils.program_fpgas([(host, host.boffile) for host in
                                     (self.fhosts + self.xhosts)],
                                    progfile=None, timeout=15)
        else:
            self.logger.info('Loading design information')
            THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=10,
                               target_function='get_system_information')

        # remove test hardware from designs
        utils.disable_test_gbes(self)
        utils.remove_test_objects(self)

        # run configuration on the parts of the instrument
        self.configure()

        # run post-programming initialisation
        self._initialise(program, qdr_cal)

        # reset all counters on fhosts and xhosts
        self.fops.clear_status_all()
        self.xops.clear_status_all()

        # set an initialised flag
        self._initialised = True

    def _gbe_setup(self):
        """
        Set up the 10gbe ports on the hosts
        :return:
        """
        feng_port = int(self.configd['fengine']['10gbe_port'])
        xeng_port = int(self.configd['xengine']['10gbe_port'])
        info_dict = {host.host: (feng_port, 'fhost') for host in self.fhosts}
        info_dict.update(
            {host.host: (xeng_port, 'xhost') for host in self.xhosts})
        timeout = len(self.fhosts[0].tengbes) * 30 * 1.1
        THREADED_FPGA_FUNC(
                self.fhosts + self.xhosts, timeout=timeout,
                target_function=('setup_host_gbes',
                                 (self.logger, info_dict), {}))

    def _initialise(self, program, qdr_cal):
        """
        Run this if boards in the system have been programmed. Basic setup
        of devices.
        :return:
        """
        if not program:
            # only run the contents of this function after programming.
            return

        # cal the qdr on all boards
        # logging.getLogger('casperfpga.qdr').setLevel(logging.INFO + 7)
        if qdr_cal:
            self.qdr_calibrate()
        else:
            self.logger.info('Skipping QDR cal - are you sure you '
                             'want to do this?')

        # init the engines
        self.fops.initialise_pre_gbe()
        self.xops.initialise_pre_gbe()

        # set up the tengbe ports in parallel
        self._gbe_setup()

        # continue with init
        self.fops.initialise_post_gbe()
        self.xops.initialise_post_gbe()
        if self.found_beamformer:
            self.bops.initialise()

        # subscribe all the engines to the multicast groups
        self.fops.subscribe_to_multicast()
        self.xops.subscribe_to_multicast()

        # start f-engine TX
        self.logger.info('Starting f-engine datastream')
        self.fops.tx_enable()

        # jason's hack to force a reset on the f-engines
        time.sleep(1)
        self.fops.sys_reset()

        # wait for switches to learn, um, stuff
        self.logger.info('post mess-with-the-switch delay of %is' %
                         self.post_switch_delay)
        time.sleep(self.post_switch_delay)

        # jason's hack to force a reset on the f-engines
        self.fops.sys_reset()
        time.sleep(1)

        # reset all counters on fhosts and xhosts
        self.fops.clear_status_all()
        self.xops.clear_status_all()

        # check to see if the f engines are receiving all their data
        if not self.fops.check_rx():
            raise RuntimeError('The f-engines RX have a problem.')

        # check that the timestamps received on the f-engines make sense
        result, times, times_unix = self.fops.check_rx_timestamps()
        if not result:
            raise RuntimeError('The timestamps received by the f-engines '
                               'are not okay. Check the logs')

        # check the f-engine QDR uses for parity errors
        if ((not self.fops.check_ct_parity()) or
                (not self.fops.check_cd_parity())):
            raise RuntimeError('The f-engine QDRs are reporting errors.')

        # check that the F-engines are transmitting data correctly
        if not self.fops.check_tx():
            raise RuntimeError('The f-engines TX have a problem.')

        # check that the X-engines are receiving data
        if not self.xops.check_rx():
            raise RuntimeError('The x-engines RX have a problem.')

        # arm the vaccs on the x-engines
        self.xops.vacc_sync()

    def configure(self):
        """
        Operations to run to configure the instrument, after programming.
        :return:
        """
        self.fops.configure()
        self.xops.configure()
        if self.found_beamformer:
            self.bops.configure()

    def get_scale_factor(self):
        """
        By what number do we divide timestamps to get seconds?
        :return:
        """
        return self.sample_rate_hz

    def set_synch_time(self, new_synch_time):
        """
        Set the last sync time for this system
        :param new_synch_time: UNIX time
        :return: <nothing>
        """
        time_now = time.time()
        # future?
        if new_synch_time > time_now:
            _err = 'Synch time in the future makes no sense? %.3f > %.3f' % (
                new_synch_time, time_now)
            self.logger.error(_err)
            raise RuntimeError(_err)
        # too far in the past?
        if new_synch_time < time_now - (2**self.timestamp_bits):
            _err = 'Synch epoch supplied is too far in the past: now(%.3f) ' \
                   'epoch(%.3f)' % (time_now, new_synch_time)
            self.logger.error(_err)
            raise RuntimeError(_err)
        self._synchronisation_epoch = new_synch_time
        if self.sensor_manager:
            sensor = self.sensor_manager.sensor_get('synch-epoch')
            sensor.set_value(self._synchronisation_epoch)
        self.logger.info('Set synch epoch to %.5f' % new_synch_time)

    def est_synch_epoch(self):
        """
        Estimates the synchronisation epoch based on current F-engine
        timestamp, and the system time.
        """
        self.logger.info('Estimating synchronisation epoch:')
        # get current time from an f-engine
        feng_mcnt = self.fhosts[0].get_local_time()
        self.logger.info('\tcurrent f-engine mcnt: %i' % feng_mcnt)
        if feng_mcnt & 0xfff != 0:
            _err = 'Bottom 12 bits of timestamp from f-engine are not ' \
                   'zero?! feng_mcnt(%i)' % feng_mcnt
            self.logger.error(_err)
            raise RuntimeError(_err)
        t_now = time.time()
        self.set_synch_time(t_now - feng_mcnt / float(self.sample_rate_hz))
        self.logger.info('\tnew epoch: %.3f' % self.get_synch_time())

    def time_from_mcnt(self, mcnt):
        """
        Returns the unix time UTC equivalent to the board timestamp. Does
        NOT account for wrapping timestamps.
        :param mcnt: the time from system boards (ADC ticks since startup)
        """
        if self.get_synch_time() < 0:
            self.logger.info('time_from_mcnt: synch epoch unset, estimating')
            self.est_synch_epoch()
        return self.get_synch_time() + (
            float(mcnt) / float(self.sample_rate_hz))

    def mcnt_from_time(self, time_seconds):
        """
        Returns the board timestamp from a given UTC system time
        (seconds since Unix Epoch). Accounts for wrapping timestamps.
        :param time_seconds: UNIX time
        """
        if self.get_synch_time() < 0:
            self.logger.info('mcnt_from_time: synch epoch unset, estimating')
            self.est_synch_epoch()
        time_diff_from_synch_epoch = time_seconds - self.get_synch_time()
        time_diff_in_samples = int(time_diff_from_synch_epoch *
                                   self.sample_rate_hz)
        _tmp = 2**self.timestamp_bits
        return time_diff_in_samples % _tmp

    def qdr_calibrate(self, timeout=120 * MAX_QDR_ATTEMPTS):
        """
        Run a software calibration routine on all the FPGA hosts.
        Do it on F- and X-hosts in parallel.
        :param timeout: how many seconds to wait for it to complete
        :return:
        """
        def _qdr_cal(_fpga):
            """
            Calibrate the QDRs found on a given FPGA.
            :param _fpga:
            :return:
            """
            _tic = time.time()
            attempts = 0
            _results = {_qdr.name: False for _qdr in _fpga.qdrs}
            while True:
                all_passed = True
                for _qdr in _fpga.qdrs:
                    if not _results[_qdr.name]:
                        try:
                            _res = _qdr.qdr_cal(fail_hard=False)
                        except RuntimeError:
                            _res = False
                        try:
                            _resval = _res[0]
                        except TypeError:
                            _resval = _res
                        if not _resval:
                            all_passed = False
                        _results[_qdr.name] = _resval
                attempts += 1
                if all_passed or (attempts >= MAX_QDR_ATTEMPTS):
                    break
            _toc = time.time()
            return {'results': _results, 'tic': _tic, 'toc': _toc,
                    'attempts': attempts}

        self.logger.info('Calibrating QDR on F- and X-engines, this may '
                         'take a while.')
        qdr_calfail = False
        results = THREADED_FPGA_OP(
            self.fhosts + self.xhosts, timeout, (_qdr_cal,))
        for fpga, result in results.items():
            _time_taken = result['toc'] - result['tic']
            self.logger.info('FPGA %s QDR cal: %.3fs, %i attempts' %
                             (fpga, _time_taken, result['attempts']))
            for qdr, qdrres in result['results'].items():
                if not qdrres:
                    qdr_calfail = True
                    break
                self.logger.info('\t%s: cal okay: %s' %
                                 (qdr, 'True' if qdrres else 'False'))
        if qdr_calfail:
            raise RuntimeError('QDR calibration failure.')
        # for host in self.fhosts:
        #     for qdr in host.qdrs:
        #         qdr.qdr_delay_in_step(0b111111111111111111111111111111111111,
        # -1)
        # for host in self.xhosts:
        #     for qdr in host.qdrs:
        #         qdr.qdr_delay_in_step(0b111111111111111111111111111111111111,
        # -1)

    def set_labels(self, newlist):
        """
        Apply new source labels to the configured fengine sources.
        :param newlist:
        :return:
        """
        old_labels = self.get_labels()
        if len(newlist) != len(old_labels):
            errstr = 'Number of supplied source labels (%i) does not match ' \
                     'number of configured sources (%i).' % \
                     (len(newlist), len(old_labels))
            self.logger.error(errstr)
            raise ValueError(errstr)
        new_dict = {}
        for ctr, source_name in enumerate(old_labels):
            _source = self.fengine_sources[source_name]
            _source.update_name(newlist[ctr])
            new_dict[newlist[ctr]] = _source
        self.fengine_sources = new_dict

        # update the list of baselines
        self.baselines = utils.baselines_from_source_list(newlist)

        # update the hostname and baseline sensors
        if self.sensor_manager:
            sm = self.sensor_manager
            try:
                for fhost in self.fhosts:
                    sensor = sm.sensor_get('%s-input-mapping' % fhost.host)
                    rv = [dsrc.name for dsrc in fhost.data_sources]
                    sensor.set_value(str(rv))
            except Exception as ve:
                self.logger.warning('Could not update input_mapping '
                                    'sensors!\n%s' % ve.message)
            sm.sensor_get('baseline-ordering').set_value(str(self.baselines))

        # update the beam input labels
        if self.found_beamformer:
            self.bops.update_labels(old_labels, self.get_labels())
        self.speadops.update_metadata([0x100e])

        self.logger.info('Source labels updated from %s to %s' % (
            old_labels, self.get_labels()))

    def get_labels(self):
        """
        Get the current fengine source labels as a list of label names.
        :return:
        """
        return sorted(self.fengine_sources,
                      key=lambda k: self.fengine_sources[k].source_number)

    def _read_config(self):
        """
        Read the instrument configuration from self.config_source.
        :return:
        """
        Instrument._read_config(self)
        _d = self.configd

        # check that the bitstream names are present
        try:
            open(_d['fengine']['bitstream'], 'r').close()
            open(_d['xengine']['bitstream'], 'r').close()
        except IOError:
            self.logger.error('xengine bitstream: '
                              '%s' % _d['xengine']['bitstream'])
            self.logger.error('fengine bitstream: '
                              '%s' % _d['fengine']['bitstream'])
            self.logger.error('One or more bitstream files not found.')
            raise IOError('One or more bitstream files not found.')

        # TODO: Load config values from the bitstream meta information -
        # f per fpga, x per fpga, etc
        _fxcorr_d = self.configd['FxCorrelator']
        self.arp_wait_time = int(_fxcorr_d['arp_wait_time'])
        self.sensor_poll_time = int(_fxcorr_d['sensor_poll_time'])
        self.katcp_port = int(_fxcorr_d['katcp_port'])
        self.sample_rate_hz = int(_fxcorr_d['sample_rate_hz'])
        self.timestamp_bits = int(_fxcorr_d['timestamp_bits'])
        self.time_jitter_allowed_ms = int(_fxcorr_d['time_jitter_allowed_ms'])
        self.time_offset_allowed_s = int(_fxcorr_d['time_offset_allowed_s'])
        try:
            self.post_switch_delay = int(_fxcorr_d['switch_delay'])
        except KeyError:
            self.post_switch_delay = 10

        _feng_d = self.configd['fengine']
        self.adc_demux_factor = int(_feng_d['adc_demux_factor'])
        self.n_chans = int(_feng_d['n_chans'])
        self.n_antennas = int(_feng_d['n_antennas'])
        self.min_load_time = float(_feng_d['min_load_time'])
        self.f_per_fpga = int(_feng_d['f_per_fpga'])
        self.ports_per_fengine = int(_feng_d['ports_per_fengine'])
        self.analogue_bandwidth = int(_feng_d['bandwidth'])
        self.true_cf = float(_feng_d['true_cf'])
        self.quant_format = _feng_d['quant_format']
        self.adc_bitwidth = int(_feng_d['sample_bits'])
        self.fft_shift = int(_feng_d['fft_shift'])

        try:
            self.qdr_ct_error_threshold = int(_feng_d['qdr_ct_error_threshold'])
        except KeyError:
            self.qdr_ct_error_threshold = 100
        try:
            self.qdr_cd_error_threshold = int(_feng_d['qdr_cd_error_threshold'])
        except KeyError:
            self.qdr_cd_error_threshold = 100

        _xeng_d = self.configd['xengine']
        self.x_per_fpga = int(_xeng_d['x_per_fpga'])
        self.accumulation_len = int(_xeng_d['accumulation_len'])
        self.xeng_accumulation_len = int(_xeng_d['xeng_accumulation_len'])
        try:
            self.qdr_vacc_error_threshold = int(
                _xeng_d['qdr_vacc_error_threshold'])
        except KeyError:
            self.qdr_vacc_error_threshold = 100

        # get this from the running x-engines?
        self.xeng_clk = int(_xeng_d['x_fpga_clock'])
        self.xeng_outbits = int(_xeng_d['xeng_outbits'])

        # check if beamformer exists with x-engines
        self.found_beamformer = False
        if 'bengine' in self.configd.keys():
            self.found_beamformer = True
            self.beng_outbits = 8

        # set up hosts and engines based on the configuration in the ini file
        _targetClass = fhost_fpga.FpgaFHost
        self.fhosts = []
        for host in _feng_d['hosts'].split(','):
            host = host.strip()
            fpgahost = _targetClass.from_config_source(host, self.katcp_port,
                                                       config_source=_feng_d)
            self.fhosts.append(fpgahost)

        # choose class (b-engine inherits x-engine functionality)
        if self.found_beamformer:
            _targetClass = bhost_fpga.FpgaBHost
        else:
            _targetClass = xhost_fpga.FpgaXHost
        self.xhosts = []
        hostlist = _xeng_d['hosts'].split(',')
        for hostindex, host in enumerate(hostlist):
            host = host.strip()
            fpgahost = _targetClass.from_config_source(
                host, hostindex, self.katcp_port, config_source=_xeng_d)
            self.xhosts.append(fpgahost)

        # check that no hosts overlap
        for _fh in self.fhosts:
            for _xh in self.xhosts:
                if _fh.host == _xh.host:
                    self.logger.error('Host %s is assigned to '
                                      'both X- and F-engines' % _fh.host)
                    raise RuntimeError

        # update the list of baselines on this system
        self.baselines = utils.baselines_from_config(config=self.configd)

        # what data sources have we been allocated?
        self._handle_sources()

        # turn the stream names into a list
        prodlist = _xeng_d['output_products'].replace('[', '')
        prodlist = prodlist.replace(']', '').split(',')
        _xeng_d['output_products'] = [prod.strip(' ') for prod in prodlist]

    def _handle_sources(self):
        """
        Sort out sources and eqs for them
        :return:
        """
        assert len(self.fhosts) > 0
        _feng_cfg = self.configd['fengine']
        source_names = utils.sources_from_config(config=self.configd)
        source_mcast = _feng_cfg['source_mcast_ips'].strip().split(',')
        assert len(source_mcast) == len(source_names), (
            'Source names (%d) must be paired with multicast source '
            'addresses (%d)' % (len(source_names), len(source_mcast)))

        # match eq polys to source names
        eq_polys = {}
        for src_name in source_names:
            eq_polys[src_name] = utils.process_new_eq(
                _feng_cfg['eq_poly_%s' % src_name])

        # assemble the sources given into a list
        _feng_src_temp = []
        for source_ctr, address in enumerate(source_mcast):
            new_source = FengineSource.from_mcast_string(address)
            new_source.name = source_names[source_ctr]
            new_source.source_number = source_ctr
            new_source.offset = source_ctr % self.f_per_fpga
            new_source.eq_poly = eq_polys[new_source.name]
            new_source.eq_bram_name = 'eq%i' % new_source.offset
            assert new_source.ip_range == self.ports_per_fengine, (
                'F-engines should be receiving from %d streams.' %
                self.ports_per_fengine)
            _feng_src_temp.append(new_source)

        # check that the sources all have the same IP ranges
        for _source in _feng_src_temp:
            assert _source.ip_range == _feng_src_temp[0].ip_range, (
                'All f-engines should be receiving from %d streams.' %
                self.ports_per_fengine)

        # assign sources to fhosts
        self.logger.info('Assigning FengineSources to f-hosts')
        _src_ctr = 0
        self.fengine_sources = {}
        for fhost in self.fhosts:
            self.logger.info('\t%s:' % fhost.host)
            for fengnum in range(0, self.f_per_fpga):
                _src = _feng_src_temp[_src_ctr]
                _src.host = fhost
                self.fengine_sources[_src.name] = _src
                fhost.add_source(_src)
                self.logger.info('\t\t%s' % _src)
                _src_ctr += 1
        if _src_ctr != len(self.fhosts) * self.f_per_fpga:
            raise RuntimeError('We have different numbers of sources (%d) and '
                               'f-engines (%d). Problem.', _src_ctr,
                               len(self.fhosts) * self.f_per_fpga)
        self.logger.info('done.')

    def _read_config_file(self):
        """
        Read the instrument configuration from self.config_source.
        :return: True if we read the file successfully, False if not
        """
        self.configd = utils.parse_ini_file(self.config_source)

    def _read_config_server(self):
        """
        Get instance-specific setup information from a given server. Via KATCP?
        :return:
        """
        raise NotImplementedError('_read_config_server not implemented')

    def stream_set_destination(self, stream_name, txip_str=None, txport=None):
        """
        Set the destination for a data stream.
        :param stream_name:
        :param txip_str: A dotted-decimal string representation of the
        IP address. e.g. '1.2.3.4'
        :param txport: An integer port number.
        :return: <nothing>
        """
        stream = self.get_data_stream(stream_name)
        stream.set_destination(txip_str, txport)
        stream.set_meta_destination(txip_str, txport)
        if self.sensor_manager:
            sensor_name = '%s-destination' % stream.name
            sensor = self.sensor_manager.sensor_get(sensor_name)
            sensor.set_value(str(stream.destination))
            sensor_name = '%s-meta-destination' % stream.name
            sensor = self.sensor_manager.sensor_get(sensor_name)
            sensor.set_value(str(stream.meta_destination))