class FxCorrelator(Instrument): """ A generic FxCorrelator composed of fengines that channelise antenna inputs and xengines that each produce cross products from a continuous portion of the channels and accumulate the result. SPEAD data products are produced. """ def __init__(self, descriptor, identifier=-1, config_source=None, logger=LOGGER): """ An abstract base class for instruments. :param descriptor: A text description of the instrument. Required. :param identifier: An optional integer identifier. :param config_source: The instrument configuration source. Can be a text file, hostname, whatever. :param logger: Use the module logger by default, unless something else is given. :return: <nothing> """ self.logger = logger self.loghandler = None # we know about f and x hosts and engines, not just engines and hosts self.fhosts = [] self.xhosts = [] self.filthosts = None self.found_beamformer = False self.fops = None self.xops = None self.bops = None self.filtops = None # attributes self.katcp_port = None self.f_per_fpga = None self.x_per_fpga = None self.accumulation_len = None self.xeng_accumulation_len = None self.xeng_tx_destination = None self.meta_destination = None self.fengine_sources = None self.spead_tx = None self.spead_meta_ig = None self._sensors = {} # parent constructor Instrument.__init__(self, descriptor, identifier, config_source) self._initialised = False def standard_log_config(self, log_level=logging.INFO, silence_spead=True): """Convenience method for setting up logging in scripts etc. :param logger: The loglevel to use by default, logging.INFO. :param silence_spead: Set 'spead' logger to level WARNING if True """ # Set the root logging handler - submodules loggers will automatically # use it self.loghandler = log.Corr2LogHandler() logging.root.addHandler(self.loghandler) logging.root.setLevel(log_level) self.logger.setLevel(log_level) self.logger.addHandler(self.loghandler) if silence_spead: # set the SPEAD logger to warning only spead_logger = logging.getLogger('spead') spead_logger.setLevel(logging.WARNING) def initialise(self, program=True, qdr_cal=True): """ Set up the correlator using the information in the config file. :return: """ # set up the F, X, B and filter handlers self.fops = FEngineOperations(self) self.xops = XEngineOperations(self) # self.bops = BEngineOperations(self) self.filtops = FilterOperations(self) # set up the filter boards if we need to if 'filter' in self.configd: try: self.filtops.initialise(program=program) except Exception as e: raise e # connect to the other hosts that make up this correlator THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=5, target_function='connect') igmp_version = self.configd['FxCorrelator'].get('igmp_version') if igmp_version is not None: self.logger.info('Setting FPGA hosts IGMP version to ' '%s' % igmp_version) THREADED_FPGA_FUNC( self.fhosts + self.xhosts, timeout=5, target_function=( 'set_igmp_version', (igmp_version, ), {})) # if we need to program the FPGAs, do so if program: self.logger.info('Programming FPGA hosts') fpgautils.program_fpgas([(host, host.boffile) for host in (self.fhosts + self.xhosts)], progfile=None, timeout=15) else: self.logger.info('Loading design information') THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=7, target_function='get_system_information') # remove test hardware from designs utils.disable_test_gbes(self) utils.remove_test_objects(self) if program: # cal the qdr on all boards if qdr_cal: self.qdr_calibrate() else: self.logger.info('Skipping QDR cal - are you sure you' ' want to do this?') # init the engines self.fops.initialise() self.xops.initialise() # are there beamformers? if self.found_beamformer: bengops.beng_initialise(self) # for fpga_ in self.fhosts: # fpga_.tap_arp_reload() # for fpga_ in self.xhosts: # fpga_.tap_arp_reload() # subscribe all the engines to the multicast groups self.fops.subscribe_to_multicast() self.xops.subscribe_to_multicast() post_mess_delay = 10 self.logger.info('post mess-with-the-switch delay of ' '%is' % post_mess_delay) time.sleep(post_mess_delay) # force a reset on the f-engines self.fops.sys_reset(sleeptime=1) # reset all counters on fhosts and xhosts self.fops.clear_status_all() self.xops.clear_status_all() # check to see if the f engines are receiving all their data if not self.fops.check_rx(): raise RuntimeError('The f-engines RX have a problem.') # start f-engine TX self.logger.info('Starting f-engine datastream') self.fops.tx_enable() # check that the F-engines are transmitting data correctly if not self.fops.check_tx(): raise RuntimeError('The f-engines TX have a problem.') # check that the X-engines are receiving data if not self.xops.check_rx(): raise RuntimeError('The x-engines RX have a problem.') # arm the vaccs on the x-engines self.xops.vacc_sync() # reset all counters on fhosts and xhosts self.fops.clear_status_all() self.xops.clear_status_all() # set an initialised flag self._initialised = True def initialised(self): """ Has initialise successfully passed? :return: """ return self._initialised def est_sync_epoch(self): """ Estimates the synchronisation epoch based on current F-engine timestamp, and the system time. """ self.logger.warn("Estimating synchronisation epoch...") # get current time from an f-engine mcnt_before = self.fhosts[0].get_local_time() self.synchronisation_epoch = (time.time() - mcnt_before / float(self.sample_rate_hz)) self.logger.info('Current f engine timetamp: %i' % mcnt_before) assert mcnt_before & 0xfff == 0, 'Bottom 12 bits of timestamp from ' \ 'f-engine are not zero?!' def time_from_mcnt(self, mcnt): """ Returns the unix time UTC equivalent to the board timestamp. Does NOT account for wrapping timestamps. """ if self.synchronisation_epoch < 0: self.est_sync_epoch() return (self.synchronisation_epoch + (float(mcnt) / float(self.sample_rate_hz))) def mcnt_from_time(self, time_seconds): """ Returns the board timestamp from a given UTC system time (seconds since Unix Epoch). Accounts for wrapping timestamps. """ if self.synchronisation_epoch < 0: self.est_sync_epoch() time_diff_from_synch_epoch = time_seconds - self.synchronisation_epoch time_diff_in_samples = int(time_diff_from_synch_epoch * self.sample_rate_hz) _tmp = 2**int(self.configd['FxCorrelator']['timestamp_bits']) return time_diff_in_samples % _tmp def qdr_calibrate(self): """ Run a software calibration routine on all the FPGA hosts. Do it on F- and X-hosts in parallel. :return: """ def _qdr_cal(_fpga): _tic = time.time() _results = {} for _qdr in _fpga.qdrs: _results[_qdr.name] = _qdr.qdr_cal(fail_hard=False) _toc = time.time() return {'results': _results, 'tic': _tic, 'toc': _toc} self.logger.info('Calibrating QDR on F- and X-engines, this takes a ' 'while.') qdr_calfail = False results = THREADED_FPGA_OP(self.fhosts + self.xhosts, timeout=30, target_function=(_qdr_cal,)) for fpga, result in results.items(): self.logger.info('FPGA %s QDR cal results: start(%.3f) end(%.3f) ' 'took(%.3f)' % (fpga, result['tic'], result['toc'], result['toc'] - result['tic'])) for qdr, qdrres in result['results'].items(): if not qdrres: qdr_calfail = True break self.logger.info('\t%s: cal okay: ' '%s' % (qdr, 'True' if qdrres else 'False')) if qdr_calfail: break if qdr_calfail: raise RuntimeError('QDR calibration failure.') # for host in self.fhosts: # for qdr in host.qdrs: # qdr.qdr_delay_in_step(0b111111111111111111111111111111111111, # -1) # for host in self.xhosts: # for qdr in host.qdrs: # qdr.qdr_delay_in_step(0b111111111111111111111111111111111111, # -1) def set_labels(self, newlist): """ Apply new source labels to the configured fengine sources. :param newlist: :return: """ if len(newlist) != len(self.fengine_sources): errstr = 'Number of supplied source labels (%i) does not match ' \ 'number of configured sources (%i).' % \ (len(newlist), len(self.fengine_sources)) self.logger.error(errstr) raise ValueError(errstr) for ctr, source in enumerate(self.fengine_sources): _source = source['source'] # update the source name old_name = _source.name _source.name = newlist[ctr] # update the eq associated with that name found_eq = False for fhost in self.fhosts: if old_name in fhost.eqs.keys(): fhost.eqs[_source.name] = fhost.eqs.pop(old_name) found_eq = True break if not found_eq: raise ValueError('Could not find the old EQ value, %s, to ' 'update to new name, %s.' % (old_name, _source.name)) if self.spead_meta_ig is not None: metalist = numpy.array(self._spead_meta_get_labelling()) self.spead_meta_ig['input_labelling'] = metalist self.spead_tx.send_heap(self.spead_meta_ig.get_heap()) def get_labels(self): """ Get the current fengine source labels as a string. :return: """ source_names = '' for fsrc in self.fengine_sources: source_names += fsrc['source'].name + ' ' source_names = source_names.strip() return source_names def _read_config(self): """ Read the instrument configuration from self.config_source. :return: True if the instrument read a config successfully, raise an error if not? """ Instrument._read_config(self) _d = self.configd # check that the bitstream names are present try: open(_d['fengine']['bitstream'], 'r').close() open(_d['xengine']['bitstream'], 'r').close() except IOError: self.logger.error('xengine bitstream: ' '%s' % _d['xengine']['bitstream']) self.logger.error('fengine bitstream: ' '%s' % _d['fengine']['bitstream']) self.logger.error('One or more bitstream files not found.') raise IOError('One or more bitstream files not found.') # TODO: Load config values from the bitstream meta information - # f per fpga, x per fpga, etc self.arp_wait_time = int(_d['FxCorrelator']['arp_wait_time']) self.sensor_poll_time = int(_d['FxCorrelator']['sensor_poll_time']) self.katcp_port = int(_d['FxCorrelator']['katcp_port']) self.f_per_fpga = int(_d['fengine']['f_per_fpga']) self.x_per_fpga = int(_d['xengine']['x_per_fpga']) self.sample_rate_hz = int(_d['FxCorrelator']['sample_rate_hz']) self.adc_demux_factor = int(_d['fengine']['adc_demux_factor']) self.accumulation_len = int(_d['xengine']['accumulation_len']) self.xeng_accumulation_len = int(_d['xengine']['xeng_accumulation_len']) self.n_chans = int(_d['fengine']['n_chans']) self.n_antennas = int(_d['fengine']['n_antennas']) self.min_load_time = float(_d['fengine']['min_load_time']) self.set_stream_destination( _d['xengine']['output_destination_ip'], int(_d['xengine']['output_destination_port'])) self.set_meta_destination(_d['xengine']['output_destination_ip'], int(_d['xengine']['output_destination_port'])) # get this from the running x-engines? self.xeng_clk = int(_d['xengine']['x_fpga_clock']) self.xeng_outbits = int(_d['xengine']['xeng_outbits']) # the f-engines have this many 10Gbe ports per f-engine # unit of operation self.ports_per_fengine = int(_d['fengine']['ports_per_fengine']) # check if beamformer exists with x-engines self.found_beamformer = False if 'bengine' in self.configd.keys(): self.found_beamformer = True # set up the hosts and engines based on the configuration in the ini file self.fhosts = [] for host in _d['fengine']['hosts'].split(','): host = host.strip() fpgahost = fhost_fpga.FpgaFHost.from_config_source( host, self.katcp_port, config_source=_d['fengine']) self.fhosts.append(fpgahost) # choose class (b-engine inherits x-engine functionality) if self.found_beamformer: _targetClass = bhost_fpga.FpgaBHost else: _targetClass = xhost_fpga.FpgaXHost self.xhosts = [] for host in _d['xengine']['hosts'].split(','): host = host.strip() fpgahost = xhost_fpga.FpgaXHost.from_config_source( host, self.katcp_port, config_source=_d['xengine']) self.xhosts.append(fpgahost) self.bhosts = [] # x-eng host b-eng for host in _d['xengine']['hosts'].split(','): host = host.strip() fpgahost = bhost_fpga.FpgaBHost.from_config_source( host, self.katcp_port, config_source=_d['xengine']) self.bhosts.append(fpgahost) # check that no hosts overlap for _fh in self.fhosts: for _xh in self.xhosts: if _fh.host == _xh.host: self.logger.error('Host %s is assigned to both X- and ' 'F-engines' % _fh.host) raise RuntimeError # what data sources have we been allocated? self._handle_sources() # turn the product names into a list prodlist = _d['xengine']['output_products'].replace('[', '').replace(']', '').split(',') _d['xengine']['output_products'] = [] for prod in prodlist: _d['xengine']['output_products'].append(prod.strip('')) def _handle_sources(self): """ Sort out sources and eqs for them :return: """ assert len(self.fhosts) > 0 _feng_cfg = self.configd['fengine'] source_names = _feng_cfg['source_names'].strip().split(',') source_mcast = _feng_cfg['source_mcast_ips'].strip().split(',') assert len(source_mcast) == len(source_names), ( 'Source names (%d) must be paired with multicast source ' 'addresses (%d)' % (len(source_names), len(source_mcast))) # match eq polys to source names eq_polys = {} for src_name in source_names: eq_polys[src_name] = utils.process_new_eq( _feng_cfg['eq_poly_%s' % src_name]) # assemble the sources given into a list _fengine_sources = [] for source_ctr, address in enumerate(source_mcast): new_source = DataSource.from_mcast_string(address) new_source.name = source_names[source_ctr] assert new_source.ip_range == self.ports_per_fengine, ( 'F-engines should be receiving from %d streams.' % self.ports_per_fengine) _fengine_sources.append(new_source) # assign sources and eqs to fhosts self.logger.info('Assigning DataSources, EQs and DelayTrackers to ' 'f-engines...') source_ctr = 0 self.fengine_sources = [] for fhost in self.fhosts: self.logger.info('\t%s:' % fhost.host) _eq_dict = {} for fengnum in range(0, self.f_per_fpga): _source = _fengine_sources[source_ctr] _eq_dict[_source.name] = {'eq': eq_polys[_source.name], 'bram_num': fengnum} assert _source.ip_range == _fengine_sources[0].ip_range, ( 'All f-engines should be receiving from %d streams.' % self.ports_per_fengine) self.fengine_sources.append({'source': _source, 'source_num': source_ctr, 'host': fhost, 'numonhost': fengnum}) fhost.add_source(_source) self.logger.info('\t\t%s' % _source) source_ctr += 1 fhost.eqs = _eq_dict if source_ctr != len(self.fhosts) * self.f_per_fpga: raise RuntimeError('We have different numbers of sources (%d) and ' 'f-engines (%d). Problem.', source_ctr, len(self.fhosts) * self.f_per_fpga) self.logger.info('done.') def _read_config_file(self): """ Read the instrument configuration from self.config_source. :return: True if we read the file successfully, False if not """ self.configd = utils.parse_ini_file(self.config_source) def _read_config_server(self): """ Get instance-specific setup information from a given server. Via KATCP? :return: """ raise NotImplementedError('_read_config_server not implemented') def set_stream_destination(self, txip_str=None, txport=None): """ Set destination for output of fxcorrelator. :param txip_str: A dotted-decimal string representation of the IP address. e.g. '1.2.3.4' :param txport: An integer port number. :return: <nothing> """ if txip_str is None: txip = tengbe.IpAddress.str2ip(self.xeng_tx_destination[0]) else: txip = tengbe.IpAddress.str2ip(txip_str) if txport is None: txport = self.xeng_tx_destination[1] else: txport = int(txport) self.logger.info('Setting stream destination to %s:%d' % (tengbe.IpAddress.ip2str(txip), txport)) try: THREADED_FPGA_OP(self.xhosts, timeout=10, target_function=(lambda fpga_: fpga_.registers.gbe_iptx.write(reg=txip),)) THREADED_FPGA_OP(self.xhosts, timeout=10, target_function=(lambda fpga_: fpga_.registers.gbe_porttx.write(reg=txport),)) except AttributeError: self.logger.warning('Set SPEAD stream destination called, but ' 'devices NOT written! Have they been created?') self.xeng_tx_destination = (tengbe.IpAddress.ip2str(txip), txport) def set_meta_destination(self, txip_str=None, txport=None): """ Set destination for meta info output of fxcorrelator. :param txip_str: A dotted-decimal string representation of the IP address. e.g. '1.2.3.4' :param txport: An integer port number. :return: <nothing> """ if txip_str is None: txip_str = self.meta_destination[0] if txport is None: txport = self.meta_destination[1] else: txport = int(txport) if txport is None or txip_str is None: self.logger.error('Cannot set part of meta destination to None - %s:%d' % (txip_str, txport)) raise RuntimeError('Cannot set part of meta destination to None - %s:%d' % (txip_str, txport)) self.meta_destination = (txip_str, txport) self.logger.info('Setting meta destination to %s:%d' % (txip_str, txport)) def tx_start(self, issue_spead=True): """ Turns on xengine output pipes needed to start data flow from xengines """ self.logger.info('Starting transmission') if issue_spead: self.spead_issue_meta() # start tx on the x-engines for f in self.xhosts: f.registers.control.write(gbe_txen=True) def tx_stop(self, stop_f=False): """ Turns off output pipes to start data flow from xengines :param stop_f: stop output of fengines as well """ self.logger.info('Stopping X transmission') THREADED_FPGA_OP( self.xhosts, timeout=10, target_function=( lambda fpga_: fpga_.registers.control.write(gbe_txen=False),)) if stop_f: self.logger.info('Stopping F transmission') THREADED_FPGA_OP( self.fhosts, timeout=10, target_function=( lambda fpga_: fpga_.registers.control.write(comms_en=False),)) def _spead_meta_get_labelling(self): """ Get a list of the source labelling for SPEAD metadata :return: """ metalist = [] for fsrc in self.fengine_sources: metalist.append((fsrc['source'].name, fsrc['source_num'], fsrc['host'].host, fsrc['numonhost'])) return metalist def spead_issue_meta(self): """ All FxCorrelators issued SPEAD in the same way, with tweakings that are implemented by the child class. :return: <nothing> """ if self.meta_destination is None: logging.info('SPEAD meta destination is still unset, NOT sending ' 'metadata at this time.') return # make a new SPEAD transmitter del self.spead_tx, self.spead_meta_ig self.spead_tx = spead.Transmitter(spead.TransportUDPtx(*self.meta_destination)) # update the multicast socket option to use a TTL of 2, # in order to traverse the L3 network on site. ttl_bin = struct.pack('@i', 2) self.spead_tx.t._udp_out.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl_bin) #mcast_interface = self.configd['xengine']['multicast_interface_address'] #self.spead_tx.t._udp_out.setsockopt(socket.SOL_IP, socket.IP_MULTICAST_IF, # socket.inet_aton(mcast_interface)) # self.spead_tx.t._udp_out.setsockopt(socket.SOL_IP, socket.IP_ADD_MEMBERSHIP, # socket.inet_aton(txip_str) + socket.inet_aton(mcast_interface)) # make the item group we're going to use self.spead_meta_ig = spead.ItemGroup() self.spead_meta_ig.add_item(name='adc_sample_rate', id=0x1007, description='The expected ADC sample rate (samples ' 'per second) of incoming data.', shape=[], fmt=spead.mkfmt(('u', 64)), init_val=self.sample_rate_hz) self.spead_meta_ig.add_item(name='n_bls', id=0x1008, description='Number of baselines in the data product.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=len(self.xops.get_baseline_order())) self.spead_meta_ig.add_item(name='n_chans', id=0x1009, description='Number of frequency channels in ' 'an integration.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=self.n_chans) self.spead_meta_ig.add_item(name='n_ants', id=0x100A, description='The number of antennas in the system.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=self.n_antennas) self.spead_meta_ig.add_item(name='n_xengs', id=0x100B, description='The number of x-engines in the system.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=(len(self.xhosts) * self.x_per_fpga)) self.spead_meta_ig.add_item(name='bls_ordering', id=0x100C, description='The baseline ordering in the output ' 'data product.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=numpy.array( [baseline for baseline in self.xops.get_baseline_order()])) # spead_ig.add_item(name='crosspol_ordering', id=0x100D, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) metalist = numpy.array(self._spead_meta_get_labelling()) self.spead_meta_ig.add_item(name='input_labelling', id=0x100E, description='input labels and numbers', init_val=metalist) # spead_ig.add_item(name='n_bengs', id=0x100F, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) self.spead_meta_ig.add_item(name='center_freq', id=0x1011, description='The on-sky centre-frequency.', shape=[], fmt=spead.mkfmt(('f', 64)), init_val=int(self.configd['fengine']['true_cf'])) self.spead_meta_ig.add_item(name='bandwidth', id=0x1013, description='The input (analogue) bandwidth of ' 'the system.', shape=[], fmt=spead.mkfmt(('f', 64)), init_val=int(self.configd['fengine']['bandwidth'])) self.spead_meta_ig.add_item(name='n_accs', id=0x1015, description='The number of spectra that are ' 'accumulated per X-engine dump.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=self.accumulation_len * self.xeng_accumulation_len) self.spead_meta_ig.add_item(name='int_time', id=0x1016, description='The time per integration, in seconds.', shape=[], fmt=spead.mkfmt(('f', 64)), init_val=self.xops.get_acc_time()) # spead_ig.add_item(name='coarse_chans', id=0x1017, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # # spead_ig.add_item(name='current_coarse_chan', id=0x1018, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # # spead_ig.add_item(name='fft_shift_fine', id=0x101C, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # # spead_ig.add_item(name='fft_shift_coarse', id=0x101D, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) self.spead_meta_ig.add_item(name='fft_shift', id=0x101E, description='The FFT bitshift pattern. F-engine correlator internals.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=int(self.configd['fengine']['fft_shift'])) self.spead_meta_ig.add_item(name='xeng_acc_len', id=0x101F, description='Number of spectra accumulated inside X engine. ' 'Determines minimum integration time and ' 'user-configurable integration time stepsize. ' 'X-engine correlator internals.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=self.xeng_accumulation_len) quant_bits = int(self.configd['fengine']['quant_format'].split('.')[0]) self.spead_meta_ig.add_item(name='requant_bits', id=0x1020, description='Number of bits after requantisation in the ' 'F engines (post FFT and any ' 'phasing stages).', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=quant_bits) pkt_len = int(self.configd['fengine']['10gbe_pkt_len']) self.spead_meta_ig.add_item(name='feng_pkt_len', id=0x1021, description='Payload size of 10GbE packet exchange between ' 'F and X engines in 64 bit words. Usually equal ' 'to the number of spectra accumulated inside X ' 'engine. F-engine correlator internals.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=pkt_len) # port = int(self.configd['xengine']['output_destination_port']) # spead_ig.add_item(name='rx_udp_port', id=0x1022, # description='Destination UDP port for X engine output.', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=port) port = int(self.configd['fengine']['10gbe_port']) self.spead_meta_ig.add_item(name='feng_udp_port', id=0x1023, description='Port for F-engines 10Gbe links in the system.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=port) # ip = self.configd['xengine']['output_destination_ip'] # spead_ig.add_item(name='rx_udp_ip_str', id=0x1024, # description='Destination UDP IP for X engine output.', # shape=[-1], fmt=spead.STR_FMT, # init_val=ip) ip = struct.unpack('>I', socket.inet_aton(self.configd['fengine']['10gbe_start_ip']))[0] self.spead_meta_ig.add_item(name='feng_start_ip', id=0x1025, description='Start IP address for F-engines in the system.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=ip) self.spead_meta_ig.add_item(name='xeng_rate', id=0x1026, description='Target clock rate of processing engines (xeng).', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=self.xeng_clk) self.spead_meta_ig.add_item(name='sync_time', id=0x1027, description='The time at which the digitisers were synchronised. ' 'Seconds since the Unix Epoch.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=self.synchronisation_epoch) # spead_ig.add_item(name='n_stokes', id=0x1040, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) x_per_fpga = int(self.configd['xengine']['x_per_fpga']) self.spead_meta_ig.add_item(name='x_per_fpga', id=0x1041, description='Number of X engines per FPGA host.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=x_per_fpga) # n_ants_per_xaui = 1 # spead_ig.add_item(name='n_ants_per_xaui', id=0x1042, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=n_ants_per_xaui) # spead_ig.add_item(name='ddc_mix_freq', id=0x1043, # description='', # shape=[],fmt=spead.mkfmt(('f', 64)), # init_val=) # spead_ig.add_item(name='ddc_bandwidth', id=0x1044, # description='', # shape=[],fmt=spead.mkfmt(('f', 64)), # init_val=) sample_bits = int(self.configd['fengine']['sample_bits']) self.spead_meta_ig.add_item(name='adc_bits', id=0x1045, description='How many bits per ADC sample.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=sample_bits) self.spead_meta_ig.add_item(name='scale_factor_timestamp', id=0x1046, description='Timestamp scaling factor. Divide the SPEAD ' 'data packet timestamp by this number to get ' 'back to seconds since last sync.', shape=[], fmt=spead.mkfmt(('f', 64)), init_val=self.sample_rate_hz) self.spead_meta_ig.add_item(name='xeng_out_bits_per_sample', id=0x1048, description='The number of bits per value of the xeng ' 'accumulator output. Note this is for a ' 'single value, not the combined complex size.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=self.xeng_outbits) self.spead_meta_ig.add_item(name='f_per_fpga', id=0x1049, description='Number of F engines per FPGA host.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), init_val=self.f_per_fpga) # spead_ig.add_item(name='rf_gain_MyAntStr ', id=0x1200+inputN, # description='', # shape=[], fmt=spead.mkfmt(('f', 64)), # init_val=) # 0x1400 +++ self.fops.eq_update_metadata() # spead_ig.add_item(name='eq_coef_MyAntStr', id=0x1400+inputN, # description='', # shape=[], fmt=spead.mkfmt(('u', 32)), # init_val=) # ndarray = numpy.dtype(numpy.int64), (4096 * 40 * 1, 1, 1) self.spead_meta_ig.add_item(name='timestamp', id=0x1600, description='Timestamp of start of this integration. uint counting multiples ' 'of ADC samples since last sync (sync_time, id=0x1027). Divide this ' 'number by timestamp_scale (id=0x1046) to get back to seconds since ' 'last sync when this integration was actually started.', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE))) self.spead_meta_ig.add_item(name='flags_xeng_raw', id=0x1601, description='Flags associated with xeng_raw data output.' 'bit 34 - corruption or data missing during integration ' 'bit 33 - overrange in data path ' 'bit 32 - noise diode on during integration ' 'bits 0 - 31 reserved for internal debugging', shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE))) ndarray = numpy.dtype(numpy.int32), (self.n_chans, len(self.xops.get_baseline_order()), 2) self.spead_meta_ig.add_item(name='xeng_raw', id=0x1800, description='Raw data for %i xengines in the system. This item represents a ' 'full spectrum (all frequency channels) assembled from lowest ' 'frequency to highest frequency. Each frequency channel contains ' 'the data for all baselines (n_bls given by SPEAD ID 0x100b). ' 'Each value is a complex number -- two (real and imaginary) ' 'unsigned integers.' % len(self.xhosts * self.x_per_fpga), ndarray=ndarray) # TODO hard-coded !!!!!!! :( # self.spead_ig.add_item(name=("xeng_raw"),id=0x1800, # description="Raw data for %i xengines in the system. This # item represents a full spectrum (all frequency channels) assembled from lowest frequency # to highest frequency. Each frequency channel contains the data for all baselines # (n_bls given by SPEAD ID 0x100B). Each value is a complex number -- two # (real and imaginary) unsigned integers."%(32), # ndarray=(numpy.dtype(numpy.int32),(4096,((4*(4+1))/2)*4,2))) # spead_ig.add_item(name='beamweight_MyAntStr', id=0x2000+inputN, # description='', # shape=[], fmt=spead.mkfmt(('u', 32)), # init_val=) # spead_ig.add_item(name='incoherent_sum', id=0x3000, # description='', # shape=[], fmt=spead.mkfmt(('u', 32)), # init_val=) # spead_ig.add_item(name='n_inputs', id=0x3100, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # spead_ig.add_item(name='digitiser_id', id=0x3101, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # spead_ig.add_item(name='digitiser_status', id=0x3102, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # spead_ig.add_item(name='pld_len', id=0x3103, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # spead_ig.add_item(name='raw_data_MyAntStr', id=0x3300+inputN, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # spead_ig.add_item(name='Reserved for SP-CAM meta-data', id=0x7000-0x7fff, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # spead_ig.add_item(name='feng_id', id=0xf101, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # spead_ig.add_item(name='feng_status', id=0xf102, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # spead_ig.add_item(name='frequency', id=0xf103, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # spead_ig.add_item(name='raw_freq_MyAntStr', id=0xf300+inputN, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) # spead_ig.add_item(name='bf_MyBeamName', id=0xb000+beamN, # description='', # shape=[], fmt=spead.mkfmt(('u', spead.ADDRSIZE)), # init_val=) self.spead_tx.send_heap(self.spead_meta_ig.get_heap()) self.logger.info('Issued SPEAD data descriptor to %s:%i.' % (self.meta_destination[0], self.meta_destination[1]))
def initialise(self, program=True, qdr_cal=True, require_epoch=False): """ Set up the correlator using the information in the config file. :param program: program the FPGA boards if True :param qdr_cal: perform QDR cal if True :param require_epoch: the synch epoch MUST be set before init if True :return: """ # check that the instrument's synch epoch has been set if require_epoch: if self.get_synch_time() == -1: raise RuntimeError('System synch epoch has not been set prior' ' to initialisation!') # set up the F, X, B and filter handlers self.fops = FEngineOperations(self) self.xops = XEngineOperations(self) self.bops = BEngineOperations(self) self.filtops = FilterOperations(self) self.speadops = SpeadOperations(self) # set up the filter boards if we need to if 'filter' in self.configd: try: self.filtops.initialise(program=program) except Exception as e: raise e # connect to the other hosts that make up this correlator THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=5, target_function='connect') igmp_version = self.configd['FxCorrelator'].get('igmp_version') if igmp_version is not None: self.logger.info('Setting FPGA hosts IGMP version ' 'to %s' % igmp_version) THREADED_FPGA_FUNC( self.fhosts + self.xhosts, timeout=5, target_function=('set_igmp_version', (igmp_version, ), {})) # if we need to program the FPGAs, do so if program: self.logger.info('Programming FPGA hosts') fpgautils.program_fpgas([(host, host.boffile) for host in (self.fhosts + self.xhosts)], progfile=None, timeout=15) else: self.logger.info('Loading design information') THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=10, target_function='get_system_information') # remove test hardware from designs utils.disable_test_gbes(self) utils.remove_test_objects(self) # run configuration on the parts of the instrument self.configure() # run post-programming initialisation self._initialise(program, qdr_cal) # reset all counters on fhosts and xhosts self.fops.clear_status_all() self.xops.clear_status_all() # set an initialised flag self._initialised = True
def initialise(self, program=True, qdr_cal=True): """ Set up the correlator using the information in the config file. :return: """ # set up the F, X, B and filter handlers self.fops = FEngineOperations(self) self.xops = XEngineOperations(self) # self.bops = BEngineOperations(self) self.filtops = FilterOperations(self) # set up the filter boards if we need to if 'filter' in self.configd: try: self.filtops.initialise(program=program) except Exception as e: raise e # connect to the other hosts that make up this correlator THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=5, target_function='connect') igmp_version = self.configd['FxCorrelator'].get('igmp_version') if igmp_version is not None: self.logger.info('Setting FPGA hosts IGMP version to ' '%s' % igmp_version) THREADED_FPGA_FUNC( self.fhosts + self.xhosts, timeout=5, target_function=( 'set_igmp_version', (igmp_version, ), {})) # if we need to program the FPGAs, do so if program: self.logger.info('Programming FPGA hosts') fpgautils.program_fpgas([(host, host.boffile) for host in (self.fhosts + self.xhosts)], progfile=None, timeout=15) else: self.logger.info('Loading design information') THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=7, target_function='get_system_information') # remove test hardware from designs utils.disable_test_gbes(self) utils.remove_test_objects(self) if program: # cal the qdr on all boards if qdr_cal: self.qdr_calibrate() else: self.logger.info('Skipping QDR cal - are you sure you' ' want to do this?') # init the engines self.fops.initialise() self.xops.initialise() # are there beamformers? if self.found_beamformer: bengops.beng_initialise(self) # for fpga_ in self.fhosts: # fpga_.tap_arp_reload() # for fpga_ in self.xhosts: # fpga_.tap_arp_reload() # subscribe all the engines to the multicast groups self.fops.subscribe_to_multicast() self.xops.subscribe_to_multicast() post_mess_delay = 10 self.logger.info('post mess-with-the-switch delay of ' '%is' % post_mess_delay) time.sleep(post_mess_delay) # force a reset on the f-engines self.fops.sys_reset(sleeptime=1) # reset all counters on fhosts and xhosts self.fops.clear_status_all() self.xops.clear_status_all() # check to see if the f engines are receiving all their data if not self.fops.check_rx(): raise RuntimeError('The f-engines RX have a problem.') # start f-engine TX self.logger.info('Starting f-engine datastream') self.fops.tx_enable() # check that the F-engines are transmitting data correctly if not self.fops.check_tx(): raise RuntimeError('The f-engines TX have a problem.') # check that the X-engines are receiving data if not self.xops.check_rx(): raise RuntimeError('The x-engines RX have a problem.') # arm the vaccs on the x-engines self.xops.vacc_sync() # reset all counters on fhosts and xhosts self.fops.clear_status_all() self.xops.clear_status_all() # set an initialised flag self._initialised = True
class FxCorrelator(Instrument): """ A generic FxCorrelator composed of fengines that channelise antenna inputs and xengines that each produce cross products from a continuous portion of the channels and accumulate the result. SPEAD data streams are produced. """ def __init__(self, descriptor, identifier=-1, config_source=None, logger=LOGGER): """ An abstract base class for instruments. :param descriptor: A text description of the instrument. Required. :param identifier: An optional integer identifier. :param config_source: The instrument configuration source. Can be a text file, hostname, whatever. :param logger: Use the module logger by default, unless something else is given. :return: <nothing> """ self.logger = logger # we know about f and x hosts and engines, not just engines and hosts self.fhosts = [] self.xhosts = [] self.filthosts = None self.found_beamformer = False self.fops = None self.xops = None self.bops = None self.filtops = None self.speadops = None # attributes self.katcp_port = None self.f_per_fpga = None self.x_per_fpga = None self.accumulation_len = None self.xeng_accumulation_len = None self.fengine_sources = None self.baselines = None self.sensor_manager = None # parent constructor Instrument.__init__(self, descriptor, identifier, config_source, logger) def initialise(self, program=True, qdr_cal=True, require_epoch=False): """ Set up the correlator using the information in the config file. :param program: program the FPGA boards if True :param qdr_cal: perform QDR cal if True :param require_epoch: the synch epoch MUST be set before init if True :return: """ # check that the instrument's synch epoch has been set if require_epoch: if self.get_synch_time() == -1: raise RuntimeError('System synch epoch has not been set prior' ' to initialisation!') # set up the F, X, B and filter handlers self.fops = FEngineOperations(self) self.xops = XEngineOperations(self) self.bops = BEngineOperations(self) self.filtops = FilterOperations(self) self.speadops = SpeadOperations(self) # set up the filter boards if we need to if 'filter' in self.configd: try: self.filtops.initialise(program=program) except Exception as e: raise e # connect to the other hosts that make up this correlator THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=5, target_function='connect') igmp_version = self.configd['FxCorrelator'].get('igmp_version') if igmp_version is not None: self.logger.info('Setting FPGA hosts IGMP version ' 'to %s' % igmp_version) THREADED_FPGA_FUNC( self.fhosts + self.xhosts, timeout=5, target_function=('set_igmp_version', (igmp_version, ), {})) # if we need to program the FPGAs, do so if program: self.logger.info('Programming FPGA hosts') fpgautils.program_fpgas([(host, host.boffile) for host in (self.fhosts + self.xhosts)], progfile=None, timeout=15) else: self.logger.info('Loading design information') THREADED_FPGA_FUNC(self.fhosts + self.xhosts, timeout=10, target_function='get_system_information') # remove test hardware from designs utils.disable_test_gbes(self) utils.remove_test_objects(self) # run configuration on the parts of the instrument self.configure() # run post-programming initialisation self._initialise(program, qdr_cal) # reset all counters on fhosts and xhosts self.fops.clear_status_all() self.xops.clear_status_all() # set an initialised flag self._initialised = True def _gbe_setup(self): """ Set up the 10gbe ports on the hosts :return: """ feng_port = int(self.configd['fengine']['10gbe_port']) xeng_port = int(self.configd['xengine']['10gbe_port']) info_dict = {host.host: (feng_port, 'fhost') for host in self.fhosts} info_dict.update( {host.host: (xeng_port, 'xhost') for host in self.xhosts}) timeout = len(self.fhosts[0].tengbes) * 30 * 1.1 THREADED_FPGA_FUNC( self.fhosts + self.xhosts, timeout=timeout, target_function=('setup_host_gbes', (self.logger, info_dict), {})) def _initialise(self, program, qdr_cal): """ Run this if boards in the system have been programmed. Basic setup of devices. :return: """ if not program: # only run the contents of this function after programming. return # cal the qdr on all boards # logging.getLogger('casperfpga.qdr').setLevel(logging.INFO + 7) if qdr_cal: self.qdr_calibrate() else: self.logger.info('Skipping QDR cal - are you sure you ' 'want to do this?') # init the engines self.fops.initialise_pre_gbe() self.xops.initialise_pre_gbe() # set up the tengbe ports in parallel self._gbe_setup() # continue with init self.fops.initialise_post_gbe() self.xops.initialise_post_gbe() if self.found_beamformer: self.bops.initialise() # subscribe all the engines to the multicast groups self.fops.subscribe_to_multicast() self.xops.subscribe_to_multicast() # start f-engine TX self.logger.info('Starting f-engine datastream') self.fops.tx_enable() # jason's hack to force a reset on the f-engines time.sleep(1) self.fops.sys_reset() # wait for switches to learn, um, stuff self.logger.info('post mess-with-the-switch delay of %is' % self.post_switch_delay) time.sleep(self.post_switch_delay) # jason's hack to force a reset on the f-engines self.fops.sys_reset() time.sleep(1) # reset all counters on fhosts and xhosts self.fops.clear_status_all() self.xops.clear_status_all() # check to see if the f engines are receiving all their data if not self.fops.check_rx(): raise RuntimeError('The f-engines RX have a problem.') # check that the timestamps received on the f-engines make sense result, times, times_unix = self.fops.check_rx_timestamps() if not result: raise RuntimeError('The timestamps received by the f-engines ' 'are not okay. Check the logs') # check the f-engine QDR uses for parity errors if ((not self.fops.check_ct_parity()) or (not self.fops.check_cd_parity())): raise RuntimeError('The f-engine QDRs are reporting errors.') # check that the F-engines are transmitting data correctly if not self.fops.check_tx(): raise RuntimeError('The f-engines TX have a problem.') # check that the X-engines are receiving data if not self.xops.check_rx(): raise RuntimeError('The x-engines RX have a problem.') # arm the vaccs on the x-engines self.xops.vacc_sync() def configure(self): """ Operations to run to configure the instrument, after programming. :return: """ self.fops.configure() self.xops.configure() if self.found_beamformer: self.bops.configure() def get_scale_factor(self): """ By what number do we divide timestamps to get seconds? :return: """ return self.sample_rate_hz def set_synch_time(self, new_synch_time): """ Set the last sync time for this system :param new_synch_time: UNIX time :return: <nothing> """ time_now = time.time() # future? if new_synch_time > time_now: _err = 'Synch time in the future makes no sense? %.3f > %.3f' % ( new_synch_time, time_now) self.logger.error(_err) raise RuntimeError(_err) # too far in the past? if new_synch_time < time_now - (2**self.timestamp_bits): _err = 'Synch epoch supplied is too far in the past: now(%.3f) ' \ 'epoch(%.3f)' % (time_now, new_synch_time) self.logger.error(_err) raise RuntimeError(_err) self._synchronisation_epoch = new_synch_time if self.sensor_manager: sensor = self.sensor_manager.sensor_get('synch-epoch') sensor.set_value(self._synchronisation_epoch) self.logger.info('Set synch epoch to %.5f' % new_synch_time) def est_synch_epoch(self): """ Estimates the synchronisation epoch based on current F-engine timestamp, and the system time. """ self.logger.info('Estimating synchronisation epoch:') # get current time from an f-engine feng_mcnt = self.fhosts[0].get_local_time() self.logger.info('\tcurrent f-engine mcnt: %i' % feng_mcnt) if feng_mcnt & 0xfff != 0: _err = 'Bottom 12 bits of timestamp from f-engine are not ' \ 'zero?! feng_mcnt(%i)' % feng_mcnt self.logger.error(_err) raise RuntimeError(_err) t_now = time.time() self.set_synch_time(t_now - feng_mcnt / float(self.sample_rate_hz)) self.logger.info('\tnew epoch: %.3f' % self.get_synch_time()) def time_from_mcnt(self, mcnt): """ Returns the unix time UTC equivalent to the board timestamp. Does NOT account for wrapping timestamps. :param mcnt: the time from system boards (ADC ticks since startup) """ if self.get_synch_time() < 0: self.logger.info('time_from_mcnt: synch epoch unset, estimating') self.est_synch_epoch() return self.get_synch_time() + ( float(mcnt) / float(self.sample_rate_hz)) def mcnt_from_time(self, time_seconds): """ Returns the board timestamp from a given UTC system time (seconds since Unix Epoch). Accounts for wrapping timestamps. :param time_seconds: UNIX time """ if self.get_synch_time() < 0: self.logger.info('mcnt_from_time: synch epoch unset, estimating') self.est_synch_epoch() time_diff_from_synch_epoch = time_seconds - self.get_synch_time() time_diff_in_samples = int(time_diff_from_synch_epoch * self.sample_rate_hz) _tmp = 2**self.timestamp_bits return time_diff_in_samples % _tmp def qdr_calibrate(self, timeout=120 * MAX_QDR_ATTEMPTS): """ Run a software calibration routine on all the FPGA hosts. Do it on F- and X-hosts in parallel. :param timeout: how many seconds to wait for it to complete :return: """ def _qdr_cal(_fpga): """ Calibrate the QDRs found on a given FPGA. :param _fpga: :return: """ _tic = time.time() attempts = 0 _results = {_qdr.name: False for _qdr in _fpga.qdrs} while True: all_passed = True for _qdr in _fpga.qdrs: if not _results[_qdr.name]: try: _res = _qdr.qdr_cal(fail_hard=False) except RuntimeError: _res = False try: _resval = _res[0] except TypeError: _resval = _res if not _resval: all_passed = False _results[_qdr.name] = _resval attempts += 1 if all_passed or (attempts >= MAX_QDR_ATTEMPTS): break _toc = time.time() return {'results': _results, 'tic': _tic, 'toc': _toc, 'attempts': attempts} self.logger.info('Calibrating QDR on F- and X-engines, this may ' 'take a while.') qdr_calfail = False results = THREADED_FPGA_OP( self.fhosts + self.xhosts, timeout, (_qdr_cal,)) for fpga, result in results.items(): _time_taken = result['toc'] - result['tic'] self.logger.info('FPGA %s QDR cal: %.3fs, %i attempts' % (fpga, _time_taken, result['attempts'])) for qdr, qdrres in result['results'].items(): if not qdrres: qdr_calfail = True break self.logger.info('\t%s: cal okay: %s' % (qdr, 'True' if qdrres else 'False')) if qdr_calfail: raise RuntimeError('QDR calibration failure.') # for host in self.fhosts: # for qdr in host.qdrs: # qdr.qdr_delay_in_step(0b111111111111111111111111111111111111, # -1) # for host in self.xhosts: # for qdr in host.qdrs: # qdr.qdr_delay_in_step(0b111111111111111111111111111111111111, # -1) def set_labels(self, newlist): """ Apply new source labels to the configured fengine sources. :param newlist: :return: """ old_labels = self.get_labels() if len(newlist) != len(old_labels): errstr = 'Number of supplied source labels (%i) does not match ' \ 'number of configured sources (%i).' % \ (len(newlist), len(old_labels)) self.logger.error(errstr) raise ValueError(errstr) new_dict = {} for ctr, source_name in enumerate(old_labels): _source = self.fengine_sources[source_name] _source.update_name(newlist[ctr]) new_dict[newlist[ctr]] = _source self.fengine_sources = new_dict # update the list of baselines self.baselines = utils.baselines_from_source_list(newlist) # update the hostname and baseline sensors if self.sensor_manager: sm = self.sensor_manager try: for fhost in self.fhosts: sensor = sm.sensor_get('%s-input-mapping' % fhost.host) rv = [dsrc.name for dsrc in fhost.data_sources] sensor.set_value(str(rv)) except Exception as ve: self.logger.warning('Could not update input_mapping ' 'sensors!\n%s' % ve.message) sm.sensor_get('baseline-ordering').set_value(str(self.baselines)) # update the beam input labels if self.found_beamformer: self.bops.update_labels(old_labels, self.get_labels()) self.speadops.update_metadata([0x100e]) self.logger.info('Source labels updated from %s to %s' % ( old_labels, self.get_labels())) def get_labels(self): """ Get the current fengine source labels as a list of label names. :return: """ return sorted(self.fengine_sources, key=lambda k: self.fengine_sources[k].source_number) def _read_config(self): """ Read the instrument configuration from self.config_source. :return: """ Instrument._read_config(self) _d = self.configd # check that the bitstream names are present try: open(_d['fengine']['bitstream'], 'r').close() open(_d['xengine']['bitstream'], 'r').close() except IOError: self.logger.error('xengine bitstream: ' '%s' % _d['xengine']['bitstream']) self.logger.error('fengine bitstream: ' '%s' % _d['fengine']['bitstream']) self.logger.error('One or more bitstream files not found.') raise IOError('One or more bitstream files not found.') # TODO: Load config values from the bitstream meta information - # f per fpga, x per fpga, etc _fxcorr_d = self.configd['FxCorrelator'] self.arp_wait_time = int(_fxcorr_d['arp_wait_time']) self.sensor_poll_time = int(_fxcorr_d['sensor_poll_time']) self.katcp_port = int(_fxcorr_d['katcp_port']) self.sample_rate_hz = int(_fxcorr_d['sample_rate_hz']) self.timestamp_bits = int(_fxcorr_d['timestamp_bits']) self.time_jitter_allowed_ms = int(_fxcorr_d['time_jitter_allowed_ms']) self.time_offset_allowed_s = int(_fxcorr_d['time_offset_allowed_s']) try: self.post_switch_delay = int(_fxcorr_d['switch_delay']) except KeyError: self.post_switch_delay = 10 _feng_d = self.configd['fengine'] self.adc_demux_factor = int(_feng_d['adc_demux_factor']) self.n_chans = int(_feng_d['n_chans']) self.n_antennas = int(_feng_d['n_antennas']) self.min_load_time = float(_feng_d['min_load_time']) self.f_per_fpga = int(_feng_d['f_per_fpga']) self.ports_per_fengine = int(_feng_d['ports_per_fengine']) self.analogue_bandwidth = int(_feng_d['bandwidth']) self.true_cf = float(_feng_d['true_cf']) self.quant_format = _feng_d['quant_format'] self.adc_bitwidth = int(_feng_d['sample_bits']) self.fft_shift = int(_feng_d['fft_shift']) try: self.qdr_ct_error_threshold = int(_feng_d['qdr_ct_error_threshold']) except KeyError: self.qdr_ct_error_threshold = 100 try: self.qdr_cd_error_threshold = int(_feng_d['qdr_cd_error_threshold']) except KeyError: self.qdr_cd_error_threshold = 100 _xeng_d = self.configd['xengine'] self.x_per_fpga = int(_xeng_d['x_per_fpga']) self.accumulation_len = int(_xeng_d['accumulation_len']) self.xeng_accumulation_len = int(_xeng_d['xeng_accumulation_len']) try: self.qdr_vacc_error_threshold = int( _xeng_d['qdr_vacc_error_threshold']) except KeyError: self.qdr_vacc_error_threshold = 100 # get this from the running x-engines? self.xeng_clk = int(_xeng_d['x_fpga_clock']) self.xeng_outbits = int(_xeng_d['xeng_outbits']) # check if beamformer exists with x-engines self.found_beamformer = False if 'bengine' in self.configd.keys(): self.found_beamformer = True self.beng_outbits = 8 # set up hosts and engines based on the configuration in the ini file _targetClass = fhost_fpga.FpgaFHost self.fhosts = [] for host in _feng_d['hosts'].split(','): host = host.strip() fpgahost = _targetClass.from_config_source(host, self.katcp_port, config_source=_feng_d) self.fhosts.append(fpgahost) # choose class (b-engine inherits x-engine functionality) if self.found_beamformer: _targetClass = bhost_fpga.FpgaBHost else: _targetClass = xhost_fpga.FpgaXHost self.xhosts = [] hostlist = _xeng_d['hosts'].split(',') for hostindex, host in enumerate(hostlist): host = host.strip() fpgahost = _targetClass.from_config_source( host, hostindex, self.katcp_port, config_source=_xeng_d) self.xhosts.append(fpgahost) # check that no hosts overlap for _fh in self.fhosts: for _xh in self.xhosts: if _fh.host == _xh.host: self.logger.error('Host %s is assigned to ' 'both X- and F-engines' % _fh.host) raise RuntimeError # update the list of baselines on this system self.baselines = utils.baselines_from_config(config=self.configd) # what data sources have we been allocated? self._handle_sources() # turn the stream names into a list prodlist = _xeng_d['output_products'].replace('[', '') prodlist = prodlist.replace(']', '').split(',') _xeng_d['output_products'] = [prod.strip(' ') for prod in prodlist] def _handle_sources(self): """ Sort out sources and eqs for them :return: """ assert len(self.fhosts) > 0 _feng_cfg = self.configd['fengine'] source_names = utils.sources_from_config(config=self.configd) source_mcast = _feng_cfg['source_mcast_ips'].strip().split(',') assert len(source_mcast) == len(source_names), ( 'Source names (%d) must be paired with multicast source ' 'addresses (%d)' % (len(source_names), len(source_mcast))) # match eq polys to source names eq_polys = {} for src_name in source_names: eq_polys[src_name] = utils.process_new_eq( _feng_cfg['eq_poly_%s' % src_name]) # assemble the sources given into a list _feng_src_temp = [] for source_ctr, address in enumerate(source_mcast): new_source = FengineSource.from_mcast_string(address) new_source.name = source_names[source_ctr] new_source.source_number = source_ctr new_source.offset = source_ctr % self.f_per_fpga new_source.eq_poly = eq_polys[new_source.name] new_source.eq_bram_name = 'eq%i' % new_source.offset assert new_source.ip_range == self.ports_per_fengine, ( 'F-engines should be receiving from %d streams.' % self.ports_per_fengine) _feng_src_temp.append(new_source) # check that the sources all have the same IP ranges for _source in _feng_src_temp: assert _source.ip_range == _feng_src_temp[0].ip_range, ( 'All f-engines should be receiving from %d streams.' % self.ports_per_fengine) # assign sources to fhosts self.logger.info('Assigning FengineSources to f-hosts') _src_ctr = 0 self.fengine_sources = {} for fhost in self.fhosts: self.logger.info('\t%s:' % fhost.host) for fengnum in range(0, self.f_per_fpga): _src = _feng_src_temp[_src_ctr] _src.host = fhost self.fengine_sources[_src.name] = _src fhost.add_source(_src) self.logger.info('\t\t%s' % _src) _src_ctr += 1 if _src_ctr != len(self.fhosts) * self.f_per_fpga: raise RuntimeError('We have different numbers of sources (%d) and ' 'f-engines (%d). Problem.', _src_ctr, len(self.fhosts) * self.f_per_fpga) self.logger.info('done.') def _read_config_file(self): """ Read the instrument configuration from self.config_source. :return: True if we read the file successfully, False if not """ self.configd = utils.parse_ini_file(self.config_source) def _read_config_server(self): """ Get instance-specific setup information from a given server. Via KATCP? :return: """ raise NotImplementedError('_read_config_server not implemented') def stream_set_destination(self, stream_name, txip_str=None, txport=None): """ Set the destination for a data stream. :param stream_name: :param txip_str: A dotted-decimal string representation of the IP address. e.g. '1.2.3.4' :param txport: An integer port number. :return: <nothing> """ stream = self.get_data_stream(stream_name) stream.set_destination(txip_str, txport) stream.set_meta_destination(txip_str, txport) if self.sensor_manager: sensor_name = '%s-destination' % stream.name sensor = self.sensor_manager.sensor_get(sensor_name) sensor.set_value(str(stream.destination)) sensor_name = '%s-meta-destination' % stream.name sensor = self.sensor_manager.sensor_get(sensor_name) sensor.set_value(str(stream.meta_destination))