예제 #1
0
 def dhost(self, program=False):
     if self._dhost is not None:
         return self._dhost
     else:
         self.config_filename = '/etc/corr/{}-{}'.format(self.array_name, self.instrument)
         if os.path.exists(self.config_filename):
             LOGGER.info('Retrieving dsim engine info from config file: %s' % self.config_filename)
             self.corr_config = corr2.utils.parse_ini_file(self.config_filename)
             self.dsim_conf = self.corr_config['dsimengine']
         elif self.instrument is not None:
             self.corr2ini_path = '/etc/corr/templates/{}'.format(self.instrument)
             LOGGER.info('Setting CORR2INI system enviroment to point to %s' % self.corr2ini_path)
             os.environ['CORR2INI'] = self.corr2ini_path
             self.corr_config = corr2.utils.parse_ini_file(self.corr2ini_path)
             self.dsim_conf = self.corr_config['dsimengine']
         else:
             errmsg = ('Could not retrieve dsim information from running config file in /etc/corr, '
                       'Perhaps, restart CBF manually and ensure dsim is running.\n'
                       'File:%s Line:%s' % (getframeinfo(currentframe()).filename.split('/')[-1],
                                            getframeinfo(currentframe()).lineno))
             LOGGER.error(errmsg)
             sys.exit(errmsg)
         try:
             dig_host = self.dsim_conf['host']
             self._dhost = FpgaDsimHost(dig_host, config=self.dsim_conf)
         except Exception:
             errmsg = 'Digitiser Simulator failed to retrieve information'
             LOGGER.exception(errmsg)
             sys.exit(errmsg)
         else:
             # Check if D-eng is running else start it.
             if self._dhost.is_running():
                 LOGGER.info('D-Eng is already running.')
             # Disabled DSim programming as it would alter the systems sync epoch
             elif program and not self._dhost.is_running():
                 LOGGER.info('Programming and starting the Digitiser Simulator.')
                 self._dhost.initialise()
                 self._dhost.enable_data_output(enabled=True)
                 self._dhost.registers.control.write(gbe_txen=True)
             else:
                 LOGGER.info('D-Eng started succesfully')
             return self._dhost
예제 #2
0
 def dhost(self, program=False):
     if self._dhost is not None:
         return self._dhost
     else:
         if os.path.exists(self.config_filename):
             LOGGER.info(
                 'Retrieving dsim engine info from config file: %s' %
                 self.config_filename)
             self.corr_config = parse_ini_file(self.config_filename)
             self.dsim_conf = self.corr_config['dsimengine']
         elif self.instrument is not None:
             self.corr2ini_path = '/etc/corr/templates/{}'.format(
                 self.instrument)
             LOGGER.info(
                 'Setting CORR2INI system environment to point to %s' %
                 self.corr2ini_path)
             os.environ['CORR2INI'] = self.corr2ini_path
             self.corr_config = parse_ini_file(self.corr2ini_path)
             self.dsim_conf = self.corr_config['dsimengine']
         else:
             errmsg = (
                 'Could not retrieve dsim information from running config file in /etc/corr, '
                 'Perhaps, restart CBF manually and ensure dsim is running.\n'
             )
             LOGGER.error(errmsg)
             sys.exit(errmsg)
         try:
             dig_host = self.dsim_conf['host']
             self._dhost = None
             self._dhost = FpgaDsimHost(dig_host, config=self.dsim_conf)
         except Exception:
             errmsg = 'Digitiser Simulator failed to retrieve information'
             LOGGER.exception(errmsg)
             sys.exit(errmsg)
         else:
             # Check if D-eng is running else start it.
             if self._dhost.is_running():
                 LOGGER.info('D-Eng is already running.')
                 return self._dhost
예제 #3
0
 def setUp(self):
     self.receiver = CorrRx(port=8888)
     start_thread_with_cleanup(self, self.receiver, start_timeout=1)
     self.correlator = correlator_fixture.correlator
     self.corr_fix = correlator_fixture
     self.corr_freqs = CorrelatorFrequencyInfo(self.correlator.configd)
     dsim_conf = self.correlator.configd['dsimengine']
     dig_host = dsim_conf['host']
     self.dhost = FpgaDsimHost(dig_host, config=dsim_conf)
     self.dhost.get_system_information()
     # Increase the dump rate so tests can run faster
     self.correlator.xeng_set_acc_time(0.2)
     self.addCleanup(self.corr_fix.stop_x_data)
     self.corr_fix.start_x_data()
     self.corr_fix.issue_metadata()
예제 #4
0
            help='Constant wave output scale')
    parser.add_argument(
            '-n', '--noise_scale', type=float, default=0.05,
            help='Noise output scale')

    args = parser.parse_args()
    cw_scale = args.cw_scale
    n_scale = args.noise_scale
    freq = args.frequency

    #Connect to DSIM
    corr_conf = utils.parse_ini_file(args.config, ['dsimengine'])
    dsim_conf = corr_conf['dsimengine']
    dig_host = dsim_conf['host']
    dsim_fpg = dsim_conf['bitstream']
    dhost = FpgaDsimHost(dig_host, config=dsim_conf)
    if dig_host.lower().find('skarab') != -1:
        dhost.get_system_information(filename=dsim_fpg)
    else:
        dhost.get_system_information()


    #Clear all noise sources
    sources_names = dhost.noise_sources.names()
    for source in sources_names:
        try:
            noise_source = getattr(dhost.noise_sources, '{}'.format(source))
            noise_source.set(0)
            print("noise source {}, set to {}.".format(noise_source.name,
                                                       noise_source.scale))
        except:
예제 #5
0
    """
    Based on JSObject : Python Objects that act like Javascript Objects
    based on James Robert blog entry:
    Making Python Objects that act like Javascript Objects
    http://jiaaro.com/making-python-objects-that-act-like-javascrip
    """

#logging.basicConfig(
#    format='%(asctime)s %(name)s %(levelname)s %(filename)s:%(lineno)s %(message)s',
#    level=logging.INFO)

corr_conf = utils.parse_ini_file(config, ['dsimengine'])
dsim_conf = corr_conf['dsimengine']
dig_host = dsim_conf['host']

dhost = FpgaDsimHost(dig_host, config=dsim_conf)
if dhost.is_running():
    dhost.get_system_information()
    print 'Dsim is running'

config_link = '/etc/corr/array0-bc8n856M4k'
config_link2 = '/etc/corr/array0-bc8n856M32k'
config_link3 = '/etc/corr/templates/bc8n856M4k'
config_link4 = '/etc/corr/templates/bc8n856M32k'
if os.path.exists(config_link):
    correlator = fxcorrelator.FxCorrelator('rts correlator', config_source=config_link)
elif os.path.exists(config_link2):
    correlator = fxcorrelator.FxCorrelator('rts correlator', config_source=config_link2)
elif os.path.exists(config_link3):
    correlator = fxcorrelator.FxCorrelator('rts correlator', config_source=config_link3)
else:
예제 #6
0
class CorrelatorFixture(object):
    def __init__(self, katcp_clt=None, product_name=None):
        self.katcp_clt = katcp_clt
        self.corr_config = None
        self.corr2ini_path = None
        self._correlator = None
        self._dhost = None
        self._katcp_rct = None
        self._rct = None
        self.katcp_array_port = None
        self.product_name = product_name
        self.halt_wait_time = 5
        # Assume the correlator is already started if start_correlator is False
        nose_test_config = {}
        self._correlator_started = not int(
            nose_test_config.get('start_correlator', False))
        self.test_config = self._test_config_file
        self.array_name, self.instrument = self._get_instrument

    @property
    def rct(self):
        if self._rct is not None:
            return self._rct
        else:
            self.io_manager = ioloop_manager.IOLoopManager()
            self.io_wrapper = resource_client.IOLoopThreadWrapper(
                self.io_manager.get_ioloop())
            LOGGER.info('Cleanup function \'self.io_manager\': File: %s line: %s' % (
                getframeinfo(currentframe()).filename.split('/')[-1],
                getframeinfo(currentframe()).lineno))
            add_cleanup(self.io_manager.stop)
            self.io_wrapper.default_timeout = _timeout
            self.io_manager.start()
            self.rc = resource_client.KATCPClientResource(
                dict(name='{}'.format(self.katcp_clt),
                     address=('{}'.format(self.katcp_clt),
                              '7147'),
                     controlled=True))
            self.rc.set_ioloop(self.io_manager.get_ioloop())
            self._rct = (resource_client.ThreadSafeKATCPClientResourceWrapper(self.rc,
                                                                              self.io_wrapper))
            self._rct.start()
            LOGGER.info('Cleanup function \'self._rct\': File: %s line: %s' % (
                getframeinfo(currentframe()).filename.split('/')[-1],
                getframeinfo(currentframe()).lineno))
            add_cleanup(self._rct.stop)
            try:
                self._rct.until_synced(timeout=_timeout)
            except TimeoutError:
                self._rct.stop()
        return self._rct

    @property
    def dhost(self, program=False):
        if self._dhost is not None:
            return self._dhost
        else:
            self.config_filename = '/etc/corr/{}-{}'.format(self.array_name, self.instrument)
            if os.path.exists(self.config_filename):
                LOGGER.info('Retrieving dsim engine info from config file: %s' % self.config_filename)
                self.corr_config = corr2.utils.parse_ini_file(self.config_filename)
                self.dsim_conf = self.corr_config['dsimengine']
            elif self.instrument is not None:
                self.corr2ini_path = '/etc/corr/templates/{}'.format(self.instrument)
                LOGGER.info('Setting CORR2INI system enviroment to point to %s' % self.corr2ini_path)
                os.environ['CORR2INI'] = self.corr2ini_path
                self.corr_config = corr2.utils.parse_ini_file(self.corr2ini_path)
                self.dsim_conf = self.corr_config['dsimengine']
            else:
                errmsg = ('Could not retrieve dsim information from running config file in /etc/corr, '
                          'Perhaps, restart CBF manually and ensure dsim is running.\n'
                          'File:%s Line:%s' % (getframeinfo(currentframe()).filename.split('/')[-1],
                                               getframeinfo(currentframe()).lineno))
                LOGGER.error(errmsg)
                sys.exit(errmsg)
            try:
                dig_host = self.dsim_conf['host']
                self._dhost = FpgaDsimHost(dig_host, config=self.dsim_conf)
            except Exception:
                errmsg = 'Digitiser Simulator failed to retrieve information'
                LOGGER.exception(errmsg)
                sys.exit(errmsg)
            else:
                # Check if D-eng is running else start it.
                if self._dhost.is_running():
                    LOGGER.info('D-Eng is already running.')
                # Disabled DSim programming as it would alter the systems sync epoch
                elif program and not self._dhost.is_running():
                    LOGGER.info('Programming and starting the Digitiser Simulator.')
                    self._dhost.initialise()
                    self._dhost.enable_data_output(enabled=True)
                    self._dhost.registers.control.write(gbe_txen=True)
                else:
                    LOGGER.info('D-Eng started succesfully')
                return self._dhost

    @property
    def correlator(self):
        if self._correlator is not None:
            LOGGER.info('Using cached correlator instance')
            return self._correlator
        else:  # include a check, if correlator is running else start it.
            if not self._correlator_started:
                LOGGER.info('Correlator not running, now starting.')
                self.start_correlator()

            # We assume either start_correlator() above has been called, or the
            # instrument was started with the name contained in self.array_name
            # before running the test.

            self.config_filename = '/etc/corr/{}-{}'.format(self.array_name, self.instrument)
            _retries = 3
            if os.path.exists(self.config_filename):
                LOGGER.info('Making new correlator instance')
                # for _retry in xrange(_retries):
                while True:
                    _retries -= 1
                    try:
                        self._correlator = fxcorrelator.FxCorrelator(
                            'test correlator', config_source=self.config_filename)
                        time.sleep(1)
                        self.correlator.initialise(program=False)
                        return self._correlator
                    except KatcpRequestFail as e:
                        LOGGER.exception('Did not get a response from roach/host: %s' %str(e))
                        continue
                    except Exception as e:
                        LOGGER.exception('Failed to create new correlator instance with error: %s, '
                            'Will now try to start correlator with config: %s-%s' % (str(e),
                                self.array_name, self.instrument))
                        continue
                    if _retries == 0:
                        break
                if _retries == 0:
                    self.start_correlator(instrument=self.instrument)
            else:
                LOGGER.error('No Config file (/etc/corr/array*-instrument), '
                             'Starting correlator with default instrument: %s' % (self.instrument))
                self.start_correlator(instrument=self.instrument)

    @property
    def halt_array(self):
        """
        Halting of primary and secondary katcp arrays and ensure that the correlator
        object is teared-down
        """
        LOGGER.info('Halting primary array: %s.' % self.array_name)
        try:
            reply, informs = self.katcp_rct.req.halt(timeout=_timeout)
            LOGGER.info(str(reply))
            assert reply.reply_ok()
            assert self._katcp_rct.is_active()
        except AssertionError:
            msg = 'Failed to halt katcp connection'
            LOGGER.error(msg)
        except AttributeError:
            raise RuntimeError('Failing to halt array, investigate halt array function.')

        self._katcp_rct.stop()
        self._katcp_rct = None

        try:
            reply, informs = self.rct.req.subordinate_list(timeout=_timeout)
            assert reply.reply_ok()
            if informs:
                informs = informs[0]
                if len(informs.arguments) >= 10 and self.array_name == informs.arguments[0]:
                    reply, informs = self.rct.req.subordinate_halt(self.array_name, timeout=_timeout)
                    assert reply.reply_ok()
        except AssertionError:
            msg = 'Failed to halt array: %s, STOPPING resource client' %self.array_name
            LOGGER.exception(msg)
        except IndexError:
            pass
        # if self.rct.is_active():
        #     self.rct.stop()
        #     self._rct.stop()
        #     self._rct = None
        #     self.rct = None

        self._correlator_started = False
        self._correlator = None
        LOGGER.info('Array %s halted and teared-down' % (self.array_name))
        time.sleep(self.halt_wait_time)

    @property
    def katcp_rct(self):
        try:
            katcp_prot = self.test_config['inst_param']['katcp_protocol']
        except TypeError:
            LOGGER.error('Failed to read katcp protocol from test config file')
        else:
            _major, _minor, _flags = katcp_prot.split(',')
            protocol_flags = ProtocolFlags(int(_major), int(_minor), _flags)
        multicast_ip = self.get_multicast_ips
        if not multicast_ip:
            LOGGER.error('Failed to calculate multicast IP\'s')
            self._katcp_rct = None
            return

        if self._katcp_rct is None:
            try:
                reply, informs = self.rct.req.subordinate_list(self.array_name)
            except TypeError:
                msg = 'Failed to list all arrays with name: %s' %self.array_name
                LOGGER.exception(msg)
            # If no sub-array present create one, but this could cause problems
            # if more than one sub-array is present. Update this to check for
            # required sub-array.
            if reply.reply_ok():
                self.katcp_array_port = int(informs[0].arguments[1])
            else:
                LOGGER.info('Array has not been assigned yet, will try to assign.'
                            ' File:%s Line:%s' % (
                                getframeinfo(currentframe()).filename.split('/')[-1],
                                getframeinfo(currentframe()).lineno))
                try:
                    reply, _informs = self.rct.req.subordinate_create(self.array_name,
                                                                *multicast_ip)
                    assert reply.reply_ok()
                except (ValueError, TypeError, AssertionError):
                    try:
                        reply, informs = self.rct.req.subordinate_list()
                        assert reply.reply_ok()
                        informs = informs[0]
                        if len(informs.arguments) >= 10 and self.array_name == informs.arguments[0]:
                            msg = 'Array assigned successfully: %s' %str(informs)
                            LOGGER.info(msg)
                        else:
                            LOGGER.error('Halting array.')
                            reply, informs = self.rct.req.subordinate_halt(self.array_name,
                                timeout=_timeout)
                            assert reply.reply_ok()
                    except AssertionError:
                        LOGGER.exception('Failed to assign multicast ip on array: %s: \n\nReply: %s' % (
                            self.array_name, str(reply)))
                else:
                    if len(reply.arguments) == 2:
                        try:
                            self.katcp_array_port = int(reply.arguments[-1])
                            LOGGER.info('Array %s assigned successfully' % (self.katcp_array_port))
                        except ValueError:
                            # self.rct.req.subordinate_halt(self.array_name)
                            # self.rct.stop()
                            # self.rct.start()
                            # self.rct.until_synced(timeout=_timeout)
                            # reply, informs = self.rct.req.subordinate_create(self.array_name,
                            # *multicast_ip)
                            errmsg = 'Investigate as to why this thing failed.'
                            LOGGER.exception(errmsg)
                            sys.exit(errmsg)

            katcp_rc = resource_client.KATCPClientResource(
                dict(name='{}'.format(self.katcp_clt),
                     address=('{}'.format(self.katcp_clt),
                              '{}'.format(self.katcp_array_port)),
                     preset_protocol_flags=protocol_flags, controlled=True))
            katcp_rc.set_ioloop(self.io_manager.get_ioloop())
            self._katcp_rct = (
                resource_client.ThreadSafeKATCPClientResourceWrapper(
                    katcp_rc, self.io_wrapper))
            self._katcp_rct.start()
            try:
                self._katcp_rct.until_synced(timeout=_timeout)
            except Exception as e:
                self._katcp_rct.stop()
                LOGGER.exception('Failed to connect to katcp due to %s' %str(e))
            LOGGER.info('Cleanup function \'self._katcp_rct\': File: %s line: %s' % (
                getframeinfo(currentframe()).filename.split('/')[-1],
                getframeinfo(currentframe()).lineno))
            # add_cleanup(self._katcp_rct.stop)
        else:
            self._katcp_rct.start()
            try:
                time.sleep(1)
                self._katcp_rct.until_synced(timeout=_timeout)
            except Exception as e:
                self._katcp_rct.stop()
                LOGGER.exception('Failed to connect to katcp due to %s' %str(e))
        return self._katcp_rct

    @property
    def issue_metadata(self):
        """Issue Spead metadata"""
        try:
            reply, informs = self.katcp_rct.req.capture_meta(self.product_name, timeout=_timeout)
            assert reply.reply_ok()
        except Exception:
            LOGGER.exception('Failed to issue new metadata: File:%s Line:%s' % (
                getframeinfo(currentframe()).filename.split('/')[-1],
                getframeinfo(currentframe()).lineno))
            return False
        else:
            return True

    @property
    def start_x_data(self):
        """
        Enable/Start output product capture
        """
        try:
            assert isinstance(self.katcp_rct, resource_client.ThreadSafeKATCPClientResourceWrapper)
            reply, informs = self.katcp_rct.req.capture_list(timeout=_timeout)
            assert reply.reply_ok()
            self.product_name = [i.arguments[0] for i in informs
                                   if self.corr_config['xengine']['output_products'] in i.arguments][0]
            assert self.product_name is not None
            LOGGER.info('Capturing %s product' %self.product_name)
        except Exception:
            self.product_name = self.corr_config['xengine']['output_products']
            LOGGER.exception('Failed to retrieve capture list via CAM interface, got it from config file.'
                             '\nFile:%s Line:%s' % (getframeinfo(currentframe()).filename.split('/')[-1],
                                getframeinfo(currentframe()).lineno))

        try:
            reply, informs = self.katcp_rct.req.capture_start(self.product_name)
            assert reply.reply_ok()
            LOGGER.info(' %s' % str(reply))
            Aqf.progress(str(reply)+'\n')
            return True
        except Exception:
            LOGGER.exception('Failed to capture start: %s' %str(reply))
            return False

    def stop_x_data(self):
        """
        Disable/Stop output product capture
        """
        try:
            assert self.product_name is not None
            assert isinstance(self.katcp_rct,
                              resource_client.ThreadSafeKATCPClientResourceWrapper)
            reply, informs = self.katcp_rct.req.capture_stop(self.product_name, timeout=_timeout)
            assert reply.reply_ok()
            LOGGER.info(' %s' %str(reply))
            Aqf.progress(str(reply))
            return True
        except Exception:
            LOGGER.exception('Failed to capture stop, might be because config file does not contain '
                             'Xengine output products.\n: File:%s Line:%s' % (
                                getframeinfo(currentframe()).filename.split('/')[-1],
                                getframeinfo(currentframe()).lineno))
            return False


    def get_running_instrument(self):
        """
        Returns currently running instrument listed on the sensor(s)
        """
        try:
            reply = None
            reply = self.katcp_rct.sensor.instrument_state.get_reading()
            assert reply.istatus
            return reply.value
        except Exception as e:
            LOGGER.exception('KATCP Request failed due to error: %s/%s'
                         '\n\t File:%s Line:%s' % (str(e), str(reply),
                             getframeinfo(currentframe()).filename.split('/')[-1],
                             getframeinfo(currentframe()).lineno))
            return False

        except KATCPSensorError:
            LOGGER.exception('KATCP Error polling sensor\n\t File:%s Line:%s' % (
                getframeinfo(currentframe()).filename.split('/')[-1],
                getframeinfo(currentframe()).lineno))
            return False
        except AssertionError:
            LOGGER.exception('Sensor request failed: %s , Forcefully Halting the Array\n\t '
                'File:%s Line:%s' % (str(reply), getframeinfo(currentframe()).filename.split('/')[-1],
                getframeinfo(currentframe()).lineno))
            return False

    def ensure_instrument(self, instrument, retries=5, force_reinit=False, **kwargs):
        """Ensure that named instrument is active on the correlator array

        Will pass `kwargs` to self.start_correlator if a start is required
        :param self: Object
        :param instrument: CBF Instrument
        :param retries: No of instrument validation retries
        :param force_reinit: Force an instrument re-initialisation
        :rtype: Boolean
        """
        self.instrument = instrument
        if force_reinit:
            LOGGER.info('Forcing an instrument(%s) re-initialisation' %self.instrument)
            corr_success = self.start_correlator(self.instrument, **kwargs)
            return corr_success

        success = False
        while retries and not success:
            check_ins = self.check_instrument(self.instrument)
            msg = 'Retries left to check instrument: %s' %retries
            LOGGER.info(msg)
            if check_ins is True:
                success = True
                LOGGER.info('Return true if named instrument is enabled on correlator array after '
                        ' %s retries' %retries)
                return success
            retries -= 1

        if self.check_instrument(self.instrument) is False:
            LOGGER.info('Correlator not running requested instrument, will restart.')
            reply = self.katcp_rct.sensor.instrument_state.get_reading()
            if reply.value == self.instrument:
                self.halt_array
            corr_success = self.start_correlator(self.instrument, **kwargs)
            return True if corr_success is True else False

    def check_instrument(self, instrument):
        """Return true if named instrument is enabled on correlator array

        Uses the correlator array KATCP interface to check if the requested
        instrument is active
        :param instrument: Correlator

        """
        self.instrument = instrument
        self._errmsg = None
        try:
            self._errmsg = 'Instrument cannot be None.'
            assert instrument is not None, self._errmsg
            self._errmsg = 'katcp client is not an instance of resource client'
            assert isinstance(self.katcp_rct,
                              resource_client.ThreadSafeKATCPClientResourceWrapper), self._errmsg
            self._errmsg = 'katcp client failed to establish a connection'
            assert self.katcp_rct.wait_connected(), self._errmsg
            self._errmsg = 'katcp client failed to sync after establishing a connection'
            assert self.katcp_rct.until_state('synced', timeout=60), self._errmsg
        except AssertionError:
            # This probably means that no array has been defined yet and therefore the
            # katcp_rct client cannot be created. IOW, the desired instrument would
            # not be available
            LOGGER.exception(self._errmsg)
            return False
        else:
            try:
                reply, informs = self.katcp_rct.req.instrument_list()
                assert reply.reply_ok()
                instruments_available = [instrument_avail.arguments[0] for instrument_avail in informs]
                # Test to see if requested instrument is available on the instrument list
                assert instrument in instruments_available
            except Exception:
                LOGGER.exception('Array request failed might have timedout\n\tFile:%s Line:%s' % (
                    getframeinfo(currentframe()).filename.split('/')[-1],
                    getframeinfo(currentframe()).lineno))
            except AssertionError:
                LOGGER.exception('Array request failed: %s or Instrument: %s is not in instrument '
                                 'list: %s \n\tFile:%s Line:%s' % (instrument,
                                    instruments_available, str(reply),
                                    getframeinfo(currentframe()).filename.split('/')[-1],
                                    getframeinfo(currentframe()).lineno))
                return False

            try:
                reply = self.katcp_rct.sensor.instrument_state.get_reading()
                assert reply.istatus == 1
            except AttributeError:
                LOGGER.exception('Instrument state could not be retrieved from the sensors')
                return False
            except AssertionError:
                LOGGER.error(' %s: Seems like the might be no current running instrument' %str(reply))
                return False

            else:
                running_intrument = reply.value
                instrument_present = instrument == running_intrument
                if instrument_present:
                    self.instrument = instrument
                    LOGGER.info('Confirmed that the named instrument %s is enabled on '
                                'correlator %s.' % (self.instrument, self.array_name))
                return instrument_present

    @property
    def _get_instrument(self):
        """
        Retrieve currently running instrument from /etc/corr
        return: List
        """
        try:
            # ToDo (MM) 06-10-2017 Hardcoded array, fix it
            running_instr = max(glob.iglob('/etc/corr/array0-*'), key=os.path.getctime).split('/')[-1]
            self.array_name, self.instrument = running_instr.split('-')
            if (self.instrument.startswith('bc') or self.instrument.startswith('c')) and \
                self.array_name.startswith('array'):
                LOGGER.info('Currenly running instrument %s as per /etc/corr' %running_instr)
                return running_instr.split('-')
        except Exception:
            LOGGER.exception('Could not retrieve information from config file, resorting to default:\n'
                             'File:%s Line:%s' % (getframeinfo(currentframe()).filename.split('/')[-1],
                            getframeinfo(currentframe()).lineno))
            return ['array0', 'bc8n856M4k']
        except ValueError:
            LOGGER.exception('Directory missing array config file.')

    @property
    def _test_config_file(self):
        """
        Configuration file containing information such as dsim, pdu and dataswitch ip's
        return: Dict
        """
        path, _none = os.path.split(__file__)
        path, _none = os.path.split(path)
        try:
            assert os.uname()[1].startswith('dbelab')
        except AssertionError:
            conf_path = 'config/test_conf_site.ini'
        else:
            conf_path = 'config/test_conf_lab.ini'

        config_file = os.path.join(path, conf_path)
        if os.path.isfile(config_file) or os.path.exists(config_file):
            try:
                config = corr2.utils.parse_ini_file(config_file)
                return config
            except (IOError, ValueError, TypeError):
                errmsg = ('Failed to read test config file %s, Test will exit'
                          '\n\t File:%s Line:%s' % (config_file,
                              getframeinfo(currentframe()).filename.split('/')[-1],
                              getframeinfo(currentframe()).lineno))
                LOGGER.exception(errmsg)
                return False
        else:
            LOGGER.error('Test config path: %s does not exist' % config_file)
            return False

    @property
    def get_multicast_ips(self):
        """
        Retrieves multicast ips from test configuration file and calculates the number
        of inputs depending on which instrument is being initialised
        :param instrument: Correlator
        """
        if self.instrument is None:
            return False
        try:
            multicast_ip_inp = (
                self.test_config['inst_param']['source_mcast_ips'].split(','))
        except TypeError:
            msg = ('Could not read and split the multicast ip\'s in the test config file')
            LOGGER.exception(msg)
            return False
        else:
            if self.instrument.startswith('bc') or self.instrument.startswith('c'):
                if self.instrument[0] == 'b':
                    try:
                        # multicast_ip = multicast_ip_inp * (int(self.instrument[2]) / 2)
                        multicast_ip = multicast_ip_inp * (
                            int(self.instrument.replace('bc', '').split('n')[0]) / 2)
                        return multicast_ip
                    except Exception:
                        LOGGER.error('Could not calculate multicast ips from config file')
                        return False
                else:
                    try:
                        # multicast_ip = multicast_ip_inp * (int(self.instrument[1]) / 2)
                        multicast_ip = multicast_ip_inp * (
                            int(self.instrument.replace('c', '').split('n')[0]) / 2)
                        return multicast_ip
                    except Exception:
                        LOGGER.error('Could not calculate multicast ips from config file')
                        return False


    @property
    def subscribe_multicast(self):
        """Automated multicasting subscription"""
        parse_address = StreamAddress._parse_address_string
        try:
            n_xengs = self.katcp_rct.sensor.n_xengs.get_value()
        except Exception:
            n_xengs = len(self.get_multicast_ips) * 2

        if self.config_filename is None:
            return
        config = self.corr_config
        if config is None:
            LOGGER.error('Failed to retrieve correlator config file, ensure that the cbf is running')
            return False

        def confirm_multicast_subs(mul_ip='239.100.0.10'):
            """"""
            # or use [netstat -g | grep eth2]
            list_inets = subprocess.check_output(['ip', 'maddr', 'show'])
            return True if mul_ip in list_inets else False

        outputIPs = {}
        for i in [key for key, value in config.items() if 'output_destinations_base' in value]:
            _IP, _num, _Port = list(parse_address(config[i]['output_destinations_base']))
            outputIPs[i] = [tengbe.IpAddress(_IP), int(_Port)]
        if outputIPs.get('xengine'):
            LOGGER.info('Multicast subscription is only valid for xengines')
            multicastIP, DataPort = outputIPs.get('xengine')
            if multicastIP.is_multicast():
                LOGGER.info('source is multicast %s.' % (multicastIP))
                # look up multicast group address in name server and find out IP version
                addrinfo = socket.getaddrinfo(str(multicastIP), None)[0]
                # create a socket
                try:
                    mcast_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
                except Exception:
                    mcast_sock = socket.socket(addrinfo[0], socket.SOCK_DGRAM)
                mcast_sock.setblocking(False)
                # Do not bind as this will cause a conflict when instantiating a receiver on the main script
                # Join group
                # mcast_sock.bind(('', DataPort))
                add_cleanup(mcast_sock.close)
                def join_mcast_group(address):
                    group_bin = socket.inet_pton(socket.AF_INET, address)
                    if addrinfo[0] == socket.AF_INET:  # IPv4
                        mreq = group_bin + struct.pack('=I', socket.INADDR_ANY)
                        mcast_sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
                        LOGGER.info('Successfully subscribed to %s:%s.' % (str(multicastIP), DataPort))
                    else:
                        mreq = group_bin + struct.pack('@I', 0)
                        mcast_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
                        LOGGER.info('Successfully subscribed to %s:%s.' % (str(multicastIP), DataPort))
                for addcntr in range(n_xengs):
                    _address = tengbe.IpAddress(multicastIP.ip_int + addcntr)
                    join_mcast_group(str(_address))
            else:
                mcast_sock = None
                LOGGER.info('Source is not multicast: %s:%s' % (str(multicastIP), DataPort))
                return False
        return confirm_multicast_subs(mul_ip=str(_address))

    def start_correlator(self, instrument=None, retries=10):
        LOGGER.debug('CBF instrument(%s) re-initialisation.' %instrument)
        success = False
        self.katcp_array_port = None
        if instrument is not None:
            self.instrument = instrument
        self._correlator = None  # Invalidate cached correlator instance
        LOGGER.info('Confirm DEngine is running before starting correlator')
        if not self.dhost.is_running():
            raise RuntimeError('DEngine: %s not running.' % (self.dhost.host))

        multicast_ip = self.get_multicast_ips
        if not multicast_ip:
            LOGGER.error('Failed to calculate multicast IP\'s, investigate')
            self._katcp_rct = None
        self.rct.start()
        try:
            self.rct.until_synced(timeout=_timeout)
            reply, informs = self.rct.req.subordinate_list(self.array_name)
            assert reply.reply_ok()
        except TimeoutError:
            self.rct.stop()
            LOGGER.exception('Resource client timed-out after %s s' % _timeout)
            return False
        except AssertionError:
            LOGGER.exception('Failed to get subordinate-list, might not have been assigned, '
                             'Will try to assign. \n\t File:%s Line:%s' % (
                               getframeinfo(currentframe()).filename.split('/')[-1],
                               getframeinfo(currentframe()).lineno))
        else:
            try:
                informs = informs[0]
                self.katcp_array_port = int(
                    [i for i in informs.arguments if len(i) == 5][0])
            except ValueError:
                LOGGER.exception('Failed to assign katcp port: Reply: %s' % (reply))
                reply = self.rct.req.subordinate_halt(self.array_name)
                if not reply.succeeded:
                    LOGGER.error('Unable to halt array %s: %s \n\t File:%s Line:%s' % (
                        self.array_name, reply,
                        getframeinfo(currentframe()).filename.split('/')[-1],
                        getframeinfo(currentframe()).lineno))
                    return False

        while retries and not success:
            try:
                if self.katcp_array_port is None:
                    LOGGER.info('Assigning array port number')
                    # self.rct.start()
                    # self.rct.until_synced(timeout=_timeout)
                    try:
                        reply, _informs = self.rct.req.subordinate_create(self.array_name,
                                                                    *multicast_ip, timeout=_timeout)
                        assert reply.reply_ok()
                    except Exception:
                        self.katcp_array_port = None
                        LOGGER.exception('Failed to assign new array: %s' %self.array_name)
                    else:
                        self.katcp_array_port = int(reply.arguments[-1])
                        LOGGER.info('Successfully created %s-%s' % (self.array_name, self.katcp_array_port))

                # try:
                #     #self.katcp_array_port = int(reply.arguments[-1])
                #     LOGGER.info('Array port assigned: {}'.format(self.katcp_array_port))
                # except ValueError:
                #     LOGGER.fatal('Failed to assign array port number on {}'.format(
                #           self.array_name))
                # else:
                #     if not reply.reply_ok():
                #         LOGGER.fatal('Failed to assign array port number on {}'.format(
                #                self.array_name))
                #         return False

                instrument_param = (
                    [int(i) for i in self.test_config['inst_param']['instrument_param'] if i != ','])
                LOGGER.info('Starting %s with %s parameters. Try #%s' % (self.instrument,
                                                                         instrument_param,
                                                                         retries))

                reply = self.katcp_rct.req.instrument_activate(self.instrument, *instrument_param,
                                                               timeout=500)
                success = reply.succeeded
                retries -= 1

                try:
                    assert success
                    LOGGER.info('Instrument %s started succesfully' % (self.instrument))
                except AssertionError:
                    LOGGER.exception('Failed to start correlator, %s attempts left. '
                                     'Restarting Correlator. Reply:%s' % (retries, reply))
                    self.halt_array
                    success = False


            except Exception:
                try:
                    self.rct.req.subordinate_halt(self.array_name)
                    assert isinstance(self.katcp_rct,
                                      resource_client.ThreadSafeKATCPClientResourceWrapper)
                except Exception:
                    LOGGER.exception('Unable to halt array: Empty Array number: '
                                 'File:%s Line:%s' % (
                                     getframeinfo(currentframe()).filename.split('/')[-1],
                                     getframeinfo(currentframe()).lineno))
                except AssertionError:
                    LOGGER.exception('self.katcp_rct has not been initiated successfully')
                    return False
                else:
                    try:
                        self.katcp_rct.stop()
                    except AttributeError:
                        LOGGER.error('KATCP request does not contain attributes: File:%s Line:%s' % (
                            getframeinfo(currentframe()).filename.split('/')[-1],
                            getframeinfo(currentframe()).lineno))
                        return False
                    else:
                        retries -= 1
                        LOGGER.warn('Failed to start correlator, %s attempts left.\n' % (retries))

            if retries < 0:
                success = False
                return success

        if success:
            self._correlator_started = True
            return self._correlator_started
        else:
            try:
                self.halt_array
                self._correlator_started = False
                self.katcp_rct.stop()
                self.rct.stop()
                self._katcp_rct = None
                self._correlator = None
            except:
                self.halt_array
                msg = ('Could not successfully start correlator within %s retries' % (retries))
                LOGGER.critical(msg)
                return False
            return False
예제 #7
0
def SpeadRx(config_file, rx_port, dsim_start, capture_start, debug, verbose):
    """
    Receive data from Correlator and play
    """
    if config_file is None:
        sys.exit("Usage: %s --help" % __file__)

    try:
        assert verbose
        _level = 'DEBUG'
        logging.basicConfig(
            level=logging.DEBUG,
            format=
            '%(asctime)s - %(name)s - %(levelname)s - %(module)s - %(pathname)s : '
            '%(lineno)d - %(message)s')
        corr_rx_logger.setLevel(logging.DEBUG)
        spead2_logger.setLevel(logging.DEBUG)
        logging.debug('DEBUG MODE ENABLED')
    except:
        _level = 'INFO'
        logging.basicConfig(
            level=logging.INFO,
            format=
            '%(asctime)s - %(name)s - %(levelname)s - %(module)s - %(pathname)s : '
            '%(lineno)d - %(message)s')
        corr_rx_logger.setLevel(logging.INFO)
        spead2_logger.setLevel(logging.INFO)
    finally:
        logger = logging.getLogger(__name__)
        coloredlogs.install(level=_level, logger=logger)

    if dsim_start and config_file:
        corr_conf = utils.parse_ini_file(config_file, ['dsimengine'])
        dsim_conf = corr_conf['dsimengine']
        dhost = FpgaDsimHost(dsim_conf['host'], config=dsim_conf)
        try:
            assert dhost.is_running()
            dhost.get_system_information()
        except:
            logger.error('DEngine Not Running!')

    try:
        assert capture_start
    except AssertionError:

        @atexit.register
        def Cleanup():
            logger.info('baseline-correlation-products capture stopped!')
            receiver.stop()
            receiver.join()
            logger.info('Receiver stopped.')
            fab.local(
                "kcpcmd -t 60 -s localhost:$(kcpcmd array-list | grep -a array-list | cut -f3 -d ' ' ) capture-stop baseline-correlation-products"
            )
    else:
        logger.info('baseline-correlation-products capture started!')
        fab.local(
            "kcpcmd -t 60 -s localhost:$(kcpcmd array-list | grep -a array-list | cut -f3 -d ' ' ) capture-start baseline-correlation-products"
        )

    try:
        receiver = CorrRx(port=rx_port, queue_size=5)
        _multicast_ips = corr_conf['xengine'].get(
            'multicast_interface_address', '239.100.0.1')
        # import IPython; IPython.embed(header='Python Debugger')
    except Exception as ex:
        template = "An exception of type {0} occured while trying to instantiate receiver. Arguments:\n{1!r}"
        message = template.format(type(ex), ex.args)
        logger.info(message)
    else:
        logger.info('Waiting for receiver to report running')
        receiver.daemon = True
        receiver.start(timeout=10)
        if receiver.running_event.wait(timeout=10):
            logger.info('Receiver ready')
        else:
            msg = 'Receiver not ready'
            logger.info(msg)
            raise RuntimeError(msg)
        try:
            raw_input('Press Enter get clean dump')
            dump = receiver.get_clean_dump()
        except KeyboardInterrupt:
            logger.info('Keyboard interrupt')
        except Exception:
            raise
        else:
            logger.info('Dump received')
        if debug:
            corr_rx_logger.setLevel(logging.FATAL)
            spead2_logger.setLevel(logging.FATAL)
            import IPython
            IPython.embed(header='Python Debugger')
예제 #8
0
class test_CBF(unittest.TestCase):
    def setUp(self):
        self.receiver = CorrRx(port=8888)
        start_thread_with_cleanup(self, self.receiver, start_timeout=1)
        self.correlator = correlator_fixture.correlator
        self.corr_fix = correlator_fixture
        self.corr_freqs = CorrelatorFrequencyInfo(self.correlator.configd)
        dsim_conf = self.correlator.configd['dsimengine']
        dig_host = dsim_conf['host']
        self.dhost = FpgaDsimHost(dig_host, config=dsim_conf)
        self.dhost.get_system_information()
        # Increase the dump rate so tests can run faster
        self.correlator.xeng_set_acc_time(0.2)
        self.addCleanup(self.corr_fix.stop_x_data)
        self.corr_fix.start_x_data()
        self.corr_fix.issue_metadata()

    # TODO 2015-05-27 (NM) Do test using get_vacc_offset(test_dump['xeng_raw']) to see if
    # the VACC is rotated. Run this test first so that we know immediately that other
    # tests will be b0rked.

    def test_channelisation(self):
        """TP.C.1.19 CBF Channelisation Wideband Coarse L-band"""
        test_chan = 1500
        expected_fc = self.corr_freqs.chan_freqs[test_chan]

        init_dsim_sources(self.dhost)
        self.dhost.sine_sources.sin_0.set(frequency=expected_fc, scale=0.25)
        # The signal source is going to quantise the requested freqency, so see what we
        # actually got
        source_fc = self.dhost.sine_sources.sin_0.frequency

        # Get baseline 0 data, i.e. auto-corr of m000h
        test_baseline = 0
        test_data = self.receiver.get_clean_dump(DUMP_TIMEOUT)['xeng_raw']
        b_mag = normalised_magnitude(test_data[:, test_baseline, :])
        # find channel with max power
        max_chan = np.argmax(b_mag)
        # self.assertEqual(max_chan, test_chan,
        #                  'Channel with max power is not the test channel')

        requested_test_freqs = self.corr_freqs.calc_freq_samples(
            test_chan, samples_per_chan=101, chans_around=5)

        # Placeholder of actual frequencies that the signal generator produces
        actual_test_freqs = []
        # Channel magnitude responses for each frequency
        chan_responses = []
        last_source_freq = None
        for i, freq in enumerate(requested_test_freqs):
            # LOGGER.info('Getting channel response for freq {}/{}: {} MHz.'.format(
            #     i+1, len(requested_test_freqs), freq/1e6))
            print ('Getting channel response for freq {}/{}: {} MHz.'.format(
                i+1, len(requested_test_freqs), freq/1e6))
            if freq == expected_fc:
                # We've already done this one!
                this_source_freq = source_fc
                this_freq_result = b_mag
            else:
                self.dhost.sine_sources.sin_0.set(frequency=freq, scale=0.125)
                this_source_freq = self.dhost.sine_sources.sin_0.frequency
                if this_source_freq == last_source_freq:
                    LOGGER.info('Skipping channel response for freq {}/{}: {} MHz.\n'
                                'Digitiser frequency is same as previous.'.format(
                                    i+1, len(requested_test_freqs), freq/1e6))
                    continue    # Already calculated this one
                else:
                    last_source_freq = this_source_freq
                this_freq_data = self.receiver.get_clean_dump(DUMP_TIMEOUT)['xeng_raw']
                this_freq_response = normalised_magnitude(
                    this_freq_data[:, test_baseline, :])
            actual_test_freqs.append(this_source_freq)
            chan_responses.append(this_freq_response)
        self.corr_fix.stop_x_data()

        # Convert the lists to numpy arrays for easier working
        actual_test_freqs = np.array(actual_test_freqs)
        chan_responses = np.array(chan_responses)

        def plot_and_save(freqs, data, plot_filename):
            df = self.corr_freqs.delta_f
            fig = plt.plot(freqs, data)[0]
            axes = fig.get_axes()
            ybound = axes.get_ybound()
            yb_diff = abs(ybound[1] - ybound[0])
            new_ybound = [ybound[0] - yb_diff*1.1, ybound[1] + yb_diff * 1.1]
            plt.vlines(expected_fc, *new_ybound, colors='r', label='chan fc')
            plt.vlines(expected_fc - df / 2, *new_ybound, label='chan min/max')
            plt.vlines(expected_fc - 0.8*df / 2, *new_ybound, label='chan +-40%',
                       linestyles='dashed')
            plt.vlines(expected_fc + df / 2, *new_ybound, label='_chan max')
            plt.vlines(expected_fc + 0.8*df / 2, *new_ybound, label='_chan +40%',
                       linestyles='dashed')
            plt.legend()
            plt.title('Channel {} ({} MHz) response'.format(
                test_chan, expected_fc/1e6))
            axes.set_ybound(*new_ybound)
            plt.grid(True)
            plt.ylabel('dB relative to VACC max')
            # TODO Normalise plot to frequency bins
            plt.xlabel('Frequency (Hz)')
            plt.savefig(plot_filename)
            plt.close()

        graph_name = '{}.{}.channel_response.svg'.format(strclass(self.__class__),
                                                         self._testMethodName)
        plot_data_all  = loggerise(chan_responses[:, test_chan], dynamic_range=90)
        plot_and_save(actual_test_freqs, plot_data_all, graph_name)

        # Get responses for central 80% of channel
        df = self.corr_freqs.delta_f
        central_indices = (
            (actual_test_freqs <= expected_fc + 0.8*df) &
            (actual_test_freqs >= expected_fc - 0.8*df))
        central_chan_responses = chan_responses[central_indices]
        central_chan_test_freqs = actual_test_freqs[central_indices]

        # Test responses in central 80% of channel
        for i, freq in enumerate(central_chan_test_freqs):
            max_chan = np.argmax(np.abs(central_chan_responses[i]))
            self.assertEqual(max_chan, test_chan, 'Source freq {} peak not in channel '
                             '{} as expected but in {}.'
                             .format(freq, test_chan, max_chan))

        # TODO Graph the central 80%  too.
        self.assertLess(
            np.max(np.abs(central_chan_responses[:, test_chan])), 0.99,
            'VACC output at > 99% of maximum value, indicates that '
            'something, somewhere, is probably overranging.')
        max_central_chan_response = np.max(10*np.log10(central_chan_responses[:, test_chan]))
        min_central_chan_response = np.min(10*np.log10(central_chan_responses[:, test_chan]))
        chan_ripple = max_chan_response - min_chan_response
        acceptable_ripple_lt = 0.3


        self.assertLess(chan_ripple, acceptable_ripple_lt,
                        'ripple {} dB within 80% of channel fc is >= {} dB'
                        .format(chan_ripple, acceptable_ripple_lt))

        # from matplotlib import pyplot
        # colour_cycle = 'rgbyk'
        # style_cycle = ['-', '--']
        # linestyles = itertools.cycle(itertools.product(style_cycle, colour_cycle))
        # for i, freq in enumerate(actual_test_freqs):
        #     style, colour = linestyles.next()
        #     pyplot.plot(loggerise(chan_responses[:, i], dynamic_range=60), color=colour, ls=style)
        # pyplot.ion()
        # pyplot.show()
        # import IPython ; IPython.embed()

    def test_product_baselines(self):
        """CBF Baseline Correlation Products: VR.C.19, TP.C.1.3"""

        init_dsim_sources(self.dhost)
        # Put some correlated noise on both outputs
        self.dhost.noise_sources.noise_corr.set(scale=0.5)
        test_dump = self.receiver.get_clean_dump(DUMP_TIMEOUT)

        # Get list of all the correlator input labels
        input_labels = sorted(tuple(test_dump['input_labelling'][:,0]))
        # Get list of all the baselines present in the correlator output
        present_baselines = sorted(
            set(tuple(bl) for bl in test_dump['bls_ordering']))

        # Make a list of all possible baselines (including redundant baselines) for the
        # given list of inputs
        possible_baselines = set()
        for li in input_labels:
            for lj in input_labels:
                possible_baselines.add((li, lj))

        test_bl = sorted(list(possible_baselines))
        # Test that each baseline (or its reverse-order counterpart) is present in the
        # correlator output
        baseline_is_present = {}

        for test_bl in possible_baselines:
           baseline_is_present[test_bl] = (test_bl in present_baselines or
                                           test_bl[::-1] in present_baselines)
        self.assertTrue(all(baseline_is_present.values()),
                        "Not all baselines are present in correlator output.")

        test_data = test_dump['xeng_raw']
        # Expect all baselines and all channels to be non-zero
        self.assertFalse(zero_baselines(test_data))
        self.assertEqual(nonzero_baselines(test_data),
                         all_nonzero_baselines(test_data))

        # Save initial f-engine equalisations
        initial_equalisations = {input: eq_info['eq'] for input, eq_info
                                in self.correlator.feng_eq_get().items()}
        def restore_initial_equalisations():
            for input, eq in initial_equalisations.items():
                self.correlator.feng_eq_set(source_name=input, new_eq=eq)
        self.addCleanup(restore_initial_equalisations)

        # Set all inputs to zero, and check that output product is all-zero
        for input in input_labels:
            self.correlator.feng_eq_set(source_name=input, new_eq=0)
        test_data = self.receiver.get_clean_dump(DUMP_TIMEOUT)['xeng_raw']
        self.assertFalse(nonzero_baselines(test_data))
        #-----------------------------------
        all_inputs = sorted(set(input_labels))
        zero_inputs = set(input_labels)
        nonzero_inputs = set()

        def calc_zero_and_nonzero_baselines(nonzero_inputs):
            nonzeros = set()
            zeros = set()
            for inp_i in all_inputs:
                for inp_j in all_inputs:
                    if inp_i in nonzero_inputs and inp_j in nonzero_inputs:
                        nonzeros.add((inp_i, inp_j))
                    else:
                        zeros.add((inp_i, inp_j))
            return zeros, nonzeros

        #zero_baseline, nonzero_baseline = calc_zero_and_nonzero_baselines(nonzero_inputs)
        def print_baselines():
            print ('zeros: {}\n\nnonzeros: {}\n\nnonzero-baselines: {}\n\n '
                'zero-baselines: {}\n\n'.format(
                    sorted(zero_inputs), sorted(nonzero_inputs),
                    sorted(nonzero_baseline), sorted(zero_baseline)))
        #print_baselines()
        for inp in input_labels:
            old_eqs = initial_equalisations[inp]
            self.correlator.feng_eq_set(source_name=inp, new_eq=old_eqs)
            zero_inputs.remove(inp)
            nonzero_inputs.add(inp)
예제 #9
0
        raise RuntimeError('No such log level: %s' % log_level)

if args.config:
    config_filename = args.config
else:
    try:
        config_filename = os.environ['CORR2INI']
    except KeyError:
        raise RuntimeError(
            'No config file speficied and environment var "CORR2INI" not defined')

corr_conf = utils.parse_ini_file(config_filename, ['dsimengine'])
dsim_conf = corr_conf['dsimengine']
dig_host = dsim_conf['host']

dhost = FpgaDsimHost(dig_host, config=dsim_conf)
print 'Connected to %s.' % dhost.host

if args.start and args.stop:
    raise RuntimeError('Start and stop? You must be crazy!')

something_happened = False
if args.program:
    dhost.initialise()
    something_happened = True
else:
    dhost.get_system_information()

if args.deprogram:
    dhost.deprogram()
    something_happened = True
예제 #10
0
# make the fpga
# dfpga = utils.script_get_fpga(args, host_index=0, host_type='dsimengine',
#                               fpga_class=FpgaDsimHost)
if 'CORR2INI' in os.environ.keys() and (args.config == '' or args.config is None):
    args.config = os.environ['CORR2INI']

try:
    config, host_detail = utils.hosts_and_bitstreams_from_config(
        config_file=args.config, section='dsimengine')
    section, host_list, bitstream = host_detail[0]
    if args.bitstream:
        bitstream = str(args.bitstream)
        print ('Starting Digitiser with bitstream: {}'.format(bitstream))
    # Note: We have hardcoded the transport to be SkarabTransport - Might change in the future!
    dfpga = FpgaDsimHost(host_list[0], bitstream=bitstream, config=config.get('dsimengine'),
        transport=SkarabTransport)
    print('Connected to {}.'.format(dfpga.host))
except TypeError:
    raise RuntimeError('Config template was not parsed!!!')

if args.start and args.stop:
    raise RuntimeError('Start and stop? You must be crazy!')

something_happened = False
if args.program:
    dfpga.initialise()
    something_happened = True
else:
    dfpga.get_system_information(bitstream)

# # TODO HACK
예제 #11
0
class CorrelatorFixture(object):
    def __init__(self, katcp_client=None, product_name=None):
        self.katcp_client = katcp_client
        self.corr_config = None
        self.corr2ini_path = None
        self._correlator = None
        self._dhost = None
        self._katcp_rct = None
        self._katcp_rct_sensor = None
        self._rct = None
        self.katcp_array_port = None
        self.katcp_sensor_port = None
        self.product_name = product_name
        self.halt_wait_time = 5
        # Assume the correlator is already started if start_correlator is False
        nose_test_config = {}
        self._correlator_started = not int(
            nose_test_config.get('start_correlator', False))
        self.test_config = self._test_config_file
        self.subarray = self.test_config['instrument_params']['subarray']
        self.config_filename = max(iglob('/etc/corr/{}-*'.format(
            self.subarray)),
                                   key=os.path.getctime)
        self.array_name, self.instrument = self._get_instrument

    @property
    def rct(self):
        if self._rct is not None:
            return self._rct
        else:
            self.io_manager = ioloop_manager.IOLoopManager()
            self.io_wrapper = resource_client.IOLoopThreadWrapper(
                self.io_manager.get_ioloop())
            add_cleanup(self.io_manager.stop)
            self.io_wrapper.default_timeout = _timeout
            self.io_manager.start()
            self.rc = resource_client.KATCPClientResource(
                dict(name='{}'.format(self.katcp_client),
                     address=('{}'.format(self.katcp_client), '7147'),
                     controlled=True))
            self.rc.set_ioloop(self.io_manager.get_ioloop())
            self._rct = (resource_client.ThreadSafeKATCPClientResourceWrapper(
                self.rc, self.io_wrapper))
            self._rct.start()
            add_cleanup(self._rct.stop)
            try:
                self._rct.until_synced(timeout=_timeout)
            except TimeoutError:
                self._rct.stop()
        return self._rct

    @property
    def dhost(self, program=False):
        if self._dhost is not None:
            return self._dhost
        else:
            if os.path.exists(self.config_filename):
                LOGGER.info(
                    'Retrieving dsim engine info from config file: %s' %
                    self.config_filename)
                self.corr_config = parse_ini_file(self.config_filename)
                self.dsim_conf = self.corr_config['dsimengine']
            elif self.instrument is not None:
                self.corr2ini_path = '/etc/corr/templates/{}'.format(
                    self.instrument)
                LOGGER.info(
                    'Setting CORR2INI system environment to point to %s' %
                    self.corr2ini_path)
                os.environ['CORR2INI'] = self.corr2ini_path
                self.corr_config = parse_ini_file(self.corr2ini_path)
                self.dsim_conf = self.corr_config['dsimengine']
            else:
                errmsg = (
                    'Could not retrieve dsim information from running config file in /etc/corr, '
                    'Perhaps, restart CBF manually and ensure dsim is running.\n'
                )
                LOGGER.error(errmsg)
                sys.exit(errmsg)
            try:
                dig_host = self.dsim_conf['host']
                self._dhost = None
                self._dhost = FpgaDsimHost(dig_host, config=self.dsim_conf)
            except Exception:
                errmsg = 'Digitiser Simulator failed to retrieve information'
                LOGGER.exception(errmsg)
                sys.exit(errmsg)
            else:
                # Check if D-eng is running else start it.
                if self._dhost.is_running():
                    LOGGER.info('D-Eng is already running.')
                    return self._dhost
                # Disabled DSim programming as it would alter the systems sync epoch
                # elif program and not self._dhost.is_running():
                #     LOGGER.info('Programming and starting the Digitiser Simulator.')
                #     self._dhost.initialise()
                #     self._dhost.enable_data_output(enabled=True)
                #     self._dhost.registers.control.write(gbe_txen=True)
                # else:
                #     LOGGER.info('D-Eng started successfully')
                # return self._dhost

    @property
    def correlator(self):
        if self._correlator is not None:
            LOGGER.info('Using cached correlator instance')
            return self._correlator
        else:  # include a check, if correlator is running else start it.
            if not self._correlator_started:
                LOGGER.info('Correlator not running, now starting.')
                print 'Correlator not running: This shouldnt happen, fix it\n' * 10
                #self.start_correlator()

            # We assume either start_correlator() above has been called, or the
            # instrument was started with the name contained in self.array_name
            # before running the test.
            _retries = 3
            if os.path.exists(self.config_filename):
                LOGGER.info('Making new correlator instance')
                while True:
                    _retries -= 1
                    try:
                        self._correlator = fxcorrelator.FxCorrelator(
                            'test correlator',
                            config_source=self.config_filename)
                        time.sleep(1)
                        try:
                            self.correlator.initialise(program=False,
                                                       configure=False)
                        except TypeError:
                            self.correlator.initialise(program=False)
                        return self._correlator
                    except Exception as e:
                        LOGGER.exception(
                            'Failed to create new correlator instance with error: %s, '
                            'Will now try to start correlator with config: %s-%s'
                            % (str(e), self.array_name, self.instrument))
                        continue
                    if _retries == 0:
                        break
                if _retries == 0:
                    self.start_correlator(instrument=self.instrument)
            else:
                LOGGER.error(
                    'No Config file (/etc/corr/array*-instrument), '
                    'Starting correlator with default instrument: %s' %
                    (self.instrument))
                self.start_correlator(instrument=self.instrument)

    @property
    def katcp_rct(self):
        if self._katcp_rct is None:
            try:
                katcp_prot = self.test_config['instrument_params'][
                    'katcp_protocol']
                _major, _minor, _flags = katcp_prot.split(',')
                protocol_flags = ProtocolFlags(int(_major), int(_minor),
                                               _flags)
                LOGGER.info('katcp protocol flags %s' % protocol_flags)

                LOGGER.info('Getting running array.')
                reply, informs = self.rct.req.subordinate_list(self.array_name)
                assert reply.reply_ok()
                # If no sub-array present create one, but this could cause problems
                # if more than one sub-array is present. Update this to check for
                # required sub-array.
            except Exception:
                LOGGER.exception('Failed to list all arrays with name: %s' %
                                 self.array_name)
            else:
                try:
                    try:
                        self.katcp_array_port = int(informs[0].arguments[1])
                        LOGGER.info(
                            'Current running array name: %s, port: %s' %
                            (self.array_name, self.katcp_array_port))
                    except ValueError:
                        self.katcp_array_port, self.katcp_sensor_port = informs[
                            0].arguments[1].split(',')
                        LOGGER.info(
                            'Current running array name: %s, port: %s, sensor port: %s'
                            % (self.array_name, self.katcp_array_port,
                               self.katcp_sensor_port))
                except Exception:
                    errmsg = (
                        'Failed to retrieve running array, ensure one has been created and running'
                    )
                    LOGGER.exception(errmsg)
                    sys.exit(errmsg)
                else:
                    katcp_rc = resource_client.KATCPClientResource(
                        dict(name='{}'.format(self.katcp_client),
                             address=('{}'.format(self.katcp_client),
                                      '{}'.format(self.katcp_array_port)),
                             preset_protocol_flags=protocol_flags,
                             controlled=True))
                    katcp_rc.set_ioloop(self.io_manager.get_ioloop())
                    self._katcp_rct = (
                        resource_client.ThreadSafeKATCPClientResourceWrapper(
                            katcp_rc, self.io_wrapper))
                    self._katcp_rct.start()
                    try:
                        self._katcp_rct.until_synced(timeout=_timeout)
                    except Exception as e:
                        self._katcp_rct.stop()
                        LOGGER.exception(
                            'Failed to connect to katcp due to %s' % str(e))
                    else:
                        return self._katcp_rct
        else:
            if not self._katcp_rct.is_active():
                LOGGER.info(
                    'katcp resource client wasnt running, hence we need to start it.'
                )
                self._katcp_rct.start()
                try:
                    time.sleep(1)
                    self._katcp_rct.until_synced(timeout=_timeout)
                    return self._katcp_rct
                except Exception:
                    self._katcp_rct.stop()
                    LOGGER.exception('Failed to connect to katcp')
            else:
                return self._katcp_rct

    @property
    def katcp_rct_sensor(self):
        if self._katcp_rct_sensor is None:
            try:
                katcp_prot = self.test_config['instrument_params'][
                    'katcp_protocol']
                _major, _minor, _flags = katcp_prot.split(',')
                protocol_flags = ProtocolFlags(int(_major), int(_minor),
                                               _flags)
                LOGGER.info('katcp protocol flags %s' % protocol_flags)

                LOGGER.info('Getting running array.')
                reply, informs = self.rct.req.subordinate_list(self.array_name)
                assert reply.reply_ok()
                # If no sub-array present create one, but this could cause problems
                # if more than one sub-array is present. Update this to check for
                # required sub-array.
            except Exception:
                LOGGER.exception('Failed to list all arrays with name: %s' %
                                 self.array_name)
            else:
                try:
                    try:
                        self.katcp_array_port = int(informs[0].arguments[1])
                        LOGGER.info(
                            'Current running array name: %s, port: %s' %
                            (self.array_name, self.katcp_array_port))
                    except ValueError:
                        self.katcp_array_port, self.katcp_sensor_port = informs[
                            0].arguments[1].split(',')
                        LOGGER.info(
                            'Current running array name: %s, port: %s, sensor port: %s'
                            % (self.array_name, self.katcp_array_port,
                               self.katcp_sensor_port))
                except Exception:
                    errmsg = (
                        'Failed to retrieve running array, ensure one has been created and running'
                    )
                    LOGGER.exception(errmsg)
                    sys.exit(errmsg)
                else:
                    katcp_rc = resource_client.KATCPClientResource(
                        dict(name='{}'.format(self.katcp_client),
                             address=('{}'.format(self.katcp_client),
                                      '{}'.format(self.katcp_sensor_port)),
                             preset_protocol_flags=protocol_flags,
                             controlled=True))
                    katcp_rc.set_ioloop(self.io_manager.get_ioloop())
                    self._katcp_rct_sensor = (
                        resource_client.ThreadSafeKATCPClientResourceWrapper(
                            katcp_rc, self.io_wrapper))
                    self._katcp_rct_sensor.start()
                    try:
                        self._katcp_rct_sensor.until_synced(timeout=_timeout)
                    except Exception as e:
                        self._katcp_rct_sensor.stop()
                        LOGGER.exception(
                            'Failed to connect to katcp due to %s' % str(e))
                    else:
                        return self._katcp_rct_sensor
        else:
            if not self._katcp_rct_sensor.is_active():
                LOGGER.info(
                    'katcp resource client wasnt running, hence we need to start it.'
                )
                self._katcp_rct_sensor.start()
                try:
                    time.sleep(1)
                    self._katcp_rct_sensor.until_synced(timeout=_timeout)
                    return self._katcp_rct_sensor
                except Exception:
                    self._katcp_rct_sensor.stop()
                    LOGGER.exception('Failed to connect to katcp')
            else:
                return self._katcp_rct_sensor

    @property
    def issue_metadata(self):
        """Issue Spead metadata"""
        try:
            reply, informs = self.katcp_rct.req.capture_meta(self.product_name,
                                                             timeout=_timeout)
            assert reply.reply_ok()
        except Exception:
            LOGGER.exception('Failed to issue new metadata')
            return False
        else:
            return True

    @property
    def start_x_data(self):
        """
        Enable/Start output product capture
        """
        try:
            assert isinstance(
                self.katcp_rct,
                resource_client.ThreadSafeKATCPClientResourceWrapper)
            reply, informs = self.katcp_rct.req.capture_list(timeout=_timeout)
            assert reply.reply_ok()
            self.product_name = [
                i.arguments[0] for i in informs if self.corr_config['xengine']
                ['output_products'] in i.arguments
            ][0]
            assert self.product_name is not None
            LOGGER.info('Capturing %s product' % self.product_name)
        except Exception:
            self.product_name = self.corr_config['xengine']['output_products']
            LOGGER.exception(
                'Failed to retrieve capture list via CAM interface, '
                'got it from config file.')

        try:
            reply, informs = self.katcp_rct.req.capture_start(
                self.product_name)
            assert reply.reply_ok()
            LOGGER.info(' %s' % str(reply))
            Aqf.progress(str(reply) + '\n')
            return True
        except Exception:
            LOGGER.exception('Failed to capture start: %s' % str(reply))
            return False

    def stop_x_data(self):
        """
        Disable/Stop output product capture
        """
        try:
            assert self.product_name is not None
            assert isinstance(
                self.katcp_rct,
                resource_client.ThreadSafeKATCPClientResourceWrapper)
            reply, informs = self.katcp_rct.req.capture_stop(self.product_name,
                                                             timeout=_timeout)
            assert reply.reply_ok()
            LOGGER.info(' %s' % str(reply))
            Aqf.progress(str(reply))
            return True
        except Exception:
            LOGGER.exception(
                'Failed to capture stop, might be because config file does not contain '
                'Xengine output products.')
            return False

    def get_running_instrument(self):
        """
        Returns currently running instrument listed on the sensor(s)
        """
        try:
            reply = None
            reply = self.katcp_rct.sensor.instrument_state.get_reading()
            assert reply.istatus
            return reply.value
        except Exception as e:
            LOGGER.exception('KATCP Request failed due to error: %s/%s' %
                             (str(e), str(reply)))
            return False

        except KATCPSensorError:
            LOGGER.exception('KATCP Error polling sensor\n')
            return False
        except AssertionError:
            LOGGER.exception(
                'Sensor request failed: %s, Forcefully Halting the Array' %
                (str(reply)))
            return False

    def ensure_instrument(self,
                          instrument,
                          retries=5,
                          force_reinit=False,
                          **kwargs):
        """Ensure that named instrument is active on the correlator array

        Will pass `kwargs` to self.start_correlator if a start is required
        :param self: Object
        :param instrument: CBF Instrument
        :param retries: No of instrument validation retries
        :param force_reinit: Force an instrument re-initialisation
        :rtype: Boolean
        """
        try:
            assert '_' in self.instrument
            self.instrument = self.instrument.split('_')[0]
        except AssertionError:
            pass

        if force_reinit:
            LOGGER.info('Forcing an instrument(%s) re-initialisation' %
                        self.instrument)
            corr_success = self.start_correlator(self.instrument, **kwargs)
            return corr_success

        success = False
        while retries and not success:
            check_ins = self.check_instrument(self.instrument)
            msg = 'Will retry to check if the instrument is up: #%s retry.' % retries
            LOGGER.info(msg)
            if check_ins is True:
                success = True
                LOGGER.info('Named instrument (%s) is currently running' %
                            self.instrument)
                return success
            retries -= 1

        if self.check_instrument(self.instrument) is False:
            LOGGER.info(
                'Correlator not running requested instrument, will restart.')
            reply = self.katcp_rct.sensor.instrument_state.get_reading()
            if reply.value == self.instrument:
                pass
            corr_success = self.start_correlator(self.instrument, **kwargs)
            return True if corr_success is True else False

    def check_instrument(self, instrument):
        """Return true if named instrument is enabled on correlator array

        Uses the correlator array KATCP interface to check if the requested
        instrument is active
        :param instrument: Correlator

        """
        self.instrument = instrument
        self._errmsg = None
        try:
            self._errmsg = 'Instrument cannot be None.'
            assert instrument is not None, self._errmsg
            self._errmsg = 'katcp client is not an instance of resource client'
            assert isinstance(
                self.katcp_rct, resource_client.
                ThreadSafeKATCPClientResourceWrapper), self._errmsg
            self._errmsg = 'katcp client failed to establish a connection'
            assert self.katcp_rct.wait_connected(), self._errmsg
            self._errmsg = 'katcp client failed to sync after establishing a connection'
            assert self.katcp_rct.until_state('synced',
                                              timeout=60), self._errmsg
        except AssertionError:
            # This probably means that no array has been defined yet and therefore the
            # katcp_rct client cannot be created. IOW, the desired instrument would
            # not be available
            LOGGER.exception(self._errmsg)
            return False
        else:
            try:
                reply, informs = self.katcp_rct.req.instrument_list()
                assert reply.reply_ok()
                instruments_available = [
                    instrument_avail.arguments[0].split('_')[0]
                    for instrument_avail in informs
                ]
                # Test to see if requested instrument is available on the instrument list
                assert instrument in instruments_available
            except Exception:
                LOGGER.exception('Array request failed might have timed-out')
            except AssertionError:
                LOGGER.exception(
                    'Array request failed: %s or Instrument: %s is not in instrument '
                    'list: %s' %
                    (instrument, instruments_available, str(reply)))
                return False

            try:
                reply = self.katcp_rct.sensor.instrument_state.get_reading()
                assert reply.istatus
            except AttributeError:
                LOGGER.exception(
                    'Instrument state could not be retrieved from the sensors')
                return False
            except AssertionError:
                LOGGER.error('%s: No running instrument' % str(reply))
                return False

            else:
                running_intrument = reply.value.split('_')[0]
                instrument_present = instrument == running_intrument
                if instrument_present:
                    self.instrument = instrument
                    LOGGER.info(
                        'Confirmed that the named instrument %s is running' %
                        self.instrument)
                return instrument_present

    @property
    def _get_instrument(self):
        """
        Retrieve currently running instrument from /etc/corr
        return: List
        """
        try:
            running_instr = self.config_filename.split('/')[-1]
            self.array_name, self.instrument = running_instr.split('-')
            try:
                assert '_' in self.instrument
                self.instrument = self.instrument.split('_')[0]
            except AssertionError:
                pass

            if (self.instrument.startswith('bc') or self.instrument.startswith('c')) and \
                self.array_name.startswith('array'):
                LOGGER.info(
                    'Currently running instrument %s as per /etc/corr' %
                    self.instrument)
                return [self.array_name, self.instrument]
        except Exception:
            LOGGER.exception(
                'Could not retrieve information from config file, resorting to default'
            )
            return ['array0', 'bc8n856M4k']
        except ValueError:
            LOGGER.exception('Directory missing array config file.')

    @property
    def _test_config_file(self):
        """
        Configuration file containing information such as dsim, pdu and dataswitch ip's
        return: Dict
        """
        path, _none = os.path.split(__file__)
        path, _none = os.path.split(path)
        try:
            assert os.uname()[1].startswith('dbelab')
        except AssertionError:
            conf_path = 'config/test_conf_site.ini'
        else:
            conf_path = 'config/test_conf_lab.ini'

        config_file = os.path.join(path, conf_path)
        if os.path.isfile(config_file) or os.path.exists(config_file):
            try:
                config = parse_ini_file(config_file)
                return config
            except (IOError, ValueError, TypeError):
                errmsg = (
                    'Failed to read test config file %s, Test will exit' %
                    (config_file))
                LOGGER.exception(errmsg)
                return False
        else:
            LOGGER.error('Test config path: %s does not exist' % config_file)
            return False

    @property
    def get_multicast_ips(self):
        """
        Retrieves multicast ips from test configuration file and calculates the number
        of inputs depending on which instrument is being initialised
        :param instrument: Correlator
        """
        if self.instrument is None:
            return False
        try:
            multicast_ip_inp = [
                self.corr_config['dsimengine'].get('pol0_destination_start_ip',
                                                   '239.101.0.64'),
                self.corr_config['dsimengine'].get('pol1_destination_start_ip',
                                                   '239.101.0.66')
            ]

        except TypeError:
            msg = (
                'Could not read and split the multicast IPs in the test config file'
            )
            LOGGER.exception(msg)
            return False
        else:
            if self.instrument.startswith('bc') or self.instrument.startswith(
                    'c'):
                if self.instrument[0] == 'b':
                    try:
                        # multicast_ip = multicast_ip_inp * (int(self.instrument[2]) / 2)
                        multicast_ip = multicast_ip_inp * (int(
                            self.instrument.replace('bc', '').split('n')[0]) /
                                                           2)
                        return multicast_ip
                    except Exception:
                        LOGGER.error(
                            'Could not calculate multicast IPs from config file'
                        )
                        return False
                else:
                    try:
                        multicast_ip = multicast_ip_inp * (int(
                            self.instrument.replace('c', '').split('n')[0]) /
                                                           2)
                        return multicast_ip
                    except Exception:
                        LOGGER.error(
                            'Could not calculate multicast IPs from config file'
                        )
                        return False

    def start_correlator(self, instrument=None, retries=10):
        LOGGER.debug('CBF instrument(%s) re-initialisation.' % instrument)
        success = False
        self.katcp_array_port = None
        if instrument is not None:
            self.instrument = instrument
        self._correlator = None  # Invalidate cached correlator instance
        LOGGER.info('Confirm DEngine is running before starting correlator')
        if not self.dhost.is_running():
            raise RuntimeError('DEngine: %s not running.' % (self.dhost.host))

        multicast_ip = self.get_multicast_ips
        if not multicast_ip:
            LOGGER.error('Failed to calculate multicast IP\'s, investigate')
            self._katcp_rct = None
        self.rct.start()
        try:
            self.rct.until_synced(timeout=_timeout)
            reply, informs = self.rct.req.subordinate_list(self.array_name)
            assert reply.reply_ok()
        except TimeoutError:
            self.rct.stop()
            LOGGER.exception('Resource client timed-out after %s s' % _timeout)
            return False
        except AssertionError:
            LOGGER.exception(
                'Failed to get subordinate-list, might not have been assigned, '
                'Will try to assign.')
        else:
            try:
                informs = informs[0]
                self.katcp_array_port = int(
                    [i for i in informs.arguments if len(i) == 5][0])
            except ValueError:
                LOGGER.exception('Failed to assign katcp port: Reply: %s' %
                                 (reply))
                reply = self.rct.req.subordinate_halt(self.array_name)
                if not reply.succeeded:
                    LOGGER.error('Unable to halt array %s: %s' %
                                 (self.array_name, reply))
                    return False

        while retries and not success:
            try:
                if self.katcp_array_port is None:
                    LOGGER.info('Assigning array port number')
                    try:
                        pass
                        # reply, _informs = self.rct.req.subordinate_create(self.array_name,
                        #                                             *multicast_ip, timeout=_timeout)
                        # assert reply.reply_ok()
                    except Exception:
                        self.katcp_array_port = None
                        LOGGER.exception('Failed to assign new array: %s' %
                                         self.array_name)
                    else:
                        self.katcp_array_port = int(reply.arguments[-1])
                        LOGGER.info('Successfully created %s-%s' %
                                    (self.array_name, self.katcp_array_port))

                instrument_param = ([
                    int(i) for i in self.test_config['instrument_params']
                    ['instrument_param'] if i != ','
                ])
                LOGGER.info('Starting %s with %s parameters. Try #%s' %
                            (self.instrument, instrument_param, retries))

                # TODO add timeout
                reply = self.katcp_rct.req.instrument_activate(
                    self.instrument, *instrument_param, timeout=500)
                success = reply.succeeded
                retries -= 1

                try:
                    assert success
                    LOGGER.info('Instrument %s started successfully' %
                                (self.instrument))
                except AssertionError:
                    LOGGER.exception(
                        'Failed to start correlator, %s attempts left. '
                        'Restarting Correlator. Reply:%s' % (retries, reply))
                    success = False

            except Exception:
                try:
                    self.rct.req.subordinate_halt(self.array_name)
                    assert isinstance(
                        self.katcp_rct,
                        resource_client.ThreadSafeKATCPClientResourceWrapper)
                except Exception:
                    LOGGER.exception(
                        'Unable to halt array: Empty Array number')
                except AssertionError:
                    LOGGER.exception(
                        'self.katcp_rct has not been initiated successfully')
                    return False
                else:
                    try:
                        self.katcp_rct.stop()
                    except AttributeError:
                        LOGGER.error(
                            'KATCP request does not contain attributes')
                        return False
                    else:
                        retries -= 1
                        LOGGER.warn(
                            'Failed to start correlator, %s attempts left.\n' %
                            (retries))

            if retries < 0:
                success = False
                return success

        if success:
            self._correlator_started = True
            return self._correlator_started
        else:
            try:
                self._correlator_started = False
                self.katcp_rct.stop()
                self.rct.stop()
                self._katcp_rct = None
                self._correlator = None
            except:
                msg = (
                    'Could not successfully start correlator within %s retries'
                    % (retries))
                LOGGER.critical(msg)
                return False
            return False