def dhost(self, program=False):
if self._dhost is not None:
return self._dhost
else:
self.config_filename = '/etc/corr/{}-{}'.format(self.array_name, self.instrument)
if os.path.exists(self.config_filename):
LOGGER.info('Retrieving dsim engine info from config file: %s' % self.config_filename)
self.corr_config = corr2.utils.parse_ini_file(self.config_filename)
self.dsim_conf = self.corr_config['dsimengine']
elif self.instrument is not None:
self.corr2ini_path = '/etc/corr/templates/{}'.format(self.instrument)
LOGGER.info('Setting CORR2INI system enviroment to point to %s' % self.corr2ini_path)
os.environ['CORR2INI'] = self.corr2ini_path
self.corr_config = corr2.utils.parse_ini_file(self.corr2ini_path)
self.dsim_conf = self.corr_config['dsimengine']
else:
errmsg = ('Could not retrieve dsim information from running config file in /etc/corr, '
'Perhaps, restart CBF manually and ensure dsim is running.\n'
'File:%s Line:%s' % (getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
LOGGER.error(errmsg)
sys.exit(errmsg)
try:
dig_host = self.dsim_conf['host']
self._dhost = FpgaDsimHost(dig_host, config=self.dsim_conf)
except Exception:
errmsg = 'Digitiser Simulator failed to retrieve information'
LOGGER.exception(errmsg)
sys.exit(errmsg)
else:
# Check if D-eng is running else start it.
if self._dhost.is_running():
LOGGER.info('D-Eng is already running.')
# Disabled DSim programming as it would alter the systems sync epoch
elif program and not self._dhost.is_running():
LOGGER.info('Programming and starting the Digitiser Simulator.')
self._dhost.initialise()
self._dhost.enable_data_output(enabled=True)
self._dhost.registers.control.write(gbe_txen=True)
else:
LOGGER.info('D-Eng started succesfully')
return self._dhost
def dhost(self, program=False):
if self._dhost is not None:
return self._dhost
else:
if os.path.exists(self.config_filename):
LOGGER.info(
'Retrieving dsim engine info from config file: %s' %
self.config_filename)
self.corr_config = parse_ini_file(self.config_filename)
self.dsim_conf = self.corr_config['dsimengine']
elif self.instrument is not None:
self.corr2ini_path = '/etc/corr/templates/{}'.format(
self.instrument)
LOGGER.info(
'Setting CORR2INI system environment to point to %s' %
self.corr2ini_path)
os.environ['CORR2INI'] = self.corr2ini_path
self.corr_config = parse_ini_file(self.corr2ini_path)
self.dsim_conf = self.corr_config['dsimengine']
else:
errmsg = (
'Could not retrieve dsim information from running config file in /etc/corr, '
'Perhaps, restart CBF manually and ensure dsim is running.\n'
)
LOGGER.error(errmsg)
sys.exit(errmsg)
try:
dig_host = self.dsim_conf['host']
self._dhost = None
self._dhost = FpgaDsimHost(dig_host, config=self.dsim_conf)
except Exception:
errmsg = 'Digitiser Simulator failed to retrieve information'
LOGGER.exception(errmsg)
sys.exit(errmsg)
else:
# Check if D-eng is running else start it.
if self._dhost.is_running():
LOGGER.info('D-Eng is already running.')
return self._dhost
def setUp(self):
self.receiver = CorrRx(port=8888)
start_thread_with_cleanup(self, self.receiver, start_timeout=1)
self.correlator = correlator_fixture.correlator
self.corr_fix = correlator_fixture
self.corr_freqs = CorrelatorFrequencyInfo(self.correlator.configd)
dsim_conf = self.correlator.configd['dsimengine']
dig_host = dsim_conf['host']
self.dhost = FpgaDsimHost(dig_host, config=dsim_conf)
self.dhost.get_system_information()
# Increase the dump rate so tests can run faster
self.correlator.xeng_set_acc_time(0.2)
self.addCleanup(self.corr_fix.stop_x_data)
self.corr_fix.start_x_data()
self.corr_fix.issue_metadata()
help='Constant wave output scale')
parser.add_argument(
'-n', '--noise_scale', type=float, default=0.05,
help='Noise output scale')
args = parser.parse_args()
cw_scale = args.cw_scale
n_scale = args.noise_scale
freq = args.frequency
#Connect to DSIM
corr_conf = utils.parse_ini_file(args.config, ['dsimengine'])
dsim_conf = corr_conf['dsimengine']
dig_host = dsim_conf['host']
dsim_fpg = dsim_conf['bitstream']
dhost = FpgaDsimHost(dig_host, config=dsim_conf)
if dig_host.lower().find('skarab') != -1:
dhost.get_system_information(filename=dsim_fpg)
else:
dhost.get_system_information()
#Clear all noise sources
sources_names = dhost.noise_sources.names()
for source in sources_names:
try:
noise_source = getattr(dhost.noise_sources, '{}'.format(source))
noise_source.set(0)
print("noise source {}, set to {}.".format(noise_source.name,
noise_source.scale))
except:
"""
Based on JSObject : Python Objects that act like Javascript Objects
based on James Robert blog entry:
Making Python Objects that act like Javascript Objects
http://jiaaro.com/making-python-objects-that-act-like-javascrip
"""
#logging.basicConfig(
# format='%(asctime)s %(name)s %(levelname)s %(filename)s:%(lineno)s %(message)s',
# level=logging.INFO)
corr_conf = utils.parse_ini_file(config, ['dsimengine'])
dsim_conf = corr_conf['dsimengine']
dig_host = dsim_conf['host']
dhost = FpgaDsimHost(dig_host, config=dsim_conf)
if dhost.is_running():
dhost.get_system_information()
print 'Dsim is running'
config_link = '/etc/corr/array0-bc8n856M4k'
config_link2 = '/etc/corr/array0-bc8n856M32k'
config_link3 = '/etc/corr/templates/bc8n856M4k'
config_link4 = '/etc/corr/templates/bc8n856M32k'
if os.path.exists(config_link):
correlator = fxcorrelator.FxCorrelator('rts correlator', config_source=config_link)
elif os.path.exists(config_link2):
correlator = fxcorrelator.FxCorrelator('rts correlator', config_source=config_link2)
elif os.path.exists(config_link3):
correlator = fxcorrelator.FxCorrelator('rts correlator', config_source=config_link3)
else:
class CorrelatorFixture(object):
def __init__(self, katcp_clt=None, product_name=None):
self.katcp_clt = katcp_clt
self.corr_config = None
self.corr2ini_path = None
self._correlator = None
self._dhost = None
self._katcp_rct = None
self._rct = None
self.katcp_array_port = None
self.product_name = product_name
self.halt_wait_time = 5
# Assume the correlator is already started if start_correlator is False
nose_test_config = {}
self._correlator_started = not int(
nose_test_config.get('start_correlator', False))
self.test_config = self._test_config_file
self.array_name, self.instrument = self._get_instrument
@property
def rct(self):
if self._rct is not None:
return self._rct
else:
self.io_manager = ioloop_manager.IOLoopManager()
self.io_wrapper = resource_client.IOLoopThreadWrapper(
self.io_manager.get_ioloop())
LOGGER.info('Cleanup function \'self.io_manager\': File: %s line: %s' % (
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
add_cleanup(self.io_manager.stop)
self.io_wrapper.default_timeout = _timeout
self.io_manager.start()
self.rc = resource_client.KATCPClientResource(
dict(name='{}'.format(self.katcp_clt),
address=('{}'.format(self.katcp_clt),
'7147'),
controlled=True))
self.rc.set_ioloop(self.io_manager.get_ioloop())
self._rct = (resource_client.ThreadSafeKATCPClientResourceWrapper(self.rc,
self.io_wrapper))
self._rct.start()
LOGGER.info('Cleanup function \'self._rct\': File: %s line: %s' % (
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
add_cleanup(self._rct.stop)
try:
self._rct.until_synced(timeout=_timeout)
except TimeoutError:
self._rct.stop()
return self._rct
@property
def dhost(self, program=False):
if self._dhost is not None:
return self._dhost
else:
self.config_filename = '/etc/corr/{}-{}'.format(self.array_name, self.instrument)
if os.path.exists(self.config_filename):
LOGGER.info('Retrieving dsim engine info from config file: %s' % self.config_filename)
self.corr_config = corr2.utils.parse_ini_file(self.config_filename)
self.dsim_conf = self.corr_config['dsimengine']
elif self.instrument is not None:
self.corr2ini_path = '/etc/corr/templates/{}'.format(self.instrument)
LOGGER.info('Setting CORR2INI system enviroment to point to %s' % self.corr2ini_path)
os.environ['CORR2INI'] = self.corr2ini_path
self.corr_config = corr2.utils.parse_ini_file(self.corr2ini_path)
self.dsim_conf = self.corr_config['dsimengine']
else:
errmsg = ('Could not retrieve dsim information from running config file in /etc/corr, '
'Perhaps, restart CBF manually and ensure dsim is running.\n'
'File:%s Line:%s' % (getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
LOGGER.error(errmsg)
sys.exit(errmsg)
try:
dig_host = self.dsim_conf['host']
self._dhost = FpgaDsimHost(dig_host, config=self.dsim_conf)
except Exception:
errmsg = 'Digitiser Simulator failed to retrieve information'
LOGGER.exception(errmsg)
sys.exit(errmsg)
else:
# Check if D-eng is running else start it.
if self._dhost.is_running():
LOGGER.info('D-Eng is already running.')
# Disabled DSim programming as it would alter the systems sync epoch
elif program and not self._dhost.is_running():
LOGGER.info('Programming and starting the Digitiser Simulator.')
self._dhost.initialise()
self._dhost.enable_data_output(enabled=True)
self._dhost.registers.control.write(gbe_txen=True)
else:
LOGGER.info('D-Eng started succesfully')
return self._dhost
@property
def correlator(self):
if self._correlator is not None:
LOGGER.info('Using cached correlator instance')
return self._correlator
else: # include a check, if correlator is running else start it.
if not self._correlator_started:
LOGGER.info('Correlator not running, now starting.')
self.start_correlator()
# We assume either start_correlator() above has been called, or the
# instrument was started with the name contained in self.array_name
# before running the test.
self.config_filename = '/etc/corr/{}-{}'.format(self.array_name, self.instrument)
_retries = 3
if os.path.exists(self.config_filename):
LOGGER.info('Making new correlator instance')
# for _retry in xrange(_retries):
while True:
_retries -= 1
try:
self._correlator = fxcorrelator.FxCorrelator(
'test correlator', config_source=self.config_filename)
time.sleep(1)
self.correlator.initialise(program=False)
return self._correlator
except KatcpRequestFail as e:
LOGGER.exception('Did not get a response from roach/host: %s' %str(e))
continue
except Exception as e:
LOGGER.exception('Failed to create new correlator instance with error: %s, '
'Will now try to start correlator with config: %s-%s' % (str(e),
self.array_name, self.instrument))
continue
if _retries == 0:
break
if _retries == 0:
self.start_correlator(instrument=self.instrument)
else:
LOGGER.error('No Config file (/etc/corr/array*-instrument), '
'Starting correlator with default instrument: %s' % (self.instrument))
self.start_correlator(instrument=self.instrument)
@property
def halt_array(self):
"""
Halting of primary and secondary katcp arrays and ensure that the correlator
object is teared-down
"""
LOGGER.info('Halting primary array: %s.' % self.array_name)
try:
reply, informs = self.katcp_rct.req.halt(timeout=_timeout)
LOGGER.info(str(reply))
assert reply.reply_ok()
assert self._katcp_rct.is_active()
except AssertionError:
msg = 'Failed to halt katcp connection'
LOGGER.error(msg)
except AttributeError:
raise RuntimeError('Failing to halt array, investigate halt array function.')
self._katcp_rct.stop()
self._katcp_rct = None
try:
reply, informs = self.rct.req.subordinate_list(timeout=_timeout)
assert reply.reply_ok()
if informs:
informs = informs[0]
if len(informs.arguments) >= 10 and self.array_name == informs.arguments[0]:
reply, informs = self.rct.req.subordinate_halt(self.array_name, timeout=_timeout)
assert reply.reply_ok()
except AssertionError:
msg = 'Failed to halt array: %s, STOPPING resource client' %self.array_name
LOGGER.exception(msg)
except IndexError:
pass
# if self.rct.is_active():
# self.rct.stop()
# self._rct.stop()
# self._rct = None
# self.rct = None
self._correlator_started = False
self._correlator = None
LOGGER.info('Array %s halted and teared-down' % (self.array_name))
time.sleep(self.halt_wait_time)
@property
def katcp_rct(self):
try:
katcp_prot = self.test_config['inst_param']['katcp_protocol']
except TypeError:
LOGGER.error('Failed to read katcp protocol from test config file')
else:
_major, _minor, _flags = katcp_prot.split(',')
protocol_flags = ProtocolFlags(int(_major), int(_minor), _flags)
multicast_ip = self.get_multicast_ips
if not multicast_ip:
LOGGER.error('Failed to calculate multicast IP\'s')
self._katcp_rct = None
return
if self._katcp_rct is None:
try:
reply, informs = self.rct.req.subordinate_list(self.array_name)
except TypeError:
msg = 'Failed to list all arrays with name: %s' %self.array_name
LOGGER.exception(msg)
# If no sub-array present create one, but this could cause problems
# if more than one sub-array is present. Update this to check for
# required sub-array.
if reply.reply_ok():
self.katcp_array_port = int(informs[0].arguments[1])
else:
LOGGER.info('Array has not been assigned yet, will try to assign.'
' File:%s Line:%s' % (
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
try:
reply, _informs = self.rct.req.subordinate_create(self.array_name,
*multicast_ip)
assert reply.reply_ok()
except (ValueError, TypeError, AssertionError):
try:
reply, informs = self.rct.req.subordinate_list()
assert reply.reply_ok()
informs = informs[0]
if len(informs.arguments) >= 10 and self.array_name == informs.arguments[0]:
msg = 'Array assigned successfully: %s' %str(informs)
LOGGER.info(msg)
else:
LOGGER.error('Halting array.')
reply, informs = self.rct.req.subordinate_halt(self.array_name,
timeout=_timeout)
assert reply.reply_ok()
except AssertionError:
LOGGER.exception('Failed to assign multicast ip on array: %s: \n\nReply: %s' % (
self.array_name, str(reply)))
else:
if len(reply.arguments) == 2:
try:
self.katcp_array_port = int(reply.arguments[-1])
LOGGER.info('Array %s assigned successfully' % (self.katcp_array_port))
except ValueError:
# self.rct.req.subordinate_halt(self.array_name)
# self.rct.stop()
# self.rct.start()
# self.rct.until_synced(timeout=_timeout)
# reply, informs = self.rct.req.subordinate_create(self.array_name,
# *multicast_ip)
errmsg = 'Investigate as to why this thing failed.'
LOGGER.exception(errmsg)
sys.exit(errmsg)
katcp_rc = resource_client.KATCPClientResource(
dict(name='{}'.format(self.katcp_clt),
address=('{}'.format(self.katcp_clt),
'{}'.format(self.katcp_array_port)),
preset_protocol_flags=protocol_flags, controlled=True))
katcp_rc.set_ioloop(self.io_manager.get_ioloop())
self._katcp_rct = (
resource_client.ThreadSafeKATCPClientResourceWrapper(
katcp_rc, self.io_wrapper))
self._katcp_rct.start()
try:
self._katcp_rct.until_synced(timeout=_timeout)
except Exception as e:
self._katcp_rct.stop()
LOGGER.exception('Failed to connect to katcp due to %s' %str(e))
LOGGER.info('Cleanup function \'self._katcp_rct\': File: %s line: %s' % (
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
# add_cleanup(self._katcp_rct.stop)
else:
self._katcp_rct.start()
try:
time.sleep(1)
self._katcp_rct.until_synced(timeout=_timeout)
except Exception as e:
self._katcp_rct.stop()
LOGGER.exception('Failed to connect to katcp due to %s' %str(e))
return self._katcp_rct
@property
def issue_metadata(self):
"""Issue Spead metadata"""
try:
reply, informs = self.katcp_rct.req.capture_meta(self.product_name, timeout=_timeout)
assert reply.reply_ok()
except Exception:
LOGGER.exception('Failed to issue new metadata: File:%s Line:%s' % (
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
return False
else:
return True
@property
def start_x_data(self):
"""
Enable/Start output product capture
"""
try:
assert isinstance(self.katcp_rct, resource_client.ThreadSafeKATCPClientResourceWrapper)
reply, informs = self.katcp_rct.req.capture_list(timeout=_timeout)
assert reply.reply_ok()
self.product_name = [i.arguments[0] for i in informs
if self.corr_config['xengine']['output_products'] in i.arguments][0]
assert self.product_name is not None
LOGGER.info('Capturing %s product' %self.product_name)
except Exception:
self.product_name = self.corr_config['xengine']['output_products']
LOGGER.exception('Failed to retrieve capture list via CAM interface, got it from config file.'
'\nFile:%s Line:%s' % (getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
try:
reply, informs = self.katcp_rct.req.capture_start(self.product_name)
assert reply.reply_ok()
LOGGER.info(' %s' % str(reply))
Aqf.progress(str(reply)+'\n')
return True
except Exception:
LOGGER.exception('Failed to capture start: %s' %str(reply))
return False
def stop_x_data(self):
"""
Disable/Stop output product capture
"""
try:
assert self.product_name is not None
assert isinstance(self.katcp_rct,
resource_client.ThreadSafeKATCPClientResourceWrapper)
reply, informs = self.katcp_rct.req.capture_stop(self.product_name, timeout=_timeout)
assert reply.reply_ok()
LOGGER.info(' %s' %str(reply))
Aqf.progress(str(reply))
return True
except Exception:
LOGGER.exception('Failed to capture stop, might be because config file does not contain '
'Xengine output products.\n: File:%s Line:%s' % (
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
return False
def get_running_instrument(self):
"""
Returns currently running instrument listed on the sensor(s)
"""
try:
reply = None
reply = self.katcp_rct.sensor.instrument_state.get_reading()
assert reply.istatus
return reply.value
except Exception as e:
LOGGER.exception('KATCP Request failed due to error: %s/%s'
'\n\t File:%s Line:%s' % (str(e), str(reply),
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
return False
except KATCPSensorError:
LOGGER.exception('KATCP Error polling sensor\n\t File:%s Line:%s' % (
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
return False
except AssertionError:
LOGGER.exception('Sensor request failed: %s , Forcefully Halting the Array\n\t '
'File:%s Line:%s' % (str(reply), getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
return False
def ensure_instrument(self, instrument, retries=5, force_reinit=False, **kwargs):
"""Ensure that named instrument is active on the correlator array
Will pass `kwargs` to self.start_correlator if a start is required
:param self: Object
:param instrument: CBF Instrument
:param retries: No of instrument validation retries
:param force_reinit: Force an instrument re-initialisation
:rtype: Boolean
"""
self.instrument = instrument
if force_reinit:
LOGGER.info('Forcing an instrument(%s) re-initialisation' %self.instrument)
corr_success = self.start_correlator(self.instrument, **kwargs)
return corr_success
success = False
while retries and not success:
check_ins = self.check_instrument(self.instrument)
msg = 'Retries left to check instrument: %s' %retries
LOGGER.info(msg)
if check_ins is True:
success = True
LOGGER.info('Return true if named instrument is enabled on correlator array after '
' %s retries' %retries)
return success
retries -= 1
if self.check_instrument(self.instrument) is False:
LOGGER.info('Correlator not running requested instrument, will restart.')
reply = self.katcp_rct.sensor.instrument_state.get_reading()
if reply.value == self.instrument:
self.halt_array
corr_success = self.start_correlator(self.instrument, **kwargs)
return True if corr_success is True else False
def check_instrument(self, instrument):
"""Return true if named instrument is enabled on correlator array
Uses the correlator array KATCP interface to check if the requested
instrument is active
:param instrument: Correlator
"""
self.instrument = instrument
self._errmsg = None
try:
self._errmsg = 'Instrument cannot be None.'
assert instrument is not None, self._errmsg
self._errmsg = 'katcp client is not an instance of resource client'
assert isinstance(self.katcp_rct,
resource_client.ThreadSafeKATCPClientResourceWrapper), self._errmsg
self._errmsg = 'katcp client failed to establish a connection'
assert self.katcp_rct.wait_connected(), self._errmsg
self._errmsg = 'katcp client failed to sync after establishing a connection'
assert self.katcp_rct.until_state('synced', timeout=60), self._errmsg
except AssertionError:
# This probably means that no array has been defined yet and therefore the
# katcp_rct client cannot be created. IOW, the desired instrument would
# not be available
LOGGER.exception(self._errmsg)
return False
else:
try:
reply, informs = self.katcp_rct.req.instrument_list()
assert reply.reply_ok()
instruments_available = [instrument_avail.arguments[0] for instrument_avail in informs]
# Test to see if requested instrument is available on the instrument list
assert instrument in instruments_available
except Exception:
LOGGER.exception('Array request failed might have timedout\n\tFile:%s Line:%s' % (
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
except AssertionError:
LOGGER.exception('Array request failed: %s or Instrument: %s is not in instrument '
'list: %s \n\tFile:%s Line:%s' % (instrument,
instruments_available, str(reply),
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
return False
try:
reply = self.katcp_rct.sensor.instrument_state.get_reading()
assert reply.istatus == 1
except AttributeError:
LOGGER.exception('Instrument state could not be retrieved from the sensors')
return False
except AssertionError:
LOGGER.error(' %s: Seems like the might be no current running instrument' %str(reply))
return False
else:
running_intrument = reply.value
instrument_present = instrument == running_intrument
if instrument_present:
self.instrument = instrument
LOGGER.info('Confirmed that the named instrument %s is enabled on '
'correlator %s.' % (self.instrument, self.array_name))
return instrument_present
@property
def _get_instrument(self):
"""
Retrieve currently running instrument from /etc/corr
return: List
"""
try:
# ToDo (MM) 06-10-2017 Hardcoded array, fix it
running_instr = max(glob.iglob('/etc/corr/array0-*'), key=os.path.getctime).split('/')[-1]
self.array_name, self.instrument = running_instr.split('-')
if (self.instrument.startswith('bc') or self.instrument.startswith('c')) and \
self.array_name.startswith('array'):
LOGGER.info('Currenly running instrument %s as per /etc/corr' %running_instr)
return running_instr.split('-')
except Exception:
LOGGER.exception('Could not retrieve information from config file, resorting to default:\n'
'File:%s Line:%s' % (getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
return ['array0', 'bc8n856M4k']
except ValueError:
LOGGER.exception('Directory missing array config file.')
@property
def _test_config_file(self):
"""
Configuration file containing information such as dsim, pdu and dataswitch ip's
return: Dict
"""
path, _none = os.path.split(__file__)
path, _none = os.path.split(path)
try:
assert os.uname()[1].startswith('dbelab')
except AssertionError:
conf_path = 'config/test_conf_site.ini'
else:
conf_path = 'config/test_conf_lab.ini'
config_file = os.path.join(path, conf_path)
if os.path.isfile(config_file) or os.path.exists(config_file):
try:
config = corr2.utils.parse_ini_file(config_file)
return config
except (IOError, ValueError, TypeError):
errmsg = ('Failed to read test config file %s, Test will exit'
'\n\t File:%s Line:%s' % (config_file,
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
LOGGER.exception(errmsg)
return False
else:
LOGGER.error('Test config path: %s does not exist' % config_file)
return False
@property
def get_multicast_ips(self):
"""
Retrieves multicast ips from test configuration file and calculates the number
of inputs depending on which instrument is being initialised
:param instrument: Correlator
"""
if self.instrument is None:
return False
try:
multicast_ip_inp = (
self.test_config['inst_param']['source_mcast_ips'].split(','))
except TypeError:
msg = ('Could not read and split the multicast ip\'s in the test config file')
LOGGER.exception(msg)
return False
else:
if self.instrument.startswith('bc') or self.instrument.startswith('c'):
if self.instrument[0] == 'b':
try:
# multicast_ip = multicast_ip_inp * (int(self.instrument[2]) / 2)
multicast_ip = multicast_ip_inp * (
int(self.instrument.replace('bc', '').split('n')[0]) / 2)
return multicast_ip
except Exception:
LOGGER.error('Could not calculate multicast ips from config file')
return False
else:
try:
# multicast_ip = multicast_ip_inp * (int(self.instrument[1]) / 2)
multicast_ip = multicast_ip_inp * (
int(self.instrument.replace('c', '').split('n')[0]) / 2)
return multicast_ip
except Exception:
LOGGER.error('Could not calculate multicast ips from config file')
return False
@property
def subscribe_multicast(self):
"""Automated multicasting subscription"""
parse_address = StreamAddress._parse_address_string
try:
n_xengs = self.katcp_rct.sensor.n_xengs.get_value()
except Exception:
n_xengs = len(self.get_multicast_ips) * 2
if self.config_filename is None:
return
config = self.corr_config
if config is None:
LOGGER.error('Failed to retrieve correlator config file, ensure that the cbf is running')
return False
def confirm_multicast_subs(mul_ip='239.100.0.10'):
""""""
# or use [netstat -g | grep eth2]
list_inets = subprocess.check_output(['ip', 'maddr', 'show'])
return True if mul_ip in list_inets else False
outputIPs = {}
for i in [key for key, value in config.items() if 'output_destinations_base' in value]:
_IP, _num, _Port = list(parse_address(config[i]['output_destinations_base']))
outputIPs[i] = [tengbe.IpAddress(_IP), int(_Port)]
if outputIPs.get('xengine'):
LOGGER.info('Multicast subscription is only valid for xengines')
multicastIP, DataPort = outputIPs.get('xengine')
if multicastIP.is_multicast():
LOGGER.info('source is multicast %s.' % (multicastIP))
# look up multicast group address in name server and find out IP version
addrinfo = socket.getaddrinfo(str(multicastIP), None)[0]
# create a socket
try:
mcast_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
except Exception:
mcast_sock = socket.socket(addrinfo[0], socket.SOCK_DGRAM)
mcast_sock.setblocking(False)
# Do not bind as this will cause a conflict when instantiating a receiver on the main script
# Join group
# mcast_sock.bind(('', DataPort))
add_cleanup(mcast_sock.close)
def join_mcast_group(address):
group_bin = socket.inet_pton(socket.AF_INET, address)
if addrinfo[0] == socket.AF_INET: # IPv4
mreq = group_bin + struct.pack('=I', socket.INADDR_ANY)
mcast_sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
LOGGER.info('Successfully subscribed to %s:%s.' % (str(multicastIP), DataPort))
else:
mreq = group_bin + struct.pack('@I', 0)
mcast_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
LOGGER.info('Successfully subscribed to %s:%s.' % (str(multicastIP), DataPort))
for addcntr in range(n_xengs):
_address = tengbe.IpAddress(multicastIP.ip_int + addcntr)
join_mcast_group(str(_address))
else:
mcast_sock = None
LOGGER.info('Source is not multicast: %s:%s' % (str(multicastIP), DataPort))
return False
return confirm_multicast_subs(mul_ip=str(_address))
def start_correlator(self, instrument=None, retries=10):
LOGGER.debug('CBF instrument(%s) re-initialisation.' %instrument)
success = False
self.katcp_array_port = None
if instrument is not None:
self.instrument = instrument
self._correlator = None # Invalidate cached correlator instance
LOGGER.info('Confirm DEngine is running before starting correlator')
if not self.dhost.is_running():
raise RuntimeError('DEngine: %s not running.' % (self.dhost.host))
multicast_ip = self.get_multicast_ips
if not multicast_ip:
LOGGER.error('Failed to calculate multicast IP\'s, investigate')
self._katcp_rct = None
self.rct.start()
try:
self.rct.until_synced(timeout=_timeout)
reply, informs = self.rct.req.subordinate_list(self.array_name)
assert reply.reply_ok()
except TimeoutError:
self.rct.stop()
LOGGER.exception('Resource client timed-out after %s s' % _timeout)
return False
except AssertionError:
LOGGER.exception('Failed to get subordinate-list, might not have been assigned, '
'Will try to assign. \n\t File:%s Line:%s' % (
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
else:
try:
informs = informs[0]
self.katcp_array_port = int(
[i for i in informs.arguments if len(i) == 5][0])
except ValueError:
LOGGER.exception('Failed to assign katcp port: Reply: %s' % (reply))
reply = self.rct.req.subordinate_halt(self.array_name)
if not reply.succeeded:
LOGGER.error('Unable to halt array %s: %s \n\t File:%s Line:%s' % (
self.array_name, reply,
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
return False
while retries and not success:
try:
if self.katcp_array_port is None:
LOGGER.info('Assigning array port number')
# self.rct.start()
# self.rct.until_synced(timeout=_timeout)
try:
reply, _informs = self.rct.req.subordinate_create(self.array_name,
*multicast_ip, timeout=_timeout)
assert reply.reply_ok()
except Exception:
self.katcp_array_port = None
LOGGER.exception('Failed to assign new array: %s' %self.array_name)
else:
self.katcp_array_port = int(reply.arguments[-1])
LOGGER.info('Successfully created %s-%s' % (self.array_name, self.katcp_array_port))
# try:
# #self.katcp_array_port = int(reply.arguments[-1])
# LOGGER.info('Array port assigned: {}'.format(self.katcp_array_port))
# except ValueError:
# LOGGER.fatal('Failed to assign array port number on {}'.format(
# self.array_name))
# else:
# if not reply.reply_ok():
# LOGGER.fatal('Failed to assign array port number on {}'.format(
# self.array_name))
# return False
instrument_param = (
[int(i) for i in self.test_config['inst_param']['instrument_param'] if i != ','])
LOGGER.info('Starting %s with %s parameters. Try #%s' % (self.instrument,
instrument_param,
retries))
reply = self.katcp_rct.req.instrument_activate(self.instrument, *instrument_param,
timeout=500)
success = reply.succeeded
retries -= 1
try:
assert success
LOGGER.info('Instrument %s started succesfully' % (self.instrument))
except AssertionError:
LOGGER.exception('Failed to start correlator, %s attempts left. '
'Restarting Correlator. Reply:%s' % (retries, reply))
self.halt_array
success = False
except Exception:
try:
self.rct.req.subordinate_halt(self.array_name)
assert isinstance(self.katcp_rct,
resource_client.ThreadSafeKATCPClientResourceWrapper)
except Exception:
LOGGER.exception('Unable to halt array: Empty Array number: '
'File:%s Line:%s' % (
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
except AssertionError:
LOGGER.exception('self.katcp_rct has not been initiated successfully')
return False
else:
try:
self.katcp_rct.stop()
except AttributeError:
LOGGER.error('KATCP request does not contain attributes: File:%s Line:%s' % (
getframeinfo(currentframe()).filename.split('/')[-1],
getframeinfo(currentframe()).lineno))
return False
else:
retries -= 1
LOGGER.warn('Failed to start correlator, %s attempts left.\n' % (retries))
if retries < 0:
success = False
return success
if success:
self._correlator_started = True
return self._correlator_started
else:
try:
self.halt_array
self._correlator_started = False
self.katcp_rct.stop()
self.rct.stop()
self._katcp_rct = None
self._correlator = None
except:
self.halt_array
msg = ('Could not successfully start correlator within %s retries' % (retries))
LOGGER.critical(msg)
return False
return False
def SpeadRx(config_file, rx_port, dsim_start, capture_start, debug, verbose):
"""
Receive data from Correlator and play
"""
if config_file is None:
sys.exit("Usage: %s --help" % __file__)
try:
assert verbose
_level = 'DEBUG'
logging.basicConfig(
level=logging.DEBUG,
format=
'%(asctime)s - %(name)s - %(levelname)s - %(module)s - %(pathname)s : '
'%(lineno)d - %(message)s')
corr_rx_logger.setLevel(logging.DEBUG)
spead2_logger.setLevel(logging.DEBUG)
logging.debug('DEBUG MODE ENABLED')
except:
_level = 'INFO'
logging.basicConfig(
level=logging.INFO,
format=
'%(asctime)s - %(name)s - %(levelname)s - %(module)s - %(pathname)s : '
'%(lineno)d - %(message)s')
corr_rx_logger.setLevel(logging.INFO)
spead2_logger.setLevel(logging.INFO)
finally:
logger = logging.getLogger(__name__)
coloredlogs.install(level=_level, logger=logger)
if dsim_start and config_file:
corr_conf = utils.parse_ini_file(config_file, ['dsimengine'])
dsim_conf = corr_conf['dsimengine']
dhost = FpgaDsimHost(dsim_conf['host'], config=dsim_conf)
try:
assert dhost.is_running()
dhost.get_system_information()
except:
logger.error('DEngine Not Running!')
try:
assert capture_start
except AssertionError:
@atexit.register
def Cleanup():
logger.info('baseline-correlation-products capture stopped!')
receiver.stop()
receiver.join()
logger.info('Receiver stopped.')
fab.local(
"kcpcmd -t 60 -s localhost:$(kcpcmd array-list | grep -a array-list | cut -f3 -d ' ' ) capture-stop baseline-correlation-products"
)
else:
logger.info('baseline-correlation-products capture started!')
fab.local(
"kcpcmd -t 60 -s localhost:$(kcpcmd array-list | grep -a array-list | cut -f3 -d ' ' ) capture-start baseline-correlation-products"
)
try:
receiver = CorrRx(port=rx_port, queue_size=5)
_multicast_ips = corr_conf['xengine'].get(
'multicast_interface_address', '239.100.0.1')
# import IPython; IPython.embed(header='Python Debugger')
except Exception as ex:
template = "An exception of type {0} occured while trying to instantiate receiver. Arguments:\n{1!r}"
message = template.format(type(ex), ex.args)
logger.info(message)
else:
logger.info('Waiting for receiver to report running')
receiver.daemon = True
receiver.start(timeout=10)
if receiver.running_event.wait(timeout=10):
logger.info('Receiver ready')
else:
msg = 'Receiver not ready'
logger.info(msg)
raise RuntimeError(msg)
try:
raw_input('Press Enter get clean dump')
dump = receiver.get_clean_dump()
except KeyboardInterrupt:
logger.info('Keyboard interrupt')
except Exception:
raise
else:
logger.info('Dump received')
if debug:
corr_rx_logger.setLevel(logging.FATAL)
spead2_logger.setLevel(logging.FATAL)
import IPython
IPython.embed(header='Python Debugger')
class test_CBF(unittest.TestCase):
def setUp(self):
self.receiver = CorrRx(port=8888)
start_thread_with_cleanup(self, self.receiver, start_timeout=1)
self.correlator = correlator_fixture.correlator
self.corr_fix = correlator_fixture
self.corr_freqs = CorrelatorFrequencyInfo(self.correlator.configd)
dsim_conf = self.correlator.configd['dsimengine']
dig_host = dsim_conf['host']
self.dhost = FpgaDsimHost(dig_host, config=dsim_conf)
self.dhost.get_system_information()
# Increase the dump rate so tests can run faster
self.correlator.xeng_set_acc_time(0.2)
self.addCleanup(self.corr_fix.stop_x_data)
self.corr_fix.start_x_data()
self.corr_fix.issue_metadata()
# TODO 2015-05-27 (NM) Do test using get_vacc_offset(test_dump['xeng_raw']) to see if
# the VACC is rotated. Run this test first so that we know immediately that other
# tests will be b0rked.
def test_channelisation(self):
"""TP.C.1.19 CBF Channelisation Wideband Coarse L-band"""
test_chan = 1500
expected_fc = self.corr_freqs.chan_freqs[test_chan]
init_dsim_sources(self.dhost)
self.dhost.sine_sources.sin_0.set(frequency=expected_fc, scale=0.25)
# The signal source is going to quantise the requested freqency, so see what we
# actually got
source_fc = self.dhost.sine_sources.sin_0.frequency
# Get baseline 0 data, i.e. auto-corr of m000h
test_baseline = 0
test_data = self.receiver.get_clean_dump(DUMP_TIMEOUT)['xeng_raw']
b_mag = normalised_magnitude(test_data[:, test_baseline, :])
# find channel with max power
max_chan = np.argmax(b_mag)
# self.assertEqual(max_chan, test_chan,
# 'Channel with max power is not the test channel')
requested_test_freqs = self.corr_freqs.calc_freq_samples(
test_chan, samples_per_chan=101, chans_around=5)
# Placeholder of actual frequencies that the signal generator produces
actual_test_freqs = []
# Channel magnitude responses for each frequency
chan_responses = []
last_source_freq = None
for i, freq in enumerate(requested_test_freqs):
# LOGGER.info('Getting channel response for freq {}/{}: {} MHz.'.format(
# i+1, len(requested_test_freqs), freq/1e6))
print ('Getting channel response for freq {}/{}: {} MHz.'.format(
i+1, len(requested_test_freqs), freq/1e6))
if freq == expected_fc:
# We've already done this one!
this_source_freq = source_fc
this_freq_result = b_mag
else:
self.dhost.sine_sources.sin_0.set(frequency=freq, scale=0.125)
this_source_freq = self.dhost.sine_sources.sin_0.frequency
if this_source_freq == last_source_freq:
LOGGER.info('Skipping channel response for freq {}/{}: {} MHz.\n'
'Digitiser frequency is same as previous.'.format(
i+1, len(requested_test_freqs), freq/1e6))
continue # Already calculated this one
else:
last_source_freq = this_source_freq
this_freq_data = self.receiver.get_clean_dump(DUMP_TIMEOUT)['xeng_raw']
this_freq_response = normalised_magnitude(
this_freq_data[:, test_baseline, :])
actual_test_freqs.append(this_source_freq)
chan_responses.append(this_freq_response)
self.corr_fix.stop_x_data()
# Convert the lists to numpy arrays for easier working
actual_test_freqs = np.array(actual_test_freqs)
chan_responses = np.array(chan_responses)
def plot_and_save(freqs, data, plot_filename):
df = self.corr_freqs.delta_f
fig = plt.plot(freqs, data)[0]
axes = fig.get_axes()
ybound = axes.get_ybound()
yb_diff = abs(ybound[1] - ybound[0])
new_ybound = [ybound[0] - yb_diff*1.1, ybound[1] + yb_diff * 1.1]
plt.vlines(expected_fc, *new_ybound, colors='r', label='chan fc')
plt.vlines(expected_fc - df / 2, *new_ybound, label='chan min/max')
plt.vlines(expected_fc - 0.8*df / 2, *new_ybound, label='chan +-40%',
linestyles='dashed')
plt.vlines(expected_fc + df / 2, *new_ybound, label='_chan max')
plt.vlines(expected_fc + 0.8*df / 2, *new_ybound, label='_chan +40%',
linestyles='dashed')
plt.legend()
plt.title('Channel {} ({} MHz) response'.format(
test_chan, expected_fc/1e6))
axes.set_ybound(*new_ybound)
plt.grid(True)
plt.ylabel('dB relative to VACC max')
# TODO Normalise plot to frequency bins
plt.xlabel('Frequency (Hz)')
plt.savefig(plot_filename)
plt.close()
graph_name = '{}.{}.channel_response.svg'.format(strclass(self.__class__),
self._testMethodName)
plot_data_all = loggerise(chan_responses[:, test_chan], dynamic_range=90)
plot_and_save(actual_test_freqs, plot_data_all, graph_name)
# Get responses for central 80% of channel
df = self.corr_freqs.delta_f
central_indices = (
(actual_test_freqs <= expected_fc + 0.8*df) &
(actual_test_freqs >= expected_fc - 0.8*df))
central_chan_responses = chan_responses[central_indices]
central_chan_test_freqs = actual_test_freqs[central_indices]
# Test responses in central 80% of channel
for i, freq in enumerate(central_chan_test_freqs):
max_chan = np.argmax(np.abs(central_chan_responses[i]))
self.assertEqual(max_chan, test_chan, 'Source freq {} peak not in channel '
'{} as expected but in {}.'
.format(freq, test_chan, max_chan))
# TODO Graph the central 80% too.
self.assertLess(
np.max(np.abs(central_chan_responses[:, test_chan])), 0.99,
'VACC output at > 99% of maximum value, indicates that '
'something, somewhere, is probably overranging.')
max_central_chan_response = np.max(10*np.log10(central_chan_responses[:, test_chan]))
min_central_chan_response = np.min(10*np.log10(central_chan_responses[:, test_chan]))
chan_ripple = max_chan_response - min_chan_response
acceptable_ripple_lt = 0.3
self.assertLess(chan_ripple, acceptable_ripple_lt,
'ripple {} dB within 80% of channel fc is >= {} dB'
.format(chan_ripple, acceptable_ripple_lt))
# from matplotlib import pyplot
# colour_cycle = 'rgbyk'
# style_cycle = ['-', '--']
# linestyles = itertools.cycle(itertools.product(style_cycle, colour_cycle))
# for i, freq in enumerate(actual_test_freqs):
# style, colour = linestyles.next()
# pyplot.plot(loggerise(chan_responses[:, i], dynamic_range=60), color=colour, ls=style)
# pyplot.ion()
# pyplot.show()
# import IPython ; IPython.embed()
def test_product_baselines(self):
"""CBF Baseline Correlation Products: VR.C.19, TP.C.1.3"""
init_dsim_sources(self.dhost)
# Put some correlated noise on both outputs
self.dhost.noise_sources.noise_corr.set(scale=0.5)
test_dump = self.receiver.get_clean_dump(DUMP_TIMEOUT)
# Get list of all the correlator input labels
input_labels = sorted(tuple(test_dump['input_labelling'][:,0]))
# Get list of all the baselines present in the correlator output
present_baselines = sorted(
set(tuple(bl) for bl in test_dump['bls_ordering']))
# Make a list of all possible baselines (including redundant baselines) for the
# given list of inputs
possible_baselines = set()
for li in input_labels:
for lj in input_labels:
possible_baselines.add((li, lj))
test_bl = sorted(list(possible_baselines))
# Test that each baseline (or its reverse-order counterpart) is present in the
# correlator output
baseline_is_present = {}
for test_bl in possible_baselines:
baseline_is_present[test_bl] = (test_bl in present_baselines or
test_bl[::-1] in present_baselines)
self.assertTrue(all(baseline_is_present.values()),
"Not all baselines are present in correlator output.")
test_data = test_dump['xeng_raw']
# Expect all baselines and all channels to be non-zero
self.assertFalse(zero_baselines(test_data))
self.assertEqual(nonzero_baselines(test_data),
all_nonzero_baselines(test_data))
# Save initial f-engine equalisations
initial_equalisations = {input: eq_info['eq'] for input, eq_info
in self.correlator.feng_eq_get().items()}
def restore_initial_equalisations():
for input, eq in initial_equalisations.items():
self.correlator.feng_eq_set(source_name=input, new_eq=eq)
self.addCleanup(restore_initial_equalisations)
# Set all inputs to zero, and check that output product is all-zero
for input in input_labels:
self.correlator.feng_eq_set(source_name=input, new_eq=0)
test_data = self.receiver.get_clean_dump(DUMP_TIMEOUT)['xeng_raw']
self.assertFalse(nonzero_baselines(test_data))
#-----------------------------------
all_inputs = sorted(set(input_labels))
zero_inputs = set(input_labels)
nonzero_inputs = set()
def calc_zero_and_nonzero_baselines(nonzero_inputs):
nonzeros = set()
zeros = set()
for inp_i in all_inputs:
for inp_j in all_inputs:
if inp_i in nonzero_inputs and inp_j in nonzero_inputs:
nonzeros.add((inp_i, inp_j))
else:
zeros.add((inp_i, inp_j))
return zeros, nonzeros
#zero_baseline, nonzero_baseline = calc_zero_and_nonzero_baselines(nonzero_inputs)
def print_baselines():
print ('zeros: {}\n\nnonzeros: {}\n\nnonzero-baselines: {}\n\n '
'zero-baselines: {}\n\n'.format(
sorted(zero_inputs), sorted(nonzero_inputs),
sorted(nonzero_baseline), sorted(zero_baseline)))
#print_baselines()
for inp in input_labels:
old_eqs = initial_equalisations[inp]
self.correlator.feng_eq_set(source_name=inp, new_eq=old_eqs)
zero_inputs.remove(inp)
nonzero_inputs.add(inp)
raise RuntimeError('No such log level: %s' % log_level)
if args.config:
config_filename = args.config
else:
try:
config_filename = os.environ['CORR2INI']
except KeyError:
raise RuntimeError(
'No config file speficied and environment var "CORR2INI" not defined')
corr_conf = utils.parse_ini_file(config_filename, ['dsimengine'])
dsim_conf = corr_conf['dsimengine']
dig_host = dsim_conf['host']
dhost = FpgaDsimHost(dig_host, config=dsim_conf)
print 'Connected to %s.' % dhost.host
if args.start and args.stop:
raise RuntimeError('Start and stop? You must be crazy!')
something_happened = False
if args.program:
dhost.initialise()
something_happened = True
else:
dhost.get_system_information()
if args.deprogram:
dhost.deprogram()
something_happened = True
# make the fpga
# dfpga = utils.script_get_fpga(args, host_index=0, host_type='dsimengine',
# fpga_class=FpgaDsimHost)
if 'CORR2INI' in os.environ.keys() and (args.config == '' or args.config is None):
args.config = os.environ['CORR2INI']
try:
config, host_detail = utils.hosts_and_bitstreams_from_config(
config_file=args.config, section='dsimengine')
section, host_list, bitstream = host_detail[0]
if args.bitstream:
bitstream = str(args.bitstream)
print ('Starting Digitiser with bitstream: {}'.format(bitstream))
# Note: We have hardcoded the transport to be SkarabTransport - Might change in the future!
dfpga = FpgaDsimHost(host_list[0], bitstream=bitstream, config=config.get('dsimengine'),
transport=SkarabTransport)
print('Connected to {}.'.format(dfpga.host))
except TypeError:
raise RuntimeError('Config template was not parsed!!!')
if args.start and args.stop:
raise RuntimeError('Start and stop? You must be crazy!')
something_happened = False
if args.program:
dfpga.initialise()
something_happened = True
else:
dfpga.get_system_information(bitstream)
# # TODO HACK
class CorrelatorFixture(object):
def __init__(self, katcp_client=None, product_name=None):
self.katcp_client = katcp_client
self.corr_config = None
self.corr2ini_path = None
self._correlator = None
self._dhost = None
self._katcp_rct = None
self._katcp_rct_sensor = None
self._rct = None
self.katcp_array_port = None
self.katcp_sensor_port = None
self.product_name = product_name
self.halt_wait_time = 5
# Assume the correlator is already started if start_correlator is False
nose_test_config = {}
self._correlator_started = not int(
nose_test_config.get('start_correlator', False))
self.test_config = self._test_config_file
self.subarray = self.test_config['instrument_params']['subarray']
self.config_filename = max(iglob('/etc/corr/{}-*'.format(
self.subarray)),
key=os.path.getctime)
self.array_name, self.instrument = self._get_instrument
@property
def rct(self):
if self._rct is not None:
return self._rct
else:
self.io_manager = ioloop_manager.IOLoopManager()
self.io_wrapper = resource_client.IOLoopThreadWrapper(
self.io_manager.get_ioloop())
add_cleanup(self.io_manager.stop)
self.io_wrapper.default_timeout = _timeout
self.io_manager.start()
self.rc = resource_client.KATCPClientResource(
dict(name='{}'.format(self.katcp_client),
address=('{}'.format(self.katcp_client), '7147'),
controlled=True))
self.rc.set_ioloop(self.io_manager.get_ioloop())
self._rct = (resource_client.ThreadSafeKATCPClientResourceWrapper(
self.rc, self.io_wrapper))
self._rct.start()
add_cleanup(self._rct.stop)
try:
self._rct.until_synced(timeout=_timeout)
except TimeoutError:
self._rct.stop()
return self._rct
@property
def dhost(self, program=False):
if self._dhost is not None:
return self._dhost
else:
if os.path.exists(self.config_filename):
LOGGER.info(
'Retrieving dsim engine info from config file: %s' %
self.config_filename)
self.corr_config = parse_ini_file(self.config_filename)
self.dsim_conf = self.corr_config['dsimengine']
elif self.instrument is not None:
self.corr2ini_path = '/etc/corr/templates/{}'.format(
self.instrument)
LOGGER.info(
'Setting CORR2INI system environment to point to %s' %
self.corr2ini_path)
os.environ['CORR2INI'] = self.corr2ini_path
self.corr_config = parse_ini_file(self.corr2ini_path)
self.dsim_conf = self.corr_config['dsimengine']
else:
errmsg = (
'Could not retrieve dsim information from running config file in /etc/corr, '
'Perhaps, restart CBF manually and ensure dsim is running.\n'
)
LOGGER.error(errmsg)
sys.exit(errmsg)
try:
dig_host = self.dsim_conf['host']
self._dhost = None
self._dhost = FpgaDsimHost(dig_host, config=self.dsim_conf)
except Exception:
errmsg = 'Digitiser Simulator failed to retrieve information'
LOGGER.exception(errmsg)
sys.exit(errmsg)
else:
# Check if D-eng is running else start it.
if self._dhost.is_running():
LOGGER.info('D-Eng is already running.')
return self._dhost
# Disabled DSim programming as it would alter the systems sync epoch
# elif program and not self._dhost.is_running():
# LOGGER.info('Programming and starting the Digitiser Simulator.')
# self._dhost.initialise()
# self._dhost.enable_data_output(enabled=True)
# self._dhost.registers.control.write(gbe_txen=True)
# else:
# LOGGER.info('D-Eng started successfully')
# return self._dhost
@property
def correlator(self):
if self._correlator is not None:
LOGGER.info('Using cached correlator instance')
return self._correlator
else: # include a check, if correlator is running else start it.
if not self._correlator_started:
LOGGER.info('Correlator not running, now starting.')
print 'Correlator not running: This shouldnt happen, fix it\n' * 10
#self.start_correlator()
# We assume either start_correlator() above has been called, or the
# instrument was started with the name contained in self.array_name
# before running the test.
_retries = 3
if os.path.exists(self.config_filename):
LOGGER.info('Making new correlator instance')
while True:
_retries -= 1
try:
self._correlator = fxcorrelator.FxCorrelator(
'test correlator',
config_source=self.config_filename)
time.sleep(1)
try:
self.correlator.initialise(program=False,
configure=False)
except TypeError:
self.correlator.initialise(program=False)
return self._correlator
except Exception as e:
LOGGER.exception(
'Failed to create new correlator instance with error: %s, '
'Will now try to start correlator with config: %s-%s'
% (str(e), self.array_name, self.instrument))
continue
if _retries == 0:
break
if _retries == 0:
self.start_correlator(instrument=self.instrument)
else:
LOGGER.error(
'No Config file (/etc/corr/array*-instrument), '
'Starting correlator with default instrument: %s' %
(self.instrument))
self.start_correlator(instrument=self.instrument)
@property
def katcp_rct(self):
if self._katcp_rct is None:
try:
katcp_prot = self.test_config['instrument_params'][
'katcp_protocol']
_major, _minor, _flags = katcp_prot.split(',')
protocol_flags = ProtocolFlags(int(_major), int(_minor),
_flags)
LOGGER.info('katcp protocol flags %s' % protocol_flags)
LOGGER.info('Getting running array.')
reply, informs = self.rct.req.subordinate_list(self.array_name)
assert reply.reply_ok()
# If no sub-array present create one, but this could cause problems
# if more than one sub-array is present. Update this to check for
# required sub-array.
except Exception:
LOGGER.exception('Failed to list all arrays with name: %s' %
self.array_name)
else:
try:
try:
self.katcp_array_port = int(informs[0].arguments[1])
LOGGER.info(
'Current running array name: %s, port: %s' %
(self.array_name, self.katcp_array_port))
except ValueError:
self.katcp_array_port, self.katcp_sensor_port = informs[
0].arguments[1].split(',')
LOGGER.info(
'Current running array name: %s, port: %s, sensor port: %s'
% (self.array_name, self.katcp_array_port,
self.katcp_sensor_port))
except Exception:
errmsg = (
'Failed to retrieve running array, ensure one has been created and running'
)
LOGGER.exception(errmsg)
sys.exit(errmsg)
else:
katcp_rc = resource_client.KATCPClientResource(
dict(name='{}'.format(self.katcp_client),
address=('{}'.format(self.katcp_client),
'{}'.format(self.katcp_array_port)),
preset_protocol_flags=protocol_flags,
controlled=True))
katcp_rc.set_ioloop(self.io_manager.get_ioloop())
self._katcp_rct = (
resource_client.ThreadSafeKATCPClientResourceWrapper(
katcp_rc, self.io_wrapper))
self._katcp_rct.start()
try:
self._katcp_rct.until_synced(timeout=_timeout)
except Exception as e:
self._katcp_rct.stop()
LOGGER.exception(
'Failed to connect to katcp due to %s' % str(e))
else:
return self._katcp_rct
else:
if not self._katcp_rct.is_active():
LOGGER.info(
'katcp resource client wasnt running, hence we need to start it.'
)
self._katcp_rct.start()
try:
time.sleep(1)
self._katcp_rct.until_synced(timeout=_timeout)
return self._katcp_rct
except Exception:
self._katcp_rct.stop()
LOGGER.exception('Failed to connect to katcp')
else:
return self._katcp_rct
@property
def katcp_rct_sensor(self):
if self._katcp_rct_sensor is None:
try:
katcp_prot = self.test_config['instrument_params'][
'katcp_protocol']
_major, _minor, _flags = katcp_prot.split(',')
protocol_flags = ProtocolFlags(int(_major), int(_minor),
_flags)
LOGGER.info('katcp protocol flags %s' % protocol_flags)
LOGGER.info('Getting running array.')
reply, informs = self.rct.req.subordinate_list(self.array_name)
assert reply.reply_ok()
# If no sub-array present create one, but this could cause problems
# if more than one sub-array is present. Update this to check for
# required sub-array.
except Exception:
LOGGER.exception('Failed to list all arrays with name: %s' %
self.array_name)
else:
try:
try:
self.katcp_array_port = int(informs[0].arguments[1])
LOGGER.info(
'Current running array name: %s, port: %s' %
(self.array_name, self.katcp_array_port))
except ValueError:
self.katcp_array_port, self.katcp_sensor_port = informs[
0].arguments[1].split(',')
LOGGER.info(
'Current running array name: %s, port: %s, sensor port: %s'
% (self.array_name, self.katcp_array_port,
self.katcp_sensor_port))
except Exception:
errmsg = (
'Failed to retrieve running array, ensure one has been created and running'
)
LOGGER.exception(errmsg)
sys.exit(errmsg)
else:
katcp_rc = resource_client.KATCPClientResource(
dict(name='{}'.format(self.katcp_client),
address=('{}'.format(self.katcp_client),
'{}'.format(self.katcp_sensor_port)),
preset_protocol_flags=protocol_flags,
controlled=True))
katcp_rc.set_ioloop(self.io_manager.get_ioloop())
self._katcp_rct_sensor = (
resource_client.ThreadSafeKATCPClientResourceWrapper(
katcp_rc, self.io_wrapper))
self._katcp_rct_sensor.start()
try:
self._katcp_rct_sensor.until_synced(timeout=_timeout)
except Exception as e:
self._katcp_rct_sensor.stop()
LOGGER.exception(
'Failed to connect to katcp due to %s' % str(e))
else:
return self._katcp_rct_sensor
else:
if not self._katcp_rct_sensor.is_active():
LOGGER.info(
'katcp resource client wasnt running, hence we need to start it.'
)
self._katcp_rct_sensor.start()
try:
time.sleep(1)
self._katcp_rct_sensor.until_synced(timeout=_timeout)
return self._katcp_rct_sensor
except Exception:
self._katcp_rct_sensor.stop()
LOGGER.exception('Failed to connect to katcp')
else:
return self._katcp_rct_sensor
@property
def issue_metadata(self):
"""Issue Spead metadata"""
try:
reply, informs = self.katcp_rct.req.capture_meta(self.product_name,
timeout=_timeout)
assert reply.reply_ok()
except Exception:
LOGGER.exception('Failed to issue new metadata')
return False
else:
return True
@property
def start_x_data(self):
"""
Enable/Start output product capture
"""
try:
assert isinstance(
self.katcp_rct,
resource_client.ThreadSafeKATCPClientResourceWrapper)
reply, informs = self.katcp_rct.req.capture_list(timeout=_timeout)
assert reply.reply_ok()
self.product_name = [
i.arguments[0] for i in informs if self.corr_config['xengine']
['output_products'] in i.arguments
][0]
assert self.product_name is not None
LOGGER.info('Capturing %s product' % self.product_name)
except Exception:
self.product_name = self.corr_config['xengine']['output_products']
LOGGER.exception(
'Failed to retrieve capture list via CAM interface, '
'got it from config file.')
try:
reply, informs = self.katcp_rct.req.capture_start(
self.product_name)
assert reply.reply_ok()
LOGGER.info(' %s' % str(reply))
Aqf.progress(str(reply) + '\n')
return True
except Exception:
LOGGER.exception('Failed to capture start: %s' % str(reply))
return False
def stop_x_data(self):
"""
Disable/Stop output product capture
"""
try:
assert self.product_name is not None
assert isinstance(
self.katcp_rct,
resource_client.ThreadSafeKATCPClientResourceWrapper)
reply, informs = self.katcp_rct.req.capture_stop(self.product_name,
timeout=_timeout)
assert reply.reply_ok()
LOGGER.info(' %s' % str(reply))
Aqf.progress(str(reply))
return True
except Exception:
LOGGER.exception(
'Failed to capture stop, might be because config file does not contain '
'Xengine output products.')
return False
def get_running_instrument(self):
"""
Returns currently running instrument listed on the sensor(s)
"""
try:
reply = None
reply = self.katcp_rct.sensor.instrument_state.get_reading()
assert reply.istatus
return reply.value
except Exception as e:
LOGGER.exception('KATCP Request failed due to error: %s/%s' %
(str(e), str(reply)))
return False
except KATCPSensorError:
LOGGER.exception('KATCP Error polling sensor\n')
return False
except AssertionError:
LOGGER.exception(
'Sensor request failed: %s, Forcefully Halting the Array' %
(str(reply)))
return False
def ensure_instrument(self,
instrument,
retries=5,
force_reinit=False,
**kwargs):
"""Ensure that named instrument is active on the correlator array
Will pass `kwargs` to self.start_correlator if a start is required
:param self: Object
:param instrument: CBF Instrument
:param retries: No of instrument validation retries
:param force_reinit: Force an instrument re-initialisation
:rtype: Boolean
"""
try:
assert '_' in self.instrument
self.instrument = self.instrument.split('_')[0]
except AssertionError:
pass
if force_reinit:
LOGGER.info('Forcing an instrument(%s) re-initialisation' %
self.instrument)
corr_success = self.start_correlator(self.instrument, **kwargs)
return corr_success
success = False
while retries and not success:
check_ins = self.check_instrument(self.instrument)
msg = 'Will retry to check if the instrument is up: #%s retry.' % retries
LOGGER.info(msg)
if check_ins is True:
success = True
LOGGER.info('Named instrument (%s) is currently running' %
self.instrument)
return success
retries -= 1
if self.check_instrument(self.instrument) is False:
LOGGER.info(
'Correlator not running requested instrument, will restart.')
reply = self.katcp_rct.sensor.instrument_state.get_reading()
if reply.value == self.instrument:
pass
corr_success = self.start_correlator(self.instrument, **kwargs)
return True if corr_success is True else False
def check_instrument(self, instrument):
"""Return true if named instrument is enabled on correlator array
Uses the correlator array KATCP interface to check if the requested
instrument is active
:param instrument: Correlator
"""
self.instrument = instrument
self._errmsg = None
try:
self._errmsg = 'Instrument cannot be None.'
assert instrument is not None, self._errmsg
self._errmsg = 'katcp client is not an instance of resource client'
assert isinstance(
self.katcp_rct, resource_client.
ThreadSafeKATCPClientResourceWrapper), self._errmsg
self._errmsg = 'katcp client failed to establish a connection'
assert self.katcp_rct.wait_connected(), self._errmsg
self._errmsg = 'katcp client failed to sync after establishing a connection'
assert self.katcp_rct.until_state('synced',
timeout=60), self._errmsg
except AssertionError:
# This probably means that no array has been defined yet and therefore the
# katcp_rct client cannot be created. IOW, the desired instrument would
# not be available
LOGGER.exception(self._errmsg)
return False
else:
try:
reply, informs = self.katcp_rct.req.instrument_list()
assert reply.reply_ok()
instruments_available = [
instrument_avail.arguments[0].split('_')[0]
for instrument_avail in informs
]
# Test to see if requested instrument is available on the instrument list
assert instrument in instruments_available
except Exception:
LOGGER.exception('Array request failed might have timed-out')
except AssertionError:
LOGGER.exception(
'Array request failed: %s or Instrument: %s is not in instrument '
'list: %s' %
(instrument, instruments_available, str(reply)))
return False
try:
reply = self.katcp_rct.sensor.instrument_state.get_reading()
assert reply.istatus
except AttributeError:
LOGGER.exception(
'Instrument state could not be retrieved from the sensors')
return False
except AssertionError:
LOGGER.error('%s: No running instrument' % str(reply))
return False
else:
running_intrument = reply.value.split('_')[0]
instrument_present = instrument == running_intrument
if instrument_present:
self.instrument = instrument
LOGGER.info(
'Confirmed that the named instrument %s is running' %
self.instrument)
return instrument_present
@property
def _get_instrument(self):
"""
Retrieve currently running instrument from /etc/corr
return: List
"""
try:
running_instr = self.config_filename.split('/')[-1]
self.array_name, self.instrument = running_instr.split('-')
try:
assert '_' in self.instrument
self.instrument = self.instrument.split('_')[0]
except AssertionError:
pass
if (self.instrument.startswith('bc') or self.instrument.startswith('c')) and \
self.array_name.startswith('array'):
LOGGER.info(
'Currently running instrument %s as per /etc/corr' %
self.instrument)
return [self.array_name, self.instrument]
except Exception:
LOGGER.exception(
'Could not retrieve information from config file, resorting to default'
)
return ['array0', 'bc8n856M4k']
except ValueError:
LOGGER.exception('Directory missing array config file.')
@property
def _test_config_file(self):
"""
Configuration file containing information such as dsim, pdu and dataswitch ip's
return: Dict
"""
path, _none = os.path.split(__file__)
path, _none = os.path.split(path)
try:
assert os.uname()[1].startswith('dbelab')
except AssertionError:
conf_path = 'config/test_conf_site.ini'
else:
conf_path = 'config/test_conf_lab.ini'
config_file = os.path.join(path, conf_path)
if os.path.isfile(config_file) or os.path.exists(config_file):
try:
config = parse_ini_file(config_file)
return config
except (IOError, ValueError, TypeError):
errmsg = (
'Failed to read test config file %s, Test will exit' %
(config_file))
LOGGER.exception(errmsg)
return False
else:
LOGGER.error('Test config path: %s does not exist' % config_file)
return False
@property
def get_multicast_ips(self):
"""
Retrieves multicast ips from test configuration file and calculates the number
of inputs depending on which instrument is being initialised
:param instrument: Correlator
"""
if self.instrument is None:
return False
try:
multicast_ip_inp = [
self.corr_config['dsimengine'].get('pol0_destination_start_ip',
'239.101.0.64'),
self.corr_config['dsimengine'].get('pol1_destination_start_ip',
'239.101.0.66')
]
except TypeError:
msg = (
'Could not read and split the multicast IPs in the test config file'
)
LOGGER.exception(msg)
return False
else:
if self.instrument.startswith('bc') or self.instrument.startswith(
'c'):
if self.instrument[0] == 'b':
try:
# multicast_ip = multicast_ip_inp * (int(self.instrument[2]) / 2)
multicast_ip = multicast_ip_inp * (int(
self.instrument.replace('bc', '').split('n')[0]) /
2)
return multicast_ip
except Exception:
LOGGER.error(
'Could not calculate multicast IPs from config file'
)
return False
else:
try:
multicast_ip = multicast_ip_inp * (int(
self.instrument.replace('c', '').split('n')[0]) /
2)
return multicast_ip
except Exception:
LOGGER.error(
'Could not calculate multicast IPs from config file'
)
return False
def start_correlator(self, instrument=None, retries=10):
LOGGER.debug('CBF instrument(%s) re-initialisation.' % instrument)
success = False
self.katcp_array_port = None
if instrument is not None:
self.instrument = instrument
self._correlator = None # Invalidate cached correlator instance
LOGGER.info('Confirm DEngine is running before starting correlator')
if not self.dhost.is_running():
raise RuntimeError('DEngine: %s not running.' % (self.dhost.host))
multicast_ip = self.get_multicast_ips
if not multicast_ip:
LOGGER.error('Failed to calculate multicast IP\'s, investigate')
self._katcp_rct = None
self.rct.start()
try:
self.rct.until_synced(timeout=_timeout)
reply, informs = self.rct.req.subordinate_list(self.array_name)
assert reply.reply_ok()
except TimeoutError:
self.rct.stop()
LOGGER.exception('Resource client timed-out after %s s' % _timeout)
return False
except AssertionError:
LOGGER.exception(
'Failed to get subordinate-list, might not have been assigned, '
'Will try to assign.')
else:
try:
informs = informs[0]
self.katcp_array_port = int(
[i for i in informs.arguments if len(i) == 5][0])
except ValueError:
LOGGER.exception('Failed to assign katcp port: Reply: %s' %
(reply))
reply = self.rct.req.subordinate_halt(self.array_name)
if not reply.succeeded:
LOGGER.error('Unable to halt array %s: %s' %
(self.array_name, reply))
return False
while retries and not success:
try:
if self.katcp_array_port is None:
LOGGER.info('Assigning array port number')
try:
pass
# reply, _informs = self.rct.req.subordinate_create(self.array_name,
# *multicast_ip, timeout=_timeout)
# assert reply.reply_ok()
except Exception:
self.katcp_array_port = None
LOGGER.exception('Failed to assign new array: %s' %
self.array_name)
else:
self.katcp_array_port = int(reply.arguments[-1])
LOGGER.info('Successfully created %s-%s' %
(self.array_name, self.katcp_array_port))
instrument_param = ([
int(i) for i in self.test_config['instrument_params']
['instrument_param'] if i != ','
])
LOGGER.info('Starting %s with %s parameters. Try #%s' %
(self.instrument, instrument_param, retries))
# TODO add timeout
reply = self.katcp_rct.req.instrument_activate(
self.instrument, *instrument_param, timeout=500)
success = reply.succeeded
retries -= 1
try:
assert success
LOGGER.info('Instrument %s started successfully' %
(self.instrument))
except AssertionError:
LOGGER.exception(
'Failed to start correlator, %s attempts left. '
'Restarting Correlator. Reply:%s' % (retries, reply))
success = False
except Exception:
try:
self.rct.req.subordinate_halt(self.array_name)
assert isinstance(
self.katcp_rct,
resource_client.ThreadSafeKATCPClientResourceWrapper)
except Exception:
LOGGER.exception(
'Unable to halt array: Empty Array number')
except AssertionError:
LOGGER.exception(
'self.katcp_rct has not been initiated successfully')
return False
else:
try:
self.katcp_rct.stop()
except AttributeError:
LOGGER.error(
'KATCP request does not contain attributes')
return False
else:
retries -= 1
LOGGER.warn(
'Failed to start correlator, %s attempts left.\n' %
(retries))
if retries < 0:
success = False
return success
if success:
self._correlator_started = True
return self._correlator_started
else:
try:
self._correlator_started = False
self.katcp_rct.stop()
self.rct.stop()
self._katcp_rct = None
self._correlator = None
except:
msg = (
'Could not successfully start correlator within %s retries'
% (retries))
LOGGER.critical(msg)
return False
return False
