def printMessages(self):
for i in self._records:
if i.exc_info:
print termcolors.colorize("%s: %s Exception: " % (i.name, i.msg), i.exc_info[0:-1], fg="red")
else:
if i.levelno < logging.WARNING:
print termcolors.colorize("%s: %s" % (i.name, i.msg), fg="green")
elif (i.levelno >= logging.WARNING) and (i.levelno < logging.ERROR):
print termcolors.colorize("%s: %s" % (i.name, i.msg), fg="yellow")
elif i.levelno >= logging.ERROR:
print termcolors.colorize("%s: %s" % (i.name, i.msg), fg="red")
else:
print "%s: %s" % (i.name, i.msg)
def printMessages(self):
for i in self._records:
if i.exc_info:
print termcolors.colorize('%s: %s Exception: ' %
(i.name, i.msg),
i.exc_info[0:-1],
fg='red')
else:
if i.levelno < logging.WARNING:
print termcolors.colorize('%s: %s' % (i.name, i.msg),
fg='green')
elif (i.levelno >= logging.WARNING) and (i.levelno <
logging.ERROR):
print termcolors.colorize('%s: %s' % (i.name, i.msg),
fg='yellow')
elif i.levelno >= logging.ERROR:
print termcolors.colorize('%s: %s' % (i.name, i.msg),
fg='red')
else:
print '%s: %s' % (i.name, i.msg)
def trans_test(roach, desired_rate = 8.0, pkt_size = 1024, skip = False, debug = False, clk_chk = True, auto_test = False):
""" Test SFP+ Boards connected to a ROACH2 rev2
Keyword arguments:
roach -- ROACH2 ip address or host name
desired_rate -- Set the datarate in Gbps. Default: 8
pkt_size -- Set the packet size in bytes. Will be rounded to nearest 8 bytes. Default: 1024
skip -- Skip initialisation and reprogramming of FPGAs.
debug -- Print out complete details of each 10GbE core, including ARP tables.
clk_chk -- Skip the clock check.
auto_test -- Automatic test; just run the test for a few seconds and then print a summary'
"""
try:
loggers = []
lh=corr.log_handlers.DebugLogHandler()
logger = logging.getLogger(roach)
logger.addHandler(lh)
logger.setLevel(logging.ERROR)
print('Connecting to server %s on port %i... '%(roach,katcp_port)),
sys.stdout.flush()
fpga = corr.katcp_wrapper.FpgaClient(roach, katcp_port, timeout=10,logger=logger)
time.sleep(1)
conn_err = False
if fpga.is_connected():
print 'ok'
else:
print 'ERROR connecting to server %s on port %i.\n'%(roach,katcp_port)
conn_err = True
connections=[float("-inf"),float("-inf"),float("inf"),float("inf"),float("-inf"),float("-inf"),float("-inf"),float("-inf")]
print '------------------------'
print '\tProgramming FPGA...',
sys.stdout.flush()
if not skip:
for tx_n in range(8):
try:
fpga.tap_stop('gbe%i'%tx_n)
except:
continue
fpga.progdev(bitstream)
print 'done'
print '\tEstimated FPGA clock rate:',
sys.stdout.flush()
if clk_chk:
est_fpga_clk=fpga.est_brd_clk()
print ' %iMHz (expecting %iMHz).'%(est_fpga_clk,200),
if est_fpga_clk > 201 or est_fpga_clk < 199:
raise RuntimeError("FPGA clock rate of %i is outside allowable limits (expecting 200MHz). Sysclk failure."%est_fpga_clk)
else:
print 'ok.'
else:
print 'Skipped'
else:
print 'Skipped.'
if not skip:
print '\tStarting tap devices on ports 0 through 7...',
sys.stdout.flush()
for tx_n in range(8):
fpga.tap_start(tap_dev=('gbe%i'%tx_n),device='gbe%i'%tx_n,mac=(20015998304256+tx_n),ip=(10<<24)+10+tx_n,port=10000)
print 'done'
print "\tWaiting 12s for ARP to complete...",
sys.stdout.flush()
time.sleep(12)
print 'done'
if debug:
for rx_n in range(8):
print '============================================'
print '\t\tPORT %i'%rx_n
print '============================================'
fpga.print_10gbe_core_details('gbe%i'%rx_n,arp=True)
print '\tGrabbing port statuses...',
sys.stdout.flush()
stat=[]
for tx_n in range(8):
stat.append(fpga.get_10gbe_core_details('gbe%i'%tx_n))
print 'done'
print '\tChecking RX bonding of link from mezzanine cards...',
sys.stdout.flush()
for tx_n in range(8):
if not stat[tx_n]['xaui_chan_bond']:
print termcolors.colorize('\t\tChannel %i lane bonding FAIL! Bad RX links on mezzanine? No mezzanine present? Lane status: %i %i %i %i'%(tx_n,stat[tx_n]['xaui_lane_sync'][0],stat[tx_n]['xaui_lane_sync'][1],stat[tx_n]['xaui_lane_sync'][2],stat[tx_n]['xaui_lane_sync'][3]),fg='red')
time.sleep(5)
#raise RuntimeError('Channel %i lane bonding FAIL! Lane status:'%tx_n,stat[tx_n]['xaui_lane_sync'])
print 'done'
print '\tChecking tgtap correctly configured cores...',
sys.stdout.flush()
for tx_n in range(8):
if not ((stat[tx_n]['arp'][tx_n+10] == 20015998304256+tx_n) and (stat[tx_n]['my_ip'] ==(10<<24)+10+tx_n)):
print 'Channel %i config FAIL!'%tx_n
fpga.print_10gbe_core_details('gbe%i'%tx_n,arp=True)
print 'done'
print '\tChecking ARP entries for valid cable links...'
found_so_far = []
sys.stdout.flush()
for rx_n in range(8):
found_this_one=False
ip=0
while ip<255 and not found_this_one:
ip += 1
if stat[rx_n]['arp'][ip] != 0xffffffffffff:
# print 'Found an IP at %i...'%ip,
if (stat[rx_n]['arp'][ip] != (20015998304256+rx_n)) and not (ip in found_so_far): #check that it's not our address that we've found!
found_so_far.append(ip)
connections[ip-10]=rx_n
found_this_one = True
print termcolors.colorize('\t\tPort %i is receiving from IP %s (port %i).'%(rx_n,ip2str(ip-10),ip-10),fg='green')
# else:
# print 'This is our IP or is already accounted for. MAC is %i.'%stat[rx_n]['arp'][ip]
if found_this_one == False:
print termcolors.colorize('\t\tERROR: Port %i is not receiving ARP packets from any other port.'%rx_n,fg='red')
connections[rx_n]=float("-inf")
reset=False
packet_words=pkt_size/8
on_wire_pkt_size=(packet_words*8)+80
print '\tSetting the packet size to %i bytes (%i 64-bit words)...'%(packet_words*8,packet_words),
sys.stdout.flush()
for tx_n,rx_n in enumerate(connections):
old_size=fpga.read_uint('pkt_size%i'%tx_n)
if old_size != packet_words:
fpga.write_int('pkt_size%i'%tx_n,packet_words)
sys.stdout.flush()
reset=True
print tx_n,
if reset:
print ''
else:
print 'skipped'
desired_dn=desired_rate*1000/200./64.
desired_n=packet_words
desired_d=max(2,(desired_n/desired_dn))
rate_d=int(desired_d)-1
rate_n=int(desired_n)
act_payld_rate=200.*64*rate_n/(rate_d+1)
act_rate=act_payld_rate*(on_wire_pkt_size)/pkt_size
print '\tSetting the payload data rate to %6.2f Mbps in both directions on all ports...'%act_payld_rate,
sys.stdout.flush()
for tx_n,rx_n in enumerate(connections):
old_n=fpga.read_uint('pkt_rate%i_n'%tx_n)
old_d=fpga.read_uint('pkt_rate%i_d'%tx_n)
if (old_n != rate_n) or (old_d != rate_d):
fpga.write_int('pkt_rate%i_n'%tx_n,rate_n)
fpga.write_int('pkt_rate%i_d'%tx_n,rate_d)
reset=True
print tx_n,
sys.stdout.flush()
if reset:
print ''
else:
print 'skipped'
print '\tSetting destination IP addresses and ports...',
sys.stdout.flush()
for tx_n,rx_n in enumerate(connections):
if (rx_n<0):
print termcolors.colorize('\t\tDisabling TX on port %i due to no associated receiver.'%(tx_n),fg='red')
fpga.write_int('pkt_size%i'%tx_n,0)
else:
fpga.write_int('ip%i'%tx_n,(10<<24)+10+rx_n)
fpga.write_int('port%i'%tx_n,10000)
print 'done'
if reset==True:
print '\tResetting counters...',
sys.stdout.flush()
#fpga.write_int('control',4)
fpga.write_int('control',0)
fpga.write_int('control',2)
fpga.write_int('control',0)
#fpga.write_int('control',4)
print 'done'
print '\tStarting transmission...',
sys.stdout.flush()
fpga.write_int('control',4)
print 'done'
print '\tChecking that packets are being sent...',
sys.stdout.flush()
for tx_n,rx_n in enumerate(connections):
if not (rx_n<0):
if fpga.read_uint('tx_pkt_cnt%i'%tx_n)<10:
print 'ERROR: Port %i is not sending any data!'%tx_n
else:
print '%i...'%tx_n,
sys.stdout.flush()
print 'done'
cont=15
core_stat=range(8)
link_stat=range(8)
link=range(8)
tx_cnt=range(8)
tx_tib=range(8)
rx_cnt_ok=range(8)
rx_cnt_bad=range(8)
rx_wrd_cnt_fail=range(8)
rx_wrd_cnt_missing=range(8)
error_rate=range(8)
start_time=time.time()
while(cont>=0):
for core in range(8):
core_stat[core]=fpga.get_10gbe_core_details('gbe%i'%core)
link_stat[core]=fpga.read_uint('status%i'%core)
link[core]=True
for i in range(4):
if not core_stat[core]['xaui_lane_sync'][i]: link[core]=False
if not core_stat[core]['xaui_chan_bond']: link[core]=False
rx_cnt_ok[core] = fpga.read_uint('rx_pkt_cnt_ok%i'%core)
rx_cnt_bad[core] = fpga.read_uint('rx_pkt_cnt_bad%i'%core)
rx_wrd_cnt_fail[core] = fpga.read_uint('rx_wrd_cnt_bad%i'%core)
rx_wrd_cnt_missing[core] = fpga.read_uint('rx_wrd_cnt_missing%i'%core)
for core in range(8):
tx_cnt[core]=fpga.read_uint('tx_pkt_cnt%i'%core)
tx_tib[core]=tx_cnt[core]*(pkt_size+80)/1024/1024/1024.
for lnk in range(8):
try:
rx_n=connections[lnk]
tx_n=lnk
if not (rx_n<0):
error_rate[lnk]=float(rx_wrd_cnt_fail[rx_n] + rx_wrd_cnt_missing[rx_n] + rx_cnt_bad[rx_n]*packet_words)/float(tx_cnt[tx_n]*packet_words)
else:
error_rate[lnk]=-1.
except ZeroDivisionError:
#print 'No errors detected on port %i.'%lnk
error_rate[lnk]=-1.
#Move cursor home:
print '%c[H'%chr(27)
#clear the screen:
print '%c[2J'%chr(27)
print 'Running time: %is.'%(time.time()-start_time)
print ''
print 'Packet payload size %6i bytes'%(packet_words*8)
print 'On wire packet size %6i bytes'%on_wire_pkt_size
print 'Packet payload efficiency %7.2f %%'%(float(packet_words*800)/on_wire_pkt_size)
print 'TX payload rate %7.2f Mbps'%act_payld_rate
print 'TX total rate %7.2f Mbps'%act_rate
print ''
print 'Mezzanine mapping:'
print '0: Card0, port 0'
print '1: Card0, port 1'
print '2: Card0, port 2'
print '3: Card0, port 3'
print '4: Card1, port 0'
print '5: Card1, port 1'
print '6: Card1, port 2'
print '7: Card1, port 3'
print ''
for lnk in range(8):
tx_n=lnk
if link_stat[tx_n]&0b1: print termcolors.colorize('ERR: RX overflow on port %i!'%tx_n,fg='red')
if link_stat[tx_n]&0b10: print termcolors.colorize('ERR: TX overflow on port %i!'%tx_n,fg='red')
if not link_stat[tx_n]&(1<<5): print termcolors.colorize('ERR: Link down on port %i!'%tx_n,fg='red')
print 'LINK TX (GiB) TX_pkts RX_pkts_ok RX_pkts_bad 64b_fails Dropped_pkts BER'
for lnk in range(8):
rx_n=connections[lnk]
tx_n=lnk
if (rx_n>=0):
print termcolors.colorize('%2i ->%2i'%(tx_n,rx_n),fg='green'),
else:
print termcolors.colorize('%2i ->??'%(tx_n),fg='red'),
#print '%12s'%(ip2str(core_stat[tx_n]['my_ip'])),
print '%12.2f'%(tx_tib[tx_n]),
print '%12i'%(tx_cnt[tx_n]),
try:
if (link[lnk] and rx_n>=0):
print termcolors.colorize('%12i'%(rx_cnt_ok[rx_n]),fg='green' if rx_cnt_ok[rx_n]>0 else 'red'),
print termcolors.colorize('%12i'%(rx_cnt_bad[rx_n]),fg='green' if rx_cnt_bad[rx_n]==0 else 'red'),
#rx_wrd_cnt_Fail sometimes contain 1, this is due to initialization error and is not conciderd a link error. So only show errors if the count is more than 1
print termcolors.colorize('%12i'%(rx_wrd_cnt_fail[rx_n]),fg='green' if rx_wrd_cnt_fail[rx_n]<2 else 'red'),
print termcolors.colorize('%12i'%(rx_wrd_cnt_missing[rx_n]),fg='green' if rx_wrd_cnt_missing[rx_n]==0 else 'red'),
print termcolors.colorize(' %12e'%(error_rate[lnk]),fg='green' if error_rate[lnk]<1e-5 else 'red')
else:
print termcolors.colorize(' Link down or no packets received.',fg='red')
except:
print termcolors.colorize(' Link down or no packets received.',fg='red')
if auto_test:
cont-=1
if cont<=0:
print_report(connections, link, rx_cnt_ok, tx_cnt, rx_cnt_bad, rx_wrd_cnt_fail, error_rate, rx_wrd_cnt_missing)
return True
else:
time.sleep(1)
#TODO: output to log file
except KeyboardInterrupt:
print_report(connections, link, rx_cnt_ok, tx_cnt, rx_cnt_bad, rx_wrd_cnt_fail, error_rate, rx_wrd_cnt_missing)
return True
finally:
try:
fp.close()
fpga.stop()
except: pass
def print_report(connections, link, rx_cnt_ok, tx_cnt, rx_cnt_bad, rx_wrd_cnt_fail, error_rate, rx_wrd_cnt_missing):
print "\n\n======================="
print '\tREPORT'
print "======================="
for lnk in range(8):
rx_n=connections[lnk]
tx_n=lnk
failed=False
if rx_n >= 0:
if link[rx_n]>0:
print termcolors.colorize('%2i ->%2i ARP OK!'%(tx_n,rx_n),fg='green'),
if (rx_cnt_ok[rx_n]>0):
print termcolors.colorize('%5.1f%% pkts received ok.'%(round(float(rx_cnt_ok[rx_n])/tx_cnt[tx_n]*100,1)),fg='green'),
else:
failed=True
if (rx_cnt_bad[rx_n]>10) or (rx_wrd_cnt_fail[rx_n]>1):
print termcolors.colorize('Some bad data received! BER: %f.'%(error_rate[lnk]),fg='red'),
failed=True
if (rx_wrd_cnt_missing[rx_n]>5000):
print termcolors.colorize('Lost %i packets.'%(rx_wrd_cnt_missing[rx_n]),fg='red'),
failed=True
if failed==True:
print termcolors.colorize('Problem with TX lanes on port %i or RX lanes on port %i.'%(tx_n,rx_n),fg='red'),
else:
print termcolors.colorize('%2i ->?? link failed! Could not determine where TX of port %i is routed.'%(tx_n,tx_n),fg='red'),
print ''
