def parseIbcheckerrors(allErrs, ibCheckFile=None):
   f = ibCheckFile
   if f == None:
       suffix = 'ibcheckerrors'
       f, fTime = findMostRecentFile( ibDir, suffix )
   print 'using', f, 'for errors'
   lines = open( ibDir + '/' + f, 'r' ).readlines()
   #print lines

   # big problems with the fabric ->
   #
   # ibwarn: [22811] _do_madrpc: recv failed: Connection timed out
   # ibwarn: [22811] mad_rpc: _do_madrpc failed; dport (DR path slid 0; dlid 0; 0,1,31,18,3,31,17)
   # ibwarn: [22811] handle_port: NodeInfo on DR path slid 0; dlid 0; 0,1,31,18,3,31,17 failed, skipping port
   # ibwarn: [22811] _do_madrpc: recv failed: Invalid argument
   # ibwarn: [22811] mad_rpc: _do_madrpc failed; dport (DR path slid 0; dlid 0; 0,1,31,18,3,31)
   # ibwarn: [22811] discover: can't reach node DR path slid 0; dlid 0; 0,1,31,18,3,31 port 18
   # 
   # normal stuff ->
   #
   # #warn: counter SymbolErrors = 65534 (threshold 10) lid 2 port 255
   # #warn: counter LinkDowned = 254 (threshold 10) lid 2 port 255
   # Error check on lid 2 (0x0021283a89190040 qnem-13-3a) port all:  FAILED 
   # #warn: counter SymbolErrors = 722 (threshold 10) lid 259 port 255
   # Error check on lid 259 (0x0021283a87820050 qnem-13-1b) port all:  FAILED 
   # #warn: counter SymbolErrors = 722 (threshold 10) lid 259 port 21
   # Error check on lid 259 (0x0021283a87820050 qnem-13-1b) port 21:  FAILED 
   # #warn: counter SymbolErrors = 11251     (threshold 10) lid 1447 port 12
   # #warn: counter LinkRecovers = 255       (threshold 10) lid 1447 port 12
   # #warn: counter RcvErrors = 2342         (threshold 10) lid 1447 port 12
   # Error check on lid 1447 (0x0021283a87760040 qnem-05-4a) port 12: FAILED
   # #warn: counter SymbolErrors = 13        (threshold 10) lid 989 port 1
   # #warn: counter RcvErrors = 13   (threshold 10) lid 989 port 1
   # Error check on lid 989 (v679 HCA-1) port 1: FAILED
   # #warn: counter SymbolErrors = 19        (threshold 10) lid 974 port 1
   # #warn: counter RcvErrors = 18   (threshold 10) lid 974 port 1
   # Error check on lid 974 (v668 HCA-1) port 1: FAILED

   # (later) also can get many lines like this which I'll ignore ->
   # ibwarn: [6329] dump_perfcounters: PortXmitWait not indicated so ignore this counter
   # ibwarn: [6381] dump_perfcounters: PortXmitWait not indicated so ignore this counter
   # ibwarn: [6438] dump_perfcounters: PortXmitWait not indicated so ignore this counter

   errs = {}
   for ll in lines:
      ll = ll.strip()
      l = ll.split()

      # ignore final summary lines
      if len(l) == 0 or l[0][:2] == "##":
         continue

      # important errors - TODO - return these...
      if l[0] == 'ibwarn:':
         if 'dump_perfcounters:' in l and 'PortXmitWait' in l:
            continue
         print ll
         continue

      #print l
      if l[0] == "#warn:":
         if l[-2] != "port":
            print 'expected a port in a #warn, not "', ll, '"'
            continue
         lid = int(l[-3])
         port = int(l[-1])
         err = l[2]
         errCnt = int(l[4])
         if port == 255:
            continue
         if not allErrs and err not in normalErrors:
            continue
         #print 'lid', lid, 'port', port, 'err', err, 'errCnt', errCnt
         key = ( lid, port )
         if key not in errs.keys():
            errs[key] = {}
            errs[key]['errs'] = []
         errs[key]['errs'].append( ( err, errCnt ) )
      elif l[0] == "Error":
         # ignore 255,all
         lid = int(l[4])
         port = l[-2].split(':')[0]
         if port == 'all':
            port = 255
         port = int(port)
         if port == 255:
            continue
         key = ( lid, port )
         name = ll.split('(')[1].split(')')[0]
         #print 'lid', lid, 'port', port, 'name', name
         if key not in errs.keys(): # all errs for this (lid,port) were skipped
            continue
         errs[key]['name'] = name

   return errs
#!/usr/bin/env python

# find and print nodes that have been rebooted since the last ibclearerrors
# used by IB error checking scripts to eliminate nodes rebooted during that interval from error sweeps

from ibTracePorts import findMostRecentFile
from ibFlagErrors import uptimes, filterHosts
from hms import hms
import time, sys

ibDir = '/root/ib'
suffix = 'ibclearerrors'

f, fTime = findMostRecentFile( ibDir, suffix )
#print 'using', f, 'for errors, time', fTime, 'now', time.time(), 'diff', time.time() - fTime, 'hrs', (time.time() - fTime)/3600.0

uptime, down = uptimes()
if uptime == None:  # failed
    sys.exit(1)

#print 'len(uptime)', len(uptime) #, 'uptime', uptime   # uptimes by hostname
#print 'len(down)', len(down), 'down', down

ignore = filterHosts( uptime, fTime )
ignore.sort()
#print 'recently rebooted - ignore hosts', ignore, 'len', len(ignore)

print '# nodes rebooted in last', hms(time.time() - fTime)
for i in ignore:
    if i in down:
        continue