Example #1
0
    def __init__( self, xmldoc, parsed ):
        
        super( DCubeTester, self ).__init__( self )
        self.xmldoc = xmldoc
        self.opts, self.args = parsed
        self.tests = { "KS"   : self.__testKS,
                       "bbb"  : self.__testBBB,
                       "chi2" : self.__testChi2,
                       "meanY": self.__testMeanY }
       

        if ( self.opts.makeplots ):
            self.info("will produce plot files")
            self.__plotter = DCubePlotter( xmldoc, parsed )
        else:
            self.warn("making of plots disabled")
Example #2
0
    def __init__( self, xmldoc, parsed ):
        
        super( DCubeTester, self ).__init__( self )
        self.xmldoc = xmldoc
        self.opts, self.args = parsed
        self.tests = { "KS"   : self.__testKS,
                       "bbb"  : self.__testBBB,
                       "chi2" : self.__testChi2,
                       "meanY": self.__testMeanY }
       

        if ( self.opts.makeplots ):
            self.info("will produce plot files")
            self.__plotter = DCubePlotter( xmldoc, parsed )
        else:
            self.warn("making of plots disabled")
Example #3
0
class DCubeTester( DCubeObject ):

    ## XML DOM Document instance
    xmldoc = None

    ## XML DOM Element instance 
    node = None

    ## handle for monitored root object
    mon = None

    ## handle for reference root object
    ref = None

    ## errors counters
    errors = { }

    ## intarnal histogram counter
    __nbObjs = 0
    
    ## statistics summary table
    sumTable = { "KS"   : { "OK"   : 0,
                            "WARN" : 0,
                            "FAIL" : 0 },
                 "chi2" : { "OK"   : 0,
                            "WARN" : 0,
                            "FAIL" : 0 },
                 "bbb"  : { "OK"   : 0,
                            "WARN" : 0,
                            "FAIL" : 0 },
                 "meanY": { "OK"   : 0,
                            "WARN" : 0,
                            "FAIL" : 0 }}


    ## chi2 error communicates
    chi2igood = { 0 : "no problems",
                  1 : "there is bin in mon hist with low then 1 exp number of event",
                  2 : "there is bin in ref hist with low then 1 exp number of event",
                  3 : "there are bins in both histograms with less than 1 event" }

    
    ## summary status
    __status = ""
    
    ## c'tor
    # @param self "Me, myself and Irene"
    # @param xmldoc XML DOM Document instance
    def __init__( self, xmldoc, parsed ):
        
        super( DCubeTester, self ).__init__( self )
        self.xmldoc = xmldoc
        self.opts, self.args = parsed
        self.tests = { "KS"   : self.__testKS,
                       "bbb"  : self.__testBBB,
                       "chi2" : self.__testChi2,
                       "meanY": self.__testMeanY }
       

        if ( self.opts.makeplots ):
            self.info("will produce plot files")
            self.__plotter = DCubePlotter( xmldoc, parsed )
        else:
            self.warn("making of plots disabled")
            
    ## 
    # @param self "Me, myself and Irene"
    # @param node DOM XML  node 
    # @param mon monitored ROOT object
    # @param ref reference ROOT object
    # @return modified XML node 
    def test( self, node, mon=None, ref=None ):

        self.__nbObjs += 1

        self.node = self.mon = self.ref = None

        self.node = node
        self.parent = self.node.parentNode

        self.mon = mon
        self.ref = ref

        # mon exists?
        if ( not self.mon ):
            status = "FAIL;monitored not found"
            self.node.setAttribute( "status", status )
            if status not in self.errors.keys():
                self.errors[ status ] = 1
            else:
                self.errors[ status ] = self.errors[ status ] + 1
            self.error("monitored object not found!")
            return "FAIL"

        cl = self.mon.Class().GetName() 
        isHist = False
        if ( cl[:2] in ( "TH", "TP") ): isHist = True
       
        # dimension OK?
        if ( isHist and self.mon.GetDimension() > 2 ):
            status = "FAIL;unsupported object, dimension bigger than 2"
            self.node.setAttribute( "status", status )
            if ( status not in self.errors.keys() ):
                self.errors[ status ] = 1
            else:
                self.errors[ status ] = self.errors[ status ] + 1
            self.error( "unsuported object found" )
            return "FAIL"

        # reference exists?
        if ( not self.ref ):
            self.warn( "reference object not found!" )
            status = "WARN;reference histogram not found"
            self.node.setAttribute( "status", status )
            if ( status not in self.errors.keys() ):
                self.errors[ status ] = 1
            else:
                self.errors[ status ] = self.errors[ status ] + 1
    
        if ( self.mon and self.ref ):
            # same class?
            monClassName = self.mon.Class().GetName()
            refClassName = self.ref.Class().GetName()
            if ( monClassName != refClassName ):
                status = "FAIL;different root types"
                self.node.setAttribute( "status", status + " mon=%s ref=%s" % ( monClassName,
                                                                                refClassName ) )
                if ( status not in self.errors.keys() ):
                    self.errors[ status ] = 1
                else:
                    self.errors[ status ] = self.errors[ status ] + 1
                self.error("different types for mon and ref objects!")
                return "FAIL"


        if ( isHist ):
            self.monBins = [ self.mon.GetNbinsX(), self.mon.GetNbinsY() ]  
        if ( isHist and self.ref ):
            self.refBins = [ self.ref.GetNbinsX(), self.ref.GetNbinsY() ]
      
        if ( isHist and self.mon and self.ref ):
            # same binnig?
            monBins = [ "%s=%d" % ( x, y) for ( x, y) in zip(["x", "y"], self.monBins ) ]
            refBins = [ "%s=%d" % ( x, y) for ( x, y) in zip(["x", "y"], self.refBins ) ]                         
            diffBins = [ "mon %s ref %s" % (x, y) for (x,y) in zip(monBins,refBins) if x != y ]
            if ( diffBins  ):
                status = "FAIL;different binnig"
                self.node.setAttribute( "status", status + " %s" % " ".join(diffBins) )
                if ( status not in self.errors.keys() ):
                    self.errors[ status ] = 1
                else:
                    self.errors[ status ] = self.errors[ status ] + 1
                self.error("different binning for mon and ref objects!")
                return "FAIL" 

        status = "OK"                    
        if ( isHist ):
            status = self.__grabHistogramStat()
        else:
            status = self.__grabGraphStat() 

        plotStatus = "OK"
        if ( self.opts.makeplots ):
            plotStatus = self.__makePlots() 
        else:
            plotStatus = "WARN;plots not reqested"
        self.node.setAttribute( "plots", plotStatus)

        return status
        

    ## grab statistic info for TGraphXXX  
    # @param self "Me, myself and Irene"
    def __grabGraphStat( self ):
        statNode = self.xmldoc.createElement( "stat" )
        self.node.appendChild( statNode )

        nbPointsNode = self.xmldoc.createElement( "points" )
        nbPointsCData = self.xmldoc.createTextNode( "%d" % self.mon.GetN() )
        if ( self.ref ):
            nbPointsNode.setAttribute( "ref" , "%d" % self.ref.GetN() )
        nbPointsNode.appendChild( nbPointsCData )

        meanNode = self.xmldoc.createElement( "mean" )
        meanCData = self.xmldoc.createTextNode( "%4.3f" % self.mon.GetMean() )
        if ( self.ref ):
            meanNode.setAttribute( "ref" , "%4.3f" % self.ref.GetMean() )
        meanNode.appendChild( meanCData )

        rmsNode = self.xmldoc.createElement( "rms" )
        rmsCData = self.xmldoc.createTextNode( "%d" % self.mon.GetRMS() )
        if ( self.ref ):
            rmsNode.setAttribute( "ref" , "%d" % self.ref.GetRMS() )
        rmsNode.appendChild( rmsCData )

        statNode.appendChild( nbPointsNode )
        statNode.appendChild( meanNode )
        statNode.appendChild( rmsNode )

        return "OK"
        
        
        
    ## grab basic statistics - nb of entries, mean and RMS
    # @param self "Me, myself and Irene"
    def __grabHistogramStat( self ):

        statNode = self.xmldoc.createElement( "stat" )

        self.node.appendChild( statNode )

        entriesNode = self.xmldoc.createElement("entries")
        entriesCDataNode = self.xmldoc.createTextNode( str( self.mon.GetEntries() ) )
        if ( self.ref ):
            entriesNode.setAttribute( "ref", str( self.ref.GetEntries() ) )
        entriesNode.appendChild( entriesCDataNode )

        statNode.appendChild( entriesNode )

        dim = self.mon.GetDimension() 

        ### To dump MeanY values
        if ( "TProfile" in self.mon.Class().GetName() ): dim = 2

        # store under and overflows 
        monU = []
        monO = []
        refU = []
        refO = []

        if ( dim == 1 ):
            monU.append( self.mon.GetBinContent(0) )
            monO.append( self.mon.GetBinContent( self.mon.GetNbinsX()+1 ) )
            if ( self.ref ):
                refU.append( self.ref.GetBinContent(0) )
                refO.append( self.ref.GetBinContent( self.ref.GetNbinsX()+1 ) )
        elif ( dim == 2 ):
            monU.append( self.mon.GetBinContent( 0, 0 ) )
            monO.append( self.mon.GetBinContent( self.mon.GetNbinsX()+1, 0 ) )
            monU.append( self.mon.GetBinContent( 0,  self.mon.GetNbinsY()+1) )
            monO.append( self.mon.GetBinContent( self.mon.GetNbinsX()+1, self.mon.GetNbinsY()+1 ) )  
            if ( self.ref ):
                refU.append( self.ref.GetBinContent( 0, 0 ) )
                refO.append( self.ref.GetBinContent( self.ref.GetNbinsX()+1, 0 ) )
                refU.append( self.ref.GetBinContent( 0,  self.ref.GetNbinsY()+1) )
                refO.append( self.ref.GetBinContent( self.ref.GetNbinsX()+1, self.ref.GetNbinsY()+1 ) )  

        underflowNode = self.xmldoc.createElement( "underflow" )
        underflowCData = self.xmldoc.createTextNode( "%d" % sum( monU ) )
        if ( self.ref ):
            underflowNode.setAttribute( "ref", "%d" % sum( refU) )
        underflowNode.appendChild( underflowCData )

        overflowNode = self.xmldoc.createElement( "overflow" )
        overflowCData = self.xmldoc.createTextNode( "%d" % sum( monO ) )
        if ( self.ref ):
            overflowNode.setAttribute( "ref", "%d" % sum( refO ) )
        overflowNode.appendChild( overflowCData )

        statNode.appendChild( underflowNode )
        statNode.appendChild( overflowNode )
        
        dimName = [ "x", "y", "z" ]
        for i in range( 1, dim+1 ):
            self.debug( "gathering statistics along %s axis" % dimName[i-1] )

            dimNode = self.xmldoc.createElement( "dim" )
            dimNode.setAttribute( "name", dimName[i-1] )
            dimNode.setAttribute( "bins", "%d" % self.monBins[i-1])
        
            underflowNode = self.xmldoc.createElement( "underflow" )
            underflowCData  = self.xmldoc.createTextNode( "%d" % monU[i-1] )
            if ( self.ref ):
                underflowNode.setAttribute( "ref", "%d" % refU[i-1] )
            underflowNode.appendChild( underflowCData )

            dimNode.appendChild( underflowNode )
        
            overflowNode = self.xmldoc.createElement( "overflow" )
            overflowCData  = self.xmldoc.createTextNode( "%d" % monO[i-1] )
            if ( self.ref ):
                overflowNode.setAttribute( "ref", "%d" % refO[i-1] )
            overflowNode.appendChild( overflowCData )

            dimNode.appendChild( overflowNode )
            
            meanNode = self.xmldoc.createElement( "mean" )
            if ( self.ref ):
                meanNode.setAttribute( "ref", "%3.2e" % self.ref.GetMean(i) )

            meanCData = self.xmldoc.createTextNode( "%3.2e" % self.mon.GetMean( i ) )
            meanNode.appendChild( meanCData )
            
            meanErrorNode = self.xmldoc.createElement( "mean_unc" )
            if ( self.ref ):
                meanErrorNode.setAttribute( "ref", "%3.2e" % self.ref.GetMeanError(i) )

            meanErrorCData = self.xmldoc.createTextNode( "%3.2e" % self.mon.GetMeanError( i ) )
            meanErrorNode.appendChild( meanErrorCData )
            
            rmsNode = self.xmldoc.createElement( "rms" )
            if ( self.ref ):
                rmsNode.setAttribute( "ref", "%3.2e" % self.ref.GetRMS(i) )
            rmsCData = self.xmldoc.createTextNode( "%3.2e" % self.mon.GetRMS(i) )
            rmsNode.appendChild( rmsCData )
        
            rmsErrorNode = self.xmldoc.createElement( "rms_unc" )
            if ( self.ref ):
                rmsErrorNode.setAttribute( "ref", "%3.2e" % self.ref.GetRMSError(i) )
            rmsErrorCData = self.xmldoc.createTextNode( "%3.2e" % self.mon.GetRMSError(i))
            rmsErrorNode.appendChild( rmsErrorCData )

            dimNode.appendChild( meanNode )
            dimNode.appendChild( meanErrorNode )
            dimNode.appendChild( rmsNode )
            dimNode.appendChild( rmsErrorNode )

            statNode.appendChild( dimNode )



        status = [ ]
        tests = self.node.getAttribute("tests").split(",")
        if ( "all" in tests ): tests = [ "KS", "chi2", "bbb", "meany" ]

        if ( None not in ( self.mon, self.ref ) ):
             
            if ( "TProfile" in self.mon.Class().GetName() ):
                if ( "meany" in tests ):
                    status.append( self.__testMeanY.__call__( statNode ) )
            else:
                for test in tests:
                    if ( test != "meany" ):
                        status.append( self.tests[ test ].__call__( statNode ) )
                    
        statusAttr = "OK"
        if ( "FAIL" in status ): statusAttr = "FAIL"
        elif ( "WARN" in status ): statusAttr = "WARN"
        
        self.node.setAttribute( "stest", statusAttr )

        return statusAttr
                    
    ## perform @f$\chi^2@f$ test
    # @param self "Me, myself and Irene"
    # @param statNode <stat> element
    def __testChi2( self, statNode ):

        chi2 = ROOT.Double( 0.0 )
        igood = ROOT.Long( 0 )
        ndf = ROOT.Long( 0 )

        pval = self.mon.Chi2TestX( self.ref, chi2, ndf, igood, "UUDNORM")

        self.debug( "*** Pearson's chi2 *** (UUDNORM) p-value= %4.3f" % pval )
        ig = "*** Pearson's chi2 *** igood = %d, %s" % ( igood, self.chi2igood[igood] )
        if ( igood == 0 ):
            self.debug( ig )
        else:
            self.warn( ig )

        status = self.__getTestStatus( pval ) 
        self.sumTable["chi2"][status] = self.sumTable["chi2"][status] + 1   

        if ( ndf != 0 ):
            self.info( "*** Pearson's chi2 *** chi2 (/ndf) = %4.3f (/%d = %4.3f) status=%s" % ( chi2,
                                                                                                ndf,
                                                                                                chi2/ndf,
                                                                                                status ) )
        else:
            self.info( "*** Pearson's chi2 *** chi2 = %4.3f ndf = %d status=%s" % ( chi2,
                                                                                    ndf,
                                                                                    status ) )

        pValueNode = self.xmldoc.createElement( "pvalue" )
        pValueCData = self.xmldoc.createTextNode( "%4.3f" % pval )

        pValueNode.setAttribute( "test", "chi2" )
        pValueNode.setAttribute( "status", status )
       
        counter = self.parent.getAttribute( "chi2" + status )
        if ( not counter ): counter = "0"
        self.parent.setAttribute( "chi2" + status, "%d" % ( int(counter) + 1 )  )

        pValueNode.appendChild( pValueCData )

        statNode.appendChild( pValueNode )


        return status

    ## perform Kolmogorov-Smirnoff test
    # @param self "Me, myself and Irene"
    # @param statNode <stat> element
    def __testKS( self, statNode ):

        pval = self.mon.KolmogorovTest( self.ref, "NDX" )
        
        status = self.__getTestStatus( pval ) 
        self.sumTable["KS"][status] = self.sumTable["KS"][status] + 1     

        self.info( "*** Kolmogorov-Smirnoff *** (NDX) p-value=%4.3f status=%s" % ( pval, status ) )

        pValueNode = self.xmldoc.createElement( "pvalue" )
        pValueCData = self.xmldoc.createTextNode( "%4.3f" % pval )

        pValueNode.setAttribute( "test", "KS" )
        pValueNode.setAttribute( "status", status )

        counter = self.parent.getAttribute( "KS" + status )
        if ( not counter ): counter = "0"
        self.parent.setAttribute( "KS" + status, "%d" % ( int(counter) + 1 )  )

        pValueNode.appendChild( pValueCData )

        statNode.appendChild( pValueNode )
        return status
        

    ## perform "bin-by-bin" statistic test
    # @param self "Me, myself and Irene"
    # @param statNode <stat> element
    def __testBBB( self, statNode ):
       
        nbBins = 0.0
        nbBinsSame = 0.0

        binX = self.mon.GetNbinsX()
        binY = self.mon.GetNbinsY()

        for i in range( binX+1 ):
            for j in range( binY + 1 ):
                mon = self.mon.GetBinContent( i, j )
                ref = self.ref.GetBinContent( i, j )
                if ( mon or ref ):
                    nbBins += 1
                    if ( mon == ref ):
                        nbBinsSame += 1

        self.debug("*** bin-by-bin *** all none-empty bins=%d all equal non-empty bins=%d" % ( nbBins, nbBinsSame ) )

        pval = 0.0
        if ( nbBins != 0.0 ):
            pval = nbBinsSame / nbBins  
            self.debug("*** bin-by-bin *** p-value=%4.3f" % pval )
        elif ( nbBinsSame == 0.0 ):
            pval = 1.0
            self.warn( "*** bin-by-bin *** both histograms are empty!" )
            self.debug( "*** bin-by-bin *** p-value=%4.3f" % pval )
        else:
            self.warn( "*** bin-by-bin *** reference histogram is empty, while monitored has some entries" )
            self.debug( "*** bin-by-bin *** test failed" )
        
        status = self.__getTestStatus( pval )
        self.info("*** bin-by-bin *** p-value=%4.3f status=%s" % ( pval, status ) )

        self.sumTable["bbb"][status] = self.sumTable["bbb"][status] + 1     

        pValueNode = self.xmldoc.createElement( "pvalue" )
        pValueCData = self.xmldoc.createTextNode( "%4.3f" % pval )

        pValueNode.setAttribute( "test", "bbb" )
        pValueNode.setAttribute( "status", status )

        counter = self.parent.getAttribute( "bbb" + status )
        if ( not counter ): counter = "0"
        self.parent.setAttribute( "bbb" + status, "%d" % ( int(counter) + 1 )  )

        pValueNode.appendChild( pValueCData )

        statNode.appendChild( pValueNode )

        return status


    ## perform "diff(MeanY)" test for TProfile histos
    # @param self "Me, myself and Irene"
    # @param statNode <stat> element
    def __testMeanY( self, statNode ):

        avgEffmon = self.mon.GetMean( 2 ) * 100
        avgEffref = self.ref.GetMean( 2 ) * 100

        self.debug("*** MeanY Test *** refMean=%4.3f and monMean=%4.3f" % ( avgEffref, avgEffmon ) )
        
        pval = abs(avgEffmon - avgEffref)

        status = self.__getMeanTestStatus( pval )
        self.info("*** Mean Test *** p-value=%4.3f status=%s" % ( pval, status ) )

        self.sumTable["meanY"][status] = self.sumTable["meanY"][status] + 1     

        pValueNode = self.xmldoc.createElement( "pvalue" )
        pValueCData = self.xmldoc.createTextNode( "%4.3f" % pval )

        pValueNode.setAttribute( "test", "meanY" )
        pValueNode.setAttribute( "status", status )

        counter = self.parent.getAttribute( "meanY" + status )
        if ( not counter ): counter = "0"
        self.parent.setAttribute( "meanY" + status, "%d" % ( int(counter) + 1 )  )

        pValueNode.appendChild( pValueCData )

        statNode.appendChild( pValueNode )

        return status
    
    
    ## get test status for given pvalue 
    # @param self "Me, myself and Irene"
    # @param pval p-value from test
    def __getTestStatus( self, pval ):
        if ( ( pval < 0.0 or pval > 1.0 ) or  
             ( pval <= self.opts.pfail ) ): return "FAIL"
        if ( pval > self.opts.pwarn ): return "OK"
        return "WARN"


    ## get test status for given pvalue 
    # @param self "Me, myself and Irene"
    # @param pval p-value from test
    def __getMeanTestStatus( self, pval ):
        if ( pval > 1.0 and pval <= 5.0 ): return "WARN"
        if ( pval <= 1.0  ): return "OK"
        return "FAIL"


    ## make plots
    # @param self "Me, myself and Irene"
    def __makePlots( self ):
        if ( self.__plotter ):
            plotOpts = self.node.getAttribute( "plotopts" )
            status = self.__plotter.plot( self.node, self.mon, self.ref, plotOpts  )
            return status
    

    ## string representation of DCubeTester
    # @param self "Me, myself and Irene" 
    def __str__( self ):
        out  = "*"*61 + "\n"
        out += "* RUN SUMMARY\n"
        out += "*"*61 + "\n"
        out += "* objects processed = %d\n" % self.__nbObjs
        out += "* STATISTICS TESTS TABLE\n"
        out += "* " + "-"*45 + "\n"
        out += "* | %-8s | %8s | %8s | %8s |\n" % ( "test", "OK", "WARN", "FAIL") 
        self.allGOOD = 0
        self.allWARN = 0
        self.allFAIL = 0
        for key, value  in self.sumTable.iteritems():
            nbGOOD = value["OK"]
            nbWARN = value["WARN"]
            nbFAIL = value["FAIL"]

            self.allGOOD += nbGOOD
            self.allWARN += nbWARN
            self.allFAIL += nbFAIL
            out += "* " + "-"*45+"\n"
            out += "* | %-8s | %8d | %8d | %8d |\n" % ( key, nbGOOD, nbWARN, nbFAIL )
            
        out += "* " + "-"*45+"\n"

        
        out += "* | %-8s | %8d | %8d | %8d |\n" % ( "all", self.allGOOD, self.allWARN, self.allFAIL )
        out += "* " + "-"*45+"\n"

        self.fracGOOD = 0.0
        self.fracWARN = 0.0
        self.fracFAIL = 0.0
        all = float( self.allGOOD + self.allWARN + self.allFAIL ) 
        if ( all ):
            self.fracGOOD = 100 * float( self.allGOOD ) / all 
            self.fracWARN = 100 * float( self.allWARN ) / all 
            self.fracFAIL = 100 * float( self.allFAIL ) / all
        out += "* | %%        |    %04.2f |    %04.2f |    %04.2f |\n" % ( self.fracGOOD, self.fracWARN, self.fracFAIL )
        out += "* " + "-"*45+"\n"

        out += "*"*61+"\n"
        self.nbErrors = sum( self.errors.itervalues() )
        out += "* WARNINGS = %3d\n" % self.nbErrors
        i = 1
        for key, value in self.errors.iteritems( ):
            sev, what = key.split(";")
            
            out += "* [%02d] %4s %-30s - occured %d " %  (i, sev, what, value )
            if ( value == 1 ): out += "time\n"
            else: out += "times\n"
            i += 1

        out += "*"*61+"\n"
        
        self.__status = ""
        
        if ( self.allFAIL != 0 ):
            self.__status = "FAIL"
        elif ( self.allWARN != 0 or self.nbErrors != 0 ):
            self.__status = "WARN"
        else:
            self.__status = "OK"

        out += "* OVERALL STATISTICS STATUS: %-4s\n" % self.__status
                    
        out += "*"*61
        
        return out

    ## put summary to the logger and create summary node 
    # @param self "Me, myself and Irene"
    # @return summary XML Element
    def summary( self ):
        
        for line in str(self).split("\n"): 
            self.info( line )

        summaryNode = self.xmldoc.createElement( "summary" )
        summaryNode.setAttribute( "status" , self.__status )
        summaryNode.setAttribute( "objs", "%d" % self.__nbObjs )
        summaryNode.setAttribute( "errors", "%d" % self.nbErrors  )

        testTable = self.xmldoc.createElement( "table" )
        testTable.appendChild( self.__sumTableRow( [ "test", "OK", "WARN", "FAIL" ] ) )
        for test, results in self.sumTable.iteritems(): 
            testTable.appendChild( self.__sumTableRow( [ test,
                                                         results["OK"],
                                                         results["WARN"],
                                                         results["FAIL"] ] ) )
            
        testTable.appendChild( self.__sumTableRow( ["sum", self.allGOOD, self.allWARN, self.allFAIL ] ) )
        testTable.appendChild( self.__sumTableRow( ["fraction",
                                                    "%4.2f" % self.fracGOOD,
                                                    "%4.2f" % self.fracWARN,
                                                    "%4.2f" % self.fracFAIL ] ) )
        
        summaryNode.appendChild( testTable )

        errorsNode = self.xmldoc.createElement( "errors" )
        errorsNode.setAttribute( "nb", "%d" % self.nbErrors )

        for error,times in self.errors.iteritems():
            error = error.replace(";", " ")
            errorNode = self.xmldoc.createElement( "error" )
            errorNode.setAttribute( "times", "%d" % times )
            errorNode.setAttribute( "what", error )
            errorsNode.appendChild( errorNode )
            
        summaryNode.appendChild( errorsNode )
        
        return summaryNode 

    ## produce summary table row 
    # @param self "Me, myself and Irene"
    def __sumTableRow( self, what ):
        row = self.xmldoc.createElement( "tr" )
        for item in what:
            cellNode = self.xmldoc.createElement("td")
            cellCData = self.xmldoc.createTextNode( str(item) )
            cellNode.appendChild( cellCData )
            row.appendChild( cellNode )
        return row

    ## return overall summary as string
    # @param self "Me, myself and Irene"
    def status( self ):
        return self.__status
Example #4
0
class DCubeTester( DCubeObject ):

    ## XML DOM Document instance
    xmldoc = None

    ## XML DOM Element instance 
    node = None

    ## handle for monitored root object
    mon = None

    ## handle for reference root object
    ref = None

    ## errors counters
    errors = { }

    ## intarnal histogram counter
    __nbObjs = 0
    
    ## statistics summary table
    sumTable = { "KS"   : { "OK"   : 0,
                            "WARN" : 0,
                            "FAIL" : 0 },
                 "chi2" : { "OK"   : 0,
                            "WARN" : 0,
                            "FAIL" : 0 },
                 "bbb"  : { "OK"   : 0,
                            "WARN" : 0,
                            "FAIL" : 0 },
                 "meanY": { "OK"   : 0,
                            "WARN" : 0,
                            "FAIL" : 0 }}


    ## chi2 error communicates
    chi2igood = { 0 : "no problems",
                  1 : "there is bin in mon hist with low then 1 exp number of event",
                  2 : "there is bin in ref hist with low then 1 exp number of event" }

    
    ## summary status
    __status = ""
    
    ## c'tor
    # @param self "Me, myself and Irene"
    # @param xmldoc XML DOM Document instance
    def __init__( self, xmldoc, parsed ):
        
        super( DCubeTester, self ).__init__( self )
        self.xmldoc = xmldoc
        self.opts, self.args = parsed
        self.tests = { "KS"   : self.__testKS,
                       "bbb"  : self.__testBBB,
                       "chi2" : self.__testChi2,
                       "meanY": self.__testMeanY }
       

        if ( self.opts.makeplots ):
            self.info("will produce plot files")
            self.__plotter = DCubePlotter( xmldoc, parsed )
        else:
            self.warn("making of plots disabled")
            
    ## 
    # @param self "Me, myself and Irene"
    # @param node DOM XML  node 
    # @param mon monitored ROOT object
    # @param ref reference ROOT object
    # @return modified XML node 
    def test( self, node, mon=None, ref=None ):

        self.__nbObjs += 1

        self.node = self.mon = self.ref = None

        self.node = node
        self.parent = self.node.parentNode

        self.mon = mon
        self.ref = ref

        # mon exists?
        if ( not self.mon ):
            status = "FAIL;monitored not found"
            self.node.setAttribute( "status", status )
            if status not in self.errors.keys():
                self.errors[ status ] = 1
            else:
                self.errors[ status ] = self.errors[ status ] + 1
            return "FAIL"


        cl = self.mon.Class().GetName() 
        isHist = False
        if ( cl[:2] in ( "TH", "TP") ): isHist = True
       
        # dimension OK?
        if ( isHist and self.mon.GetDimension() > 2 ):
            status = "FAIL;unsupported object, dimension bigger than 2"
            self.node.setAttribute( "status", status )
            if ( status not in self.errors.keys() ):
                self.errors[ status ] = 1
            else:
                self.errors[ status ] = self.errors[ status ] + 1
            return "FAIL"

        # reference exists?
        if ( not self.ref ):
            status = "WARN;reference histogram not found"
            self.node.setAttribute( "status", status )
            if ( status not in self.errors.keys() ):
                self.errors[ status ] = 1
            else:
                self.errors[ status ] = self.errors[ status ] + 1
            
    
        if ( self.mon and self.ref ):

            # same class?
            monClassName = self.mon.Class().GetName()
            refClassName = self.ref.Class().GetName()
           
            if ( monClassName != refClassName ):
                status = "FAIL;different root types"
                self.node.setAttribute( "status", status + " mon=%s ref=%s" % ( monClassName,
                                                                                refClassName ) )
                if ( status not in self.errors.keys() ):
                    self.errors[ status ] = 1
                else:
                    self.errors[ status ] = self.errors[ status ] + 1 
                return "FAIL"

        if ( isHist and self.mon and self.ref ):
            # same binnig?
            self.monBins = [ self.mon.GetNbinsX(), self.mon.GetNbinsY() ]
            
            self.refBins = [ self.ref.GetNbinsX(), self.ref.GetNbinsY() ]

            monBins = [ "%s=%d" % ( x, y) for ( x, y) in zip(["x", "y"], self.monBins ) ]
            refBins = [ "%s=%d" % ( x, y) for ( x, y) in zip(["x", "y"], self.refBins ) ]
                             
            diffBins = [ "mon %s ref %s" % (x, y) for (x,y) in zip(monBins,refBins) if x != y ]

            if ( diffBins  ):
                status = "FAIL;different binnig"
                self.node.setAttribute( "status", status + " %s" % " ".join(diffBins) )
                if ( status not in self.errors.keys() ):
                    self.errors[ status ] = 1
                else:
                    self.errors[ status ] = self.errors[ status ] + 1
                return "FAIL" 
            
        status = "OK"
                    
        if ( isHist ):
            status = self.__grabHistogramStat()
        else:
            status = self.__grabGraphStat() 

        plotStatus = "OK"
        if ( self.opts.makeplots ):
            plotStatus = self.__makePlots() 
        else:
            plotStatus = "WARN;plots not reqested"
        self.node.setAttribute( "plots", plotStatus)

        return status
        

    ## grab statistic info for TGraphXXX  
    # @param self "Me, myself and Irene"
    def __grabGraphStat( self ):
        statNode = self.xmldoc.createElement( "stat" )
        self.node.appendChild( statNode )

        nbPointsNode = self.xmldoc.createElement( "points" )
        nbPointsCData = self.xmldoc.createTextNode( "%d" % self.mon.GetN() )
        if ( self.ref ):
            nbPointsNode.setAttribute( "ref" , "%d" % self.ref.GetN() )
        nbPointsNode.appendChild( nbPointsCData )

        meanNode = self.xmldoc.createElement( "mean" )
        meanCData = self.xmldoc.createTextNode( "%4.3f" % self.mon.GetMean() )
        if ( self.ref ):
            meanNode.setAttribute( "ref" , "%4.3f" % self.ref.GetMean() )
        meanNode.appendChild( meanCData )

        rmsNode = self.xmldoc.createElement( "rms" )
        rmsCData = self.xmldoc.createTextNode( "%d" % self.mon.GetRMS() )
        if ( self.ref ):
            rmsNode.setAttribute( "ref" , "%d" % self.ref.GetRMS() )
        rmsNode.appendChild( rmsCData )

        statNode.appendChild( nbPointsNode )
        statNode.appendChild( meanNode )
        statNode.appendChild( rmsNode )

        return "OK"
        
        
        
    ## grab basic statistics - nb of entries, mean and RMS
    # @param self "Me, myself and Irene"
    def __grabHistogramStat( self ):

        statNode = self.xmldoc.createElement( "stat" )

        self.node.appendChild( statNode )

        entriesNode = self.xmldoc.createElement("entries")
        entriesCDataNode = self.xmldoc.createTextNode( str( self.mon.GetEntries() ) )
        if ( self.ref ):
            entriesNode.setAttribute( "ref", str( self.ref.GetEntries() ) )
        entriesNode.appendChild( entriesCDataNode )

        statNode.appendChild( entriesNode )

        dim = self.mon.GetDimension() 

        ### To dump MeanY values
        if ( "TProfile" in self.mon.Class().GetName() ): dim = 2

        # store under and overflows 
        monU = []
        monO = []
        refU = []
        refO = []

        if ( dim == 1 ):
            monU.append( self.mon.GetBinContent(0) )
            monO.append( self.mon.GetBinContent( self.mon.GetNbinsX()+1 ) )
            if ( self.ref ):
                refU.append( self.ref.GetBinContent(0) )
                refO.append( self.ref.GetBinContent( self.ref.GetNbinsX()+1 ) )
        elif ( dim == 2 ):
            monU.append( self.mon.GetBinContent( 0, 0 ) )
            monO.append( self.mon.GetBinContent( self.mon.GetNbinsX()+1, 0 ) )
            monU.append( self.mon.GetBinContent( 0,  self.mon.GetNbinsY()+1) )
            monO.append( self.mon.GetBinContent( self.mon.GetNbinsX()+1, self.mon.GetNbinsY()+1 ) )  
            if ( self.ref ):
                refU.append( self.ref.GetBinContent( 0, 0 ) )
                refO.append( self.ref.GetBinContent( self.ref.GetNbinsX()+1, 0 ) )
                refU.append( self.ref.GetBinContent( 0,  self.ref.GetNbinsY()+1) )
                refO.append( self.ref.GetBinContent( self.ref.GetNbinsX()+1, self.ref.GetNbinsY()+1 ) )  

        underflowNode = self.xmldoc.createElement( "underflow" )
        underflowCData = self.xmldoc.createTextNode( "%d" % sum( monU ) )
        if ( self.ref ):
            underflowNode.setAttribute( "ref", "%d" % sum( refU) )
        underflowNode.appendChild( underflowCData )

        overflowNode = self.xmldoc.createElement( "overflow" )
        overflowCData = self.xmldoc.createTextNode( "%d" % sum( monO ) )
        if ( self.ref ):
            overflowNode.setAttribute( "ref", "%d" % sum( refO ) )
        overflowNode.appendChild( overflowCData )

        statNode.appendChild( underflowNode )
        statNode.appendChild( overflowNode )
        

        dimName = [ "x", "y", "z" ]
        for i in range( 1, dim+1 ):
            self.debug( "gathering statistics along %s axis" % dimName[i-1] )

            dimNode = self.xmldoc.createElement( "dim" )
            dimNode.setAttribute( "name", dimName[i-1] )
            dimNode.setAttribute( "bins", "%d" % self.monBins[i-1])
        
            underflowNode = self.xmldoc.createElement( "underflow" )
            underflowCData  = self.xmldoc.createTextNode( "%d" % monU[i-1] )
            if ( self.ref ):
                underflowNode.setAttribute( "ref", "%d" % refU[i-1] )
            underflowNode.appendChild( underflowCData )

            dimNode.appendChild( underflowNode )
        
            overflowNode = self.xmldoc.createElement( "overflow" )
            overflowCData  = self.xmldoc.createTextNode( "%d" % monO[i-1] )
            if ( self.ref ):
                overflowNode.setAttribute( "ref", "%d" % refO[i-1] )
            overflowNode.appendChild( overflowCData )

            dimNode.appendChild( overflowNode )
            
            meanNode = self.xmldoc.createElement( "mean" )
            if ( self.ref ):
                meanNode.setAttribute( "ref", "%3.2e" % self.ref.GetMean(i) )

            meanCData = self.xmldoc.createTextNode( "%3.2e" % self.mon.GetMean( i ) )
            meanNode.appendChild( meanCData )
            
            meanErrorNode = self.xmldoc.createElement( "mean_unc" )
            if ( self.ref ):
                meanErrorNode.setAttribute( "ref", "%3.2e" % self.ref.GetMeanError(i) )

            meanErrorCData = self.xmldoc.createTextNode( "%3.2e" % self.mon.GetMeanError( i ) )
            meanErrorNode.appendChild( meanErrorCData )
            
            rmsNode = self.xmldoc.createElement( "rms" )
            if ( self.ref ):
                rmsNode.setAttribute( "ref", "%3.2e" % self.ref.GetRMS(i) )
            rmsCData = self.xmldoc.createTextNode( "%3.2e" % self.mon.GetRMS(i) )
            rmsNode.appendChild( rmsCData )
        
            rmsErrorNode = self.xmldoc.createElement( "rms_unc" )
            if ( self.ref ):
                rmsErrorNode.setAttribute( "ref", "%3.2e" % self.ref.GetRMSError(i) )
            rmsErrorCData = self.xmldoc.createTextNode( "%3.2e" % self.mon.GetRMSError(i))
            rmsErrorNode.appendChild( rmsErrorCData )

            dimNode.appendChild( meanNode )
            dimNode.appendChild( meanErrorNode )
            dimNode.appendChild( rmsNode )
            dimNode.appendChild( rmsErrorNode )

            statNode.appendChild( dimNode )

        status = [ ]
        if ( "TProfile" in self.mon.Class().GetName() ):
            #self.warn( "will skip statistics testing for TProfile objects...")
            self.info( "will run Mean test for TProfile objects...")
            status.append( self.__testMeanY.__call__( statNode ) )

        else:

            tests = self.node.getAttribute("tests").split(",")
            if ( "all" in tests ): tests = [ "KS", "chi2", "bbb" ]

            if ( None not in ( self.mon, self.ref )  ):
                for test in tests:
                    status.append( self.tests[ test ].__call__( statNode ) )
                    
        statusAttr = "OK"
        if ( "FAIL" in status ): statusAttr = "FAIL"
        elif ( "WARN" in status ): statusAttr = "WARN"
        
        self.node.setAttribute( "stest", statusAttr )

        return statusAttr
                    
    ## perform @f$\chi^2@f$ test
    # @param self "Me, myself and Irene"
    # @param statNode <stat> element
    def __testChi2( self, statNode ):

        chi2 = ROOT.Double( 0.0 )
        igood = ROOT.Long( 0 )
        ndf = ROOT.Long( 0 )

        pval = self.mon.Chi2TestX( self.ref, chi2, ndf, igood, "UUDNORM")

        self.debug( "*** Pearson's chi2 *** (UUDNORM) p-value= %4.3f" % pval )
        ig = "*** Pearson's chi2 *** igood = %d, %s" % ( igood, self.chi2igood[igood] )
        if ( igood == 0 ):
            self.debug( ig )
        else:
            self.warn( ig )

        status = self.__getTestStatus( pval ) 
        self.sumTable["chi2"][status] = self.sumTable["chi2"][status] + 1   

        if ( ndf != 0 ):
            self.info( "*** Pearson's chi2 *** chi2 (/ndf) = %4.3f (/%d = %4.3f) status=%s" % ( chi2,
                                                                                                ndf,
                                                                                                chi2/ndf,
                                                                                                status ) )
        else:
            self.info( "*** Pearson's chi2 *** chi2 = %4.3f ndf = %d status=%s" % ( chi2,
                                                                                    ndf,
                                                                                    status ) )

        pValueNode = self.xmldoc.createElement( "pvalue" )
        pValueCData = self.xmldoc.createTextNode( "%4.3f" % pval )

        pValueNode.setAttribute( "test", "chi2" )
        pValueNode.setAttribute( "status", status )
       
        counter = self.parent.getAttribute( "chi2" + status )
        if ( not counter ): counter = "0"
        self.parent.setAttribute( "chi2" + status, "%d" % ( int(counter) + 1 )  )

        pValueNode.appendChild( pValueCData )

        statNode.appendChild( pValueNode )


        return status

    ## perform Kolmogorov-Smirnoff test
    # @param self "Me, myself and Irene"
    # @param statNode <stat> element
    def __testKS( self, statNode ):

        pval = self.mon.KolmogorovTest( self.ref, "NDX" )
        
        status = self.__getTestStatus( pval ) 
        self.sumTable["KS"][status] = self.sumTable["KS"][status] + 1     

        self.info( "*** Kolmogorov-Smirnoff *** (NDX) p-value=%4.3f status=%s" % ( pval, status ) )

        pValueNode = self.xmldoc.createElement( "pvalue" )
        pValueCData = self.xmldoc.createTextNode( "%4.3f" % pval )

        pValueNode.setAttribute( "test", "KS" )
        pValueNode.setAttribute( "status", status )

        counter = self.parent.getAttribute( "KS" + status )
        if ( not counter ): counter = "0"
        self.parent.setAttribute( "KS" + status, "%d" % ( int(counter) + 1 )  )

        pValueNode.appendChild( pValueCData )

        statNode.appendChild( pValueNode )
        return status
        

    ## perform "bin-by-bin" statistic test
    # @param self "Me, myself and Irene"
    # @param statNode <stat> element
    def __testBBB( self, statNode ):
       
        nbBins = 0.0
        nbBinsSame = 0.0

        binX = self.mon.GetNbinsX()
        binY = self.mon.GetNbinsY()

        for i in range( binX+1 ):
            for j in range( binY +1 ):
                mon = self.mon.GetBinContent( i, j )
                ref = self.ref.GetBinContent( i, j )
                if ( mon or ref ):
                    nbBins += 1
                    if ( mon == ref ):
                        nbBinsSame += 1

        self.debug("*** bin-by-bin *** all none-empty bins=%d all equal non-empty bins=%d" % ( nbBins, nbBinsSame ) )

        pval = 0.0
        if ( nbBins ):
            pval = nbBinsSame / nbBins  
            self.debug("*** bin-by-bin *** p-value=%4.3f" % pval )
        elif ( not nbBinsSame ):
            pval = 1.0
            self.warn( "*** bin-by-bin *** both histograms are empty!" )
            self.debug( "*** bin-by-bin *** p-value=%4.3f" % pval )
        else:
            self.warn( "*** bin-by-bin *** reference histogram is empty, while monitored has some entries" )
            self.debug( "*** bin-by-bin *** test failed" )
        
        status = self.__getTestStatus( pval )
        self.info("*** bin-by-bin *** p-value=%4.3f status=%s" % ( pval, status ) )

        self.sumTable["bbb"][status] = self.sumTable["bbb"][status] + 1     

        pValueNode = self.xmldoc.createElement( "pvalue" )
        pValueCData = self.xmldoc.createTextNode( "%4.3f" % pval )

        pValueNode.setAttribute( "test", "bbb" )
        pValueNode.setAttribute( "status", status )

        counter = self.parent.getAttribute( "bbb" + status )
        if ( not counter ): counter = "0"
        self.parent.setAttribute( "bbb" + status, "%d" % ( int(counter) + 1 )  )

        pValueNode.appendChild( pValueCData )

        statNode.appendChild( pValueNode )

        return status


    ## perform "diff(MeanY)" test for TProfile histos
    # @param self "Me, myself and Irene"
    # @param statNode <stat> element
    def __testMeanY( self, statNode ):

        avgEffmon = self.mon.GetMean( 2 ) * 100
        avgEffref = self.ref.GetMean( 2 ) * 100

        self.debug("*** MeanY Test *** refMean=%4.3f and monMean=%4.3f" % ( avgEffref, avgEffmon ) )
        
        pval = abs(avgEffmon - avgEffref)

        status = self.__getMeanTestStatus( pval )
        self.info("*** Mean Test *** p-value=%4.3f status=%s" % ( pval, status ) )

        self.sumTable["meanY"][status] = self.sumTable["meanY"][status] + 1     

        pValueNode = self.xmldoc.createElement( "pvalue" )
        pValueCData = self.xmldoc.createTextNode( "%4.3f" % pval )

        pValueNode.setAttribute( "test", "meanY" )
        pValueNode.setAttribute( "status", status )

        counter = self.parent.getAttribute( "meanY" + status )
        if ( not counter ): counter = "0"
        self.parent.setAttribute( "meanY" + status, "%d" % ( int(counter) + 1 )  )

        pValueNode.appendChild( pValueCData )

        statNode.appendChild( pValueNode )

        return status
    
    
    ## get test status for given pvalue 
    # @param self "Me, myself and Irene"
    # @param pval p-value from test
    def __getTestStatus( self, pval ):
        if ( ( pval < 0.0 or pval > 1.0 ) or  
             ( pval <= self.opts.pfail ) ): return "FAIL"
        if ( pval > self.opts.pwarn ): return "OK"
        return "WARN"


    ## get test status for given pvalue 
    # @param self "Me, myself and Irene"
    # @param pval p-value from test
    def __getMeanTestStatus( self, pval ):
        if ( pval > 1.0 and pval <= 5.0 ): return "WARN"
        if ( pval <= 1.0  ): return "OK"
        return "FAIL"


    ## make plots
    # @param self "Me, myself and Irene"
    def __makePlots( self ):
        if ( self.__plotter ):
            status = self.__plotter.plot( self.node, self.mon, self.ref )
            return status
    

    ## string representation of DCubeTester
    # @param self "Me, myself and Irene" 
    def __str__( self ):
        out  = "*"*61 + "\n"
        out += "* RUN SUMMARY\n"
        out += "*"*61 + "\n"
        out += "* objects processed = %d\n" % self.__nbObjs
        out += "* STATISTICS TESTS TABLE\n"
        out += "* " + "-"*45 + "\n"
        out += "* | %-8s | %8s | %8s | %8s |\n" % ( "test", "OK", "WARN", "FAIL") 
        self.allGOOD = 0
        self.allWARN = 0
        self.allFAIL = 0
        for key, value  in self.sumTable.iteritems():
            nbGOOD = value["OK"]
            nbWARN = value["WARN"]
            nbFAIL = value["FAIL"]

            self.allGOOD += nbGOOD
            self.allWARN += nbWARN
            self.allFAIL += nbFAIL
            out += "* " + "-"*45+"\n"
            out += "* | %-8s | %8d | %8d | %8d |\n" % ( key, nbGOOD, nbWARN, nbFAIL )
            
        out += "* " + "-"*45+"\n"

        
        out += "* | %-8s | %8d | %8d | %8d |\n" % ( "all", self.allGOOD, self.allWARN, self.allFAIL )
        out += "* " + "-"*45+"\n"

        self.fracGOOD = 0.0
        self.fracWARN = 0.0
        self.fracFAIL = 0.0
        all = float( self.allGOOD + self.allWARN + self.allFAIL ) 
        if ( all ):
            self.fracGOOD = 100 * float( self.allGOOD ) / all 
            self.fracWARN = 100 * float( self.allWARN ) / all 
            self.fracFAIL = 100 * float( self.allFAIL ) / all
        out += "* | %%        |    %04.2f |    %04.2f |    %04.2f |\n" % ( self.fracGOOD, self.fracWARN, self.fracFAIL )
        out += "* " + "-"*45+"\n"

        out += "*"*61+"\n"
        self.nbErrors = sum( self.errors.itervalues() )
        out += "* WARNINGS = %3d\n" % self.nbErrors
        i = 1
        for key, value in self.errors.iteritems( ):
            sev, what = key.split(";")
            
            out += "* [%02d] %4s %-30s - occured %d " %  (i, sev, what, value )
            if ( value == 1 ): out += "time\n"
            else: out += "times\n"
            i += 1

        out += "*"*61+"\n"
        
        self.__status = ""
        
        if ( self.allFAIL != 0 ):
            self.__status = "FAIL"
        elif ( self.allWARN != 0 or self.nbErrors != 0 ):
            self.__status = "WARN"
        else:
            self.__status = "OK"

        out += "* OVERALL STATISTICS STATUS: %-4s\n" % self.__status
                    
        out += "*"*61
        
        return out

    ## put summary to the logger and create summary node 
    # @param self "Me, myself and Irene"
    # @return summary XML Element
    def summary( self ):
        
        for line in str(self).split("\n"): 
            self.info( line )

        summaryNode = self.xmldoc.createElement( "summary" )
        summaryNode.setAttribute( "status" , self.__status )
        summaryNode.setAttribute( "objs", "%d" % self.__nbObjs )
        summaryNode.setAttribute( "errors", "%d" % self.nbErrors  )

        testTable = self.xmldoc.createElement( "table" )
        testTable.appendChild( self.__sumTableRow( [ "test", "OK", "WARN", "FAIL" ] ) )
        for test, results in self.sumTable.iteritems(): 
            testTable.appendChild( self.__sumTableRow( [ test,
                                                         results["OK"],
                                                         results["WARN"],
                                                         results["FAIL"] ] ) )
            
        testTable.appendChild( self.__sumTableRow( ["sum", self.allGOOD, self.allWARN, self.allFAIL ] ) )
        testTable.appendChild( self.__sumTableRow( ["fraction",
                                                    "%4.2f" % self.fracGOOD,
                                                    "%4.2f" % self.fracWARN,
                                                    "%4.2f" % self.fracFAIL ] ) )
        
        summaryNode.appendChild( testTable )

        errorsNode = self.xmldoc.createElement( "errors" )
        errorsNode.setAttribute( "nb", "%d" % self.nbErrors )

        for error,times in self.errors.iteritems():
            error = error.replace(";", " ")
            errorNode = self.xmldoc.createElement( "error" )
            errorNode.setAttribute( "times", "%d" % times )
            errorNode.setAttribute( "what", error )
            errorsNode.appendChild( errorNode )
            
        summaryNode.appendChild( errorsNode )
        
        return summaryNode 

    ## produce summary table row 
    # @param self "Me, myself and Irene"
    def __sumTableRow( self, what ):
        row = self.xmldoc.createElement( "tr" )
        for item in what:
            cellNode = self.xmldoc.createElement("td")
            cellCData = self.xmldoc.createTextNode( str(item) )
            cellNode.appendChild( cellCData )
            row.appendChild( cellNode )
        return row

    ## return overall summary as string
    # @param self "Me, myself and Irene"
    def status( self ):
        return self.__status