Beispiel #1
0
    def PopulateResultData(self):

        try:
            iniFileDict = BetterConfigParser()
            confFileDict = BetterConfigParser()

            iniFileSearchString = self.RawTestSessionDataPath + "/../configfiles/*.ini"
            iniFiles = glob.glob(iniFileSearchString)
            if len(iniFiles) == 1:
                iniFileDict.read(iniFiles[0])

            confFileSearchString = self.RawTestSessionDataPath + "/../configfiles/*.conf"
            confFiles = glob.glob(confFileSearchString)
            if len(confFiles) == 1:
                confFileDict.read(confFiles[0])

            try:
                self.ResultData['KeyValueDictPairs']['TestCenter']['Value'] = iniFileDict.get('OperationDetails', 'TestCenter')
            except:
                pass

            try:
                self.ResultData['KeyValueDictPairs']['Hostname']['Value'] =  iniFileDict.get('OperationDetails', 'Hostname')
            except:
                pass

        except:
            print "no .conf/.ini file found in 'configfiles' directory!"
            pass
Beispiel #2
0
    def PopulateResultData(self):

        try:
            iniFileDict = BetterConfigParser()
            confFileDict = BetterConfigParser()

            iniFileSearchString = self.RawTestSessionDataPath + "/configfiles/*.ini"
            iniFiles = glob.glob(iniFileSearchString)
            if len(iniFiles) == 1:
                iniFileDict.read(iniFiles[0])

            confFileSearchString = self.RawTestSessionDataPath + "/configfiles/*.conf"
            confFiles = glob.glob(confFileSearchString)
            if len(confFiles) == 1:
                confFileDict.read(confFiles[0])

            try:
                self.ResultData['KeyValueDictPairs']['TestCenter'][
                    'Value'] = iniFileDict.get('OperationDetails',
                                               'TestCenter')
            except:
                pass

            try:
                self.ResultData['KeyValueDictPairs']['Hostname'][
                    'Value'] = iniFileDict.get('OperationDetails', 'Hostname')
            except:
                pass

        except:
            print "no .conf/.ini file found in 'configfiles' directory!"
            pass
Beispiel #3
0
 def ReadModuleVersion(self):
     if self.verbose:
         print 'Read configParameters'
     self.check_Test_Software()
     config_format = self.HistoDict.get('ConfigParameters', 'configFormat')
     fileNames = self.HistoDict.get('ConfigParameters', 'configParameters').split(',')
     version = 'none'
     nRocs = 0
     halfModule = 0
     if config_format == 'dat':
         lines = []
         for filename in fileNames:
             fileName = '%s/%s' % (self.RawTestSessionDataPath, filename)
             try:
                 f = open(fileName)
                 lines.extend(f.readlines())
                 f.close()
             except:
                 warnings.warn('cannot open config parameters dat file {file}'.format(file=fileName))
         version = 'none'
         for line in lines:
             if line.strip().startswith('rocType'):
                 version = line.split(' ')[-1]
             elif line.strip().startswith('nRocs'):
                 nRocs = int(line.split(' ')[-1])
                 if self.verbose: print '\tnRocs: %s' % nRocs
             elif line.strip().startswith('halfModule'):
                 halfModule = int(line.split(' ')[-1])
                 if self.verbose: print '\thalfModule: %s' % halfModule
     elif config_format == 'cfg':
         config = BetterConfigParser()
         for filename in fileNames:
             fileName = '%s/%s' % (self.RawTestSessionDataPath, filename)
             config.read(fileName)
         try:
             version = config.get('ROC', 'type')
         except (KeyError,ConfigParser.NoOptionError,ConfigParser.NoSectionError):
             warnings.warn('cannot find version name {section}'.format(section=config.sections()))
             if 'ROC' in config.sections():
                 warnings.warn('cannot find type in  ROC-section: {options}'.format(options=config.options('ROC')))
             version = 'none'
         try:
             nRocs = config.getint('Module', 'rocs')
         except (KeyError,ConfigParser.NoOptionError,ConfigParser.NoSectionError):
             nRocs = 0
         try:
             halfModule = config.get('Module', 'halfModule')
         except (KeyError,ConfigParser.NoOptionError,ConfigParser.NoSectionError):
             halfModule = 0
     version = version.rstrip('\n')
     if self.verbose:
         print 'Version:    ', version
         print 'nRocs:      ', nRocs
         print 'halfModule: ', halfModule
     self.version = version
     self.nRocs = nRocs
     self.halfModule = halfModule
     return version, nRocs, halfModule
Beispiel #4
0
    def PopulateResultData(self):

        SysConfiguration = ConfigParser.ConfigParser()
        SysConfiguration.read(['Configuration/SystemConfiguration.cfg'])
        ParallelProcessing = False
        LimitProcesses = None
        try:
            if int(SysConfiguration.get('SystemConfiguration', 'ParallelProcessing').strip())>0:
                ParallelProcessing = True
        except:
            print "no 'ParallelProcessing' option found, running sequentially..."
            pass

        try:
            if int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip())>0:
                LimitProcesses = int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip())
        except:
            pass

        nRocs = self.ParentObject.Attributes['NumberOfChips']
        directory = self.ParentObject.Attributes['SCurvePaths']['HRSCurves_{Rate}'.format(Rate=self.Attributes['Rate'])]
        refit = self.TestResultEnvironmentObject.Configuration['Fitting']['refit']

        print 'SCurve fitting...'
        HistoDict = BetterConfigParser()
        HistoDict.add_section('SCurveFitting')
        HistoDict.set('SCurveFitting','nTrigs', str(10))
        HistoDict.set('SCurveFitting','dir', '')
        HistoDict.set('SCurveFitting','ignoreValidityCheck', '1')
        HistoDict.set('SCurveFitting','inputFileName', self.ParentObject.ParentObject.HistoDict.get('HighRate', 'SCurveDataFileName'))

        ePerVcal =  self.TestResultEnvironmentObject.GradingParameters['StandardVcal2ElectronConversionFactor']

        # compare current fit revision with saved revision
        FitRevDict = BetterConfigParser()
        fitRevFileName = directory + "/fits.rev"
        if os.path.isfile(fitRevFileName):

            try:
                FitRevDict.read(fitRevFileName)

                # if fit procedure changed -> refit
                if self.FitRevisionScurves > int(FitRevDict.get('FitRevisions', 'SCurves')):
                    print "info: fit procedure has changed, re-fitting..."
                    refit = True

                # if last fitting was not done completely (eg. STRG+C while fit...) -> refit
                if FitRevDict.get('Fit', 'Complete').strip().lower() != 'true':
                    print "info: last fit was incomplete! re-fitting..."
                    refit = True
            except:
                print "warning: could not read fit revision file!"

        else:
            FitRevDict.add_section('FitRevisions')
            FitRevDict.add_section('Fit')

        # unset complete flag in file
        FitRevDict.set('Fit','Complete', 'false')
        with open(fitRevFileName, 'wb') as fitRevFile:
            FitRevDict.write(fitRevFile)

        # do fitting
        fitter = SCurve_Fitting(refit=refit, HistoDict=HistoDict, ePerVcal=ePerVcal, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses)
        fitter.FitAllSCurve(directory, nRocs)

        # update saved revision
        FitRevDict.set('FitRevisions','SCurves', str(self.FitRevisionScurves))

        # set complete flag in file
        FitRevDict.set('Fit','Complete', 'true')
        with open(fitRevFileName, 'wb') as fitRevFile:
            FitRevDict.write(fitRevFile)
Beispiel #5
0
    def PopulateResultData(self):

        SysConfiguration = ConfigParser.ConfigParser()
        SysConfiguration.read(['Configuration/SystemConfiguration.cfg'])

        ParallelProcessing = False
        LimitProcesses = None
        try:
            if int(SysConfiguration.get('SystemConfiguration', 'ParallelProcessing').strip())>0:
                ParallelProcessing = True
        except:
            print "no 'ParallelProcessing' option found, running sequentially..."
            pass

        try:
            if int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip())>0:
                LimitProcesses = int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip())
        except:
            pass

        print 'do fitting...'
        nRocs = self.Attributes['NumberOfChips']
        directory = self.RawTestSessionDataPath
        refit = self.TestResultEnvironmentObject.Configuration['Fitting']['refit']

        refitScurves = refit
        refitPHlin = refit
        refitPHtan = refit

        # compare current fit revision with saved revision
        FitRevDict = BetterConfigParser()
        fitRevFileName = directory + "/fits.rev"
        if os.path.isfile(fitRevFileName):

            try:
                FitRevDict.read(fitRevFileName)

                # if fit procedure changed -> refit
                if self.FitRevisionScurves > int(FitRevDict.get('FitRevisions', 'SCurves')):
                    print "info: fit procedure has changed, re-fitting scurves..."
                    refitScurves = True
                if self.FitRevisionPHlin > int(FitRevDict.get('FitRevisions', 'PHlin')):
                    print "info: fit procedure has changed, re-fitting ph lin..."
                    refitPHlin = True
                if self.FitRevisionPHtan > int(FitRevDict.get('FitRevisions', 'PHtan')):
                    print "info: fit procedure has changed, re-fitting ph tan..."
                    refitPHtan = True

                # if last fitting was not done completely (eg. STRG+C while fit...) -> refit
                if FitRevDict.get('Fit', 'Complete').strip().lower() != 'true':
                    print "info: last fit was incomplete! re-fitting all..."
                    refitScurves = True
                    refitPHlin = True
                    refitPHtan = True
            except:
                print "warning: could not read fit revision file!"

        else:
            FitRevDict.add_section('FitRevisions')
            FitRevDict.add_section('Fit')

        # unset complete flag in file
        FitRevDict.set('Fit','Complete', 'false')
        with open(fitRevFileName, 'wb') as fitRevFile:
            FitRevDict.write(fitRevFile)

        print 'SCurve fitting...'
        ePerVcal =  self.TestResultEnvironmentObject.GradingParameters['StandardVcal2ElectronConversionFactor']
        fitter = SCurve_Fitting(refit=refitScurves, HistoDict=self.ParentObject.HistoDict,ePerVcal=ePerVcal, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses)
        fitter.FitAllSCurve(directory,nRocs)
        print 'linear PH fitting...'
        fitter = PH_Fitting(fitMode=0, refit=refitPHlin, HistoDict=self.ParentObject.HistoDict, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses)
        fitter.FitAllPHCurves(directory,nRocs)
        print 'tanh PH fitting...'
        fitter = PH_Fitting(fitMode=3, refit=refitPHtan, HistoDict=self.ParentObject.HistoDict, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses)
        fitter.FitAllPHCurves(directory,nRocs)

        # update saved revisions
        FitRevDict.set('FitRevisions','SCurves', str(self.FitRevisionScurves))
        FitRevDict.set('FitRevisions','PHlin', str(self.FitRevisionPHlin))
        FitRevDict.set('FitRevisions','PHtan', str(self.FitRevisionPHtan))

        # unset complete flag in file
        FitRevDict.set('Fit','Complete', 'true')
        with open(fitRevFileName, 'wb') as fitRevFile:
            FitRevDict.write(fitRevFile)

        print 'done'
Beispiel #6
0
class GeneralTestResult(object):
    nRows = 80
    nCols = 52
    nTotalChips = 16

    MoReWebVersion = 'unknown MoReWeb version'

    '''
        Initialization function
        @param ParentObject Reference to the Parent Object
        @param InitialModulePath Starting point of modules
    '''
    
    def __init__(self, TestResultEnvironmentObject, ParentObject=None, InitialModulePath=None,
                 InitialFinalResultsStoragePath=None, InitialAttributes=None, Key=None, DisplayOptions=None):
        ROOTConfiguration.initialise_ROOT()
        self.verbose = False

        # Name of the Test Result, only a-zA-Z0-9_
        self.Name = ''

        # Name of the test result without prefixes
        self.NameSingle = ''
        self.Key = ''

        # Title displayed in HTML, etc
        self.Title = ''
        # options which might be transfered to Attributes?
        self.testSoftware = None
        self.HistoDict = None
        self.version = None
        self.nRocs = 0
        self.halfModule = 0
        self.CommentFromFile = None
        self.AddCommentsToKeyValueDictPairs = False

        if Key:
            self.Key = Key

        # Attributes like Date, etc.
        self.Attributes = {
            # Date as UNIX timestamp, if 0, the date of the parent object is taken
            'TestDate': 0,
            # Type of the tested object
            'TestedObjectType': '',
            # ID of the tested object
            'TestedObjectID': '',
            # Key for custom storage path
            'StorageKey': '',
            # subDirectory of the test result root-files
            'TestResultSubDirectory': '',
        }

        self.Show = True
        self.Enabled = True
        self.SavePlotFile = True
        self.GzipSVG = TestResultEnvironmentObject.Configuration['GzipSVG']
        
        self.DefaultImageFormat = TestResultEnvironmentObject.Configuration['DefaultImageFormat'].strip().lower()
        if TestResultEnvironmentObject.Configuration.has_key('AdditionalImageFormats'):
            self.AdditionalImageFormats = TestResultEnvironmentObject.Configuration['AdditionalImageFormats'].strip().lower().split(',')
        else:
        	self.AdditionalImageFormats = ['root', 'pdf']
        	
        if TestResultEnvironmentObject.Configuration.has_key('OverviewHTMLLink'):
            self.OverviewHTMLLink = TestResultEnvironmentObject.Configuration['OverviewHTMLLink']
        else:
            self.OverviewHTMLLink = None
            
        # Path for current test to folder with root-files
        self.RawTestSessionDataPath = ''

        # Path for current test result files (html, svg, etc)
        self.FinalResultsStoragePath = ''

        # File handle (might be used by sub results)
        self.FileHandle = 0

        # Reference to ROOT canvas
        self.Canvas = None

        # Display Properties as in the ResultData.SubTestResultDictList
        self.DisplayOptions = {
            'Order': 0,
            'Width': 1,
            'GroupWithNext': False,
        }

        # Result array
        self.ResultData = {
            # Key / ValueDict (dict with {Value, Unit, Label}, if Label not specified, the key is used as label) Pairs
            # 'KeyValueDictPairs':{
            # 'MyKey':{
            # 'Value':'25',
            # 'NumericValue':25,
            # 'Unit': 'kg',
            # 'Label': 'My Key'
            # }
            # }
            'KeyValueDictPairs': {},

            # List of keys for sorting
            'KeyList': [],
            # Plot data
            'Plot': {
                'Enabled': 0,
                'ROOTObject': None,
                'Caption': '',
                'ImageFile': '',
                'Format': self.DefaultImageFormat,
                'AdditionalFormats':self.AdditionalImageFormats,
                'ImageFilePDF':'',
            },
            # SubTest Results
            'SubTestResults': {},
            # List of {Key, Module} dict for sorting and special attributes
            'SubTestResultDictList': [],
            # 'SubTestResultDictList':{
            # {
            # 'Key':'Noise'
            # 'Module':'Noise',
            # 'TestResultObject':object,
            # 'InitialAttributes':{'StorageKey':'Blub'},
            # 'DisplayOptions':{'Order':1, 'Width':1,'GroupWithNext':False},
            # }
            # }

            # hidden data, might be needed by other test results but is not displayed
            'HiddenData': {},

            # a data table if needed
            'Table': {
                'HEADER': [],
                'BODY': [],
                'FOOTER': [],
            }

        }
        # Reference to parent object
        self.ParentObject = None
        # Reference to the Test Result Environment
        self.TestResultEnvironmentObject = None
        self.TestResultEnvironmentObject = TestResultEnvironmentObject
        self.Canvas = self.TestResultEnvironmentObject.Canvas
        self.RawTestSessionDataPath = self.TestResultEnvironmentObject.ModuleDataDirectory

        if ParentObject:
            self.ParentObject = ParentObject
            self.RawTestSessionDataPath = self.ParentObject.RawTestSessionDataPath

        if InitialAttributes:
            self.Attributes.update(InitialAttributes)

        if DisplayOptions:
            self.DisplayOptions.update(DisplayOptions)

        if self.Attributes['TestResultSubDirectory']:
            self.RawTestSessionDataPath += '/' + self.Attributes['TestResultSubDirectory']

        self.CustomInit()

        if not self.Title:
            self.Title = self.NameSingle

        # Module Path
        self.ModulePath = self.NameSingle
        
        
        if InitialModulePath:
            self.ModulePath = InitialModulePath

        if InitialFinalResultsStoragePath:
            self.FinalResultsStoragePath = InitialFinalResultsStoragePath

        self.RelativeFinalResultsStoragePath = ''

        if self.ParentObject:
            self.FinalResultsStoragePath = self.ParentObject.FinalResultsStoragePath
            self.RelativeFinalResultsStoragePath = self.ParentObject.RelativeFinalResultsStoragePath
            self.ModulePath = self.ParentObject.ModulePath + '.' + self.ModulePath

        if self.Attributes['StorageKey']:
            self.FinalResultsStoragePath += '/' + self.Attributes['StorageKey']
            self.RelativeFinalResultsStoragePath += '/' + self.Attributes['StorageKey']
        else:
            self.FinalResultsStoragePath += '/' + self.NameSingle
            self.RelativeFinalResultsStoragePath += '/' + self.NameSingle
        self.SetFinalResultsStoragePath()

        if not os.path.exists(self.FinalResultsStoragePath):
            os.makedirs(self.FinalResultsStoragePath)

        # Load all sub test results in correct order
        i2 = len(self.ResultData['SubTestResultDictList'])
        for i in self.ResultData['SubTestResultDictList']:
            if i.has_key('Module'):
                SubModule = i['Module']
            else:
                SubModule = i['Key']
                i['Module'] = SubModule

            if i.has_key('InitialAttributes'):
                pass
            else:
                i['InitialAttributes'] = {}

            # set the test date for all test results
            i['InitialAttributes'].update({
                'TestDate': self.Attributes['TestDate']
            })

            # Ensure all keys of display properties are set
            DisplayOptions = {
                'Order': i2,
                'Width': 1,
                'Show': True,
                'GroupWithNext': False,
            }

            if i.has_key('DisplayOptions'):
                DisplayOptions.update(i['DisplayOptions'])

            i['DisplayOptions'] = DisplayOptions
           
            importdir = self.ModulePath + '.' + SubModule
            try:
                # print 'import ',importdir,SubModule
                f = __import__(importdir + '.' + SubModule, fromlist=[importdir + '.' + 'TestResult'])
            except ImportError as inst:
                # print 'could not ',importdir+'.'+SubModule,SubModule
                # print 'type',type(inst)
                # print 'inst',inst
                f = __import__(importdir + '.TestResult', fromlist=[''])
                print 'imported', f, 'please change name of file'
            pass

            self.ResultData['SubTestResults'][i['Key']] = f.TestResult(
                self.TestResultEnvironmentObject,
                self,
                None,
                None,
                i['InitialAttributes'],
                i['Key'],
                i['DisplayOptions']
            )

            i['TestResultObject'] = self.ResultData['SubTestResults'][i['Key']]
            i2 += 1

    def check_Test_Software_Pyxar(self):
        self.RawTestSessionDataPath = os.path.abspath(self.RawTestSessionDataPath)
        files = glob.glob(self.RawTestSessionDataPath + '/test.cfg') + \
                glob.glob(self.RawTestSessionDataPath + '/*/test.cfg')
        return len(files) > 0

    def check_Test_Software_Pxar(self):
        data = glob.glob(self.RawTestSessionDataPath + '/*[p,P][x,X][a,A][r,R]*.*') + \
                   glob.glob(self.RawTestSessionDataPath + '/*/*[p,P][x,X][a,A][r,R]*.*')
        if len(data):
            return True

        LogFileNames = glob.glob(self.RawTestSessionDataPath + '/*.log') + glob.glob(self.RawTestSessionDataPath + '/*/*.log')

        for LogFileName in LogFileNames:
            LogFile = open(LogFileName, "r")
            FirstLine = LogFile.readline()
            LogFile.close()
            if FirstLine.lower().find("welcome to pxar") > -1:
                return True

        return False

    def check_Test_Software(self):

        if self.check_Test_Software_Pyxar():
            self.testSoftware = 'pyxar'
        elif self.check_Test_Software_Pxar():
            self.testSoftware = 'pxar'
        else:
            data = glob.glob(self.RawTestSessionDataPath + '/*[p,P][x,X][a,A][r,R]*.*') + \
                   glob.glob(self.RawTestSessionDataPath + '/*/*[p,P][x,X][a,A][r,R]*.*')
            if len(data):
                self.testSoftware = 'pxar'
            else:
                #self.testSoftware = 'psi46expert'
                self.testSoftware = 'pxar'
        self.HistoDict = BetterConfigParser()
        fileName = 'Configuration/Software/%s.cfg' % self.testSoftware
        self.HistoDict.read(fileName)
        fileName = 'Configuration/Software/global.cfg'
        self.HistoDict.read(fileName)

    def check_for_comments(self):
        dir_name = os.path.abspath(self.RawTestSessionDataPath)
        comment_files = glob.glob(dir_name + '/comment*')
        comment = ''
        comment_short = ''
        for filename in comment_files:
            comment += '{filename}:\n'.format(filename=filename.split('/')[-1])
            with file(filename) as f:
                s = f.read()
            comment += s + '\n\n'
            comment_short += s + ' '
        # if self.ResultData:
        # if not 'KeyVaueDictPairs' in self.ResultData:
        # self.ResultData['KeyValueDictPairs'] = {}
        #
        # if not 'KeyValueList' in self.ResultData:
        # self.ResultData['KeyValueList'] = []
        if self.verbose:
            print 'checking', self.RawTestSessionDataPath, comment_files
        if comment != '':
            if self.verbose:
                print 'added comment', comment, 'to ', self.Name

            if self.AddCommentsToKeyValueDictPairs:
                self.ResultData['KeyValueDictPairs']['Comment'] = {
                    'Value': comment,
                    'Style': 'font-weight:bold;color:red;' if '!' in comment else 'font-weight:bold;',
                }
                self.ResultData['KeyList'].append('Comment')
            self.CommentFromFile = comment_short

    def check_for_manualGrade(self):
        self.RawTestSessionDataPath = os.path.abspath(self.RawTestSessionDataPath)
        gradefilename = self.RawTestSessionDataPath + '/grade.txt'
        print "checking for manual grading in '%s'"%gradefilename
        grade = ''
        if os.path.isfile(gradefilename):
            try:
                gradefile = open(gradefilename)
            except:
                warnings.warn('cannot open manual grade file {file}'.format(file=gradefilename))
            grade = gradefile.read().strip()
            gradefile.close()
            # grade can be given either as number 1,2,3 or as letter A,B,C
            GradeNames = ['A','B','C']
            if grade in GradeNames:
                grade = 1 + GradeNames.index(grade)
            print "Reading a manual grade "+str(grade)+" specified by the user in "+str(gradefilename)
        else:
            print " => not found."
        return grade
            

    def ReadModuleVersion(self):
        if self.verbose:
            print 'Read configParameters'
        self.check_Test_Software()
        config_format = self.HistoDict.get('ConfigParameters', 'configFormat')
        fileNames = self.HistoDict.get('ConfigParameters', 'configParameters').split(',')
        version = 'none'
        nRocs = 0
        halfModule = 0
        if config_format == 'dat':
            lines = []
            for filename in fileNames:
                fileName = '%s/%s' % (self.RawTestSessionDataPath, filename)
                try:
                    f = open(fileName)
                    lines.extend(f.readlines())
                    f.close()
                except:
                    warnings.warn('cannot open config parameters dat file {file}'.format(file=fileName))
            version = 'none'
            for line in lines:
                if line.strip().startswith('rocType'):
                    version = line.split(' ')[-1]
                elif line.strip().startswith('nRocs'):
                    nRocs = int(line.split(' ')[-1])
                    if self.verbose: print '\tnRocs: %s' % nRocs
                elif line.strip().startswith('halfModule'):
                    halfModule = int(line.split(' ')[-1])
                    if self.verbose: print '\thalfModule: %s' % halfModule
        elif config_format == 'cfg':
            config = BetterConfigParser()
            for filename in fileNames:
                fileName = '%s/%s' % (self.RawTestSessionDataPath, filename)
                config.read(fileName)
            try:
                version = config.get('ROC', 'type')
            except (KeyError,ConfigParser.NoOptionError,ConfigParser.NoSectionError):
                warnings.warn('cannot find version name {section}'.format(section=config.sections()))
                if 'ROC' in config.sections():
                    warnings.warn('cannot find type in  ROC-section: {options}'.format(options=config.options('ROC')))
                version = 'none'
            try:
                nRocs = config.getint('Module', 'rocs')
            except (KeyError,ConfigParser.NoOptionError,ConfigParser.NoSectionError):
                nRocs = 0
            try:
                halfModule = config.get('Module', 'halfModule')
            except (KeyError,ConfigParser.NoOptionError,ConfigParser.NoSectionError):
                halfModule = 0
        version = version.rstrip('\n')
        if self.verbose:
            print 'Version:    ', version
            print 'nRocs:      ', nRocs
            print 'halfModule: ', halfModule
        self.version = version
        self.nRocs = nRocs
        self.halfModule = halfModule
        return version, nRocs, halfModule

    '''
        Populates all necessary data
        @final
    '''

    def PopulateAllData(self):
        self.OpenFileHandle()
        for i in self.ResultData['SubTestResultDictList']:
            if i['TestResultObject'].Enabled:
                self.SetCanvasSize()
                try:
                    i['TestResultObject'].PopulateAllData()
                    # i['TestResultObject'].check_for_comments()
                except Exception as inst:
                    exc_type, exc_obj, exc_tb = sys.exc_info()
                    # Start red color
                    sys.stdout.write("\x1b[31m")
                    sys.stdout.flush()
                    # Print error message
                    print '\x1b[31mException while processing', i['TestResultObject'].FinalResultsStoragePath
                    # Print traceback
                    traceback.print_exception(exc_type, exc_obj, exc_tb)
                    # Stop red color
                    sys.stdout.write("\x1b[0m")
                    sys.stdout.flush()

                    self.TestResultEnvironmentObject.ErrorList.append(
                        {'ModulePath': i['TestResultObject'].ModulePath,
                         'ErrorCode': inst,
                         'FinalResultsStoragePath': i['TestResultObject'].FinalResultsStoragePath}
                    )
                    # todo Felix: handel exceptions

        self.SetCanvasSize()
        try:
            self.PopulateResultData()
            #self.SaveCanvas()
            self.check_for_comments()
        except Exception as inst:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            # Start red color
            sys.stdout.write("\x1b[31m")
            sys.stdout.flush()
            print '\x1b[31mException while processing', self.FinalResultsStoragePath
            # Print traceback
            traceback.print_exception(exc_type, exc_obj, exc_tb)
            # Stop red color
            sys.stdout.write("\x1b[0m")
            sys.stdout.flush()

            self.TestResultEnvironmentObject.ErrorList.append(
                {'ModulePath': self.ModulePath,
                 'ErrorCode': inst,
                 'FinalResultsStoragePath': self.FinalResultsStoragePath}
                # 'FinalResultsStoragePath':i['TestResultObject'].FinalResultsStoragePath}
            )

    def AnalyzeLogfiles(self, logfiles, DetectMessages, ErrorNames):
        KeyValueDictPairs = {}
        KeyList = []
        ErrorObjects = []

        if not type(logfiles) is list:
            logfilesList = [logfiles]
        else:
            logfilesList = logfiles

        for logfile in logfilesList:
            if logfile:
                with open(logfile, 'r') as logfile:
                    for line in logfile:
                        ErrorObject = {}

                        if 'WARNING:' in line and not 'Not unmasking DUT' in line:
                            ErrorObject['type'] = 'warning'
                        if 'ERROR:' in line:
                            ErrorObject['type'] = 'error'
                        if 'CRITICAL:' in line:
                            ErrorObject['type'] = 'critical'

                        ErrorObject['channel'] = None
                        lineParts = line.strip().split()
                        if 'Channel' in lineParts:
                            pos = lineParts.index('Channel')
                            if pos < len(lineParts) - 1:
                                try:
                                    ErrorObject['channel'] = int(lineParts[pos+1])
                                except:
                                    pass

                        if ErrorObject.has_key('type'):
                            Found = False
                            for errorname, keywords in DetectMessages.iteritems():
                                for keyword in keywords:
                                    if keyword in line:
                                        ErrorObject['subtype'] = errorname
                                        Found = True
                                        break
                            if not Found:
                                ErrorObject['subtype'] = 'other'

                            ErrorObjects.append(ErrorObject)

        KeyValueDictPairs['nCriticals'] = {'Label': '# Criticals', 'Value': '%d'%len([True for ErrorObject in ErrorObjects if ErrorObject['type'] == 'critical'])}
        KeyValueDictPairs['nErrors'] = {'Label': '# Errors', 'Value': '%d'%len([True for ErrorObject in ErrorObjects if ErrorObject['type'] == 'error'])}
        KeyValueDictPairs['nWarnings'] = {'Label': '# Warnings', 'Value': '%d'%len([True for ErrorObject in ErrorObjects if ErrorObject['type'] == 'warning'])}

        for ch in range(4):
            KeyValueDictPairs['channel_%d_count'%ch] = {'Label': 'Channel %d'%ch, 'Value': '%d'%len([True for ErrorObject in ErrorObjects if ErrorObject['channel'] == ch])}

        for errorSubtype, keywords in DetectMessages.iteritems():
            KeyValueDictPairs['message_%s_count'%errorSubtype] = {'Label': ErrorNames[errorSubtype] if errorSubtype in ErrorNames else errorSubtype, 'Value': '%d'%len([True for ErrorObject in ErrorObjects if ErrorObject['subtype'] == errorSubtype])}

        KeyList = (['nCriticals', 'nErrors', 'nWarnings', 'channel_0_count', 'channel_1_count', 'channel_2_count', 'channel_3_count', 'message_tokenchain_count', 
            'message_eventid_count', 'message_readback_count', 'message_notokenpass_count', 'message_datasize_count', 'message_missingevents_count', 'message_usbtimeout_count', 'message_deser400_count', 'message_daqerror_count'])

        return {'KeyValueDictPairs': KeyValueDictPairs, 'KeyList': KeyList}


    '''
        Manually close all file handles of the sub tests
        @final
    '''

    def CloseSubTestResultFileHandles(self, Level=0):
        for i in self.ResultData['SubTestResultDictList']:
            if i['TestResultObject'].Enabled:
                i['TestResultObject'].CloseSubTestResultFileHandles(Level + 1)

        if Level:
            if self.FileHandle:

                if not type(self.FileHandle) == list:
                    self.FileHandle = [self.FileHandle]

                for SingleFileHandle in self.FileHandle:
                    try:
                        if "ROOT.TFile" in repr(SingleFileHandle):
                            if self.verbose:
                                print '\x1b[35mclose ROOT file "%s" ...\x1b[0m'%repr(SingleFileHandle)
                            SingleFileHandle.Close()
                        else:
                            if self.verbose:
                                print '\x1b[32mclose file "%s" ...\x1b[0m'%repr(SingleFileHandle)
                            SingleFileHandle.close()
                    except:
                        print '\x1b[33m warning: can not close file "%s" \x1b[0m'%repr(SingleFileHandle)

    '''
        Reads all attributes and writes it to the memory
    '''

    def CustomInit(self):
        pass

    '''
        Opens a file handle just before populating data
    '''

    def OpenFileHandle(self):
        pass

    '''
        Create a unique ID for creating root histograms
    '''

    def GetUniqueID(self):
        return self.TestResultEnvironmentObject.GetUniqueID(self.NameSingle)

    '''
        Sets the storage path
    '''

    def SetFinalResultsStoragePath(self):
        pass

    def CanvasSize(self, canvas):
        canvas.SetCanvasSize(
            self.DisplayOptions['Width'] * self.TestResultEnvironmentObject.Configuration['DefaultValues'][
                'CanvasWidth'],
            self.TestResultEnvironmentObject.Configuration['DefaultValues']['CanvasHeight']
        )
        canvas.Draw()
        canvas.Update()
        return canvas

    '''
        Sets the canvas size
    '''

    def SetCanvasSize(self):
        self.Canvas = self.CanvasSize(self.Canvas)

        # self.Canvas.SetCanvasSize(

    # self.DisplayOptions['Width']*self.TestResultEnvironmentObject.Configuration['DefaultValues']['CanvasWidth'],
    # self.TestResultEnvironmentObject.Configuration['DefaultValues']['CanvasHeight']
    # )
    # self.Canvas.Draw()
    # self.Canvas.Update()

    '''
        Saving the Canvas
    '''
    def SaveCanvas(self):
        if self.SavePlotFile:
            if self.Canvas:
                self.Canvas.SaveAs(self.GetPlotFileName())
                if not self.ResultData['Plot']['Caption']:
                    self.ResultData['Plot']['Caption'] = self.Title
                for Suffix in self.ResultData['Plot']['AdditionalFormats']:
                	if Suffix == 'pdf':
                		self.ResultData['Plot']['ImageFilePDF'] = self.GetPlotFileName(Suffix)
                		if self.ResultData['Plot']['ROOTObject']:
                		    self.ResultData['Plot']['ROOTObject'].SetTitle(self.ResultData['Plot']['Caption'])
                	self.Canvas.SaveAs(self.GetPlotFileName(Suffix))
                self.ResultData['Plot']['Enabled'] = 1
                self.ResultData['Plot']['ImageFile'] = self.GetPlotFileName()
    '''
        Generate the filename including the full path to the plot file according to the format
    '''

    def GetPlotFileName(self,Suffix=''):
        if Suffix == '':
            Suffix = self.ResultData['Plot']['Format']
        return self.FinalResultsStoragePath + '/' + self.NameSingle + '.' + Suffix

    '''
        Get the sub test result list in the display order
    '''

    def GetSortedSubTestResultDictList(self):
        # sort sub test results according to display order
        return sorted(
            self.ResultData['SubTestResultDictList'],
            key=lambda i: i['DisplayOptions']['Order']
        )

    '''
        Reads all test results and writes it to the memory
    '''

    def PopulateResultData(self):
        pass

    '''
        Generates all output files
    '''

    def GenerateFinalOutput(self):
        for i in self.ResultData['SubTestResults']:
            self.ResultData['SubTestResults'][i].GenerateFinalOutput()

        self.GenerateDataFiles()

    '''
        Generate files like Key/Value pairs in JSON format, ASCII-files, HTML-files, etc.
        @final
    '''

    def GenerateDataFiles(self):
        self.GenerateDataFileHTML()
        self.GenerateDataFileJSON()

    '''
        Generate HTML file
        @final
    '''

    def GenerateDataFileHTML(self):
        HtmlParser = self.TestResultEnvironmentObject.HtmlParser

        HTMLFileName = 'TestResult.html'

        FinalHTML = ''
        HTMLTemplate = self.TestResultEnvironmentObject.TestResultHTMLTemplate
        FinalHTML = HTMLTemplate
        Levels = self.ModulePath[:].split('.')

        # Stylesheet

        StylesheetHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###HEAD_STYLESHEET_TEMPLATE###')
        StylesheetHTML = HtmlParser.substituteMarkerArray(
            StylesheetHTMLTemplate,
            {
                '###STYLESHEET###': self.TestResultEnvironmentObject.MainStylesheet +
                                    self.TestResultEnvironmentObject.TestResultStylesheet,
            }
        )
        FinalHTML = HtmlParser.substituteSubpart(
            FinalHTML,
            '###HEAD_STYLESHEETS###',
            StylesheetHTML
        )
        FinalHTML = HtmlParser.substituteSubpart(
            FinalHTML,
            '###HEAD_STYLESHEET_TEMPLATE###',
            ''
        )

        # Clickpath
        ClickPathEntries = []
        ClickPathEntryTemplate = HtmlParser.getSubpart(HTMLTemplate, '###CLICKPATH_ENTRY###')

        LevelPath = ''
        i = 0
        tmpTestResultObject = self

        for Level in Levels[2:]:
            LevelPath = '../' * i
            ClickPathEntries.append(HtmlParser.substituteMarkerArray(
                ClickPathEntryTemplate,
                {
                    '###URL###': HtmlParser.MaskHTML(LevelPath + HTMLFileName),
                    '###LABEL###': HtmlParser.MaskHTML(tmpTestResultObject.Title)
                }
            ))
            if self.ParentObject:
                tmpTestResultObject = tmpTestResultObject.ParentObject

            i += 1
        if not self.OverviewHTMLLink:
            OverviewHTMLLink = os.path.relpath(self.TestResultEnvironmentObject.GlobalOverviewPath + '/Overview.html',
                                               self.FinalResultsStoragePath)
        else:
            OverviewHTMLLink = self.OverviewHTMLLink
        ClickPathEntries.append(HtmlParser.substituteMarkerArray(
            ClickPathEntryTemplate,
            {
                '###URL###': HtmlParser.MaskHTML(OverviewHTMLLink),
                '###LABEL###': 'Overview'
            }
        ))

        ClickPathEntries.reverse()

        CSSClasses = ''

        # Result Data
        FinalHTML = HtmlParser.substituteSubpartArray(
            FinalHTML,
            {
                '###CLICKPATH_ENTRY###': ''.join(ClickPathEntries),
                '###RESULTDATA###': self.GenerateResultDataHTML(self, 0, self.DisplayOptions),
                '###ADDITIONALCSSCLASSES###': CSSClasses,
            }
        )
        # Do it again for the ghost
        FinalHTML = HtmlParser.substituteSubpartArray(
            FinalHTML,
            {
                '###CLICKPATH_ENTRY###': ''.join(ClickPathEntries),
            }
        )

        f = open(self.FinalResultsStoragePath + '/' + HTMLFileName, 'w')
        f.write(FinalHTML)
        f.close()

    '''
        Generate Result Data HTML for usage in HTML files
        @final
    '''

    def GenerateResultDataHTML(self, TestResultObject, RecursionLevel, DisplayOptions):
        HtmlParser = self.TestResultEnvironmentObject.HtmlParser
        HTMLTemplate = self.TestResultEnvironmentObject.TestResultHTMLTemplate
        ResultDataHTML = HtmlParser.getSubpart(HTMLTemplate, '###RESULTDATA###')
        HTMLFileName = 'TestResult.html'

        CSSClasses = self.NameSingle

        ResultDataHTML = HtmlParser.substituteMarker(ResultDataHTML,
                                                     '###RESULTDATAADDITIONALCSSCLASSES###',
                                                     HtmlParser.MaskHTML(CSSClasses)
        )

        RecursionRelativePath = ''
        if RecursionLevel > 0:
            PathParts = TestResultObject.FinalResultsStoragePath.split('/')
            # print PathParts
            RecursionRelativePath = PathParts[-1] + '/'

        # Title
        if not TestResultObject.Title:
            TestResultObject.Title = TestResultObject.NameSingle

        MyObjectTestDate = ''
        VersionInfo = ''
        if RecursionLevel == 0 and TestResultObject.Attributes['TestDate']:
            MyObjectTestDate = 'Test Date: ' + datetime.datetime.fromtimestamp(
                float(TestResultObject.Attributes['TestDate'])).strftime("%Y-%m-%d %H:%M") + '<br><span style="font-size:10.5pt;" title="' + self.TestResultEnvironmentObject.MoReWebVersion + '">Analysis date: ' + datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + '</span>'
            VersionInfo = self.TestResultEnvironmentObject.MoReWebVersion + " on branch " + self.TestResultEnvironmentObject.MoReWebBranch

        MainTestResultAdditionalClasses = ''

        if RecursionLevel == 0:
            MainTestResultAdditionalClasses += 'MainTestResult Group'
            if self.DisplayOptions['Width'] > 1:
                MainTestResultAdditionalClasses += ' Width' + str(self.DisplayOptions['Width'])

        ResultDataHTML = HtmlParser.substituteMarkerArray(
            ResultDataHTML,
            {
                '###TITLE###': HtmlParser.MaskHTML(TestResultObject.Title),
                '###TESTDATE###': MyObjectTestDate,
                '###MAINTESTRESULTADDITIONALCLASSES###': MainTestResultAdditionalClasses,
                '###VERSIONINFO###': VersionInfo,
            }
        )
        # Plot
        PlotHTML = ''
        if TestResultObject.ResultData['Plot']['Enabled']:
            PlotHTML = HtmlParser.getSubpart(HTMLTemplate, '###PLOT###')
            PlotHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###PLOT###')
            PlotImageHTML = HtmlParser.getSubpart(PlotHTMLTemplate, '###PLOT_IMAGE###')
            '''PlotImageHTML = {
                'SVG':'',
                'PNG':''
            }
            if TestResultObject.ResultData['Plot']['Format'] == 'svg':
                PlotImageHTML['SVG'] = HtmlParser.getSubpart(PlotHTMLTemplate, '###PLOT_IMAGE_SVG###')
            elif TestResultObject.ResultData['Plot']['Format'] == 'png':
                PlotImageHTML['PNG'] = HtmlParser.getSubpart(PlotHTMLTemplate, '###PLOT_IMAGE_PNG###')

            for i in PlotImageHTML:
            '''

            # file operations for svg
            if TestResultObject.ResultData['Plot']['Format'] == 'svg' and RecursionLevel == 0 and self.SavePlotFile:
                f = open(TestResultObject.ResultData['Plot']['ImageFile'], 'r')
                SVGContent = f.read()
                f.close()

                # fix an error in chrome / safari when displaying resized svg
                if SVGContent.find('<svg preserveAspectRatio') == -1:
                    SVGContent = SVGContent.replace('<svg', '<svg preserveAspectRatio="xMinYMin"', 1)

                # remove an invalid space in attribute width and height
                SVGContent = SVGContent.replace('width=" ', 'width="')
                SVGContent = SVGContent.replace('height=" ', 'height="')
                SVGContent = SVGContent.replace('x=" ', 'x="')
                SVGContent = SVGContent.replace('y=" ', 'y="')
                SVGContent = SVGContent.replace('r=" ', 'r="')
                SVGContent = SVGContent.replace('=" ', '="')
                SVGContent = SVGContent.replace('&#786', '&#176')

                if self.GzipSVG and TestResultObject.ResultData['Plot']['ImageFile'].find('.svgz') == -1:
                    os.remove(TestResultObject.ResultData['Plot']['ImageFile'])
                    TestResultObject.ResultData['Plot']['ImageFile'] += 'z'
                    f = gzip.GzipFile(TestResultObject.ResultData['Plot']['ImageFile'], 'w')

                else:
                    f = open(TestResultObject.ResultData['Plot']['ImageFile'], 'w')
                    if os.path.exists(TestResultObject.ResultData['Plot']['ImageFile'] + 'z'):
                        os.remove(TestResultObject.ResultData['Plot']['ImageFile'] + 'z')

                f.write(SVGContent)
                f.close()

            if not TestResultObject.ResultData['Plot']['Caption']:
                TestResultObject.ResultData['Plot']['Caption'] = TestResultObject.Title
            try:
                PlotImageHTML = HtmlParser.substituteMarkerArray(
                    PlotImageHTML,
                    {
                        '###FILENAME###': HtmlParser.MaskHTML(
                            RecursionRelativePath + os.path.basename(TestResultObject.ResultData['Plot']['ImageFile'])),
                        '###PDFFILENAME###': HtmlParser.MaskHTML(
                            RecursionRelativePath + os.path.basename(TestResultObject.ResultData['Plot']['ImageFilePDF'])),
                        '###IMAGELARGECONTAINERID###': HtmlParser.MaskHTML(
                            TestResultObject.Name + '_' + TestResultObject.Key),
                        '###MARGIN_TOP###': str(int(-800. / float(
                            DisplayOptions['Width'] * self.TestResultEnvironmentObject.Configuration['DefaultValues'][
                                'CanvasWidth']) *
                                                    float(
                                                        self.TestResultEnvironmentObject.Configuration['DefaultValues'][
                                                            'CanvasHeight']) / 2.)),
                        '###TITLE###': TestResultObject.ResultData['Plot']['Caption'],
                        '###WIDTH###': str(DisplayOptions['Width']),
                        '###HEIGHT###': str(1),
                    }
                )
            except TypeError:
                print TestResultObject.Name, '_', TestResultObject.Key
                raise TypeError('Canntot convert, ' + str(TestResultObject.Name) + str(TestResultObject.Key))
            # PlotHTML = HtmlParser.substituteSubpart(PlotHTML, '###PLOT_IMAGE_'+i+'###', PlotImageHTML)
            PlotHTML = HtmlParser.substituteSubpart(PlotHTML, '###PLOT_IMAGE###', PlotImageHTML)

        ResultDataHTML = HtmlParser.substituteSubpart(ResultDataHTML, '###PLOT###', PlotHTML)

        # Key Value Dict Pairs
        KeyValueDictPairsRowHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###KEYVALUEDICTPAIRS_ROW###')
        KeyValueDictPairsRows = ''

        for i in TestResultObject.ResultData['KeyList']:
            if not TestResultObject.ResultData['KeyValueDictPairs'].has_key(i):
                warnings.warn('Cannot find Key: {Key} in {Name}'.format(Key=i,Name=self.Name))
                continue
            if not TestResultObject.ResultData['KeyValueDictPairs'][i].has_key('Unit'):
                TestResultObject.ResultData['KeyValueDictPairs'][i]['Unit'] = ''
            html_value = HtmlParser.MaskHTML(str(TestResultObject.ResultData['KeyValueDictPairs'][i]['Value']))
            if TestResultObject.ResultData['KeyValueDictPairs'][i].has_key('Sigma'):
                TestResultObject.ResultData['KeyValueDictPairs'][i]['SigmaOutput'] = ' +/- %s' % \
                                                                                     TestResultObject.ResultData[
                                                                                         'KeyValueDictPairs'][i][
                                                                                         'Sigma']
                html_value += ' &plusmn; ' + HtmlParser.MaskHTML(
                    str(TestResultObject.ResultData['KeyValueDictPairs'][i]['Sigma']))
            else:
                TestResultObject.ResultData['KeyValueDictPairs'][i]['SigmaOutput'] = ''

            if not TestResultObject.ResultData['KeyValueDictPairs'][i].has_key('Label'):
                TestResultObject.ResultData['KeyValueDictPairs'][i]['Label'] = i

            KeyValueDictPairsRows += HtmlParser.substituteMarkerArray(
                KeyValueDictPairsRowHTMLTemplate,
                {
                    '###KEY###': HtmlParser.MaskHTML(i),
                    '###LABEL###': HtmlParser.MaskHTML(
                        TestResultObject.ResultData['KeyValueDictPairs'][i]['Label']
                    ),
                    '###VALUE###': html_value,
                    '###UNIT###': HtmlParser.MaskHTML(
                        TestResultObject.ResultData['KeyValueDictPairs'][i]['Unit']
                    ),
                }
            )
        ResultDataHTML = HtmlParser.substituteSubpart(ResultDataHTML,
                                                      '###KEYVALUEDICTPAIRS_ROW###',
                                                      KeyValueDictPairsRows)

        # Table
        TableHTML = ''
        if RecursionLevel == 0 or True:
            TableHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###TABLE###')
            TableHTML = HtmlParser.GenerateTableHTML(TableHTMLTemplate, TestResultObject.ResultData['Table'], {
                '###ADDITIONALCSSCLASS###': '',
                '###ID###': 'Table',
            })

        ResultDataHTML = HtmlParser.substituteSubpart(ResultDataHTML,
                                                      '###TABLE###',
                                                      TableHTML)

        # Sub Test Results
        SubTestResultListHTML = ''

        if RecursionLevel == 0:
            GroupWithNext = False
            i2 = 0
            for i in TestResultObject.GetSortedSubTestResultDictList():
                if i['TestResultObject'].DisplayOptions['Show']:

                    GroupCSSClass = ''
                    if i2 % 5 == 0:
                        GroupCSSClass += ' WidthNthChild5n'
                    if i2 % 4 == 0:
                        GroupCSSClass += ' WidthNthChild4n'
                    if i2 % 3 == 0:
                        GroupCSSClass += ' WidthNthChild3n'
                    if i2 % 2 == 0:
                        GroupCSSClass += ' WidthNthChild2n'

                    if i['TestResultObject'].DisplayOptions['Width'] > 1:
                        GroupCSSClass += ' Width' + str(i['TestResultObject'].DisplayOptions['Width'])

                    if not GroupWithNext:
                        SubTestResultListHTML += HtmlParser.substituteMarker(
                            HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTGROUP_START###'),
                            '###SUBTESTRESULTGROUP_ADDITIONALCSSCLASSES###',
                            HtmlParser.MaskHTML(GroupCSSClass)
                        )
                        # only increase the width counter for a group start
                        i2 += i['TestResultObject'].DisplayOptions['Width']

                    SubTestResultListHTML += self.GenerateResultDataHTML(i['TestResultObject'], RecursionLevel + 1,
                                                                         i['TestResultObject'].DisplayOptions)

                    if not i['TestResultObject'].DisplayOptions['GroupWithNext']:
                        # if the last element was in a group but the current not, close the group
                        SubTestResultListHTML += HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTGROUP_END###')

                    GroupWithNext = i['DisplayOptions']['GroupWithNext']

            # if the last element was in a group, close the group
            if GroupWithNext:
                SubTestResultListHTML += HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTGROUP_END###')

        else:

            SubTestResultListItemHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTLIST_ITEM###')
            SubTestResultListHTML = HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTLIST###')
            SubTestResultListItems = ''

            # Overview link
            SubTestResultOverviewLinkHTML = HtmlParser.getSubpart(SubTestResultListHTML, '###OVERVIEW_LINK###')
            if TestResultObject.ResultData['SubTestResultDictList']:
                SubTestResultOverviewLinkHTML = HtmlParser.substituteMarkerArray(
                    SubTestResultOverviewLinkHTML,
                    {
                        '###URL###': HtmlParser.MaskHTML(
                            os.path.basename(TestResultObject.FinalResultsStoragePath) + '/' + HTMLFileName
                        ),
                    }
                )
            else:
                SubTestResultOverviewLinkHTML = ''

            # Subtests
            for i in TestResultObject.GetSortedSubTestResultDictList():
                if i['DisplayOptions']['Show']:
                    SubTestResultListItems += HtmlParser.substituteMarkerArray(
                        SubTestResultListItemHTMLTemplate,
                        {
                            '###URL###': HtmlParser.MaskHTML(
                                os.path.basename(TestResultObject.FinalResultsStoragePath) + '/' +
                                os.path.basename(i['TestResultObject'].FinalResultsStoragePath) + '/' + HTMLFileName
                            ),
                            '###LABEL###': HtmlParser.MaskHTML(
                                i['TestResultObject'].Title
                            ),
                        }
                    )
            SubTestResultListHTML = HtmlParser.substituteSubpartArray(
                SubTestResultListHTML,
                {
                    '###SUBTESTRESULTLIST_ITEM###': SubTestResultListItems,
                    '###OVERVIEW_LINK###': SubTestResultOverviewLinkHTML,
                }
            )

            if not SubTestResultListItems:
                SubTestResultListHTML = ''


        if 'HTMLContent' in TestResultObject.ResultData:
            ResultDataHTML = ResultDataHTML + TestResultObject.ResultData['HTMLContent']

        ResultDataHTML = HtmlParser.substituteSubpartArray(
            ResultDataHTML,
            {
                '###SUBTESTRESULTLIST###': SubTestResultListHTML,
                '###SUBTESTRESULTGROUP_START###': '',
                '###SUBTESTRESULTGROUP_END###': '',
            }
        )
        return ResultDataHTML

    '''
        Generate file from ResultData['KeyValueDictPairs'] Key/Value pairs in JSON format
        @final
    '''

    def GenerateDataFileJSON(self):
        data = None
        key = None
        DataKeys = ['KeyValueDictPairs', 'HiddenData']
        for DataKey in DataKeys:
            try:
                data = self.ResultData[DataKey]
                for key in data:
                    if type(data[key])==dict and data[key].has_key('Value'):
                        value = data[key]['Value']
                        if type(value) == set:
                            value = list(value)
                            data[key]['Value'] = value
                    elif type(data[key])==dict:
                        data[key] = list(data[key])
                    else:
                        data_value = data[key]
                        data[key] = {'Value': str(data_value)}

                f = open(self.FinalResultsStoragePath + '/'+DataKey+'.json', 'w')
                f.write(json.dumps(self.ResultData[DataKey], sort_keys=True, indent=4, separators=(',', ': '), cls=SetEncoder))
                f.close()
            except (KeyError,IOError,TypeError):
                if data and key in data:
                    if type(data[key])==dict:
                        warnings.warn(
                            'Cannot create JSON for %s, %s' % (type(data[key]['Value']), str(self.ResultData[DataKey])))
                    else:
                        warnings.warn('Cannot create JSON for %s.' % (repr(data)))
                else:
                    warnings.warn('Cannot create JSON for %s.' % (repr(data)))

    '''
        Generate file from ResultData['KeyValueDictPairs'] Key/Value pairs in ASCII format
        @final
    '''

    def GenerateDataFileASCII(self):
        pass

    '''
        Write all test results to the database
    '''

    def WriteToDatabase(self, ParentID=0):
        ColumnMapping = {}
        ID = 0
        ID = self.CustomWriteToDatabase(ParentID)

        WriteToDBSuccess = True
        for i in self.ResultData['SubTestResults']:
            try:
                SubtestWriteToDBSuccess = self.ResultData['SubTestResults'][i].WriteToDatabase(ID)
                WriteToDBSuccess = WriteToDBSuccess and SubtestWriteToDBSuccess
            except Exception as inst:
                # Start red color
                sys.stdout.write("\x1b[31m")
                sys.stdout.flush()
                print 'Error in subtest (write to database)', self.ResultData['SubTestResults'][i].ModulePath, \
                    self.ResultData['SubTestResults'][i].FinalResultsStoragePath
                print inst
                print inst.args
                print sys.exc_info()[0]
                print "\n\n------\n"
                # Print traceback
                exc_type, exc_obj, exc_tb = sys.exc_info()
                traceback.print_exception(exc_type, exc_obj, exc_tb)
                # Reset color
                sys.stdout.write("\x1b[0m")
                sys.stdout.flush()
                WriteToDBSuccess = False

        self.PostWriteToDatabase()
        return WriteToDBSuccess

    def CustomWriteToDatabase(self, ParentID):
        pass

    def PostWriteToDatabase(self):
        pass

    def __del__(self):
        self.CloseSubTestResultFileHandles()
Beispiel #7
0
class GeneralTestResult:
    nRows = 80
    nCols = 52
    nTotalChips = 16
    '''
        Initialization function
        @param ParentObject Reference to the Parent Object
        @param InitialModulePath Starting point of modules
    '''

    def __init__(self, TestResultEnvironmentObject, ParentObject=None, InitialModulePath=None,
                 InitialFinalResultsStoragePath=None, InitialAttributes=None, Key=None, DisplayOptions=None):
        ROOTConfiguration.initialise_ROOT()
        self.verbose = False

        # Name of the Test Result, only a-zA-Z0-9_
        self.Name = ''

        # Name of the test result without prefixes
        self.NameSingle = ''
        self.Key = ''

        # Title displayed in HTML, etc
        self.Title = ''

        if Key:
            self.Key = Key

        # Attributes like Date, etc.
        self.Attributes = {
            # Date as UNIX timestamp, if 0, the date of the parent object is taken
            'TestDate': 0,
            # Type of the tested object
            'TestedObjectType': '',
            # ID of the tested object
            'TestedObjectID': '',
            # Key for custom storage path
            'StorageKey': '',
            # subDirectory of the test result root-files
            'TestResultSubDirectory': '',
        }

        self.Show = True
        self.Enabled = True
        self.SavePlotFile = True
        self.GzipSVG = TestResultEnvironmentObject.Configuration['GzipSVG']
        self.DefaultImageFormat = TestResultEnvironmentObject.Configuration['DefaultImageFormat'].strip().lower()
        if TestResultEnvironmentObject.Configuration.has_key('OverviewHTMLLink'):
            self.OverviewHTMLLink = TestResultEnvironmentObject.Configuration['OverviewHTMLLink']
        else:
            self.OverviewHTMLLink = None
        # Path for current test to folder with root-files
        self.RawTestSessionDataPath = ''

        # Path for current test result files (html, svg, etc)
        self.FinalResultsStoragePath = ''


        # File handle (might be used by sub results)
        self.FileHandle = 0

        # Reference to ROOT canvas
        self.Canvas = None

        # Display Properties as in the ResultData.SubTestResultDictList
        self.DisplayOptions = {
            'Order': 0,
            'Width': 1,
            'GroupWithNext': False,
        }


        # Result array
        self.ResultData = {
            # Key / ValueDict (dict with {Value, Unit, Label}, if Label not specified, the key is used as label) Pairs
            # 'KeyValueDictPairs':{
            # 'MyKey':{
            #           'Value':25,
            #           'Unit': 'kg',
            #           'Label': 'My Key'
            #       }
            #   }
            'KeyValueDictPairs': {},

            # List of keys for sorting
            'KeyList': [],
            # Plot data
            'Plot': {
                'Enabled': 0,
                'ROOTObject': None,
                'Caption': '',
                'ImageFile': '',
                'Format': self.DefaultImageFormat  #svg
            },
            # SubTest Results
            'SubTestResults': {},
            # List of {Key, Module} dict for sorting and special attributes
            'SubTestResultDictList': [],
            # 'SubTestResultDictList':{
            #   {
            #       'Key':'Noise'
            #       'Module':'Noise',
            #       'TestResultObject':object,
            #       'InitialAttributes':{'StorageKey':'Blub'},
            #       'DisplayOptions':{'Order':1, 'Width':1,'GroupWithNext':False},
            #   }
            # }

            # hidden data, might be needed by other test results but is not displayed
            'HiddenData': {},

            # a data table if needed
            'Table': {
                'HEADER': [],
                'BODY': [],
                'FOOTER': [],
            }

        }

        # Reference to parent object
        self.ParentObject = None



        # Reference to the Test Result Environment
        self.TestResultEnvironmentObject = None

        self.TestResultEnvironmentObject = TestResultEnvironmentObject
        self.Canvas = self.TestResultEnvironmentObject.Canvas
        self.RawTestSessionDataPath = self.TestResultEnvironmentObject.ModuleDataDirectory

        if ParentObject:
            self.ParentObject = ParentObject
            self.RawTestSessionDataPath = self.ParentObject.RawTestSessionDataPath

        if InitialAttributes:
            self.Attributes.update(InitialAttributes)

        if DisplayOptions:
            self.DisplayOptions.update(DisplayOptions)

        if self.Attributes['TestResultSubDirectory']:
            self.RawTestSessionDataPath += '/' + self.Attributes['TestResultSubDirectory']

        self.CustomInit();

        if not self.Title:
            self.Title = self.NameSingle

        # Module Path
        self.ModulePath = self.NameSingle

        if InitialModulePath:
            self.ModulePath = InitialModulePath

        if InitialFinalResultsStoragePath:
            self.FinalResultsStoragePath = InitialFinalResultsStoragePath

        self.RelativeFinalResultsStoragePath = ''

        if self.ParentObject:
            self.FinalResultsStoragePath = self.ParentObject.FinalResultsStoragePath
            self.RelativeFinalResultsStoragePath = self.ParentObject.RelativeFinalResultsStoragePath
            self.ModulePath = self.ParentObject.ModulePath + '.' + self.ModulePath

        if self.Attributes['StorageKey']:
            self.FinalResultsStoragePath += '/' + self.Attributes['StorageKey']
            self.RelativeFinalResultsStoragePath += '/' + self.Attributes['StorageKey']
        else:
            self.FinalResultsStoragePath += '/' + self.NameSingle
            self.RelativeFinalResultsStoragePath += '/' + self.NameSingle
        self.SetFinalResultsStoragePath()

        if not os.path.exists(self.FinalResultsStoragePath):
            os.makedirs(self.FinalResultsStoragePath)

        # Load all sub test results in correct order
        i2 = len(self.ResultData['SubTestResultDictList'])
        for i in self.ResultData['SubTestResultDictList']:
            if i.has_key('Module'):
                SubModule = i['Module']
            else:
                SubModule = i['Key']
                i['Module'] = SubModule

            if i.has_key('InitialAttributes'):
                pass
            else:
                i['InitialAttributes'] = {}

            # set the test date for all test results
            i['InitialAttributes'].update({
                'TestDate': self.Attributes['TestDate']
            })

            # Ensure all keys of display properties are set
            DisplayOptions = {
                'Order': i2,
                'Width': 1,
                'Show': True,
                'GroupWithNext': False,
            }

            if i.has_key('DisplayOptions'):
                DisplayOptions.update(i['DisplayOptions'])

            i['DisplayOptions'] = DisplayOptions

            try:
                importdir = self.ModulePath + '.' + SubModule
                # print 'import ',importdir,SubModule
                f = __import__(importdir + '.' + SubModule, fromlist=[importdir + '.' + 'TestResult'])
            except Exception as inst:
                # print 'could not ',importdir+'.'+SubModule,SubModule
                # print 'type',type(inst)
                # print 'inst',inst
                f = __import__(importdir + '.TestResult', fromlist=[''])
                print 'imported', f, 'please change name of file'
            pass

            self.ResultData['SubTestResults'][i['Key']] = f.TestResult(
                self.TestResultEnvironmentObject,
                self,
                None,
                None,
                i['InitialAttributes'],
                i['Key'],
                i['DisplayOptions']
            )

            i['TestResultObject'] = self.ResultData['SubTestResults'][i['Key']]
            i2 += 1

    def check_Test_Software(self):
        # file = self.RawTestSessionDataPath + '/test.cfg'
        # print file
        self.RawTestSessionDataPath = os.path.abspath(self.RawTestSessionDataPath)
        files = glob.glob(self.RawTestSessionDataPath + '/test.cfg') + \
                glob.glob(self.RawTestSessionDataPath + '/*/test.cfg')
        # print 'pyxar:',files
        if len(files) > 0:
            self.testSoftware = 'pyxar'
        else:
            data = glob.glob(self.RawTestSessionDataPath + '/*[p,P][x,X][a,A][r,R]*.*') + \
                   glob.glob(self.RawTestSessionDataPath + '/*/*[p,P][x,X][a,A][r,R]*.*')
            if len(data):
                self.testSoftware = 'pxar'
            else:
                self.testSoftware = 'psi46expert'
        self.HistoDict = BetterConfigParser()
        fileName = 'Configuration/Software/%s.cfg' % self.testSoftware
        self.HistoDict.read(fileName)
        fileName = 'Configuration/Software/global.cfg'
        self.HistoDict.read(fileName)

    def check_for_comments(self):
        dir = os.path.abspath(self.RawTestSessionDataPath)
        comment_files = glob.glob(dir + '/comment*')
        comment = ''
        for filename in comment_files:
            comment += '{filename}:\n'.format(filename=filename.split('/')[-1])
            with file(filename) as f:
                s = f.read()
            comment += s + '\n\n'
        # if self.ResultData:
        # if not 'KeyVaueDictPairs' in self.ResultData:
        #         self.ResultData['KeyValueDictPairs'] = {}
        #
        #     if not 'KeyValueList' in self.ResultData:
        #         self.ResultData['KeyValueList'] = []
        if self.verbose:
            print 'checking', self.RawTestSessionDataPath, comment_files
        if comment != '':
            if self.verbose:
                print 'added comment', comment, 'to ', self.Name
            self.ResultData['KeyValueDictPairs']['Comment'] = {
                'Value': comment,
            }
            self.ResultData['KeyList'].append('Comment')

    def ReadModuleVersion(self):
        if self.verbose:
            print 'Read configParameters'
        self.check_Test_Software()
        format = self.HistoDict.get('ConfigParameters', 'configFormat')
        fileNames = self.HistoDict.get('ConfigParameters', 'configParameters').split(',')
        if format == 'dat':
            lines = []
            for filename in fileNames:
                fileName = '%s/%s' % (self.RawTestSessionDataPath, filename)
                f = open(fileName)
                lines.extend(f.readlines())
                f.close()
            version = 'none'
            for line in lines:
                if line.strip().startswith('rocType'):
                    version = line.split(' ')[-1]
                elif line.strip().startswith('nRocs'):
                    nRocs = int(line.split(' ')[-1])
                    if self.verbose: print '\tnRocs: %s' % nRocs
                elif line.strip().startswith('halfModule'):
                    halfModule = int(line.split(' ')[-1])
                    if self.verbose: print '\thalfModule: %s' % halfModule
        elif format == 'cfg':
            config = BetterConfigParser()
            for filename in fileNames:
                fileName = '%s/%s' % (self.RawTestSessionDataPath, filename)
                config.read(fileName)
            try:
                version = config.get('ROC', 'type')
            except:
                warnings.warn('cannot find version name {section}'.format(section=config.sections()))
                if 'ROC' in config.sections():
                    warnings.warn('cannot find version ROC-section {section}'.format(section=config.options('ROC')))
                version = 'none'
            try:
                nRocs = config.getint('Module', 'rocs')
            except:
                nRocs = 0
            try:
                halfModule = config.get('Module', 'halfModule')
            except:
                halfModule = 0
        version = version.rstrip('\n')
        if self.verbose:
            print 'Version:    ', version
            print 'nRocs:      ', nRocs
            print 'halfModule: ', halfModule
        self.version = version
        self.nRocs = nRocs
        self.halfModule = halfModule
        return (version, nRocs, halfModule)

    '''
        Populates all necessary data
        @final
    '''

    def PopulateAllData(self):
        self.OpenFileHandle();
        for i in self.ResultData['SubTestResultDictList']:
            if i['TestResultObject'].Enabled:
                self.SetCanvasSize()
                try:
                    i['TestResultObject'].PopulateAllData()
                    # i['TestResultObject'].check_for_comments()
                except Exception as inst:
                    exc_type, exc_obj, exc_tb = sys.exc_info()
                    # Start red color
                    sys.stdout.write("\x1b[31m")
                    sys.stdout.flush()
                    # Print error message
                    print '\x1b[31mException while processing', i['TestResultObject'].FinalResultsStoragePath
                    # Print traceback
                    traceback.print_exception(exc_type, exc_obj, exc_tb)
                    # Stop red color
                    sys.stdout.write("\x1b[0m")
                    sys.stdout.flush()

                    self.TestResultEnvironmentObject.ErrorList.append(
                        {'ModulePath': i['TestResultObject'].ModulePath,
                         'ErrorCode': inst,
                         'FinalResultsStoragePath': i['TestResultObject'].FinalResultsStoragePath}
                    )
                    # todo Felix: handel exceptions

        self.SetCanvasSize()
        try:
            self.PopulateResultData()
            self.check_for_comments()
        except Exception as inst:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            # Start red color
            sys.stdout.write("\x1b[31m")
            sys.stdout.flush()
            print '\x1b[31mException while processing', self.FinalResultsStoragePath
            # Print traceback
            traceback.print_exception(exc_type, exc_obj, exc_tb)
            # Stop red color
            sys.stdout.write("\x1b[0m")
            sys.stdout.flush()

            self.TestResultEnvironmentObject.ErrorList.append(
                {'ModulePath': self.ModulePath,
                 'ErrorCode': inst,
                 'FinalResultsStoragePath': self.FinalResultsStoragePath}
                # 'FinalResultsStoragePath':i['TestResultObject'].FinalResultsStoragePath}
            )

    '''
        Manually close all file handles of the sub tests
        @final
    '''

    def CloseSubTestResultFileHandles(self, Level=0):
        for i in self.ResultData['SubTestResultDictList']:
            if i['TestResultObject'].Enabled:
                i['TestResultObject'].CloseSubTestResultFileHandles(Level + 1)

        if Level:
            if self.FileHandle:
                try:
                    self.FileHandle.close()
                except:
                    try:
                        self.FileHandle.Close()
                    except:
                        pass

    '''
        Reads all attributes and writes it to the memory
    '''

    def CustomInit(self):
        pass

    '''
        Opens a file handle just before populating data
    '''

    def OpenFileHandle(self):
        pass


    '''
        Create a unique ID for creating root histograms
    '''

    def GetUniqueID(self):
        return self.TestResultEnvironmentObject.GetUniqueID(self.NameSingle)

    '''
        Sets the storage path
    '''

    def SetFinalResultsStoragePath(self):
        pass

    def CanvasSize(self, canvas):
        canvas.SetCanvasSize(
            self.DisplayOptions['Width'] * self.TestResultEnvironmentObject.Configuration['DefaultValues'][
                'CanvasWidth'],
            self.TestResultEnvironmentObject.Configuration['DefaultValues']['CanvasHeight']
        )
        canvas.Draw()
        canvas.Update()
        return canvas

    '''
        Sets the canvas size
    '''

    def SetCanvasSize(self):
        self.Canvas = self.CanvasSize(self.Canvas)

        # self.Canvas.SetCanvasSize(

    #             self.DisplayOptions['Width']*self.TestResultEnvironmentObject.Configuration['DefaultValues']['CanvasWidth'],
    #             self.TestResultEnvironmentObject.Configuration['DefaultValues']['CanvasHeight']
    #         )
    #         self.Canvas.Draw()
    #         self.Canvas.Update()


    '''
        Generate the filename including the full path to the plot file according to the format
    '''

    def GetPlotFileName(self):
        Suffix = self.ResultData['Plot']['Format']
        return self.FinalResultsStoragePath + '/' + self.NameSingle + '.' + Suffix

    '''
        Get the sub test result list in the display order
    '''

    def GetSortedSubTestResultDictList(self):
        # sort sub test results according to display order
        return sorted(
            self.ResultData['SubTestResultDictList'],
            key=lambda i: i['DisplayOptions']['Order']
        )


    '''
        Reads all test results and writes it to the memory
    '''

    def PopulateResultData(self):
        pass

    '''
        Generates all output files
    '''

    def GenerateFinalOutput(self):
        for i in self.ResultData['SubTestResults']:
            self.ResultData['SubTestResults'][i].GenerateFinalOutput()

        self.GenerateDataFiles()

    '''
        Generate files like Key/Value pairs in JSON format, ASCII-files, HTML-files, etc.
        @final
    '''

    def GenerateDataFiles(self):
        self.GenerateDataFileHTML()
        self.GenerateDataFileJSON()

    '''
        Generate HTML file
        @final
    '''

    def GenerateDataFileHTML(self):
        HtmlParser = self.TestResultEnvironmentObject.HtmlParser

        HTMLFileName = 'TestResult.html'

        FinalHTML = ''
        HTMLTemplate = self.TestResultEnvironmentObject.TestResultHTMLTemplate
        FinalHTML = HTMLTemplate
        Levels = self.ModulePath[:].split('.')

        # Stylesheet

        StylesheetHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###HEAD_STYLESHEET_TEMPLATE###')
        StylesheetHTML = HtmlParser.substituteMarkerArray(
            StylesheetHTMLTemplate,
            {
                '###STYLESHEET###': self.TestResultEnvironmentObject.MainStylesheet +
                                    self.TestResultEnvironmentObject.TestResultStylesheet,
            }
        )
        FinalHTML = HtmlParser.substituteSubpart(
            FinalHTML,
            '###HEAD_STYLESHEETS###',
            StylesheetHTML
        )
        FinalHTML = HtmlParser.substituteSubpart(
            FinalHTML,
            '###HEAD_STYLESHEET_TEMPLATE###',
            ''
        )


        # Clickpath

        ClickPathEntries = []
        ClickPathEntryTemplate = HtmlParser.getSubpart(HTMLTemplate, '###CLICKPATH_ENTRY###')

        LevelPath = ''
        i = 0
        tmpTestResultObject = self

        for Level in Levels[2:]:
            LevelPath = '../' * i
            ClickPathEntries.append(HtmlParser.substituteMarkerArray(
                ClickPathEntryTemplate,
                {
                    '###URL###': HtmlParser.MaskHTML(LevelPath + HTMLFileName),
                    '###LABEL###': HtmlParser.MaskHTML(tmpTestResultObject.Title)
                }
            ))
            if self.ParentObject:
                tmpTestResultObject = tmpTestResultObject.ParentObject

            i += 1
        if not self.OverviewHTMLLink:
            OverviewHTMLLink = os.path.relpath(self.TestResultEnvironmentObject.GlobalOverviewPath + '/Overview.html',
                                               self.FinalResultsStoragePath)
        else:
            OverviewHTMLLink = self.OverviewHTMLLink
        ClickPathEntries.append(HtmlParser.substituteMarkerArray(
            ClickPathEntryTemplate,
            {
                '###URL###': HtmlParser.MaskHTML(OverviewHTMLLink),
                '###LABEL###': 'Overview'
            }
        ))

        ClickPathEntries.reverse()

        CSSClasses = ''

        #Result Data
        FinalHTML = HtmlParser.substituteSubpartArray(
            FinalHTML,
            {
                '###CLICKPATH_ENTRY###': ''.join(ClickPathEntries),
                '###RESULTDATA###': self.GenerateResultDataHTML(self, 0, self.DisplayOptions),
                '###ADDITIONALCSSCLASSES###': CSSClasses,
            }
        )
        # Do it again for the ghost
        FinalHTML = HtmlParser.substituteSubpartArray(
            FinalHTML,
            {
                '###CLICKPATH_ENTRY###': ''.join(ClickPathEntries),
            }
        )

        f = open(self.FinalResultsStoragePath + '/' + HTMLFileName, 'w')
        f.write(FinalHTML)
        f.close()


    '''
        Generate Result Data HTML for usage in HTML files
        @final
    '''

    def GenerateResultDataHTML(self, TestResultObject, RecursionLevel, DisplayOptions):
        HtmlParser = self.TestResultEnvironmentObject.HtmlParser
        HTMLTemplate = self.TestResultEnvironmentObject.TestResultHTMLTemplate
        ResultDataHTML = HtmlParser.getSubpart(HTMLTemplate, '###RESULTDATA###')
        HTMLFileName = 'TestResult.html'

        CSSClasses = self.NameSingle

        ResultDataHTML = HtmlParser.substituteMarker(ResultDataHTML,
                                                     '###RESULTDATAADDITIONALCSSCLASSES###',
                                                     HtmlParser.MaskHTML(CSSClasses)
        )

        RecursionRelativePath = ''
        if RecursionLevel > 0:
            PathParts = TestResultObject.FinalResultsStoragePath.split('/')
            #print PathParts
            RecursionRelativePath = PathParts[-1] + '/'

        # Title
        if not TestResultObject.Title:
            TestResultObject.Title = TestResultObject.NameSingle
        MyObjectTestDate = '';
        if RecursionLevel == 0 and TestResultObject.Attributes['TestDate']:
            MyObjectTestDate = 'Test Date: ' + datetime.datetime.fromtimestamp(
                float(TestResultObject.Attributes['TestDate'])).strftime("%Y-%m-%d %H:%m")

        MainTestResultAdditionalClasses = ''

        if RecursionLevel == 0:
            MainTestResultAdditionalClasses += 'MainTestResult Group'
            if self.DisplayOptions['Width'] > 1:
                MainTestResultAdditionalClasses += ' Width' + str(self.DisplayOptions['Width'])

        ResultDataHTML = HtmlParser.substituteMarkerArray(
            ResultDataHTML,
            {
                '###TITLE###': HtmlParser.MaskHTML(TestResultObject.Title),
                '###TESTDATE###': MyObjectTestDate,
                '###MAINTESTRESULTADDITIONALCLASSES###': MainTestResultAdditionalClasses,
            }
        )
        # Plot
        PlotHTML = ''
        if TestResultObject.ResultData['Plot']['Enabled']:
            PlotHTML = HtmlParser.getSubpart(HTMLTemplate, '###PLOT###')
            PlotHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###PLOT###')
            PlotImageHTML = HtmlParser.getSubpart(PlotHTMLTemplate, '###PLOT_IMAGE###')
            '''PlotImageHTML = {
                'SVG':'',
                'PNG':''
            }
            if TestResultObject.ResultData['Plot']['Format'] == 'svg':
                PlotImageHTML['SVG'] = HtmlParser.getSubpart(PlotHTMLTemplate, '###PLOT_IMAGE_SVG###')
            elif TestResultObject.ResultData['Plot']['Format'] == 'png':
                PlotImageHTML['PNG'] = HtmlParser.getSubpart(PlotHTMLTemplate, '###PLOT_IMAGE_PNG###')

            for i in PlotImageHTML:
            '''

            # file operations for svg
            if TestResultObject.ResultData['Plot']['Format'] == 'svg' and RecursionLevel == 0 and self.SavePlotFile:
                f = open(TestResultObject.ResultData['Plot']['ImageFile'], 'r')
                SVGContent = f.read()
                f.close()

                # fix an error in chrome / safari when displaying resized svg
                if SVGContent.find('<svg preserveAspectRatio') == -1:
                    SVGContent = SVGContent.replace('<svg', '<svg preserveAspectRatio="xMinYMin"', 1)

                # remove an invalid space in attribute width and height
                SVGContent = SVGContent.replace('width=" ', 'width="')
                SVGContent = SVGContent.replace('height=" ', 'height="')
                SVGContent = SVGContent.replace('x=" ', 'x="')
                SVGContent = SVGContent.replace('y=" ', 'y="')
                SVGContent = SVGContent.replace('r=" ', 'r="')
                SVGContent = SVGContent.replace('=" ', '="')
                SVGContent = SVGContent.replace('&#786', '&#176')

                if self.GzipSVG and TestResultObject.ResultData['Plot']['ImageFile'].find('.svgz') == -1:
                    os.remove(TestResultObject.ResultData['Plot']['ImageFile'])
                    TestResultObject.ResultData['Plot']['ImageFile'] += 'z'
                    f = gzip.GzipFile(TestResultObject.ResultData['Plot']['ImageFile'], 'w')

                else:
                    f = open(TestResultObject.ResultData['Plot']['ImageFile'], 'w')
                    if os.path.exists(TestResultObject.ResultData['Plot']['ImageFile'] + 'z'):
                        os.remove(TestResultObject.ResultData['Plot']['ImageFile'] + 'z')

                f.write(SVGContent)
                f.close()

            if not TestResultObject.ResultData['Plot']['Caption']:
                TestResultObject.ResultData['Plot']['Caption'] = TestResultObject.Title
            try:
                PlotImageHTML = HtmlParser.substituteMarkerArray(
                    PlotImageHTML,
                    {
                        '###FILENAME###': HtmlParser.MaskHTML(
                            RecursionRelativePath + os.path.basename(TestResultObject.ResultData['Plot']['ImageFile'])),
                        '###IMAGELARGECONTAINERID###': HtmlParser.MaskHTML(
                            TestResultObject.Name + '_' + TestResultObject.Key),
                        '###MARGIN_TOP###': str(int(-800. / float(
                            DisplayOptions['Width'] * self.TestResultEnvironmentObject.Configuration['DefaultValues'][
                                'CanvasWidth']) *
                                                    float(
                                                        self.TestResultEnvironmentObject.Configuration['DefaultValues'][
                                                            'CanvasHeight']) / 2.)),
                        '###TITLE###': TestResultObject.ResultData['Plot']['Caption'],
                        '###WIDTH###': str(DisplayOptions['Width']),
                        '###HEIGHT###': str(1),
                    }
                )
            except TypeError:
                print TestResultObject.Name, '_', TestResultObject.Key
                raise TypeError('Canntot convert, ' + str(TestResultObject.Name) + str(TestResultObject.Key))
            #PlotHTML = HtmlParser.substituteSubpart(PlotHTML, '###PLOT_IMAGE_'+i+'###', PlotImageHTML)
            PlotHTML = HtmlParser.substituteSubpart(PlotHTML, '###PLOT_IMAGE###', PlotImageHTML)

        ResultDataHTML = HtmlParser.substituteSubpart(ResultDataHTML, '###PLOT###', PlotHTML)

        #Key Value Dict Pairs
        KeyValueDictPairsRowHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###KEYVALUEDICTPAIRS_ROW###')
        KeyValueDictPairsRows = ''

        for i in TestResultObject.ResultData['KeyList']:
            if not TestResultObject.ResultData['KeyValueDictPairs'].has_key(i):
                warnings.warn('Cannot find Key: {Key}'.format(Key=i))
                continue
            if not TestResultObject.ResultData['KeyValueDictPairs'][i].has_key('Unit'):
                TestResultObject.ResultData['KeyValueDictPairs'][i]['Unit'] = ''
            html_value = HtmlParser.MaskHTML(str(TestResultObject.ResultData['KeyValueDictPairs'][i]['Value']))
            if TestResultObject.ResultData['KeyValueDictPairs'][i].has_key('Sigma'):
                TestResultObject.ResultData['KeyValueDictPairs'][i]['SigmaOutput'] = ' +/- %s' % \
                                                                                     TestResultObject.ResultData[
                                                                                         'KeyValueDictPairs'][i][
                                                                                         'Sigma']
                html_value += ' &plusmn; ' + HtmlParser.MaskHTML(
                    str(TestResultObject.ResultData['KeyValueDictPairs'][i]['Sigma']))
            else:
                TestResultObject.ResultData['KeyValueDictPairs'][i]['SigmaOutput'] = ''

            if not TestResultObject.ResultData['KeyValueDictPairs'][i].has_key('Label'):
                TestResultObject.ResultData['KeyValueDictPairs'][i]['Label'] = i

            KeyValueDictPairsRows += HtmlParser.substituteMarkerArray(
                KeyValueDictPairsRowHTMLTemplate,
                {
                    '###KEY###': HtmlParser.MaskHTML(i),
                    '###LABEL###': HtmlParser.MaskHTML(
                        TestResultObject.ResultData['KeyValueDictPairs'][i]['Label']
                    ),
                    '###VALUE###': html_value,
                    '###UNIT###': HtmlParser.MaskHTML(
                        TestResultObject.ResultData['KeyValueDictPairs'][i]['Unit']
                    ),
                }
            )
        ResultDataHTML = HtmlParser.substituteSubpart(ResultDataHTML,
                                                      '###KEYVALUEDICTPAIRS_ROW###',
                                                      KeyValueDictPairsRows)

        #Table
        TableHTML = ''
        if RecursionLevel == 0 or True:
            TableHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###TABLE###')
            TableHTML = HtmlParser.GenerateTableHTML(TableHTMLTemplate, TestResultObject.ResultData['Table'], {
                '###ADDITIONALCSSCLASS###': '',
                '###ID###': 'Table',
            })

        ResultDataHTML = HtmlParser.substituteSubpart(ResultDataHTML,
                                                      '###TABLE###',
                                                      TableHTML)

        # Sub Test Results
        SubTestResultListHTML = ''

        if RecursionLevel == 0:
            GroupWithNext = False
            i2 = 0
            for i in TestResultObject.GetSortedSubTestResultDictList():
                if i['TestResultObject'].DisplayOptions['Show']:

                    GroupCSSClass = ''
                    if i2 % 5 == 0:
                        GroupCSSClass += ' WidthNthChild5n'
                    if i2 % 4 == 0:
                        GroupCSSClass += ' WidthNthChild4n'
                    if i2 % 3 == 0:
                        GroupCSSClass += ' WidthNthChild3n'
                    if i2 % 2 == 0:
                        GroupCSSClass += ' WidthNthChild2n'

                    if i['TestResultObject'].DisplayOptions['Width'] > 1:
                        GroupCSSClass += ' Width' + str(i['TestResultObject'].DisplayOptions['Width'])

                    if not GroupWithNext:
                        SubTestResultListHTML += HtmlParser.substituteMarker(
                            HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTGROUP_START###'),
                            '###SUBTESTRESULTGROUP_ADDITIONALCSSCLASSES###',
                            HtmlParser.MaskHTML(GroupCSSClass)
                        )
                        # only increase the width counter for a group start
                        i2 += i['TestResultObject'].DisplayOptions['Width']

                    SubTestResultListHTML += self.GenerateResultDataHTML(i['TestResultObject'], RecursionLevel + 1,
                                                                         i['TestResultObject'].DisplayOptions)

                    if not i['TestResultObject'].DisplayOptions['GroupWithNext']:
                        # if the last element was in a group but the current not, close the group
                        SubTestResultListHTML += HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTGROUP_END###')

                    GroupWithNext = i['DisplayOptions']['GroupWithNext']



            # if the last element was in a group, close the group
            if GroupWithNext:
                SubTestResultListHTML += HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTGROUP_END###')

        else:

            SubTestResultListItemHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTLIST_ITEM###')
            SubTestResultListHTML = HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTLIST###')
            SubTestResultListItems = ''

            # Overview link
            SubTestResultOverviewLinkHTML = HtmlParser.getSubpart(SubTestResultListHTML, '###OVERVIEW_LINK###')
            if TestResultObject.ResultData['SubTestResultDictList']:
                SubTestResultOverviewLinkHTML = HtmlParser.substituteMarkerArray(
                    SubTestResultOverviewLinkHTML,
                    {
                        '###URL###': HtmlParser.MaskHTML(
                            os.path.basename(TestResultObject.FinalResultsStoragePath) + '/' + HTMLFileName
                        ),
                    }
                )
            else:
                SubTestResultOverviewLinkHTML = ''

            # Subtests
            for i in TestResultObject.GetSortedSubTestResultDictList():
                if i['DisplayOptions']['Show']:
                    SubTestResultListItems += HtmlParser.substituteMarkerArray(
                        SubTestResultListItemHTMLTemplate,
                        {
                            '###URL###': HtmlParser.MaskHTML(
                                os.path.basename(TestResultObject.FinalResultsStoragePath) + '/' +
                                os.path.basename(i['TestResultObject'].FinalResultsStoragePath) + '/' + HTMLFileName
                            ),
                            '###LABEL###': HtmlParser.MaskHTML(
                                i['TestResultObject'].Title
                            ),
                        }
                    )
            SubTestResultListHTML = HtmlParser.substituteSubpartArray(
                SubTestResultListHTML,
                {
                    '###SUBTESTRESULTLIST_ITEM###': SubTestResultListItems,
                    '###OVERVIEW_LINK###': SubTestResultOverviewLinkHTML,
                }
            )

            if not SubTestResultListItems:
                SubTestResultListHTML = ''

        ResultDataHTML = HtmlParser.substituteSubpartArray(
            ResultDataHTML,
            {
                '###SUBTESTRESULTLIST###': SubTestResultListHTML,
                '###SUBTESTRESULTGROUP_START###': '',
                '###SUBTESTRESULTGROUP_END###': '',
            }
        )
        return ResultDataHTML


    '''
        Generate file from ResultData['KeyValueDictPairs'] Key/Value pairs in JSON format
        @final
    '''

    def GenerateDataFileJSON(self):
        try:
            data = self.ResultData['KeyValueDictPairs']
            for key in data:
                #{'NotAlivePixels': {'SigmaOutput': '', 'Unit': '', 'Value': Set([(0, 21, 31)]), 'Label': 'Pixels'}, 'MaskDefects': {'SigmaOutput': '', 'Unit': '', 'Value': Set([]), 'Label': 'Pixels'}, 'NoisyPixels': {'SigmaOutput': '', 'Unit': '', 'Value': Set([]), 'Label': 'Pixels'}, 'DeadPixels': {'SigmaOutput': '', 'Unit': '', 'Value': Set([(0, 21, 31)]), 'Label': 'Pixels'}, 'InefficentPixels': {'SigmaOutput': '', 'Unit': '', 'Value': Set([]), 'Label': 'Pixels'}}
                if data[key].has_key('Value'):
                    value = data[key]['Value']
                    if type(value) == set:
                        value = list(value)
                        data[key]['Value'] = value
                pass
            #self.ResultData['KeyValueDictPairs']
            f = open(self.FinalResultsStoragePath + '/KeyValueDictPairs.json', 'w')
            f.write(json.dumps(self.ResultData['KeyValueDictPairs'], sort_keys=True, indent=4, separators=(',', ': ')))
            f.close()
        except:
            print 'Cannot create JSON for %s, %s' % (type(data[key]['Value']), self.ResultData['KeyValueDictPairs'])

    '''
        Generate file from ResultData['KeyValueDictPairs'] Key/Value pairs in ASCII format
        @final
    '''

    def GenerateDataFileASCII(self):
        pass

    '''
        Write all test results to the database
    '''

    def WriteToDatabase(self, ParentID=0):
        ColumnMapping = {}
        ID = 0
        ID = self.CustomWriteToDatabase(ParentID)

        for i in self.ResultData['SubTestResults']:
            try:
                self.ResultData['SubTestResults'][i].WriteToDatabase(ID)
            except Exception as inst:
                print 'Error in subtest (write to database)', self.ResultData['SubTestResults'][i].ModulePath, \
                self.ResultData['SubTestResults'][i].FinalResultsStoragePath
                print inst
                print inst.args
                print sys.exc_info()[0]
                print "\n\n------\n"

        self.PostWriteToDatabase()

    def CustomWriteToDatabase(self, ParentID):
        pass

    def PostWriteToDatabase(self):
        pass

    def __del__(self):
        self.CloseSubTestResultFileHandles()
Beispiel #8
0
    def PopulateResultData(self):

        SysConfiguration = ConfigParser.ConfigParser()
        SysConfiguration.read(['Configuration/SystemConfiguration.cfg'])

        ParallelProcessing = False
        LimitProcesses = None
        try:
            if int(SysConfiguration.get('SystemConfiguration', 'ParallelProcessing').strip())>0:
                ParallelProcessing = True
        except:
            print "no 'ParallelProcessing' option found, running sequentially..."
            pass

        try:
            if int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip())>0:
                LimitProcesses = int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip())
        except:
            pass

        print 'do fitting...'
        nRocs = self.Attributes['NumberOfChips']
        directory = self.RawTestSessionDataPath
        refit = self.TestResultEnvironmentObject.Configuration['Fitting']['refit']

        refitScurves = refit
        refitPHlin = refit
        refitPHtan = refit

        # compare current fit revision with saved revision
        FitRevDict = BetterConfigParser()
        fitRevFileName = directory + "/fits.rev"
        if os.path.isfile(fitRevFileName):

            try:
                FitRevDict.read(fitRevFileName)

                # if fit procedure changed -> refit
                if self.FitRevisionScurves > int(FitRevDict.get('FitRevisions', 'SCurves')):
                    print "info: fit procedure has changed, re-fitting scurves..."
                    refitScurves = True
                if self.FitRevisionPHlin > int(FitRevDict.get('FitRevisions', 'PHlin')):
                    print "info: fit procedure has changed, re-fitting ph lin..."
                    refitPHlin = True
                if self.FitRevisionPHtan > int(FitRevDict.get('FitRevisions', 'PHtan')):
                    print "info: fit procedure has changed, re-fitting ph tan..."
                    refitPHtan = True

                # if last fitting was not done completely (eg. STRG+C while fit...) -> refit
                if FitRevDict.get('Fit', 'Complete').strip().lower() != 'true':
                    print "info: last fit was incomplete! re-fitting all..."
                    refitScurves = True
                    refitPHlin = True
                    refitPHtan = True
            except:
                print "warning: could not read fit revision file! => needs re-fitting"
                refitScurves = True
                refitPHlin = True
                refitPHtan = True

        else:
            FitRevDict.add_section('FitRevisions')
            FitRevDict.add_section('Fit')
            print "info: this folder has never been analyzed with a recent version of MoReWeb => needs re-fitting"
            refitScurves = True
            refitPHlin = True
            refitPHtan = True

        # unset complete flag in file
        FitRevDict.set('Fit','Complete', 'false')
        with open(fitRevFileName, 'wb') as fitRevFile:
            FitRevDict.write(fitRevFile)

        print 'SCurve fitting...'
        ePerVcal =  self.TestResultEnvironmentObject.GradingParameters['StandardVcal2ElectronConversionFactor']
        fitter = SCurve_Fitting(refit=refitScurves, HistoDict=self.ParentObject.HistoDict,ePerVcal=ePerVcal, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses)
        fitter.FitAllSCurve(directory,nRocs)
        print 'linear PH fitting...'
        fitter = PH_Fitting(fitMode=0, refit=refitPHlin, HistoDict=self.ParentObject.HistoDict, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses)
        fitter.FitAllPHCurves(directory,nRocs)
        print 'tanh PH fitting...'
        fitter = PH_Fitting(fitMode=3, refit=refitPHtan, HistoDict=self.ParentObject.HistoDict, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses)
        fitter.FitAllPHCurves(directory,nRocs)

        # update saved revisions
        FitRevDict.set('FitRevisions','SCurves', str(self.FitRevisionScurves))
        FitRevDict.set('FitRevisions','PHlin', str(self.FitRevisionPHlin))
        FitRevDict.set('FitRevisions','PHtan', str(self.FitRevisionPHtan))

        # unset complete flag in file
        FitRevDict.set('Fit','Complete', 'true')
        with open(fitRevFileName, 'wb') as fitRevFile:
            FitRevDict.write(fitRevFile)

        print 'done'
Beispiel #9
0
    def PopulateResultData(self):

        SysConfiguration = ConfigParser.ConfigParser()
        SysConfiguration.read(['Configuration/SystemConfiguration.cfg'])
        ParallelProcessing = False
        LimitProcesses = None
        try:
            if int(
                    SysConfiguration.get('SystemConfiguration',
                                         'ParallelProcessing').strip()) > 0:
                ParallelProcessing = True
        except:
            print "no 'ParallelProcessing' option found, running sequentially..."
            pass

        try:
            if int(
                    SysConfiguration.get('SystemConfiguration',
                                         'LimitProcesses').strip()) > 0:
                LimitProcesses = int(
                    SysConfiguration.get('SystemConfiguration',
                                         'LimitProcesses').strip())
        except:
            pass

        nRocs = self.ParentObject.Attributes['NumberOfChips']
        directory = self.ParentObject.Attributes['SCurvePaths'][
            'HRSCurves_{Rate}'.format(Rate=self.Attributes['Rate'])]
        refit = self.TestResultEnvironmentObject.Configuration['Fitting'][
            'refit']

        print 'SCurve fitting...'
        HistoDict = BetterConfigParser()
        HistoDict.add_section('SCurveFitting')
        HistoDict.set('SCurveFitting', 'nTrigs', str(10))
        HistoDict.set('SCurveFitting', 'dir', '')
        HistoDict.set('SCurveFitting', 'ignoreValidityCheck', '1')
        HistoDict.set(
            'SCurveFitting', 'inputFileName',
            self.ParentObject.ParentObject.HistoDict.get(
                'HighRate', 'SCurveDataFileName'))

        ePerVcal = self.TestResultEnvironmentObject.GradingParameters[
            'StandardVcal2ElectronConversionFactor']

        # compare current fit revision with saved revision
        FitRevDict = BetterConfigParser()
        fitRevFileName = directory + "/fits.rev"
        if os.path.isfile(fitRevFileName):

            try:
                FitRevDict.read(fitRevFileName)

                # if fit procedure changed -> refit
                if self.FitRevisionScurves > int(
                        FitRevDict.get('FitRevisions', 'SCurves')):
                    print "info: fit procedure has changed, re-fitting..."
                    refit = True

                # if last fitting was not done completely (eg. STRG+C while fit...) -> refit
                if FitRevDict.get('Fit', 'Complete').strip().lower() != 'true':
                    print "info: last fit was incomplete! re-fitting..."
                    refit = True
            except:
                print "warning: could not read fit revision file! => needs re-fitting"
                refit = True

        else:
            FitRevDict.add_section('FitRevisions')
            FitRevDict.add_section('Fit')
            print "info: never analyzed with recent version of MoReWeb => needs re-fitting"
            refit = True

        # unset complete flag in file
        FitRevDict.set('Fit', 'Complete', 'false')
        with open(fitRevFileName, 'wb') as fitRevFile:
            FitRevDict.write(fitRevFile)

        # do fitting
        fitter = SCurve_Fitting(refit=refit,
                                HistoDict=HistoDict,
                                ePerVcal=ePerVcal,
                                ParallelProcessing=ParallelProcessing,
                                LimitProcesses=LimitProcesses)
        fitter.FitAllSCurve(directory, nRocs)

        # update saved revision
        FitRevDict.set('FitRevisions', 'SCurves', str(self.FitRevisionScurves))

        # set complete flag in file
        FitRevDict.set('Fit', 'Complete', 'true')
        with open(fitRevFileName, 'wb') as fitRevFile:
            FitRevDict.write(fitRevFile)