def PopulateResultData(self): try: iniFileDict = BetterConfigParser() confFileDict = BetterConfigParser() iniFileSearchString = self.RawTestSessionDataPath + "/configfiles/*.ini" iniFiles = glob.glob(iniFileSearchString) if len(iniFiles) == 1: iniFileDict.read(iniFiles[0]) confFileSearchString = self.RawTestSessionDataPath + "/configfiles/*.conf" confFiles = glob.glob(confFileSearchString) if len(confFiles) == 1: confFileDict.read(confFiles[0]) try: self.ResultData['KeyValueDictPairs']['TestCenter'][ 'Value'] = iniFileDict.get('OperationDetails', 'TestCenter') except: pass try: self.ResultData['KeyValueDictPairs']['Hostname'][ 'Value'] = iniFileDict.get('OperationDetails', 'Hostname') except: pass except: print "no .conf/.ini file found in 'configfiles' directory!" pass
def PopulateResultData(self): try: iniFileDict = BetterConfigParser() confFileDict = BetterConfigParser() iniFileSearchString = self.RawTestSessionDataPath + "/../configfiles/*.ini" iniFiles = glob.glob(iniFileSearchString) if len(iniFiles) == 1: iniFileDict.read(iniFiles[0]) confFileSearchString = self.RawTestSessionDataPath + "/../configfiles/*.conf" confFiles = glob.glob(confFileSearchString) if len(confFiles) == 1: confFileDict.read(confFiles[0]) try: self.ResultData['KeyValueDictPairs']['TestCenter']['Value'] = iniFileDict.get('OperationDetails', 'TestCenter') except: pass try: self.ResultData['KeyValueDictPairs']['Hostname']['Value'] = iniFileDict.get('OperationDetails', 'Hostname') except: pass except: print "no .conf/.ini file found in 'configfiles' directory!" pass
def ReadModuleVersion(self): if self.verbose: print 'Read configParameters' self.check_Test_Software() config_format = self.HistoDict.get('ConfigParameters', 'configFormat') fileNames = self.HistoDict.get('ConfigParameters', 'configParameters').split(',') version = 'none' nRocs = 0 halfModule = 0 if config_format == 'dat': lines = [] for filename in fileNames: fileName = '%s/%s' % (self.RawTestSessionDataPath, filename) try: f = open(fileName) lines.extend(f.readlines()) f.close() except: warnings.warn('cannot open config parameters dat file {file}'.format(file=fileName)) version = 'none' for line in lines: if line.strip().startswith('rocType'): version = line.split(' ')[-1] elif line.strip().startswith('nRocs'): nRocs = int(line.split(' ')[-1]) if self.verbose: print '\tnRocs: %s' % nRocs elif line.strip().startswith('halfModule'): halfModule = int(line.split(' ')[-1]) if self.verbose: print '\thalfModule: %s' % halfModule elif config_format == 'cfg': config = BetterConfigParser() for filename in fileNames: fileName = '%s/%s' % (self.RawTestSessionDataPath, filename) config.read(fileName) try: version = config.get('ROC', 'type') except (KeyError,ConfigParser.NoOptionError,ConfigParser.NoSectionError): warnings.warn('cannot find version name {section}'.format(section=config.sections())) if 'ROC' in config.sections(): warnings.warn('cannot find type in ROC-section: {options}'.format(options=config.options('ROC'))) version = 'none' try: nRocs = config.getint('Module', 'rocs') except (KeyError,ConfigParser.NoOptionError,ConfigParser.NoSectionError): nRocs = 0 try: halfModule = config.get('Module', 'halfModule') except (KeyError,ConfigParser.NoOptionError,ConfigParser.NoSectionError): halfModule = 0 version = version.rstrip('\n') if self.verbose: print 'Version: ', version print 'nRocs: ', nRocs print 'halfModule: ', halfModule self.version = version self.nRocs = nRocs self.halfModule = halfModule return version, nRocs, halfModule
def check_Test_Software(self): if self.check_Test_Software_Pyxar(): self.testSoftware = 'pyxar' elif self.check_Test_Software_Pxar(): self.testSoftware = 'pxar' else: self.testSoftware = 'psi46expert' self.HistoDict = BetterConfigParser() fileName = 'Configuration/Software/%s.cfg' % self.testSoftware self.HistoDict.read(fileName) fileName = 'Configuration/Software/global.cfg' self.HistoDict.read(fileName)
def check_Test_Software(self): if self.check_Test_Software_Pyxar(): self.testSoftware = 'pyxar' elif self.check_Test_Software_Pxar(): self.testSoftware = 'pxar' else: data = glob.glob(self.RawTestSessionDataPath + '/*[p,P][x,X][a,A][r,R]*.*') + \ glob.glob(self.RawTestSessionDataPath + '/*/*[p,P][x,X][a,A][r,R]*.*') if len(data): self.testSoftware = 'pxar' else: #self.testSoftware = 'psi46expert' self.testSoftware = 'pxar' self.HistoDict = BetterConfigParser() fileName = 'Configuration/Software/%s.cfg' % self.testSoftware self.HistoDict.read(fileName) fileName = 'Configuration/Software/global.cfg' self.HistoDict.read(fileName)
def check_Test_Software(self): # file = self.RawTestSessionDataPath + '/test.cfg' # print file self.RawTestSessionDataPath = os.path.abspath(self.RawTestSessionDataPath) files = glob.glob(self.RawTestSessionDataPath + '/test.cfg') + \ glob.glob(self.RawTestSessionDataPath + '/*/test.cfg') # print 'pyxar:',files if len(files) > 0: self.testSoftware = 'pyxar' else: data = glob.glob(self.RawTestSessionDataPath + '/*[p,P][x,X][a,A][r,R]*.*') + \ glob.glob(self.RawTestSessionDataPath + '/*/*[p,P][x,X][a,A][r,R]*.*') if len(data): self.testSoftware = 'pxar' else: self.testSoftware = 'psi46expert' self.HistoDict = BetterConfigParser() fileName = 'Configuration/Software/%s.cfg' % self.testSoftware self.HistoDict.read(fileName) fileName = 'Configuration/Software/global.cfg' self.HistoDict.read(fileName)
def PopulateResultData(self): SysConfiguration = ConfigParser.ConfigParser() SysConfiguration.read(['Configuration/SystemConfiguration.cfg']) ParallelProcessing = False LimitProcesses = None try: if int(SysConfiguration.get('SystemConfiguration', 'ParallelProcessing').strip())>0: ParallelProcessing = True except: print "no 'ParallelProcessing' option found, running sequentially..." pass try: if int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip())>0: LimitProcesses = int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip()) except: pass nRocs = self.ParentObject.Attributes['NumberOfChips'] directory = self.ParentObject.Attributes['SCurvePaths']['HRSCurves_{Rate}'.format(Rate=self.Attributes['Rate'])] refit = self.TestResultEnvironmentObject.Configuration['Fitting']['refit'] print 'SCurve fitting...' HistoDict = BetterConfigParser() HistoDict.add_section('SCurveFitting') HistoDict.set('SCurveFitting','nTrigs', str(10)) HistoDict.set('SCurveFitting','dir', '') HistoDict.set('SCurveFitting','ignoreValidityCheck', '1') HistoDict.set('SCurveFitting','inputFileName', self.ParentObject.ParentObject.HistoDict.get('HighRate', 'SCurveDataFileName')) ePerVcal = self.TestResultEnvironmentObject.GradingParameters['StandardVcal2ElectronConversionFactor'] # compare current fit revision with saved revision FitRevDict = BetterConfigParser() fitRevFileName = directory + "/fits.rev" if os.path.isfile(fitRevFileName): try: FitRevDict.read(fitRevFileName) # if fit procedure changed -> refit if self.FitRevisionScurves > int(FitRevDict.get('FitRevisions', 'SCurves')): print "info: fit procedure has changed, re-fitting..." refit = True # if last fitting was not done completely (eg. STRG+C while fit...) -> refit if FitRevDict.get('Fit', 'Complete').strip().lower() != 'true': print "info: last fit was incomplete! re-fitting..." refit = True except: print "warning: could not read fit revision file!" else: FitRevDict.add_section('FitRevisions') FitRevDict.add_section('Fit') # unset complete flag in file FitRevDict.set('Fit','Complete', 'false') with open(fitRevFileName, 'wb') as fitRevFile: FitRevDict.write(fitRevFile) # do fitting fitter = SCurve_Fitting(refit=refit, HistoDict=HistoDict, ePerVcal=ePerVcal, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses) fitter.FitAllSCurve(directory, nRocs) # update saved revision FitRevDict.set('FitRevisions','SCurves', str(self.FitRevisionScurves)) # set complete flag in file FitRevDict.set('Fit','Complete', 'true') with open(fitRevFileName, 'wb') as fitRevFile: FitRevDict.write(fitRevFile)
def PopulateResultData(self): SysConfiguration = ConfigParser.ConfigParser() SysConfiguration.read(['Configuration/SystemConfiguration.cfg']) ParallelProcessing = False LimitProcesses = None try: if int(SysConfiguration.get('SystemConfiguration', 'ParallelProcessing').strip())>0: ParallelProcessing = True except: print "no 'ParallelProcessing' option found, running sequentially..." pass try: if int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip())>0: LimitProcesses = int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip()) except: pass print 'do fitting...' nRocs = self.Attributes['NumberOfChips'] directory = self.RawTestSessionDataPath refit = self.TestResultEnvironmentObject.Configuration['Fitting']['refit'] refitScurves = refit refitPHlin = refit refitPHtan = refit # compare current fit revision with saved revision FitRevDict = BetterConfigParser() fitRevFileName = directory + "/fits.rev" if os.path.isfile(fitRevFileName): try: FitRevDict.read(fitRevFileName) # if fit procedure changed -> refit if self.FitRevisionScurves > int(FitRevDict.get('FitRevisions', 'SCurves')): print "info: fit procedure has changed, re-fitting scurves..." refitScurves = True if self.FitRevisionPHlin > int(FitRevDict.get('FitRevisions', 'PHlin')): print "info: fit procedure has changed, re-fitting ph lin..." refitPHlin = True if self.FitRevisionPHtan > int(FitRevDict.get('FitRevisions', 'PHtan')): print "info: fit procedure has changed, re-fitting ph tan..." refitPHtan = True # if last fitting was not done completely (eg. STRG+C while fit...) -> refit if FitRevDict.get('Fit', 'Complete').strip().lower() != 'true': print "info: last fit was incomplete! re-fitting all..." refitScurves = True refitPHlin = True refitPHtan = True except: print "warning: could not read fit revision file!" else: FitRevDict.add_section('FitRevisions') FitRevDict.add_section('Fit') # unset complete flag in file FitRevDict.set('Fit','Complete', 'false') with open(fitRevFileName, 'wb') as fitRevFile: FitRevDict.write(fitRevFile) print 'SCurve fitting...' ePerVcal = self.TestResultEnvironmentObject.GradingParameters['StandardVcal2ElectronConversionFactor'] fitter = SCurve_Fitting(refit=refitScurves, HistoDict=self.ParentObject.HistoDict,ePerVcal=ePerVcal, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses) fitter.FitAllSCurve(directory,nRocs) print 'linear PH fitting...' fitter = PH_Fitting(fitMode=0, refit=refitPHlin, HistoDict=self.ParentObject.HistoDict, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses) fitter.FitAllPHCurves(directory,nRocs) print 'tanh PH fitting...' fitter = PH_Fitting(fitMode=3, refit=refitPHtan, HistoDict=self.ParentObject.HistoDict, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses) fitter.FitAllPHCurves(directory,nRocs) # update saved revisions FitRevDict.set('FitRevisions','SCurves', str(self.FitRevisionScurves)) FitRevDict.set('FitRevisions','PHlin', str(self.FitRevisionPHlin)) FitRevDict.set('FitRevisions','PHtan', str(self.FitRevisionPHtan)) # unset complete flag in file FitRevDict.set('Fit','Complete', 'true') with open(fitRevFileName, 'wb') as fitRevFile: FitRevDict.write(fitRevFile) print 'done'
class GeneralTestResult(object): nRows = 80 nCols = 52 nTotalChips = 16 MoReWebVersion = 'unknown MoReWeb version' ''' Initialization function @param ParentObject Reference to the Parent Object @param InitialModulePath Starting point of modules ''' def __init__(self, TestResultEnvironmentObject, ParentObject=None, InitialModulePath=None, InitialFinalResultsStoragePath=None, InitialAttributes=None, Key=None, DisplayOptions=None): ROOTConfiguration.initialise_ROOT() self.verbose = False # Name of the Test Result, only a-zA-Z0-9_ self.Name = '' # Name of the test result without prefixes self.NameSingle = '' self.Key = '' # Title displayed in HTML, etc self.Title = '' # options which might be transfered to Attributes? self.testSoftware = None self.HistoDict = None self.version = None self.nRocs = 0 self.halfModule = 0 self.CommentFromFile = None self.AddCommentsToKeyValueDictPairs = False if Key: self.Key = Key # Attributes like Date, etc. self.Attributes = { # Date as UNIX timestamp, if 0, the date of the parent object is taken 'TestDate': 0, # Type of the tested object 'TestedObjectType': '', # ID of the tested object 'TestedObjectID': '', # Key for custom storage path 'StorageKey': '', # subDirectory of the test result root-files 'TestResultSubDirectory': '', } self.Show = True self.Enabled = True self.SavePlotFile = True self.GzipSVG = TestResultEnvironmentObject.Configuration['GzipSVG'] self.DefaultImageFormat = TestResultEnvironmentObject.Configuration['DefaultImageFormat'].strip().lower() if TestResultEnvironmentObject.Configuration.has_key('AdditionalImageFormats'): self.AdditionalImageFormats = TestResultEnvironmentObject.Configuration['AdditionalImageFormats'].strip().lower().split(',') else: self.AdditionalImageFormats = ['root', 'pdf'] if TestResultEnvironmentObject.Configuration.has_key('OverviewHTMLLink'): self.OverviewHTMLLink = TestResultEnvironmentObject.Configuration['OverviewHTMLLink'] else: self.OverviewHTMLLink = None # Path for current test to folder with root-files self.RawTestSessionDataPath = '' # Path for current test result files (html, svg, etc) self.FinalResultsStoragePath = '' # File handle (might be used by sub results) self.FileHandle = 0 # Reference to ROOT canvas self.Canvas = None # Display Properties as in the ResultData.SubTestResultDictList self.DisplayOptions = { 'Order': 0, 'Width': 1, 'GroupWithNext': False, } # Result array self.ResultData = { # Key / ValueDict (dict with {Value, Unit, Label}, if Label not specified, the key is used as label) Pairs # 'KeyValueDictPairs':{ # 'MyKey':{ # 'Value':'25', # 'NumericValue':25, # 'Unit': 'kg', # 'Label': 'My Key' # } # } 'KeyValueDictPairs': {}, # List of keys for sorting 'KeyList': [], # Plot data 'Plot': { 'Enabled': 0, 'ROOTObject': None, 'Caption': '', 'ImageFile': '', 'Format': self.DefaultImageFormat, 'AdditionalFormats':self.AdditionalImageFormats, 'ImageFilePDF':'', }, # SubTest Results 'SubTestResults': {}, # List of {Key, Module} dict for sorting and special attributes 'SubTestResultDictList': [], # 'SubTestResultDictList':{ # { # 'Key':'Noise' # 'Module':'Noise', # 'TestResultObject':object, # 'InitialAttributes':{'StorageKey':'Blub'}, # 'DisplayOptions':{'Order':1, 'Width':1,'GroupWithNext':False}, # } # } # hidden data, might be needed by other test results but is not displayed 'HiddenData': {}, # a data table if needed 'Table': { 'HEADER': [], 'BODY': [], 'FOOTER': [], } } # Reference to parent object self.ParentObject = None # Reference to the Test Result Environment self.TestResultEnvironmentObject = None self.TestResultEnvironmentObject = TestResultEnvironmentObject self.Canvas = self.TestResultEnvironmentObject.Canvas self.RawTestSessionDataPath = self.TestResultEnvironmentObject.ModuleDataDirectory if ParentObject: self.ParentObject = ParentObject self.RawTestSessionDataPath = self.ParentObject.RawTestSessionDataPath if InitialAttributes: self.Attributes.update(InitialAttributes) if DisplayOptions: self.DisplayOptions.update(DisplayOptions) if self.Attributes['TestResultSubDirectory']: self.RawTestSessionDataPath += '/' + self.Attributes['TestResultSubDirectory'] self.CustomInit() if not self.Title: self.Title = self.NameSingle # Module Path self.ModulePath = self.NameSingle if InitialModulePath: self.ModulePath = InitialModulePath if InitialFinalResultsStoragePath: self.FinalResultsStoragePath = InitialFinalResultsStoragePath self.RelativeFinalResultsStoragePath = '' if self.ParentObject: self.FinalResultsStoragePath = self.ParentObject.FinalResultsStoragePath self.RelativeFinalResultsStoragePath = self.ParentObject.RelativeFinalResultsStoragePath self.ModulePath = self.ParentObject.ModulePath + '.' + self.ModulePath if self.Attributes['StorageKey']: self.FinalResultsStoragePath += '/' + self.Attributes['StorageKey'] self.RelativeFinalResultsStoragePath += '/' + self.Attributes['StorageKey'] else: self.FinalResultsStoragePath += '/' + self.NameSingle self.RelativeFinalResultsStoragePath += '/' + self.NameSingle self.SetFinalResultsStoragePath() if not os.path.exists(self.FinalResultsStoragePath): os.makedirs(self.FinalResultsStoragePath) # Load all sub test results in correct order i2 = len(self.ResultData['SubTestResultDictList']) for i in self.ResultData['SubTestResultDictList']: if i.has_key('Module'): SubModule = i['Module'] else: SubModule = i['Key'] i['Module'] = SubModule if i.has_key('InitialAttributes'): pass else: i['InitialAttributes'] = {} # set the test date for all test results i['InitialAttributes'].update({ 'TestDate': self.Attributes['TestDate'] }) # Ensure all keys of display properties are set DisplayOptions = { 'Order': i2, 'Width': 1, 'Show': True, 'GroupWithNext': False, } if i.has_key('DisplayOptions'): DisplayOptions.update(i['DisplayOptions']) i['DisplayOptions'] = DisplayOptions importdir = self.ModulePath + '.' + SubModule try: # print 'import ',importdir,SubModule f = __import__(importdir + '.' + SubModule, fromlist=[importdir + '.' + 'TestResult']) except ImportError as inst: # print 'could not ',importdir+'.'+SubModule,SubModule # print 'type',type(inst) # print 'inst',inst f = __import__(importdir + '.TestResult', fromlist=['']) print 'imported', f, 'please change name of file' pass self.ResultData['SubTestResults'][i['Key']] = f.TestResult( self.TestResultEnvironmentObject, self, None, None, i['InitialAttributes'], i['Key'], i['DisplayOptions'] ) i['TestResultObject'] = self.ResultData['SubTestResults'][i['Key']] i2 += 1 def check_Test_Software_Pyxar(self): self.RawTestSessionDataPath = os.path.abspath(self.RawTestSessionDataPath) files = glob.glob(self.RawTestSessionDataPath + '/test.cfg') + \ glob.glob(self.RawTestSessionDataPath + '/*/test.cfg') return len(files) > 0 def check_Test_Software_Pxar(self): data = glob.glob(self.RawTestSessionDataPath + '/*[p,P][x,X][a,A][r,R]*.*') + \ glob.glob(self.RawTestSessionDataPath + '/*/*[p,P][x,X][a,A][r,R]*.*') if len(data): return True LogFileNames = glob.glob(self.RawTestSessionDataPath + '/*.log') + glob.glob(self.RawTestSessionDataPath + '/*/*.log') for LogFileName in LogFileNames: LogFile = open(LogFileName, "r") FirstLine = LogFile.readline() LogFile.close() if FirstLine.lower().find("welcome to pxar") > -1: return True return False def check_Test_Software(self): if self.check_Test_Software_Pyxar(): self.testSoftware = 'pyxar' elif self.check_Test_Software_Pxar(): self.testSoftware = 'pxar' else: data = glob.glob(self.RawTestSessionDataPath + '/*[p,P][x,X][a,A][r,R]*.*') + \ glob.glob(self.RawTestSessionDataPath + '/*/*[p,P][x,X][a,A][r,R]*.*') if len(data): self.testSoftware = 'pxar' else: #self.testSoftware = 'psi46expert' self.testSoftware = 'pxar' self.HistoDict = BetterConfigParser() fileName = 'Configuration/Software/%s.cfg' % self.testSoftware self.HistoDict.read(fileName) fileName = 'Configuration/Software/global.cfg' self.HistoDict.read(fileName) def check_for_comments(self): dir_name = os.path.abspath(self.RawTestSessionDataPath) comment_files = glob.glob(dir_name + '/comment*') comment = '' comment_short = '' for filename in comment_files: comment += '{filename}:\n'.format(filename=filename.split('/')[-1]) with file(filename) as f: s = f.read() comment += s + '\n\n' comment_short += s + ' ' # if self.ResultData: # if not 'KeyVaueDictPairs' in self.ResultData: # self.ResultData['KeyValueDictPairs'] = {} # # if not 'KeyValueList' in self.ResultData: # self.ResultData['KeyValueList'] = [] if self.verbose: print 'checking', self.RawTestSessionDataPath, comment_files if comment != '': if self.verbose: print 'added comment', comment, 'to ', self.Name if self.AddCommentsToKeyValueDictPairs: self.ResultData['KeyValueDictPairs']['Comment'] = { 'Value': comment, 'Style': 'font-weight:bold;color:red;' if '!' in comment else 'font-weight:bold;', } self.ResultData['KeyList'].append('Comment') self.CommentFromFile = comment_short def check_for_manualGrade(self): self.RawTestSessionDataPath = os.path.abspath(self.RawTestSessionDataPath) gradefilename = self.RawTestSessionDataPath + '/grade.txt' print "checking for manual grading in '%s'"%gradefilename grade = '' if os.path.isfile(gradefilename): try: gradefile = open(gradefilename) except: warnings.warn('cannot open manual grade file {file}'.format(file=gradefilename)) grade = gradefile.read().strip() gradefile.close() # grade can be given either as number 1,2,3 or as letter A,B,C GradeNames = ['A','B','C'] if grade in GradeNames: grade = 1 + GradeNames.index(grade) print "Reading a manual grade "+str(grade)+" specified by the user in "+str(gradefilename) else: print " => not found." return grade def ReadModuleVersion(self): if self.verbose: print 'Read configParameters' self.check_Test_Software() config_format = self.HistoDict.get('ConfigParameters', 'configFormat') fileNames = self.HistoDict.get('ConfigParameters', 'configParameters').split(',') version = 'none' nRocs = 0 halfModule = 0 if config_format == 'dat': lines = [] for filename in fileNames: fileName = '%s/%s' % (self.RawTestSessionDataPath, filename) try: f = open(fileName) lines.extend(f.readlines()) f.close() except: warnings.warn('cannot open config parameters dat file {file}'.format(file=fileName)) version = 'none' for line in lines: if line.strip().startswith('rocType'): version = line.split(' ')[-1] elif line.strip().startswith('nRocs'): nRocs = int(line.split(' ')[-1]) if self.verbose: print '\tnRocs: %s' % nRocs elif line.strip().startswith('halfModule'): halfModule = int(line.split(' ')[-1]) if self.verbose: print '\thalfModule: %s' % halfModule elif config_format == 'cfg': config = BetterConfigParser() for filename in fileNames: fileName = '%s/%s' % (self.RawTestSessionDataPath, filename) config.read(fileName) try: version = config.get('ROC', 'type') except (KeyError,ConfigParser.NoOptionError,ConfigParser.NoSectionError): warnings.warn('cannot find version name {section}'.format(section=config.sections())) if 'ROC' in config.sections(): warnings.warn('cannot find type in ROC-section: {options}'.format(options=config.options('ROC'))) version = 'none' try: nRocs = config.getint('Module', 'rocs') except (KeyError,ConfigParser.NoOptionError,ConfigParser.NoSectionError): nRocs = 0 try: halfModule = config.get('Module', 'halfModule') except (KeyError,ConfigParser.NoOptionError,ConfigParser.NoSectionError): halfModule = 0 version = version.rstrip('\n') if self.verbose: print 'Version: ', version print 'nRocs: ', nRocs print 'halfModule: ', halfModule self.version = version self.nRocs = nRocs self.halfModule = halfModule return version, nRocs, halfModule ''' Populates all necessary data @final ''' def PopulateAllData(self): self.OpenFileHandle() for i in self.ResultData['SubTestResultDictList']: if i['TestResultObject'].Enabled: self.SetCanvasSize() try: i['TestResultObject'].PopulateAllData() # i['TestResultObject'].check_for_comments() except Exception as inst: exc_type, exc_obj, exc_tb = sys.exc_info() # Start red color sys.stdout.write("\x1b[31m") sys.stdout.flush() # Print error message print '\x1b[31mException while processing', i['TestResultObject'].FinalResultsStoragePath # Print traceback traceback.print_exception(exc_type, exc_obj, exc_tb) # Stop red color sys.stdout.write("\x1b[0m") sys.stdout.flush() self.TestResultEnvironmentObject.ErrorList.append( {'ModulePath': i['TestResultObject'].ModulePath, 'ErrorCode': inst, 'FinalResultsStoragePath': i['TestResultObject'].FinalResultsStoragePath} ) # todo Felix: handel exceptions self.SetCanvasSize() try: self.PopulateResultData() #self.SaveCanvas() self.check_for_comments() except Exception as inst: exc_type, exc_obj, exc_tb = sys.exc_info() # Start red color sys.stdout.write("\x1b[31m") sys.stdout.flush() print '\x1b[31mException while processing', self.FinalResultsStoragePath # Print traceback traceback.print_exception(exc_type, exc_obj, exc_tb) # Stop red color sys.stdout.write("\x1b[0m") sys.stdout.flush() self.TestResultEnvironmentObject.ErrorList.append( {'ModulePath': self.ModulePath, 'ErrorCode': inst, 'FinalResultsStoragePath': self.FinalResultsStoragePath} # 'FinalResultsStoragePath':i['TestResultObject'].FinalResultsStoragePath} ) def AnalyzeLogfiles(self, logfiles, DetectMessages, ErrorNames): KeyValueDictPairs = {} KeyList = [] ErrorObjects = [] if not type(logfiles) is list: logfilesList = [logfiles] else: logfilesList = logfiles for logfile in logfilesList: if logfile: with open(logfile, 'r') as logfile: for line in logfile: ErrorObject = {} if 'WARNING:' in line and not 'Not unmasking DUT' in line: ErrorObject['type'] = 'warning' if 'ERROR:' in line: ErrorObject['type'] = 'error' if 'CRITICAL:' in line: ErrorObject['type'] = 'critical' ErrorObject['channel'] = None lineParts = line.strip().split() if 'Channel' in lineParts: pos = lineParts.index('Channel') if pos < len(lineParts) - 1: try: ErrorObject['channel'] = int(lineParts[pos+1]) except: pass if ErrorObject.has_key('type'): Found = False for errorname, keywords in DetectMessages.iteritems(): for keyword in keywords: if keyword in line: ErrorObject['subtype'] = errorname Found = True break if not Found: ErrorObject['subtype'] = 'other' ErrorObjects.append(ErrorObject) KeyValueDictPairs['nCriticals'] = {'Label': '# Criticals', 'Value': '%d'%len([True for ErrorObject in ErrorObjects if ErrorObject['type'] == 'critical'])} KeyValueDictPairs['nErrors'] = {'Label': '# Errors', 'Value': '%d'%len([True for ErrorObject in ErrorObjects if ErrorObject['type'] == 'error'])} KeyValueDictPairs['nWarnings'] = {'Label': '# Warnings', 'Value': '%d'%len([True for ErrorObject in ErrorObjects if ErrorObject['type'] == 'warning'])} for ch in range(4): KeyValueDictPairs['channel_%d_count'%ch] = {'Label': 'Channel %d'%ch, 'Value': '%d'%len([True for ErrorObject in ErrorObjects if ErrorObject['channel'] == ch])} for errorSubtype, keywords in DetectMessages.iteritems(): KeyValueDictPairs['message_%s_count'%errorSubtype] = {'Label': ErrorNames[errorSubtype] if errorSubtype in ErrorNames else errorSubtype, 'Value': '%d'%len([True for ErrorObject in ErrorObjects if ErrorObject['subtype'] == errorSubtype])} KeyList = (['nCriticals', 'nErrors', 'nWarnings', 'channel_0_count', 'channel_1_count', 'channel_2_count', 'channel_3_count', 'message_tokenchain_count', 'message_eventid_count', 'message_readback_count', 'message_notokenpass_count', 'message_datasize_count', 'message_missingevents_count', 'message_usbtimeout_count', 'message_deser400_count', 'message_daqerror_count']) return {'KeyValueDictPairs': KeyValueDictPairs, 'KeyList': KeyList} ''' Manually close all file handles of the sub tests @final ''' def CloseSubTestResultFileHandles(self, Level=0): for i in self.ResultData['SubTestResultDictList']: if i['TestResultObject'].Enabled: i['TestResultObject'].CloseSubTestResultFileHandles(Level + 1) if Level: if self.FileHandle: if not type(self.FileHandle) == list: self.FileHandle = [self.FileHandle] for SingleFileHandle in self.FileHandle: try: if "ROOT.TFile" in repr(SingleFileHandle): if self.verbose: print '\x1b[35mclose ROOT file "%s" ...\x1b[0m'%repr(SingleFileHandle) SingleFileHandle.Close() else: if self.verbose: print '\x1b[32mclose file "%s" ...\x1b[0m'%repr(SingleFileHandle) SingleFileHandle.close() except: print '\x1b[33m warning: can not close file "%s" \x1b[0m'%repr(SingleFileHandle) ''' Reads all attributes and writes it to the memory ''' def CustomInit(self): pass ''' Opens a file handle just before populating data ''' def OpenFileHandle(self): pass ''' Create a unique ID for creating root histograms ''' def GetUniqueID(self): return self.TestResultEnvironmentObject.GetUniqueID(self.NameSingle) ''' Sets the storage path ''' def SetFinalResultsStoragePath(self): pass def CanvasSize(self, canvas): canvas.SetCanvasSize( self.DisplayOptions['Width'] * self.TestResultEnvironmentObject.Configuration['DefaultValues'][ 'CanvasWidth'], self.TestResultEnvironmentObject.Configuration['DefaultValues']['CanvasHeight'] ) canvas.Draw() canvas.Update() return canvas ''' Sets the canvas size ''' def SetCanvasSize(self): self.Canvas = self.CanvasSize(self.Canvas) # self.Canvas.SetCanvasSize( # self.DisplayOptions['Width']*self.TestResultEnvironmentObject.Configuration['DefaultValues']['CanvasWidth'], # self.TestResultEnvironmentObject.Configuration['DefaultValues']['CanvasHeight'] # ) # self.Canvas.Draw() # self.Canvas.Update() ''' Saving the Canvas ''' def SaveCanvas(self): if self.SavePlotFile: if self.Canvas: self.Canvas.SaveAs(self.GetPlotFileName()) if not self.ResultData['Plot']['Caption']: self.ResultData['Plot']['Caption'] = self.Title for Suffix in self.ResultData['Plot']['AdditionalFormats']: if Suffix == 'pdf': self.ResultData['Plot']['ImageFilePDF'] = self.GetPlotFileName(Suffix) if self.ResultData['Plot']['ROOTObject']: self.ResultData['Plot']['ROOTObject'].SetTitle(self.ResultData['Plot']['Caption']) self.Canvas.SaveAs(self.GetPlotFileName(Suffix)) self.ResultData['Plot']['Enabled'] = 1 self.ResultData['Plot']['ImageFile'] = self.GetPlotFileName() ''' Generate the filename including the full path to the plot file according to the format ''' def GetPlotFileName(self,Suffix=''): if Suffix == '': Suffix = self.ResultData['Plot']['Format'] return self.FinalResultsStoragePath + '/' + self.NameSingle + '.' + Suffix ''' Get the sub test result list in the display order ''' def GetSortedSubTestResultDictList(self): # sort sub test results according to display order return sorted( self.ResultData['SubTestResultDictList'], key=lambda i: i['DisplayOptions']['Order'] ) ''' Reads all test results and writes it to the memory ''' def PopulateResultData(self): pass ''' Generates all output files ''' def GenerateFinalOutput(self): for i in self.ResultData['SubTestResults']: self.ResultData['SubTestResults'][i].GenerateFinalOutput() self.GenerateDataFiles() ''' Generate files like Key/Value pairs in JSON format, ASCII-files, HTML-files, etc. @final ''' def GenerateDataFiles(self): self.GenerateDataFileHTML() self.GenerateDataFileJSON() ''' Generate HTML file @final ''' def GenerateDataFileHTML(self): HtmlParser = self.TestResultEnvironmentObject.HtmlParser HTMLFileName = 'TestResult.html' FinalHTML = '' HTMLTemplate = self.TestResultEnvironmentObject.TestResultHTMLTemplate FinalHTML = HTMLTemplate Levels = self.ModulePath[:].split('.') # Stylesheet StylesheetHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###HEAD_STYLESHEET_TEMPLATE###') StylesheetHTML = HtmlParser.substituteMarkerArray( StylesheetHTMLTemplate, { '###STYLESHEET###': self.TestResultEnvironmentObject.MainStylesheet + self.TestResultEnvironmentObject.TestResultStylesheet, } ) FinalHTML = HtmlParser.substituteSubpart( FinalHTML, '###HEAD_STYLESHEETS###', StylesheetHTML ) FinalHTML = HtmlParser.substituteSubpart( FinalHTML, '###HEAD_STYLESHEET_TEMPLATE###', '' ) # Clickpath ClickPathEntries = [] ClickPathEntryTemplate = HtmlParser.getSubpart(HTMLTemplate, '###CLICKPATH_ENTRY###') LevelPath = '' i = 0 tmpTestResultObject = self for Level in Levels[2:]: LevelPath = '../' * i ClickPathEntries.append(HtmlParser.substituteMarkerArray( ClickPathEntryTemplate, { '###URL###': HtmlParser.MaskHTML(LevelPath + HTMLFileName), '###LABEL###': HtmlParser.MaskHTML(tmpTestResultObject.Title) } )) if self.ParentObject: tmpTestResultObject = tmpTestResultObject.ParentObject i += 1 if not self.OverviewHTMLLink: OverviewHTMLLink = os.path.relpath(self.TestResultEnvironmentObject.GlobalOverviewPath + '/Overview.html', self.FinalResultsStoragePath) else: OverviewHTMLLink = self.OverviewHTMLLink ClickPathEntries.append(HtmlParser.substituteMarkerArray( ClickPathEntryTemplate, { '###URL###': HtmlParser.MaskHTML(OverviewHTMLLink), '###LABEL###': 'Overview' } )) ClickPathEntries.reverse() CSSClasses = '' # Result Data FinalHTML = HtmlParser.substituteSubpartArray( FinalHTML, { '###CLICKPATH_ENTRY###': ''.join(ClickPathEntries), '###RESULTDATA###': self.GenerateResultDataHTML(self, 0, self.DisplayOptions), '###ADDITIONALCSSCLASSES###': CSSClasses, } ) # Do it again for the ghost FinalHTML = HtmlParser.substituteSubpartArray( FinalHTML, { '###CLICKPATH_ENTRY###': ''.join(ClickPathEntries), } ) f = open(self.FinalResultsStoragePath + '/' + HTMLFileName, 'w') f.write(FinalHTML) f.close() ''' Generate Result Data HTML for usage in HTML files @final ''' def GenerateResultDataHTML(self, TestResultObject, RecursionLevel, DisplayOptions): HtmlParser = self.TestResultEnvironmentObject.HtmlParser HTMLTemplate = self.TestResultEnvironmentObject.TestResultHTMLTemplate ResultDataHTML = HtmlParser.getSubpart(HTMLTemplate, '###RESULTDATA###') HTMLFileName = 'TestResult.html' CSSClasses = self.NameSingle ResultDataHTML = HtmlParser.substituteMarker(ResultDataHTML, '###RESULTDATAADDITIONALCSSCLASSES###', HtmlParser.MaskHTML(CSSClasses) ) RecursionRelativePath = '' if RecursionLevel > 0: PathParts = TestResultObject.FinalResultsStoragePath.split('/') # print PathParts RecursionRelativePath = PathParts[-1] + '/' # Title if not TestResultObject.Title: TestResultObject.Title = TestResultObject.NameSingle MyObjectTestDate = '' VersionInfo = '' if RecursionLevel == 0 and TestResultObject.Attributes['TestDate']: MyObjectTestDate = 'Test Date: ' + datetime.datetime.fromtimestamp( float(TestResultObject.Attributes['TestDate'])).strftime("%Y-%m-%d %H:%M") + '<br><span style="font-size:10.5pt;" title="' + self.TestResultEnvironmentObject.MoReWebVersion + '">Analysis date: ' + datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + '</span>' VersionInfo = self.TestResultEnvironmentObject.MoReWebVersion + " on branch " + self.TestResultEnvironmentObject.MoReWebBranch MainTestResultAdditionalClasses = '' if RecursionLevel == 0: MainTestResultAdditionalClasses += 'MainTestResult Group' if self.DisplayOptions['Width'] > 1: MainTestResultAdditionalClasses += ' Width' + str(self.DisplayOptions['Width']) ResultDataHTML = HtmlParser.substituteMarkerArray( ResultDataHTML, { '###TITLE###': HtmlParser.MaskHTML(TestResultObject.Title), '###TESTDATE###': MyObjectTestDate, '###MAINTESTRESULTADDITIONALCLASSES###': MainTestResultAdditionalClasses, '###VERSIONINFO###': VersionInfo, } ) # Plot PlotHTML = '' if TestResultObject.ResultData['Plot']['Enabled']: PlotHTML = HtmlParser.getSubpart(HTMLTemplate, '###PLOT###') PlotHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###PLOT###') PlotImageHTML = HtmlParser.getSubpart(PlotHTMLTemplate, '###PLOT_IMAGE###') '''PlotImageHTML = { 'SVG':'', 'PNG':'' } if TestResultObject.ResultData['Plot']['Format'] == 'svg': PlotImageHTML['SVG'] = HtmlParser.getSubpart(PlotHTMLTemplate, '###PLOT_IMAGE_SVG###') elif TestResultObject.ResultData['Plot']['Format'] == 'png': PlotImageHTML['PNG'] = HtmlParser.getSubpart(PlotHTMLTemplate, '###PLOT_IMAGE_PNG###') for i in PlotImageHTML: ''' # file operations for svg if TestResultObject.ResultData['Plot']['Format'] == 'svg' and RecursionLevel == 0 and self.SavePlotFile: f = open(TestResultObject.ResultData['Plot']['ImageFile'], 'r') SVGContent = f.read() f.close() # fix an error in chrome / safari when displaying resized svg if SVGContent.find('<svg preserveAspectRatio') == -1: SVGContent = SVGContent.replace('<svg', '<svg preserveAspectRatio="xMinYMin"', 1) # remove an invalid space in attribute width and height SVGContent = SVGContent.replace('width=" ', 'width="') SVGContent = SVGContent.replace('height=" ', 'height="') SVGContent = SVGContent.replace('x=" ', 'x="') SVGContent = SVGContent.replace('y=" ', 'y="') SVGContent = SVGContent.replace('r=" ', 'r="') SVGContent = SVGContent.replace('=" ', '="') SVGContent = SVGContent.replace('̒', '°') if self.GzipSVG and TestResultObject.ResultData['Plot']['ImageFile'].find('.svgz') == -1: os.remove(TestResultObject.ResultData['Plot']['ImageFile']) TestResultObject.ResultData['Plot']['ImageFile'] += 'z' f = gzip.GzipFile(TestResultObject.ResultData['Plot']['ImageFile'], 'w') else: f = open(TestResultObject.ResultData['Plot']['ImageFile'], 'w') if os.path.exists(TestResultObject.ResultData['Plot']['ImageFile'] + 'z'): os.remove(TestResultObject.ResultData['Plot']['ImageFile'] + 'z') f.write(SVGContent) f.close() if not TestResultObject.ResultData['Plot']['Caption']: TestResultObject.ResultData['Plot']['Caption'] = TestResultObject.Title try: PlotImageHTML = HtmlParser.substituteMarkerArray( PlotImageHTML, { '###FILENAME###': HtmlParser.MaskHTML( RecursionRelativePath + os.path.basename(TestResultObject.ResultData['Plot']['ImageFile'])), '###PDFFILENAME###': HtmlParser.MaskHTML( RecursionRelativePath + os.path.basename(TestResultObject.ResultData['Plot']['ImageFilePDF'])), '###IMAGELARGECONTAINERID###': HtmlParser.MaskHTML( TestResultObject.Name + '_' + TestResultObject.Key), '###MARGIN_TOP###': str(int(-800. / float( DisplayOptions['Width'] * self.TestResultEnvironmentObject.Configuration['DefaultValues'][ 'CanvasWidth']) * float( self.TestResultEnvironmentObject.Configuration['DefaultValues'][ 'CanvasHeight']) / 2.)), '###TITLE###': TestResultObject.ResultData['Plot']['Caption'], '###WIDTH###': str(DisplayOptions['Width']), '###HEIGHT###': str(1), } ) except TypeError: print TestResultObject.Name, '_', TestResultObject.Key raise TypeError('Canntot convert, ' + str(TestResultObject.Name) + str(TestResultObject.Key)) # PlotHTML = HtmlParser.substituteSubpart(PlotHTML, '###PLOT_IMAGE_'+i+'###', PlotImageHTML) PlotHTML = HtmlParser.substituteSubpart(PlotHTML, '###PLOT_IMAGE###', PlotImageHTML) ResultDataHTML = HtmlParser.substituteSubpart(ResultDataHTML, '###PLOT###', PlotHTML) # Key Value Dict Pairs KeyValueDictPairsRowHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###KEYVALUEDICTPAIRS_ROW###') KeyValueDictPairsRows = '' for i in TestResultObject.ResultData['KeyList']: if not TestResultObject.ResultData['KeyValueDictPairs'].has_key(i): warnings.warn('Cannot find Key: {Key} in {Name}'.format(Key=i,Name=self.Name)) continue if not TestResultObject.ResultData['KeyValueDictPairs'][i].has_key('Unit'): TestResultObject.ResultData['KeyValueDictPairs'][i]['Unit'] = '' html_value = HtmlParser.MaskHTML(str(TestResultObject.ResultData['KeyValueDictPairs'][i]['Value'])) if TestResultObject.ResultData['KeyValueDictPairs'][i].has_key('Sigma'): TestResultObject.ResultData['KeyValueDictPairs'][i]['SigmaOutput'] = ' +/- %s' % \ TestResultObject.ResultData[ 'KeyValueDictPairs'][i][ 'Sigma'] html_value += ' ± ' + HtmlParser.MaskHTML( str(TestResultObject.ResultData['KeyValueDictPairs'][i]['Sigma'])) else: TestResultObject.ResultData['KeyValueDictPairs'][i]['SigmaOutput'] = '' if not TestResultObject.ResultData['KeyValueDictPairs'][i].has_key('Label'): TestResultObject.ResultData['KeyValueDictPairs'][i]['Label'] = i KeyValueDictPairsRows += HtmlParser.substituteMarkerArray( KeyValueDictPairsRowHTMLTemplate, { '###KEY###': HtmlParser.MaskHTML(i), '###LABEL###': HtmlParser.MaskHTML( TestResultObject.ResultData['KeyValueDictPairs'][i]['Label'] ), '###VALUE###': html_value, '###UNIT###': HtmlParser.MaskHTML( TestResultObject.ResultData['KeyValueDictPairs'][i]['Unit'] ), } ) ResultDataHTML = HtmlParser.substituteSubpart(ResultDataHTML, '###KEYVALUEDICTPAIRS_ROW###', KeyValueDictPairsRows) # Table TableHTML = '' if RecursionLevel == 0 or True: TableHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###TABLE###') TableHTML = HtmlParser.GenerateTableHTML(TableHTMLTemplate, TestResultObject.ResultData['Table'], { '###ADDITIONALCSSCLASS###': '', '###ID###': 'Table', }) ResultDataHTML = HtmlParser.substituteSubpart(ResultDataHTML, '###TABLE###', TableHTML) # Sub Test Results SubTestResultListHTML = '' if RecursionLevel == 0: GroupWithNext = False i2 = 0 for i in TestResultObject.GetSortedSubTestResultDictList(): if i['TestResultObject'].DisplayOptions['Show']: GroupCSSClass = '' if i2 % 5 == 0: GroupCSSClass += ' WidthNthChild5n' if i2 % 4 == 0: GroupCSSClass += ' WidthNthChild4n' if i2 % 3 == 0: GroupCSSClass += ' WidthNthChild3n' if i2 % 2 == 0: GroupCSSClass += ' WidthNthChild2n' if i['TestResultObject'].DisplayOptions['Width'] > 1: GroupCSSClass += ' Width' + str(i['TestResultObject'].DisplayOptions['Width']) if not GroupWithNext: SubTestResultListHTML += HtmlParser.substituteMarker( HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTGROUP_START###'), '###SUBTESTRESULTGROUP_ADDITIONALCSSCLASSES###', HtmlParser.MaskHTML(GroupCSSClass) ) # only increase the width counter for a group start i2 += i['TestResultObject'].DisplayOptions['Width'] SubTestResultListHTML += self.GenerateResultDataHTML(i['TestResultObject'], RecursionLevel + 1, i['TestResultObject'].DisplayOptions) if not i['TestResultObject'].DisplayOptions['GroupWithNext']: # if the last element was in a group but the current not, close the group SubTestResultListHTML += HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTGROUP_END###') GroupWithNext = i['DisplayOptions']['GroupWithNext'] # if the last element was in a group, close the group if GroupWithNext: SubTestResultListHTML += HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTGROUP_END###') else: SubTestResultListItemHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTLIST_ITEM###') SubTestResultListHTML = HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTLIST###') SubTestResultListItems = '' # Overview link SubTestResultOverviewLinkHTML = HtmlParser.getSubpart(SubTestResultListHTML, '###OVERVIEW_LINK###') if TestResultObject.ResultData['SubTestResultDictList']: SubTestResultOverviewLinkHTML = HtmlParser.substituteMarkerArray( SubTestResultOverviewLinkHTML, { '###URL###': HtmlParser.MaskHTML( os.path.basename(TestResultObject.FinalResultsStoragePath) + '/' + HTMLFileName ), } ) else: SubTestResultOverviewLinkHTML = '' # Subtests for i in TestResultObject.GetSortedSubTestResultDictList(): if i['DisplayOptions']['Show']: SubTestResultListItems += HtmlParser.substituteMarkerArray( SubTestResultListItemHTMLTemplate, { '###URL###': HtmlParser.MaskHTML( os.path.basename(TestResultObject.FinalResultsStoragePath) + '/' + os.path.basename(i['TestResultObject'].FinalResultsStoragePath) + '/' + HTMLFileName ), '###LABEL###': HtmlParser.MaskHTML( i['TestResultObject'].Title ), } ) SubTestResultListHTML = HtmlParser.substituteSubpartArray( SubTestResultListHTML, { '###SUBTESTRESULTLIST_ITEM###': SubTestResultListItems, '###OVERVIEW_LINK###': SubTestResultOverviewLinkHTML, } ) if not SubTestResultListItems: SubTestResultListHTML = '' if 'HTMLContent' in TestResultObject.ResultData: ResultDataHTML = ResultDataHTML + TestResultObject.ResultData['HTMLContent'] ResultDataHTML = HtmlParser.substituteSubpartArray( ResultDataHTML, { '###SUBTESTRESULTLIST###': SubTestResultListHTML, '###SUBTESTRESULTGROUP_START###': '', '###SUBTESTRESULTGROUP_END###': '', } ) return ResultDataHTML ''' Generate file from ResultData['KeyValueDictPairs'] Key/Value pairs in JSON format @final ''' def GenerateDataFileJSON(self): data = None key = None DataKeys = ['KeyValueDictPairs', 'HiddenData'] for DataKey in DataKeys: try: data = self.ResultData[DataKey] for key in data: if type(data[key])==dict and data[key].has_key('Value'): value = data[key]['Value'] if type(value) == set: value = list(value) data[key]['Value'] = value elif type(data[key])==dict: data[key] = list(data[key]) else: data_value = data[key] data[key] = {'Value': str(data_value)} f = open(self.FinalResultsStoragePath + '/'+DataKey+'.json', 'w') f.write(json.dumps(self.ResultData[DataKey], sort_keys=True, indent=4, separators=(',', ': '), cls=SetEncoder)) f.close() except (KeyError,IOError,TypeError): if data and key in data: if type(data[key])==dict: warnings.warn( 'Cannot create JSON for %s, %s' % (type(data[key]['Value']), str(self.ResultData[DataKey]))) else: warnings.warn('Cannot create JSON for %s.' % (repr(data))) else: warnings.warn('Cannot create JSON for %s.' % (repr(data))) ''' Generate file from ResultData['KeyValueDictPairs'] Key/Value pairs in ASCII format @final ''' def GenerateDataFileASCII(self): pass ''' Write all test results to the database ''' def WriteToDatabase(self, ParentID=0): ColumnMapping = {} ID = 0 ID = self.CustomWriteToDatabase(ParentID) WriteToDBSuccess = True for i in self.ResultData['SubTestResults']: try: SubtestWriteToDBSuccess = self.ResultData['SubTestResults'][i].WriteToDatabase(ID) WriteToDBSuccess = WriteToDBSuccess and SubtestWriteToDBSuccess except Exception as inst: # Start red color sys.stdout.write("\x1b[31m") sys.stdout.flush() print 'Error in subtest (write to database)', self.ResultData['SubTestResults'][i].ModulePath, \ self.ResultData['SubTestResults'][i].FinalResultsStoragePath print inst print inst.args print sys.exc_info()[0] print "\n\n------\n" # Print traceback exc_type, exc_obj, exc_tb = sys.exc_info() traceback.print_exception(exc_type, exc_obj, exc_tb) # Reset color sys.stdout.write("\x1b[0m") sys.stdout.flush() WriteToDBSuccess = False self.PostWriteToDatabase() return WriteToDBSuccess def CustomWriteToDatabase(self, ParentID): pass def PostWriteToDatabase(self): pass def __del__(self): self.CloseSubTestResultFileHandles()
class GeneralTestResult: nRows = 80 nCols = 52 nTotalChips = 16 ''' Initialization function @param ParentObject Reference to the Parent Object @param InitialModulePath Starting point of modules ''' def __init__(self, TestResultEnvironmentObject, ParentObject=None, InitialModulePath=None, InitialFinalResultsStoragePath=None, InitialAttributes=None, Key=None, DisplayOptions=None): ROOTConfiguration.initialise_ROOT() self.verbose = False # Name of the Test Result, only a-zA-Z0-9_ self.Name = '' # Name of the test result without prefixes self.NameSingle = '' self.Key = '' # Title displayed in HTML, etc self.Title = '' if Key: self.Key = Key # Attributes like Date, etc. self.Attributes = { # Date as UNIX timestamp, if 0, the date of the parent object is taken 'TestDate': 0, # Type of the tested object 'TestedObjectType': '', # ID of the tested object 'TestedObjectID': '', # Key for custom storage path 'StorageKey': '', # subDirectory of the test result root-files 'TestResultSubDirectory': '', } self.Show = True self.Enabled = True self.SavePlotFile = True self.GzipSVG = TestResultEnvironmentObject.Configuration['GzipSVG'] self.DefaultImageFormat = TestResultEnvironmentObject.Configuration['DefaultImageFormat'].strip().lower() if TestResultEnvironmentObject.Configuration.has_key('OverviewHTMLLink'): self.OverviewHTMLLink = TestResultEnvironmentObject.Configuration['OverviewHTMLLink'] else: self.OverviewHTMLLink = None # Path for current test to folder with root-files self.RawTestSessionDataPath = '' # Path for current test result files (html, svg, etc) self.FinalResultsStoragePath = '' # File handle (might be used by sub results) self.FileHandle = 0 # Reference to ROOT canvas self.Canvas = None # Display Properties as in the ResultData.SubTestResultDictList self.DisplayOptions = { 'Order': 0, 'Width': 1, 'GroupWithNext': False, } # Result array self.ResultData = { # Key / ValueDict (dict with {Value, Unit, Label}, if Label not specified, the key is used as label) Pairs # 'KeyValueDictPairs':{ # 'MyKey':{ # 'Value':25, # 'Unit': 'kg', # 'Label': 'My Key' # } # } 'KeyValueDictPairs': {}, # List of keys for sorting 'KeyList': [], # Plot data 'Plot': { 'Enabled': 0, 'ROOTObject': None, 'Caption': '', 'ImageFile': '', 'Format': self.DefaultImageFormat #svg }, # SubTest Results 'SubTestResults': {}, # List of {Key, Module} dict for sorting and special attributes 'SubTestResultDictList': [], # 'SubTestResultDictList':{ # { # 'Key':'Noise' # 'Module':'Noise', # 'TestResultObject':object, # 'InitialAttributes':{'StorageKey':'Blub'}, # 'DisplayOptions':{'Order':1, 'Width':1,'GroupWithNext':False}, # } # } # hidden data, might be needed by other test results but is not displayed 'HiddenData': {}, # a data table if needed 'Table': { 'HEADER': [], 'BODY': [], 'FOOTER': [], } } # Reference to parent object self.ParentObject = None # Reference to the Test Result Environment self.TestResultEnvironmentObject = None self.TestResultEnvironmentObject = TestResultEnvironmentObject self.Canvas = self.TestResultEnvironmentObject.Canvas self.RawTestSessionDataPath = self.TestResultEnvironmentObject.ModuleDataDirectory if ParentObject: self.ParentObject = ParentObject self.RawTestSessionDataPath = self.ParentObject.RawTestSessionDataPath if InitialAttributes: self.Attributes.update(InitialAttributes) if DisplayOptions: self.DisplayOptions.update(DisplayOptions) if self.Attributes['TestResultSubDirectory']: self.RawTestSessionDataPath += '/' + self.Attributes['TestResultSubDirectory'] self.CustomInit(); if not self.Title: self.Title = self.NameSingle # Module Path self.ModulePath = self.NameSingle if InitialModulePath: self.ModulePath = InitialModulePath if InitialFinalResultsStoragePath: self.FinalResultsStoragePath = InitialFinalResultsStoragePath self.RelativeFinalResultsStoragePath = '' if self.ParentObject: self.FinalResultsStoragePath = self.ParentObject.FinalResultsStoragePath self.RelativeFinalResultsStoragePath = self.ParentObject.RelativeFinalResultsStoragePath self.ModulePath = self.ParentObject.ModulePath + '.' + self.ModulePath if self.Attributes['StorageKey']: self.FinalResultsStoragePath += '/' + self.Attributes['StorageKey'] self.RelativeFinalResultsStoragePath += '/' + self.Attributes['StorageKey'] else: self.FinalResultsStoragePath += '/' + self.NameSingle self.RelativeFinalResultsStoragePath += '/' + self.NameSingle self.SetFinalResultsStoragePath() if not os.path.exists(self.FinalResultsStoragePath): os.makedirs(self.FinalResultsStoragePath) # Load all sub test results in correct order i2 = len(self.ResultData['SubTestResultDictList']) for i in self.ResultData['SubTestResultDictList']: if i.has_key('Module'): SubModule = i['Module'] else: SubModule = i['Key'] i['Module'] = SubModule if i.has_key('InitialAttributes'): pass else: i['InitialAttributes'] = {} # set the test date for all test results i['InitialAttributes'].update({ 'TestDate': self.Attributes['TestDate'] }) # Ensure all keys of display properties are set DisplayOptions = { 'Order': i2, 'Width': 1, 'Show': True, 'GroupWithNext': False, } if i.has_key('DisplayOptions'): DisplayOptions.update(i['DisplayOptions']) i['DisplayOptions'] = DisplayOptions try: importdir = self.ModulePath + '.' + SubModule # print 'import ',importdir,SubModule f = __import__(importdir + '.' + SubModule, fromlist=[importdir + '.' + 'TestResult']) except Exception as inst: # print 'could not ',importdir+'.'+SubModule,SubModule # print 'type',type(inst) # print 'inst',inst f = __import__(importdir + '.TestResult', fromlist=['']) print 'imported', f, 'please change name of file' pass self.ResultData['SubTestResults'][i['Key']] = f.TestResult( self.TestResultEnvironmentObject, self, None, None, i['InitialAttributes'], i['Key'], i['DisplayOptions'] ) i['TestResultObject'] = self.ResultData['SubTestResults'][i['Key']] i2 += 1 def check_Test_Software(self): # file = self.RawTestSessionDataPath + '/test.cfg' # print file self.RawTestSessionDataPath = os.path.abspath(self.RawTestSessionDataPath) files = glob.glob(self.RawTestSessionDataPath + '/test.cfg') + \ glob.glob(self.RawTestSessionDataPath + '/*/test.cfg') # print 'pyxar:',files if len(files) > 0: self.testSoftware = 'pyxar' else: data = glob.glob(self.RawTestSessionDataPath + '/*[p,P][x,X][a,A][r,R]*.*') + \ glob.glob(self.RawTestSessionDataPath + '/*/*[p,P][x,X][a,A][r,R]*.*') if len(data): self.testSoftware = 'pxar' else: self.testSoftware = 'psi46expert' self.HistoDict = BetterConfigParser() fileName = 'Configuration/Software/%s.cfg' % self.testSoftware self.HistoDict.read(fileName) fileName = 'Configuration/Software/global.cfg' self.HistoDict.read(fileName) def check_for_comments(self): dir = os.path.abspath(self.RawTestSessionDataPath) comment_files = glob.glob(dir + '/comment*') comment = '' for filename in comment_files: comment += '{filename}:\n'.format(filename=filename.split('/')[-1]) with file(filename) as f: s = f.read() comment += s + '\n\n' # if self.ResultData: # if not 'KeyVaueDictPairs' in self.ResultData: # self.ResultData['KeyValueDictPairs'] = {} # # if not 'KeyValueList' in self.ResultData: # self.ResultData['KeyValueList'] = [] if self.verbose: print 'checking', self.RawTestSessionDataPath, comment_files if comment != '': if self.verbose: print 'added comment', comment, 'to ', self.Name self.ResultData['KeyValueDictPairs']['Comment'] = { 'Value': comment, } self.ResultData['KeyList'].append('Comment') def ReadModuleVersion(self): if self.verbose: print 'Read configParameters' self.check_Test_Software() format = self.HistoDict.get('ConfigParameters', 'configFormat') fileNames = self.HistoDict.get('ConfigParameters', 'configParameters').split(',') if format == 'dat': lines = [] for filename in fileNames: fileName = '%s/%s' % (self.RawTestSessionDataPath, filename) f = open(fileName) lines.extend(f.readlines()) f.close() version = 'none' for line in lines: if line.strip().startswith('rocType'): version = line.split(' ')[-1] elif line.strip().startswith('nRocs'): nRocs = int(line.split(' ')[-1]) if self.verbose: print '\tnRocs: %s' % nRocs elif line.strip().startswith('halfModule'): halfModule = int(line.split(' ')[-1]) if self.verbose: print '\thalfModule: %s' % halfModule elif format == 'cfg': config = BetterConfigParser() for filename in fileNames: fileName = '%s/%s' % (self.RawTestSessionDataPath, filename) config.read(fileName) try: version = config.get('ROC', 'type') except: warnings.warn('cannot find version name {section}'.format(section=config.sections())) if 'ROC' in config.sections(): warnings.warn('cannot find version ROC-section {section}'.format(section=config.options('ROC'))) version = 'none' try: nRocs = config.getint('Module', 'rocs') except: nRocs = 0 try: halfModule = config.get('Module', 'halfModule') except: halfModule = 0 version = version.rstrip('\n') if self.verbose: print 'Version: ', version print 'nRocs: ', nRocs print 'halfModule: ', halfModule self.version = version self.nRocs = nRocs self.halfModule = halfModule return (version, nRocs, halfModule) ''' Populates all necessary data @final ''' def PopulateAllData(self): self.OpenFileHandle(); for i in self.ResultData['SubTestResultDictList']: if i['TestResultObject'].Enabled: self.SetCanvasSize() try: i['TestResultObject'].PopulateAllData() # i['TestResultObject'].check_for_comments() except Exception as inst: exc_type, exc_obj, exc_tb = sys.exc_info() # Start red color sys.stdout.write("\x1b[31m") sys.stdout.flush() # Print error message print '\x1b[31mException while processing', i['TestResultObject'].FinalResultsStoragePath # Print traceback traceback.print_exception(exc_type, exc_obj, exc_tb) # Stop red color sys.stdout.write("\x1b[0m") sys.stdout.flush() self.TestResultEnvironmentObject.ErrorList.append( {'ModulePath': i['TestResultObject'].ModulePath, 'ErrorCode': inst, 'FinalResultsStoragePath': i['TestResultObject'].FinalResultsStoragePath} ) # todo Felix: handel exceptions self.SetCanvasSize() try: self.PopulateResultData() self.check_for_comments() except Exception as inst: exc_type, exc_obj, exc_tb = sys.exc_info() # Start red color sys.stdout.write("\x1b[31m") sys.stdout.flush() print '\x1b[31mException while processing', self.FinalResultsStoragePath # Print traceback traceback.print_exception(exc_type, exc_obj, exc_tb) # Stop red color sys.stdout.write("\x1b[0m") sys.stdout.flush() self.TestResultEnvironmentObject.ErrorList.append( {'ModulePath': self.ModulePath, 'ErrorCode': inst, 'FinalResultsStoragePath': self.FinalResultsStoragePath} # 'FinalResultsStoragePath':i['TestResultObject'].FinalResultsStoragePath} ) ''' Manually close all file handles of the sub tests @final ''' def CloseSubTestResultFileHandles(self, Level=0): for i in self.ResultData['SubTestResultDictList']: if i['TestResultObject'].Enabled: i['TestResultObject'].CloseSubTestResultFileHandles(Level + 1) if Level: if self.FileHandle: try: self.FileHandle.close() except: try: self.FileHandle.Close() except: pass ''' Reads all attributes and writes it to the memory ''' def CustomInit(self): pass ''' Opens a file handle just before populating data ''' def OpenFileHandle(self): pass ''' Create a unique ID for creating root histograms ''' def GetUniqueID(self): return self.TestResultEnvironmentObject.GetUniqueID(self.NameSingle) ''' Sets the storage path ''' def SetFinalResultsStoragePath(self): pass def CanvasSize(self, canvas): canvas.SetCanvasSize( self.DisplayOptions['Width'] * self.TestResultEnvironmentObject.Configuration['DefaultValues'][ 'CanvasWidth'], self.TestResultEnvironmentObject.Configuration['DefaultValues']['CanvasHeight'] ) canvas.Draw() canvas.Update() return canvas ''' Sets the canvas size ''' def SetCanvasSize(self): self.Canvas = self.CanvasSize(self.Canvas) # self.Canvas.SetCanvasSize( # self.DisplayOptions['Width']*self.TestResultEnvironmentObject.Configuration['DefaultValues']['CanvasWidth'], # self.TestResultEnvironmentObject.Configuration['DefaultValues']['CanvasHeight'] # ) # self.Canvas.Draw() # self.Canvas.Update() ''' Generate the filename including the full path to the plot file according to the format ''' def GetPlotFileName(self): Suffix = self.ResultData['Plot']['Format'] return self.FinalResultsStoragePath + '/' + self.NameSingle + '.' + Suffix ''' Get the sub test result list in the display order ''' def GetSortedSubTestResultDictList(self): # sort sub test results according to display order return sorted( self.ResultData['SubTestResultDictList'], key=lambda i: i['DisplayOptions']['Order'] ) ''' Reads all test results and writes it to the memory ''' def PopulateResultData(self): pass ''' Generates all output files ''' def GenerateFinalOutput(self): for i in self.ResultData['SubTestResults']: self.ResultData['SubTestResults'][i].GenerateFinalOutput() self.GenerateDataFiles() ''' Generate files like Key/Value pairs in JSON format, ASCII-files, HTML-files, etc. @final ''' def GenerateDataFiles(self): self.GenerateDataFileHTML() self.GenerateDataFileJSON() ''' Generate HTML file @final ''' def GenerateDataFileHTML(self): HtmlParser = self.TestResultEnvironmentObject.HtmlParser HTMLFileName = 'TestResult.html' FinalHTML = '' HTMLTemplate = self.TestResultEnvironmentObject.TestResultHTMLTemplate FinalHTML = HTMLTemplate Levels = self.ModulePath[:].split('.') # Stylesheet StylesheetHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###HEAD_STYLESHEET_TEMPLATE###') StylesheetHTML = HtmlParser.substituteMarkerArray( StylesheetHTMLTemplate, { '###STYLESHEET###': self.TestResultEnvironmentObject.MainStylesheet + self.TestResultEnvironmentObject.TestResultStylesheet, } ) FinalHTML = HtmlParser.substituteSubpart( FinalHTML, '###HEAD_STYLESHEETS###', StylesheetHTML ) FinalHTML = HtmlParser.substituteSubpart( FinalHTML, '###HEAD_STYLESHEET_TEMPLATE###', '' ) # Clickpath ClickPathEntries = [] ClickPathEntryTemplate = HtmlParser.getSubpart(HTMLTemplate, '###CLICKPATH_ENTRY###') LevelPath = '' i = 0 tmpTestResultObject = self for Level in Levels[2:]: LevelPath = '../' * i ClickPathEntries.append(HtmlParser.substituteMarkerArray( ClickPathEntryTemplate, { '###URL###': HtmlParser.MaskHTML(LevelPath + HTMLFileName), '###LABEL###': HtmlParser.MaskHTML(tmpTestResultObject.Title) } )) if self.ParentObject: tmpTestResultObject = tmpTestResultObject.ParentObject i += 1 if not self.OverviewHTMLLink: OverviewHTMLLink = os.path.relpath(self.TestResultEnvironmentObject.GlobalOverviewPath + '/Overview.html', self.FinalResultsStoragePath) else: OverviewHTMLLink = self.OverviewHTMLLink ClickPathEntries.append(HtmlParser.substituteMarkerArray( ClickPathEntryTemplate, { '###URL###': HtmlParser.MaskHTML(OverviewHTMLLink), '###LABEL###': 'Overview' } )) ClickPathEntries.reverse() CSSClasses = '' #Result Data FinalHTML = HtmlParser.substituteSubpartArray( FinalHTML, { '###CLICKPATH_ENTRY###': ''.join(ClickPathEntries), '###RESULTDATA###': self.GenerateResultDataHTML(self, 0, self.DisplayOptions), '###ADDITIONALCSSCLASSES###': CSSClasses, } ) # Do it again for the ghost FinalHTML = HtmlParser.substituteSubpartArray( FinalHTML, { '###CLICKPATH_ENTRY###': ''.join(ClickPathEntries), } ) f = open(self.FinalResultsStoragePath + '/' + HTMLFileName, 'w') f.write(FinalHTML) f.close() ''' Generate Result Data HTML for usage in HTML files @final ''' def GenerateResultDataHTML(self, TestResultObject, RecursionLevel, DisplayOptions): HtmlParser = self.TestResultEnvironmentObject.HtmlParser HTMLTemplate = self.TestResultEnvironmentObject.TestResultHTMLTemplate ResultDataHTML = HtmlParser.getSubpart(HTMLTemplate, '###RESULTDATA###') HTMLFileName = 'TestResult.html' CSSClasses = self.NameSingle ResultDataHTML = HtmlParser.substituteMarker(ResultDataHTML, '###RESULTDATAADDITIONALCSSCLASSES###', HtmlParser.MaskHTML(CSSClasses) ) RecursionRelativePath = '' if RecursionLevel > 0: PathParts = TestResultObject.FinalResultsStoragePath.split('/') #print PathParts RecursionRelativePath = PathParts[-1] + '/' # Title if not TestResultObject.Title: TestResultObject.Title = TestResultObject.NameSingle MyObjectTestDate = ''; if RecursionLevel == 0 and TestResultObject.Attributes['TestDate']: MyObjectTestDate = 'Test Date: ' + datetime.datetime.fromtimestamp( float(TestResultObject.Attributes['TestDate'])).strftime("%Y-%m-%d %H:%m") MainTestResultAdditionalClasses = '' if RecursionLevel == 0: MainTestResultAdditionalClasses += 'MainTestResult Group' if self.DisplayOptions['Width'] > 1: MainTestResultAdditionalClasses += ' Width' + str(self.DisplayOptions['Width']) ResultDataHTML = HtmlParser.substituteMarkerArray( ResultDataHTML, { '###TITLE###': HtmlParser.MaskHTML(TestResultObject.Title), '###TESTDATE###': MyObjectTestDate, '###MAINTESTRESULTADDITIONALCLASSES###': MainTestResultAdditionalClasses, } ) # Plot PlotHTML = '' if TestResultObject.ResultData['Plot']['Enabled']: PlotHTML = HtmlParser.getSubpart(HTMLTemplate, '###PLOT###') PlotHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###PLOT###') PlotImageHTML = HtmlParser.getSubpart(PlotHTMLTemplate, '###PLOT_IMAGE###') '''PlotImageHTML = { 'SVG':'', 'PNG':'' } if TestResultObject.ResultData['Plot']['Format'] == 'svg': PlotImageHTML['SVG'] = HtmlParser.getSubpart(PlotHTMLTemplate, '###PLOT_IMAGE_SVG###') elif TestResultObject.ResultData['Plot']['Format'] == 'png': PlotImageHTML['PNG'] = HtmlParser.getSubpart(PlotHTMLTemplate, '###PLOT_IMAGE_PNG###') for i in PlotImageHTML: ''' # file operations for svg if TestResultObject.ResultData['Plot']['Format'] == 'svg' and RecursionLevel == 0 and self.SavePlotFile: f = open(TestResultObject.ResultData['Plot']['ImageFile'], 'r') SVGContent = f.read() f.close() # fix an error in chrome / safari when displaying resized svg if SVGContent.find('<svg preserveAspectRatio') == -1: SVGContent = SVGContent.replace('<svg', '<svg preserveAspectRatio="xMinYMin"', 1) # remove an invalid space in attribute width and height SVGContent = SVGContent.replace('width=" ', 'width="') SVGContent = SVGContent.replace('height=" ', 'height="') SVGContent = SVGContent.replace('x=" ', 'x="') SVGContent = SVGContent.replace('y=" ', 'y="') SVGContent = SVGContent.replace('r=" ', 'r="') SVGContent = SVGContent.replace('=" ', '="') SVGContent = SVGContent.replace('̒', '°') if self.GzipSVG and TestResultObject.ResultData['Plot']['ImageFile'].find('.svgz') == -1: os.remove(TestResultObject.ResultData['Plot']['ImageFile']) TestResultObject.ResultData['Plot']['ImageFile'] += 'z' f = gzip.GzipFile(TestResultObject.ResultData['Plot']['ImageFile'], 'w') else: f = open(TestResultObject.ResultData['Plot']['ImageFile'], 'w') if os.path.exists(TestResultObject.ResultData['Plot']['ImageFile'] + 'z'): os.remove(TestResultObject.ResultData['Plot']['ImageFile'] + 'z') f.write(SVGContent) f.close() if not TestResultObject.ResultData['Plot']['Caption']: TestResultObject.ResultData['Plot']['Caption'] = TestResultObject.Title try: PlotImageHTML = HtmlParser.substituteMarkerArray( PlotImageHTML, { '###FILENAME###': HtmlParser.MaskHTML( RecursionRelativePath + os.path.basename(TestResultObject.ResultData['Plot']['ImageFile'])), '###IMAGELARGECONTAINERID###': HtmlParser.MaskHTML( TestResultObject.Name + '_' + TestResultObject.Key), '###MARGIN_TOP###': str(int(-800. / float( DisplayOptions['Width'] * self.TestResultEnvironmentObject.Configuration['DefaultValues'][ 'CanvasWidth']) * float( self.TestResultEnvironmentObject.Configuration['DefaultValues'][ 'CanvasHeight']) / 2.)), '###TITLE###': TestResultObject.ResultData['Plot']['Caption'], '###WIDTH###': str(DisplayOptions['Width']), '###HEIGHT###': str(1), } ) except TypeError: print TestResultObject.Name, '_', TestResultObject.Key raise TypeError('Canntot convert, ' + str(TestResultObject.Name) + str(TestResultObject.Key)) #PlotHTML = HtmlParser.substituteSubpart(PlotHTML, '###PLOT_IMAGE_'+i+'###', PlotImageHTML) PlotHTML = HtmlParser.substituteSubpart(PlotHTML, '###PLOT_IMAGE###', PlotImageHTML) ResultDataHTML = HtmlParser.substituteSubpart(ResultDataHTML, '###PLOT###', PlotHTML) #Key Value Dict Pairs KeyValueDictPairsRowHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###KEYVALUEDICTPAIRS_ROW###') KeyValueDictPairsRows = '' for i in TestResultObject.ResultData['KeyList']: if not TestResultObject.ResultData['KeyValueDictPairs'].has_key(i): warnings.warn('Cannot find Key: {Key}'.format(Key=i)) continue if not TestResultObject.ResultData['KeyValueDictPairs'][i].has_key('Unit'): TestResultObject.ResultData['KeyValueDictPairs'][i]['Unit'] = '' html_value = HtmlParser.MaskHTML(str(TestResultObject.ResultData['KeyValueDictPairs'][i]['Value'])) if TestResultObject.ResultData['KeyValueDictPairs'][i].has_key('Sigma'): TestResultObject.ResultData['KeyValueDictPairs'][i]['SigmaOutput'] = ' +/- %s' % \ TestResultObject.ResultData[ 'KeyValueDictPairs'][i][ 'Sigma'] html_value += ' ± ' + HtmlParser.MaskHTML( str(TestResultObject.ResultData['KeyValueDictPairs'][i]['Sigma'])) else: TestResultObject.ResultData['KeyValueDictPairs'][i]['SigmaOutput'] = '' if not TestResultObject.ResultData['KeyValueDictPairs'][i].has_key('Label'): TestResultObject.ResultData['KeyValueDictPairs'][i]['Label'] = i KeyValueDictPairsRows += HtmlParser.substituteMarkerArray( KeyValueDictPairsRowHTMLTemplate, { '###KEY###': HtmlParser.MaskHTML(i), '###LABEL###': HtmlParser.MaskHTML( TestResultObject.ResultData['KeyValueDictPairs'][i]['Label'] ), '###VALUE###': html_value, '###UNIT###': HtmlParser.MaskHTML( TestResultObject.ResultData['KeyValueDictPairs'][i]['Unit'] ), } ) ResultDataHTML = HtmlParser.substituteSubpart(ResultDataHTML, '###KEYVALUEDICTPAIRS_ROW###', KeyValueDictPairsRows) #Table TableHTML = '' if RecursionLevel == 0 or True: TableHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###TABLE###') TableHTML = HtmlParser.GenerateTableHTML(TableHTMLTemplate, TestResultObject.ResultData['Table'], { '###ADDITIONALCSSCLASS###': '', '###ID###': 'Table', }) ResultDataHTML = HtmlParser.substituteSubpart(ResultDataHTML, '###TABLE###', TableHTML) # Sub Test Results SubTestResultListHTML = '' if RecursionLevel == 0: GroupWithNext = False i2 = 0 for i in TestResultObject.GetSortedSubTestResultDictList(): if i['TestResultObject'].DisplayOptions['Show']: GroupCSSClass = '' if i2 % 5 == 0: GroupCSSClass += ' WidthNthChild5n' if i2 % 4 == 0: GroupCSSClass += ' WidthNthChild4n' if i2 % 3 == 0: GroupCSSClass += ' WidthNthChild3n' if i2 % 2 == 0: GroupCSSClass += ' WidthNthChild2n' if i['TestResultObject'].DisplayOptions['Width'] > 1: GroupCSSClass += ' Width' + str(i['TestResultObject'].DisplayOptions['Width']) if not GroupWithNext: SubTestResultListHTML += HtmlParser.substituteMarker( HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTGROUP_START###'), '###SUBTESTRESULTGROUP_ADDITIONALCSSCLASSES###', HtmlParser.MaskHTML(GroupCSSClass) ) # only increase the width counter for a group start i2 += i['TestResultObject'].DisplayOptions['Width'] SubTestResultListHTML += self.GenerateResultDataHTML(i['TestResultObject'], RecursionLevel + 1, i['TestResultObject'].DisplayOptions) if not i['TestResultObject'].DisplayOptions['GroupWithNext']: # if the last element was in a group but the current not, close the group SubTestResultListHTML += HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTGROUP_END###') GroupWithNext = i['DisplayOptions']['GroupWithNext'] # if the last element was in a group, close the group if GroupWithNext: SubTestResultListHTML += HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTGROUP_END###') else: SubTestResultListItemHTMLTemplate = HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTLIST_ITEM###') SubTestResultListHTML = HtmlParser.getSubpart(HTMLTemplate, '###SUBTESTRESULTLIST###') SubTestResultListItems = '' # Overview link SubTestResultOverviewLinkHTML = HtmlParser.getSubpart(SubTestResultListHTML, '###OVERVIEW_LINK###') if TestResultObject.ResultData['SubTestResultDictList']: SubTestResultOverviewLinkHTML = HtmlParser.substituteMarkerArray( SubTestResultOverviewLinkHTML, { '###URL###': HtmlParser.MaskHTML( os.path.basename(TestResultObject.FinalResultsStoragePath) + '/' + HTMLFileName ), } ) else: SubTestResultOverviewLinkHTML = '' # Subtests for i in TestResultObject.GetSortedSubTestResultDictList(): if i['DisplayOptions']['Show']: SubTestResultListItems += HtmlParser.substituteMarkerArray( SubTestResultListItemHTMLTemplate, { '###URL###': HtmlParser.MaskHTML( os.path.basename(TestResultObject.FinalResultsStoragePath) + '/' + os.path.basename(i['TestResultObject'].FinalResultsStoragePath) + '/' + HTMLFileName ), '###LABEL###': HtmlParser.MaskHTML( i['TestResultObject'].Title ), } ) SubTestResultListHTML = HtmlParser.substituteSubpartArray( SubTestResultListHTML, { '###SUBTESTRESULTLIST_ITEM###': SubTestResultListItems, '###OVERVIEW_LINK###': SubTestResultOverviewLinkHTML, } ) if not SubTestResultListItems: SubTestResultListHTML = '' ResultDataHTML = HtmlParser.substituteSubpartArray( ResultDataHTML, { '###SUBTESTRESULTLIST###': SubTestResultListHTML, '###SUBTESTRESULTGROUP_START###': '', '###SUBTESTRESULTGROUP_END###': '', } ) return ResultDataHTML ''' Generate file from ResultData['KeyValueDictPairs'] Key/Value pairs in JSON format @final ''' def GenerateDataFileJSON(self): try: data = self.ResultData['KeyValueDictPairs'] for key in data: #{'NotAlivePixels': {'SigmaOutput': '', 'Unit': '', 'Value': Set([(0, 21, 31)]), 'Label': 'Pixels'}, 'MaskDefects': {'SigmaOutput': '', 'Unit': '', 'Value': Set([]), 'Label': 'Pixels'}, 'NoisyPixels': {'SigmaOutput': '', 'Unit': '', 'Value': Set([]), 'Label': 'Pixels'}, 'DeadPixels': {'SigmaOutput': '', 'Unit': '', 'Value': Set([(0, 21, 31)]), 'Label': 'Pixels'}, 'InefficentPixels': {'SigmaOutput': '', 'Unit': '', 'Value': Set([]), 'Label': 'Pixels'}} if data[key].has_key('Value'): value = data[key]['Value'] if type(value) == set: value = list(value) data[key]['Value'] = value pass #self.ResultData['KeyValueDictPairs'] f = open(self.FinalResultsStoragePath + '/KeyValueDictPairs.json', 'w') f.write(json.dumps(self.ResultData['KeyValueDictPairs'], sort_keys=True, indent=4, separators=(',', ': '))) f.close() except: print 'Cannot create JSON for %s, %s' % (type(data[key]['Value']), self.ResultData['KeyValueDictPairs']) ''' Generate file from ResultData['KeyValueDictPairs'] Key/Value pairs in ASCII format @final ''' def GenerateDataFileASCII(self): pass ''' Write all test results to the database ''' def WriteToDatabase(self, ParentID=0): ColumnMapping = {} ID = 0 ID = self.CustomWriteToDatabase(ParentID) for i in self.ResultData['SubTestResults']: try: self.ResultData['SubTestResults'][i].WriteToDatabase(ID) except Exception as inst: print 'Error in subtest (write to database)', self.ResultData['SubTestResults'][i].ModulePath, \ self.ResultData['SubTestResults'][i].FinalResultsStoragePath print inst print inst.args print sys.exc_info()[0] print "\n\n------\n" self.PostWriteToDatabase() def CustomWriteToDatabase(self, ParentID): pass def PostWriteToDatabase(self): pass def __del__(self): self.CloseSubTestResultFileHandles()
def PopulateResultData(self): SysConfiguration = ConfigParser.ConfigParser() SysConfiguration.read(['Configuration/SystemConfiguration.cfg']) ParallelProcessing = False LimitProcesses = None try: if int(SysConfiguration.get('SystemConfiguration', 'ParallelProcessing').strip())>0: ParallelProcessing = True except: print "no 'ParallelProcessing' option found, running sequentially..." pass try: if int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip())>0: LimitProcesses = int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip()) except: pass print 'do fitting...' nRocs = self.Attributes['NumberOfChips'] directory = self.RawTestSessionDataPath refit = self.TestResultEnvironmentObject.Configuration['Fitting']['refit'] refitScurves = refit refitPHlin = refit refitPHtan = refit # compare current fit revision with saved revision FitRevDict = BetterConfigParser() fitRevFileName = directory + "/fits.rev" if os.path.isfile(fitRevFileName): try: FitRevDict.read(fitRevFileName) # if fit procedure changed -> refit if self.FitRevisionScurves > int(FitRevDict.get('FitRevisions', 'SCurves')): print "info: fit procedure has changed, re-fitting scurves..." refitScurves = True if self.FitRevisionPHlin > int(FitRevDict.get('FitRevisions', 'PHlin')): print "info: fit procedure has changed, re-fitting ph lin..." refitPHlin = True if self.FitRevisionPHtan > int(FitRevDict.get('FitRevisions', 'PHtan')): print "info: fit procedure has changed, re-fitting ph tan..." refitPHtan = True # if last fitting was not done completely (eg. STRG+C while fit...) -> refit if FitRevDict.get('Fit', 'Complete').strip().lower() != 'true': print "info: last fit was incomplete! re-fitting all..." refitScurves = True refitPHlin = True refitPHtan = True except: print "warning: could not read fit revision file! => needs re-fitting" refitScurves = True refitPHlin = True refitPHtan = True else: FitRevDict.add_section('FitRevisions') FitRevDict.add_section('Fit') print "info: this folder has never been analyzed with a recent version of MoReWeb => needs re-fitting" refitScurves = True refitPHlin = True refitPHtan = True # unset complete flag in file FitRevDict.set('Fit','Complete', 'false') with open(fitRevFileName, 'wb') as fitRevFile: FitRevDict.write(fitRevFile) print 'SCurve fitting...' ePerVcal = self.TestResultEnvironmentObject.GradingParameters['StandardVcal2ElectronConversionFactor'] fitter = SCurve_Fitting(refit=refitScurves, HistoDict=self.ParentObject.HistoDict,ePerVcal=ePerVcal, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses) fitter.FitAllSCurve(directory,nRocs) print 'linear PH fitting...' fitter = PH_Fitting(fitMode=0, refit=refitPHlin, HistoDict=self.ParentObject.HistoDict, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses) fitter.FitAllPHCurves(directory,nRocs) print 'tanh PH fitting...' fitter = PH_Fitting(fitMode=3, refit=refitPHtan, HistoDict=self.ParentObject.HistoDict, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses) fitter.FitAllPHCurves(directory,nRocs) # update saved revisions FitRevDict.set('FitRevisions','SCurves', str(self.FitRevisionScurves)) FitRevDict.set('FitRevisions','PHlin', str(self.FitRevisionPHlin)) FitRevDict.set('FitRevisions','PHtan', str(self.FitRevisionPHtan)) # unset complete flag in file FitRevDict.set('Fit','Complete', 'true') with open(fitRevFileName, 'wb') as fitRevFile: FitRevDict.write(fitRevFile) print 'done'
def PopulateResultData(self): SysConfiguration = ConfigParser.ConfigParser() SysConfiguration.read(['Configuration/SystemConfiguration.cfg']) ParallelProcessing = False LimitProcesses = None try: if int(SysConfiguration.get('SystemConfiguration', 'ParallelProcessing').strip())>0: ParallelProcessing = True except: print "no 'ParallelProcessing' option found, running sequentially..." pass try: if int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip())>0: LimitProcesses = int(SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip()) except: pass nRocs = self.ParentObject.Attributes['NumberOfChips'] directory = self.ParentObject.Attributes['SCurvePaths']['HRSCurves_{Rate}'.format(Rate=self.Attributes['Rate'])] refit = self.TestResultEnvironmentObject.Configuration['Fitting']['refit'] print 'SCurve fitting...' HistoDict = BetterConfigParser() HistoDict.add_section('SCurveFitting') HistoDict.set('SCurveFitting','nTrigs', str(10)) HistoDict.set('SCurveFitting','dir', '') HistoDict.set('SCurveFitting','ignoreValidityCheck', '1') HistoDict.set('SCurveFitting','inputFileName', self.ParentObject.ParentObject.HistoDict.get('HighRate', 'SCurveDataFileName')) ePerVcal = self.TestResultEnvironmentObject.GradingParameters['StandardVcal2ElectronConversionFactor'] fitter = SCurve_Fitting(refit,HistoDict = HistoDict,ePerVcal=ePerVcal, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses) fitter.FitAllSCurve(directory, nRocs)
def PopulateResultData(self): SysConfiguration = ConfigParser.ConfigParser() SysConfiguration.read(['Configuration/SystemConfiguration.cfg']) ParallelProcessing = False LimitProcesses = None try: if int( SysConfiguration.get('SystemConfiguration', 'ParallelProcessing').strip()) > 0: ParallelProcessing = True except: print "no 'ParallelProcessing' option found, running sequentially..." pass try: if int( SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip()) > 0: LimitProcesses = int( SysConfiguration.get('SystemConfiguration', 'LimitProcesses').strip()) except: pass nRocs = self.ParentObject.Attributes['NumberOfChips'] directory = self.ParentObject.Attributes['SCurvePaths'][ 'HRSCurves_{Rate}'.format(Rate=self.Attributes['Rate'])] refit = self.TestResultEnvironmentObject.Configuration['Fitting'][ 'refit'] print 'SCurve fitting...' HistoDict = BetterConfigParser() HistoDict.add_section('SCurveFitting') HistoDict.set('SCurveFitting', 'nTrigs', str(10)) HistoDict.set('SCurveFitting', 'dir', '') HistoDict.set('SCurveFitting', 'ignoreValidityCheck', '1') HistoDict.set( 'SCurveFitting', 'inputFileName', self.ParentObject.ParentObject.HistoDict.get( 'HighRate', 'SCurveDataFileName')) ePerVcal = self.TestResultEnvironmentObject.GradingParameters[ 'StandardVcal2ElectronConversionFactor'] # compare current fit revision with saved revision FitRevDict = BetterConfigParser() fitRevFileName = directory + "/fits.rev" if os.path.isfile(fitRevFileName): try: FitRevDict.read(fitRevFileName) # if fit procedure changed -> refit if self.FitRevisionScurves > int( FitRevDict.get('FitRevisions', 'SCurves')): print "info: fit procedure has changed, re-fitting..." refit = True # if last fitting was not done completely (eg. STRG+C while fit...) -> refit if FitRevDict.get('Fit', 'Complete').strip().lower() != 'true': print "info: last fit was incomplete! re-fitting..." refit = True except: print "warning: could not read fit revision file! => needs re-fitting" refit = True else: FitRevDict.add_section('FitRevisions') FitRevDict.add_section('Fit') print "info: never analyzed with recent version of MoReWeb => needs re-fitting" refit = True # unset complete flag in file FitRevDict.set('Fit', 'Complete', 'false') with open(fitRevFileName, 'wb') as fitRevFile: FitRevDict.write(fitRevFile) # do fitting fitter = SCurve_Fitting(refit=refit, HistoDict=HistoDict, ePerVcal=ePerVcal, ParallelProcessing=ParallelProcessing, LimitProcesses=LimitProcesses) fitter.FitAllSCurve(directory, nRocs) # update saved revision FitRevDict.set('FitRevisions', 'SCurves', str(self.FitRevisionScurves)) # set complete flag in file FitRevDict.set('Fit', 'Complete', 'true') with open(fitRevFileName, 'wb') as fitRevFile: FitRevDict.write(fitRevFile)