def validate(self): """ Validate the alarm configuration. """ className = self.__RootObject.__class__.__name__ if not className in self.SUPPORTED_ROOT_TYPES: abort('%s object not supported by alarm %s' %\ (className, self.name()))
def validate(self): """ Validate the alarm configuration. """ className = self.__RootObject.__class__.__name__ if not className in self.SUPPORTED_ROOT_TYPES: abort('%s object not supported by alarm %s' %\ (className, self.name()))
def e3analyzer2(binFilePath, suffix = None, opts = '-b -r -c'): """ Run the official EEE analyzer. Mind we are not doing anything, at this level, to make sure that you do have the right eee_calib.txt file---it is really your responsibility to do it. """ chrono = E3Chrono() if not os.path.exists(binFilePath): abort('Could not find %s, giving up...' % binFilePath) if not binFilePath.endswith('.bin'): abort('%s not a .bin file, giving up...' % binFilePath) cleanupTemp() listTemp() logger.info('Processing run data file %s...' % binFilePath) binFileName = os.path.basename(binFilePath) copyFilePath = os.path.join(E3PIPE_TEMP, binFileName) __utils__.cp(binFilePath, copyFilePath) # __e3analyzer2(copyFilePath, opts) __e3analyzer2(binFilePath, opts) logger.info('Run processed in %.3f s.' % chrono.stop()) baseFilePath = copyFilePath.replace('.bin', '') if suffix is not None: for extension in E3_ANALYZER_NEW_OUTPUTS: src = '%s%s' % (baseFilePath, extension) dest = '%s_%s%s' % (baseFilePath, suffix, extension) __utils__.mv(src, dest) baseFilePath = '%s_%s' % (baseFilePath, suffix) listTemp() logger.info('Returning base path: "%s"...' % baseFilePath) return baseFilePath
def uniqueRunId(station, date, runId): """ Turn a triplet (station, date, runId) into a unique ID. """ try: stationId = E3_STATION_DICT[station] except KeyError: abort('Unknown station: %s' % station) dateId = (date - E3_DATE_REFERENCE).days return stationId * STATION_MULTIPLIER + dateId * DATE_MULTIPLIER + runId
def uniqueRunId(station, date, runId): """ Turn a triplet (station, date, runId) into a unique ID. """ try: stationId = E3_STATION_DICT[station] except KeyError: abort('Unknown station: %s' % station) dateId = (date - E3_DATE_REFERENCE).days return stationId*STATION_MULTIPLIER + dateId*DATE_MULTIPLIER + runId
def __init__(self, filePath, extension = None): """ """ if not os.path.exists(filePath): abort('Could not find input file %s' % filePath) if extension is not None and not filePath.endswith(extension): abort('Wrong file extension for input file %s (%s expected)' %\ (filePath, extension)) logger.info('Opening input file %s...' % filePath) file.__init__(self, filePath)
def __init__(self, filePath, extension=None): """ """ if not os.path.exists(filePath): abort('Could not find input file %s' % filePath) if extension is not None and not filePath.endswith(extension): abort('Wrong file extension for input file %s (%s expected)' %\ (filePath, extension)) logger.info('Opening input file %s...' % filePath) file.__init__(self, filePath)
def Add(self, filePath): """ Add a file to the chain. Note that we are doing some basic check, here, in order to try and prevent simple issues. """ if not '*' in filePath: if not os.path.exists(filePath): abort('Could not find input file %s' % filePath) if not filePath.endswith(E3RootFileBase.EXTENSION): abort('Wrong file extension for input file %s (%s expected)' %\ (filePath, E3RootFileBase.EXTENSION)) logger.info('Adding %s...' % filePath) ROOT.TChain.Add(self, filePath)
def Add(self, filePath): """ Add a file to the chain. Note that we are doing some basic check, here, in order to try and prevent simple issues. """ if not '*' in filePath: if not os.path.exists(filePath): abort('Could not find input file %s' % filePath) if not filePath.endswith(E3RootFileBase.EXTENSION): abort('Wrong file extension for input file %s (%s expected)' %\ (filePath, E3RootFileBase.EXTENSION)) logger.info('Adding %s...' % filePath) ROOT.TChain.Add(self, filePath)
def __init__(self): """ Constructor. """ if not os.path.exists(self.CFG_FILE_PATH): abort('Could not find configuration file %s' % self.CFG_FILE_PATH) logger.info('Reading db configuration file %s...' % self.CFG_FILE_PATH) parser = ConfigParser.ConfigParser() parser.read(self.CFG_FILE_PATH) try: host = parser.get('General', 'host') user = parser.get('General', 'user') dbname = parser.get('General', 'dbname') pwd = parser.get('General', 'pwd') except ConfigParser.NoOptionError, e: abort(e)
def __init__(self): """ Constructor. """ if not os.path.exists(self.CFG_FILE_PATH): abort('Could not find configuration file %s' % self.CFG_FILE_PATH) logger.info('Reading db configuration file %s...' % self.CFG_FILE_PATH) parser = ConfigParser.ConfigParser() parser.read(self.CFG_FILE_PATH) try: host = parser.get('General', 'host') user = parser.get('General', 'user') dbname = parser.get('General', 'dbname') pwd = parser.get('General', 'pwd') except ConfigParser.NoOptionError, e: abort(e)
def __init__(self, filePath): """ """ if not os.path.exists(filePath): abort('Could not find input file %s' % filePath) if not filePath.endswith(E3RootFileBase.EXTENSION): abort('Wrong file extension for input file %s (%s expected)' %\ (filePath, E3RootFileBase.EXTENSION)) logger.info('Opening input file %s...' % filePath) ROOT.TFile.__init__(self, filePath) logger.info('Creator: %s' % self.creator()) logger.info('Version: %s' % self.version()) logger.info('Creation time: %s' % self.creationTime()) logger.info('Date: %s' % self.date()) logger.info('Station: %s' % self.station())
def __init__(self, filePath): """ """ if not os.path.exists(filePath): abort('Could not find input file %s' % filePath) if not filePath.endswith(E3RootFileBase.EXTENSION): abort('Wrong file extension for input file %s (%s expected)' %\ (filePath, E3RootFileBase.EXTENSION)) logger.info('Opening input file %s...' % filePath) ROOT.TFile.__init__(self, filePath) logger.info('Creator: %s' % self.creator()) logger.info('Version: %s' % self.version()) logger.info('Creation time: %s' % self.creationTime()) logger.info('Date: %s' % self.date()) logger.info('Station: %s' % self.station())
def __init__(self, filePath, creator = 'e3pipe', date = None, station = None): """ """ if not filePath.endswith(E3RootFileBase.EXTENSION): abort('Please root the %s extension for ROOT files' %\ E3RootFileBase.EXTENSION) logger.info('Opening output file %s...' % filePath) ROOT.TFile.__init__(self, filePath, 'RECREATE') writeString(E3RootFileBase.CREATOR_KEY, creator) writeString(E3RootFileBase.VERSION_KEY, TAG) writeString(E3RootFileBase.CREATION_TIME_KEY, time.asctime()) if date is not None: writeString(E3RootFileBase.DATE_KEY, date) if station is not None: writeString(E3RootFileBase.STATION_KEY, station)
def e3mergeFiles(outputFilePath, *fileList, **kwargs): """ Merge a series of DST ROOT files. """ if kwargs.get('sort', True): logger.info('Sorting file list...') fileList = list(fileList) fileList.sort() logger.info('Done.') if len(fileList) < 2: abort('No files to merge') __utils__.createFolder(os.path.dirname(outputFilePath)) outputFile = E3OutputRootFile(outputFilePath, 'e3merge', kwargs['date'], kwargs['station']) if kwargs.get('mergeHeader', True): header = E3DstHeaderChain(*fileList) branches = kwargs.get('headerBranches', None) if branches is not None: header.selectBranches(*branches) _header = header.CloneTree() _header.Write() if kwargs.get('mergeEvents', True): events = E3DstEventChain(*fileList) branches = kwargs.get('eventBranches', None) if branches is not None: events.selectBranches(*branches) _events = events.CloneTree() _events.Write() if kwargs.get('mergeTrending', True): trending = E3DstTrendingChain(*fileList) branches = kwargs.get('trendingBranches', None) if branches is not None: trending.selectBranches(*branches) _trending = trending.CloneTree() _trending.Write() if kwargs.get('mergeWeather', True): weather = E3DstWeatherChain(*fileList) branches = kwargs.get('trendingBranches', None) if branches is not None: weather.selectBranches(*branches) _weather = weather.CloneTree() _weather.Write() outputFile.Close() return outputFilePath
def e3mergeFiles(outputFilePath, *fileList, **kwargs): """ Merge a series of DST ROOT files. """ if kwargs.get('sort', True): logger.info('Sorting file list...') fileList = list(fileList) fileList.sort() logger.info('Done.') if len(fileList) < 2: abort('No files to merge') __utils__.createFolder(os.path.dirname(outputFilePath)) outputFile = E3OutputRootFile(outputFilePath, 'e3merge', kwargs['date'], kwargs['station']) if kwargs.get('mergeHeader', True): header = E3DstHeaderChain(*fileList) branches = kwargs.get('headerBranches', None) if branches is not None: header.selectBranches(*branches) _header = header.CloneTree() _header.Write() if kwargs.get('mergeEvents', True): events = E3DstEventChain(*fileList) branches = kwargs.get('eventBranches', None) if branches is not None: events.selectBranches(*branches) _events = events.CloneTree() _events.Write() if kwargs.get('mergeTrending', True): trending = E3DstTrendingChain(*fileList) branches = kwargs.get('trendingBranches', None) if branches is not None: trending.selectBranches(*branches) _trending = trending.CloneTree() _trending.Write() if kwargs.get('mergeWeather', True): weather = E3DstWeatherChain(*fileList) branches = kwargs.get('trendingBranches', None) if branches is not None: weather.selectBranches(*branches) _weather = weather.CloneTree() _weather.Write() outputFile.Close() return outputFilePath
def __init__(self, filePath): """ Constructor. """ if not filePath.endswith('.bin'): abort('%s does not look like a raw binary file' % filePath) dict.__init__(self) self['RawFilePath'] = filePath self['RawDirName'], self['RawFileName'] = os.path.split(filePath) data = splitFilePath(filePath) self['Station'] = data[0] year, month, day = [int(item) for item in data[1:4]] self['RunString'] = data[4] self['RunNumber'] = int(self.RunString) self['Date'] = datetime.date(year, month, day) self['DateString'] = date2str(self.Date) self['DstFilePath'] = self.__dstFilePath() self['CalibFilePath'] = self.__calibFilePath() self['DqmFolderPath'] = self.__dqmFolderPath() self['LogFilePath'] = self.__logFilePath() self['LockFilePath'] = self.__lockFilePath()
def __init__(self, filePath): """ Constructor. """ if not filePath.endswith('.bin'): abort('%s does not look like a raw binary file' % filePath) dict.__init__(self) self['RawFilePath'] = filePath self['RawDirName'], self['RawFileName'] = os.path.split(filePath) data = splitFilePath(filePath) self['Station'] = data[0] year, month, day = [int(item) for item in data[1:4]] self['RunString'] = data[4] self['RunNumber'] = int(self.RunString) self['Date'] = datetime.date(year, month, day) self['DateString'] = date2str(self.Date) self['DstFilePath'] = self.__dstFilePath() self['CalibFilePath'] = self.__calibFilePath() self['DqmFolderPath'] = self.__dqmFolderPath() self['LogFilePath'] = self.__logFilePath() self['LockFilePath'] = self.__lockFilePath()
def e3analyzer(binFilePath, suffix = None): """ Run the official EEE analyzer. Mind we are not doing anything, at this level, to make sure that you do have the right eee_calib.txt file---it is really your responsibility to do it. """ chrono = E3Chrono() if not os.path.exists(binFilePath): abort('Could not find %s, giving up...' % binFilePath) if not binFilePath.endswith('.bin'): abort('%s not a .bin file, giving up...' % binFilePath) cleanupTemp() listTemp() logger.info('Processing run data file %s...' % binFilePath) binFileName = os.path.basename(binFilePath) copyFilePath = os.path.join(E3PIPE_TEMP, binFileName) __utils__.cp(binFilePath, copyFilePath) # Here we need a bit of extra attention about whether we are analyzing data # from Pisa or any other station. station = splitFilePath(copyFilePath)[0] if station == 'PISA-01': __e3analyzer_pisa(copyFilePath) else: __e3analyzer_standard(copyFilePath) # End of the hack for the Pisa custom DAQ. logger.info('Run processed in %.3f s.' % chrono.stop()) baseFilePath = copyFilePath.replace('.bin', '') if suffix is not None: for extension in E3_ANALYZER_OUTPUTS: src = '%s%s' % (baseFilePath, extension) dest = '%s_%s%s' % (baseFilePath, suffix, extension) __utils__.mv(src, dest) baseFilePath = '%s_%s' % (baseFilePath, suffix) listTemp() logger.info('Returning base path: "%s"...' % baseFilePath) return baseFilePath
def e3analyzer(binFilePath, suffix=None): """ Run the official EEE analyzer. Mind we are not doing anything, at this level, to make sure that you do have the right eee_calib.txt file---it is really your responsibility to do it. """ chrono = E3Chrono() if not os.path.exists(binFilePath): abort('Could not find %s, giving up...' % binFilePath) if not binFilePath.endswith('.bin'): abort('%s not a .bin file, giving up...' % binFilePath) cleanupTemp() listTemp() logger.info('Processing run data file %s...' % binFilePath) binFileName = os.path.basename(binFilePath) copyFilePath = os.path.join(E3PIPE_TEMP, binFileName) __utils__.cp(binFilePath, copyFilePath) # Here we need a bit of extra attention about whether we are analyzing data # from Pisa or any other station. station = splitFilePath(copyFilePath)[0] if station == 'PISA-01': __e3analyzer_pisa(copyFilePath) else: __e3analyzer_standard(copyFilePath) # End of the hack for the Pisa custom DAQ. logger.info('Run processed in %.3f s.' % chrono.stop()) baseFilePath = copyFilePath.replace('.bin', '') if suffix is not None: for extension in E3_ANALYZER_OUTPUTS: src = '%s%s' % (baseFilePath, extension) dest = '%s_%s%s' % (baseFilePath, suffix, extension) __utils__.mv(src, dest) baseFilePath = '%s_%s' % (baseFilePath, suffix) listTemp() logger.info('Returning base path: "%s"...' % baseFilePath) return baseFilePath
def __validate(self): """ Validate the limits (i.e. make sure that the min and max values are consistent and that the warning margins are smaller than the error margins.) """ if self.WarningMin > self.WarningMax: abort('Warning min (%s) higher than warning max (%s).' %\ (self.WarningMin, self.WarningMax)) if self.ErrorMin > self.ErrorMax: abort('Error min (%s) higher than error max (%s).' %\ (self.ErrorMin, self.ErrorMax)) if self.WarningMin < self.ErrorMin: abort('Warning min (%s) lower than error min (%s).' %\ (self.WarningMin, self.ErrorMin)) if self.WarningMax > self.ErrorMax: abort('Warning max (%s) is higher than error max (%s).' %\ (self.WarningMax, self.ErrorMax))
def e3dst(baseFilePath): """ Parse all the output text files from the analyzer and build the actual DST in ROOT format. """ chrono = E3Chrono() logger.info('Collecting input files for the DST...') outFile = E3AnalyzerOutFile('%s.out' % baseFilePath) sumFile = E3AnalyzerSumFile('%s.sum' % baseFilePath) dstFilePath = '%s_dst.root' % baseFilePath uniqueId = uniqueRunIdFromFilePath(baseFilePath) station, year, month, day, runId = splitFilePath(baseFilePath) date = datetime.date(int(year), int(month), int(day)) logger.info('Unique run ID is %s.' % uniqueId) logger.info('Opening output ROOT file %s...' % dstFilePath) rootFile = E3OutputRootFile(dstFilePath, 'e3dst', date2str(date), station) logger.info('Initializing event tree...') eventTree = E3DstEventTree() eventTree.setUniqueRunId(uniqueId) logger.info('Filling event tree...') for row in outFile: eventTree.fillRow(row) eventStat = outFile.eventStat() tmin = outFile.minTimestamp() tmax = outFile.maxTimestamp() # If we have less than two good events there is nothing we could # possibly do, here. # Close all files and remove the output dst ROOT file so that # we know the run has not been processed. if eventStat['hits'] < 2: logger.info('Closing all files...') rootFile.Close() outFile.close() sumFile.close() __utils__.rm(dstFilePath) logger.info('No events with hits, processing terminated after %.3f s.' %\ chrono.stop()) sys.exit(E3PIPE_EXIT_CODE_NO_HITS_EVENTS) logger.info('Event stats: %s' % eventStat) logger.info('Range of timestamps in the output files: %.3f--%.3f' %\ (tmin, tmax)) duration = tmax - tmin logger.info('Corresponding run duration: %.3f s' % duration) if duration > MAX_RUN_DURATION: logger.error('Run looks way too long, something must be wrong.') sys.exit(E3PIPE_EXIT_CODE_RUN_TOO_LONG) eventTree.Write() logger.info('Done, %d event(s) filled in.' % eventTree.GetEntries()) if eventTree.GetEntries() == 0: abort('No events found (maybe an issue with eee_calib.txt?)') logger.info('Creating monitoring plots...') eventTree.doMonitoring() logger.info('Initializing weather tree...') weatherTree = E3DstWeatherTree() weatherTree.setUniqueRunId(uniqueId) binFile = E3BinFile('%s.bin' % baseFilePath) weatherRecord = binFile.weatherStationRecord() if weatherRecord is not None: logger.info('Filling weather tree...') weatherTree.fillRow(weatherRecord.data()) weatherTree.Write() logger.info('Creating trending data products...') trendingTree = eventTree.doTrending(TRENDING_TIME_BIN, tmin, tmax, weatherRecord) logger.info('Writing trending tree...') trendingTree.Write() logger.info('Writing monitoring/trending plots...') for plot in eventTree.plots(): plot.Write() logger.info('Initializing header tree...') headerTree = E3DstHeaderTree() headerTree.setUniqueRunId(uniqueId) logger.info('Filling header tree...') data = sumFile.data() # Mind we need to add a few things "by hand", here, as not all the # information that we want in the header is really coming from the # sum file. data['RunNumber'] = row['RunNumber'] data['RunStart'] = tmin data['RunStop'] = tmax data['RunDuration'] = tmax - tmin data['NumHitEvents'] = eventStat['hits'] data['NumTrackEvents'] = eventStat['track'] data['NumNoHitsEvents'] = eventStat['no_hits'] data['NumNoHitEvents'] = eventStat['no_hit'] data['NumMalformedEvents'] = eventStat['malformed'] data['NumBackwardEvents'] = eventStat['backward'] headerTree.fillRow(data) headerTree.Write() logger.info('Creating histograms...') for key in ['HitMultBot', 'HitMultMid', 'HitMultTop', 'ClusterMultBot', 'ClusterMultMid', 'ClusterMultTop']: h = data2hist(data, key, xmax = 15.5) h.Write() for key in ['HitMultTotal', 'ClusterMultTotal']: h = data2hist(data, key, xmax = 35.5) h.Write() logger.info('Closing all files...') rootFile.Close() outFile.close() sumFile.close() logger.info('DST created in %.3f s.' % chrono.stop()) listTemp() logger.info('Returning DST path: %s...' % dstFilePath) return dstFilePath
# Import the necessary stuff. from e3pipe.tasks.e3report import e3report if len(args): parser.error('This app only takes options.') if opts.statfile is None: parser.error('Please select an input file (e3report.py -F file)') # And now we are ready to go. if opts.end is None: import datetime end = datetime.date.today() else: from e3pipe.dst.__time__ import str2date end = str2date(opts.end) import os if not os.path.isfile(opts.statfile): abort('Could not find input file %s' % opts.statfile) stations = [line.split()[0] for line in open(opts.statfile)] import e3pipe.__utils__ as __utils__ for station in stations: logger.info('Creating report for %s...' % station) _cmd = 'e3report.py -s %s -E %s -N %d -o %s' %\ (station, end, opts.span, opts.out) exitCode = __utils__.cmd(_cmd)
def e3trending2(dstFilePath): """ Parse a DST root file produced by the new analyzer and add the bits that are missing for the DQM (e.g., the trending). """ chrono = E3Chrono() #uniqueId = uniqueRunIdFromFilePath(baseFilePath) #station, year, month, day, runId = splitFilePath(baseFilePath) #date = datetime.date(int(year), int(month), int(day)) #logger.info('Unique run ID is %s.' % uniqueId) logger.info('Opening output ROOT file %s...' % dstFilePath) rootFile = E3OutputRootFile(dstFilePath, 'e3dst', date2str(date), station) logger.info('Initializing event tree...') eventTree = E3DstEventTree() #eventTree.setUniqueRunId(uniqueId) #logger.info('Filling event tree...') #for row in outFile: # eventTree.fillRow(row) #eventStat = outFile.eventStat() #tmin = outFile.minTimestamp() #tmax = outFile.maxTimestamp() # If we have less than two good events there is nothing we could # possibly do, here. # Close all files and remove the output dst ROOT file so that # we know the run has not been processed. #if eventStat['hits'] < 2: # logger.info('Closing all files...') # rootFile.Close() # outFile.close() # sumFile.close() # __utils__.rm(dstFilePath) # logger.info('No events with hits, processing terminated after %.3f s.' %\ # chrono.stop()) # sys.exit(E3PIPE_EXIT_CODE_NO_HITS_EVENTS) #logger.info('Event stats: %s' % eventStat) #logger.info('Range of timestamps in the output files: %.3f--%.3f' %\ # (tmin, tmax)) #duration = tmax - tmin #logger.info('Corresponding run duration: %.3f s' % duration) #if duration > MAX_RUN_DURATION: # logger.error('Run looks way too long, something must be wrong.') # sys.exit(E3PIPE_EXIT_CODE_RUN_TOO_LONG) #eventTree.Write() logger.info('Done, %d event(s) filled in.' % eventTree.GetEntries()) if eventTree.GetEntries() == 0: abort('No events found (maybe an issue with eee_calib.txt?)') logger.info('Creating monitoring plots...') #eventTree.doMonitoring() #logger.info('Initializing weather tree...') #weatherTree = E3DstWeatherTree() #weatherTree.setUniqueRunId(uniqueId) #binFile = E3BinFile('%s.bin' % baseFilePath) #weatherRecord = binFile.weatherStationRecord() #if weatherRecord is not None: # logger.info('Filling weather tree...') # weatherTree.fillRow(weatherRecord.data()) #weatherTree.Write() #logger.info('Creating trending data products...') #trendingTree = eventTree.doTrending(TRENDING_TIME_BIN, tmin, tmax, # weatherRecord) #logger.info('Writing trending tree...') #trendingTree.Write() #logger.info('Writing monitoring/trending plots...') #for plot in eventTree.plots(): # plot.Write() #logger.info('Initializing header tree...') #headerTree = E3DstHeaderTree() #headerTree.setUniqueRunId(uniqueId) #logger.info('Filling header tree...') #data = sumFile.data() # Mind we need to add a few things "by hand", here, as not all the # information that we want in the header is really coming from the # sum file. #data['RunNumber'] = row['RunNumber'] #data['RunStart'] = tmin #data['RunStop'] = tmax #data['RunDuration'] = tmax - tmin #data['NumHitEvents'] = eventStat['hits'] #data['NumTrackEvents'] = eventStat['track'] #data['NumNoHitsEvents'] = eventStat['no_hits'] #data['NumNoHitEvents'] = eventStat['no_hit'] #data['NumMalformedEvents'] = eventStat['malformed'] #data['NumBackwardEvents'] = eventStat['backward'] #headerTree.fillRow(data) #headerTree.Write() #logger.info('Creating histograms...') #for key in ['HitMultBot', 'HitMultMid', 'HitMultTop', # 'ClusterMultBot', 'ClusterMultMid', 'ClusterMultTop']: # h = data2hist(data, key, xmax = 15.5) # h.Write() #for key in ['HitMultTotal', 'ClusterMultTotal']: # h = data2hist(data, key, xmax = 35.5) # h.Write() #logger.info('Closing all files...') #rootFile.Close() #outFile.close() #sumFile.close() logger.info('DST created in %.3f s.' % chrono.stop()) listTemp() logger.info('Returning DST path: %s...' % dstFilePath) return dstFilePath