def e3crawl(endDate = None, daysSpanned = 2, minSize = 0,
blackList = [], overwrite = False, maxNumRuns = None,
dryRun = False):
""" Crawl the raw data and process the files.
"""
logDate = datetime.datetime.today()
datestr = date2str(logDate)
timestr = logDate.strftime('%Y-%m-%d-%H-%M-%S-%f')
logFilePath = os.path.join(E3PIPE_LOG_BASE, datestr, '%s.log' % timestr)
logFolder = os.path.dirname(logFilePath)
__utils__.createFolder(logFolder)
logFileHandler = E3FileHandler(logFilePath)
crawler = E3RunDbRawFileCrawler(endDate, daysSpanned, minSize,
blackList, overwrite)
logger.info(crawler)
if dryRun:
logger.info('Just kidding, dry run :-)')
return
numFiles = maxNumRuns or len(crawler)
curFile = 1
for filePath in crawler:
logger.info('Processing file %d/%d: %s' % (curFile, numFiles, filePath))
_cmd = 'e3process.py %s' % filePath
exitCode = __utils__.cmd(_cmd)
if maxNumRuns is not None and curFile >= maxNumRuns:
break
curFile += 1
logFileHandler.close()
def e3crawl(endDate=None,
daysSpanned=2,
minSize=0,
blackList=[],
overwrite=False,
maxNumRuns=None,
dryRun=False):
""" Crawl the raw data and process the files.
"""
logDate = datetime.datetime.today()
datestr = date2str(logDate)
timestr = logDate.strftime('%Y-%m-%d-%H-%M-%S-%f')
logFilePath = os.path.join(E3PIPE_LOG_BASE, datestr, '%s.log' % timestr)
logFolder = os.path.dirname(logFilePath)
__utils__.createFolder(logFolder)
logFileHandler = E3FileHandler(logFilePath)
crawler = E3RunDbRawFileCrawler(endDate, daysSpanned, minSize, blackList,
overwrite)
logger.info(crawler)
if dryRun:
logger.info('Just kidding, dry run :-)')
return
numFiles = maxNumRuns or len(crawler)
curFile = 1
for filePath in crawler:
logger.info('Processing file %d/%d: %s' %
(curFile, numFiles, filePath))
_cmd = 'e3process.py %s' % filePath
exitCode = __utils__.cmd(_cmd)
if maxNumRuns is not None and curFile >= maxNumRuns:
break
curFile += 1
logFileHandler.close()
def e3mergeFiles(outputFilePath, *fileList, **kwargs):
""" Merge a series of DST ROOT files.
"""
if kwargs.get('sort', True):
logger.info('Sorting file list...')
fileList = list(fileList)
fileList.sort()
logger.info('Done.')
if len(fileList) < 2:
abort('No files to merge')
__utils__.createFolder(os.path.dirname(outputFilePath))
outputFile = E3OutputRootFile(outputFilePath, 'e3merge', kwargs['date'],
kwargs['station'])
if kwargs.get('mergeHeader', True):
header = E3DstHeaderChain(*fileList)
branches = kwargs.get('headerBranches', None)
if branches is not None:
header.selectBranches(*branches)
_header = header.CloneTree()
_header.Write()
if kwargs.get('mergeEvents', True):
events = E3DstEventChain(*fileList)
branches = kwargs.get('eventBranches', None)
if branches is not None:
events.selectBranches(*branches)
_events = events.CloneTree()
_events.Write()
if kwargs.get('mergeTrending', True):
trending = E3DstTrendingChain(*fileList)
branches = kwargs.get('trendingBranches', None)
if branches is not None:
trending.selectBranches(*branches)
_trending = trending.CloneTree()
_trending.Write()
if kwargs.get('mergeWeather', True):
weather = E3DstWeatherChain(*fileList)
branches = kwargs.get('trendingBranches', None)
if branches is not None:
weather.selectBranches(*branches)
_weather = weather.CloneTree()
_weather.Write()
outputFile.Close()
return outputFilePath
def e3mergeFiles(outputFilePath, *fileList, **kwargs):
""" Merge a series of DST ROOT files.
"""
if kwargs.get('sort', True):
logger.info('Sorting file list...')
fileList = list(fileList)
fileList.sort()
logger.info('Done.')
if len(fileList) < 2:
abort('No files to merge')
__utils__.createFolder(os.path.dirname(outputFilePath))
outputFile = E3OutputRootFile(outputFilePath, 'e3merge', kwargs['date'],
kwargs['station'])
if kwargs.get('mergeHeader', True):
header = E3DstHeaderChain(*fileList)
branches = kwargs.get('headerBranches', None)
if branches is not None:
header.selectBranches(*branches)
_header = header.CloneTree()
_header.Write()
if kwargs.get('mergeEvents', True):
events = E3DstEventChain(*fileList)
branches = kwargs.get('eventBranches', None)
if branches is not None:
events.selectBranches(*branches)
_events = events.CloneTree()
_events.Write()
if kwargs.get('mergeTrending', True):
trending = E3DstTrendingChain(*fileList)
branches = kwargs.get('trendingBranches', None)
if branches is not None:
trending.selectBranches(*branches)
_trending = trending.CloneTree()
_trending.Write()
if kwargs.get('mergeWeather', True):
weather = E3DstWeatherChain(*fileList)
branches = kwargs.get('trendingBranches', None)
if branches is not None:
weather.selectBranches(*branches)
_weather = weather.CloneTree()
_weather.Write()
outputFile.Close()
return outputFilePath
def run(self):
""" Run the data quality monitoring on a DST file.
TODO: the way we distinguish between plots and alarms is actually ugly,
we need to find a better way to handle this.
"""
if self.__OutputFolder is not None:
createFolder(self.__OutputFolder)
for item in DQM_BASELINE_LIST:
if item is None:
self.__ObjectList.append(item)
elif len(item) == 2:
plotName, kwargs = item
self.draw(plotName, **kwargs)
elif len(item) == 7:
objName, alarmName, errMin, warnMin, warnMax, errMax,\
kwargs = item
self.alarm(objName, alarmName, errMin, warnMin, warnMax, errMax,
**kwargs)
self.createReport()
self.createSummary()
def fill(self):
""" Fill the report.
"""
if self.__OutputFolder is not None:
__utils__.createFolder(self.__OutputFolder)
# Plots from the header tree.
header = E3DstHeaderChain(self.__InputFilePath)
numRuns = header.GetEntries()
numEvents = 0
numHitEvents = 0
numTrackEvents = 0
for i in xrange(numRuns):
header.GetEntry(i)
numEvents += header.NumEvents
numHitEvents += header.NumHitEvents
numTrackEvents += header.NumTrackEvents
self.__StatDict['num_runs'] = numRuns
self.__StatDict['num_events'] = numEvents
self.__StatDict['num_hit_events'] = numHitEvents
self.__StatDict['num_track_events'] = numTrackEvents
header.setupArrays()
header.setupTreeFormulae()
header.doSummaryPlots()
for _plot in header.plots():
_canvas = E3Canvas(self.canvasName(_plot.GetName()))
_plot.Draw()
_canvas.annotate(0.1, 0.94, self.__Label)
_canvas.Update()
if self.__OutputFolder is not None:
_canvas.save(self.__OutputFolder)
self.__ObjectList.append(_plot.GetName())
self.__ObjectList.append(None)
# Plots from the trending tree.
trending = E3DstTrendingChain(self.__InputFilePath)
trending.setupArrays()
trending.setupTreeFormulae()
trending.doSummaryPlots()
for _plot in trending.plots():
_canvas = E3Canvas(self.canvasName(_plot.GetName()))
_plot.Draw('al')
_canvas.annotate(0.1, 0.94, self.__Label)
_canvas.Update()
if self.__OutputFolder is not None:
_canvas.save(self.__OutputFolder)
self.__ObjectList.append(_plot.GetName())
# And now the summary plots.
_canvas = E3Canvas(self.canvasName('RateSummary'))
g1 = header.plot('AverageRate')
g2 = trending.plot('RateHitEvents')
g3 = trending.plot('RateTrackEvents')
ymax = max([g1.GetY()[i] for i in range(g1.GetN())])
ymin = min([g3.GetY()[i] for i in range(g3.GetN())])
yrange = ymax - ymin
ymax += yrange
ymin -= yrange
if ymin < 0:
ymin = 0.
g1.GetYaxis().SetRangeUser(ymin, ymax)
g1.GetYaxis().SetTitle('Rate [Hz]')
g2.SetLineColor(ROOT.kBlue)
g3.SetLineColor(ROOT.kRed)
g1.Draw('al')
g2.Draw('lsame')
g3.Draw('lsame')
legend = E3Legend(entries = [g1, g2, g3])
legend.Draw()
_canvas.annotate(0.1, 0.94, self.__Label)
_canvas.Update()
if self.__OutputFolder is not None:
_canvas.save(self.__OutputFolder)
_canvas = E3Canvas(self.canvasName('WeatherSummary'))
_canvas.SetRightMargin(0.15)
_canvas.SetTicky(0)
# Weather quantities.
weather = E3DstWeatherChain(self.__InputFilePath)
g1 = weather.indoorTemperature()
g2 = weather.outdoorTemperature()
g3 = weather.pressure()
store(g1)
store(g2)
store(g3)
ymax = max([g1.GetY()[i] for i in range(g1.GetN())] +\
[g2.GetY()[i] for i in range(g2.GetN())])
ymin = min([g1.GetY()[i] for i in range(g1.GetN())] +\
[g2.GetY()[i] for i in range(g2.GetN())])
yrange = ymax - ymin
if yrange == 0:
yrange = 5.
ymax += yrange
ymin -= yrange
if ymin < 0:
ymin = 0.
yrange = ymax - ymin
g1.GetYaxis().SetRangeUser(ymin, ymax)
g1.GetYaxis().SetTitle('Temperature [#circ C]')
g1.SetLineColor(ROOT.kRed)
g2.SetLineColor(ROOT.kBlue)
setupTimeDisplay(g1)
g1.Draw('al')
g2.Draw('lsame')
# Now put the pressure, with a right axis.
pmin = min([g3.GetY()[i] for i in range(g3.GetN())])
pmax = max([g3.GetY()[i] for i in range(g3.GetN())])
pmin -= 20
pmax += 20
prange = pmax - pmin
xmax = g1.GetXaxis().GetXmax()
rightAxis = ROOT.TGaxis(xmax, ymin, xmax, ymax, pmin, pmax, 410, 'L+')
rightAxis.SetTitle('Pressure [hPa]')
rightAxis.SetLabelFont(TEXT_FONT)
rightAxis.SetLabelSize(LABEL_TEXT_SIZE)
rightAxis.SetTitleFont(TEXT_FONT)
rightAxis.SetTitleSize(TEXT_SIZE)
rightAxis.SetTitleOffset(1.2)
store(rightAxis)
rightAxis.Draw()
g4 = g3.Clone('PressureScaled')
store(g4)
g4.Draw('lsame')
x = ROOT.Double()
y = ROOT.Double()
for i in xrange(g4.GetN()):
g4.GetPoint(i, x, y)
_x = float(x)
_y = (float(y) - pmin)/prange*yrange + ymin
g4.SetPoint(i, _x, _y)
legend = E3Legend(entries = [g1, g2, g4])
legend.Draw()
_canvas.annotate(0.1, 0.94, self.__Label)
_canvas.Update()
if self.__OutputFolder is not None:
_canvas.save(self.__OutputFolder)
self.createReport()
self.createSummary()
try:
E3PIPE_DQM_BASE = os.environ['E3PIPE_DQM_BASE']
except KeyError:
E3PIPE_DQM_BASE = '/dqm'
""" Base folder for the output dqm reports.
TODO: need to ask Francesco to link /reports, here.
"""
try:
E3PIPE_DQM_REPORT_BASE = os.environ['E3PIPE_DQM_REPORT_BASE']
except KeyError:
E3PIPE_DQM_REPORT_BASE = '/dqmreport'
""" Base folder for the log files.
"""
E3PIPE_LOG_BASE = os.path.expanduser(os.path.join('~', 'eeelog'))
__utils__.createFolder(E3PIPE_LOG_BASE)
""" Base folder to keep track of the failed runs.
"""
try:
E3PIPE_LOCK_BASE = os.environ['E3PIPE_LOCK_BASE']
except KeyError:
E3PIPE_LOCK_BASE = os.path.expanduser(os.path.join('~', 'eeelock'))
__utils__.createFolder(E3PIPE_LOCK_BASE)
""" Temporary folder we run the reconstruction in.
Note this is user-specific, and we do cleanup the folder after the
reconstruction has run.
We also do create the folder on the fly (if it does not exists) when
the module is imported.
"""
""" Base folder for the output dqm reports.
TODO: need to ask Francesco to link /reports, here.
"""
try:
E3PIPE_DQM_REPORT_BASE = os.environ['E3PIPE_DQM_REPORT_BASE2']
except KeyError:
E3PIPE_DQM_REPORT_BASE = '/dqmreport2'
""" Base folder for the log files.
"""
E3PIPE_LOG_BASE = os.path.expanduser(os.path.join('~', 'eeelog2'))
__utils__.createFolder(E3PIPE_LOG_BASE)
""" Base folder to keep track of the failed runs.
"""
try:
E3PIPE_LOCK_BASE = os.environ['E3PIPE_LOCK_BASE']
except KeyError:
E3PIPE_LOCK_BASE = os.path.expanduser(os.path.join('~', 'eeelock2'))
__utils__.createFolder(E3PIPE_LOCK_BASE)
""" Temporary folder we run the reconstruction in.
Note this is user-specific, and we do cleanup the folder after the
reconstruction has run.