def __init__(self, filePath, maxRecords = 4):
""" Constructor.
"""
self.__WeatherStationRecord = None
self.__GeometryRecord = None
logger.info('Opening input binary file %s...' % filePath)
file.__init__(self, filePath, 'rb')
self.__LastByteRead = None
self.readWord(self.HEADER_EEE_1)
self.readWord(self.HEADER_EEE_2)
for i in xrange(maxRecords):
try:
blockType, blockSize, data = self.readBlock()
except:
logger.error('Could not parse file content, giving up.')
break
if blockType == self.HEADER_GEO_RECORD:
self.__GeometryRecord = E3GeometryRecord(*data)
logger.info('Geometry record found!')
logger.info(self.__GeometryRecord)
elif blockType == self.HEADER_VWS_RECORD:
self.__WeatherStationRecord = E3WeatherStationRecord(*data)
logger.info('Weather station record found!')
logger.info(self.__WeatherStationRecord)
break
if self.__WeatherStationRecord is None:
logger.info('Could not find any weather station record.')
def __init__(self, filePath, maxRecords=4):
""" Constructor.
"""
self.__WeatherStationRecord = None
self.__GeometryRecord = None
logger.info('Opening input binary file %s...' % filePath)
file.__init__(self, filePath, 'rb')
self.__LastByteRead = None
self.readWord(self.HEADER_EEE_1)
self.readWord(self.HEADER_EEE_2)
for i in xrange(maxRecords):
try:
blockType, blockSize, data = self.readBlock()
except:
logger.error('Could not parse file content, giving up.')
break
if blockType == self.HEADER_GEO_RECORD:
self.__GeometryRecord = E3GeometryRecord(*data)
logger.info('Geometry record found!')
logger.info(self.__GeometryRecord)
elif blockType == self.HEADER_VWS_RECORD:
self.__WeatherStationRecord = E3WeatherStationRecord(*data)
logger.info('Weather station record found!')
logger.info(self.__WeatherStationRecord)
break
if self.__WeatherStationRecord is None:
logger.info('Could not find any weather station record.')
def readWord(self, expected=None):
""" Read a byte from the binary file.
If the expected argument is not None, the value which is read out is
checked against that.
"""
self.__LastByteRead = struct.unpack('i', self.read(4))[0]
if expected is not None and self.__LastByteRead != expected:
logger.error('Data mismatch (expected %d, found %d)' %\
(expected, self.__LastByteRead))
return self.__LastByteRead
def execute(self, query, commit=True):
""" Execute a query.
"""
logger.info('About to execute "%s"...' % query)
if commit:
try:
self.__Cursor.execute(query)
self.commit()
except Exception, e:
logger.error('%s, rolling back change...' % e)
self.rollback()
def readWord(self, expected = None):
""" Read a byte from the binary file.
If the expected argument is not None, the value which is read out is
checked against that.
"""
self.__LastByteRead = struct.unpack('i', self.read(4))[0]
if expected is not None and self.__LastByteRead != expected:
logger.error('Data mismatch (expected %d, found %d)' %\
(expected, self.__LastByteRead))
return self.__LastByteRead
def execute(self, query, commit = True):
""" Execute a query.
"""
logger.info('About to execute "%s"...' % query)
if commit:
try:
self.__Cursor.execute(query)
self.commit()
except Exception, e:
logger.error('%s, rolling back change...' % e)
self.rollback()
def rmdir(folderPath):
""" Remove an entire (empty or non empty) folder.
"""
logger.info('About to remove folder %s...' % folderPath)
try:
shutil.rmtree(folderPath)
logger.info('Folder succesfully removed.')
status = 0
except Exception as e:
logger.error('Could not remove folder (%s)' % e)
status = 1
return status
def rmdir(folderPath):
""" Remove an entire (empty or non empty) folder.
"""
logger.info('About to remove folder %s...' % folderPath)
try:
shutil.rmtree(folderPath)
logger.info('Folder succesfully removed.')
status = 0
except Exception as e:
logger.error('Could not remove folder (%s)' % e)
status = 1
return status
def addFilePath(self, station, filePath):
""" Add a file path for a specific station.
"""
if not os.path.exists(filePath):
logger.error('Could not find %s, file will not be added.' %\
filePath)
return
if station not in self.__Stations:
self.__Stations.append(station)
self.__FileDict[station] = []
self.__FileDict[station].append(filePath)
self.__FileList.append(filePath)
def scanGPS(self, maxRecords=10):
"""
"""
numRecords = 0
while numRecords < maxRecords:
try:
blockType, blockSize, data = self.readBlock()
except:
logger.error('Could not parse file content, giving up.')
break
if blockType == self.HEADER_GPS_RECORD:
print numRecords, E3GpsRecord(*data)
numRecords += 1
def scanGPS(self, maxRecords = 10):
"""
"""
numRecords = 0
while numRecords < maxRecords:
try:
blockType, blockSize, data = self.readBlock()
except:
logger.error('Could not parse file content, giving up.')
break
if blockType == self.HEADER_GPS_RECORD:
print numRecords, E3GpsRecord(*data)
numRecords += 1
def mv(source, dest):
""" Move a file.
Return 0 upon succesfull operation, 1 otherwise.
"""
logger.info('About to move %s to %s...' % (source, dest))
try:
shutil.move(source, dest)
logger.info('File succesfully copied.')
status = 0
except Exception as e:
logger.error('Could not move file (%s)' % e)
status = 1
return status
def mv(source, dest):
""" Move a file.
Return 0 upon succesfull operation, 1 otherwise.
"""
logger.info('About to move %s to %s...' % (source, dest))
try:
shutil.move(source, dest)
logger.info('File succesfully copied.')
status = 0
except Exception as e:
logger.error('Could not move file (%s)' % e)
status = 1
return status
def createFolder(folderPath):
""" Create a folder (unless it already exists).
Return 0 upon succesfull operation, 1 otherwise.
"""
if not os.path.exists(folderPath):
logger.info('About to create folder %s...' % folderPath)
try:
os.makedirs(folderPath)
logger.info('Folder succesfully created.')
status = 0
except Exception as e:
logger.error('Could not create folder (%s)' % e)
status = 1
return status
def createFolder(folderPath):
""" Create a folder (unless it already exists).
Return 0 upon succesfull operation, 1 otherwise.
"""
if not os.path.exists(folderPath):
logger.info('About to create folder %s...' % folderPath)
try:
os.makedirs(folderPath)
logger.info('Folder succesfully created.')
status = 0
except Exception as e:
logger.error('Could not create folder (%s)' % e)
status = 1
return status
def rm(filePath):
""" Remove a file.
Return 0 upon succesfull operation, 1 otherwise.
"""
logger.info('About to remove file %s...' % filePath)
if not os.path.exists(filePath):
logger.info('File is not there, giving up...')
return 0
try:
os.remove(filePath)
logger.info('File succesfully removed.')
status = 0
except Exception as e:
logger.error('Could not remove file (%s)' % e)
status = 1
return status
def rm(filePath):
""" Remove a file.
Return 0 upon succesfull operation, 1 otherwise.
"""
logger.info('About to remove file %s...' % filePath)
if not os.path.exists(filePath):
logger.info('File is not there, giving up...')
return 0
try:
os.remove(filePath)
logger.info('File succesfully removed.')
status = 0
except Exception as e:
logger.error('Could not remove file (%s)' % e)
status = 1
return status
def cmd(cmd, verbose = False, logFilePath = None, logFileMode = 'w',
dryRun = False):
""" Exec a command.
This uses subprocess internally and returns the subprocess status code
(if the dryRun option is true the function will just print the command out
through the logger and returns happily).
By default the stdout and the stderr are redirected into subprocess pipes
so that the output can be effectively used by the logger. It the logFilePath
parameter is different from None the stdout is redirected to file instead.
The rules are:
(*) if verbose is True the command output is logged onto the terminal one
line at a time;
(*) if the status code is zero we just aknowledge that before returning it;
(*) upon error we try and log out both the error code and the error message.
"""
logger.info('About to execute "%s"...' % cmd)
if dryRun:
logger.info('Just kidding (dry run).')
return 0
err = subprocess.PIPE
if logFilePath is not None:
out = open(logFilePath, logFileMode)
else:
out = subprocess.PIPE
process = subprocess.Popen(cmd, stdout = out, stderr = err, shell = True)
errorCode = process.wait()
if verbose:
if logFilePath is None:
output = process.stdout.read().strip('\n')
else:
output = open(logFilePath).read().strip('\n')
for line in output.split('\n'):
logger.info(line)
if not errorCode:
logger.info('Command executed with status code %d.' % errorCode)
else:
logger.error('Command returned status code %d.' % errorCode)
logger.error('Full error message following...\n%s' %\
process.stderr.read().strip('\n'))
return errorCode
def cp(source, dest, createTree=False):
""" Copy a file.
Return 0 upon succesfull operation, 1 otherwise.
"""
logger.info('About to copy %s to %s...' % (source, dest))
destFolder = os.path.dirname(dest)
if not os.path.exists(destFolder) and createTree:
createFolder(destFolder)
try:
if os.path.isdir(source):
shutil.copytree(source, dest)
else:
shutil.copy(source, dest)
logger.info('File succesfully copied.')
status = 0
except Exception as e:
logger.error('Could not copy file (%s)' % e)
status = 1
return status
def cp(source, dest, createTree = False):
""" Copy a file.
Return 0 upon succesfull operation, 1 otherwise.
"""
logger.info('About to copy %s to %s...' % (source, dest))
destFolder = os.path.dirname(dest)
if not os.path.exists(destFolder) and createTree:
createFolder(destFolder)
try:
if os.path.isdir(source):
shutil.copytree(source, dest)
else:
shutil.copy(source, dest)
logger.info('File succesfully copied.')
status = 0
except Exception as e:
logger.error('Could not copy file (%s)' % e)
status = 1
return status
def cmd(cmd, verbose=False, logFilePath=None, logFileMode='w', dryRun=False):
""" Exec a command.
This uses subprocess internally and returns the subprocess status code
(if the dryRun option is true the function will just print the command out
through the logger and returns happily).
By default the stdout and the stderr are redirected into subprocess pipes
so that the output can be effectively used by the logger. It the logFilePath
parameter is different from None the stdout is redirected to file instead.
The rules are:
(*) if verbose is True the command output is logged onto the terminal one
line at a time;
(*) if the status code is zero we just aknowledge that before returning it;
(*) upon error we try and log out both the error code and the error message.
"""
logger.info('About to execute "%s"...' % cmd)
if dryRun:
logger.info('Just kidding (dry run).')
return 0
err = subprocess.PIPE
if logFilePath is not None:
out = open(logFilePath, logFileMode)
else:
out = subprocess.PIPE
process = subprocess.Popen(cmd, stdout=out, stderr=err, shell=True)
errorCode = process.wait()
if verbose:
if logFilePath is None:
output = process.stdout.read().strip('\n')
else:
output = open(logFilePath).read().strip('\n')
for line in output.split('\n'):
logger.info(line)
if not errorCode:
logger.info('Command executed with status code %d.' % errorCode)
else:
logger.error('Command returned status code %d.' % errorCode)
logger.error('Full error message following...\n%s' %\
process.stderr.read().strip('\n'))
return errorCode
def e3failureStats(exitCode):
"""
"""
db = E3RunDbInterface()
query = 'SELECT station_name, run_date, run_id from runs WHERE processing_status_code = %d' % exitCode
db.execute(query, commit = False)
outputFilePath = 'failures_%d.dat' % exitCode
logger.info('Opening output file %s...' % outputFilePath)
outputFile = open(outputFilePath, 'w')
n = 0
for station, date, runId in db.fetchall():
filePath = binFilePath(station, date, runId)
if os.path.exists(filePath):
outputFile.write('%s\t%d\n' %\
(filePath, os.stat(filePath).st_size))
n += 1
else:
logger.error('Could not find %s!' % filePath)
outputFile.close()
logger.info('Done, %d rows written.' % n)
db.close()
def e3failureStats(exitCode):
"""
"""
db = E3RunDbInterface()
query = 'SELECT station_name, run_date, run_id from runs WHERE processing_status_code = %d' % exitCode
db.execute(query, commit=False)
outputFilePath = 'failures_%d.dat' % exitCode
logger.info('Opening output file %s...' % outputFilePath)
outputFile = open(outputFilePath, 'w')
n = 0
for station, date, runId in db.fetchall():
filePath = binFilePath(station, date, runId)
if os.path.exists(filePath):
outputFile.write('%s\t%d\n' %\
(filePath, os.stat(filePath).st_size))
n += 1
else:
logger.error('Could not find %s!' % filePath)
outputFile.close()
logger.info('Done, %d rows written.' % n)
db.close()
def e3dst(baseFilePath):
""" Parse all the output text files from the analyzer and build the
actual DST in ROOT format.
"""
chrono = E3Chrono()
logger.info('Collecting input files for the DST...')
outFile = E3AnalyzerOutFile('%s.out' % baseFilePath)
sumFile = E3AnalyzerSumFile('%s.sum' % baseFilePath)
dstFilePath = '%s_dst.root' % baseFilePath
uniqueId = uniqueRunIdFromFilePath(baseFilePath)
station, year, month, day, runId = splitFilePath(baseFilePath)
date = datetime.date(int(year), int(month), int(day))
logger.info('Unique run ID is %s.' % uniqueId)
logger.info('Opening output ROOT file %s...' % dstFilePath)
rootFile = E3OutputRootFile(dstFilePath, 'e3dst', date2str(date), station)
logger.info('Initializing event tree...')
eventTree = E3DstEventTree()
eventTree.setUniqueRunId(uniqueId)
logger.info('Filling event tree...')
for row in outFile:
eventTree.fillRow(row)
eventStat = outFile.eventStat()
tmin = outFile.minTimestamp()
tmax = outFile.maxTimestamp()
# If we have less than two good events there is nothing we could
# possibly do, here.
# Close all files and remove the output dst ROOT file so that
# we know the run has not been processed.
if eventStat['hits'] < 2:
logger.info('Closing all files...')
rootFile.Close()
outFile.close()
sumFile.close()
__utils__.rm(dstFilePath)
logger.info('No events with hits, processing terminated after %.3f s.' %\
chrono.stop())
sys.exit(E3PIPE_EXIT_CODE_NO_HITS_EVENTS)
logger.info('Event stats: %s' % eventStat)
logger.info('Range of timestamps in the output files: %.3f--%.3f' %\
(tmin, tmax))
duration = tmax - tmin
logger.info('Corresponding run duration: %.3f s' % duration)
if duration > MAX_RUN_DURATION:
logger.error('Run looks way too long, something must be wrong.')
sys.exit(E3PIPE_EXIT_CODE_RUN_TOO_LONG)
eventTree.Write()
logger.info('Done, %d event(s) filled in.' % eventTree.GetEntries())
if eventTree.GetEntries() == 0:
abort('No events found (maybe an issue with eee_calib.txt?)')
logger.info('Creating monitoring plots...')
eventTree.doMonitoring()
logger.info('Initializing weather tree...')
weatherTree = E3DstWeatherTree()
weatherTree.setUniqueRunId(uniqueId)
binFile = E3BinFile('%s.bin' % baseFilePath)
weatherRecord = binFile.weatherStationRecord()
if weatherRecord is not None:
logger.info('Filling weather tree...')
weatherTree.fillRow(weatherRecord.data())
weatherTree.Write()
logger.info('Creating trending data products...')
trendingTree = eventTree.doTrending(TRENDING_TIME_BIN, tmin, tmax,
weatherRecord)
logger.info('Writing trending tree...')
trendingTree.Write()
logger.info('Writing monitoring/trending plots...')
for plot in eventTree.plots():
plot.Write()
logger.info('Initializing header tree...')
headerTree = E3DstHeaderTree()
headerTree.setUniqueRunId(uniqueId)
logger.info('Filling header tree...')
data = sumFile.data()
# Mind we need to add a few things "by hand", here, as not all the
# information that we want in the header is really coming from the
# sum file.
data['RunNumber'] = row['RunNumber']
data['RunStart'] = tmin
data['RunStop'] = tmax
data['RunDuration'] = tmax - tmin
data['NumHitEvents'] = eventStat['hits']
data['NumTrackEvents'] = eventStat['track']
data['NumNoHitsEvents'] = eventStat['no_hits']
data['NumNoHitEvents'] = eventStat['no_hit']
data['NumMalformedEvents'] = eventStat['malformed']
data['NumBackwardEvents'] = eventStat['backward']
headerTree.fillRow(data)
headerTree.Write()
logger.info('Creating histograms...')
for key in [
'HitMultBot', 'HitMultMid', 'HitMultTop', 'ClusterMultBot',
'ClusterMultMid', 'ClusterMultTop'
]:
h = data2hist(data, key, xmax=15.5)
h.Write()
for key in ['HitMultTotal', 'ClusterMultTotal']:
h = data2hist(data, key, xmax=35.5)
h.Write()
logger.info('Closing all files...')
rootFile.Close()
outFile.close()
sumFile.close()
logger.info('DST created in %.3f s.' % chrono.stop())
listTemp()
logger.info('Returning DST path: %s...' % dstFilePath)
return dstFilePath
def e3dst(baseFilePath):
""" Parse all the output text files from the analyzer and build the
actual DST in ROOT format.
"""
chrono = E3Chrono()
logger.info('Collecting input files for the DST...')
outFile = E3AnalyzerOutFile('%s.out' % baseFilePath)
sumFile = E3AnalyzerSumFile('%s.sum' % baseFilePath)
dstFilePath = '%s_dst.root' % baseFilePath
uniqueId = uniqueRunIdFromFilePath(baseFilePath)
station, year, month, day, runId = splitFilePath(baseFilePath)
date = datetime.date(int(year), int(month), int(day))
logger.info('Unique run ID is %s.' % uniqueId)
logger.info('Opening output ROOT file %s...' % dstFilePath)
rootFile = E3OutputRootFile(dstFilePath, 'e3dst', date2str(date), station)
logger.info('Initializing event tree...')
eventTree = E3DstEventTree()
eventTree.setUniqueRunId(uniqueId)
logger.info('Filling event tree...')
for row in outFile:
eventTree.fillRow(row)
eventStat = outFile.eventStat()
tmin = outFile.minTimestamp()
tmax = outFile.maxTimestamp()
# If we have less than two good events there is nothing we could
# possibly do, here.
# Close all files and remove the output dst ROOT file so that
# we know the run has not been processed.
if eventStat['hits'] < 2:
logger.info('Closing all files...')
rootFile.Close()
outFile.close()
sumFile.close()
__utils__.rm(dstFilePath)
logger.info('No events with hits, processing terminated after %.3f s.' %\
chrono.stop())
sys.exit(E3PIPE_EXIT_CODE_NO_HITS_EVENTS)
logger.info('Event stats: %s' % eventStat)
logger.info('Range of timestamps in the output files: %.3f--%.3f' %\
(tmin, tmax))
duration = tmax - tmin
logger.info('Corresponding run duration: %.3f s' % duration)
if duration > MAX_RUN_DURATION:
logger.error('Run looks way too long, something must be wrong.')
sys.exit(E3PIPE_EXIT_CODE_RUN_TOO_LONG)
eventTree.Write()
logger.info('Done, %d event(s) filled in.' % eventTree.GetEntries())
if eventTree.GetEntries() == 0:
abort('No events found (maybe an issue with eee_calib.txt?)')
logger.info('Creating monitoring plots...')
eventTree.doMonitoring()
logger.info('Initializing weather tree...')
weatherTree = E3DstWeatherTree()
weatherTree.setUniqueRunId(uniqueId)
binFile = E3BinFile('%s.bin' % baseFilePath)
weatherRecord = binFile.weatherStationRecord()
if weatherRecord is not None:
logger.info('Filling weather tree...')
weatherTree.fillRow(weatherRecord.data())
weatherTree.Write()
logger.info('Creating trending data products...')
trendingTree = eventTree.doTrending(TRENDING_TIME_BIN, tmin, tmax,
weatherRecord)
logger.info('Writing trending tree...')
trendingTree.Write()
logger.info('Writing monitoring/trending plots...')
for plot in eventTree.plots():
plot.Write()
logger.info('Initializing header tree...')
headerTree = E3DstHeaderTree()
headerTree.setUniqueRunId(uniqueId)
logger.info('Filling header tree...')
data = sumFile.data()
# Mind we need to add a few things "by hand", here, as not all the
# information that we want in the header is really coming from the
# sum file.
data['RunNumber'] = row['RunNumber']
data['RunStart'] = tmin
data['RunStop'] = tmax
data['RunDuration'] = tmax - tmin
data['NumHitEvents'] = eventStat['hits']
data['NumTrackEvents'] = eventStat['track']
data['NumNoHitsEvents'] = eventStat['no_hits']
data['NumNoHitEvents'] = eventStat['no_hit']
data['NumMalformedEvents'] = eventStat['malformed']
data['NumBackwardEvents'] = eventStat['backward']
headerTree.fillRow(data)
headerTree.Write()
logger.info('Creating histograms...')
for key in ['HitMultBot', 'HitMultMid', 'HitMultTop',
'ClusterMultBot', 'ClusterMultMid', 'ClusterMultTop']:
h = data2hist(data, key, xmax = 15.5)
h.Write()
for key in ['HitMultTotal', 'ClusterMultTotal']:
h = data2hist(data, key, xmax = 35.5)
h.Write()
logger.info('Closing all files...')
rootFile.Close()
outFile.close()
sumFile.close()
logger.info('DST created in %.3f s.' % chrono.stop())
listTemp()
logger.info('Returning DST path: %s...' % dstFilePath)
return dstFilePath
def hist2d(self, expression, cut = '', **kwargs):
""" Create a 2-dimensional histogram.
"""
logger.error('Two-dimensional histograms not implemented, yet.')
_usage = 'usage: %prog [options] binfile'
_synopsis = 'Run the full e3recon in a shell and, if needed, register the results in the database'
# Set up the command-line switches.
from e3pipe.misc.E3OptionParser import E3OptionParser
parser = E3OptionParser(_usage, _synopsis)
(opts, args) = parser.parse_args()
# Make sure we are passing some argument.
if not len(args) == 1:
parser.print_help()
parser.error('Please provide a single input file.')
rawFilePath = args[0]
# Print the start message.
from e3pipe.__logging__ import startmsg
startmsg()
# Run e3recon
chrono = E3Chrono()
_cmd = 'e3recon.py %s' % rawFilePath
exitCode = __utils__.cmd(_cmd)
if exitCode == 0:
logger.info('Run processed in %.3f s.' % chrono.stop())
else:
logger.error('Processing terminated with exit code %d after %.3f s.' %\
(exitCode, chrono.stop()))
if doDbRegister():
runInfo = E3RawDataInfo(rawFilePath)
registerRun(runInfo, exitCode)
def hist2d(self, expression, cut='', **kwargs):
""" Create a 2-dimensional histogram.
"""
logger.error('Two-dimensional histograms not implemented, yet.')
_usage = 'usage: %prog [options] binfile'
_synopsis = 'Run the full e3recon in a shell and, if needed, register the results in the database'
# Set up the command-line switches.
from e3pipe.misc.E3OptionParser import E3OptionParser
parser = E3OptionParser(_usage, _synopsis)
(opts, args) = parser.parse_args()
# Make sure we are passing some argument.
if not len(args) == 1:
parser.print_help()
parser.error('Please provide a single input file.')
rawFilePath = args[0]
# Print the start message.
from e3pipe.__logging__ import startmsg
startmsg()
# Run e3recon
chrono = E3Chrono()
_cmd = 'e3recon2.py %s' % rawFilePath
exitCode = __utils__.cmd(_cmd)
if exitCode == 0:
logger.info('Run processed in %.3f s.' % chrono.stop())
else:
logger.error('Processing terminated with exit code %d after %.3f s.' %\
(exitCode, chrono.stop()))
if doDbRegister():
runInfo = E3RawDataInfo(rawFilePath)
registerRun(runInfo, exitCode)
