def printProxiesInAllStreams(streamProxyList):
"""Print proxies in all streams"""
s = ""
for x in streamProxyList:
s = s+"\n===> List of proxies in '%s'\n"%x[0]
s = s+pds_utils.printProxies(x[1])
return s
def printProxiesInAllStreams(streamProxyList):
"""Print proxies in all streams"""
s = ""
for x in streamProxyList:
s = s + "\n===> List of proxies in '%s'\n" % x[0]
s = s + pds_utils.printProxies(x[1])
return s
def dump(fileName, verbose=1):
"""Dump content of location file"""
sNames, pdsIDList, oDict, hash, dataKeysInStreams, recordSize, posOfFirstRecord, needToSwap = locationFileParser(
fileName)
if needToSwap:
print "File was produced on another endian machine, byte swapping is enabled"
print "File identifiers:",
for x in pdsIDList:
print "%d" % x,
print
print "StreamNames :",
for x in sNames:
print x,
print
print "File hash :",
for tByte in hash:
print tByte,
print "\n"
for idx in xrange(0, len(sNames)):
print "List of data keys in stream:", sNames[idx]
print pds_utils.printProxies(dataKeysInStreams[idx])
locFile = open(fileName, 'rb')
locFile.seek(posOfFirstRecord)
nFiles = len(pdsIDList)
count = 0
while 1:
try:
fOffsetList, streamIdx, dataKeyList = decodeLocationRecord(
locFile, needToSwap, recordSize, nFiles)
if verbose:
print "************"
print "# syncValue:", count,
if len(sNames):
print sNames[streamIdx]
else:
print "\n"
for index in xrange(0, len(pdsIDList)):
fileId = pdsIDList[index]
print "fileId " + str(fileId) + " at offset " + str(
fOffsetList[index * 2]) + " " + str(
fOffsetList[index * 2 + 1])
print dataKeyList
count += 1
except EOFError:
break
def dump(fileName,verbose=1):
"""Dump content of location file"""
sNames,pdsIDList,oDict,hash,dataKeysInStreams,recordSize,posOfFirstRecord,needToSwap=locationFileParser(fileName)
if needToSwap:
print "File was produced on another endian machine, byte swapping is enabled"
print "File identifiers:",
for x in pdsIDList: print "%d"%x,
print
print "StreamNames :",
for x in sNames: print x,
print
print "File hash :",
for tByte in hash:
print tByte,
print "\n"
for idx in xrange(0,len(sNames)):
print "List of data keys in stream:",sNames[idx]
print pds_utils.printProxies(dataKeysInStreams[idx])
locFile = open(fileName,'rb')
locFile.seek(posOfFirstRecord)
nFiles = len(pdsIDList)
count = 0
while 1:
try:
fOffsetList,streamIdx,dataKeyList=decodeLocationRecord(locFile,needToSwap,recordSize,nFiles)
if verbose:
print "************"
print "# syncValue:",count,
if len(sNames):
print sNames[streamIdx]
else:
print "\n"
for index in xrange(0,len(pdsIDList)):
fileId = pdsIDList[index]
print "fileId "+str(fileId)+" at offset "+str(fOffsetList[index*2])+" "+str(fOffsetList[index*2+1])
print dataKeyList
count+=1
except EOFError:
break
def dump_old(fileName, verbose=1):
"""Dump content of pds location file to stdout"""
# define signature of location file
global PDSSIGNATURE
pds_utils.PDSSIGNATURE = PDSSIGNATURE
fileDesc = open(fileName, 'rb')
headerHeader = array.array('I')
headerHeader.fromfile(fileDesc, 3)
global pdsSignature
pdsSignature = headerHeader[0] >> 8
pds_utils.pdsSignature = pdsSignature
if pdsSignature != PDSSIGNATURE:
print "File was produced on another endian machine, byte swapping is enabled"
# print "File signature",pdsSignature,"and it should be ",PDSSIGNATURE
if pdsSignature != PDSSIGNATURE: headerHeader.byteswap()
nFiles = array.array('I')
nFiles.fromfile(fileDesc, 1)
if pdsSignature != PDSSIGNATURE: nFiles.byteswap()
# print "Number of file identifiers:",nFiles
fileIDs = array.array('I')
fileIDs.fromfile(fileDesc, 2 * nFiles[0])
if pdsSignature != PDSSIGNATURE: fileIDs.byteswap()
# print "fileIDs",fileIDs
listOfFileIDs = []
for i in xrange(0, len(fileIDs), 2):
lowerMostId = fileIDs[i]
upperMostId = fileIDs[i + 1]
fileID = ((upperMostId << 32) | lowerMostId)
# print "fileID",lowerMostId,upperMostId,fileID
listOfFileIDs.append(fileID)
print "File identifiers:",
for x in listOfFileIDs:
print "%d" % x,
print
streamNames = pds_utils.name_list_from_file(fileDesc)
print "StreamNames :",
for x in streamNames:
print x,
print
# grab the hash
hash = array.array('I')
hash.fromfile(fileDesc, 5)
print "File hash :",
for tByte in hash:
print tByte,
# print hex(tByte), tByte
print "\n"
# grab the rest of the header
restOfHeader = array.array('I')
# the header size ignores the first 3 words in the event
restOfHeader.fromfile(fileDesc, headerHeader[2] - fileDesc.tell() / 4 + 3)
if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
# print restOfHeader
if restOfHeader[-1] != headerHeader[2]:
raise "header inconsistent"
dataKeysInStreams = find_datakeys_in_streams(restOfHeader[:-1],
streamNames)
for idx in xrange(0, len(streamNames)):
print "List of data keys in stream:", streamNames[idx]
print pds_utils.printProxies(dataKeysInStreams[idx])
# print dataKeysInStreams
maxDataKeys = 0
for dataKeys in dataKeysInStreams:
if maxDataKeys < len(dataKeys):
maxDataKeys = len(dataKeys)
# determine size of a record
nWordsForFileIDsInRecord = nFiles[0] * 2
nCharsForDataKeysInRecord = maxDataKeys
while ((nCharsForDataKeysInRecord + 4) % 8):
nCharsForDataKeysInRecord += 1
if verbose:
print "nWordsForFileIDsInRecord : ", nWordsForFileIDsInRecord
print " maxDataKeys : ", maxDataKeys
print "nCharsForDataKeysInRecord: ", nCharsForDataKeysInRecord
count = 0
while 1:
try:
endReason = "End Of File"
fileOffsets = array.array('I')
fileOffsets.fromfile(fileDesc, nWordsForFileIDsInRecord)
if pdsSignature != PDSSIGNATURE: fileOffsets.byteswap()
recordType = array.array('I')
recordType.fromfile(fileDesc, 1)
if pdsSignature != PDSSIGNATURE: recordType.byteswap()
endReason = "bad Record: fileIDS"
if verbose:
print "********"
print "# syncValue:", count
if len(streamNames):
print streamNames[recordType[0]]
for index in xrange(0, nFiles[0]):
print "file: " + str(fileIDs[index * 2]) + " " + str(
fileIDs[index * 2 + 1]) + "->" + str(
fileOffsets[index * 2]) + " " + str(
fileOffsets[index * 2 + 1])
dataKeys = array.array('B')
dataKeys.fromfile(fileDesc, nCharsForDataKeysInRecord)
if pdsSignature != PDSSIGNATURE: dataKeys.byteswap()
if verbose: print dataKeys
#for index in xrange(0,nCharsForDataKeysInRecord):
# print " "+str(index)+" "+str(dataKeys[index])
count += 1
except EOFError:
if verbose: print endReason
break
def dump(fileName, verbose=0, iStream='event'):
"""Dump content of pds file to stdout"""
svName, parents, verList, verDict = decodeVersionInfo(fileName)
if not parents:
parents = 'N/A'
if svName:
print "Versioning information:"
print "-----------------------"
print "data version name: '%s'" % svName
print "list of parents :", parents
PDSSIGNATURE = 3141592 # magic number for PDS format, as it should be
pdsSignature = 0 # signature we read from given PDS file
fileDesc = open(fileName, 'rb')
headerHeader = array.array('I')
headerHeader.fromfile(fileDesc, 3)
# to handle endianess, read pdsSignature
pdsSignature = headerHeader[0] >> 8
pds_utils.pdsSignature = pdsSignature
pds_utils.PDSSIGNATURE = PDSSIGNATURE
if pdsSignature != PDSSIGNATURE:
print "File was produced on another endian machine, byte swapping is enabled"
if pdsSignature != PDSSIGNATURE: headerHeader.byteswap()
print
print "List of streams in a file:",
streamNames = pds_utils.name_list_from_file(fileDesc)
for x in streamNames:
print x,
print
shProxyNames = pds_utils.name_list_from_file(fileDesc)
shNames = pds_utils.name_list_from_file(fileDesc)
if verbose:
print "List of types that storage helper proxy factories have:"
for x in shProxyNames:
print x,
print "\n\n"
print "List of types that only have storage helpers"
for x in shNames:
print x,
print
# grab the rest of the header
restOfHeader = array.array('I')
# the header size ignores the first 3 words in the event
restOfHeader.fromfile(fileDesc, headerHeader[2] - fileDesc.tell() / 4 + 3)
if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
if restOfHeader[-1] != headerHeader[2]:
raise "header inconsistent"
proxiesInStreams = pds_utils.find_proxies_in_streams(
restOfHeader, streamNames, shProxyNames)
print
for idx in xrange(0, len(streamNames)):
print "List of data keys in stream:", streamNames[idx]
print pds_utils.printProxies(proxiesInStreams[idx])
# want to only look at stuff in given stream (default is event stream)
if not streamNames.count(iStream):
print "There is no stream %s in %s" % (iStream, fileName)
return
eventIndex = streamNames.index(iStream)
# create a structure to hold our info
eventProxies = proxiesInStreams[eventIndex]
accumulatedData = []
for proxy in eventProxies:
accumulatedData.append([0, 488888888, 0])
# start to read the rest of the file
numEvents = 0
firstSV = ""
lastSV = ""
while 1:
try:
endReason = ""
pos = fileDesc.tell()
recordHeader = array.array('I')
recordHeader.fromfile(fileDesc, 5)
if pdsSignature != PDSSIGNATURE: recordHeader.byteswap()
uid = ((recordHeader[4] << 32) | recordHeader[3])
if not len(firstSV):
firstSV = "%d/%d/%d" % (recordHeader[1], recordHeader[2], uid)
lastSV = "%d/%d/%d" % (recordHeader[1], recordHeader[2], uid)
if verbose:
print "********************"
print "position : %s" % pos
print "stream : " + streamNames[recordHeader[0]]
print "SyncValue : %s/%s/%s" % (recordHeader[1],
recordHeader[2], uid)
endReason = "bad Record: record size"
recordDataLength = array.array('I')
recordDataLength.fromfile(fileDesc, 1)
if pdsSignature != PDSSIGNATURE: recordDataLength.byteswap()
if verbose:
print "size of record data: " + str(recordDataLength[0])
endReason = "bad Record: record (type " + streamNames[
recordHeader[0]] + ") (size " + str(recordDataLength[0]) + ")"
begRecPos = fileDesc.tell()
recordData = array.array('I')
recordData.fromfile(fileDesc, recordDataLength[0])
if pdsSignature != PDSSIGNATURE: recordData.byteswap()
numEvents = numEvents + 1
index = 0
endIndex = len(recordData)
dataInfo = []
dataInfoSize = []
lengthOffset = 0
while index < endIndex - 1:
proxyIndex = int(recordData[index])
index = index + 1
dataSize = recordData[index]
index = index + int(dataSize) + 1
fullDataSize = dataSize + 2
strData = str(proxiesInStreams[recordHeader[0]][proxyIndex])
if lengthOffset < len(strData):
lengthOffset = len(strData)
dataInfo.append(strData)
dataInfoSize.append(fullDataSize)
# if verbose:
# print str(proxiesInStreams[recordHeader[0]][proxyIndex]) + " size "+str(fullDataSize)
if recordHeader[0] == eventIndex:
accumulatedData[proxyIndex][
0] = accumulatedData[proxyIndex][0] + fullDataSize
if accumulatedData[proxyIndex][1] > fullDataSize:
accumulatedData[proxyIndex][1] = fullDataSize
if accumulatedData[proxyIndex][2] < fullDataSize:
accumulatedData[proxyIndex][2] = fullDataSize
if verbose:
print
for idx in xrange(0, len(dataInfo)):
s = dataInfo[idx]
print "%s size %s" % (s.ljust(lengthOffset),
dataInfoSize[idx])
except EOFError:
if verbose and endReason:
print "########################################################"
print endReason
print "########################################################"
break
index = 0
if numEvents == 0:
raise "no events found"
if verbose:
print "######### Summary ##########"
print "Proxies in %s:" % iStream
for item in eventProxies:
print "%s : %i %f %i" % (item, accumulatedData[index][1],
float(accumulatedData[index][0]) /
float(numEvents),
accumulatedData[index][2])
index = index + 1
print "First sync value :", firstSV
print "Last sync value :", lastSV
print "Number of syncValues :", numEvents
print
def dump(fileName,verbose=0,iStream='event'):
"""Dump content of pds file to stdout"""
svName,parents,verList,verDict = decodeVersionInfo(fileName)
if not parents:
parents = 'N/A'
if svName:
print "Versioning information:"
print "-----------------------"
print "data version name: '%s'"%svName
print "list of parents :",parents
PDSSIGNATURE=3141592 # magic number for PDS format, as it should be
pdsSignature=0 # signature we read from given PDS file
fileDesc = open(fileName,'rb')
headerHeader = array.array('I')
headerHeader.fromfile(fileDesc, 3)
# to handle endianess, read pdsSignature
pdsSignature = headerHeader[0]>>8
pds_utils.pdsSignature=pdsSignature
pds_utils.PDSSIGNATURE=PDSSIGNATURE
if pdsSignature != PDSSIGNATURE:
print "File was produced on another endian machine, byte swapping is enabled"
if pdsSignature != PDSSIGNATURE: headerHeader.byteswap()
print
print "List of streams in a file:",
streamNames = pds_utils.name_list_from_file(fileDesc)
for x in streamNames: print x,
print
shProxyNames = pds_utils.name_list_from_file(fileDesc)
shNames = pds_utils.name_list_from_file(fileDesc)
if verbose:
print "List of types that storage helper proxy factories have:"
for x in shProxyNames: print x,
print "\n\n"
print "List of types that only have storage helpers"
for x in shNames: print x,
print
# grab the rest of the header
restOfHeader = array.array('I')
# the header size ignores the first 3 words in the event
restOfHeader.fromfile(fileDesc, headerHeader[2] -fileDesc.tell()/4 +3)
if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
if restOfHeader[-1] != headerHeader[2]:
raise "header inconsistent"
proxiesInStreams = pds_utils.find_proxies_in_streams( restOfHeader, streamNames, shProxyNames)
print
for idx in xrange(0,len(streamNames)):
print "List of data keys in stream:",streamNames[idx]
print pds_utils.printProxies(proxiesInStreams[idx])
# want to only look at stuff in given stream (default is event stream)
if not streamNames.count(iStream):
print "There is no stream %s in %s"%(iStream,fileName)
return
eventIndex = streamNames.index(iStream)
# create a structure to hold our info
eventProxies = proxiesInStreams[eventIndex]
accumulatedData = []
for proxy in eventProxies:
accumulatedData.append([0,488888888,0])
# start to read the rest of the file
numEvents = 0
firstSV=""
lastSV =""
while 1:
try:
endReason = ""
pos = fileDesc.tell()
recordHeader = array.array('I')
recordHeader.fromfile(fileDesc,5)
if pdsSignature != PDSSIGNATURE: recordHeader.byteswap()
uid = ((recordHeader[4]<<32)|recordHeader[3])
if not len(firstSV):
firstSV="%d/%d/%d"%(recordHeader[1],recordHeader[2],uid)
lastSV="%d/%d/%d"%(recordHeader[1],recordHeader[2],uid)
if verbose:
print "********************"
print "position : %s"%pos
print "stream : "+streamNames[recordHeader[0]]
print "SyncValue : %s/%s/%s"%(recordHeader[1],recordHeader[2],uid)
endReason = "bad Record: record size"
recordDataLength = array.array('I')
recordDataLength.fromfile(fileDesc,1)
if pdsSignature != PDSSIGNATURE: recordDataLength.byteswap()
if verbose:
print "size of record data: "+str(recordDataLength[0])
endReason = "bad Record: record (type "+streamNames[recordHeader[0]]+") (size " + str(recordDataLength[0]) +")"
begRecPos = fileDesc.tell()
recordData = array.array('I')
recordData.fromfile(fileDesc, recordDataLength[0])
if pdsSignature != PDSSIGNATURE: recordData.byteswap()
numEvents = numEvents+1
index = 0
endIndex = len(recordData)
dataInfo = []
dataInfoSize = []
lengthOffset = 0
while index < endIndex - 1:
proxyIndex = int(recordData[index])
index = index + 1
dataSize = recordData[index]
index = index + int(dataSize) + 1
fullDataSize = dataSize+2
strData = str(proxiesInStreams[recordHeader[0]][proxyIndex])
if lengthOffset<len(strData):
lengthOffset=len(strData)
dataInfo.append(strData)
dataInfoSize.append(fullDataSize)
# if verbose:
# print str(proxiesInStreams[recordHeader[0]][proxyIndex]) + " size "+str(fullDataSize)
if recordHeader[0] == eventIndex:
accumulatedData[proxyIndex][0] = accumulatedData[proxyIndex][0]+fullDataSize
if accumulatedData[proxyIndex][1] > fullDataSize:
accumulatedData[proxyIndex][1] = fullDataSize
if accumulatedData[proxyIndex][2] < fullDataSize:
accumulatedData[proxyIndex][2] = fullDataSize
if verbose:
print
for idx in xrange(0,len(dataInfo)):
s = dataInfo[idx]
print "%s size %s"%(s.ljust(lengthOffset),dataInfoSize[idx])
except EOFError:
if verbose and endReason:
print "########################################################"
print endReason
print "########################################################"
break
index = 0
if numEvents == 0:
raise "no events found"
if verbose:
print "######### Summary ##########"
print "Proxies in %s:"%iStream
for item in eventProxies:
print "%s : %i %f %i" % (item, accumulatedData[index][1], float(accumulatedData[index][0])/float(numEvents), accumulatedData[index][2])
index = index + 1
print "First sync value :",firstSV
print "Last sync value :",lastSV
print "Number of syncValues :",numEvents
print
def dump_old(fileName,verbose=1):
"""Dump content of pds location file to stdout"""
# define signature of location file
global PDSSIGNATURE
pds_utils.PDSSIGNATURE=PDSSIGNATURE
fileDesc = open(fileName,'rb')
headerHeader = array.array('I')
headerHeader.fromfile(fileDesc, 3)
global pdsSignature
pdsSignature=headerHeader[0]>>8
pds_utils.pdsSignature=pdsSignature
if pdsSignature != PDSSIGNATURE:
print "File was produced on another endian machine, byte swapping is enabled"
# print "File signature",pdsSignature,"and it should be ",PDSSIGNATURE
if pdsSignature != PDSSIGNATURE: headerHeader.byteswap()
nFiles = array.array('I')
nFiles.fromfile(fileDesc,1)
if pdsSignature != PDSSIGNATURE: nFiles.byteswap()
# print "Number of file identifiers:",nFiles
fileIDs = array.array('I')
fileIDs.fromfile(fileDesc,2*nFiles[0])
if pdsSignature != PDSSIGNATURE: fileIDs.byteswap()
# print "fileIDs",fileIDs
listOfFileIDs=[]
for i in xrange(0,len(fileIDs),2):
lowerMostId=fileIDs[i]
upperMostId=fileIDs[i+1]
fileID = ((upperMostId<<32)|lowerMostId)
# print "fileID",lowerMostId,upperMostId,fileID
listOfFileIDs.append(fileID)
print "File identifiers:",
for x in listOfFileIDs: print "%d"%x,
print
streamNames = pds_utils.name_list_from_file(fileDesc)
print "StreamNames :",
for x in streamNames: print x,
print
# grab the hash
hash = array.array('I')
hash.fromfile(fileDesc,5)
print "File hash :",
for tByte in hash:
print tByte,
# print hex(tByte), tByte
print "\n"
# grab the rest of the header
restOfHeader = array.array('I')
# the header size ignores the first 3 words in the event
restOfHeader.fromfile(fileDesc, headerHeader[2] -fileDesc.tell()/4 +3)
if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
# print restOfHeader
if restOfHeader[-1] != headerHeader[2]:
raise "header inconsistent"
dataKeysInStreams = find_datakeys_in_streams(restOfHeader[:-1], streamNames)
for idx in xrange(0,len(streamNames)):
print "List of data keys in stream:",streamNames[idx]
print pds_utils.printProxies(dataKeysInStreams[idx])
# print dataKeysInStreams
maxDataKeys = 0
for dataKeys in dataKeysInStreams:
if maxDataKeys < len(dataKeys):
maxDataKeys = len(dataKeys)
# determine size of a record
nWordsForFileIDsInRecord = nFiles[0]*2
nCharsForDataKeysInRecord = maxDataKeys
while ((nCharsForDataKeysInRecord + 4) %8):
nCharsForDataKeysInRecord += 1
if verbose:
print "nWordsForFileIDsInRecord : ",nWordsForFileIDsInRecord
print " maxDataKeys : ",maxDataKeys
print "nCharsForDataKeysInRecord: ",nCharsForDataKeysInRecord
count = 0
while 1:
try:
endReason = "End Of File"
fileOffsets = array.array('I')
fileOffsets.fromfile(fileDesc,nWordsForFileIDsInRecord)
if pdsSignature != PDSSIGNATURE: fileOffsets.byteswap()
recordType = array.array('I')
recordType.fromfile(fileDesc,1)
if pdsSignature != PDSSIGNATURE: recordType.byteswap()
endReason = "bad Record: fileIDS"
if verbose:
print "********"
print "# syncValue:",count
if len(streamNames):
print streamNames[recordType[0]]
for index in xrange(0,nFiles[0]):
print "file: "+str(fileIDs[index*2])+" "+str(fileIDs[index*2+1])+"->"+str(fileOffsets[index*2])+" "+str(fileOffsets[index*2+1])
dataKeys = array.array('B')
dataKeys.fromfile(fileDesc,nCharsForDataKeysInRecord)
if pdsSignature != PDSSIGNATURE: dataKeys.byteswap()
if verbose: print dataKeys
#for index in xrange(0,nCharsForDataKeysInRecord):
# print " "+str(index)+" "+str(dataKeys[index])
count+=1
except EOFError:
if verbose: print endReason
break
