def changeFileIdsInLocFile(locFileName, fileIdList):
"""Change fileIDs in location header to provided id list"""
# define signature of location file
global LOCSIGNATURE
pds_utils.LOCSIGNATURE = LOCSIGNATURE
fileDesc = open(locFileName, 'r+w+b')
headerHeader = array.array('I')
headerHeader.fromfile(fileDesc, 3)
global locSignature
locSignature = headerHeader[0] >> 8
pds_utils.locSignature = pdsSignature
if locSignature != LOCSIGNATURE: headerHeader.byteswap()
nFiles = array.array('I')
nFiles.fromfile(fileDesc, 1)
pos = fileDesc.tell()
if locSignature != LOCSIGNATURE: nFiles.byteswap()
fileIDs = array.array('I')
fileIDs.fromfile(fileDesc, 2 * nFiles[0])
if locSignature != LOCSIGNATURE: fileIDs.byteswap()
# change file ids by new offset
# fileIDList = []
# i = 0
# while i<len(fileIDs):
# lower = fileIDs[i]
# i+=1
# upper = fileIDs[i]
# i+=1
# fileId = gen_util.form64BitNumber(lower,upper)
# newLower, newUpper = gen_util.lowerUpperBitsOfUID(fileId+fileIdOffset)
# fileIDList.append(newLower)
# fileIDList.append(newUpper)
idList = []
for fileId in fileIdList:
lower, upper = gen_util.lowerUpperBitsOfUID(fileId)
idList.append(lower)
idList.append(upper)
# seek back to fileID position in order to write new fileIDs
fileDesc.seek(pos, 0)
# write new fileIDs
fileArr = array.array('I')
fileArr.fromlist(idList)
if locSignature != LOCSIGNATURE: fileArr.byteswap()
fileArr.tofile(fileDesc)
fileDesc.close()
return
def changeFileIdsInLocFile(locFileName,fileIdList):
"""Change fileIDs in location header to provided id list"""
# define signature of location file
global LOCSIGNATURE
pds_utils.LOCSIGNATURE=LOCSIGNATURE
fileDesc = open(locFileName,'r+w+b')
headerHeader = array.array('I')
headerHeader.fromfile(fileDesc, 3)
global locSignature
locSignature=headerHeader[0]>>8
pds_utils.locSignature=pdsSignature
if locSignature != LOCSIGNATURE: headerHeader.byteswap()
nFiles = array.array('I')
nFiles.fromfile(fileDesc,1)
pos = fileDesc.tell()
if locSignature != LOCSIGNATURE: nFiles.byteswap()
fileIDs = array.array('I')
fileIDs.fromfile(fileDesc,2*nFiles[0])
if locSignature != LOCSIGNATURE: fileIDs.byteswap()
# change file ids by new offset
# fileIDList = []
# i = 0
# while i<len(fileIDs):
# lower = fileIDs[i]
# i+=1
# upper = fileIDs[i]
# i+=1
# fileId = gen_util.form64BitNumber(lower,upper)
# newLower, newUpper = gen_util.lowerUpperBitsOfUID(fileId+fileIdOffset)
# fileIDList.append(newLower)
# fileIDList.append(newUpper)
idList = []
for fileId in fileIdList:
lower, upper = gen_util.lowerUpperBitsOfUID(fileId)
idList.append(lower)
idList.append(upper)
# seek back to fileID position in order to write new fileIDs
fileDesc.seek(pos,0)
# write new fileIDs
fileArr = array.array('I')
fileArr.fromlist(idList)
if locSignature != LOCSIGNATURE: fileArr.byteswap()
fileArr.tofile(fileDesc)
fileDesc.close()
return
def buildLocationHeaderFromDict(streamDataKeysDict, fileIDList):
"""Build a PDS location header, from given components"""
locationHeader = array.array('I')
locationHeader.fromlist([LOCSIGNATURE * 256])
# file id for master index
locationHeader.append(0)
# remaining number of words in of header
locationHeader.append(0)
# associated file list
fList = []
for fileId in fileIDList:
lowerMostId, upperMostId = gen_util.lowerUpperBitsOfUID(fileId)
fList.append(lowerMostId)
fList.append(upperMostId)
fileList = array.array('I')
# for now only set the low word for the file ID
fileList.fromlist([len(fileIDList)] + fList)
locationHeader += fileList
# put streamNames into header
streamNames = streamDataKeysDict.keys()
nameWords = pds_utils.charArrayOfStreams(streamNames)
sortedStreamNames = list(streamNames)
sortedStreamNames.sort()
# streamNameString =""
# for name in sortedStreamNames:
# streamNameString = streamNameString+name +"\0"
# streamNameString = streamNameString[:-1]
# while 0 != len(streamNameString) % 4:
# streamNameString = streamNameString + "\0"
# nameWords = array.array('I')
# nameWords.fromstring(streamNameString)
nWords = array.array('I')
nWords.fromlist([len(nameWords)])
locationHeader += nWords + nameWords
# add dataKeys and hash into header
dataKeyArray = array.array('I')
nProxies = array.array('I')
nProxies.fromlist([0])
for stream in sortedStreamNames:
dataKeys = streamDataKeysDict[stream]
nProxies[0] = len(dataKeys)
dataKeyArray += nProxies
dataKeyString = ""
for dataKey in dataKeys:
dataKeyString += dataKey[0] + "\0" + dataKey[1] + "\0" + dataKey[
2] + "\0"
while (len(dataKeyString) % 4):
dataKeyString += "\0"
dataStringArray = array.array('I')
dataStringArray.fromstring(dataKeyString)
dataKeyArray += dataStringArray
# emptyHashArray = array.array('I')
# emptyHashArray.fromlist([0]*5)
hash = sha.new(dataKeyArray.tostring()).digest()
# tempArray = array.array('I')
# tempArray.fromstring(hash)
# print "hash1",tempArray.tolist()
# hash = sha.new( (emptyHashArray+dataKeyArray).tostring() ).digest()
# tempArray = array.array('I')
# tempArray.fromstring(hash)
# print "hash2",tempArray.tolist()
hashWords = array.array('I')
hashWords.fromstring(hash)
locationHeader += hashWords + dataKeyArray
# pad header so Records begin on 8 byte boundary
if not (len(locationHeader) % 2):
locationHeader.fromlist([0])
headerLength = len(locationHeader) + 1 - 3
locationHeader.fromlist([headerLength])
locationHeader[2] = headerLength
return locationHeader
def stripKeyFile(keyFileName,iNewFileName,newFileId,skimKeyFileName):
"""From given key file name, new file id and fake sv list recreate key file.
Return new file name and write out the file."""
print "stripKeyFile called - look at this code!"
sys.exit(1)
newFileName = string.split(iNewFileName,"-esdb")[0]+"-esdb.ikey"
pos,needToSwap,nRecordNames,sNames,nSV=keyFileHeaderReader(keyFileName)
origKeyFile = open(keyFileName,'r')
origKeyFile.seek(pos)
# read information from skimKeyFileName
skimPos,skimSwap,skimRecords,skimStreamNames,skimNumberSV=keyFileHeaderReader(skimKeyFileName)
skimKeyFile = open(skimKeyFileName,'r')
skimKeyFile.seek(skimPos)
streamNameString =""
for name in sNames:
streamNameString = streamNameString+name +"\0"
streamNameString = streamNameString[:-1]
while 0 != len(streamNameString) % 4:
streamNameString = streamNameString + "\0"
nameWords = array.array('I')
nameWords.fromstring(streamNameString)
nameList = nameWords.tolist()
number32BitWordsInNameList = len(nameList)
nWordsInHeader = len(sNames)+number32BitWordsInNameList+1+1+1
headerList = [2718281*256,newFileId,nWordsInHeader,len(sNames)]
headerList+= [number32BitWordsInNameList]+nameList+[nSV,nWordsInHeader]
keyFile = open(newFileName,'w+b')
keyHeader = array.array('I')
keyHeader.fromlist(headerList)
keyHeader.tofile(keyFile)
# loop over all sv's in original key file and write out those which are not in fakeSVList
i = 0
fakeIdx = 0
counter = 0
skimSV,skimIdxList = decodeKeyRecord(skimKeyFile,skimSwap,skimRecords)
print "orig",origKeyFile.name
print "skim",skimKeyFile.name
while i<nSV:
if not skimSV: break
sv,recIdxList = decodeKeyRecord(origKeyFile,needToSwap,nRecordNames)
if not sv: break
i+=1
# we event number from parent key file is become larger then from skim, advance the skim
if sv[1]>skimSV[1]:
try:
skimSV,skimIdxList = decodeKeyRecord(skimKeyFile,skimSwap,skimRecords)
except:
skimSV=()
break
if skimSV!=sv:
continue
else: # write out
run = sv[0]
evt = sv[1]
lowerId, upperId = gen_util.lowerUpperBitsOfUID(sv[2])
recordArray = array.array('I')
recordHeader = [sv[0],sv[1],lowerId,upperId]+skimIdxList
recordArray.fromlist(recordHeader)
recordArray.tofile(keyFile)
counter+=1
# read one more event from skim file
try:
skimSV,skimIdxList = decodeKeyRecord(skimKeyFile,skimSwap,skimRecords)
except:
skimSV=()
# seek to position where total SV's written and fix it to new number of SV's
keyFile.seek(pos-2*4)
pos = keyFile.tell()
nSVHeader = array.array('I')
nSVHeader.fromfile(keyFile,1)
if nSVHeader[0]!=nSV:
print "nSVHeader[0]",nSVHeader[0],nSV
print "Miss position of nSyncValues in a file"
return ""
keyFile.seek(pos)
nSVHeader[0]=counter
nSVHeader.tofile(keyFile)
keyFile.close()
return newFileName
def stripKeyFile(keyFileName, iNewFileName, newFileId, skimKeyFileName):
"""From given key file name, new file id and fake sv list recreate key file.
Return new file name and write out the file."""
print "stripKeyFile called - look at this code!"
sys.exit(1)
newFileName = string.split(iNewFileName, "-esdb")[0] + "-esdb.ikey"
pos, needToSwap, nRecordNames, sNames, nSV = keyFileHeaderReader(
keyFileName)
origKeyFile = open(keyFileName, 'r')
origKeyFile.seek(pos)
# read information from skimKeyFileName
skimPos, skimSwap, skimRecords, skimStreamNames, skimNumberSV = keyFileHeaderReader(
skimKeyFileName)
skimKeyFile = open(skimKeyFileName, 'r')
skimKeyFile.seek(skimPos)
streamNameString = ""
for name in sNames:
streamNameString = streamNameString + name + "\0"
streamNameString = streamNameString[:-1]
while 0 != len(streamNameString) % 4:
streamNameString = streamNameString + "\0"
nameWords = array.array('I')
nameWords.fromstring(streamNameString)
nameList = nameWords.tolist()
number32BitWordsInNameList = len(nameList)
nWordsInHeader = len(sNames) + number32BitWordsInNameList + 1 + 1 + 1
headerList = [2718281 * 256, newFileId, nWordsInHeader, len(sNames)]
headerList += [number32BitWordsInNameList
] + nameList + [nSV, nWordsInHeader]
keyFile = open(newFileName, 'w+b')
keyHeader = array.array('I')
keyHeader.fromlist(headerList)
keyHeader.tofile(keyFile)
# loop over all sv's in original key file and write out those which are not in fakeSVList
i = 0
fakeIdx = 0
counter = 0
skimSV, skimIdxList = decodeKeyRecord(skimKeyFile, skimSwap, skimRecords)
print "orig", origKeyFile.name
print "skim", skimKeyFile.name
while i < nSV:
if not skimSV: break
sv, recIdxList = decodeKeyRecord(origKeyFile, needToSwap, nRecordNames)
if not sv: break
i += 1
# we event number from parent key file is become larger then from skim, advance the skim
if sv[1] > skimSV[1]:
try:
skimSV, skimIdxList = decodeKeyRecord(skimKeyFile, skimSwap,
skimRecords)
except:
skimSV = ()
break
if skimSV != sv:
continue
else: # write out
run = sv[0]
evt = sv[1]
lowerId, upperId = gen_util.lowerUpperBitsOfUID(sv[2])
recordArray = array.array('I')
recordHeader = [sv[0], sv[1], lowerId, upperId] + skimIdxList
recordArray.fromlist(recordHeader)
recordArray.tofile(keyFile)
counter += 1
# read one more event from skim file
try:
skimSV, skimIdxList = decodeKeyRecord(skimKeyFile, skimSwap,
skimRecords)
except:
skimSV = ()
# seek to position where total SV's written and fix it to new number of SV's
keyFile.seek(pos - 2 * 4)
pos = keyFile.tell()
nSVHeader = array.array('I')
nSVHeader.fromfile(keyFile, 1)
if nSVHeader[0] != nSV:
print "nSVHeader[0]", nSVHeader[0], nSV
print "Miss position of nSyncValues in a file"
return ""
keyFile.seek(pos)
nSVHeader[0] = counter
nSVHeader.tofile(keyFile)
keyFile.close()
return newFileName
def buildLocationHeaderFromDict(streamDataKeysDict, fileIDList):
"""Build a PDS location header, from given components"""
locationHeader = array.array('I')
locationHeader.fromlist([LOCSIGNATURE*256])
# file id for master index
locationHeader.append(0)
# remaining number of words in of header
locationHeader.append(0)
# associated file list
fList = []
for fileId in fileIDList:
lowerMostId,upperMostId = gen_util.lowerUpperBitsOfUID(fileId)
fList.append(lowerMostId)
fList.append(upperMostId)
fileList = array.array('I')
# for now only set the low word for the file ID
fileList.fromlist([len(fileIDList)]+fList)
locationHeader += fileList
# put streamNames into header
streamNames=streamDataKeysDict.keys()
nameWords = pds_utils.charArrayOfStreams(streamNames)
sortedStreamNames=list(streamNames)
sortedStreamNames.sort()
# streamNameString =""
# for name in sortedStreamNames:
# streamNameString = streamNameString+name +"\0"
# streamNameString = streamNameString[:-1]
# while 0 != len(streamNameString) % 4:
# streamNameString = streamNameString + "\0"
# nameWords = array.array('I')
# nameWords.fromstring(streamNameString)
nWords = array.array('I')
nWords.fromlist([len(nameWords)])
locationHeader += nWords+nameWords
# add dataKeys and hash into header
dataKeyArray= array.array('I')
nProxies = array.array('I')
nProxies.fromlist([0])
for stream in sortedStreamNames:
dataKeys = streamDataKeysDict[stream]
nProxies[0] = len(dataKeys)
dataKeyArray += nProxies
dataKeyString = ""
for dataKey in dataKeys:
dataKeyString+= dataKey[0]+"\0"+dataKey[1]+"\0"+dataKey[2]+"\0"
while( len(dataKeyString) % 4 ):
dataKeyString+="\0"
dataStringArray= array.array('I')
dataStringArray.fromstring(dataKeyString)
dataKeyArray += dataStringArray
# emptyHashArray = array.array('I')
# emptyHashArray.fromlist([0]*5)
hash = sha.new( dataKeyArray.tostring() ).digest()
# tempArray = array.array('I')
# tempArray.fromstring(hash)
# print "hash1",tempArray.tolist()
# hash = sha.new( (emptyHashArray+dataKeyArray).tostring() ).digest()
# tempArray = array.array('I')
# tempArray.fromstring(hash)
# print "hash2",tempArray.tolist()
hashWords = array.array('I')
hashWords.fromstring(hash)
locationHeader += hashWords+dataKeyArray
# pad header so Records begin on 8 byte boundary
if not (len(locationHeader) % 2):
locationHeader.fromlist([0])
headerLength = len(locationHeader)+1-3
locationHeader.fromlist([headerLength])
locationHeader[2] = headerLength
return locationHeader
