Exemplo n.º 1
0
def pdsHeaderParser(fileName):
    """Parser header of PDS file"""
    svName, parents, verList, verDict = decodeVersionInfo(fileName)

    PDSSIGNATURE = 3141592  # magic number for PDS format, as it should be
    pdsSignature = 0  # signature we read from given PDS file

    fileDesc = open(fileName, 'rb')
    headerHeader = array.array('I')
    headerHeader.fromfile(fileDesc, 3)

    # to handle endianess, read pdsSignature
    pdsSignature = headerHeader[0] >> 8
    pds_utils.pdsSignature = pdsSignature
    pds_utils.PDSSIGNATURE = PDSSIGNATURE

    needToSwap = 0
    if pdsSignature != PDSSIGNATURE:
        needToSwap = 1
        headerHeader.byteswap()
    streamNames = pds_utils.name_list_from_file(fileDesc)

    shProxyNames = pds_utils.name_list_from_file(fileDesc)
    shNames = pds_utils.name_list_from_file(fileDesc)

    # grab the rest of the header
    restOfHeader = array.array('I')
    # the header size ignores the first 3 words in the event
    restOfHeader.fromfile(fileDesc, headerHeader[2] - fileDesc.tell() / 4 + 3)
    if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
    if restOfHeader[-1] != headerHeader[2]:
        raise "header inconsistent"
    proxiesInStreams = pds_utils.find_proxies_in_streams(
        restOfHeader, streamNames, shProxyNames)
    oDict = {}
    for idx in xrange(0, len(streamNames)):
        oDict[streamNames[idx]] = proxiesInStreams[idx]
    return (fileDesc, needToSwap, streamNames, oDict)
Exemplo n.º 2
0
def pdsHeaderParser(fileName):
    """Parser header of PDS file"""
    svName,parents,verList,verDict = decodeVersionInfo(fileName)
	
    PDSSIGNATURE=3141592 # magic number for PDS format, as it should be
    pdsSignature=0       # signature we read from given PDS file

    fileDesc = open(fileName,'rb')
    headerHeader = array.array('I')
    headerHeader.fromfile(fileDesc, 3)

    # to handle endianess, read pdsSignature
    pdsSignature = headerHeader[0]>>8
    pds_utils.pdsSignature=pdsSignature
    pds_utils.PDSSIGNATURE=PDSSIGNATURE

    needToSwap   = 0
    if pdsSignature != PDSSIGNATURE:
       needToSwap= 1
       headerHeader.byteswap()
    streamNames  = pds_utils.name_list_from_file(fileDesc)
    
    shProxyNames = pds_utils.name_list_from_file(fileDesc)
    shNames = pds_utils.name_list_from_file(fileDesc)

    # grab the rest of the header
    restOfHeader = array.array('I')
    # the header size ignores the first 3 words in the event
    restOfHeader.fromfile(fileDesc, headerHeader[2] -fileDesc.tell()/4 +3)
    if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
    if restOfHeader[-1] != headerHeader[2]:
       raise "header inconsistent"
    proxiesInStreams = pds_utils.find_proxies_in_streams( restOfHeader, streamNames, shProxyNames)
    oDict = {}
    for idx in xrange(0,len(streamNames)):
	oDict[streamNames[idx]]=proxiesInStreams[idx]
    return (fileDesc,needToSwap,streamNames,oDict)
Exemplo n.º 3
0
def buildLocationHeader(iPDSFileName, iFileID):
    """Build a PDS location header, from given pds file name and file id"""
    #    print "iPDSFileName",iPDSFileName

    pdsFile = open(iPDSFileName, 'rb')
    pdsFileID = int(iFileID)

    pdsHeader = array.array('I')
    pdsHeader.fromfile(pdsFile, 3)
    global pdsSignature
    pdsSignature = pdsHeader[0] >> 8
    pds_utils.PDSSIGNATURE = PDSSIGNATURE
    pds_utils.pdsSignature = pdsSignature
    if pdsSignature != PDSSIGNATURE:
        pdsHeader.byteswap()
        needToSwap = 1
    else:
        needToSwap = 0

    locationHeader = array.array('I')
    locationHeader.fromlist([2951413 * 256])
    #file id for master index
    locationHeader.append(0)
    #remaining number of words in of header
    locationHeader.append(0)
    #associated file list
    fileList = array.array('I')
    # for now only set the low word for the file ID
    fileList.fromlist([1, pdsFileID, 0])
    locationHeader = locationHeader + fileList

    # now get the names of the streams
    #  the location file and the PDS file use the same format for the
    #  stream names so just have to copy the info
    numberOfRecordNameWords = array.array('I')
    numberOfRecordNameWords.fromfile(pdsFile, 1)
    if pdsSignature != PDSSIGNATURE: numberOfRecordNameWords.byteswap()

    #need number of names
    pdsFile.seek(4 * 4)
    nameChars = array.array('c')
    nameChars.fromfile(pdsFile, numberOfRecordNameWords[0] * 4)
    if pdsSignature != PDSSIGNATURE: nameChars.byteswap()
    streamNames = pds_utils.name_list(nameChars)
    sortedStreamNames = list(streamNames)
    sortedStreamNames.sort()

    #build conversion from new stream index to old index
    newStreamIndex2OldIndex = []
    oldStreamIndex2NewIndex = [0] * len(streamNames)
    streamNameString = ""
    for name in sortedStreamNames:
        newStreamIndex2OldIndex.append(streamNames.index(name))
        oldStreamIndex2NewIndex[
            newStreamIndex2OldIndex[-1]] = len(newStreamIndex2OldIndex) - 1
        streamNameString = streamNameString + name + "\0"
    streamNameString = streamNameString[:-1]
    while 0 != len(streamNameString) % 4:
        streamNameString = streamNameString + "\0"
    nameWords = array.array('I')
    nameWords.fromstring(streamNameString)

    locationHeader = locationHeader + numberOfRecordNameWords + nameWords

    #print streamNames
    shProxyNames = pds_utils.name_list_from_file(pdsFile)
    #print shProxyNames
    #print len(shProxyNames)
    shNames = pds_utils.name_list_from_file(pdsFile)
    #print shNames

    #grab the rest of the header
    restOfHeader = array.array('I')
    #the header size ignores the first 3 words in the event
    restOfHeader.fromfile(pdsFile, pdsHeader[2] - pdsFile.tell() / 4 + 3)
    if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
    #print restOfHeader
    if restOfHeader[-1] != pdsHeader[2]:
        raise "header inconsistent"

    #create the list of 'type' 'usage' 'production' tags for each stream
    def proxySort(a, b):
        #print a, b
        temp = cmp(a[0], b[0])
        if 0 == temp:
            temp = cmp(a[1], b[1])
            if 0 == temp:
                temp = cmp(a[2], b[2])
        return temp

    proxiesInStreams = pds_utils.find_proxies_in_streams(
        restOfHeader, streamNames, shProxyNames)
    #sortProxiesInStreams = proxiesInStreams
    for proxies in proxiesInStreams:
        proxies.sort(proxySort)

    #reserve space for our hash
    dataKeyHashIndex = len(locationHeader)
    dataKeyHashArray = array.array('I')
    dataKeyHashArray.fromlist([0] * 5)
    locationHeader += dataKeyHashArray

    maxNProxies = 0
    nProxies = array.array('I')
    nProxies.fromlist([0])
    proxiesArray = []
    for oldIndex in newStreamIndex2OldIndex:
        #print oldIndex
        proxies = proxiesInStreams[oldIndex]
        nProxies[0] = len(proxies)
        if nProxies[0] > maxNProxies:
            maxNProxies = nProxies[0]
        locationHeader += nProxies
        datakeys = ""
        #now add each string
        proxyList = []
        for proxy in proxies:
            proxyList.append(proxy)
            for key in proxy:
                datakeys += key + "\0"
            while (len(datakeys) % 4):
                datakeys += "\0"
        #print len(datakeys)
        #print datakeys
        #print len(datakeys)
        dataKeysArray = array.array('I')
        dataKeysArray.fromstring(datakeys)
        #        proxiesArray+=[dataKeysArray.tolist()]
        proxiesArray += [proxyList]
        #nProxies[0] = len(dataKeysArray)
        #locationHeader += nProxies
        locationHeader += dataKeysArray
    #calculate the hash
    hash = sha.new(locationHeader[dataKeyHashIndex + 1:].tostring()).digest()
    #print sha.new( locationHeader[dataKeyHashIndex+1:].tostring() ).hexdigest()
    dataKeyHashArray = array.array('I')
    dataKeyHashArray.fromstring(hash)
    locationHeader[dataKeyHashIndex:dataKeyHashIndex + 5] = dataKeyHashArray

    locationHeaderBeforePadding = locationHeader.tolist()
    #pad header so Records begin on 8 byte boundary
    if not (len(locationHeader) % 2):
        locationHeader.fromlist([0])
    headerLength = len(locationHeader) + 1 - 3
    locationHeader.fromlist([headerLength])
    locationHeader[2] = headerLength

    #pad the max number of ASUs to be a multiple of 8
    nASUperRecord = maxNProxies
    while ((nASUperRecord + 4) % 8):
        nASUperRecord += 1

    #For each stream, create the ASU to file ID list
    whichFileForStream = []
    whichFileForStreamFake = []
    for proxies in proxiesInStreams:
        whichFile = array.array('B')
        fakeArray = array.array('B')

        whichFile.fromlist([0] * len(proxies) + [255] *
                           (nASUperRecord - len(proxies)))
        for x in xrange(0, len(whichFile)):
            fakeArray.append(255)
        #print whichFile
        whichFileForStream.append(whichFile)
        whichFileForStreamFake.append(fakeArray)

    return (pdsFile, locationHeader, locationHeaderBeforePadding, nameWords,
            dataKeyHashArray, proxiesArray, streamNames,
            oldStreamIndex2NewIndex, newStreamIndex2OldIndex,
            whichFileForStream, whichFileForStreamFake, needToSwap)
Exemplo n.º 4
0
def getStreamDataKeyDictFromPDS(iPDSFileName):
    """Extract from PDS file streamDataKey dictionary"""

    pdsFile = open(iPDSFileName, 'rb')
    pdsHeader = array.array('I')
    pdsHeader.fromfile(pdsFile, 3)
    global pdsSignature
    pdsSignature = pdsHeader[0] >> 8
    pds_utils.PDSSIGNATURE = PDSSIGNATURE
    pds_utils.pdsSignature = pdsSignature
    if pdsSignature != PDSSIGNATURE:
        pdsHeader.byteswap()
        needToSwap = 1
    else:
        needToSwap = 0

    # now get the names of the streams
    #  the location file and the PDS file use the same format for the
    #  stream names so just have to copy the info
    numberOfRecordNameWords = array.array('I')
    numberOfRecordNameWords.fromfile(pdsFile, 1)
    if pdsSignature != PDSSIGNATURE: numberOfRecordNameWords.byteswap()

    #need number of names
    pdsFile.seek(4 * 4)
    nameChars = array.array('c')
    nameChars.fromfile(pdsFile, numberOfRecordNameWords[0] * 4)
    if pdsSignature != PDSSIGNATURE: nameChars.byteswap()
    streamNames = pds_utils.name_list(nameChars)
    sortedStreamNames = list(streamNames)
    sortedStreamNames.sort()

    #build conversion from new stream index to old index
    newStreamIndex2OldIndex = []
    for name in sortedStreamNames:
        newStreamIndex2OldIndex.append(streamNames.index(name))

    #print streamNames
    shProxyNames = pds_utils.name_list_from_file(pdsFile)
    #print shProxyNames
    #print len(shProxyNames)
    shNames = pds_utils.name_list_from_file(pdsFile)
    #print shNames

    #grab the rest of the header
    restOfHeader = array.array('I')
    #the header size ignores the first 3 words in the event
    restOfHeader.fromfile(pdsFile, pdsHeader[2] - pdsFile.tell() / 4 + 3)
    if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
    #print restOfHeader
    if restOfHeader[-1] != pdsHeader[2]:
        raise "header inconsistent"

    #create the list of 'type' 'usage' 'production' tags for each stream
    def proxySort(a, b):
        #print a, b
        temp = cmp(a[0], b[0])
        if 0 == temp:
            temp = cmp(a[1], b[1])
            if 0 == temp:
                temp = cmp(a[2], b[2])
        return temp

    proxiesInStreams = pds_utils.find_proxies_in_streams(
        restOfHeader, streamNames, shProxyNames)
    #sortProxiesInStreams = proxiesInStreams
    for proxies in proxiesInStreams:
        proxies.sort(proxySort)

    streamDataKeyDict = {}
    for oldIndex in newStreamIndex2OldIndex:
        #print oldIndex
        stream = streamNames[oldIndex]
        proxies = proxiesInStreams[oldIndex]
        streamDataKeyDict[stream] = proxies

    return streamDataKeyDict
Exemplo n.º 5
0
def pdsParser(file,what=""):
    """PDS file parser. Return a list of run/uid/sync.values/proxies in given file"""
    fileDesc = open(file,'rb')
    headerHeader = array.array('I')
    headerHeader.fromfile(fileDesc, 3)
#    global pdsSignature
    pdsSignature=headerHeader[0]>>8
    pds_utils.PDSSIGNATURE=PDSSIGNATURE
    pds_utils.pdsSignature=pdsSignature
    if pdsSignature != PDSSIGNATURE: headerHeader.byteswap()
    # invoke name_list_from_file with additional argument 1 which means
    # return number of words for those names
    streamNames,streamNamesWords   = pds_utils.name_list_from_file(fileDesc,1)
    shProxyNames,shProxyNamesWords = pds_utils.name_list_from_file(fileDesc,1)
    shNames,shNamesWords           = pds_utils.name_list_from_file(fileDesc,1)
  
    #grab the rest of the header
    restOfHeader = array.array('I')
    #the header size ignores the first 3 words in the event
    restOfHeader.fromfile(fileDesc, headerHeader[2] -fileDesc.tell()/4 +3)
    if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
    if restOfHeader[-1] != headerHeader[2]:
       raise "header inconsistent"
    proxiesInStreams = pds_utils.find_proxies_in_streams( restOfHeader, streamNames, shProxyNames)

    # define proxy dictionary dict={'stream':listOfProxiesInThatStream}
    pList = []
    for str in streamNames:
        strIndex = streamNames.index(str)
	pList.append((str,proxiesInStreams[strIndex]))

    posOfLastHeaderWords = fileDesc.tell()

    #start to read the rest of the file
    runList = []
    uidList = []
    syncList= []
    while 1:
       try:
	  endReason = ""
	  recordHeader = array.array('I')
	  recordHeader.fromfile(fileDesc,5)
	  if pdsSignature != PDSSIGNATURE: recordHeader.byteswap()
	  # read stream id and convert it to stream name
	  streamId = int(recordHeader[0])
	  stream   = streamNames[streamId]
 
	  # read uid
	  uid = ((recordHeader[4]<<32)|recordHeader[3])

 	  # form syncValue
 	  syncValue = (recordHeader[1],recordHeader[2],uid,stream)
 	  syncList.append(syncValue)
	   
	  # read run record
	  runNumber = recordHeader[1]
	  if len(runList)==0 or (uid!=uidList[-1] or runNumber!=runList[-1]): 
	     runList.append(runNumber)
	     uidList.append(uid)

	  # read the record data
	  recordDataSize = array.array('I')
	  recordDataSize.fromfile(fileDesc,1)
	  if pdsSignature != PDSSIGNATURE: recordDataSize.byteswap()
#       print "recordDataSize",recordDataSize
      
	  recordData = array.array('I')
	  recordData.fromfile(fileDesc,recordDataSize[0]-1)
	  if pdsSignature != PDSSIGNATURE: recordData.byteswap()
      
	  recordDataEnd = array.array('I')
	  recordDataEnd.fromfile(fileDesc,1)
	  if pdsSignature != PDSSIGNATURE: recordDataEnd.byteswap()

	  if recordDataSize[0] != recordDataEnd[0]:
	     print "Error of reading record data"
	     sys.exit(1)
	  
       except EOFError:
	  break
#    runList.sort()
#    uidList.sort()
#    syncList.sort()
    pList.sort()
    if what=="run":
       return runList
    elif what=="uid":
       return uidList
    elif what=="syncValue":
       return syncList
    elif what=="proxies":
       return pList
    
    returnList  = [runList,uidList,pList,syncList]
    returnList += [streamNames,streamNamesWords]
    returnList += [shProxyNames,shProxyNamesWords]
    returnList += [shNames,shNamesWords]
    returnList += [proxiesInStreams,posOfLastHeaderWords]
    return returnList
Exemplo n.º 6
0
def dump(fileName, verbose=0, iStream='event'):
    """Dump content of pds file to stdout"""
    svName, parents, verList, verDict = decodeVersionInfo(fileName)
    if not parents:
        parents = 'N/A'
    if svName:
        print "Versioning information:"
        print "-----------------------"
        print "data version name: '%s'" % svName
        print "list of parents  :", parents

    PDSSIGNATURE = 3141592  # magic number for PDS format, as it should be
    pdsSignature = 0  # signature we read from given PDS file

    fileDesc = open(fileName, 'rb')
    headerHeader = array.array('I')
    headerHeader.fromfile(fileDesc, 3)

    # to handle endianess, read pdsSignature
    pdsSignature = headerHeader[0] >> 8
    pds_utils.pdsSignature = pdsSignature
    pds_utils.PDSSIGNATURE = PDSSIGNATURE

    if pdsSignature != PDSSIGNATURE:
        print "File was produced on another endian machine, byte swapping is enabled"
    if pdsSignature != PDSSIGNATURE: headerHeader.byteswap()
    print
    print "List of streams in a file:",
    streamNames = pds_utils.name_list_from_file(fileDesc)
    for x in streamNames:
        print x,
    print

    shProxyNames = pds_utils.name_list_from_file(fileDesc)
    shNames = pds_utils.name_list_from_file(fileDesc)
    if verbose:
        print "List of types that storage helper proxy factories have:"
        for x in shProxyNames:
            print x,
        print "\n\n"
        print "List of types that only have storage helpers"
        for x in shNames:
            print x,
        print

    # grab the rest of the header
    restOfHeader = array.array('I')
    # the header size ignores the first 3 words in the event
    restOfHeader.fromfile(fileDesc, headerHeader[2] - fileDesc.tell() / 4 + 3)
    if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
    if restOfHeader[-1] != headerHeader[2]:
        raise "header inconsistent"
    proxiesInStreams = pds_utils.find_proxies_in_streams(
        restOfHeader, streamNames, shProxyNames)
    print
    for idx in xrange(0, len(streamNames)):
        print "List of data keys in stream:", streamNames[idx]
        print pds_utils.printProxies(proxiesInStreams[idx])

    # want to only look at stuff in given stream (default is event stream)
    if not streamNames.count(iStream):
        print "There is no stream %s in %s" % (iStream, fileName)
        return
    eventIndex = streamNames.index(iStream)

    # create a structure to hold our info
    eventProxies = proxiesInStreams[eventIndex]
    accumulatedData = []
    for proxy in eventProxies:
        accumulatedData.append([0, 488888888, 0])

    # start to read the rest of the file
    numEvents = 0

    firstSV = ""
    lastSV = ""
    while 1:
        try:
            endReason = ""
            pos = fileDesc.tell()
            recordHeader = array.array('I')
            recordHeader.fromfile(fileDesc, 5)
            if pdsSignature != PDSSIGNATURE: recordHeader.byteswap()

            uid = ((recordHeader[4] << 32) | recordHeader[3])
            if not len(firstSV):
                firstSV = "%d/%d/%d" % (recordHeader[1], recordHeader[2], uid)
            lastSV = "%d/%d/%d" % (recordHeader[1], recordHeader[2], uid)

            if verbose:
                print "********************"
                print "position           : %s" % pos
                print "stream             : " + streamNames[recordHeader[0]]
                print "SyncValue          : %s/%s/%s" % (recordHeader[1],
                                                         recordHeader[2], uid)

            endReason = "bad Record: record size"
            recordDataLength = array.array('I')
            recordDataLength.fromfile(fileDesc, 1)
            if pdsSignature != PDSSIGNATURE: recordDataLength.byteswap()
            if verbose:
                print "size of record data: " + str(recordDataLength[0])

            endReason = "bad Record: record (type " + streamNames[
                recordHeader[0]] + ") (size " + str(recordDataLength[0]) + ")"
            begRecPos = fileDesc.tell()
            recordData = array.array('I')
            recordData.fromfile(fileDesc, recordDataLength[0])
            if pdsSignature != PDSSIGNATURE: recordData.byteswap()

            numEvents = numEvents + 1
            index = 0
            endIndex = len(recordData)
            dataInfo = []
            dataInfoSize = []
            lengthOffset = 0
            while index < endIndex - 1:
                proxyIndex = int(recordData[index])
                index = index + 1
                dataSize = recordData[index]
                index = index + int(dataSize) + 1
                fullDataSize = dataSize + 2
                strData = str(proxiesInStreams[recordHeader[0]][proxyIndex])
                if lengthOffset < len(strData):
                    lengthOffset = len(strData)
                dataInfo.append(strData)
                dataInfoSize.append(fullDataSize)
                #            if verbose:
                #               print str(proxiesInStreams[recordHeader[0]][proxyIndex]) + " size "+str(fullDataSize)
                if recordHeader[0] == eventIndex:
                    accumulatedData[proxyIndex][
                        0] = accumulatedData[proxyIndex][0] + fullDataSize
                    if accumulatedData[proxyIndex][1] > fullDataSize:
                        accumulatedData[proxyIndex][1] = fullDataSize
                    if accumulatedData[proxyIndex][2] < fullDataSize:
                        accumulatedData[proxyIndex][2] = fullDataSize
            if verbose:
                print
                for idx in xrange(0, len(dataInfo)):
                    s = dataInfo[idx]
                    print "%s size %s" % (s.ljust(lengthOffset),
                                          dataInfoSize[idx])

        except EOFError:
            if verbose and endReason:
                print "########################################################"
                print endReason
                print "########################################################"
            break

    index = 0
    if numEvents == 0:
        raise "no events found"

    if verbose:
        print "######### Summary ##########"
        print "Proxies in %s:" % iStream
        for item in eventProxies:
            print "%s : %i %f %i" % (item, accumulatedData[index][1],
                                     float(accumulatedData[index][0]) /
                                     float(numEvents),
                                     accumulatedData[index][2])
            index = index + 1

    print "First sync value     :", firstSV
    print "Last sync value      :", lastSV
    print "Number of syncValues :", numEvents
    print
Exemplo n.º 7
0
def decodeVersionInfo(fileName):
    """Decode VersionInfo from beginrun record. 
    VersionInfo consists of:
       - softwareRelease : string
       - specificVersionName : string
       - configurationHash : string
       - ordinal : unsigned int no packing
       - ancestors : container of string
    So, it always grows, e.g. post-p2 file will contains two VersionInfo's, one
    for itself and one for it's parent. So, the underlying algorithm creates
    a list of VersionInfo's in the following format:
    [(childTag,[softRel,svName,hash,id,parent1,parent2,...]),(parentTag,[...])]
    This method returns (svName,[listOfParents]) tuple
    """
    emptyTuple = ('', '', '', '')
    PDSSIGNATURE = 3141592  # magic number for PDS format, as it should be
    pdsSignature = 0  # signature we read from given PDS file

    fileDesc = open(fileName, 'rb')
    headerHeader = array.array('I')
    headerHeader.fromfile(fileDesc, 3)

    # to handle endianess, read pdsSignature
    pdsSignature = headerHeader[0] >> 8
    pds_utils.pdsSignature = pdsSignature
    pds_utils.PDSSIGNATURE = PDSSIGNATURE

    needToSwap = 0
    if pdsSignature != PDSSIGNATURE:
        needToSwap = 1
        headerHeader.byteswap()
    streamNames = pds_utils.name_list_from_file(fileDesc)
    shProxyNames = pds_utils.name_list_from_file(fileDesc)
    shNames = pds_utils.name_list_from_file(fileDesc)

    # grab the rest of the header
    restOfHeader = array.array('I')
    # the header size ignores the first 3 words in the event
    restOfHeader.fromfile(fileDesc, headerHeader[2] - fileDesc.tell() / 4 + 3)
    if needToSwap: restOfHeader.byteswap()
    if restOfHeader[-1] != headerHeader[2]:
        raise "header inconsistent"
    proxiesInStreams = pds_utils.find_proxies_in_streams(
        restOfHeader, streamNames, shProxyNames)

    # want to only look at stuff in beginrun stream
    if not streamNames.count('beginrun'):
        return emptyTuple
    eventIndex = streamNames.index('beginrun')

    # create a structure to hold our info
    eventProxies = proxiesInStreams[eventIndex]
    accumulatedData = []
    foundVersionInfo = 0
    for proxy in eventProxies:
        accumulatedData.append([0, 488888888, 0])
        if proxy[0] == 'VersionInfo':
            foundVersionInfo = 1
            break
    # check if VersionInfo is present in a proxy list
    if not foundVersionInfo:
        return emptyTuple

    # start to read the rest of the file
    numEvents = 0
    nWordSize = 4  # we use to use 32-bit words, which is 4 bytes
    firstSV = ""
    lastSV = ""
    versionInfoList = []
    versionInfoDict = {}
    while 1:
        try:
            endReason = ""
            recordHeader = array.array('I')
            recordHeader.fromfile(fileDesc, 5)
            if needToSwap: recordHeader.byteswap()

            uid = ((recordHeader[4] << 32) | recordHeader[3])
            if not len(firstSV):
                firstSV = "%d/%d/%d" % (recordHeader[1], recordHeader[2], uid)
            lastSV = "%d/%d/%d" % (recordHeader[1], recordHeader[2], uid)

            endReason = "bad Record: record size"
            recordDataLength = array.array('I')
            recordDataLength.fromfile(fileDesc, 1)
            if needToSwap: recordDataLength.byteswap()

            if recordHeader[0] > len(streamNames) - 1:
                break
            endReason = "bad Record: record (type " + streamNames[
                recordHeader[0]] + ") (size " + str(recordDataLength[0]) + ")"
            curStream = streamNames[recordHeader[0]]
            if curStream != 'beginrun':
                continue
            begRecPos = fileDesc.tell()
            recordData = array.array('I')
            recordData.fromfile(fileDesc, recordDataLength[0] - 1)
            if needToSwap: recordData.byteswap()
            testLastWord = array.array('I')
            testLastWord.fromfile(fileDesc, 1)
            if needToSwap: testLastWord.byteswap()
            endRecPos = fileDesc.tell()

            ############ remove later this block, it's a full dataRecord
            fileDesc.seek(begRecPos)
            cData = array.array('c')
            cData.fromfile(fileDesc, (recordDataLength[0]) * nWordSize)
            ################

            fileDesc.seek(begRecPos)
            nReadWords = 0
            while nReadWords < (recordDataLength[0] - 1):
                # let's read two words: proxy index and number of words used for Proxy's data
                proxyData = array.array('I')
                proxyData.fromfile(fileDesc, 2)
                nReadWords += 2
                if needToSwap: proxyData.byteswap()
                proxyIndex = proxyData[0]
                productionTag = eventProxies[proxyIndex][2]
                proxyLength = proxyData[1]
                if eventProxies[proxyIndex][0] == 'VersionInfo':
                    # now let's read proxy's data
                    versionInfo, dict = proxyReader(fileDesc, needToSwap,
                                                    proxyLength)
                    for key in dict.keys():
                        if versionInfoDict.has_key(key):
                            if not versionInfoDict[key].count(dict[key]):
                                versionInfoDict[key] += [dict[key]]
                        else:
                            versionInfoDict[key] = [dict[key]]
                    if not versionInfoList.count((productionTag, versionInfo)):
                        versionInfoList.append((productionTag, versionInfo))
                else:
                    fileDesc.seek(proxyLength * 4, 1)
                nReadWords += proxyLength
            # read last word of the recordData
            lastWord = array.array('I')
            lastWord.fromfile(fileDesc, 1)
            if needToSwap: lastWord.byteswap()
            if lastWord[0] != recordDataLength[0] or fileDesc.tell(
            ) != endRecPos:
                raise "While decoding proxies we went too far"
        # once we reach this point, we found VersionInfo in beginrun record
            break
        except EOFError:
            break
    keys = versionInfoDict.keys()
    maxKey = max(keys)
    svName = versionInfoDict[maxKey][0]
    keys.remove(maxKey)
    parents = ""
    if len(keys):
        parents = versionInfoDict[max(keys)]
    return (svName, parents, versionInfoList, versionInfoDict)
Exemplo n.º 8
0
def dump(fileName,verbose=0,iStream='event'):
    """Dump content of pds file to stdout"""
    svName,parents,verList,verDict = decodeVersionInfo(fileName)
    if not parents:
       parents = 'N/A'
    if svName:
       print "Versioning information:"
       print "-----------------------"
       print "data version name: '%s'"%svName
       print "list of parents  :",parents
	
    PDSSIGNATURE=3141592 # magic number for PDS format, as it should be
    pdsSignature=0       # signature we read from given PDS file

    fileDesc = open(fileName,'rb')
    headerHeader = array.array('I')
    headerHeader.fromfile(fileDesc, 3)

    # to handle endianess, read pdsSignature
    pdsSignature = headerHeader[0]>>8
    pds_utils.pdsSignature=pdsSignature
    pds_utils.PDSSIGNATURE=PDSSIGNATURE

    if pdsSignature != PDSSIGNATURE:
       print "File was produced on another endian machine, byte swapping is enabled"
    if pdsSignature != PDSSIGNATURE: headerHeader.byteswap()
    print
    print "List of streams in a file:",
    streamNames = pds_utils.name_list_from_file(fileDesc)
    for x in streamNames: print x,
    print
    
    shProxyNames = pds_utils.name_list_from_file(fileDesc)
    shNames = pds_utils.name_list_from_file(fileDesc)
    if verbose:
       print "List of types that storage helper proxy factories have:"
       for x in shProxyNames: print x,
       print "\n\n"
       print "List of types that only have storage helpers"
       for x in shNames: print x,
       print

    # grab the rest of the header
    restOfHeader = array.array('I')
    # the header size ignores the first 3 words in the event
    restOfHeader.fromfile(fileDesc, headerHeader[2] -fileDesc.tell()/4 +3)
    if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
    if restOfHeader[-1] != headerHeader[2]:
       raise "header inconsistent"
    proxiesInStreams = pds_utils.find_proxies_in_streams( restOfHeader, streamNames, shProxyNames)
    print
    for idx in xrange(0,len(streamNames)):
        print "List of data keys in stream:",streamNames[idx]
        print pds_utils.printProxies(proxiesInStreams[idx])

    # want to only look at stuff in given stream (default is event stream)
    if not streamNames.count(iStream):
       print "There is no stream %s in %s"%(iStream,fileName)
       return
    eventIndex = streamNames.index(iStream)

    # create a structure to hold our info
    eventProxies = proxiesInStreams[eventIndex]
    accumulatedData = []
    for proxy in eventProxies:
       accumulatedData.append([0,488888888,0])

    # start to read the rest of the file
    numEvents = 0

    firstSV=""
    lastSV =""
    while 1:
       try:
	  endReason = ""
	  pos = fileDesc.tell()
	  recordHeader = array.array('I')
	  recordHeader.fromfile(fileDesc,5)
	  if pdsSignature != PDSSIGNATURE: recordHeader.byteswap()
	
	  uid = ((recordHeader[4]<<32)|recordHeader[3])
	  if not len(firstSV):
	     firstSV="%d/%d/%d"%(recordHeader[1],recordHeader[2],uid)
	  lastSV="%d/%d/%d"%(recordHeader[1],recordHeader[2],uid)

	  if verbose:
	     print "********************"
	     print "position           : %s"%pos
	     print "stream             : "+streamNames[recordHeader[0]]
	     print "SyncValue          : %s/%s/%s"%(recordHeader[1],recordHeader[2],uid)

	  endReason = "bad Record: record size"
	  recordDataLength = array.array('I')
	  recordDataLength.fromfile(fileDesc,1)
	  if pdsSignature != PDSSIGNATURE: recordDataLength.byteswap()
	  if verbose:
	     print "size of record data: "+str(recordDataLength[0])
	  
	  endReason = "bad Record: record (type "+streamNames[recordHeader[0]]+") (size " + str(recordDataLength[0]) +")"
	  begRecPos = fileDesc.tell()
	  recordData = array.array('I')
	  recordData.fromfile(fileDesc, recordDataLength[0])
	  if pdsSignature != PDSSIGNATURE: recordData.byteswap()
	  
	  numEvents = numEvents+1
	  index = 0
	  endIndex = len(recordData)
	  dataInfo     = []
	  dataInfoSize = []
	  lengthOffset = 0
	  while index < endIndex - 1:
	    proxyIndex = int(recordData[index])
	    index = index + 1
	    dataSize =  recordData[index]
	    index = index + int(dataSize) + 1
	    fullDataSize = dataSize+2
	    strData = str(proxiesInStreams[recordHeader[0]][proxyIndex])
	    if lengthOffset<len(strData):
	       lengthOffset=len(strData)
	    dataInfo.append(strData)
	    dataInfoSize.append(fullDataSize)
#            if verbose:
#               print str(proxiesInStreams[recordHeader[0]][proxyIndex]) + " size "+str(fullDataSize)
	    if recordHeader[0] == eventIndex:
		accumulatedData[proxyIndex][0] = accumulatedData[proxyIndex][0]+fullDataSize
		if accumulatedData[proxyIndex][1] > fullDataSize:
		   accumulatedData[proxyIndex][1] = fullDataSize
		if accumulatedData[proxyIndex][2] < fullDataSize:
		   accumulatedData[proxyIndex][2] = fullDataSize
	  if verbose:
	     print
	     for idx in xrange(0,len(dataInfo)):
	         s = dataInfo[idx]
		 print "%s size %s"%(s.ljust(lengthOffset),dataInfoSize[idx])

       except EOFError:
	 if verbose and endReason:
	    print "########################################################"
	    print endReason
	    print "########################################################"
	 break


    index = 0
    if numEvents == 0:
      raise "no events found"

    if verbose:
       print "######### Summary ##########"
       print "Proxies in %s:"%iStream
       for item in eventProxies:
	  print "%s : %i %f %i" % (item, accumulatedData[index][1], float(accumulatedData[index][0])/float(numEvents), accumulatedData[index][2])
	  index = index + 1

    print "First sync value     :",firstSV
    print "Last sync value      :",lastSV
    print "Number of syncValues :",numEvents
    print
Exemplo n.º 9
0
def decodeVersionInfo(fileName):
    """Decode VersionInfo from beginrun record. 
    VersionInfo consists of:
       - softwareRelease : string
       - specificVersionName : string
       - configurationHash : string
       - ordinal : unsigned int no packing
       - ancestors : container of string
    So, it always grows, e.g. post-p2 file will contains two VersionInfo's, one
    for itself and one for it's parent. So, the underlying algorithm creates
    a list of VersionInfo's in the following format:
    [(childTag,[softRel,svName,hash,id,parent1,parent2,...]),(parentTag,[...])]
    This method returns (svName,[listOfParents]) tuple
    """
    emptyTuple  = ('','','','')
    PDSSIGNATURE=3141592 # magic number for PDS format, as it should be
    pdsSignature=0       # signature we read from given PDS file

    fileDesc = open(fileName,'rb')
    headerHeader = array.array('I')
    headerHeader.fromfile(fileDesc, 3)

    # to handle endianess, read pdsSignature
    pdsSignature = headerHeader[0]>>8
    pds_utils.pdsSignature=pdsSignature
    pds_utils.PDSSIGNATURE=PDSSIGNATURE

    needToSwap = 0
    if pdsSignature != PDSSIGNATURE:
       needToSwap = 1
       headerHeader.byteswap()
    streamNames = pds_utils.name_list_from_file(fileDesc)
    shProxyNames = pds_utils.name_list_from_file(fileDesc)
    shNames = pds_utils.name_list_from_file(fileDesc)

    # grab the rest of the header
    restOfHeader = array.array('I')
    # the header size ignores the first 3 words in the event
    restOfHeader.fromfile(fileDesc, headerHeader[2] -fileDesc.tell()/4 +3)
    if needToSwap: restOfHeader.byteswap()
    if restOfHeader[-1] != headerHeader[2]:
       raise "header inconsistent"
    proxiesInStreams = pds_utils.find_proxies_in_streams( restOfHeader, streamNames, shProxyNames)

    # want to only look at stuff in beginrun stream
    if not streamNames.count('beginrun'):
       return emptyTuple
    eventIndex = streamNames.index('beginrun')

    # create a structure to hold our info
    eventProxies    = proxiesInStreams[eventIndex]
    accumulatedData = []
    foundVersionInfo= 0
    for proxy in eventProxies:
	accumulatedData.append([0,488888888,0])
	if proxy[0]=='VersionInfo':
	   foundVersionInfo=1
           break
    # check if VersionInfo is present in a proxy list
    if not foundVersionInfo:
       return emptyTuple

    # start to read the rest of the file
    numEvents = 0
    nWordSize = 4 # we use to use 32-bit words, which is 4 bytes
    firstSV   =""
    lastSV    =""
    versionInfoList = []
    versionInfoDict = {}
    while 1:
       try:
	  endReason = ""
	  recordHeader = array.array('I')
	  recordHeader.fromfile(fileDesc,5)
	  if needToSwap: recordHeader.byteswap()
	
	  uid = ((recordHeader[4]<<32)|recordHeader[3])
	  if not len(firstSV):
	     firstSV="%d/%d/%d"%(recordHeader[1],recordHeader[2],uid)
	  lastSV="%d/%d/%d"%(recordHeader[1],recordHeader[2],uid)

	  endReason = "bad Record: record size"
	  recordDataLength = array.array('I')
	  recordDataLength.fromfile(fileDesc,1)
	  if needToSwap: recordDataLength.byteswap()
	  
	  if recordHeader[0]>len(streamNames)-1:
	     break
	  endReason = "bad Record: record (type "+streamNames[recordHeader[0]]+") (size " + str(recordDataLength[0]) +")"
	  curStream = streamNames[recordHeader[0]]
	  if curStream!='beginrun':
	     continue
	  begRecPos = fileDesc.tell()
	  recordData= array.array('I')
	  recordData.fromfile(fileDesc, recordDataLength[0]-1)
	  if needToSwap: recordData.byteswap()
	  testLastWord= array.array('I')
	  testLastWord.fromfile(fileDesc,1)
	  if needToSwap: testLastWord.byteswap()
	  endRecPos = fileDesc.tell()
	  
	  ############ remove later this block, it's a full dataRecord
	  fileDesc.seek(begRecPos)
	  cData = array.array('c')
	  cData.fromfile(fileDesc, (recordDataLength[0])*nWordSize)
	  ################
	
	  fileDesc.seek(begRecPos)
	  nReadWords = 0
	  while nReadWords<(recordDataLength[0]-1):
	     # let's read two words: proxy index and number of words used for Proxy's data
	     proxyData = array.array('I')
	     proxyData.fromfile(fileDesc,2)
	     nReadWords+=2
	     if needToSwap: proxyData.byteswap()
	     proxyIndex = proxyData[0]
	     productionTag = eventProxies[proxyIndex][2]
	     proxyLength= proxyData[1]
	     if eventProxies[proxyIndex][0]=='VersionInfo':
		# now let's read proxy's data
		versionInfo, dict = proxyReader(fileDesc,needToSwap,proxyLength)
		for key in dict.keys():
		    if versionInfoDict.has_key(key):
		       if not versionInfoDict[key].count(dict[key]):
			  versionInfoDict[key]+= [dict[key]]
		    else:
		       versionInfoDict[key] = [dict[key]]
		if not versionInfoList.count((productionTag,versionInfo)):
		   versionInfoList.append((productionTag,versionInfo))
             else:
	        fileDesc.seek(proxyLength*4,1)
	     nReadWords+=proxyLength
	  # read last word of the recordData
	  lastWord = array.array('I')
	  lastWord.fromfile(fileDesc,1)
	  if needToSwap: lastWord.byteswap()
	  if lastWord[0]!=recordDataLength[0] or fileDesc.tell()!=endRecPos:
	     raise "While decoding proxies we went too far"
          # once we reach this point, we found VersionInfo in beginrun record
          break
       except EOFError:
	 break
    keys   = versionInfoDict.keys()
    maxKey = max(keys)
    svName = versionInfoDict[maxKey][0]
    keys.remove(maxKey)
    parents= ""
    if len(keys):
       parents= versionInfoDict[max(keys)]
    return (svName,parents,versionInfoList,versionInfoDict)
Exemplo n.º 10
0
def buildLocationHeader(iPDSFileName, iFileID):
    """Build a PDS location header, from given pds file name and file id"""
#    print "iPDSFileName",iPDSFileName

    pdsFile = open(iPDSFileName,'rb')
    pdsFileID = int(iFileID)

    pdsHeader = array.array('I')
    pdsHeader.fromfile(pdsFile, 3)
    global pdsSignature
    pdsSignature=pdsHeader[0]>>8
    pds_utils.PDSSIGNATURE=PDSSIGNATURE
    pds_utils.pdsSignature=pdsSignature
    if pdsSignature != PDSSIGNATURE:
       pdsHeader.byteswap()
       needToSwap=1
    else:
       needToSwap=0
   
    locationHeader = array.array('I')
    locationHeader.fromlist([2951413*256])
    #file id for master index
    locationHeader.append(0)
    #remaining number of words in of header
    locationHeader.append(0)
    #associated file list
    fileList = array.array('I')
    # for now only set the low word for the file ID
    fileList.fromlist([1,pdsFileID,0])
    locationHeader =locationHeader + fileList

    # now get the names of the streams
    #  the location file and the PDS file use the same format for the
    #  stream names so just have to copy the info
    numberOfRecordNameWords = array.array('I')
    numberOfRecordNameWords.fromfile(pdsFile,1)
    if pdsSignature != PDSSIGNATURE: numberOfRecordNameWords.byteswap()

    #need number of names
    pdsFile.seek(4*4)
    nameChars = array.array('c')
    nameChars.fromfile(pdsFile,numberOfRecordNameWords[0]*4)
    if pdsSignature != PDSSIGNATURE: nameChars.byteswap()
    streamNames = pds_utils.name_list(nameChars)
    sortedStreamNames = list(streamNames)
    sortedStreamNames.sort()

    #build conversion from new stream index to old index
    newStreamIndex2OldIndex = []
    oldStreamIndex2NewIndex = [0]*len(streamNames)
    streamNameString =""
    for name in sortedStreamNames:
        newStreamIndex2OldIndex.append(streamNames.index(name))
        oldStreamIndex2NewIndex[newStreamIndex2OldIndex[-1] ] = len(newStreamIndex2OldIndex)-1
        streamNameString = streamNameString+name +"\0"
    streamNameString = streamNameString[:-1]
    while 0 != len(streamNameString) % 4:
        streamNameString = streamNameString + "\0"
    nameWords = array.array('I')
    nameWords.fromstring(streamNameString)

    locationHeader = locationHeader + numberOfRecordNameWords+nameWords

    #print streamNames
    shProxyNames = pds_utils.name_list_from_file(pdsFile)
    #print shProxyNames
    #print len(shProxyNames)
    shNames = pds_utils.name_list_from_file(pdsFile)
    #print shNames

    #grab the rest of the header
    restOfHeader = array.array('I')
    #the header size ignores the first 3 words in the event
    restOfHeader.fromfile(pdsFile, pdsHeader[2] -pdsFile.tell()/4 +3)
    if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
    #print restOfHeader
    if restOfHeader[-1] != pdsHeader[2]:
      raise "header inconsistent"

    #create the list of 'type' 'usage' 'production' tags for each stream
    def proxySort(a,b):
        #print a, b
        temp = cmp(a[0],b[0])
        if 0 == temp:
	  temp=cmp(a[1],b[1])
	  if 0 == temp:
	     temp=cmp(a[2],b[2])
        return temp
    proxiesInStreams = pds_utils.find_proxies_in_streams( restOfHeader, streamNames, shProxyNames)
#sortProxiesInStreams = proxiesInStreams
    for proxies in proxiesInStreams:
        proxies.sort(proxySort)

    #reserve space for our hash
    dataKeyHashIndex = len(locationHeader)
    dataKeyHashArray = array.array('I')
    dataKeyHashArray.fromlist([0]*5)
    locationHeader += dataKeyHashArray

    maxNProxies = 0
    nProxies = array.array('I')
    nProxies.fromlist([0])
    proxiesArray = []
    for oldIndex in newStreamIndex2OldIndex:
	#print oldIndex
	proxies = proxiesInStreams[oldIndex]
	nProxies[0] = len(proxies)
	if nProxies[0] > maxNProxies:
	   maxNProxies = nProxies[0]
	locationHeader +=nProxies
	datakeys = ""
	#now add each string
	proxyList= []
	for proxy in proxies:
	    proxyList.append(proxy)
	    for key in proxy:
	        datakeys +=key+"\0"
	    while( len(datakeys) % 4 ):
	        datakeys +="\0"
	#print len(datakeys)
	#print datakeys
	#print len(datakeys)
	dataKeysArray=array.array('I')
	dataKeysArray.fromstring(datakeys)
#        proxiesArray+=[dataKeysArray.tolist()]
	proxiesArray+=[proxyList]
	#nProxies[0] = len(dataKeysArray)
	#locationHeader += nProxies
	locationHeader +=dataKeysArray
    #calculate the hash
    hash = sha.new( locationHeader[dataKeyHashIndex+1:].tostring() ).digest()
    #print sha.new( locationHeader[dataKeyHashIndex+1:].tostring() ).hexdigest()
    dataKeyHashArray = array.array('I')
    dataKeyHashArray.fromstring(hash)
    locationHeader[dataKeyHashIndex:dataKeyHashIndex+5]=dataKeyHashArray

    locationHeaderBeforePadding = locationHeader.tolist()
    #pad header so Records begin on 8 byte boundary
    if not (len(locationHeader) % 2):
       locationHeader.fromlist([0])
    headerLength = len(locationHeader)+1-3
    locationHeader.fromlist([headerLength])
    locationHeader[2] = headerLength

    #pad the max number of ASUs to be a multiple of 8
    nASUperRecord = maxNProxies
    while ((nASUperRecord + 4) % 8):
      nASUperRecord +=1
      
    #For each stream, create the ASU to file ID list
    whichFileForStream = []
    whichFileForStreamFake = []
    for proxies in proxiesInStreams:
      whichFile = array.array('B')
      fakeArray = array.array('B')

      whichFile.fromlist([0]*len(proxies)+[255]*(nASUperRecord-len(proxies)))
      for x in xrange(0,len(whichFile)):
          fakeArray.append(255)
      #print whichFile
      whichFileForStream.append(whichFile)
      whichFileForStreamFake.append(fakeArray)

    return (pdsFile,locationHeader,locationHeaderBeforePadding,nameWords,dataKeyHashArray,proxiesArray,streamNames,oldStreamIndex2NewIndex,newStreamIndex2OldIndex,whichFileForStream,whichFileForStreamFake,needToSwap)
Exemplo n.º 11
0
def getStreamDataKeyDictFromPDS(iPDSFileName):
    """Extract from PDS file streamDataKey dictionary"""

    pdsFile = open(iPDSFileName,'rb')
    pdsHeader = array.array('I')
    pdsHeader.fromfile(pdsFile, 3)
    global pdsSignature
    pdsSignature=pdsHeader[0]>>8
    pds_utils.PDSSIGNATURE=PDSSIGNATURE
    pds_utils.pdsSignature=pdsSignature
    if pdsSignature != PDSSIGNATURE:
       pdsHeader.byteswap()
       needToSwap=1
    else:
       needToSwap=0
   
    # now get the names of the streams
    #  the location file and the PDS file use the same format for the
    #  stream names so just have to copy the info
    numberOfRecordNameWords = array.array('I')
    numberOfRecordNameWords.fromfile(pdsFile,1)
    if pdsSignature != PDSSIGNATURE: numberOfRecordNameWords.byteswap()

    #need number of names
    pdsFile.seek(4*4)
    nameChars = array.array('c')
    nameChars.fromfile(pdsFile,numberOfRecordNameWords[0]*4)
    if pdsSignature != PDSSIGNATURE: nameChars.byteswap()
    streamNames = pds_utils.name_list(nameChars)
    sortedStreamNames = list(streamNames)
    sortedStreamNames.sort()

    #build conversion from new stream index to old index
    newStreamIndex2OldIndex = []
    for name in sortedStreamNames:
        newStreamIndex2OldIndex.append(streamNames.index(name))

    #print streamNames
    shProxyNames = pds_utils.name_list_from_file(pdsFile)
    #print shProxyNames
    #print len(shProxyNames)
    shNames = pds_utils.name_list_from_file(pdsFile)
    #print shNames

    #grab the rest of the header
    restOfHeader = array.array('I')
    #the header size ignores the first 3 words in the event
    restOfHeader.fromfile(pdsFile, pdsHeader[2] -pdsFile.tell()/4 +3)
    if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
    #print restOfHeader
    if restOfHeader[-1] != pdsHeader[2]:
      raise "header inconsistent"

    #create the list of 'type' 'usage' 'production' tags for each stream
    def proxySort(a,b):
        #print a, b
        temp = cmp(a[0],b[0])
        if 0 == temp:
	  temp=cmp(a[1],b[1])
	  if 0 == temp:
	     temp=cmp(a[2],b[2])
        return temp
    proxiesInStreams = pds_utils.find_proxies_in_streams( restOfHeader, streamNames, shProxyNames)
#sortProxiesInStreams = proxiesInStreams
    for proxies in proxiesInStreams:
        proxies.sort(proxySort)

    streamDataKeyDict = {}
    for oldIndex in newStreamIndex2OldIndex:
	#print oldIndex
	stream  = streamNames[oldIndex] 
	proxies = proxiesInStreams[oldIndex]
	streamDataKeyDict[stream] = proxies

    return streamDataKeyDict
Exemplo n.º 12
0
def pdsParser(file, what=""):
    """PDS file parser. Return a list of run/uid/sync.values/proxies in given file"""
    fileDesc = open(file, 'rb')
    headerHeader = array.array('I')
    headerHeader.fromfile(fileDesc, 3)
    #    global pdsSignature
    pdsSignature = headerHeader[0] >> 8
    pds_utils.PDSSIGNATURE = PDSSIGNATURE
    pds_utils.pdsSignature = pdsSignature
    if pdsSignature != PDSSIGNATURE: headerHeader.byteswap()
    # invoke name_list_from_file with additional argument 1 which means
    # return number of words for those names
    streamNames, streamNamesWords = pds_utils.name_list_from_file(fileDesc, 1)
    shProxyNames, shProxyNamesWords = pds_utils.name_list_from_file(
        fileDesc, 1)
    shNames, shNamesWords = pds_utils.name_list_from_file(fileDesc, 1)

    #grab the rest of the header
    restOfHeader = array.array('I')
    #the header size ignores the first 3 words in the event
    restOfHeader.fromfile(fileDesc, headerHeader[2] - fileDesc.tell() / 4 + 3)
    if pdsSignature != PDSSIGNATURE: restOfHeader.byteswap()
    if restOfHeader[-1] != headerHeader[2]:
        raise "header inconsistent"
    proxiesInStreams = pds_utils.find_proxies_in_streams(
        restOfHeader, streamNames, shProxyNames)

    # define proxy dictionary dict={'stream':listOfProxiesInThatStream}
    pList = []
    for str in streamNames:
        strIndex = streamNames.index(str)
        pList.append((str, proxiesInStreams[strIndex]))

    posOfLastHeaderWords = fileDesc.tell()

    #start to read the rest of the file
    runList = []
    uidList = []
    syncList = []
    while 1:
        try:
            endReason = ""
            recordHeader = array.array('I')
            recordHeader.fromfile(fileDesc, 5)
            if pdsSignature != PDSSIGNATURE: recordHeader.byteswap()
            # read stream id and convert it to stream name
            streamId = int(recordHeader[0])
            stream = streamNames[streamId]

            # read uid
            uid = ((recordHeader[4] << 32) | recordHeader[3])

            # form syncValue
            syncValue = (recordHeader[1], recordHeader[2], uid, stream)
            syncList.append(syncValue)

            # read run record
            runNumber = recordHeader[1]
            if len(runList) == 0 or (uid != uidList[-1]
                                     or runNumber != runList[-1]):
                runList.append(runNumber)
                uidList.append(uid)

            # read the record data
            recordDataSize = array.array('I')
            recordDataSize.fromfile(fileDesc, 1)
            if pdsSignature != PDSSIGNATURE: recordDataSize.byteswap()
            #       print "recordDataSize",recordDataSize

            recordData = array.array('I')
            recordData.fromfile(fileDesc, recordDataSize[0] - 1)
            if pdsSignature != PDSSIGNATURE: recordData.byteswap()

            recordDataEnd = array.array('I')
            recordDataEnd.fromfile(fileDesc, 1)
            if pdsSignature != PDSSIGNATURE: recordDataEnd.byteswap()

            if recordDataSize[0] != recordDataEnd[0]:
                print "Error of reading record data"
                sys.exit(1)

        except EOFError:
            break


#    runList.sort()
#    uidList.sort()
#    syncList.sort()
    pList.sort()
    if what == "run":
        return runList
    elif what == "uid":
        return uidList
    elif what == "syncValue":
        return syncList
    elif what == "proxies":
        return pList

    returnList = [runList, uidList, pList, syncList]
    returnList += [streamNames, streamNamesWords]
    returnList += [shProxyNames, shProxyNamesWords]
    returnList += [shNames, shNamesWords]
    returnList += [proxiesInStreams, posOfLastHeaderWords]
    return returnList