def __getValueForKeyInIndex__(self, key, nestedIndex): try: offsetData = self.indexedOffsetTable[nestedIndex].Get(key) except TypeError: raise KeyError('Key (%s) not found in %s' % (str(key), self.__path__)) if offsetData is None: raise KeyError('Key (%s) not found in %s' % (str(key), self.__path__)) offset = offsetData[0] size = offsetData[1] valueSchema = self.indexedSchemas[nestedIndex]['valueTypes'] offsetToStartOfData = 4 + offset + self.offsetToData if valueSchema.get('buildIndex', False): v = IndexLoader(self.fileObject, 100, valueSchema, FsdDataPathObject('[%s]' % str(key), parent=self.__path__), self.__extraState__, offsetToData=offsetToStartOfData, offsetToFooter=offsetToStartOfData + size) return v else: itemData = readBinaryDataFromFileAtOffset(self.fileObject, offsetToStartOfData, size) dataAsBuffer = ctypes.create_string_buffer(itemData, len(itemData)) v = self.__extraState__.RepresentSchemaNode( dataAsBuffer, 0, FsdDataPathObject('[%s]' % str(key), parent=self.__path__), valueSchema) return v
def next(self): self.index += 1 if self.index == self.count: raise StopIteration() return self.__extraState__.RepresentSchemaNode( self.data, self.offset + self.itemSize * self.index, FsdDataPathObject('[%s]' % str(self.index), parent=self.__path__), self.itemSchema)
def CreatePythonDictOffset(schema, binaryFooterData, path, extraState): useOptimizedPythonOffsetStructure = schema['keyTypes']['type'] == 'int' buff = ctypes.create_string_buffer(binaryFooterData, len(binaryFooterData)) if useOptimizedPythonOffsetStructure: return StandardFSDOptimizedDictFooter(buff, schema) else: return StandardFSDDictFooter( buff, 0, schema['keyFooter'], FsdDataPathObject('<keyFooter>', parent=path), extraState)
def __getitem__(self, key): if type(key) not in (int, long): raise TypeError('Invalid key type') if key < 0 or key >= self.count: raise IndexError('Invalid item index %i for list of length %i' % (key, self.count)) countOffset = 0 if self.fixedLength else 4 totalOffset = self.offset + countOffset + self.itemSize * key return self.__extraState__.RepresentSchemaNode( self.data, totalOffset, FsdDataPathObject('[%s]' % str(key), parent=self.__path__), self.itemSchema)
def __getitem__(self, key): if type(key) not in (int, long): raise TypeError('Invalid key type') if key < 0 or key >= self.count: raise IndexError('Invalid item index %i for list of length %i' % (key, self.count)) countOffset = 0 if self.fixedLength else 4 dataOffsetFromObjectStart = structTypes.uint32.unpack_from( self.data, self.offset + countOffset + 4 * key)[0] return self.__extraState__.RepresentSchemaNode( self.data, self.offset + dataOffsetFromObjectStart, FsdDataPathObject('[%s]' % str(key), parent=self.__path__), self.itemSchema)
def __GetItemFromOffsetAndSize__(self, key, itemOffset, itemSize): newOffset = 4 + self.offsetToData + itemOffset valueSchema = self.schema['valueTypes'] if valueSchema.get('buildIndex', False): v = IndexLoader(self.fileObject, self.cacheSize, valueSchema, FsdDataPathObject('[%s]' % str(key), parent=self.__path__), self.__extraState__, offsetToData=newOffset, offsetToFooter=newOffset + itemSize) return v else: itemData = readBinaryDataFromFileAtOffset(self.fileObject, newOffset, itemSize) dataAsBuffer = ctypes.create_string_buffer(itemData, len(itemData)) v = self.__extraState__.RepresentSchemaNode( dataAsBuffer, 0, FsdDataPathObject('[%s]' % str(key), parent=self.__path__), valueSchema) return v
def __getitem__(self, key): if key not in self.__schema__['attributes']: raise KeyError( "Object: %s - Attribute '%s' is not in the schema for this object. It may be removed by the 'usage' flag under the build configuration that produced this data." % (self.__path__, key)) attributeSchema = self.__schema__['attributes'][key] if key in self.__schema__['constantAttributeOffsets']: return self.__extraState__.RepresentSchemaNode( self.__data__, self.__offset__ + self.__schema__['constantAttributeOffsets'][key], FsdDataPathObject('.%s' % str(key), parent=self.__path__), attributeSchema) else: if key not in self.__offsetAttributesOffsetLookupTable__: if 'default' in attributeSchema: return attributeSchema['default'] raise KeyError( "Object: %s - Attribute '%s' is not present on this instance." % (self.__path__, key)) return self.__extraState__.RepresentSchemaNode( self.__data__, self.__variableDataOffsetBase__ + self.__offsetAttributesOffsetLookupTable__[key], FsdDataPathObject('.%s' % str(key), parent=self.__path__), attributeSchema)
def __init__(self, fileObject, cacheSize, schema, path, extraState, offsetToData=0): IndexLoader.__init__(self, fileObject, cacheSize, schema, path, extraState, offsetToData=offsetToData) offsetToAttributeLookupTableSize = self.offsetToData + 4 + self.fileObjectSize - 4 - self.footerDataSize - 4 self.attributeLookupTableSize = readIntFromFileAtOffset( self.fileObject, offsetToAttributeLookupTableSize) startOfAttributeLookupTable = offsetToAttributeLookupTableSize - self.attributeLookupTableSize attributeLookupTable = readBinaryDataFromFileAtOffset( self.fileObject, startOfAttributeLookupTable, self.attributeLookupTableSize) attributeLookupTable = ctypes.create_string_buffer( attributeLookupTable, len(attributeLookupTable)) self.nestedIndexIdToOffsetTable = self.__extraState__.RepresentSchemaNode( attributeLookupTable, 0, FsdDataPathObject('<MultiIndexAttributes>', parent=self.__path__), self.schema['subIndexOffsetLookup']) log.info('Attribute lookup table for MultiIndex %s. %s', self.__path__, self.__extraState__.FormatSize(len(attributeLookupTable))) self.attributeCache = {} nestedIndexIdToOffsetData = {} combinedNestedOffsetDataSize = 0 for index, offsetAndSizeOfOffsetData in self.nestedIndexIdToOffsetTable.iteritems( ): offsetToNestedDictionaryOffsetData = offsetToData + offsetAndSizeOfOffsetData.offset binaryNestedDictionaryOffsetData = readBinaryDataFromFileAtOffset( self.fileObject, offsetToNestedDictionaryOffsetData, offsetAndSizeOfOffsetData.size) combinedNestedOffsetDataSize += len( binaryNestedDictionaryOffsetData) nestedIndexIdToOffsetData[index] = CreateDictFooter( self.schema['indexableSchemas'][index], binaryNestedDictionaryOffsetData, FsdDataPathObject('<MultiIndexAttributes>.footer[%i]' % index, parent=self.__path__), self.__extraState__) log.info( 'Total size of nested offset data kept in memory %s. %s' % (self.__path__, self.__extraState__.FormatSize(combinedNestedOffsetDataSize))) for indexName, indices in self.schema['indexNameToIds'].iteritems(): indexToDictOffsetTable = {} indexToSchema = {} for index in indices: indexToDictOffsetTable[index] = nestedIndexIdToOffsetData[ index] indexToSchema[index] = self.schema['indexableSchemas'][index] self.attributeCache[indexName] = SubIndexLoader( indexToDictOffsetTable, indexToSchema, offsetToData, self.fileObject, self.__extraState__, FsdDataPathObject('<MultiIndexAttributes>.%s' % indexName, parent=self.__path__))
def __GetItemFromOffset__(self, key, offset): return self.__extraState__.RepresentSchemaNode( self.data, self.offset + 4 + offset, FsdDataPathObject('[%s]' % str(key), parent=self.__path__), self.schema['valueTypes'])