def __GetItemFromOffsetAndSize__(self, key, itemOffset, itemSize):
newOffset = 4 + self.offsetToData + itemOffset
valueSchema = self.schema['valueTypes']
if valueSchema.get('buildIndex', False):
v = IndexLoader(self.fileObject, self.cacheSize, valueSchema, FsdDataPathObject('[%s]' % str(key), parent=self.__path__), self.__extraState__, offsetToData=newOffset, offsetToFooter=newOffset + itemSize)
return v
else:
itemData = readBinaryDataFromFileAtOffset(self.fileObject, newOffset, itemSize)
dataAsBuffer = ctypes.create_string_buffer(itemData, len(itemData))
v = self.__extraState__.RepresentSchemaNode(dataAsBuffer, 0, FsdDataPathObject('[%s]' % str(key), parent=self.__path__), valueSchema)
return v
def CreatePythonDictOffset(schema, binaryFooterData, path, extraState):
useOptimizedPythonOffsetStructure = schema['keyTypes']['type'] == 'int'
buff = ctypes.create_string_buffer(binaryFooterData, len(binaryFooterData))
if useOptimizedPythonOffsetStructure:
return StandardFSDOptimizedDictFooter(buff, schema)
else:
return StandardFSDDictFooter(buff, 0, schema['keyFooter'], FsdDataPathObject('<keyFooter>', parent=path), extraState)
def next(self):
self.index += 1
if self.index == self.count:
raise StopIteration()
return self.__extraState__.RepresentSchemaNode(
self.data, self.offset + self.itemSize * self.index,
FsdDataPathObject('[%s]' % str(self.index), parent=self.__path__),
self.itemSchema)
def __getValueForKeyInIndex__(self, key, nestedIndex):
try:
offsetData = self.indexedOffsetTable[nestedIndex].Get(key)
except TypeError:
raise KeyError('Key (%s) not found in %s' % (str(key), self.__path__))
if offsetData is None:
raise KeyError('Key (%s) not found in %s' % (str(key), self.__path__))
offset = offsetData[0]
size = offsetData[1]
valueSchema = self.indexedSchemas[nestedIndex]['valueTypes']
offsetToStartOfData = 4 + offset + self.offsetToData
if valueSchema.get('buildIndex', False):
v = IndexLoader(self.fileObject, 100, valueSchema, FsdDataPathObject('[%s]' % str(key), parent=self.__path__), self.__extraState__, offsetToData=offsetToStartOfData, offsetToFooter=offsetToStartOfData + size)
return v
else:
itemData = readBinaryDataFromFileAtOffset(self.fileObject, offsetToStartOfData, size)
dataAsBuffer = ctypes.create_string_buffer(itemData, len(itemData))
v = self.__extraState__.RepresentSchemaNode(dataAsBuffer, 0, FsdDataPathObject('[%s]' % str(key), parent=self.__path__), valueSchema)
return v
def __getitem__(self, key):
if type(key) not in (int, long):
raise TypeError('Invalid key type')
if key < 0 or key >= self.count:
raise IndexError('Invalid item index %i for list of length %i' %
(key, self.count))
countOffset = 0 if self.fixedLength else 4
totalOffset = self.offset + countOffset + self.itemSize * key
return self.__extraState__.RepresentSchemaNode(
self.data, totalOffset,
FsdDataPathObject('[%s]' % str(key), parent=self.__path__),
self.itemSchema)
def __getitem__(self, key):
if type(key) not in (int, long):
raise TypeError('Invalid key type')
if key < 0 or key >= self.count:
raise IndexError('Invalid item index %i for list of length %i' %
(key, self.count))
countOffset = 0 if self.fixedLength else 4
dataOffsetFromObjectStart = structTypes.uint32.unpack_from(
self.data, self.offset + countOffset + 4 * key)[0]
return self.__extraState__.RepresentSchemaNode(
self.data, self.offset + dataOffsetFromObjectStart,
FsdDataPathObject('[%s]' % str(key), parent=self.__path__),
self.itemSchema)
def __getitem__(self, key):
if key not in self.__schema__['attributes']:
raise KeyError("Object: %s - Attribute '%s' is not in the schema for this object. It may be removed by the 'usage' flag under the build configuration that produced this data." % (self.__path__, key))
attributeSchema = self.__schema__['attributes'][key]
if key in self.__schema__['constantAttributeOffsets']:
return self.__extraState__.RepresentSchemaNode(self.__data__, self.__offset__ + self.__schema__['constantAttributeOffsets'][key], FsdDataPathObject('.%s' % str(key), parent=self.__path__), attributeSchema)
else:
if key not in self.__offsetAttributesOffsetLookupTable__:
if 'default' in attributeSchema:
return attributeSchema['default']
raise KeyError("Object: %s - Attribute '%s' is not present on this instance." % (self.__path__, key))
return self.__extraState__.RepresentSchemaNode(self.__data__, self.__variableDataOffsetBase__ + self.__offsetAttributesOffsetLookupTable__[key], FsdDataPathObject('.%s' % str(key), parent=self.__path__), attributeSchema)
def __init__(self, fileObject, cacheSize, schema, path, extraState, offsetToData = 0):
IndexLoader.__init__(self, fileObject, cacheSize, schema, path, extraState, offsetToData=offsetToData)
offsetToAttributeLookupTableSize = self.offsetToData + 4 + self.fileObjectSize - 4 - self.footerDataSize - 4
self.attributeLookupTableSize = readIntFromFileAtOffset(self.fileObject, offsetToAttributeLookupTableSize)
startOfAttributeLookupTable = offsetToAttributeLookupTableSize - self.attributeLookupTableSize
attributeLookupTable = readBinaryDataFromFileAtOffset(self.fileObject, startOfAttributeLookupTable, self.attributeLookupTableSize)
attributeLookupTable = ctypes.create_string_buffer(attributeLookupTable, len(attributeLookupTable))
self.nestedIndexIdToOffsetTable = self.__extraState__.RepresentSchemaNode(attributeLookupTable, 0, FsdDataPathObject('<MultiIndexAttributes>', parent=self.__path__), self.schema['subIndexOffsetLookup'])
log.info('Attribute lookup table for MultiIndex %s. %s', self.__path__, self.__extraState__.FormatSize(len(attributeLookupTable)))
self.attributeCache = {}
nestedIndexIdToOffsetData = {}
combinedNestedOffsetDataSize = 0
for index, offsetAndSizeOfOffsetData in self.nestedIndexIdToOffsetTable.iteritems():
offsetToNestedDictionaryOffsetData = offsetToData + offsetAndSizeOfOffsetData.offset
binaryNestedDictionaryOffsetData = readBinaryDataFromFileAtOffset(self.fileObject, offsetToNestedDictionaryOffsetData, offsetAndSizeOfOffsetData.size)
combinedNestedOffsetDataSize += len(binaryNestedDictionaryOffsetData)
nestedIndexIdToOffsetData[index] = CreateDictFooter(self.schema['indexableSchemas'][index], binaryNestedDictionaryOffsetData, FsdDataPathObject('<MultiIndexAttributes>.footer[%i]' % index, parent=self.__path__), self.__extraState__)
log.info('Total size of nested offset data kept in memory %s. %s' % (self.__path__, self.__extraState__.FormatSize(combinedNestedOffsetDataSize)))
for indexName, indices in self.schema['indexNameToIds'].iteritems():
indexToDictOffsetTable = {}
indexToSchema = {}
for index in indices:
indexToDictOffsetTable[index] = nestedIndexIdToOffsetData[index]
indexToSchema[index] = self.schema['indexableSchemas'][index]
self.attributeCache[indexName] = SubIndexLoader(indexToDictOffsetTable, indexToSchema, offsetToData, self.fileObject, self.__extraState__, FsdDataPathObject('<MultiIndexAttributes>.%s' % indexName, parent=self.__path__))
def __GetItemFromOffset__(self, key, offset):
return self.__extraState__.RepresentSchemaNode(self.data, self.offset + 4 + offset, FsdDataPathObject('[%s]' % str(key), parent=self.__path__), self.schema['valueTypes'])