def __getValueForKeyInIndex__(self, key, nestedIndex):
try:
offsetData = self.indexedOffsetTable[nestedIndex].Get(key)
except TypeError:
raise KeyError('Key (%s) not found in %s' %
(str(key), self.__path__))
if offsetData is None:
raise KeyError('Key (%s) not found in %s' %
(str(key), self.__path__))
offset = offsetData[0]
size = offsetData[1]
valueSchema = self.indexedSchemas[nestedIndex]['valueTypes']
offsetToStartOfData = 4 + offset + self.offsetToData
if valueSchema.get('buildIndex', False):
v = IndexLoader(self.fileObject,
100,
valueSchema,
FsdDataPathObject('[%s]' % str(key),
parent=self.__path__),
self.__extraState__,
offsetToData=offsetToStartOfData,
offsetToFooter=offsetToStartOfData + size)
return v
else:
itemData = readBinaryDataFromFileAtOffset(self.fileObject,
offsetToStartOfData,
size)
dataAsBuffer = ctypes.create_string_buffer(itemData, len(itemData))
v = self.__extraState__.RepresentSchemaNode(
dataAsBuffer, 0,
FsdDataPathObject('[%s]' % str(key), parent=self.__path__),
valueSchema)
return v
def next(self):
self.index += 1
if self.index == self.count:
raise StopIteration()
return self.__extraState__.RepresentSchemaNode(
self.data, self.offset + self.itemSize * self.index,
FsdDataPathObject('[%s]' % str(self.index), parent=self.__path__),
self.itemSchema)
def CreatePythonDictOffset(schema, binaryFooterData, path, extraState):
useOptimizedPythonOffsetStructure = schema['keyTypes']['type'] == 'int'
buff = ctypes.create_string_buffer(binaryFooterData, len(binaryFooterData))
if useOptimizedPythonOffsetStructure:
return StandardFSDOptimizedDictFooter(buff, schema)
else:
return StandardFSDDictFooter(
buff, 0, schema['keyFooter'],
FsdDataPathObject('<keyFooter>', parent=path), extraState)
def __getitem__(self, key):
if type(key) not in (int, long):
raise TypeError('Invalid key type')
if key < 0 or key >= self.count:
raise IndexError('Invalid item index %i for list of length %i' %
(key, self.count))
countOffset = 0 if self.fixedLength else 4
totalOffset = self.offset + countOffset + self.itemSize * key
return self.__extraState__.RepresentSchemaNode(
self.data, totalOffset,
FsdDataPathObject('[%s]' % str(key), parent=self.__path__),
self.itemSchema)
def __getitem__(self, key):
if type(key) not in (int, long):
raise TypeError('Invalid key type')
if key < 0 or key >= self.count:
raise IndexError('Invalid item index %i for list of length %i' %
(key, self.count))
countOffset = 0 if self.fixedLength else 4
dataOffsetFromObjectStart = structTypes.uint32.unpack_from(
self.data, self.offset + countOffset + 4 * key)[0]
return self.__extraState__.RepresentSchemaNode(
self.data, self.offset + dataOffsetFromObjectStart,
FsdDataPathObject('[%s]' % str(key), parent=self.__path__),
self.itemSchema)
def __GetItemFromOffsetAndSize__(self, key, itemOffset, itemSize):
newOffset = 4 + self.offsetToData + itemOffset
valueSchema = self.schema['valueTypes']
if valueSchema.get('buildIndex', False):
v = IndexLoader(self.fileObject,
self.cacheSize,
valueSchema,
FsdDataPathObject('[%s]' % str(key),
parent=self.__path__),
self.__extraState__,
offsetToData=newOffset,
offsetToFooter=newOffset + itemSize)
return v
else:
itemData = readBinaryDataFromFileAtOffset(self.fileObject,
newOffset, itemSize)
dataAsBuffer = ctypes.create_string_buffer(itemData, len(itemData))
v = self.__extraState__.RepresentSchemaNode(
dataAsBuffer, 0,
FsdDataPathObject('[%s]' % str(key), parent=self.__path__),
valueSchema)
return v
def __getitem__(self, key):
if key not in self.__schema__['attributes']:
raise KeyError(
"Object: %s - Attribute '%s' is not in the schema for this object. It may be removed by the 'usage' flag under the build configuration that produced this data."
% (self.__path__, key))
attributeSchema = self.__schema__['attributes'][key]
if key in self.__schema__['constantAttributeOffsets']:
return self.__extraState__.RepresentSchemaNode(
self.__data__, self.__offset__ +
self.__schema__['constantAttributeOffsets'][key],
FsdDataPathObject('.%s' % str(key),
parent=self.__path__), attributeSchema)
else:
if key not in self.__offsetAttributesOffsetLookupTable__:
if 'default' in attributeSchema:
return attributeSchema['default']
raise KeyError(
"Object: %s - Attribute '%s' is not present on this instance."
% (self.__path__, key))
return self.__extraState__.RepresentSchemaNode(
self.__data__, self.__variableDataOffsetBase__ +
self.__offsetAttributesOffsetLookupTable__[key],
FsdDataPathObject('.%s' % str(key),
parent=self.__path__), attributeSchema)
def __init__(self,
fileObject,
cacheSize,
schema,
path,
extraState,
offsetToData=0):
IndexLoader.__init__(self,
fileObject,
cacheSize,
schema,
path,
extraState,
offsetToData=offsetToData)
offsetToAttributeLookupTableSize = self.offsetToData + 4 + self.fileObjectSize - 4 - self.footerDataSize - 4
self.attributeLookupTableSize = readIntFromFileAtOffset(
self.fileObject, offsetToAttributeLookupTableSize)
startOfAttributeLookupTable = offsetToAttributeLookupTableSize - self.attributeLookupTableSize
attributeLookupTable = readBinaryDataFromFileAtOffset(
self.fileObject, startOfAttributeLookupTable,
self.attributeLookupTableSize)
attributeLookupTable = ctypes.create_string_buffer(
attributeLookupTable, len(attributeLookupTable))
self.nestedIndexIdToOffsetTable = self.__extraState__.RepresentSchemaNode(
attributeLookupTable, 0,
FsdDataPathObject('<MultiIndexAttributes>', parent=self.__path__),
self.schema['subIndexOffsetLookup'])
log.info('Attribute lookup table for MultiIndex %s. %s', self.__path__,
self.__extraState__.FormatSize(len(attributeLookupTable)))
self.attributeCache = {}
nestedIndexIdToOffsetData = {}
combinedNestedOffsetDataSize = 0
for index, offsetAndSizeOfOffsetData in self.nestedIndexIdToOffsetTable.iteritems(
):
offsetToNestedDictionaryOffsetData = offsetToData + offsetAndSizeOfOffsetData.offset
binaryNestedDictionaryOffsetData = readBinaryDataFromFileAtOffset(
self.fileObject, offsetToNestedDictionaryOffsetData,
offsetAndSizeOfOffsetData.size)
combinedNestedOffsetDataSize += len(
binaryNestedDictionaryOffsetData)
nestedIndexIdToOffsetData[index] = CreateDictFooter(
self.schema['indexableSchemas'][index],
binaryNestedDictionaryOffsetData,
FsdDataPathObject('<MultiIndexAttributes>.footer[%i]' % index,
parent=self.__path__), self.__extraState__)
log.info(
'Total size of nested offset data kept in memory %s. %s' %
(self.__path__,
self.__extraState__.FormatSize(combinedNestedOffsetDataSize)))
for indexName, indices in self.schema['indexNameToIds'].iteritems():
indexToDictOffsetTable = {}
indexToSchema = {}
for index in indices:
indexToDictOffsetTable[index] = nestedIndexIdToOffsetData[
index]
indexToSchema[index] = self.schema['indexableSchemas'][index]
self.attributeCache[indexName] = SubIndexLoader(
indexToDictOffsetTable, indexToSchema, offsetToData,
self.fileObject, self.__extraState__,
FsdDataPathObject('<MultiIndexAttributes>.%s' % indexName,
parent=self.__path__))
def __GetItemFromOffset__(self, key, offset):
return self.__extraState__.RepresentSchemaNode(
self.data, self.offset + 4 + offset,
FsdDataPathObject('[%s]' % str(key), parent=self.__path__),
self.schema['valueTypes'])