def ObjectAsBinaryStringWithIndexedOffsetData(o, schema, path):
representation = cStringIO.StringIO()
attributesByOffset = [
(off, attr)
for attr, off in schema['constantAttributeOffsets'].iteritems()
]
attributesByOffset.sort()
indexedOffsetDataForObjectAttributes = IndexedOffsetData()
for offset, attr in attributesByOffset:
b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(
o[attr], schema['attributes'][attr],
FsdDataPathObject('.%s' % attr, parent=path))
representation.write(b)
if not indexedOffsetData.isEmpty():
indexedOffsetData.AddOffset(offset)
indexedOffsetDataForObjectAttributes.AddNestedIndexedOffsetData(
indexedOffsetData)
if 'size' in schema:
return (representation.getvalue(),
indexedOffsetDataForObjectAttributes)
variableAndOptionalAttributeOrder = schema[
'attributesWithVariableOffsets'][:]
optionalAttributesField = 0
for i in schema['optionalValueLookups']:
if i in o:
optionalAttributesField |= schema['optionalValueLookups'][i]
else:
variableAndOptionalAttributeOrder.remove(i)
offsets = (ctypes.c_uint32 * len(variableAndOptionalAttributeOrder))()
attributesWithOffsets = cStringIO.StringIO()
for i, attr in enumerate(variableAndOptionalAttributeOrder):
offsets[i] = attributesWithOffsets.tell()
b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(
o[attr], schema['attributes'][attr],
FsdDataPathObject('.%s' % attr, parent=path))
attributesWithOffsets.write(b)
if not indexedOffsetData.isEmpty():
indexedOffsetData.AddOffset(offsets[i] + representation.tell())
indexedOffsetDataForObjectAttributes.AddNestedIndexedOffsetData(
indexedOffsetData)
offsetToListData = len(uint32.pack(optionalAttributesField)) + len(
ctypes.string_at(ctypes.addressof(offsets), ctypes.sizeof(offsets)))
indexedOffsetDataForObjectAttributes.AddOffset(offsetToListData)
representation.write(uint32.pack(optionalAttributesField))
representation.write(
ctypes.string_at(ctypes.addressof(offsets), ctypes.sizeof(offsets)))
representation.write(attributesWithOffsets.getvalue())
return (representation.getvalue(), indexedOffsetDataForObjectAttributes)
def StreamSortedIndexToFile(fileObjectOut, keyValueAndMaybeIndexedOffsetData, totalCount, schema, embedSchema = False):
keyFooterSchema = schema['keyFooter']
if 'fixedItemSize' not in keyFooterSchema:
raise ValueError('keys must have a known size to stream the index')
startingOffset = 0
if embedSchema:
schemaToRepresent = {}
schemaToRepresent.update(schema)
if schema['valueTypes']['type'] == 'binary' and 'schema' in schema['valueTypes']:
schemaToRepresent['valueTypes'] = schema['valueTypes']['schema']
fileObjectOut.write(RepresentSchemaAsBinary(schemaToRepresent))
startingOffset = fileObjectOut.tell()
fileObjectOut.write(uint32.pack(0))
footerList = []
path = FsdDataPathObject('[Stream]')
offset = 0
lengthOfFileObjectHeader = 4
indexedOffsetDataForStream = IndexedOffsetData()
for d in keyValueAndMaybeIndexedOffsetData:
key, value = d[:2]
indexedOffsetDataContainedInKeyValueSet = len(d) == 3
prebuiltIndexedOffsetData = d[2] if indexedOffsetDataContainedInKeyValueSet else IndexedOffsetData()
currentFilePos = fileObjectOut.tell()
pathToValues = FsdDataPathObject('[%s]' % key, path)
b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(value, schema['valueTypes'], pathToValues)
fileObjectOut.write(b)
footer = {'key': key,
'offset': offset}
if schema.get('buildIndex', False):
footer['size'] = len(b)
footerList.append(footer)
if not prebuiltIndexedOffsetData.isEmpty():
prebuiltIndexedOffsetData.AddOffset(offset + lengthOfFileObjectHeader)
indexedOffsetDataForStream.AddNestedIndexedOffsetData(prebuiltIndexedOffsetData)
if not indexedOffsetData.isEmpty():
indexedOffsetData.AddOffset(offset + lengthOfFileObjectHeader)
indexedOffsetDataForStream.AddNestedIndexedOffsetData(indexedOffsetData)
offset += fileObjectOut.tell() - currentFilePos
if 'multiIndex' in schema:
multiIndexAttributes = RepresentIndexedOffsetDataAsBinary(indexedOffsetDataForStream, schema, fileObjectOut.tell() - startingOffset)
fileObjectOut.write(multiIndexAttributes)
primaryIndexAsBinary = RepresentAsBinary(footerList, keyFooterSchema, FsdDataPathObject('<Footer>', path))
fileObjectOut.write(primaryIndexAsBinary)
fileObjectOut.write(uint32.pack(len(primaryIndexAsBinary)))
totalFileObjectSize = fileObjectOut.tell() - startingOffset
fileObjectOut.seek(startingOffset)
fileObjectOut.write(uint32.pack(totalFileObjectSize - 4))
def ListAsBinaryStringWithIndexedOffsetData(l, schema, path):
representation = cStringIO.StringIO()
listLength = len(l)
indexedOffsetDataForListItems = IndexedOffsetData()
if 'length' not in schema:
representation.write(uint32.pack(listLength))
if 'fixedItemSize' in schema:
for i, item in enumerate(l):
b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(item, schema['itemTypes'], FsdDataPathObject('[%s]' % i, parent=path))
representation.write(b)
else:
baseOffset = ctypes.sizeof(ctypes.c_uint32) * (listLength + 1)
offset = baseOffset
offsetArray = (ctypes.c_uint32 * listLength)()
listData = cStringIO.StringIO()
for i, item in enumerate(l):
b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(item, schema['itemTypes'], FsdDataPathObject('[%s]' % i, parent=path))
offsetArray[i] = offset
offset += len(b)
listData.write(b)
if not indexedOffsetData.isEmpty():
indexedOffsetData.AddOffset(offsetArray[i])
indexedOffsetDataForListItems.AddNestedIndexedOffsetData(indexedOffsetData)
offsetHeader = ctypes.string_at(ctypes.addressof(offsetArray), ctypes.sizeof(offsetArray))
representation.write(offsetHeader)
representation.write(listData.getvalue())
return (representation.getvalue(), indexedOffsetDataForListItems)
def DictAsBinaryStringWithIndexedOffsetData(d, schema, path):
keySchema = schema['keyFooter']
sortedKeyList = [ k for k in sorted(d.keys()) ]
offsets = []
indexedOffsetData = IndexedOffsetData()
dataRepresentation = cStringIO.StringIO()
offsetToData = 4
includeDataItemSizeInKeyList = 'size' in keySchema['itemTypes']['attributes']
for key in sortedKeyList:
b, nestedOffsetData = RepresentAsBinaryWithIndexedOffsetData(d[key], schema['valueTypes'], FsdDataPathObject('[%s]' % key, parent=path))
offset = dataRepresentation.tell()
footerData = {'key': key,
'offset': offset}
if not nestedOffsetData.isEmpty():
nestedOffsetData.AddOffset(offset + offsetToData)
indexedOffsetData.AddNestedIndexedOffsetData(nestedOffsetData)
if includeDataItemSizeInKeyList:
footerData['size'] = len(b)
offsets.append(footerData)
dataRepresentation.write(b)
offsetTable = RepresentAsBinary(offsets, keySchema, path)
dictFooter = offsetTable + uint32.pack(len(offsetTable))
binaryData = dataRepresentation.getvalue()
if 'indexBy' in schema:
for offset in offsets:
indexedOffsetData.AddKeyOffsetSizeAndPathToNestedIndexId(offset['key'], offset['offset'], offset['size'], path, schema['nestedIndexId'])
elif schema.get('multiIndex', False):
offsetToIndexedOffset = 4 + len(binaryData)
nestedOffsets = RepresentIndexedOffsetDataAsBinary(indexedOffsetData, schema, offsetToIndexedOffset)
dictFooter = nestedOffsets + dictFooter
return (uint32.pack(len(binaryData + dictFooter)) + binaryData + dictFooter, indexedOffsetData)