def LoadFSDDataInPython(dataResPath,
schemaResPath=None,
optimize=None,
cacheSize=100):
import schemaOptimizer
if schemaResPath is not None:
with open(schemaResPath, 'r') as schemaFile:
schema = persistence.LoadSchema(schemaFile.read())
if optimize:
schema = persistence.GetUnOptimizedRuntimeSchema(schema)
schema = schemaOptimizer.OptimizeSchema(schema, optimize)
else:
schema = None
dataFile = open(dataResPath, 'rb')
if schema is None:
peekSchema, size = GetEmbeddedSchemaAndSizeFromFile(dataFile)
dataFile.seek(0)
else:
peekSchema = schema
if peekSchema['type'] == 'dict' and peekSchema.get('buildIndex', False):
return LoadIndexFromFile(dataFile,
schema,
cacheSize,
path=FsdDataPathObject('<file %s>' %
dataResPath))
else:
s = dataFile.read()
log.info('Loading FSD data file %s into memory. %s', dataResPath,
sizeof_fmt(len(s)))
dataFile.close()
return LoadFromString(s,
schema,
path=FsdDataPathObject('<file %s>' %
dataResPath))
def ListAsBinaryStringWithIndexedOffsetData(l, schema, path):
representation = cStringIO.StringIO()
listLength = len(l)
indexedOffsetDataForListItems = IndexedOffsetData()
if 'length' not in schema:
representation.write(uint32.pack(listLength))
if 'fixedItemSize' in schema:
for i, item in enumerate(l):
b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(
item, schema['itemTypes'],
FsdDataPathObject('[%s]' % i, parent=path))
representation.write(b)
else:
baseOffset = ctypes.sizeof(ctypes.c_uint32) * (listLength + 1)
offset = baseOffset
offsetArray = (ctypes.c_uint32 * listLength)()
listData = cStringIO.StringIO()
for i, item in enumerate(l):
b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(
item, schema['itemTypes'],
FsdDataPathObject('[%s]' % i, parent=path))
offsetArray[i] = offset
offset += len(b)
listData.write(b)
if not indexedOffsetData.isEmpty():
indexedOffsetData.AddOffset(offsetArray[i])
indexedOffsetDataForListItems.AddNestedIndexedOffsetData(
indexedOffsetData)
offsetHeader = ctypes.string_at(ctypes.addressof(offsetArray),
ctypes.sizeof(offsetArray))
representation.write(offsetHeader)
representation.write(listData.getvalue())
return (representation.getvalue(), indexedOffsetDataForListItems)
def ObjectAsBinaryStringWithIndexedOffsetData(o, schema, path):
representation = cStringIO.StringIO()
attributesByOffset = [
(off, attr)
for attr, off in schema['constantAttributeOffsets'].iteritems()
]
attributesByOffset.sort()
indexedOffsetDataForObjectAttributes = IndexedOffsetData()
for offset, attr in attributesByOffset:
b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(
o[attr], schema['attributes'][attr],
FsdDataPathObject('.%s' % attr, parent=path))
representation.write(b)
if not indexedOffsetData.isEmpty():
indexedOffsetData.AddOffset(offset)
indexedOffsetDataForObjectAttributes.AddNestedIndexedOffsetData(
indexedOffsetData)
if 'size' in schema:
return (representation.getvalue(),
indexedOffsetDataForObjectAttributes)
variableAndOptionalAttributeOrder = schema[
'attributesWithVariableOffsets'][:]
optionalAttributesField = 0
for i in schema['optionalValueLookups']:
if i in o:
optionalAttributesField |= schema['optionalValueLookups'][i]
else:
variableAndOptionalAttributeOrder.remove(i)
offsets = (ctypes.c_uint32 * len(variableAndOptionalAttributeOrder))()
attributesWithOffsets = cStringIO.StringIO()
for i, attr in enumerate(variableAndOptionalAttributeOrder):
offsets[i] = attributesWithOffsets.tell()
b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(
o[attr], schema['attributes'][attr],
FsdDataPathObject('.%s' % attr, parent=path))
attributesWithOffsets.write(b)
if not indexedOffsetData.isEmpty():
indexedOffsetData.AddOffset(offsets[i] + representation.tell())
indexedOffsetDataForObjectAttributes.AddNestedIndexedOffsetData(
indexedOffsetData)
offsetToListData = len(uint32.pack(optionalAttributesField)) + len(
ctypes.string_at(ctypes.addressof(offsets), ctypes.sizeof(offsets)))
indexedOffsetDataForObjectAttributes.AddOffset(offsetToListData)
representation.write(uint32.pack(optionalAttributesField))
representation.write(
ctypes.string_at(ctypes.addressof(offsets), ctypes.sizeof(offsets)))
representation.write(attributesWithOffsets.getvalue())
return (representation.getvalue(), indexedOffsetDataForObjectAttributes)
def StreamSortedIndexToFile(fileObjectOut, keyValueAndMaybeIndexedOffsetData, totalCount, schema, embedSchema = False):
keyFooterSchema = schema['keyFooter']
if 'fixedItemSize' not in keyFooterSchema:
raise ValueError('keys must have a known size to stream the index')
startingOffset = 0
if embedSchema:
schemaToRepresent = {}
schemaToRepresent.update(schema)
if schema['valueTypes']['type'] == 'binary' and 'schema' in schema['valueTypes']:
schemaToRepresent['valueTypes'] = schema['valueTypes']['schema']
fileObjectOut.write(RepresentSchemaAsBinary(schemaToRepresent))
startingOffset = fileObjectOut.tell()
fileObjectOut.write(uint32.pack(0))
footerList = []
path = FsdDataPathObject('[Stream]')
offset = 0
lengthOfFileObjectHeader = 4
indexedOffsetDataForStream = IndexedOffsetData()
for d in keyValueAndMaybeIndexedOffsetData:
key, value = d[:2]
indexedOffsetDataContainedInKeyValueSet = len(d) == 3
prebuiltIndexedOffsetData = d[2] if indexedOffsetDataContainedInKeyValueSet else IndexedOffsetData()
currentFilePos = fileObjectOut.tell()
pathToValues = FsdDataPathObject('[%s]' % key, path)
b, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(value, schema['valueTypes'], pathToValues)
fileObjectOut.write(b)
footer = {'key': key,
'offset': offset}
if schema.get('buildIndex', False):
footer['size'] = len(b)
footerList.append(footer)
if not prebuiltIndexedOffsetData.isEmpty():
prebuiltIndexedOffsetData.AddOffset(offset + lengthOfFileObjectHeader)
indexedOffsetDataForStream.AddNestedIndexedOffsetData(prebuiltIndexedOffsetData)
if not indexedOffsetData.isEmpty():
indexedOffsetData.AddOffset(offset + lengthOfFileObjectHeader)
indexedOffsetDataForStream.AddNestedIndexedOffsetData(indexedOffsetData)
offset += fileObjectOut.tell() - currentFilePos
if 'multiIndex' in schema:
multiIndexAttributes = RepresentIndexedOffsetDataAsBinary(indexedOffsetDataForStream, schema, fileObjectOut.tell() - startingOffset)
fileObjectOut.write(multiIndexAttributes)
primaryIndexAsBinary = RepresentAsBinary(footerList, keyFooterSchema, FsdDataPathObject('<Footer>', path))
fileObjectOut.write(primaryIndexAsBinary)
fileObjectOut.write(uint32.pack(len(primaryIndexAsBinary)))
totalFileObjectSize = fileObjectOut.tell() - startingOffset
fileObjectOut.seek(startingOffset)
fileObjectOut.write(uint32.pack(totalFileObjectSize - 4))
def LoadIndexFromFile(fileObject,
optimizedSchema=None,
cacheItems=100,
path=None,
extraState=None):
if extraState is None:
extraState = LoaderState(defaultLoaderFactories, None)
if path is None:
path = FsdDataPathObject('<file input>')
offsetToData = 0
if optimizedSchema is None:
fileObject.seek(0)
optimizedSchema, schemaSize = GetEmbeddedSchemaAndSizeFromFile(
fileObject)
offsetToData = schemaSize + 4
if optimizedSchema.get('multiIndex', False):
return loaders.dictLoader.MultiIndexLoader(fileObject,
cacheItems,
optimizedSchema,
path,
extraState,
offsetToData=offsetToData)
else:
return loaders.dictLoader.IndexLoader(fileObject,
cacheItems,
optimizedSchema,
path,
extraState,
offsetToData=offsetToData)
def LoadFSDDataForCFG(dataResPath,
schemaResPath=None,
optimize=True,
cacheSize=100):
import blue
import schemaOptimizer
res = blue.ResFile()
if schemaResPath is not None:
if not res.Open(schemaResPath):
log.error('Could not load FSD data schema: %s', schemaResPath)
else:
schema = persistence.LoadSchema(res.Read())
if optimize:
schema = persistence.GetUnOptimizedRuntimeSchema(schema)
schema = schemaOptimizer.OptimizeSchema(
schema, boot.role.capitalize())
else:
schema = None
if not res.Open(dataResPath):
log.error('Could not load FSD data file: %s', dataResPath)
else:
wrappedRes = BlueResFileIOWrapper(res)
if schema is None:
peekSchema, size = GetEmbeddedSchemaAndSizeFromFile(wrappedRes)
wrappedRes.seek(0)
else:
peekSchema = schema
if peekSchema['type'] == 'dict' and peekSchema.get(
'buildIndex', False):
return LoadIndexFromFile(wrappedRes,
schema,
cacheSize,
path=FsdDataPathObject('<file %s>' %
dataResPath))
s = res.Read()
log.info('Loading FSD data file %s into memory. %s', dataResPath,
sizeof_fmt(len(s)))
return LoadFromString(s,
schema,
path=FsdDataPathObject('<file %s>' %
dataResPath))
def LoadFromString(dataString,
optimizedSchema=None,
path=None,
extraState=None):
if path is None:
path = FsdDataPathObject('<string input>')
offsetToData = 0
if optimizedSchema is None:
schemaSize = uint32.unpack_from(dataString, 0)[0]
optimizedSchema = cPickle.loads(dataString[4:schemaSize + 4])
offsetToData = schemaSize + 4
dataBuffer = ctypes.create_string_buffer(dataString, len(dataString))
return RepresentSchemaNode(dataBuffer, offsetToData, optimizedSchema, path,
extraState)
def UnionAsBinaryStringWithIndexedOffsetData(o, schema, path):
representation = cStringIO.StringIO()
for idx, possibleSchemaType in enumerate(schema['optionTypes']):
if len(validator.Validate(possibleSchemaType, o)) == 0:
binaryIndex = uint32.pack(idx)
representation.write(binaryIndex)
itemRepresentation, indexedOffsetData = RepresentAsBinaryWithIndexedOffsetData(
o, possibleSchemaType,
FsdDataPathObject('<%s>' % possibleSchemaType['type'],
parent=path))
if not indexedOffsetData.isEmpty():
indexedOffsetData.AddOffset(len(binaryIndex))
representation.write(itemRepresentation)
return (representation.getvalue(), indexedOffsetData)
raise ValueError('Could not represent %s as any type in union' % str(o))
def DictAsBinaryStringWithIndexedOffsetData(d, schema, path):
keySchema = schema['keyFooter']
sortedKeyList = [k for k in sorted(d.keys())]
offsets = []
indexedOffsetData = IndexedOffsetData()
dataRepresentation = cStringIO.StringIO()
offsetToData = 4
includeDataItemSizeInKeyList = 'size' in keySchema['itemTypes'][
'attributes']
for key in sortedKeyList:
b, nestedOffsetData = RepresentAsBinaryWithIndexedOffsetData(
d[key], schema['valueTypes'],
FsdDataPathObject('[%s]' % key, parent=path))
offset = dataRepresentation.tell()
footerData = {'key': key, 'offset': offset}
if not nestedOffsetData.isEmpty():
nestedOffsetData.AddOffset(offset + offsetToData)
indexedOffsetData.AddNestedIndexedOffsetData(nestedOffsetData)
if includeDataItemSizeInKeyList:
footerData['size'] = len(b)
offsets.append(footerData)
dataRepresentation.write(b)
offsetTable = RepresentAsBinary(offsets, keySchema, path)
dictFooter = offsetTable + uint32.pack(len(offsetTable))
binaryData = dataRepresentation.getvalue()
if 'indexBy' in schema:
for offset in offsets:
indexedOffsetData.AddKeyOffsetSizeAndPathToNestedIndexId(
offset['key'], offset['offset'], offset['size'], path,
schema['nestedIndexId'])
elif schema.get('multiIndex', False):
offsetToIndexedOffset = 4 + len(binaryData)
nestedOffsets = RepresentIndexedOffsetDataAsBinary(
indexedOffsetData, schema, offsetToIndexedOffset)
dictFooter = nestedOffsets + dictFooter
return (uint32.pack(len(binaryData + dictFooter)) + binaryData +
dictFooter, indexedOffsetData)
def RepresentAsBinary(o, schema, path = None):
if path is None:
path = FsdDataPathObject('<root>')
binaryData = RepresentAsBinaryWithIndexedOffsetData(o, schema, path)[0]
return binaryData