def serbin_arr(field):
assert field.is_array()
n = len(field.val.values)
d = gob_uvarint_encode(n)
for val in field.val.values:
assert isinstance(val, Struct)
off = val.offset
assert type(off) in (types.IntType, types.LongType)
d += gob_uvarint_encode(off)
return d
def serbin_arr(field):
assert field.is_array()
n = len(field.val.values)
d = gob_uvarint_encode(n)
for val in field.val.values:
assert isinstance(val, Struct)
off = val.offset
assert type(off) in (types.IntType, types.LongType)
d += gob_uvarint_encode(off)
return d
def serialize_string(val):
# empty strings are encoded as 0 (0 length)
# non-empty strings are encoded as uvariant(len(s)+1)
# (+1 for terminating 0), followed by string data (including terminating 0)
if val == None:
data = gob_uvarint_encode(0)
else:
data = gob_uvarint_encode(len(val)+1)
data = data + val + chr(0)
return data
def serbin_str(val):
# empty strings are encoded as 0 (0 length)
# non-empty strings are encoded as uvariant(len(s)+1)
# (+1 for terminating 0), followed by string data (including terminating 0)
if val == None:
return gob_uvarint_encode(0)
if type(val) == type(u""):
val = val.encode("utf-8")
data = gob_uvarint_encode(len(val) + 1)
data = data + val + chr(0)
return data
def serbin_str(val):
# empty strings are encoded as 0 (0 length)
# non-empty strings are encoded as uvariant(len(s)+1)
# (+1 for terminating 0), followed by string data (including terminating 0)
if val == None:
return gob_uvarint_encode(0)
if type(val) == type(u""):
val = val.encode("utf-8")
data = gob_uvarint_encode(len(val)+1)
data = data + val + chr(0)
return data
def _serialize(self):
if self.is_signed():
return gob_varint_encode(long(self.val))
if self.is_unsigned():
return gob_uvarint_encode(long(self.val))
if self.is_string():
return serialize_string(self.val)
# floats are serialied as strings
if self.is_float():
return serialize_string(str(self.val))
if self.is_struct():
off = self.val.offset
assert type(off) in (types.IntType, types.LongType)
return gob_uvarint_encode(off)
assert False, "don't know how to serialize %s" % str(self.typ)
def _serbin(field):
assert isinstance(field, Field)
if field.is_no_store():
return None
if field.is_signed():
return gob_varint_encode(long(field.val))
if field.is_unsigned():
return gob_uvarint_encode(long(field.val))
if field.is_string():
return serbin_str(field.val)
# floats are serialied as strings
if field.is_float():
return serbin_str(str(field.val))
if field.is_struct():
off = field.val.offset
assert type(off) in (types.IntType, types.LongType)
return gob_uvarint_encode(off)
if field.is_array():
return serbin_arr(field)
assert False, "don't know how to serialize %s" % str(field.typ)
def _serbin(field):
assert isinstance(field, Field)
if field.is_no_store():
return None
if field.is_signed():
return gob_varint_encode(long(field.val))
if field.is_unsigned():
return gob_uvarint_encode(long(field.val))
if field.is_string():
return serbin_str(field.val)
# floats are serialied as strings
if field.is_float():
return serbin_str(str(field.val))
if field.is_struct():
off = field.val.offset
assert type(off) in (types.IntType, types.LongType)
return gob_uvarint_encode(off)
if field.is_array():
return serbin_arr(field)
assert False, "don't know how to serialize %s" % str(field.typ)
def get_cpp_data_for_struct(stru):
assert isinstance(stru, Struct)
name = stru.name()
offset = stru.offset
lines = ["", " // offset: %s %s" % (hex(offset), short_object_id(stru))]
assert stru.values not in (None, [])
# magic id
data = struct.pack("<I", g_magic_id)
comment = "magic id '%s'" % g_magic_id_str
lines += [data_with_comment_as_c(data, comment)]
# number of fields as uvarint
fields_count = len(stru.values)
data = gob_uvarint_encode(fields_count)
lines += [data_with_comment_as_c(data, "%d fields" % fields_count)]
size = 4 + len(data)
for val in stru.values:
data = val.serialized()
size += len(data)
data_hex = data_to_hex(data)
var_type = val.c_type()
var_name = val.name
if val.is_struct():
val_str = "NULL"
if val.val.offset != 0:
val_str = short_object_id(val.val)
else:
if val.val == None:
try:
assert isinstance(val.typ, String)
except:
print(val)
print(val.name)
print(val.typ)
print(val.val)
raise
val_str = ""
elif type(val.val) == type(""):
val_str = val.val
else:
try:
val_str = hex(val.val)
except:
print(val)
print(val.name)
print(val.typ)
print(val.val)
raise
s = " %(data_hex)s, // %(var_type)s %(var_name)s = %(val_str)s" % locals()
lines += [s]
return (lines, size)
def serialize_top_level_value(top_level_val, serialized_vals):
vals = flatten_values(top_level_val)
# the first 12 bytes are:
# - 4 bytes magic constant (for robustness)
# - 4 bytes for version
# - 4 bytes offset pointing to a top-level structure
# within the data
offset = 12
for stru in vals:
stru.offset = offset
for field in stru.values:
data = serbin(field, serialized_vals)
if None != data:
offset += len(data)
fields_count = len(stru.values)
offset = offset + 4 + len(gob_uvarint_encode(fields_count))
return vals
def serialize_top_level_struct(top_level_struct):
vals = flatten_struct(top_level_struct)
# the first 12 bytes are:
# - 4 bytes magic constant (for robustness)
# - 4 bytes for version
# - 4 bytes offset pointing to a top-level structure
# within the data
offset = 12
for stru in vals:
stru.offset = offset
for field in stru.values:
data = field.serialized()
offset += len(data)
fields_count = len(stru.values)
offset = offset + 4 + len(gob_uvarint_encode(fields_count))
return vals
def get_cpp_data_for_struct(stru, serialized_vals):
assert isinstance(stru, Struct)
name = stru.name()
offset = stru.offset
lines = ["", " // offset: %s %s" % (hex(offset), short_object_id(stru))]
assert stru.values not in (None, [])
# magic id
data = struct.pack("<I", g_magic_id)
comment = "magic id '%s'" % g_magic_id_str
lines += [data_with_comment_as_c(data, comment)]
# number of fields as uvarint
fields_count = len(stru.values)
data = gob_uvarint_encode(fields_count)
lines += [data_with_comment_as_c(data, "%d fields" % fields_count)]
size = 4 + len(data)
for field in stru.values:
if field.is_no_store():
continue
data = serbin(field, serialized_vals)
size += len(data)
data_hex = data_to_hex(data)
var_type = field.c_type()
var_name = field.name
if field.is_struct():
val_str = "NULL"
if field.val.offset != 0:
val_str = short_object_id(field.val)
elif field.is_array():
val_str = short_object_id(field.val)
else:
val_str = field_val_as_str(field)
if val_str == None:
assert isinstance(field.typ, String) or isinstance(
field.typ, WString)
val_str = "NULL"
val_str = str_c_safe(val_str)
s = " %(data_hex)s, // %(var_type)s %(var_name)s = %(val_str)s" % locals(
)
lines += [s]
return (lines, size)
def get_cpp_data_for_struct(stru, serialized_vals):
assert isinstance(stru, Struct)
name = stru.name()
offset = stru.offset
lines = ["", " // offset: %s %s" % (hex(offset), short_object_id(stru))]
assert stru.values not in (None, [])
# magic id
data = struct.pack("<I", g_magic_id)
comment = "magic id '%s'" % g_magic_id_str
lines += [data_with_comment_as_c(data, comment)]
# number of fields as uvarint
fields_count = len(stru.values)
data = gob_uvarint_encode(fields_count)
lines += [data_with_comment_as_c(data, "%d fields" % fields_count)]
size = 4 + len(data)
for field in stru.values:
if field.is_no_store():
continue
data = serbin(field, serialized_vals)
size += len(data)
data_hex = data_to_hex(data)
var_type = field.c_type()
var_name = field.name
if field.is_struct():
val_str = "NULL"
if field.val.offset != 0:
val_str = short_object_id(field.val)
elif field.is_array():
val_str = short_object_id(field.val)
else:
val_str = field_val_as_str(field)
if val_str == None:
assert isinstance(field.typ, String) or isinstance(field.typ, WString)
val_str = "NULL"
val_str = str_c_safe(val_str)
s = " %(data_hex)s, // %(var_type)s %(var_name)s = %(val_str)s" % locals()
lines += [s]
return (lines, size)