def _construct_header(self, message):
field_offset = 0
# All messages start with a message size field
message_id = int(message['id'])
schema_block_length = int(message['block_length'])
message_type = type(message['description'], (SBEMessage,), {'message_id': message_id,
'schema_block_length': schema_block_length})
self.message_map[message_id] = message_type
setattr(message_type, 'fields', [])
# All messages start with a message size field
message_size_field = TypeMessageField(name='message_size', original_name='message_size',
description="Header Message Size",
unpack_fmt='<H', field_offset=field_offset, field_length=2)
field_offset += message_size_field.field_length
message_type.fields.append(message_size_field)
setattr(message_type, 'message_size', message_size_field)
# Now grab the messageHeader type, it has to exist and populate the remaining header fields
message_header_type = self.type_map['messageHeader']
for header_field_type in message_header_type.get('children', []):
primitive_type_fmt, primitive_type_size = self.primitive_type_map[header_field_type['primitive_type']]
message_header_field = TypeMessageField(name=convert_to_underscore(header_field_type['name']),
original_name = header_field_type['name'],
description='Header ' + header_field_type['name'],
unpack_fmt=primitive_type_fmt,
field_offset=field_offset,
field_length=primitive_type_size)
field_offset += message_header_field.field_length
message_type.fields.append(message_header_field)
setattr(message_type, message_header_field.name, message_header_field)
setattr(message_type, 'header_size', field_offset)
return field_offset
def _add_groups(self, entity, entity_type, endian):
# Now figure out the message groups
repeating_groups = []
for group_type in entity.get('groups', []):
group_name = convert_to_underscore(group_type['name'])
group_original_name = group_type['name']
group_since_version = int(group_type.get('since_version','0'))
dimension_type = self.type_map[group_type['dimension_type']]
# There are two fields we care about, block_length and num_in_group
block_length_field = None
num_in_group_field = None
block_field_offset = 0
for child in dimension_type['children']:
if child['name'] == 'blockLength':
primitive_type = child['primitive_type']
primitive_type_fmt, primitive_type_size = self.primitive_type_map[primitive_type]
block_length_field = TypeMessageField(name=convert_to_underscore(child['name']),
original_name=child['name'],
description=child['name'],
unpack_fmt=endian + primitive_type_fmt,
field_offset=block_field_offset,
field_length=primitive_type_size,
semantic_type=child.get('semantic_type'))
block_field_offset += primitive_type_size
elif child['name'] == 'numInGroup':
primitive_type = child['primitive_type']
if 'offset' in child:
block_field_offset = int(child['offset'])
primitive_type_fmt, primitive_type_size = self.primitive_type_map[primitive_type]
num_in_group_field = TypeMessageField(name=convert_to_underscore(child['name']),
original_name=child['name'],
description=child['name'],
unpack_fmt=endian + primitive_type_fmt,
field_offset=block_field_offset,
field_length=primitive_type_size,
semantic_type=child.get('semantic_type'))
block_field_offset += primitive_type_size
group_field_offset = 0
repeating_group = SBERepeatingGroupContainer(name=group_name, original_name=group_original_name,
id=int(group_type['id']),
block_length_field=block_length_field,
num_in_group_field=num_in_group_field,
dimension_size=block_field_offset,
since_version=group_since_version)
self._add_fields(group_field_offset, group_type, repeating_group, endian, add_header_size=False)
repeating_groups.append(repeating_group)
setattr(entity_type, repeating_group.name, repeating_group)
# handle nested groups
self._add_groups(group_type, repeating_group, endian)
setattr(entity_type, 'groups', repeating_groups)
def parse(self,
xml_file,
message_tag="message",
types_tag="types",
endian='<'):
self.type_map = self._parse_types(xml_file, types_tag=types_tag)
self.messages = self._parse_messages(xml_file, message_tag=message_tag)
# Now construct each message with its expected field types
for message in self.messages:
field_offset = 0
# All messages start with a message size field
message_id = int(message['id'])
description = message[
'description'] if 'description' in message else message['id']
if 'block_length' in message:
schema_block_length = int(message['block_length'])
message_type = type(
description, (SBEMessage, ), {
'message_id': message_id,
'schema_block_length': schema_block_length
})
else:
message_type = type(description, (SBEMessage, ),
{'message_id': message_id})
message_fields = []
# All messages start with a message size field
message_size_field = TypeMessageField(
name='message_size',
description="Header Message Size",
unpack_fmt='<H',
field_offset=field_offset,
field_length=2)
field_offset += message_size_field.field_length
message_fields.append(message_size_field)
# Now grab the messageHeader type, it has to exist and populate the remaining header fields
message_header_type = self.type_map['messageHeader']
for header_field_type in message_header_type.get('children', []):
primitive_type_fmt, primitive_type_size = self.primitive_type_map[
header_field_type['primitive_type']]
message_header_field = TypeMessageField(
name=convert_to_underscore(header_field_type['name']),
description='Header ' + header_field_type['name'],
unpack_fmt=primitive_type_fmt,
field_offset=field_offset,
field_length=primitive_type_size)
field_offset += message_header_field.field_length
message_fields.append(message_header_field)
setattr(message_type, 'header_size', field_offset)
# Now run through the remaining types and update the fields
for message_field_type in message.get('fields', []):
message_field = self._build_message_field(
message_field_type, field_offset)
field_offset += message_field.field_length
message_fields.append(message_field)
# Assign all the fields to class type
for message_field in message_fields:
setattr(message_type, message_field.name, message_field)
# Assign the fields array to keep around
setattr(message_type, 'fields', message_fields)
# Now figure out the message groups
repeating_groups = []
for group in message.get('groups', []):
group_name = convert_to_underscore(group['name'])
dimension_type = self.type_map[group['dimension_type']]
# There are two fields we care about, block_length and num_in_group
block_length_field = None
num_in_group_field = None
block_field_offset = 0
for child in dimension_type['children']:
if child['name'] == 'blockLength':
primitive_type = child['primitive_type']
primitive_type_fmt, primitive_type_size = self.primitive_type_map[
primitive_type]
block_length_field = TypeMessageField(
name=convert_to_underscore(child['name']),
description=child['name'],
unpack_fmt=endian + primitive_type_fmt,
field_offset=block_field_offset,
field_length=primitive_type_size)
block_field_offset += primitive_type_size
elif child['name'] == 'numInGroup':
primitive_type = child['primitive_type']
if 'offset' in child:
block_field_offset = int(child['offset'])
primitive_type_fmt, primitive_type_size = self.primitive_type_map[
primitive_type]
num_in_group_field = TypeMessageField(
name=convert_to_underscore(child['name']),
description=child['name'],
unpack_fmt=endian + primitive_type_fmt,
field_offset=block_field_offset,
field_length=primitive_type_size)
block_field_offset += primitive_type_size
group_field_offset = 0
group_fields = []
for group_field_def in group.get('fields', []):
group_field = self._build_message_field(
group_field_def,
group_field_offset,
add_header_size=False)
group_field_offset += group_field.field_length
group_fields.append(group_field)
repeating_group = SBERepeatingGroupIterator(
name=group_name,
block_length_field=block_length_field,
num_in_group_field=num_in_group_field,
dimension_size=block_field_offset,
group_fields=group_fields)
repeating_groups.append(repeating_group)
for repeating_group in repeating_groups:
setattr(message_type, repeating_group.name, repeating_group)
setattr(message_type, 'iterators', repeating_groups)
self.message_map[message_id] = message_type
def _build_message_field(self,
field_definition,
offset,
header_size=10,
endian='<',
add_header_size=True):
field_name = convert_to_underscore(field_definition['name'])
field_description = field_definition[
'description'] if 'description' in field_definition else ''
if field_definition['type'] in self.type_map:
field_type = self.type_map[field_definition['type']]
else:
field_type = self.primitive_type_definitions[
field_definition['type']]
field_type_type = field_type['type']
field_semantic_type = field_type.get('semantic_type', None)
is_string_type = (field_semantic_type == 'String')
message_field = None
if field_type_type == 'type':
field_offset = offset
if field_definition.get('offset', None) is not None:
field_offset = int(field_definition.get('offset', None))
if add_header_size:
field_offset += header_size
primitive_type_fmt, primitive_type_size = self.get_type_format(
field_type['primitive_type'])
unpack_fmt = endian
field_length = field_type.get('length', None)
if field_length is not None:
field_length = int(field_length)
if is_string_type:
unpack_fmt += '%ss' % (str(field_length), )
else:
unpack_fmt += '%s%s' % (str(field_length),
primitive_type_fmt)
else:
# Field length is just the primitive type length
field_length = primitive_type_size
unpack_fmt += primitive_type_fmt
constant = None
optional = False
if 'presence' in field_type:
if field_type['presence'] == 'constant':
constant_prim_type = field_type['primitive_type']
if constant_prim_type == 'char':
constant = str(field_type['text'])
else:
constant = int(field_type['text'])
elif field_type['presence'] == 'optional':
optional = True
null_value = None
if 'null_value' in field_type:
null_value = long(field_type['null_value'])
message_field = TypeMessageField(name=field_name,
description=field_description,
unpack_fmt=unpack_fmt,
field_offset=field_offset,
field_length=field_length,
null_value=null_value,
constant=constant,
optional=optional,
is_string_type=is_string_type)
elif field_type_type == 'enum':
encoding_type = field_type['encoding_type']
primitive_type_fmt, primitive_type_size = self.get_type_format(
encoding_type)
field_offset = offset
if field_definition.get('offset', None) is not None:
field_offset = int(field_definition.get('offset', None))
if add_header_size:
field_offset += header_size
unpack_fmt = endian
field_length = field_type.get('length', None)
if field_length is not None:
field_length = int(field_length)
for i in xrange(field_length):
unpack_fmt += primitive_type_fmt
else:
# Field length is just the primitive type length
field_length = primitive_type_size
unpack_fmt += primitive_type_fmt
enum_values = field_type['children']
message_field = EnumMessageField(name=field_name,
description=field_description,
unpack_fmt=unpack_fmt,
field_offset=field_offset,
enum_values=enum_values,
field_length=field_length)
elif field_type_type == 'set':
encoding_type = field_type['encoding_type']
primitive_type_fmt, primitive_type_size = self.get_type_format(
encoding_type)
field_offset = offset
if field_definition.get('offset', None) is not None:
field_offset = int(field_definition.get('offset', None))
if add_header_size:
field_offset += header_size
unpack_fmt = endian
field_length = field_type.get('length', None)
if field_length is not None:
field_length = int(field_length)
for i in xrange(field_length):
unpack_fmt += primitive_type_fmt
else:
# Field length is just the primitive type length
field_length = primitive_type_size
unpack_fmt += primitive_type_fmt
choice_values = field_type['children']
message_field = SetMessageField(field_name, field_description,
unpack_fmt, field_offset,
choice_values, field_length)
elif field_type_type == 'composite':
composite_parts = []
field_offset = offset
if field_definition.get('offset', None) is not None:
field_offset = int(field_definition.get('offset', None))
if add_header_size:
field_offset += header_size
float_composite = False
field_length = 0
for child in field_type['children']:
primitive_type_fmt, primitive_type_size = self.get_type_format(
child['primitive_type'])
unpack_fmt = endian + primitive_type_fmt
constant = None
optional = False
if 'presence' in child:
if child['presence'] == 'constant':
constant_prim_type = child['primitive_type']
if constant_prim_type == 'char':
constant = str(child['text'])
else:
constant = int(child['text'])
elif child['presence'] == 'optional':
optional = True
null_value = None
if 'null_value' in child:
null_value = long(child['null_value'])
# If a 'mantissa' field exists, assume we are working with a floating point value
if child['name'] == 'mantissa':
float_composite = True
composite_field = TypeMessageField(
name=child['name'],
description=child['description'],
unpack_fmt=unpack_fmt,
field_offset=field_offset,
field_length=primitive_type_size,
null_value=null_value,
constant=constant,
optional=optional)
field_offset += primitive_type_size
field_length += primitive_type_size
composite_parts.append(composite_field)
message_field = CompositeMessageField(
name=field_name,
description=field_description,
field_offset=field_offset,
field_length=field_length,
parts=composite_parts,
float_value=float_composite)
return message_field
def _build_message_field(self,
field_definition,
offset,
header_size=10,
endian='<',
add_header_size=True):
field_original_name = field_definition['name']
field_name = convert_to_underscore(field_original_name)
field_id = field_definition['id']
field_description = field_definition.get('description', '')
field_type = self.type_map[field_definition['type']]
field_type_type = field_type['type']
field_semantic_type = field_definition.get('semantic_type', None)
field_since_version = int(field_definition.get('since_version', '0'))
message_field = None
if field_type_type == 'type':
is_string_type = field_type[
'primitive_type'] == 'char' and 'length' in field_type and int(
field_type['length']) > 1
field_offset = offset
if field_definition.get('offset', None) is not None:
field_offset = int(field_definition.get('offset', None))
if add_header_size:
field_offset += header_size
primitive_type_fmt, primitive_type_size = self.primitive_type_map[
field_type['primitive_type']]
field_length = field_type.get('length', None)
if field_length is not None:
field_length = int(field_length)
if is_string_type:
unpack_fmt = '%ds' % field_length # unpack as string (which may be null-terminated if shorter)
else:
unpack_fmt = '%s%s%s' % (endian, str(field_length),
primitive_type_fmt)
else:
# Field length is just the primitive type length
field_length = primitive_type_size
unpack_fmt = '%s%s' % (endian, primitive_type_fmt)
constant = None
optional = False
if 'presence' in field_type:
if field_type['presence'] == 'constant':
constant_prim_type = field_type['primitive_type']
if constant_prim_type == 'char':
constant = str(field_type['text'])
else:
constant = int(field_type['text'])
elif field_type['presence'] == 'optional':
optional = True
null_value = None
if 'null_value' in field_type:
null_value = int(field_type['null_value'])
message_field = TypeMessageField(name=field_name,
original_name=field_original_name,
id=field_id,
description=field_description,
unpack_fmt=unpack_fmt,
field_offset=field_offset,
field_length=field_length,
null_value=null_value,
constant=constant,
optional=optional,
is_string_type=is_string_type,
semantic_type=field_semantic_type,
since_version=field_since_version)
elif field_type_type == 'enum':
encoding_type = field_type['encoding_type']
encoding_type_type = self.type_map[encoding_type]
primitive_type_fmt, primitive_type_size = self.primitive_type_map[
encoding_type_type['primitive_type']]
field_offset = offset
if field_definition.get('offset', None) is not None:
field_offset = int(field_definition.get('offset', None))
if add_header_size:
field_offset += header_size
unpack_fmt = endian
field_length = field_type.get('length', None)
if field_length is not None:
field_length = int(field_length)
for i in range(field_length):
unpack_fmt += primitive_type_fmt
else:
# Field length is just the primitive type length
field_length = primitive_type_size
unpack_fmt += primitive_type_fmt
enum_values = field_type['children']
message_field = EnumMessageField(name=field_name,
original_name=field_original_name,
id=field_id,
description=field_description,
unpack_fmt=unpack_fmt,
field_offset=field_offset,
enum_values=enum_values,
field_length=field_length,
semantic_type=field_semantic_type,
since_version=field_since_version)
elif field_type_type == 'set':
encoding_type = field_type['encoding_type']
encoding_type_type = self.type_map[encoding_type]
primitive_type_fmt, primitive_type_size = self.primitive_type_map[
encoding_type_type['primitive_type']]
field_offset = offset
if field_definition.get('offset', None) is not None:
field_offset = int(field_definition.get('offset', None))
if add_header_size:
field_offset += header_size
unpack_fmt = endian
field_length = field_type.get('length', None)
if field_length is not None:
field_length = int(field_length)
for i in range(field_length):
unpack_fmt += primitive_type_fmt
else:
# Field length is just the primitive type length
field_length = primitive_type_size
unpack_fmt += primitive_type_fmt
choice_values = field_type['children']
message_field = SetMessageField(name=field_name,
original_name=field_original_name,
id=field_id,
description=field_description,
unpack_fmt=unpack_fmt,
field_offset=field_offset,
choices=choice_values,
field_length=field_length,
semantic_type=field_semantic_type,
since_version=field_since_version)
elif field_type_type == 'composite':
composite_parts = []
field_offset = offset
if field_definition.get('offset', None) is not None:
field_offset = int(field_definition.get('offset', None))
if add_header_size:
field_offset += header_size
float_composite = False
field_length = 0
for child in field_type['children']:
primitive_type_fmt, primitive_type_size = self.primitive_type_map[
child['primitive_type']]
unpack_fmt = endian + primitive_type_fmt
child_since_version = int(child.get('since_version', '0'))
constant = None
optional = False
if 'presence' in child:
if child['presence'] == 'constant':
constant_prim_type = child['primitive_type']
if constant_prim_type == 'char':
constant = str(child['text'])
else:
constant = int(child['text'])
elif child['presence'] == 'optional':
optional = True
null_value = None
if 'null_value' in child:
null_value = int(child['null_value'])
# If a 'mantissa' field exists, assume we are working with a floating point value
if child['name'] == 'mantissa':
float_composite = True
composite_field = TypeMessageField(
name=child['name'],
original_name=child['name'],
description=child.get('description', ''),
unpack_fmt=unpack_fmt,
field_offset=field_offset,
field_length=primitive_type_size,
null_value=null_value,
constant=constant,
optional=optional,
semantic_type=field_semantic_type,
since_version=child_since_version)
field_offset += primitive_type_size
field_length += primitive_type_size
composite_parts.append(composite_field)
message_field = CompositeMessageField(
name=field_name,
original_name=field_original_name,
id=field_id,
description=field_description,
field_offset=field_offset,
field_length=field_length,
parts=composite_parts,
float_value=float_composite,
semantic_type=field_semantic_type,
since_version=field_since_version)
return message_field