def build_bitmapped_descriptors(self, bitmap):
"""
Build the bitmapped descriptors based on the given bitmap. Also build
the back referenced descriptors if it is not already defined.
"""
# Second get all the back referenced descriptors if it does not already exist
if not self.back_referenced_descriptors:
self.back_referenced_descriptors = []
for idx in range(self.back_reference_boundary - 1, -1, -1):
descriptor = self.decoded_descriptors[idx]
# The type has to be an exact match, not just isinstance
if type(descriptor) is ElementDescriptor:
self.back_referenced_descriptors.insert(
0, (idx, descriptor))
if len(self.back_referenced_descriptors) == len(bitmap):
break
if len(self.back_referenced_descriptors) != len(bitmap):
raise PyBufrKitError(
'Back referenced descriptors not matching defined Bitmap')
# Lastly, get all the descriptors that has a corresponding Zero bit value
self.bitmapped_descriptors = [
(idx, d)
for bit, (idx, d) in zip(bitmap, self.back_referenced_descriptors)
if bit == 0
]
self.next_bitmapped_descriptor = functools.partial(
next, iter(self.bitmapped_descriptors))
def wire_members(self, members):
for member in members:
# 221 YYY data not present for following YYY descriptors except class 0-9 and 31
if self.data_not_present_count:
self.data_not_present_count -= 1
if isinstance(member, ElementDescriptor):
X = member.X
if not (1 <= X <= 9 or X == 31): # skipping
self.add_node(NoValueDataNode(member))
continue
# Now process normally
if isinstance(member, ElementDescriptor):
self.wire_element_descriptor(member)
elif isinstance(member, FixedReplicationDescriptor):
self.wire_fixed_replication_descriptor(member)
elif isinstance(member, DelayedReplicationDescriptor):
self.wire_delayed_replication_descriptor(member)
elif isinstance(member, OperatorDescriptor):
self.wire_operator_descriptor(member)
elif isinstance(member, SequenceDescriptor):
self.wire_sequence_descriptor(member)
elif isinstance(member, SkippedLocalDescriptor):
self.wire_skippable_local_descriptor()
else:
raise PyBufrKitError('Cannot wire descriptor type: {}'.format(
type(member)))
def process(self,
s,
file_path='<string>',
start_signature=MESSAGE_START_SIGNATURE,
info_only=False,
ignore_value_expectation=False,
wire_template_data=True):
"""
Decoding the given message string.
:param s: Message string that contains the BUFR Message
:param file_path: The file where this string is read from.
:param start_signature: Locate the starting position of the message
string with the given signature.
:param info_only: Only show information up to template data (exclusive)
:param ignore_value_expectation: Do not validate the expected value
:param wire_template_data: Whether to wire the template data to construct
a fully hierarchical structure from the flat lists. Only takes effect
when it is NOT info_only.
:return: A BufrMessage object that contains the decoded information.
"""
idx = s.find(start_signature) if start_signature is not None else 0
if idx == -1:
raise PyBufrKitError(
'Cannot find start signature: {}'.format(start_signature))
s = s[idx:]
bit_reader = get_bit_reader(s)
bufr_message = BufrMessage(file_path)
configuration_transformers = (
self.section_configurer.info_configuration, ) if info_only else ()
if ignore_value_expectation:
configuration_transformers += (
self.section_configurer.ignore_value_expectation, )
nbits_decoded = 0
section_index = 0 # Always start decoding from section 0
while True:
section = self.section_configurer.configure_section(
bufr_message, section_index, configuration_transformers)
section_index += 1
if section is None: # when optional section is not present
continue
nbits_decoded += self.process_section(bufr_message, bit_reader,
section)
if section.end_of_message:
break
# The exact bytes that have been decoded
bufr_message.serialized_bytes = s[:nbits_decoded // NBITS_PER_BYTE]
if not info_only and wire_template_data:
bufr_message.wire()
return bufr_message
def process_statements(coder, state, bit_operator, statements):
"""
Process through a list of statements. Recursively call itself if the sub-statement
is itself a list of statements.
"""
for statement in statements:
if isinstance(statement, MethodCall):
# Populate any necessary state properties. This is to re-create the
# modifier effects of operator descriptors.
if statement.state_properties is not None:
for k, v in statement.state_properties.items():
setattr(state, k, v)
if type(statement) is StateMethodCall:
getattr(state, statement.method_name)(*statement.args)
elif type(statement) is CoderMethodCall:
if issubclass(type(statement.args[0]), Descriptor):
getattr(coder, statement.method_name)(state, bit_operator,
*statement.args)
else:
getattr(coder, statement.method_name)(state,
*statement.args)
else:
raise PyBufrKitError('Unknown statement: {}'.format(statement))
elif isinstance(statement, State031031Reset):
state.n_031031 = 0
elif isinstance(statement, State031031Increment):
state.n_031031 += 1
elif type(statement) is Loop:
if isinstance(statement.repeat, CoderMethodCall):
repeat = getattr(coder, statement.repeat.method_name)(state)
else:
repeat = statement.repeat
for _ in range(repeat):
process_statements(coder, state, bit_operator,
statement.statements)
else:
raise PyBufrKitError('Unknown statement: {}'.format(statement))
def process_section(self, bufr_message, bit_reader, section):
"""
Decode the given configured Section.
:param bufr_message: The BUFR message object.
:param section: The BUFR section object.
:param bit_reader:
:return: Number of bits decoded for this section.
"""
section.set_metadata(BITPOS_START, bit_reader.get_pos())
for parameter in section:
if parameter.type == PARAMETER_TYPE_UNEXPANDED_DESCRIPTORS:
parameter.value = self.process_unexpanded_descriptors(
bit_reader, section)
elif parameter.type == PARAMETER_TYPE_TEMPLATE_DATA:
parameter.value = self.process_template_data(
bufr_message, bit_reader)
elif parameter.nbits == 0:
# Zero number of bits means to read all bits till the end of the section
parameter.value = bit_reader.read(
parameter.type,
section.section_length.value * NBITS_PER_BYTE -
(bit_reader.get_pos() -
section.get_metadata(BITPOS_START)))
else:
parameter.value = bit_reader.read(parameter.type,
parameter.nbits)
log.debug('{} = {!r}'.format(parameter.name, parameter.value))
# Make available as a property of the overall message object
if parameter.as_property:
setattr(bufr_message, parameter.name, parameter)
if parameter.expected is not None:
assert parameter.value == parameter.expected, 'Value ({!r}) not as expected ({!r})'.format(
parameter.value, parameter.expected)
# TODO: option to ignore the declared length?
# TODO: this depends on a specific parameter name, need change to parameter type?
if 'section_length' in section:
nbits_read = bit_reader.get_pos() - section.get_metadata(
BITPOS_START)
nbits_unread = section.section_length.value * NBITS_PER_BYTE - nbits_read
if nbits_unread > 0:
log.debug('Skipping {} bits to end of the section'.format(
nbits_unread))
bit_reader.read_bin(nbits_unread)
elif nbits_unread < 0:
raise PyBufrKitError(
'Read exceeds declared section {} length: {} by {} bits'.
format(section.get_metadata('index'),
section.section_length.value, -nbits_unread))
return bit_reader.get_pos() - section.get_metadata(BITPOS_START)
def process_section(self, bufr_message, bit_writer, section):
section.set_metadata(BITPOS_START, bit_writer.get_pos())
for parameter in section:
if parameter.type == PARAMETER_TYPE_UNEXPANDED_DESCRIPTORS:
self.process_unexpanded_descriptors(bit_writer, parameter)
elif parameter.type == PARAMETER_TYPE_TEMPLATE_DATA:
self.process_template_data(bufr_message, bit_writer, parameter)
else:
bit_writer.write(parameter.value, parameter.type, parameter.nbits)
log.debug('{} = {!r}'.format(parameter.name, parameter.value))
# Make available as a property of the overall message object
if parameter.as_property:
setattr(bufr_message, parameter.name, parameter)
nbits_write = (bit_writer.get_pos() - section.get_metadata(BITPOS_START))
nbytes_write, nbits_residue = nbits_write // NBITS_PER_BYTE, nbits_write % NBITS_PER_BYTE
# For edition 3 and earlier, ensure each section has an even number of octets.
# This is done by padding Zeros to the required number of octets.
if bufr_message.edition.value <= 3:
if nbytes_write % 2 != 0:
nbits_padding_for_octet = (NBITS_PER_BYTE - nbits_residue)
else:
nbits_padding_for_octet = 0 if nbits_residue == 0 else (2 * NBITS_PER_BYTE - nbits_residue)
else: # otherwise just padding to any integer multiple of octet
nbits_padding_for_octet = 0 if nbits_residue == 0 else (NBITS_PER_BYTE - nbits_residue)
if nbits_padding_for_octet != 0:
log.debug('Padding {} bits for complete Octets'.format(nbits_padding_for_octet))
bit_writer.write_bin('0' * nbits_padding_for_octet)
if 'section_length' in section:
nbits_write = bit_writer.get_pos() - section.get_metadata(BITPOS_START)
# A zero length means the length should be calculated
if section.section_length.value == 0 or self.ignore_declared_length:
section.section_length.value = nbits_write // NBITS_PER_BYTE
bit_writer.set_uint(
section.section_length.value,
section.section_length.nbits,
section.get_metadata(BITPOS_START) + section.get_parameter_offset('section_length')
)
else:
nbits_unwrite = section.section_length.value * NBITS_PER_BYTE - nbits_write
if nbits_unwrite > 0:
log.debug('Padding {} bits to for declared length of the section'.format(nbits_unwrite))
bit_writer.skip(nbits_unwrite)
elif nbits_unwrite < 0:
raise PyBufrKitError('Writing exceeds declared section length {} by {} bytes'.format(
section.section_length.value, -nbits_unwrite // NBITS_PER_BYTE
))
return bit_writer.get_pos() - section.get_metadata(BITPOS_START)
def get_value_for_delayed_replication_factor(self, idx):
if self.is_compressed:
self._assert_equal_values_of_index(idx)
value = self.decoded_values_all_subsets[0][idx]
else:
value = self.decoded_values[idx]
if value is None or value < 0:
raise PyBufrKitError('Delayed replication factor must be >= 0: got ({!r})'.format(value))
return value
def process_bitmap_definition(self, state, bit_operator, descriptor):
n_031031 = state.n_031031
super(TemplateCompiler,
self).process_bitmap_definition(state, bit_operator, descriptor)
if state.n_031031 == 0:
state.add_statement(State031031Reset())
elif state.n_031031 == n_031031 + 1:
state.add_statement(State031031Increment())
elif state.n_031031 == n_031031:
pass
else:
raise PyBufrKitError('erroneous n_031031 change')
def process_define_new_refval(self, state, bit_operator, descriptor):
"""
Process defining a new reference value for the given descriptor.
:param state:
:param bit_operator:
:param descriptor:
"""
log.debug('Defining new reference value for {}'.format(descriptor))
if descriptor.unit == UNITS_STRING:
raise PyBufrKitError('Cannot define new reference value for descriptor of string value')
self.process_new_refval(state, bit_operator, descriptor, state.nbits_of_new_refval)
def subset(self, subset_indices):
if max(subset_indices) >= self.n_subsets.value:
raise PyBufrKitError('maximum subset index out of range')
if min(subset_indices) < 0:
raise PyBufrKitError('minimum subset index out of range')
data = []
for section in self.sections:
section_data = []
for parameter in section:
if parameter.type == PARAMETER_TYPE_TEMPLATE_DATA:
section_data.append([
v for i, v in enumerate(
parameter.value.decoded_values_all_subsets)
if i in subset_indices
])
else:
section_data.append(
len(subset_indices) if parameter.name ==
'n_subsets' else parameter.value)
data.append(section_data)
return data
def get_parameter_offset(self, parameter_name):
"""
Get the bit offset from the beginning of the section for parameter of
the given name.
:return: The bit offset.
:rtype: int
"""
nbits_offset = 0
for parameter in self:
if parameter.name == parameter_name:
return nbits_offset
else:
nbits_offset += parameter.nbits
else:
raise PyBufrKitError(
'Parameter "{}" not found'.format(parameter_name))
def render(self, obj):
"""
Render the given object as string.
:param object obj: The object to render
:return: A string representation of the given object.
:rtype: str
"""
if isinstance(obj, BufrMessage):
return self._render_bufr_message(obj)
elif isinstance(obj, TemplateData):
return self._render_template_data(obj)
elif isinstance(obj, Descriptor):
return self._render_descriptor(obj)
elif isinstance(obj, QueryResult):
return self._render_query_result(obj)
else:
raise PyBufrKitError('Unknown object {} for rendering'.format(type(obj)))
def command_encode(ns):
"""
Command to encode given JSON file from command line into BUFR file.
"""
encoder = Encoder(
definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory,
compiled_template_cache_max=ns.compiled_template_cache_max,
master_table_version=ns.master_table_version)
if ns.filename != '-':
with open(ns.filename) as ins:
s = ins.read()
else: # read from stdin, this is useful for piping
s = sys.stdin.read()
messages = {
(True, True): 'Nested JSON',
(True, False): 'Nested Text',
(False, True): 'Flat JSON',
(False, False): 'Flat Text',
}
try:
if ns.json:
data = json.loads(s)
if ns.attributed:
data = nested_json_to_flat_json(data)
else:
if ns.attributed:
data = nested_text_to_flat_json(s)
else:
data = flat_text_to_flat_json(s)
except (ValueError, SyntaxError):
raise PyBufrKitError('Invalid input: Is it in {} format?'.format(
messages[(ns.attributed, ns.json)]))
bufr_message = encoder.process(data,
'<stdin>' if ns.filename else ns.filename,
wire_template_data=False)
if ns.output_filename:
fmode = 'ab' if ns.append else 'wb'
with open(ns.output_filename, fmode) as outs:
if ns.preamble:
outs.write(str.encode(ns.preamble, encoding='utf8'))
outs.write(bufr_message.serialized_bytes)
def process(self, s, file_path='<string>', wire_template_data=True):
"""
Entry point for the encoding process. The process encodes a JSON format
message to BUFR message.
:param s: A JSON or its string serialized form
:param file_path: The file path to the JSON file.
:param wire_template_data: Whether to wire the template data to construct
a fully hierarchical structure from the flat lists.
:return: A bitstring object of the encoded message.
"""
if isinstance(s, (six.binary_type, six.text_type)):
# TODO: ensure all strings are loaded as plain ascii instead of unicode from JSON
json_data = json.loads(s) if six.PY3 else json.loads(s, encoding='latin-1')
else:
json_data = s
bit_writer = get_bit_writer()
bufr_message = BufrMessage(filename=file_path)
nbits_encoded = 0
section_index = 0
# When a section is not present in the json data. The index must not be
# increase for the section.
index_offset = 0
while True:
section = self.section_configurer.configure_section_with_values(
bufr_message, section_index, json_data[section_index - index_offset],
self.overrides)
section_index += 1
if section is None: # optional section is not present
index_offset += 1 # This section should not be counted
continue
nbits_encoded += self.process_section(bufr_message, bit_writer, section)
if section.end_of_message:
break
# A zero length means the actual length must be calculated
# Fix is needed for both the message object and the serialized bits
nbytes_write = bit_writer.get_pos() // NBITS_PER_BYTE
if bufr_message.length.value == 0 or self.ignore_declared_length:
bufr_message.length.value = nbytes_write
section = bufr_message.length.parent
bit_writer.set_uint(
bufr_message.length.value,
bufr_message.length.nbits,
section.get_metadata(BITPOS_START) + section.get_parameter_offset('length')
)
elif bufr_message.length.value != nbytes_write:
raise PyBufrKitError('Write exceeds declared total length {} by {} bytes'.format(
bufr_message.length.value, nbytes_write - bufr_message.length.value
))
bufr_message.serialized_bytes = bit_writer.to_bytes()
if wire_template_data:
bufr_message.wire()
return bufr_message
def n_repeats(self):
raise PyBufrKitError('Cannot access n_repeats for Delayed Replication')
