def build_bitmapped_descriptors(self, bitmap): """ Build the bitmapped descriptors based on the given bitmap. Also build the back referenced descriptors if it is not already defined. """ # Second get all the back referenced descriptors if it does not already exist if not self.back_referenced_descriptors: self.back_referenced_descriptors = [] for idx in range(self.back_reference_boundary - 1, -1, -1): descriptor = self.decoded_descriptors[idx] # The type has to be an exact match, not just isinstance if type(descriptor) is ElementDescriptor: self.back_referenced_descriptors.insert( 0, (idx, descriptor)) if len(self.back_referenced_descriptors) == len(bitmap): break if len(self.back_referenced_descriptors) != len(bitmap): raise PyBufrKitError( 'Back referenced descriptors not matching defined Bitmap') # Lastly, get all the descriptors that has a corresponding Zero bit value self.bitmapped_descriptors = [ (idx, d) for bit, (idx, d) in zip(bitmap, self.back_referenced_descriptors) if bit == 0 ] self.next_bitmapped_descriptor = functools.partial( next, iter(self.bitmapped_descriptors))
def wire_members(self, members): for member in members: # 221 YYY data not present for following YYY descriptors except class 0-9 and 31 if self.data_not_present_count: self.data_not_present_count -= 1 if isinstance(member, ElementDescriptor): X = member.X if not (1 <= X <= 9 or X == 31): # skipping self.add_node(NoValueDataNode(member)) continue # Now process normally if isinstance(member, ElementDescriptor): self.wire_element_descriptor(member) elif isinstance(member, FixedReplicationDescriptor): self.wire_fixed_replication_descriptor(member) elif isinstance(member, DelayedReplicationDescriptor): self.wire_delayed_replication_descriptor(member) elif isinstance(member, OperatorDescriptor): self.wire_operator_descriptor(member) elif isinstance(member, SequenceDescriptor): self.wire_sequence_descriptor(member) elif isinstance(member, SkippedLocalDescriptor): self.wire_skippable_local_descriptor() else: raise PyBufrKitError('Cannot wire descriptor type: {}'.format( type(member)))
def process(self, s, file_path='<string>', start_signature=MESSAGE_START_SIGNATURE, info_only=False, ignore_value_expectation=False, wire_template_data=True): """ Decoding the given message string. :param s: Message string that contains the BUFR Message :param file_path: The file where this string is read from. :param start_signature: Locate the starting position of the message string with the given signature. :param info_only: Only show information up to template data (exclusive) :param ignore_value_expectation: Do not validate the expected value :param wire_template_data: Whether to wire the template data to construct a fully hierarchical structure from the flat lists. Only takes effect when it is NOT info_only. :return: A BufrMessage object that contains the decoded information. """ idx = s.find(start_signature) if start_signature is not None else 0 if idx == -1: raise PyBufrKitError( 'Cannot find start signature: {}'.format(start_signature)) s = s[idx:] bit_reader = get_bit_reader(s) bufr_message = BufrMessage(file_path) configuration_transformers = ( self.section_configurer.info_configuration, ) if info_only else () if ignore_value_expectation: configuration_transformers += ( self.section_configurer.ignore_value_expectation, ) nbits_decoded = 0 section_index = 0 # Always start decoding from section 0 while True: section = self.section_configurer.configure_section( bufr_message, section_index, configuration_transformers) section_index += 1 if section is None: # when optional section is not present continue nbits_decoded += self.process_section(bufr_message, bit_reader, section) if section.end_of_message: break # The exact bytes that have been decoded bufr_message.serialized_bytes = s[:nbits_decoded // NBITS_PER_BYTE] if not info_only and wire_template_data: bufr_message.wire() return bufr_message
def process_statements(coder, state, bit_operator, statements): """ Process through a list of statements. Recursively call itself if the sub-statement is itself a list of statements. """ for statement in statements: if isinstance(statement, MethodCall): # Populate any necessary state properties. This is to re-create the # modifier effects of operator descriptors. if statement.state_properties is not None: for k, v in statement.state_properties.items(): setattr(state, k, v) if type(statement) is StateMethodCall: getattr(state, statement.method_name)(*statement.args) elif type(statement) is CoderMethodCall: if issubclass(type(statement.args[0]), Descriptor): getattr(coder, statement.method_name)(state, bit_operator, *statement.args) else: getattr(coder, statement.method_name)(state, *statement.args) else: raise PyBufrKitError('Unknown statement: {}'.format(statement)) elif isinstance(statement, State031031Reset): state.n_031031 = 0 elif isinstance(statement, State031031Increment): state.n_031031 += 1 elif type(statement) is Loop: if isinstance(statement.repeat, CoderMethodCall): repeat = getattr(coder, statement.repeat.method_name)(state) else: repeat = statement.repeat for _ in range(repeat): process_statements(coder, state, bit_operator, statement.statements) else: raise PyBufrKitError('Unknown statement: {}'.format(statement))
def process_section(self, bufr_message, bit_reader, section): """ Decode the given configured Section. :param bufr_message: The BUFR message object. :param section: The BUFR section object. :param bit_reader: :return: Number of bits decoded for this section. """ section.set_metadata(BITPOS_START, bit_reader.get_pos()) for parameter in section: if parameter.type == PARAMETER_TYPE_UNEXPANDED_DESCRIPTORS: parameter.value = self.process_unexpanded_descriptors( bit_reader, section) elif parameter.type == PARAMETER_TYPE_TEMPLATE_DATA: parameter.value = self.process_template_data( bufr_message, bit_reader) elif parameter.nbits == 0: # Zero number of bits means to read all bits till the end of the section parameter.value = bit_reader.read( parameter.type, section.section_length.value * NBITS_PER_BYTE - (bit_reader.get_pos() - section.get_metadata(BITPOS_START))) else: parameter.value = bit_reader.read(parameter.type, parameter.nbits) log.debug('{} = {!r}'.format(parameter.name, parameter.value)) # Make available as a property of the overall message object if parameter.as_property: setattr(bufr_message, parameter.name, parameter) if parameter.expected is not None: assert parameter.value == parameter.expected, 'Value ({!r}) not as expected ({!r})'.format( parameter.value, parameter.expected) # TODO: option to ignore the declared length? # TODO: this depends on a specific parameter name, need change to parameter type? if 'section_length' in section: nbits_read = bit_reader.get_pos() - section.get_metadata( BITPOS_START) nbits_unread = section.section_length.value * NBITS_PER_BYTE - nbits_read if nbits_unread > 0: log.debug('Skipping {} bits to end of the section'.format( nbits_unread)) bit_reader.read_bin(nbits_unread) elif nbits_unread < 0: raise PyBufrKitError( 'Read exceeds declared section {} length: {} by {} bits'. format(section.get_metadata('index'), section.section_length.value, -nbits_unread)) return bit_reader.get_pos() - section.get_metadata(BITPOS_START)
def process_section(self, bufr_message, bit_writer, section): section.set_metadata(BITPOS_START, bit_writer.get_pos()) for parameter in section: if parameter.type == PARAMETER_TYPE_UNEXPANDED_DESCRIPTORS: self.process_unexpanded_descriptors(bit_writer, parameter) elif parameter.type == PARAMETER_TYPE_TEMPLATE_DATA: self.process_template_data(bufr_message, bit_writer, parameter) else: bit_writer.write(parameter.value, parameter.type, parameter.nbits) log.debug('{} = {!r}'.format(parameter.name, parameter.value)) # Make available as a property of the overall message object if parameter.as_property: setattr(bufr_message, parameter.name, parameter) nbits_write = (bit_writer.get_pos() - section.get_metadata(BITPOS_START)) nbytes_write, nbits_residue = nbits_write // NBITS_PER_BYTE, nbits_write % NBITS_PER_BYTE # For edition 3 and earlier, ensure each section has an even number of octets. # This is done by padding Zeros to the required number of octets. if bufr_message.edition.value <= 3: if nbytes_write % 2 != 0: nbits_padding_for_octet = (NBITS_PER_BYTE - nbits_residue) else: nbits_padding_for_octet = 0 if nbits_residue == 0 else (2 * NBITS_PER_BYTE - nbits_residue) else: # otherwise just padding to any integer multiple of octet nbits_padding_for_octet = 0 if nbits_residue == 0 else (NBITS_PER_BYTE - nbits_residue) if nbits_padding_for_octet != 0: log.debug('Padding {} bits for complete Octets'.format(nbits_padding_for_octet)) bit_writer.write_bin('0' * nbits_padding_for_octet) if 'section_length' in section: nbits_write = bit_writer.get_pos() - section.get_metadata(BITPOS_START) # A zero length means the length should be calculated if section.section_length.value == 0 or self.ignore_declared_length: section.section_length.value = nbits_write // NBITS_PER_BYTE bit_writer.set_uint( section.section_length.value, section.section_length.nbits, section.get_metadata(BITPOS_START) + section.get_parameter_offset('section_length') ) else: nbits_unwrite = section.section_length.value * NBITS_PER_BYTE - nbits_write if nbits_unwrite > 0: log.debug('Padding {} bits to for declared length of the section'.format(nbits_unwrite)) bit_writer.skip(nbits_unwrite) elif nbits_unwrite < 0: raise PyBufrKitError('Writing exceeds declared section length {} by {} bytes'.format( section.section_length.value, -nbits_unwrite // NBITS_PER_BYTE )) return bit_writer.get_pos() - section.get_metadata(BITPOS_START)
def get_value_for_delayed_replication_factor(self, idx): if self.is_compressed: self._assert_equal_values_of_index(idx) value = self.decoded_values_all_subsets[0][idx] else: value = self.decoded_values[idx] if value is None or value < 0: raise PyBufrKitError('Delayed replication factor must be >= 0: got ({!r})'.format(value)) return value
def process_bitmap_definition(self, state, bit_operator, descriptor): n_031031 = state.n_031031 super(TemplateCompiler, self).process_bitmap_definition(state, bit_operator, descriptor) if state.n_031031 == 0: state.add_statement(State031031Reset()) elif state.n_031031 == n_031031 + 1: state.add_statement(State031031Increment()) elif state.n_031031 == n_031031: pass else: raise PyBufrKitError('erroneous n_031031 change')
def process_define_new_refval(self, state, bit_operator, descriptor): """ Process defining a new reference value for the given descriptor. :param state: :param bit_operator: :param descriptor: """ log.debug('Defining new reference value for {}'.format(descriptor)) if descriptor.unit == UNITS_STRING: raise PyBufrKitError('Cannot define new reference value for descriptor of string value') self.process_new_refval(state, bit_operator, descriptor, state.nbits_of_new_refval)
def subset(self, subset_indices): if max(subset_indices) >= self.n_subsets.value: raise PyBufrKitError('maximum subset index out of range') if min(subset_indices) < 0: raise PyBufrKitError('minimum subset index out of range') data = [] for section in self.sections: section_data = [] for parameter in section: if parameter.type == PARAMETER_TYPE_TEMPLATE_DATA: section_data.append([ v for i, v in enumerate( parameter.value.decoded_values_all_subsets) if i in subset_indices ]) else: section_data.append( len(subset_indices) if parameter.name == 'n_subsets' else parameter.value) data.append(section_data) return data
def get_parameter_offset(self, parameter_name): """ Get the bit offset from the beginning of the section for parameter of the given name. :return: The bit offset. :rtype: int """ nbits_offset = 0 for parameter in self: if parameter.name == parameter_name: return nbits_offset else: nbits_offset += parameter.nbits else: raise PyBufrKitError( 'Parameter "{}" not found'.format(parameter_name))
def render(self, obj): """ Render the given object as string. :param object obj: The object to render :return: A string representation of the given object. :rtype: str """ if isinstance(obj, BufrMessage): return self._render_bufr_message(obj) elif isinstance(obj, TemplateData): return self._render_template_data(obj) elif isinstance(obj, Descriptor): return self._render_descriptor(obj) elif isinstance(obj, QueryResult): return self._render_query_result(obj) else: raise PyBufrKitError('Unknown object {} for rendering'.format(type(obj)))
def command_encode(ns): """ Command to encode given JSON file from command line into BUFR file. """ encoder = Encoder( definitions_dir=ns.definitions_directory, tables_root_dir=ns.tables_root_directory, compiled_template_cache_max=ns.compiled_template_cache_max, master_table_version=ns.master_table_version) if ns.filename != '-': with open(ns.filename) as ins: s = ins.read() else: # read from stdin, this is useful for piping s = sys.stdin.read() messages = { (True, True): 'Nested JSON', (True, False): 'Nested Text', (False, True): 'Flat JSON', (False, False): 'Flat Text', } try: if ns.json: data = json.loads(s) if ns.attributed: data = nested_json_to_flat_json(data) else: if ns.attributed: data = nested_text_to_flat_json(s) else: data = flat_text_to_flat_json(s) except (ValueError, SyntaxError): raise PyBufrKitError('Invalid input: Is it in {} format?'.format( messages[(ns.attributed, ns.json)])) bufr_message = encoder.process(data, '<stdin>' if ns.filename else ns.filename, wire_template_data=False) if ns.output_filename: fmode = 'ab' if ns.append else 'wb' with open(ns.output_filename, fmode) as outs: if ns.preamble: outs.write(str.encode(ns.preamble, encoding='utf8')) outs.write(bufr_message.serialized_bytes)
def process(self, s, file_path='<string>', wire_template_data=True): """ Entry point for the encoding process. The process encodes a JSON format message to BUFR message. :param s: A JSON or its string serialized form :param file_path: The file path to the JSON file. :param wire_template_data: Whether to wire the template data to construct a fully hierarchical structure from the flat lists. :return: A bitstring object of the encoded message. """ if isinstance(s, (six.binary_type, six.text_type)): # TODO: ensure all strings are loaded as plain ascii instead of unicode from JSON json_data = json.loads(s) if six.PY3 else json.loads(s, encoding='latin-1') else: json_data = s bit_writer = get_bit_writer() bufr_message = BufrMessage(filename=file_path) nbits_encoded = 0 section_index = 0 # When a section is not present in the json data. The index must not be # increase for the section. index_offset = 0 while True: section = self.section_configurer.configure_section_with_values( bufr_message, section_index, json_data[section_index - index_offset], self.overrides) section_index += 1 if section is None: # optional section is not present index_offset += 1 # This section should not be counted continue nbits_encoded += self.process_section(bufr_message, bit_writer, section) if section.end_of_message: break # A zero length means the actual length must be calculated # Fix is needed for both the message object and the serialized bits nbytes_write = bit_writer.get_pos() // NBITS_PER_BYTE if bufr_message.length.value == 0 or self.ignore_declared_length: bufr_message.length.value = nbytes_write section = bufr_message.length.parent bit_writer.set_uint( bufr_message.length.value, bufr_message.length.nbits, section.get_metadata(BITPOS_START) + section.get_parameter_offset('length') ) elif bufr_message.length.value != nbytes_write: raise PyBufrKitError('Write exceeds declared total length {} by {} bytes'.format( bufr_message.length.value, nbytes_write - bufr_message.length.value )) bufr_message.serialized_bytes = bit_writer.to_bytes() if wire_template_data: bufr_message.wire() return bufr_message
def n_repeats(self): raise PyBufrKitError('Cannot access n_repeats for Delayed Replication')