def command_subset(ns):
"""
Command to subset and save the given BUFR file.
"""
decoder = Decoder(
definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory,
compiled_template_cache_max=ns.compiled_template_cache_max)
encoder = Encoder(
definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory,
compiled_template_cache_max=ns.compiled_template_cache_max)
subset_indices = [int(x) for x in ns.subset_indices.split(',')]
with open(ns.filename, 'rb') as ins:
s = ins.read()
bufr_message = decoder.process(
s,
file_path=ns.filename,
wire_template_data=False,
ignore_value_expectation=ns.ignore_value_expectation)
data = bufr_message.subset(subset_indices)
nb = encoder.process(data,
file_path=ns.output_filename,
wire_template_data=False)
with open(ns.output_filename, 'wb') as outs:
outs.write(nb.serialized_bytes)
def setUp(self):
self.encoder = Encoder()
self.decoder = Decoder()
self.filename_stubs = [
'IUSK73_AMMC_182300',
'rado_250', # uncompressed with 222000, 224000, 236000
'207003', # compressed with delayed replication
'amv2_87', # compressed with 222000
'b005_89', # compressed with 222000 and 224000 (1st order stats)
'profiler_european', # uncompressed with 204001 associated fields
'jaso_214', # compressed with 204001 associated fields
'uegabe', # uncompressed with 204004 associated fields
'asr3_190', # compressed with complex replication and 222000, 224000
'b002_95', # uncompressed with skipped local descriptors
'g2nd_208', # compressed with identical string values for all subsets
'ISMD01_OKPR', # compressed with different string values for subsets
'mpco_217',
]
def command_encode(ns):
"""
Command to encode given JSON file from command line into BUFR file.
"""
encoder = Encoder(
definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory,
compiled_template_cache_max=ns.compiled_template_cache_max,
master_table_version=ns.master_table_version)
if ns.filename != '-':
with open(ns.filename) as ins:
s = ins.read()
else: # read from stdin, this is useful for piping
s = sys.stdin.read()
messages = {
(True, True): 'Nested JSON',
(True, False): 'Nested Text',
(False, True): 'Flat JSON',
(False, False): 'Flat Text',
}
try:
if ns.json:
data = json.loads(s)
if ns.attributed:
data = nested_json_to_flat_json(data)
else:
if ns.attributed:
data = nested_text_to_flat_json(s)
else:
data = flat_text_to_flat_json(s)
except (ValueError, SyntaxError):
raise PyBufrKitError('Invalid input: Is it in {} format?'.format(
messages[(ns.attributed, ns.json)]))
bufr_message = encoder.process(data,
'<stdin>' if ns.filename else ns.filename,
wire_template_data=False)
if ns.output_filename:
fmode = 'ab' if ns.append else 'wb'
with open(ns.output_filename, fmode) as outs:
if ns.preamble:
outs.write(str.encode(ns.preamble, encoding='utf8'))
outs.write(bufr_message.serialized_bytes)
class EncoderTests(unittest.TestCase):
def setUp(self):
self.encoder = Encoder()
self.decoder = Decoder()
self.filename_stubs = [
'IUSK73_AMMC_182300',
'rado_250', # uncompressed with 222000, 224000, 236000
'207003', # compressed with delayed replication
'amv2_87', # compressed with 222000
'b005_89', # compressed with 222000 and 224000 (1st order stats)
'profiler_european', # uncompressed with 204001 associated fields
'jaso_214', # compressed with 204001 associated fields
'uegabe', # uncompressed with 204004 associated fields
'asr3_190', # compressed with complex replication and 222000, 224000
'b002_95', # uncompressed with skipped local descriptors
'g2nd_208', # compressed with identical string values for all subsets
'ISMD01_OKPR', # compressed with different string values for subsets
'mpco_217',
]
def tearDown(self):
pass
def do_test(self, filename_stub):
with open(os.path.join(DATA_DIR, filename_stub + '.json')) as ins:
s = ins.read()
bins = self.encoder.encode(s)
self.decoder.decode(bins.bytes)
assert len(self.encoder.decoded_values_all_subsets) == len(
self.decoder.decoded_values_all_subsets)
for idx_subset in range(len(self.encoder.decoded_values_all_subsets)):
encoder_values = self.encoder.decoded_values_all_subsets[
idx_subset]
decoder_values = self.decoder.decoded_values_all_subsets[
idx_subset]
assert len(encoder_values) == len(decoder_values)
for idx_value in range(len(encoder_values)):
if isinstance(encoder_values[idx_value], six.text_type):
encoder_value = encoder_values[idx_value].encode('latin-1')
else:
encoder_value = encoder_values[idx_value]
assert encoder_value == decoder_values[idx_value], \
'{!r} != {!r}'.format(encoder_value, decoder_values[idx_value])
def test_encode(self):
print()
for filename_stub in self.filename_stubs:
print(filename_stub)
self.do_test(filename_stub)
def test_overriding_master_table_version():
encoder = Encoder(master_table_version=31)
bufr_message = encoder.process(CONTRIVED_JSON)
assert 31 == bufr_message.master_table_version.value
assert 31 == Decoder().process(bufr_message.serialized_bytes).master_table_version.value