class EncoderTests(unittest.TestCase):
def setUp(self):
self.encoder = Encoder()
self.decoder = Decoder()
self.filename_stubs = [
'IUSK73_AMMC_182300',
'rado_250', # uncompressed with 222000, 224000, 236000
'207003', # compressed with delayed replication
'amv2_87', # compressed with 222000
'b005_89', # compressed with 222000 and 224000 (1st order stats)
'profiler_european', # uncompressed with 204001 associated fields
'jaso_214', # compressed with 204001 associated fields
'uegabe', # uncompressed with 204004 associated fields
'asr3_190', # compressed with complex replication and 222000, 224000
'b002_95', # uncompressed with skipped local descriptors
'g2nd_208', # compressed with identical string values for all subsets
'ISMD01_OKPR', # compressed with different string values for subsets
'mpco_217',
]
def tearDown(self):
pass
def do_test(self, filename_stub):
with open(os.path.join(DATA_DIR, filename_stub + '.json')) as ins:
s = ins.read()
bins = self.encoder.encode(s)
self.decoder.decode(bins.bytes)
assert len(self.encoder.decoded_values_all_subsets) == len(
self.decoder.decoded_values_all_subsets)
for idx_subset in range(len(self.encoder.decoded_values_all_subsets)):
encoder_values = self.encoder.decoded_values_all_subsets[
idx_subset]
decoder_values = self.decoder.decoded_values_all_subsets[
idx_subset]
assert len(encoder_values) == len(decoder_values)
for idx_value in range(len(encoder_values)):
if isinstance(encoder_values[idx_value], six.text_type):
encoder_value = encoder_values[idx_value].encode('latin-1')
else:
encoder_value = encoder_values[idx_value]
assert encoder_value == decoder_values[idx_value], \
'{!r} != {!r}'.format(encoder_value, decoder_values[idx_value])
def test_encode(self):
print()
for filename_stub in self.filename_stubs:
print(filename_stub)
self.do_test(filename_stub)
class BufrDataTests(unittest.TestCase):
def setUp(self):
self.decoder = Decoder()
self.filename_stubs = [
'IUSK73_AMMC_182300',
'rado_250', # uncompressed with 222000, 224000, 236000
'207003', # compressed with delayed replication
'amv2_87', # compressed with 222000
'b005_89', # compressed with 222000 and 224000 (1st order stats)
'profiler_european', # uncompressed with 204001 associated fields
'jaso_214', # compressed with 204001 associated fields
'uegabe', # uncompressed with 204004 associated fields
'asr3_190', # compressed with complex replication and 222000, 224000
'b002_95', # uncompressed with skipped local descriptors
'g2nd_208', # compressed with identical string values for all subsets
'ISMD01_OKPR', # compressed with different string values for subsets
'mpco_217',
]
def do_test(self, filename_stub):
s = read_bufr_file(filename_stub + '.bufr')
bufr = self.decoder.decode(s, filename_stub)
bufr_data = bufr.wire_data()
if filename_stub in ('207003', 'rado_250'):
with open(
os.path.join(
DATA_DIR,
'{}.datadump.cmp'.format(filename_stub))) as ins:
cmp_str = ins.read()
dump_str = bufr_data.dumps()
# TODO: this is to fix the inconsistent int and long of bitstring on different OS
dump_str = dump_str.replace('005040 ORBIT NUMBER 5258\n',
'005040 ORBIT NUMBER 5258L\n')
assert dump_str == cmp_str, dump_str
else:
bufr_data.dumps()
def test_bufr_data(self):
print()
for filename_stub in self.filename_stubs:
print(filename_stub)
self.do_test(filename_stub)
def test_path_string_parsing(self):
path = parse_position_string('1, 2, 3, 4')
assert path == (1, ((1, None), (2, None), (3, None), (4, None)), ())
path = parse_position_string('2,3[0:10:2],5[2].7[2].8')
assert path == (1, ((2, None), (3, slice(0, 10, 2)), (5, 2)),
((7, 2), (8, None)))
path = parse_position_string('#121, 1, 3, 5[3:].1. 3')
assert path == (121, ((1, None), (3, None), (5, slice(3, None, None))),
((1, None), (3, None)))
def test_query_by_path(self):
s = read_bufr_file('asr3_190.bufr')
bufr = self.decoder.decode(s, 'asr3_190')
bufr_data = bufr.wire_data()
assert bufr_data.query_by_position('1,1,3')[1] == 333
assert bufr_data.query_by_position('#1,1,1,3')[1] == 333
assert bufr_data.query_by_position('#128,1,1,3')[1] == 333
assert bufr_data.query_by_position('#1, 1, 4, 1')[1] == 24.87108
assert bufr_data.query_by_position('#3, 1, 4, 1')[1] == 24.87502
assert bufr_data.query_by_position('#1, 1, 11, 1, 2')[1] == [
236391100000000.0, 131971700000000.0, 64338200000000.0,
36184200000000.0, 30148700000000.0, 11316000000000.0,
6395700000000.0, 3612800000000.0, 10653400000000.0,
8571400000000.0, 6835300000000.0
]
assert bufr_data.query_by_position('#1, 1, 11, 1, 3.2.1')[1] == [
10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10
]
assert bufr_data.query_by_position('1, 11, 1, 3.1')[1] == \
bufr_data.query_by_position('7, 1')[1][0::6]
assert bufr_data.query_by_position('#128, 1, 11, 1, 14')[1] == [
None, None, None, 240.7, 215.7, 220.3, 227.1, 228.3, 224.2, 221.5,
218.3
]
def test_query_by_name(self):
s = read_bufr_file('asr3_190.bufr')
bufr = self.decoder.decode(s, 'asr3_190')
bufr_data = bufr.wire_data()
assert bufr_data.query_by_name('001007')[1] == [[57]] * 128
assert bufr_data.query_by_name('012063')[1][-1] == [
None, None, None, None, None, None, None, None, None, None, None,
None, None, None, None, None, None, None, 240.7, None, 240.7, None,
None, 240.7, 215.7, None, 215.7, None, None, 215.7, 220.3, None,
220.3, None, None, 220.3, 227.1, None, 227.1, None, None, 227.1,
228.3, None, 228.3, None, None, 228.3, 224.2, None, 224.2, None,
None, 224.2, 221.5, None, 221.5, None, None, 221.5, 218.3, None,
218.3, None, None, 218.3
]
assert bufr_data.query_by_name('012063.F12063.008023')[1] == [[10] * 66
] * 128
class DecoderTests(unittest.TestCase):
def setUp(self):
self.decoder = Decoder()
self.filename_stubs = [
'IUSK73_AMMC_182300',
'rado_250', # uncompressed with 222000, 224000, 236000
'207003', # compressed with delayed replication
'amv2_87', # compressed with 222000
'b005_89', # compressed with 222000 and 224000 (1st order stats)
'profiler_european', # uncompressed with 204001 associated fields
'jaso_214', # compressed with 204001 associated fields
'uegabe', # uncompressed with 204004 associated fields
'asr3_190', # compressed with complex replication and 222000, 224000
'b002_95', # uncompressed with skipped local descriptors
'g2nd_208', # compressed with identical string values for all subsets
'ISMD01_OKPR', # compressed with different string values for subsets
'mpco_217',
]
def tearDown(self):
pass
def _compare(self, cmp_file_name):
with open(os.path.join(DATA_DIR, cmp_file_name)) as ins:
lines = ins.readlines()
next_line = functools.partial(next, iter(lines))
for idx_subset in range(len(self.decoder.decoded_values_all_subsets)):
for idx, value in enumerate(
self.decoder.decoded_values_all_subsets[idx_subset]):
cmp_line = next_line().strip()
if value is None:
line = '{} {}'.format(idx + 1, repr(value))
assert line == cmp_line, \
'At line {}: {} != {}'.format(idx + 1, line, cmp_line)
elif isinstance(value, (binary_type, text_type)):
# TODO: better to decode all ascii bytes to unicode string
if isinstance(value, binary_type) and PY3:
line = '{} {}'.format(idx + 1, repr(value)[1:])
else:
line = '{} {}'.format(idx + 1, repr(value))
assert line == cmp_line, \
'At line {}: {} != {}'.format(idx + 1, line, cmp_line)
else:
field = cmp_line.split()[1]
if field.endswith('L'):
field = field[:-1]
cmp_value = eval(field)
assert abs(value - cmp_value) < 1.0e6, \
'At line {}: {} != {}'.format(idx + 1, value, cmp_value)
def _print_values(self):
for idx_subset in range(len(self.decoder.decoded_values_all_subsets)):
for idx, value in enumerate(
self.decoder.decoded_values_all_subsets[idx_subset]):
print(idx + 1, repr(value))
def do_test(self, filename_stub):
s = read_bufr_file(filename_stub + '.bufr')
self.decoder.decode(s, filename_stub)
self._compare(filename_stub + '.values.cmp')
def test_decode(self):
print()
for filename_stub in self.filename_stubs:
print(filename_stub)
self.do_test(filename_stub)