Ejemplo n.º 1
0
class EncoderTests(unittest.TestCase):
    def setUp(self):
        self.encoder = Encoder()
        self.decoder = Decoder()
        self.filename_stubs = [
            'IUSK73_AMMC_182300',
            'rado_250',  # uncompressed with 222000, 224000, 236000
            '207003',  # compressed with delayed replication
            'amv2_87',  # compressed with 222000
            'b005_89',  # compressed with 222000 and 224000 (1st order stats)
            'profiler_european',  # uncompressed with 204001 associated fields
            'jaso_214',  # compressed with 204001 associated fields
            'uegabe',  # uncompressed with 204004 associated fields
            'asr3_190',  # compressed with complex replication and 222000, 224000
            'b002_95',  # uncompressed with skipped local descriptors
            'g2nd_208',  # compressed with identical string values for all subsets
            'ISMD01_OKPR',  # compressed with different string values for subsets
            'mpco_217',
        ]

    def tearDown(self):
        pass

    def do_test(self, filename_stub):
        with open(os.path.join(DATA_DIR, filename_stub + '.json')) as ins:
            s = ins.read()
        bins = self.encoder.encode(s)
        self.decoder.decode(bins.bytes)

        assert len(self.encoder.decoded_values_all_subsets) == len(
            self.decoder.decoded_values_all_subsets)

        for idx_subset in range(len(self.encoder.decoded_values_all_subsets)):
            encoder_values = self.encoder.decoded_values_all_subsets[
                idx_subset]
            decoder_values = self.decoder.decoded_values_all_subsets[
                idx_subset]
            assert len(encoder_values) == len(decoder_values)
            for idx_value in range(len(encoder_values)):
                if isinstance(encoder_values[idx_value], six.text_type):
                    encoder_value = encoder_values[idx_value].encode('latin-1')
                else:
                    encoder_value = encoder_values[idx_value]
                assert encoder_value == decoder_values[idx_value], \
                    '{!r} != {!r}'.format(encoder_value, decoder_values[idx_value])

    def test_encode(self):
        print()
        for filename_stub in self.filename_stubs:
            print(filename_stub)
            self.do_test(filename_stub)
Ejemplo n.º 2
0
class BufrDataTests(unittest.TestCase):
    def setUp(self):
        self.decoder = Decoder()
        self.filename_stubs = [
            'IUSK73_AMMC_182300',
            'rado_250',  # uncompressed with 222000, 224000, 236000
            '207003',  # compressed with delayed replication
            'amv2_87',  # compressed with 222000
            'b005_89',  # compressed with 222000 and 224000 (1st order stats)
            'profiler_european',  # uncompressed with 204001 associated fields
            'jaso_214',  # compressed with 204001 associated fields
            'uegabe',  # uncompressed with 204004 associated fields
            'asr3_190',  # compressed with complex replication and 222000, 224000
            'b002_95',  # uncompressed with skipped local descriptors
            'g2nd_208',  # compressed with identical string values for all subsets
            'ISMD01_OKPR',  # compressed with different string values for subsets
            'mpco_217',
        ]

    def do_test(self, filename_stub):
        s = read_bufr_file(filename_stub + '.bufr')
        bufr = self.decoder.decode(s, filename_stub)
        bufr_data = bufr.wire_data()

        if filename_stub in ('207003', 'rado_250'):
            with open(
                    os.path.join(
                        DATA_DIR,
                        '{}.datadump.cmp'.format(filename_stub))) as ins:
                cmp_str = ins.read()
            dump_str = bufr_data.dumps()
            # TODO: this is to fix the inconsistent int and long of bitstring on different OS
            dump_str = dump_str.replace('005040 ORBIT NUMBER 5258\n',
                                        '005040 ORBIT NUMBER 5258L\n')
            assert dump_str == cmp_str, dump_str
        else:
            bufr_data.dumps()

    def test_bufr_data(self):
        print()
        for filename_stub in self.filename_stubs:
            print(filename_stub)
            self.do_test(filename_stub)

    def test_path_string_parsing(self):
        path = parse_position_string('1, 2, 3, 4')
        assert path == (1, ((1, None), (2, None), (3, None), (4, None)), ())

        path = parse_position_string('2,3[0:10:2],5[2].7[2].8')
        assert path == (1, ((2, None), (3, slice(0, 10, 2)), (5, 2)),
                        ((7, 2), (8, None)))

        path = parse_position_string('#121, 1, 3, 5[3:].1. 3')
        assert path == (121, ((1, None), (3, None), (5, slice(3, None, None))),
                        ((1, None), (3, None)))

    def test_query_by_path(self):
        s = read_bufr_file('asr3_190.bufr')
        bufr = self.decoder.decode(s, 'asr3_190')
        bufr_data = bufr.wire_data()

        assert bufr_data.query_by_position('1,1,3')[1] == 333
        assert bufr_data.query_by_position('#1,1,1,3')[1] == 333
        assert bufr_data.query_by_position('#128,1,1,3')[1] == 333

        assert bufr_data.query_by_position('#1, 1, 4, 1')[1] == 24.87108
        assert bufr_data.query_by_position('#3, 1, 4, 1')[1] == 24.87502

        assert bufr_data.query_by_position('#1, 1, 11, 1, 2')[1] == [
            236391100000000.0, 131971700000000.0, 64338200000000.0,
            36184200000000.0, 30148700000000.0, 11316000000000.0,
            6395700000000.0, 3612800000000.0, 10653400000000.0,
            8571400000000.0, 6835300000000.0
        ]

        assert bufr_data.query_by_position('#1, 1, 11, 1, 3.2.1')[1] == [
            10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10
        ]

        assert bufr_data.query_by_position('1, 11, 1, 3.1')[1] == \
               bufr_data.query_by_position('7, 1')[1][0::6]

        assert bufr_data.query_by_position('#128, 1, 11, 1, 14')[1] == [
            None, None, None, 240.7, 215.7, 220.3, 227.1, 228.3, 224.2, 221.5,
            218.3
        ]

    def test_query_by_name(self):
        s = read_bufr_file('asr3_190.bufr')
        bufr = self.decoder.decode(s, 'asr3_190')
        bufr_data = bufr.wire_data()

        assert bufr_data.query_by_name('001007')[1] == [[57]] * 128
        assert bufr_data.query_by_name('012063')[1][-1] == [
            None, None, None, None, None, None, None, None, None, None, None,
            None, None, None, None, None, None, None, 240.7, None, 240.7, None,
            None, 240.7, 215.7, None, 215.7, None, None, 215.7, 220.3, None,
            220.3, None, None, 220.3, 227.1, None, 227.1, None, None, 227.1,
            228.3, None, 228.3, None, None, 228.3, 224.2, None, 224.2, None,
            None, 224.2, 221.5, None, 221.5, None, None, 221.5, 218.3, None,
            218.3, None, None, 218.3
        ]

        assert bufr_data.query_by_name('012063.F12063.008023')[1] == [[10] * 66
                                                                      ] * 128
Ejemplo n.º 3
0
class DecoderTests(unittest.TestCase):
    def setUp(self):
        self.decoder = Decoder()
        self.filename_stubs = [
            'IUSK73_AMMC_182300',
            'rado_250',  # uncompressed with 222000, 224000, 236000
            '207003',  # compressed with delayed replication
            'amv2_87',  # compressed with 222000
            'b005_89',  # compressed with 222000 and 224000 (1st order stats)
            'profiler_european',  # uncompressed with 204001 associated fields
            'jaso_214',  # compressed with 204001 associated fields
            'uegabe',  # uncompressed with 204004 associated fields
            'asr3_190',  # compressed with complex replication and 222000, 224000
            'b002_95',  # uncompressed with skipped local descriptors
            'g2nd_208',  # compressed with identical string values for all subsets
            'ISMD01_OKPR',  # compressed with different string values for subsets
            'mpco_217',
        ]

    def tearDown(self):
        pass

    def _compare(self, cmp_file_name):
        with open(os.path.join(DATA_DIR, cmp_file_name)) as ins:
            lines = ins.readlines()

        next_line = functools.partial(next, iter(lines))
        for idx_subset in range(len(self.decoder.decoded_values_all_subsets)):
            for idx, value in enumerate(
                    self.decoder.decoded_values_all_subsets[idx_subset]):
                cmp_line = next_line().strip()
                if value is None:
                    line = '{} {}'.format(idx + 1, repr(value))
                    assert line == cmp_line, \
                        'At line {}: {} != {}'.format(idx + 1, line, cmp_line)
                elif isinstance(value, (binary_type, text_type)):
                    # TODO: better to decode all ascii bytes to unicode string
                    if isinstance(value, binary_type) and PY3:
                        line = '{} {}'.format(idx + 1, repr(value)[1:])
                    else:
                        line = '{} {}'.format(idx + 1, repr(value))
                    assert line == cmp_line, \
                        'At line {}: {} != {}'.format(idx + 1, line, cmp_line)
                else:
                    field = cmp_line.split()[1]
                    if field.endswith('L'):
                        field = field[:-1]
                    cmp_value = eval(field)
                    assert abs(value - cmp_value) < 1.0e6, \
                        'At line {}: {} != {}'.format(idx + 1, value, cmp_value)

    def _print_values(self):
        for idx_subset in range(len(self.decoder.decoded_values_all_subsets)):
            for idx, value in enumerate(
                    self.decoder.decoded_values_all_subsets[idx_subset]):
                print(idx + 1, repr(value))

    def do_test(self, filename_stub):
        s = read_bufr_file(filename_stub + '.bufr')
        self.decoder.decode(s, filename_stub)
        self._compare(filename_stub + '.values.cmp')

    def test_decode(self):
        print()
        for filename_stub in self.filename_stubs:
            print(filename_stub)
            self.do_test(filename_stub)