Пример #1
0
def read_bufr_file(input_bufr_file):
    #  #[ read a bufr file
    """
    read the file using the BUFRReader class and get the data
    with the get_values_as_2d_array method
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bufr = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
    msg_nr = -1
    for msg_nr, msg in enumerate(bufr):
        num_subsets = msg.get_num_subsets()
        for subs, msg_or_subset_data in enumerate(msg):
            #names = msg_or_subset_data.names
            #units = msg_or_subset_data.units
            data = msg_or_subset_data.data
            if data.shape[0] == 0:
                print('NO DATA FOUND! this seems an empty BUFR message !')
                continue

            print('loaded BUFR msg nr. ', msg_nr, 'shape = ', data.shape)

            if len(data.shape) == 1:
                print('data[:2] = ', data[:2])
            else:
                print('data[:2,:2] = ', data[:2, :2])

            if subs > 1:
                break

    # close the file
    bufr.close()
    if msg_nr == -1:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Пример #2
0
def print_bufr_content1(input_bufr_file, output_fd, separator, max_msg_nr):
    #  #[ implementation 1
    """
    example implementation using the BUFRReader class
    combined with the get_values_as_2d_array method
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        # add header strings
        # print 'DEBUG: bob.msg_loaded ',bob.msg_loaded
        if bob.msg_loaded == 1:
            list_of_names = []
            list_of_units = []
            list_of_names.extend(bob.get_names())
            list_of_units.extend(bob.get_units())
            # print 'DEBUG: ',separator.join(list_of_names)
            # print 'DEBUG: ',separator.join(list_of_units)
            output_fd.write(separator.join(list_of_names) + "\n")
            output_fd.write(separator.join(list_of_units) + "\n")

        data = bob.get_values_as_2d_array()
        # print 'DEBUG: data.shape = ', data.shape
        if data.shape[0] * data.shape[1] == 0:
            print 'NO DATA FOUND! this seems an empty BUFR message !'
            continue

        for subs in range(len(data[:, 0])):
            output_fd.write(
                str(subs) + separator +
                separator.join(str(val) for val in data[subs, :]) + "\n")
        print 'converted BUFR msg nr. ', msg_nr
        if ((max_msg_nr > 0) and (msg_nr >= max_msg_nr)):
            print 'skipping remainder of this BUFR file'
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print 'no BUFR messages found, are you sure this is a BUFR file?'
def read_bufr_file(input_bufr_file):
    #  #[ read a bufr file
    """
    read the file using the BUFRReader class and get the data
    with the get_values_as_2d_array method
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        nsubsets = bob.get_num_subsets()
        data1 = bob.get_subset_values(1)

        if numpy.shape(data1)[0] == 0:
            print('NO DATA FOUND! this seems an empty BUFR message !')
            continue

        print('loaded BUFR msg nr. ', msg_nr, 'shape = ', numpy.shape(data1))
        print('data1[:2] = ', data1[:2])

        if nsubsets > 1:
            data2 = bob.get_subset_values(2)

            if numpy.shape(data2)[0] == 0:
                print('NO DATA FOUND! this seems an empty BUFR message !')
                continue

            print('data2[:2] = ', data2[:2])

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
def reopen_bufr_file(input_bufr_file):
    #  #[ open bufr file
    '''
    open a bufr file for reading and print its content
    '''
    print '*'*50
    from pybufr_ecmwf.bufr import BUFRReader

    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
    bob.setup_tables(table_b_to_use='B'+TABLE_NAME,
                     table_d_to_use='D'+TABLE_NAME)

    bob.get_next_msg()
    print 'num_subsets:  ', bob.get_num_subsets()

    if USE_DELAYED_REPLICATION:
        data1 = bob.get_subset_values(0)
        print 'data1 = ', data1
        data2 = bob.get_subset_values(1)
        print 'data2 = ', data2
    else:
        print 'num_elements: ', bob.get_num_elements()
        print bob.get_names()
        print bob.get_units()
        data = bob.get_values_as_2d_array()
        print data.shape
        print data
        print 'bob.bufr_obj.values = '
        print bob.bufr_obj.values

    textdata = bob.get_value(3, 0)
    print 'textdata(3,0)', textdata
    textdata = bob.get_value(3, 0, get_cval=True)
    print 'textdata(3,0)', textdata
    textdata = bob.get_values(3, get_cval=True)
    print 'textdata(3,:)', textdata

    bob.close()
Пример #5
0
def reopen_bufr_file(input_bufr_file):
    #  #[ open bufr file
    '''
    open a bufr file for reading and print its content
    '''
    print '*'*50
    from pybufr_ecmwf.bufr import BUFRReader

    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
    bob.setup_tables(table_b_to_use='B'+TABLE_NAME,
                     table_d_to_use='D'+TABLE_NAME)

    bob.get_next_msg()
    print 'num_subsets:  ', bob.get_num_subsets()

    if USE_DELAYED_REPLICATION:
        data1 = bob.get_subset_values(0)
        print 'data1 = ', data1
        data2 = bob.get_subset_values(1)
        print 'data2 = ', data2
    else:
        print 'num_elements: ', bob.get_num_elements()
        print bob.get_names()
        print bob.get_units()
        data = bob.get_values_as_2d_array()
        print data.shape
        print data
        print 'bob.bufr_obj.values = '
        print bob.bufr_obj.values

    textdata = bob.get_value(3, 0)
    print 'textdata(3,0)', textdata
    textdata = bob.get_value(3, 0, get_cval=True)
    print 'textdata(3,0)', textdata
    textdata = bob.get_values(3, get_cval=True)
    print 'textdata(3,:)', textdata

    bob.close()
def read_bufr_file(input_bufr_file):
    #  #[ read a bufr file
    """
    read the file using the BUFRReader class and get the data
    with the get_values_as_2d_array method
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bufr = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
    msg_nr = -1
    for msg_nr, msg in enumerate(bufr):
        num_subsets = msg.get_num_subsets()
        for subs, msg_or_subset_data in enumerate(msg):
            #names = msg_or_subset_data.names
            #units = msg_or_subset_data.units
            data = msg_or_subset_data.data
            if data.shape[0] == 0:
                print('NO DATA FOUND! this seems an empty BUFR message !')
                continue

            print('loaded BUFR msg nr. ', msg_nr,
                  'shape = ', data.shape)

            if len(data.shape) == 1:
                print('data[:2] = ', data[:2].tolist())
            else:
                print('data[:2,:2] = ', data[:2,:2].tolist())

            if subs > 1:
                break

    # close the file
    bufr.close()
    if msg_nr == -1:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Пример #7
0
def print_bufr_content5(input_bufr_file, output_fd, separator, max_msg_nr,
                        expand_flags):
    #  #[ implementation 5
    """
    example implementation using the BUFRReader class
    to decode a bufr file using delayed replication.
    Since these files may have different descriptor lists
    for each subset, a different call pattern is needed.
    """

    # testcases:
    # ./example_programs/bufr_to_ascii.py -5 -c -o tmp.csv \
    #     -i ./pybufr_ecmwf/ecmwf_bufr_lib/bufrdc_000403/data/syno_1.bufr
    #
    # ./example_programs/bufr_to_ascii.py -5 -c -o tmp.csv \
    #     -i ../BUFR_test_files/synop_knmi_via_ko_janssen/MSSAEOL_00002950.b
    #

    names_to_be_selected = ['temperature', 'wind']
    names_to_be_excluded = ['minimum', 'maximum']

    write_names_and_units_just_once = True

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file,
                     warn_about_bufr_size=False,
                     verbose=False,
                     expand_flags=expand_flags)

    msg_nr = 0
    not_yet_printed = True
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        # since this example assumes a bufr file using delayed replication
        # always request and add the header for each subset
        nsubsets = bob.get_num_subsets()
        for subs in range(1, nsubsets + 1):

            print('==> subset ', subs)

            # add header strings
            (list_of_names, list_of_units) = bob.get_names_and_units(subs)
            data = bob.get_subset_values(subs)  #,autoget_cval=True)

            selected_names = []
            selected_units = []
            selected_values = []
            for i, name in enumerate(list_of_names):
                selected = False
                for name in names_to_be_selected:
                    if name in name.lower():
                        selected = True
                for name in names_to_be_excluded:
                    if name in name.lower():
                        selected = False

                if selected:
                    # print(' '*10,name,'=',data[i],list_of_units[i])
                    selected_names.append(list_of_names[i])
                    selected_units.append(list_of_units[i])
                    selected_values.append(data[i])

            if len(selected_values) == 0:
                print('NO DATA SELECTED for BUFR message %d and subset %d!' %
                      (msg_nr, subs))
                continue

            if write_names_and_units_just_once and not_yet_printed:
                output_fd.write('"subset nr"' + separator +
                                separator.join(selected_names) + "\n")
                output_fd.write('""' + separator +
                                separator.join(selected_units) + "\n")
                not_yet_printed = False

            output_fd.write(
                str(subs) + separator +
                separator.join(str(val) for val in selected_values) + "\n")

        print('=' * 25)
        print('converted BUFR msg nr. ', msg_nr)
        print('=' * 25)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Пример #8
0
def print_bufr_content4(input_bufr_file, output_fd, separator, max_msg_nr,
                        expand_flags, expand_strings, descr_multiplier):
    #  #[ implementation 4
    """
    example implementation using the BUFRReader class
    to decode a bufr file using delayed replication.
    Since these files may have different descriptor lists
    for each subset, a different call pattern is needed.
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bufr = BUFRReader(
        input_bufr_file,
        warn_about_bufr_size=False,
        #verbose=True, expand_flags=expand_flags,
        verbose=False,
        expand_flags=expand_flags,
        expand_strings=expand_strings,
        descr_multiplyer=descr_multiplier)

    msg_nr = -1
    for msg_nr, msg in enumerate(bufr):
        # since this example assumes a bufr file using delayed replication
        # always request and add the header for each subset
        nsubsets = msg.get_num_subsets()
        # print('nsubsets = ', nsubsets)

        for msg_or_subs_nr, msg_or_subset_data in enumerate(msg):
            # add header strings
            list_of_names = msg_or_subset_data.names
            list_of_units = msg_or_subset_data.units
            data = msg_or_subset_data.data
            if numpy.shape(data)[0] == 0:
                print('NO DATA FOUND! this seems an empty BUFR message !')
                continue
            output_fd.write('"subset nr"' + separator +
                            separator.join(list_of_names) + "\n")
            output_fd.write('""' + separator + separator.join(list_of_units) +
                            "\n")

            # print(data.shape)
            if len(data.shape) == 1:
                # we are walking over subsets
                subs = msg_or_subs_nr
                output_fd.write(
                    str(subs + 1) + separator +
                    separator.join(to_str(val) for val in data[:]) + "\n")
            else:
                # we are getting a 2D array as result
                for subs in range(data.shape[0]):
                    output_fd.write(
                        str(subs + 1) + separator +
                        separator.join(to_str(val)
                                       for val in data[subs, :]) + "\n")

        print('converted BUFR msg nr. ', msg_nr + 1)
        if (max_msg_nr >= 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bufr.close()
    if msg_nr == -1:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Пример #9
0
def print_bufr_content5(input_bufr_file, output_fd, separator,
                        max_msg_nr, expand_flags):
    #  #[ implementation 5
    """
    example implementation using the BUFRReader class
    to decode a bufr file using delayed replication.
    Since these files may have different descriptor lists
    for each subset, a different call pattern is needed.
    """

    # testcases:
    # ./example_programs/bufr_to_ascii.py -5 -c -o tmp.csv \
    #     -i ./pybufr_ecmwf/ecmwf_bufr_lib/bufrdc_000403/data/syno_1.bufr
    #
    # ./example_programs/bufr_to_ascii.py -5 -c -o tmp.csv \
    #     -i ../BUFR_test_files/synop_knmi_via_ko_janssen/MSSAEOL_00002950.b
    #

    names_to_be_selected = ['temperature', 'wind']
    names_to_be_excluded = ['minimum', 'maximum']

    write_names_and_units_just_once = True

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False,
                     verbose=False, expand_flags=expand_flags)

    msg_nr = 0
    not_yet_printed = True
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        # since this example assumes a bufr file using delayed replication
        # always request and add the header for each subset
        nsubsets = bob.get_num_subsets()
        for subs in range(1, nsubsets+1):

            print('==> subset ', subs)

            # add header strings
            (list_of_names, list_of_units) = bob.get_names_and_units(subs)
            data = bob.get_subset_values(subs) #,autoget_cval=True)

            selected_names = []
            selected_units = []
            selected_values = []
            for i, name in enumerate(list_of_names):
                selected = False
                for name in names_to_be_selected:
                    if name in name.lower():
                        selected = True
                for name in names_to_be_excluded:
                    if name in name.lower():
                        selected = False

                if selected:
                    # print(' '*10,name,'=',data[i],list_of_units[i])
                    selected_names.append(list_of_names[i])
                    selected_units.append(list_of_units[i])
                    selected_values.append(data[i])

            if len(selected_values) == 0:
                print('NO DATA SELECTED for BUFR message %d and subset %d!' %
                      (msg_nr, subs))
                continue

            if write_names_and_units_just_once and not_yet_printed:
                output_fd.write('"subset nr"'+separator+
                                separator.join(selected_names) + "\n")
                output_fd.write('""'+separator+
                                separator.join(selected_units) + "\n")
                not_yet_printed = False

            output_fd.write(str(subs)+separator+
                            separator.join(str(val) for val in selected_values)+
                            "\n")

        print('='*25)
        print('converted BUFR msg nr. ', msg_nr)
        print('='*25)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Пример #10
0
def print_bufr_content1(input_bufr_file, output_fd, separator,
                        max_msg_nr, expand_flags):
    #  #[ implementation 1
    """
    example implementation using the BUFRReader class
    combined with the get_values_as_2d_array method
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False,
                     expand_flags=expand_flags)

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        # add header strings
        # print('DEBUG: bob.msg_loaded ',bob.msg_loaded)
        list_of_names = []
        list_of_units = []
        list_of_names.extend(bob.get_names())
        list_of_units.extend(bob.get_units())
        list_of_unexp_descr = bob.get_unexp_descr_list()

        #print('list_of_names = ',list_of_names)
        #print('list_of_units = ',list_of_units)

        if bob.msg_loaded == 1:
            output_fd.write('"subset nr"'+separator)
            if list_of_names:
                for name in list_of_names[:-1]:
                    output_fd.write('"'+name+'"'+separator)
                name = list_of_names[-1]
                output_fd.write('"'+name+'"\n')
            else:
                output_fd.write('"[NO DATA]"\n')

            output_fd.write('""'+separator)
            if list_of_units:
                for unit in list_of_units[:-1]:
                    output_fd.write('"'+unit+'"'+separator)
                unit = list_of_units[-1]
                output_fd.write('"'+unit+'"\n')
            else:
                output_fd.write('"[NO DATA]"\n')

            list_of_unexp_descr_first_msg = bob.bufr_obj.py_unexp_descr_list
            #print('list_of_unexp_descr_first_msg = ',
            #      list_of_unexp_descr_first_msg)

        data = bob.get_values_as_2d_array()

        if list_of_unexp_descr != list_of_unexp_descr_first_msg:
            print('\n\n')
            print('WARNING: it seems different types of BUFR messages')
            print('are mixed in this BUFR file, meaning that the list of')
            print('descriptor names and units printed on the first 2 output')
            print('lines will not match with all lines of data.')
            print('To prevent confusion, therefore decoding is halted')
            print('It is recommended to first sort BUFR messages by type')
            print('before converting them to ascii or csv.')
            print('The example script sort_bufr_msgs.py can be used')
            print('to sort a BUFR file.')
            print('\n\n')
            print('Detailed info:')
            print('list_of_unexp_descr != list_of_unexp_descr_first_msg !')
            print('list_of_unexp_descr           = ',
                  list_of_unexp_descr)
            print('list_of_unexp_descr_first_msg = ',
                  list_of_unexp_descr_first_msg)
            sys.exit(0)

        if data.shape[0]*data.shape[1] == 0:
            print('NO DATA FOUND! this seems an empty BUFR message !')
            continue

        for subs in range(len(data[:, 0])):
            output_fd.write(str(subs+1)+separator+
                            separator.join(str(val) for val in data[subs, :])+
                            "\n")
        print('converted BUFR msg nr. ', msg_nr)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Пример #11
0
def print_bufr_content1(input_bufr_file, output_fd, separator,
                        max_msg_nr, expand_flags):
    #  #[ implementation 1
    """
    example implementation using the BUFRReader class
    combined with the get_values_as_2d_array method
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bufr = BUFRReader(input_bufr_file, warn_about_bufr_size=False,
                      expand_flags=expand_flags)
    msg_nr = -1
    for msg_nr, msg in enumerate(bufr):
        num_subsets = msg.get_num_subsets()
        list_of_unexp_descr = msg.get_unexp_descr_list()

        for subs, msg_or_subset_data in enumerate(msg):
            # get the actual values
            data = msg_or_subset_data.data

            if msg_nr == 0 and subs == 0:
                list_of_unexp_descr_first_msg = list_of_unexp_descr[:]

                # add header strings
                list_of_names = msg_or_subset_data.names
                list_of_units = msg_or_subset_data.units
                output_fd.write('"subset nr"'+separator)
                if list_of_names:
                    for name in list_of_names[:-1]:
                        output_fd.write('"'+name+'"'+separator)
                    name = list_of_names[-1]
                    output_fd.write('"'+name+'"\n')
                else:
                    output_fd.write('"[NO DATA]"\n')

                output_fd.write('""'+separator)
                if list_of_units:
                    for unit in list_of_units[:-1]:
                        output_fd.write('"'+unit+'"'+separator)
                    unit = list_of_units[-1]
                    output_fd.write('"'+unit+'"\n')
                else:
                    output_fd.write('"[NO DATA]"\n')

            try:
                data_is_2d = True
                if len(numpy.shape(data)) == 1:
                    data_is_2d = False

                if data_is_2d:
                    ns = numpy.shape(data)[0]
                    for subs_cnt in range(ns):
                        output_fd.write(str(subs_cnt+1)+separator+
                                        separator.join(str(val)
                                                       for val in
                                                       data[subs_cnt, :])+
                                        "\n")
                else:
                    # 1D data is returned if character values may be present
                    # in the data (in case autoget_cval or expand_flags
                    # are active)
                    # it may also happen if the message uses delayed
                    # replication and has variable lengths for the
                    # different subsets
                    subs_cnt = msg_or_subset_data.current_subset
                    output_fd.write(str(subs_cnt)+separator+
                                    separator.join(str(val)
                                                   for val in
                                                   data[:])+
                                    "\n")

            except TypeError:
                # in case of delayed replication or when character strings are
                # present (i.e. expand_flags = True) data will be returned as
                # a 1D list in stead of a 2D numpy array.
                # This generates a TypeError in the data[subs, :] indexing above
                output_fd.write(str(subs+1)+separator+
                                separator.join(str(val) for val in data[:])+
                                "\n")

        if list(list_of_unexp_descr) != list(list_of_unexp_descr_first_msg):
            print('\n\n')
            print('ERROR: it seems different types of BUFR messages')
            print('are mixed in this BUFR file, meaning that the list of')
            print('descriptor names and units printed on the first 2 output')
            print('lines will not match with all lines of data.')
            print('To prevent confusion, therefore decoding is halted')
            print('It is recommended to first sort BUFR messages by type')
            print('before converting them to ascii or csv.')
            print('The example script sort_bufr_msgs.py can be used')
            print('to sort a BUFR file.')
            print('\n\n')
            print('Detailed info:')
            print('list_of_unexp_descr != list_of_unexp_descr_first_msg !')
            print('list_of_unexp_descr           = ',
                  list_of_unexp_descr)
            print('list_of_unexp_descr_first_msg = ',
                  list_of_unexp_descr_first_msg)
            sys.exit(1)

        if numpy.shape(data)[0] == 0:
            print('NO DATA FOUND! this seems an empty BUFR message !')
            continue

        print('converted BUFR msg nr. ', msg_nr+1)
        if msg_nr >= max_msg_nr:
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bufr.close()
    if msg_nr == -1:
        print('no BUFR messages found, are you sure this is a BUFR file?')
def select_subsets(input_bufr_file, output_bufr_file):
    #  #[ select on subsets
    """
    select data and write out again
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)

    # open the file for writing
    rbf_out = RawBUFRFile()
    rbf_out.open(output_bufr_file, 'wb')

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        data = bob.get_values_as_2d_array()
        print 'data.shape = ', data.shape

        if data.shape[0]*data.shape[1] == 0:
            print 'NO DATA FOUND! this seems an empty BUFR message !'
            continue

        # select every other subset
        new_data = data[::2, :]

        print 'new_data.shape = ', new_data.shape
        #bob.bufr_obj

        nsub = bob.bufr_obj.get_num_subsets()
        n_exp_descr = len(bob.bufr_obj.values)/nsub
        bob.bufr_obj.fill_descriptor_list(nr_of_expanded_descriptors=
                                          n_exp_descr)
        bob.bufr_obj.ktdlst = bob.bufr_obj.get_descriptor_list()

        delayed_repl_data = bob.bufr_obj.derive_delayed_repl_factors()
        bob.bufr_obj.fill_delayed_repl_data(delayed_repl_data)

        new_nsub = new_data.shape[0]
        bob.bufr_obj.nr_subsets = new_nsub
        btm = BufrTemplate()
        btm.add_descriptors(*bob.bufr_obj.ktdlst)#[:self.ktdlen])
        btm.nr_of_delayed_repl_factors = 1
        btm.del_repl_max_nr_of_repeats_list = list(delayed_repl_data)
        bob.bufr_obj.register_and_expand_descriptors(btm)

        # activate this one if the encoding crashes without clear cause:
        # bob.bufr_obj.estimated_num_bytes_for_encoding = 25000

        bob.bufr_obj.kdate = new_nsub*list(delayed_repl_data)

        print 'bob.bufr_obj.cvals.shape = ', bob.bufr_obj.cvals.shape
        bob.bufr_obj.encode_data(new_data, bob.bufr_obj.cvals[:32, :])
        rbf_out.write_raw_bufr_msg(bob.bufr_obj.encoded_message)

        #for subs in range(len(data[:, 0])):
        #    output_fd.write(str(subs)+separator+
        #                    separator.join(str(val) for val in data[subs, :])+
        #                    "\n")
        print 'converted BUFR msg nr. ', msg_nr


    # close the file
    bob.close()
    if msg_nr == 0:
        print 'no BUFR messages found, are you sure this is a BUFR file?'

    rbf_out.close()
Пример #13
0
def select_subsets(input_bufr_file, output_bufr_file):
    #  #[ select on subsets
    """
    select data and write out again
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)

    # open the file for writing
    rbf_out = RawBUFRFile()
    rbf_out.open(output_bufr_file, 'wb')

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        data = bob.get_values_as_2d_array()
        print('data.shape = ', data.shape)

        if data.shape[0] * data.shape[1] == 0:
            print('NO DATA FOUND! this seems an empty BUFR message !')
            continue

        # select every other subset
        new_data = data[::2, :]

        print('new_data.shape = ', new_data.shape)
        #bob.bufr_obj

        nsub = bob.bufr_obj.get_num_subsets()
        n_exp_descr = len(bob.bufr_obj.values) / nsub
        bob.bufr_obj.fill_descriptor_list(
            nr_of_expanded_descriptors=n_exp_descr)
        bob.bufr_obj.ktdlst = bob.bufr_obj.get_descriptor_list()

        delayed_repl_data = bob.bufr_obj.derive_delayed_repl_factors()
        bob.bufr_obj.fill_delayed_repl_data(delayed_repl_data)

        new_nsub = new_data.shape[0]
        bob.bufr_obj.nr_subsets = new_nsub
        btm = BufrTemplate()
        btm.add_descriptors(*bob.bufr_obj.ktdlst)  #[:self.ktdlen])
        btm.nr_of_delayed_repl_factors = 1
        btm.del_repl_max_nr_of_repeats_list = list(delayed_repl_data)
        bob.bufr_obj.register_and_expand_descriptors(btm)

        # activate this one if the encoding crashes without clear cause:
        # bob.bufr_obj.estimated_num_bytes_for_encoding = 25000

        bob.bufr_obj.kdate = new_nsub * list(delayed_repl_data)

        print('bob.bufr_obj.cvals.shape = ', bob.bufr_obj.cvals.shape)
        bob.bufr_obj.encode_data(new_data, bob.bufr_obj.cvals[:32, :])
        rbf_out.write_raw_bufr_msg(bob.bufr_obj.encoded_message)

        #for subs in range(len(data[:, 0])):
        #    output_fd.write(str(subs)+separator+
        #                    separator.join(str(val) for val in data[subs, :])+
        #                    "\n")
        print('converted BUFR msg nr. ', msg_nr)

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')

    rbf_out.close()
print('values[-25:]: ', msg.values[-25:].tolist())

msg.write_msg_to_file()
bwr.close()

##############################
# reopen the BUFR file as test
##############################

print('*'*50)

input_bufr_file = output_bufr_file
bufr = BUFRReader(input_bufr_file, warn_about_bufr_size=False)

# just 1 msg in this test file, so no looping needed
for msg in bufr:
    print('num_subsets = ', msg.get_num_subsets())
    for subs, msg_or_subset_data in enumerate(msg):
        list_of_names = msg_or_subset_data.names
        list_of_units = msg_or_subset_data.units
        data = msg_or_subset_data.data
        print('"subset nr"'+','+','.join(list_of_names))
        print('""'+','+','.join(list_of_units))
        print('data.shape = ', data.shape)
        for irow, row in enumerate(data):
            print('{}, {} : {}'.
                  format(subs+1, irow,
                         ', '.join('{}'.format(val) for val in row.tolist())))

bufr.close()
Пример #15
0
def print_bufr_content1(input_bufr_file, output_fd, separator, max_msg_nr,
                        expand_flags):
    #  #[ implementation 1
    """
    example implementation using the BUFRReader class
    combined with the get_values_as_2d_array method
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bufr = BUFRReader(input_bufr_file,
                      warn_about_bufr_size=False,
                      expand_flags=expand_flags)
    msg_nr = -1
    for msg_nr, msg in enumerate(bufr):
        num_subsets = msg.get_num_subsets()
        list_of_unexp_descr = msg.get_unexp_descr_list()

        for subs, msg_or_subset_data in enumerate(msg):
            # get the actual values
            data = msg_or_subset_data.data

            if msg_nr == 0 and subs == 0:
                list_of_unexp_descr_first_msg = list_of_unexp_descr[:]

                # add header strings
                list_of_names = msg_or_subset_data.names
                list_of_units = msg_or_subset_data.units
                output_fd.write('"subset nr"' + separator)
                if list_of_names:
                    for name in list_of_names[:-1]:
                        output_fd.write('"' + name + '"' + separator)
                    name = list_of_names[-1]
                    output_fd.write('"' + name + '"\n')
                else:
                    output_fd.write('"[NO DATA]"\n')

                output_fd.write('""' + separator)
                if list_of_units:
                    for unit in list_of_units[:-1]:
                        output_fd.write('"' + unit + '"' + separator)
                    unit = list_of_units[-1]
                    output_fd.write('"' + unit + '"\n')
                else:
                    output_fd.write('"[NO DATA]"\n')

            try:
                data_is_2d = True
                if len(numpy.shape(data)) == 1:
                    data_is_2d = False

                if data_is_2d:
                    ns = numpy.shape(data)[0]
                    for subs_cnt in range(ns):
                        output_fd.write(
                            str(subs_cnt + 1) + separator + separator.join(
                                str(val) for val in data[subs_cnt, :]) + "\n")
                else:
                    # 1D data is returned if character values may be present
                    # in the data (in case autoget_cval or expand_flags
                    # are active)
                    # it may also happen if the message uses delayed
                    # replication and has variable lengths for the
                    # different subsets
                    subs_cnt = msg_or_subset_data.current_subset
                    output_fd.write(
                        str(subs_cnt) + separator +
                        separator.join(str(val) for val in data[:]) + "\n")

            except TypeError:
                # in case of delayed replication or when character strings are
                # present (i.e. expand_flags = True) data will be returned as
                # a 1D list in stead of a 2D numpy array.
                # This generates a TypeError in the data[subs, :] indexing above
                output_fd.write(
                    str(subs + 1) + separator +
                    separator.join(str(val) for val in data[:]) + "\n")

        if list(list_of_unexp_descr) != list(list_of_unexp_descr_first_msg):
            print('\n\n')
            print('ERROR: it seems different types of BUFR messages')
            print('are mixed in this BUFR file, meaning that the list of')
            print('descriptor names and units printed on the first 2 output')
            print('lines will not match with all lines of data.')
            print('To prevent confusion, therefore decoding is halted')
            print('It is recommended to first sort BUFR messages by type')
            print('before converting them to ascii or csv.')
            print('The example script sort_bufr_msgs.py can be used')
            print('to sort a BUFR file.')
            print('\n\n')
            print('Detailed info:')
            print('list_of_unexp_descr != list_of_unexp_descr_first_msg !')
            print('list_of_unexp_descr           = ', list_of_unexp_descr)
            print('list_of_unexp_descr_first_msg = ',
                  list_of_unexp_descr_first_msg)
            sys.exit(1)

        if numpy.shape(data)[0] == 0:
            print('NO DATA FOUND! this seems an empty BUFR message !')
            continue

        print('converted BUFR msg nr. ', msg_nr + 1)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bufr.close()
    if msg_nr == -1:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Пример #16
0
def print_bufr_content2(input_bufr_file, output_fd, separator,
                        max_msg_nr, expand_flags):
    #  #[ implementation 2
    """
    example implementation using the BUFRReader class
    combined with the get_value method
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, expand_flags=expand_flags)

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        # add header strings
        if bob.msg_loaded == 1:
            list_of_names = bob.get_names()
            list_of_units = bob.get_units()

            output_fd.write('"subset nr"'+separator)
            if list_of_names:
                for name in list_of_names[:-1]:
                    output_fd.write('"'+name+'"'+separator)
                name = list_of_names[-1]
                output_fd.write('"'+name+'"\n')
            else:
                output_fd.write('"[NO DATA]"\n')

            output_fd.write('""'+separator)
            if list_of_units:
                for unit in list_of_units[:-1]:
                    output_fd.write('"'+unit+'"'+separator)
                unit = list_of_units[-1]
                output_fd.write('"'+unit+'"\n')
            else:
                output_fd.write('"[NO DATA]"\n')

        nsubsets = bob.get_num_subsets()
        for subs in range(1, nsubsets+1):
            nelements = bob.get_num_elements()
            data_list = []
            for descr_nr in range(nelements):
                data = bob.get_value(descr_nr, subs, autoget_cval=True)
                data_list.append(data)
            output_fd.write(str(subs)+separator+
                            separator.join(str(val) for val in data_list)+
                            "\n")

        print('converted BUFR msg nr. ', msg_nr)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Пример #17
0
def print_bufr_content4(input_bufr_file, output_fd, separator,
                        max_msg_nr, expand_flags):
    #  #[ implementation 4
    """
    example implementation using the BUFRReader class
    to decode a bufr file using delayed replication.
    Since these files may have different descriptor lists
    for each subset, a different call pattern is needed.
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False,
#                     verbose=True, expand_flags=expand_flags)
                     verbose=False, expand_flags=expand_flags)

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        # since this example assumes a bufr file using delayed replication
        # always request and add the header for each subset
        nsubsets = bob.get_num_subsets()
        for subs in range(1, nsubsets+1):

            # add header strings
            (list_of_names, list_of_units) = bob.get_names_and_units(subs)

            # currently not used
            # list_of_unexp_descr = bob.bufr_obj.py_unexp_descr_list

            data = bob.get_subset_values(subs, autoget_cval=True)

            # print('len(list_of_names) = ', len(list_of_names))
            # print('len(list_of_units) = ', len(list_of_units))
            # print('len(data) = ', len(data))

            if numpy.shape(data)[0] == 0:
                print('NO DATA FOUND! this seems an empty BUFR message !')
                continue

            output_fd.write('"subset nr"'+separator+
                            separator.join(list_of_names) + "\n")
            output_fd.write('""'+separator+
                            separator.join(list_of_units) + "\n")
            output_fd.write(str(subs)+separator+
                            separator.join(str(val) for val in data[:])+
                            "\n")

        print('converted BUFR msg nr. ', msg_nr)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Пример #18
0
msg['LATI'] = [55.2, 66.3, 77.4]
msg['LONG'] = [5.1, 6.2, 7.3]

# debug
print('values: ', list(msg.values))

msg.write_msg_to_file()
bwr.close()

##############################
# reopen the BUFR file as test
##############################

print('*' * 50)

input_bufr_file = output_bufr_file
bufr = BUFRReader(input_bufr_file, warn_about_bufr_size=False)

# just 1 msg in this test file, so no looping needed
for msg in bufr:
    print('num_subsets = ', msg.get_num_subsets())
    for subs, msg_or_subset_data in enumerate(msg):
        list_of_names = msg_or_subset_data.names
        list_of_units = msg_or_subset_data.units
        data = msg_or_subset_data.data
        print('"subset nr"' + ',' + ','.join(list_of_names))
        print('""' + ',' + ','.join(list_of_units))
        print(str(subs + 1) + ',' + ','.join(str(val) for val in data[:]))

bufr.close()
Пример #19
0
def print_bufr_content4(input_bufr_file, output_fd, separator,
                        max_msg_nr, expand_flags, expand_strings,
                        descr_multiplier):
    #  #[ implementation 4
    """
    example implementation using the BUFRReader class
    to decode a bufr file using delayed replication.
    Since these files may have different descriptor lists
    for each subset, a different call pattern is needed.
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bufr = BUFRReader(input_bufr_file, warn_about_bufr_size=False,
                      #verbose=True, expand_flags=expand_flags,
                      verbose=False, expand_flags=expand_flags,
                      expand_strings=expand_strings)
    
    bufr.tune_decoding_parameters(nr_of_descriptors_multiplier=descr_multiplier)

    msg_nr = -1
    for msg_nr, msg in enumerate(bufr):
        # since this example assumes a bufr file using delayed replication
        # always request and add the header for each subset
        nsubsets = msg.get_num_subsets()
        # print('nsubsets = ', nsubsets)

        for msg_or_subs_nr, msg_or_subset_data in enumerate(msg):
            # add header strings
            list_of_names = msg_or_subset_data.names
            list_of_units = msg_or_subset_data.units
            data = msg_or_subset_data.data
            if numpy.shape(data)[0] == 0:
                print('NO DATA FOUND! this seems an empty BUFR message !')
                continue
            output_fd.write('"subset nr"'+separator+
                            separator.join(list_of_names) + "\n")
            output_fd.write('""'+separator+
                            separator.join(list_of_units) + "\n")

            # print(data.shape)
            if len(data.shape) == 1:
                # we are walking over subsets
                subs = msg_or_subs_nr
                output_fd.write(str(subs+1)+separator+
                                separator.join(to_str(val)
                                               for val in data[:])+
                                "\n")
            else:
                # we are getting a 2D array as result
                for subs in range(data.shape[0]):
                    output_fd.write(str(subs+1)+separator+
                                    separator.join(to_str(val)
                                                   for val in data[subs, :])+
                                    "\n")
                    
                
        print('converted BUFR msg nr. ', msg_nr+1)
        if msg_nr >= max_msg_nr:
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bufr.close()
    if msg_nr == -1:
        print('no BUFR messages found, are you sure this is a BUFR file?')