Beispiel #1
0
def print_bufr_content1(input_bufr_file, output_fd, separator, max_msg_nr):
    #  #[ implementation 1
    """
    example implementation using the BUFRReader class
    combined with the get_values_as_2d_array method
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        # add header strings
        # print 'DEBUG: bob.msg_loaded ',bob.msg_loaded
        if bob.msg_loaded == 1:
            list_of_names = []
            list_of_units = []
            list_of_names.extend(bob.get_names())
            list_of_units.extend(bob.get_units())
            # print 'DEBUG: ',separator.join(list_of_names)
            # print 'DEBUG: ',separator.join(list_of_units)
            output_fd.write(separator.join(list_of_names) + "\n")
            output_fd.write(separator.join(list_of_units) + "\n")

        data = bob.get_values_as_2d_array()
        # print 'DEBUG: data.shape = ', data.shape
        if data.shape[0] * data.shape[1] == 0:
            print 'NO DATA FOUND! this seems an empty BUFR message !'
            continue

        for subs in range(len(data[:, 0])):
            output_fd.write(
                str(subs) + separator +
                separator.join(str(val) for val in data[subs, :]) + "\n")
        print 'converted BUFR msg nr. ', msg_nr
        if ((max_msg_nr > 0) and (msg_nr >= max_msg_nr)):
            print 'skipping remainder of this BUFR file'
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print 'no BUFR messages found, are you sure this is a BUFR file?'
def read_bufr_file(input_bufr_file):
    #  #[ read a bufr file
    """
    read the file using the BUFRReader class and get the data
    with the get_values_as_2d_array method
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        nsubsets = bob.get_num_subsets()
        data1 = bob.get_subset_values(1)

        if numpy.shape(data1)[0] == 0:
            print('NO DATA FOUND! this seems an empty BUFR message !')
            continue

        print('loaded BUFR msg nr. ', msg_nr, 'shape = ', numpy.shape(data1))
        print('data1[:2] = ', data1[:2])

        if nsubsets > 1:
            data2 = bob.get_subset_values(2)

            if numpy.shape(data2)[0] == 0:
                print('NO DATA FOUND! this seems an empty BUFR message !')
                continue

            print('data2[:2] = ', data2[:2])

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
def sort_msgs(input_bufr_file):
    #  #[
    """
    a little example routine to demonstrate how to extract
    BUFR messages from a BUFR file, sort them, and write them
    out again to another file.
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
    files_dict = {}

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        print 'handling message nr ', msg_nr
        list_of_unexp_descr = bob.bufr_obj.py_unexp_descr_list
        output_filename = '_'.join(d for d in list_of_unexp_descr)
        if files_dict.has_key(output_filename):
            fdescr = files_dict[output_filename][0]
            files_dict[output_filename][1] += 1 # increment count
        else:
            fdescr = open(output_filename, 'wb')
            count = 1
            files_dict[output_filename] = [fdescr, count]
        fdescr.write(bob.bufr_obj.encoded_message)

    generated_files = files_dict.keys()
    for k in files_dict.keys():
        count = files_dict[k][1]
        print 'file ', k, ' contains ', count, ' messages'
        files_dict[k][0].close()

    return generated_files
def reopen_bufr_file(input_bufr_file):
    #  #[ open bufr file
    '''
    open a bufr file for reading and print its content
    '''
    print '*'*50
    from pybufr_ecmwf.bufr import BUFRReader

    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
    bob.setup_tables(table_b_to_use='B'+TABLE_NAME,
                     table_d_to_use='D'+TABLE_NAME)

    bob.get_next_msg()
    print 'num_subsets:  ', bob.get_num_subsets()

    if USE_DELAYED_REPLICATION:
        data1 = bob.get_subset_values(0)
        print 'data1 = ', data1
        data2 = bob.get_subset_values(1)
        print 'data2 = ', data2
    else:
        print 'num_elements: ', bob.get_num_elements()
        print bob.get_names()
        print bob.get_units()
        data = bob.get_values_as_2d_array()
        print data.shape
        print data
        print 'bob.bufr_obj.values = '
        print bob.bufr_obj.values

    textdata = bob.get_value(3, 0)
    print 'textdata(3,0)', textdata
    textdata = bob.get_value(3, 0, get_cval=True)
    print 'textdata(3,0)', textdata
    textdata = bob.get_values(3, get_cval=True)
    print 'textdata(3,:)', textdata

    bob.close()
Beispiel #5
0
def reopen_bufr_file(input_bufr_file):
    #  #[ open bufr file
    '''
    open a bufr file for reading and print its content
    '''
    print '*'*50
    from pybufr_ecmwf.bufr import BUFRReader

    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
    bob.setup_tables(table_b_to_use='B'+TABLE_NAME,
                     table_d_to_use='D'+TABLE_NAME)

    bob.get_next_msg()
    print 'num_subsets:  ', bob.get_num_subsets()

    if USE_DELAYED_REPLICATION:
        data1 = bob.get_subset_values(0)
        print 'data1 = ', data1
        data2 = bob.get_subset_values(1)
        print 'data2 = ', data2
    else:
        print 'num_elements: ', bob.get_num_elements()
        print bob.get_names()
        print bob.get_units()
        data = bob.get_values_as_2d_array()
        print data.shape
        print data
        print 'bob.bufr_obj.values = '
        print bob.bufr_obj.values

    textdata = bob.get_value(3, 0)
    print 'textdata(3,0)', textdata
    textdata = bob.get_value(3, 0, get_cval=True)
    print 'textdata(3,0)', textdata
    textdata = bob.get_values(3, get_cval=True)
    print 'textdata(3,:)', textdata

    bob.close()
Beispiel #6
0
def print_bufr_content5(input_bufr_file, output_fd, separator, max_msg_nr,
                        expand_flags):
    #  #[ implementation 5
    """
    example implementation using the BUFRReader class
    to decode a bufr file using delayed replication.
    Since these files may have different descriptor lists
    for each subset, a different call pattern is needed.
    """

    # testcases:
    # ./example_programs/bufr_to_ascii.py -5 -c -o tmp.csv \
    #     -i ./pybufr_ecmwf/ecmwf_bufr_lib/bufrdc_000403/data/syno_1.bufr
    #
    # ./example_programs/bufr_to_ascii.py -5 -c -o tmp.csv \
    #     -i ../BUFR_test_files/synop_knmi_via_ko_janssen/MSSAEOL_00002950.b
    #

    names_to_be_selected = ['temperature', 'wind']
    names_to_be_excluded = ['minimum', 'maximum']

    write_names_and_units_just_once = True

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file,
                     warn_about_bufr_size=False,
                     verbose=False,
                     expand_flags=expand_flags)

    msg_nr = 0
    not_yet_printed = True
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        # since this example assumes a bufr file using delayed replication
        # always request and add the header for each subset
        nsubsets = bob.get_num_subsets()
        for subs in range(1, nsubsets + 1):

            print('==> subset ', subs)

            # add header strings
            (list_of_names, list_of_units) = bob.get_names_and_units(subs)
            data = bob.get_subset_values(subs)  #,autoget_cval=True)

            selected_names = []
            selected_units = []
            selected_values = []
            for i, name in enumerate(list_of_names):
                selected = False
                for name in names_to_be_selected:
                    if name in name.lower():
                        selected = True
                for name in names_to_be_excluded:
                    if name in name.lower():
                        selected = False

                if selected:
                    # print(' '*10,name,'=',data[i],list_of_units[i])
                    selected_names.append(list_of_names[i])
                    selected_units.append(list_of_units[i])
                    selected_values.append(data[i])

            if len(selected_values) == 0:
                print('NO DATA SELECTED for BUFR message %d and subset %d!' %
                      (msg_nr, subs))
                continue

            if write_names_and_units_just_once and not_yet_printed:
                output_fd.write('"subset nr"' + separator +
                                separator.join(selected_names) + "\n")
                output_fd.write('""' + separator +
                                separator.join(selected_units) + "\n")
                not_yet_printed = False

            output_fd.write(
                str(subs) + separator +
                separator.join(str(val) for val in selected_values) + "\n")

        print('=' * 25)
        print('converted BUFR msg nr. ', msg_nr)
        print('=' * 25)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Beispiel #7
0
def print_bufr_content5(input_bufr_file, output_fd, separator,
                        max_msg_nr, expand_flags):
    #  #[ implementation 5
    """
    example implementation using the BUFRReader class
    to decode a bufr file using delayed replication.
    Since these files may have different descriptor lists
    for each subset, a different call pattern is needed.
    """

    # testcases:
    # ./example_programs/bufr_to_ascii.py -5 -c -o tmp.csv \
    #     -i ./pybufr_ecmwf/ecmwf_bufr_lib/bufrdc_000403/data/syno_1.bufr
    #
    # ./example_programs/bufr_to_ascii.py -5 -c -o tmp.csv \
    #     -i ../BUFR_test_files/synop_knmi_via_ko_janssen/MSSAEOL_00002950.b
    #

    names_to_be_selected = ['temperature', 'wind']
    names_to_be_excluded = ['minimum', 'maximum']

    write_names_and_units_just_once = True

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False,
                     verbose=False, expand_flags=expand_flags)

    msg_nr = 0
    not_yet_printed = True
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        # since this example assumes a bufr file using delayed replication
        # always request and add the header for each subset
        nsubsets = bob.get_num_subsets()
        for subs in range(1, nsubsets+1):

            print('==> subset ', subs)

            # add header strings
            (list_of_names, list_of_units) = bob.get_names_and_units(subs)
            data = bob.get_subset_values(subs) #,autoget_cval=True)

            selected_names = []
            selected_units = []
            selected_values = []
            for i, name in enumerate(list_of_names):
                selected = False
                for name in names_to_be_selected:
                    if name in name.lower():
                        selected = True
                for name in names_to_be_excluded:
                    if name in name.lower():
                        selected = False

                if selected:
                    # print(' '*10,name,'=',data[i],list_of_units[i])
                    selected_names.append(list_of_names[i])
                    selected_units.append(list_of_units[i])
                    selected_values.append(data[i])

            if len(selected_values) == 0:
                print('NO DATA SELECTED for BUFR message %d and subset %d!' %
                      (msg_nr, subs))
                continue

            if write_names_and_units_just_once and not_yet_printed:
                output_fd.write('"subset nr"'+separator+
                                separator.join(selected_names) + "\n")
                output_fd.write('""'+separator+
                                separator.join(selected_units) + "\n")
                not_yet_printed = False

            output_fd.write(str(subs)+separator+
                            separator.join(str(val) for val in selected_values)+
                            "\n")

        print('='*25)
        print('converted BUFR msg nr. ', msg_nr)
        print('='*25)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Beispiel #8
0
def print_bufr_content1(input_bufr_file, output_fd, separator,
                        max_msg_nr, expand_flags):
    #  #[ implementation 1
    """
    example implementation using the BUFRReader class
    combined with the get_values_as_2d_array method
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False,
                     expand_flags=expand_flags)

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        # add header strings
        # print('DEBUG: bob.msg_loaded ',bob.msg_loaded)
        list_of_names = []
        list_of_units = []
        list_of_names.extend(bob.get_names())
        list_of_units.extend(bob.get_units())
        list_of_unexp_descr = bob.get_unexp_descr_list()

        #print('list_of_names = ',list_of_names)
        #print('list_of_units = ',list_of_units)

        if bob.msg_loaded == 1:
            output_fd.write('"subset nr"'+separator)
            if list_of_names:
                for name in list_of_names[:-1]:
                    output_fd.write('"'+name+'"'+separator)
                name = list_of_names[-1]
                output_fd.write('"'+name+'"\n')
            else:
                output_fd.write('"[NO DATA]"\n')

            output_fd.write('""'+separator)
            if list_of_units:
                for unit in list_of_units[:-1]:
                    output_fd.write('"'+unit+'"'+separator)
                unit = list_of_units[-1]
                output_fd.write('"'+unit+'"\n')
            else:
                output_fd.write('"[NO DATA]"\n')

            list_of_unexp_descr_first_msg = bob.bufr_obj.py_unexp_descr_list
            #print('list_of_unexp_descr_first_msg = ',
            #      list_of_unexp_descr_first_msg)

        data = bob.get_values_as_2d_array()

        if list_of_unexp_descr != list_of_unexp_descr_first_msg:
            print('\n\n')
            print('WARNING: it seems different types of BUFR messages')
            print('are mixed in this BUFR file, meaning that the list of')
            print('descriptor names and units printed on the first 2 output')
            print('lines will not match with all lines of data.')
            print('To prevent confusion, therefore decoding is halted')
            print('It is recommended to first sort BUFR messages by type')
            print('before converting them to ascii or csv.')
            print('The example script sort_bufr_msgs.py can be used')
            print('to sort a BUFR file.')
            print('\n\n')
            print('Detailed info:')
            print('list_of_unexp_descr != list_of_unexp_descr_first_msg !')
            print('list_of_unexp_descr           = ',
                  list_of_unexp_descr)
            print('list_of_unexp_descr_first_msg = ',
                  list_of_unexp_descr_first_msg)
            sys.exit(0)

        if data.shape[0]*data.shape[1] == 0:
            print('NO DATA FOUND! this seems an empty BUFR message !')
            continue

        for subs in range(len(data[:, 0])):
            output_fd.write(str(subs+1)+separator+
                            separator.join(str(val) for val in data[subs, :])+
                            "\n")
        print('converted BUFR msg nr. ', msg_nr)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Beispiel #9
0
def print_bufr_content4(input_bufr_file, output_fd, separator,
                        max_msg_nr, expand_flags):
    #  #[ implementation 4
    """
    example implementation using the BUFRReader class
    to decode a bufr file using delayed replication.
    Since these files may have different descriptor lists
    for each subset, a different call pattern is needed.
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False,
#                     verbose=True, expand_flags=expand_flags)
                     verbose=False, expand_flags=expand_flags)

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        # since this example assumes a bufr file using delayed replication
        # always request and add the header for each subset
        nsubsets = bob.get_num_subsets()
        for subs in range(1, nsubsets+1):

            # add header strings
            (list_of_names, list_of_units) = bob.get_names_and_units(subs)

            # currently not used
            # list_of_unexp_descr = bob.bufr_obj.py_unexp_descr_list

            data = bob.get_subset_values(subs, autoget_cval=True)

            # print('len(list_of_names) = ', len(list_of_names))
            # print('len(list_of_units) = ', len(list_of_units))
            # print('len(data) = ', len(data))

            if numpy.shape(data)[0] == 0:
                print('NO DATA FOUND! this seems an empty BUFR message !')
                continue

            output_fd.write('"subset nr"'+separator+
                            separator.join(list_of_names) + "\n")
            output_fd.write('""'+separator+
                            separator.join(list_of_units) + "\n")
            output_fd.write(str(subs)+separator+
                            separator.join(str(val) for val in data[:])+
                            "\n")

        print('converted BUFR msg nr. ', msg_nr)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Beispiel #10
0
def print_bufr_content2(input_bufr_file, output_fd, separator,
                        max_msg_nr, expand_flags):
    #  #[ implementation 2
    """
    example implementation using the BUFRReader class
    combined with the get_value method
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, expand_flags=expand_flags)

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        # add header strings
        if bob.msg_loaded == 1:
            list_of_names = bob.get_names()
            list_of_units = bob.get_units()

            output_fd.write('"subset nr"'+separator)
            if list_of_names:
                for name in list_of_names[:-1]:
                    output_fd.write('"'+name+'"'+separator)
                name = list_of_names[-1]
                output_fd.write('"'+name+'"\n')
            else:
                output_fd.write('"[NO DATA]"\n')

            output_fd.write('""'+separator)
            if list_of_units:
                for unit in list_of_units[:-1]:
                    output_fd.write('"'+unit+'"'+separator)
                unit = list_of_units[-1]
                output_fd.write('"'+unit+'"\n')
            else:
                output_fd.write('"[NO DATA]"\n')

        nsubsets = bob.get_num_subsets()
        for subs in range(1, nsubsets+1):
            nelements = bob.get_num_elements()
            data_list = []
            for descr_nr in range(nelements):
                data = bob.get_value(descr_nr, subs, autoget_cval=True)
                data_list.append(data)
            output_fd.write(str(subs)+separator+
                            separator.join(str(val) for val in data_list)+
                            "\n")

        print('converted BUFR msg nr. ', msg_nr)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Beispiel #11
0
        if ((max_msg_nr > 0) and (msg_nr >= max_msg_nr)):
            print 'skipping remainder of this BUFR file'
            break

    # close the file
    bob.close()
    if msg_nr == 0:
        print 'no BUFR messages found, are you sure this is a BUFR file?'
    #  #]


if __name__ == '__main__':

    bufrdir = '/net/bhw379/nobackup/users/plas/temp/BUFRdata/'
    bufrfile = os.path.join(bufrdir, 'BUFR.radarv.bewid')
    os.environ['BUFR_TABLES'] = os.path.join(bufrdir, 'BUFRtables')

    bob = BUFRReader(bufrfile, warn_about_bufr_size=False)
    bob.get_next_msg()
    names = bob.get_names()
    units = bob.get_units()

    print set(names)
    print 20 * '='
    print set(units)
    for n, u in zip(names, units):
        if "dB" in u: print n, u

    outf = open('radarBUFR.dat', 'w')
    print_bufr_content1(bufrfile, outf, ',', 5)
Beispiel #12
0
def sort_msgs(input_bufr_file):
    #  #[
    """
    a little example routine to demonstrate how to extract
    BUFR messages from a BUFR file, sort them, and write them
    out again to another file.
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
    num_msgs = bob.rbf.get_num_bufr_msgs()
    progress_step = max(1,int(num_msgs/20))
    
    files_dict = {}
    msg_nr = 0
    while True:
        can_be_decoded = False
        try:
            bob.get_next_msg()
            can_be_decoded = True
        except EOFError:
            break
        except KeyError:
            # allow sorting of BUFR messages that cannot be decoded
            # because the needed key is not in the available set of
            # BUFR table files. This tool only uses unexpanded descriptors
            # so ability to decode should not be required.
            # A user still may wish to first sort, and then decode
            # a subset of messages that can be decoded.
            pass
        
        msg_nr += 1
        if progress_step*int(msg_nr/progress_step) == msg_nr:
            print('handling message nr {} out of {}'.format(msg_nr, num_msgs))
            
        list_of_unexp_descr = bob.bufr_obj.py_unexp_descr_list
        output_filename = construct_unique_filename(list_of_unexp_descr)
        if files_dict.has_key(output_filename):
            fdescr = files_dict[output_filename][0]
            files_dict[output_filename][1] += 1 # increment count
        else:
            fdescr = open(output_filename, 'wb')
            count = 1
            files_dict[output_filename] = [fdescr, count, can_be_decoded,
                                           list_of_unexp_descr]
        fdescr.write(bob.bufr_obj.encoded_message)

    generated_files = files_dict.keys()
    num_that_can_be_decoded = 0
    num_that_cannot_be_decoded = 0
    for k in files_dict:
        fdescr, count, can_be_decoded, list_of_unexp_descr = files_dict[k]
        print('file {} contains {} messages'.format(k[:25], count))
        files_dict[k][0].close()
        if can_be_decoded:
            num_that_can_be_decoded += 1
        else:
            num_that_cannot_be_decoded += 1
            # check to see if local descriptors are present
            for d in list_of_unexp_descr:
                if int(d[3:]) >= 192:
                    print('==>A local descriptor definition is present: ', d)
            print('==>this template cannot be decoded with '+
                  'standard WMO BUFR tables.')
            
    print('Sorting results:')
    print('BUFR messages with {} different templates are present in this file'.
          format(num_that_can_be_decoded+num_that_cannot_be_decoded))
    if num_that_cannot_be_decoded > 0:
        print('decoding is not possible for {} templates.'.
              format(num_that_cannot_be_decoded))
        
    return generated_files
def select_subsets(input_bufr_file, output_bufr_file):
    #  #[ select on subsets
    """
    select data and write out again
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)

    # open the file for writing
    rbf_out = RawBUFRFile()
    rbf_out.open(output_bufr_file, 'wb')

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        data = bob.get_values_as_2d_array()
        print 'data.shape = ', data.shape

        if data.shape[0]*data.shape[1] == 0:
            print 'NO DATA FOUND! this seems an empty BUFR message !'
            continue

        # select every other subset
        new_data = data[::2, :]

        print 'new_data.shape = ', new_data.shape
        #bob.bufr_obj

        nsub = bob.bufr_obj.get_num_subsets()
        n_exp_descr = len(bob.bufr_obj.values)/nsub
        bob.bufr_obj.fill_descriptor_list(nr_of_expanded_descriptors=
                                          n_exp_descr)
        bob.bufr_obj.ktdlst = bob.bufr_obj.get_descriptor_list()

        delayed_repl_data = bob.bufr_obj.derive_delayed_repl_factors()
        bob.bufr_obj.fill_delayed_repl_data(delayed_repl_data)

        new_nsub = new_data.shape[0]
        bob.bufr_obj.nr_subsets = new_nsub
        btm = BufrTemplate()
        btm.add_descriptors(*bob.bufr_obj.ktdlst)#[:self.ktdlen])
        btm.nr_of_delayed_repl_factors = 1
        btm.del_repl_max_nr_of_repeats_list = list(delayed_repl_data)
        bob.bufr_obj.register_and_expand_descriptors(btm)

        # activate this one if the encoding crashes without clear cause:
        # bob.bufr_obj.estimated_num_bytes_for_encoding = 25000

        bob.bufr_obj.kdate = new_nsub*list(delayed_repl_data)

        print 'bob.bufr_obj.cvals.shape = ', bob.bufr_obj.cvals.shape
        bob.bufr_obj.encode_data(new_data, bob.bufr_obj.cvals[:32, :])
        rbf_out.write_raw_bufr_msg(bob.bufr_obj.encoded_message)

        #for subs in range(len(data[:, 0])):
        #    output_fd.write(str(subs)+separator+
        #                    separator.join(str(val) for val in data[subs, :])+
        #                    "\n")
        print 'converted BUFR msg nr. ', msg_nr


    # close the file
    bob.close()
    if msg_nr == 0:
        print 'no BUFR messages found, are you sure this is a BUFR file?'

    rbf_out.close()
Beispiel #14
0
def select_subsets(input_bufr_file, output_bufr_file):
    #  #[ select on subsets
    """
    select data and write out again
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)

    # open the file for writing
    rbf_out = RawBUFRFile()
    rbf_out.open(output_bufr_file, 'wb')

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        data = bob.get_values_as_2d_array()
        print('data.shape = ', data.shape)

        if data.shape[0] * data.shape[1] == 0:
            print('NO DATA FOUND! this seems an empty BUFR message !')
            continue

        # select every other subset
        new_data = data[::2, :]

        print('new_data.shape = ', new_data.shape)
        #bob.bufr_obj

        nsub = bob.bufr_obj.get_num_subsets()
        n_exp_descr = len(bob.bufr_obj.values) / nsub
        bob.bufr_obj.fill_descriptor_list(
            nr_of_expanded_descriptors=n_exp_descr)
        bob.bufr_obj.ktdlst = bob.bufr_obj.get_descriptor_list()

        delayed_repl_data = bob.bufr_obj.derive_delayed_repl_factors()
        bob.bufr_obj.fill_delayed_repl_data(delayed_repl_data)

        new_nsub = new_data.shape[0]
        bob.bufr_obj.nr_subsets = new_nsub
        btm = BufrTemplate()
        btm.add_descriptors(*bob.bufr_obj.ktdlst)  #[:self.ktdlen])
        btm.nr_of_delayed_repl_factors = 1
        btm.del_repl_max_nr_of_repeats_list = list(delayed_repl_data)
        bob.bufr_obj.register_and_expand_descriptors(btm)

        # activate this one if the encoding crashes without clear cause:
        # bob.bufr_obj.estimated_num_bytes_for_encoding = 25000

        bob.bufr_obj.kdate = new_nsub * list(delayed_repl_data)

        print('bob.bufr_obj.cvals.shape = ', bob.bufr_obj.cvals.shape)
        bob.bufr_obj.encode_data(new_data, bob.bufr_obj.cvals[:32, :])
        rbf_out.write_raw_bufr_msg(bob.bufr_obj.encoded_message)

        #for subs in range(len(data[:, 0])):
        #    output_fd.write(str(subs)+separator+
        #                    separator.join(str(val) for val in data[subs, :])+
        #                    "\n")
        print('converted BUFR msg nr. ', msg_nr)

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')

    rbf_out.close()
Beispiel #15
0
def sort_msgs(input_bufr_file):
    #  #[
    """
    a little example routine to demonstrate how to extract
    BUFR messages from a BUFR file, sort them, and write them
    out again to another file.
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
    num_msgs = bob.rbf.get_num_bufr_msgs()
    progress_step = max(1, int(num_msgs / 20))

    files_dict = {}
    msg_nr = 0
    while True:
        can_be_decoded = False
        try:
            bob.get_next_msg()
            can_be_decoded = True
        except EOFError:
            break
        except KeyError:
            # allow sorting of BUFR messages that cannot be decoded
            # because the needed key is not in the available set of
            # BUFR table files. This tool only uses unexpanded descriptors
            # so ability to decode should not be required.
            # A user still may wish to first sort, and then decode
            # a subset of messages that can be decoded.
            pass

        msg_nr += 1
        if progress_step * int(msg_nr / progress_step) == msg_nr:
            print('handling message nr {} out of {}'.format(msg_nr, num_msgs))

        list_of_unexp_descr = bob.bufr_obj.py_unexp_descr_list
        output_filename = construct_unique_filename(list_of_unexp_descr)
        if files_dict.has_key(output_filename):
            fdescr = files_dict[output_filename][0]
            files_dict[output_filename][1] += 1  # increment count
        else:
            fdescr = open(output_filename, 'wb')
            count = 1
            files_dict[output_filename] = [
                fdescr, count, can_be_decoded, list_of_unexp_descr
            ]
        fdescr.write(bob.bufr_obj.encoded_message)

    generated_files = files_dict.keys()
    num_that_can_be_decoded = 0
    num_that_cannot_be_decoded = 0
    for k in files_dict:
        fdescr, count, can_be_decoded, list_of_unexp_descr = files_dict[k]
        print('file {} contains {} messages'.format(k[:25], count))
        files_dict[k][0].close()
        if can_be_decoded:
            num_that_can_be_decoded += 1
        else:
            num_that_cannot_be_decoded += 1
            # check to see if local descriptors are present
            for descr in list_of_unexp_descr:
                if int(descr[3:]) >= 192:
                    print('==>A local descriptor definition is present: ',
                          descr)
            print('==>this template cannot be decoded with ' +
                  'standard WMO BUFR tables.')

    print('Sorting results:')
    print('BUFR messages with {} different templates are present in this file'.
          format(num_that_can_be_decoded + num_that_cannot_be_decoded))
    if num_that_cannot_be_decoded > 0:
        print('decoding is not possible for {} templates.'.format(
            num_that_cannot_be_decoded))

    return generated_files