def test_close(self):
        #  #[
        """
        test opening and closing a BUFR file
        """
        bufrfile = RawBUFRFile(verbose=False)
        bufrfile.open(self.corruptedtestinputfile, 'rb')
        bufrfile.close()

        # check that a second close fails
        self.assertRaises(AttributeError, bufrfile.close)
Beispiel #2
0
def print_bufr_data_category(input_bufr_file):
    #  #[
    """
    an example routine to demonstrate how to retrieve
    some meta datafrom the BUFR messages in a BUFR file
    """
    # get an instance of the RawBUFRFile class
    rbf = RawBUFRFile()

    # open the file for reading, count nr of BUFR messages in it
    # and store its content in memory, together with
    # an array of pointers to the start and end of each BUFR message
    rbf.open(input_bufr_file, 'rb')

    # extract the number of BUFR messages from the file
    num_msgs = rbf.get_num_bufr_msgs()

    for msg_nr in range(1, num_msgs + 1):
        raw_msg = rbf.get_raw_bufr_msg(msg_nr)[0]
        bufr_obj = BUFRInterfaceECMWF(encoded_message=raw_msg)
        bufr_obj.decode_sections_012()
        data_category = bufr_obj.ksec1[11 - 1]
        print('BUFR msg %i has data category %i' % (msg_nr, data_category))

    # close the file
    rbf.close()
def print_bufr_edition_number(input_bufr_file):
    #  #[
    """
    an example routine to demonstrate how to retrieve
    some meta data from the BUFR messages in a BUFR file
    """
    # get an instance of the RawBUFRFile class
    rbf = RawBUFRFile()

    # open the file for reading, count nr of BUFR messages in it
    # and store its content in memory, together with
    # an array of pointers to the start and end of each BUFR message
    rbf.open(input_bufr_file, 'rb')

    # extract the number of BUFR messages from the file
    num_msgs = rbf.get_num_bufr_msgs()

    for msg_nr in range(1, num_msgs+1):
        raw_msg = rbf.get_raw_bufr_msg(msg_nr)[0]
        bufr_obj = BUFRInterfaceECMWF(encoded_message=raw_msg)
        bufr_obj.decode_sections_012()
        bufr_edition = bufr_obj.ksec0[3-1]
        print('BUFR msg %i has version %i' % (msg_nr, bufr_edition))

    # close the file
    rbf.close()
Beispiel #4
0
def count_msgs(input_bufr_file):
    #  #[
    """
    a little example routine to demonstrate how to extract
    the number of messages from a BUFR file
    """

    # get an instance of the RawBUFRFile class
    rbf = RawBUFRFile()
    #rbf = RawBUFRFile(verbose=True)

    # open the file for reading, count nr of BUFR messages in it
    # and store its content in memory, together with
    # an array of pointers to the start and end of each BUFR message
    rbf.open(input_bufr_file, 'rb')

    # extract the number of BUFR messages from the file
    num_msgs = rbf.get_num_bufr_msgs()

    # print('rbf.nr_of_bufr_messages = ',rbf.nr_of_bufr_messages)

    # close the file
    rbf.close()

    # delete the class instance
    # (just as test, not really needed here since this scope is about
    #  to be deleted anyway)
    del rbf

    return num_msgs
def count_msgs(input_bufr_file):
    #  #[
    """
    a little example routine to demonstrate how to extract
    the number of messages from a BUFR file
    """

    # get an instance of the RawBUFRFile class
    rbf = RawBUFRFile()
    #rbf = RawBUFRFile(verbose=True)

    # open the file for reading, count nr of BUFR messages in it
    # and store its content in memory, together with
    # an array of pointers to the start and end of each BUFR message
    rbf.open(input_bufr_file, 'rb')

    # extract the number of BUFR messages from the file
    num_msgs = rbf.get_num_bufr_msgs()

    # print 'rbf.nr_of_bufr_messages = ',rbf.nr_of_bufr_messages

    # close the file
    rbf.close()

    # delete the class instance
    # (just as test, not really needed here since this scope is about
    #  to be deleted anyway)
    del rbf

    return num_msgs
def decoding_example(input_bufr_file, custom_bufr_tables=None):
    #  #[
    """
    wrap the example in a function to circumvent the pylint
    convention of requiring capitals for constants in the global
    scope (since most of these variables are not constants at all))
    """

    # read the binary data using the BUFRFile class
    print('loading testfile: ', input_bufr_file)
    rbf = RawBUFRFile(verbose=False)
    rbf.open(input_bufr_file, 'rb')
    (words, section_sizes, section_start_locations) = \
            rbf.get_next_raw_bufr_msg()
    rbf.close()

    if words is None:
        print('No valid BUFR messages found')
        sys.exit(0)

    print('------------------------------')
    bufr = BUFRInterfaceECMWF(encoded_message=words,
                              section_sizes=section_sizes,
                              section_start_locations=section_start_locations,
                              verbose=True)

    print("calling: decode_sections_012():")
    bufr.decode_sections_012()

    print("Metadata for decoded BUFR message:")
    bufr.print_sections_012_metadata()

    print("calling: setup_tables()")
    if custom_bufr_tables:
        bufr.setup_tables(table_b_to_use=custom_bufr_tables[0],
                          table_c_to_use=custom_bufr_tables[1],
                          table_d_to_use=custom_bufr_tables[2])
    else:
        bufr.setup_tables()

    print("calling: print_sections_012():")
    bufr.print_sections_012()

    # seems not to work correctly now ...
    #bufr.fill_descriptor_list()
    #bufr.print_descriptors()

    print('------------------------------')
    print("calling: bufr.decode_data():")
    bufr.decode_data()

    bufr.decode_sections_0123()
    bufr.fill_descriptor_list_subset(subset=1)

    return bufr
def raw_file_appending_example(output_bufr_file, msg3):
    #  #[
    """
    example for appending a BUFR message
    """

    # get an instance of the RawBUFRFile class
    bf2 = RawBUFRFile()

    # open the test file for appending
    bf2.open(output_bufr_file, 'ab')

    # write a third raw (encoded) BUFR messages
    if msg3 is not None:
        bf2.write_raw_bufr_msg(msg3)

    # print the internal data of the class instance
    bf2.print_properties(prefix="RawBUFRFile2 (opened for appending)")

    # close the file
    bf2.close()

    # delete the class instance
    del bf2
def decoding_example(input_bufr_file, custom_bufr_tables=None):
    #  #[
    """
    wrap the example in a function to circumvent the pylint
    convention of requiring capitals for constants in the global
    scope (since most of these variables are not constants at all))
    """

    # read the binary data using the BUFRFile class
    print('loading testfile: ', input_bufr_file)
    rbf = RawBUFRFile(verbose=False)
    rbf.open(input_bufr_file, 'rb')
    (words, section_sizes, section_start_locations) = \
            rbf.get_next_raw_bufr_msg()
    rbf.close()

    if words is None:
        print('No valid BUFR messages found')
        sys.exit(0)

    print('------------------------------')
    bufr = BUFRInterfaceECMWF(encoded_message=words,
                              section_sizes=section_sizes,
                              section_start_locations=section_start_locations,
                              verbose=True)

    print("calling: decode_sections_012():")
    bufr.decode_sections_012()

    print("Metadata for decoded BUFR message:")
    bufr.print_sections_012_metadata()

    print("calling: setup_tables()")
    if custom_bufr_tables:
        bufr.setup_tables(table_b_to_use=custom_bufr_tables[0],
                          table_c_to_use=custom_bufr_tables[1],
                          table_d_to_use=custom_bufr_tables[2])
    else:
        bufr.setup_tables()

    print("calling: print_sections_012():")
    bufr.print_sections_012()

    # seems not to work correctly now ...
    #bufr.fill_descriptor_list()
    #bufr.print_descriptors()

    print('------------------------------')
    print("calling: bufr.decode_data():")
    bufr.decode_data()

    bufr.decode_sections_0123()
    bufr.fill_descriptor_list_subset(subset=1)

    return bufr
def raw_file_appending_example(output_bufr_file, msg3):
    #  #[
    """
    example for appending a BUFR message
    """

    # get an instance of the RawBUFRFile class
    bf2 = RawBUFRFile()

    # open the test file for appending
    bf2.open(output_bufr_file, 'ab')

    # write a third raw (encoded) BUFR messages
    if msg3 is not None:
        bf2.write_raw_bufr_msg(msg3)

    # print the internal data of the class instance
    bf2.print_properties(prefix="RawBUFRFile2 (opened for appending)")

    # close the file
    bf2.close()

    # delete the class instance
    del bf2
Beispiel #10
0
def encoding_example(output_bufr_file):
    #  #[
    """
    wrap the example in a function to circumvent the pylint
    convention of requiring capitals for constants in the global
    scope (since most of these variables are not constants at all))
    """
    # define the needed constants
    max_nr_descriptors = 20  # 300
    max_nr_expanded_descriptors = 140  # 160000
    max_nr_subsets = 361  # 25

    ktdlen = max_nr_descriptors
    # krdlen = max_nr_delayed_replication_factors
    kelem = max_nr_expanded_descriptors
    kvals = max_nr_expanded_descriptors * max_nr_subsets
    # jbufl  = max_bufr_msg_size
    # jsup   = length_ksup

    #  initialise all arrays
    print('------------------------------')
    print('reinitialising all arrays...')
    print('------------------------------')
    #ksup   = np.zeros(          9, dtype = np.int)
    ksec0 = np.zeros(3, dtype=np.int)
    ksec1 = np.zeros(40, dtype=np.int)
    ksec2 = np.zeros(4096, dtype=np.int)
    #key    = np.zeros(         52, dtype = np.int)
    ksec3 = np.zeros(4, dtype=np.int)
    ksec4 = np.zeros(2, dtype=np.int)
    cnames = np.zeros((kelem, 64), dtype=np.character)
    cunits = np.zeros((kelem, 24), dtype=np.character)
    values = np.zeros(kvals, dtype=np.float64)  # this is the default
    cvals = np.zeros((kvals, 80), dtype=np.character)
    ktdlen = 0
    ktdlst = np.zeros(max_nr_descriptors, dtype=np.int)
    ktdexl = 0
    ktdexp = np.zeros(max_nr_expanded_descriptors, dtype=np.int)
    kerr = 0

    # handle BUFR tables
    print('------------------------------')

    # define our own location for storing (symlinks to) the BUFR tables
    private_bufr_tables_dir = os.path.abspath("./tmp_BUFR_TABLES")
    if not os.path.exists(private_bufr_tables_dir):
        os.mkdir(private_bufr_tables_dir)

    # make the needed symlinks to bufr tables

    # inspect the location of the ecmwfbufr.so file, and derive
    # from this the location of the BUFR tables that are delivered
    # with the ECMWF BUFR library software
    ecmwfbufr_path = os.path.split(ecmwfbufr.__file__)[0]
    path1 = os.path.join(ecmwfbufr_path, "ecmwf_bufrtables")
    path2 = os.path.join(ecmwfbufr_path, '..', "ecmwf_bufrtables")

    if os.path.exists(path1):
        ecmwf_bufr_tables_dir = path1
    elif os.path.exists(path2):
        ecmwf_bufr_tables_dir = path2
    else:
        print("Error: could not find BUFR tables directory")
        raise IOError

    # make sure the path is absolute, otherwise the ECMWF library
    # might fail when it attempts to use it ...
    ecmwf_bufr_tables_dir = os.path.abspath(ecmwf_bufr_tables_dir)

    needed_b_table = "B0000000000098015001.TXT"
    needed_d_table = "D0000000000098015001.TXT"
    available_b_table = "B0000000000098013001.TXT"
    available_d_table = "D0000000000098013001.TXT"

    source = os.path.join(ecmwf_bufr_tables_dir, available_b_table)
    destination = os.path.join(private_bufr_tables_dir, needed_b_table)
    ensure_symlink_exists(source, destination)

    source = os.path.join(ecmwf_bufr_tables_dir, available_d_table)
    destination = os.path.join(private_bufr_tables_dir, needed_d_table)
    ensure_symlink_exists(source, destination)

    # make sure the BUFR tables can be found
    # also, force a slash at the end, otherwise the library fails
    # to find the tables
    envir = os.environ
    envir["BUFR_TABLES"] = private_bufr_tables_dir + os.path.sep

    # redirect all fortran stdout to fileunit 12, so the text
    # will end-up in a file called 'fort.12'
    # This is needed to get reproducible output for my unittests.
    # Without this trick the stdout stream from fortran and the stdout
    # stream from c/python may be mixed in inpredictable ways
    # (depending on the size of the buffer used to store the streams)
    os.environ['STD_OUT'] = '12'

    # suppres the default ECMWF welcome message which
    # is not yet redirected to the above defined fileunit
    os.environ['PRINT_TABLE_NAMES'] = 'FALSE'

    # fill sections 0,1,2 and 3
    bufr_edition = 4
    bufr_code_centre = 98  # ECMWF
    bufr_obstype = 3  # sounding
    bufr_subtype_l1b = 251  # L1B
    bufr_table_local_version = 1
    bufr_table_master = 0
    bufr_table_master_version = 15
    bufr_code_subcentre = 0  # L2B processing facility
    bufr_compression_flag = 0  # 64=compression/0=no compression

    (year, month, day, hour, minute) = time.localtime()[:5]
    #(year, month, day, hour, minute, second,
    # weekday, julianday, isdaylightsavingstime) = time.localtime()

    num_subsets = 4

    # fill section 0
    ksec0[1 - 1] = 0
    ksec0[2 - 1] = 0
    ksec0[3 - 1] = bufr_edition

    # fill section 1
    ksec1[1 - 1] = 22  # length sec1 bytes
    #                                        [filled by the encoder]
    # however,a minimum of 22 is obliged here
    ksec1[2 - 1] = bufr_edition  # bufr edition
    ksec1[3 - 1] = bufr_code_centre  # originating centre
    ksec1[4 - 1] = 1  # update sequence
    ksec1[5 - 1] = 0  # (PRESENCE SECT 2)
    #                                        (0/128 = no/yes)
    ksec1[6 - 1] = bufr_obstype  # message type
    ksec1[7 - 1] = bufr_subtype_l1b  # subtype
    ksec1[8 - 1] = bufr_table_local_version  # version of local table
    ksec1[9 - 1] = (year - 2000)  # Without offset year - 2000
    ksec1[10 - 1] = month  # month
    ksec1[11 - 1] = day  # day
    ksec1[12 - 1] = hour  # hour
    ksec1[13 - 1] = minute  # minute
    ksec1[14 - 1] = bufr_table_master  # master table
    ksec1[15 - 1] = bufr_table_master_version  # version of master table
    ksec1[16 - 1] = bufr_code_subcentre  # originating subcentre
    ksec1[17 - 1] = 0
    ksec1[18 - 1] = 0

    # a test for ksec2 is not yet defined

    # fill section 3
    ksec3[1 - 1] = 0
    ksec3[2 - 1] = 0
    ksec3[3 - 1] = num_subsets  # no of data subsets
    ksec3[4 - 1] = bufr_compression_flag  # compression flag

    # define a descriptor list
    ktdlen = 9  # length of unexpanded descriptor list
    ktdlst = np.zeros(ktdlen, dtype=np.int)

    # define descriptor 1
    dd_d_date_yyyymmdd = 301011  # date
    # this defines the sequence:
    # 004001 ! year
    # 004002 ! month
    # 004003 ! day

    # define descriptor 2
    dd_d_time_hhmm = 301012  # time
    # this defines the sequence:
    # 004004 ! hour
    # 004005 ! minute

    # define descriptor 3
    dd_pressure = int('007004', 10)  # pressure [pa]

    # WARNING: filling the descriptor variable with 007004 will fail
    # because python will interpret this as an octal value, and thus
    # automatically convert 007004 to the decimal value 3588

    # define descriptor 4
    dd_temperature = int('012001', 10)  # [dry-bulb] temperature [K]

    # define descriptor 5
    dd_latitude_high_accuracy = int('005001', 10)
    # latitude (high accuracy) [degree]

    # define descriptor 6
    dd_longitude_high_accuracy = int('006001', 10)
    # longitude (high accuracy) [degree]

    # define the delayed replication code
    delayed_descr_repl_factor = int('031001', 10)

    def get_replication_code(num_descriptors, num_repeats):
        """
        calculate the replication note needed to describe
        how many times a descriptor sequence is to be repeated
        """
        repl_factor = 100000 + num_descriptors * 1000 + num_repeats
        # for example replicating 2 descriptors 25 times will
        # be encoded as: 102025
        # for delayed replication, set num_repeats to 0
        # then add the delayed_descr_repl_factor after this code
        return repl_factor

    ktdlst[0] = dd_d_date_yyyymmdd
    ktdlst[1] = dd_d_time_hhmm

    # delay replication for the next 2 descriptors
    ktdlst[2] = get_replication_code(2, 0)
    ktdlst[3] = delayed_descr_repl_factor  # = 031001

    ktdlst[4] = dd_pressure
    ktdlst[5] = dd_temperature

    # replicate the next 2 descriptors 3 times
    ktdlst[6] = get_replication_code(2, 3)

    ktdlst[7] = dd_latitude_high_accuracy
    ktdlst[8] = dd_longitude_high_accuracy

    # call BUXDES
    # buxdes: expand the descriptor list
    #         and fill the array ktdexp and the variable ktdexp
    #         [only usefull when creating a bufr msg with table D entries

    # iprint = 0 # default is to be silent
    iprint = 1
    if iprint == 1:
        print("------------------------")
        print(" printing BUFR template ")
        print("------------------------")

    # define and fill the list of replication factors
    num_del_repl_factors = 1
    kdata = np.zeros(num_subsets * num_del_repl_factors, dtype=np.int)
    for i in range(num_subsets):
        # Warning: just set the whole array to the maximum you wish to have.
        # Letting this number vary seems not to work with the current
        # ECMWF library. It will allways just look at the first element
        # in the kdata array. (or do I misunderstand the BUFR format here?)
        kdata[i] = 2  # i+1
    print("delayed replication factors: ", kdata)

    ecmwfbufr.buxdes(iprint, ksec1, ktdlst, kdata, ktdexl, ktdexp, cnames,
                     cunits, kerr)
    print("ktdlst = ", ktdlst)
    selection = np.where(ktdexp > 0)
    #print("selection = ",selection)
    print("ktdexp = ", ktdexp[selection])
    print("ktdexl = ", ktdexl)  # this one seems not to be filled ...?
    if kerr != 0:
        print("kerr = ", kerr)
        sys.exit(1)

    # print("cnames = ", cnames)
    # print("cunits = ", cunits)

    # retrieve the length of the expanded descriptor list
    exp_descr_list_length = len(np.where(ktdexp > 0)[0])
    print("exp_descr_list_length = ", exp_descr_list_length)

    # fill the values array with some dummy varying data
    num_values = exp_descr_list_length * num_subsets
    values = np.zeros(num_values, dtype=np.float64)  # this is the default

    for subset in range(num_subsets):
        # note that python starts counting with 0, unlike fortran,
        # so there is no need to take (subset-1)
        i = subset * exp_descr_list_length
        values[i] = 1999  # year

        i = i + 1
        values[i] = 12  # month

        i = i + 1
        values[i] = 31  # day

        i = i + 1
        values[i] = 23  # hour

        i = i + 1
        values[i] = 59 - subset  # minute

        i = i + 1
        values[i] = 2  # delayed replication factor
        # this delayed replication factor determines the actual number
        # of values to be stored for this particular subset
        # even if it is less then the number given in kdata above !
        for repl in range(2):
            i = i + 1
            values[i] = 1013.e2 - 100.e2 * subset + i + repl  # pressure [pa]

            i = i + 1
            values[i] = 273.15 - 10. * subset + i + repl  # temperature [K]

        for repl in range(3):
            i = i + 1
            values[i] = 51.82 + 0.05 * subset + i + repl  # latitude

            i = i + 1
            values[i] = 5.25 + 0.1 * subset + i + repl  # longitude

    # call BUFREN
    #   bufren: encode a bufr message
    #sizewords = 200
    #kbuff = np.zeros(num_values, dtype = np.int)
    cvals = np.zeros((num_values, 80), dtype=np.character)
    # define the output buffer
    num_bytes = 5000
    num_words = num_bytes // 4
    words = np.zeros(num_words, dtype=np.int)

    ecmwfbufr.bufren(ksec0, ksec1, ksec2, ksec3, ksec4, ktdlst, kdata,
                     exp_descr_list_length, values, cvals, words, kerr)

    print("bufren call finished")
    if kerr != 0:
        print("kerr = ", kerr)
        sys.exit(1)

    # convert the arrays to a list before printing, since
    # the standard print for numpy arrays is no longer reproducible
    # (old versions use spaces, new versions use commas as separator)
    print("words[:5] =", words[:5].tolist())
    # dont print word 7, since it contains the current datetime
    # of the encoding, and makes the test irreproducible
    print("words[10:25] =", words[10:25].tolist())

    nonzero_locations = np.where(words != 0)
    #print('nonzero_locations = ',nonzero_locations[0])

    numwords = nonzero_locations[0][-1] + 1
    print("encoded size: ", numwords, " words or ", numwords * 4, " bytes")

    encoded_message = words[:numwords]

    # get an instance of the RawBUFRFile class
    bf1 = RawBUFRFile()
    # open the file for writing
    bf1.open(output_bufr_file, 'wb')
    # write the encoded BUFR message
    bf1.write_raw_bufr_msg(encoded_message)
    # close the file
    bf1.close()
Beispiel #11
0
        for l in range(num_replications2[subset][j]):

            # fill dummy var
            values[i] = 45678.
            i = i + 1

# debug
print('values: ', list(values))  #numpy.where(values != 0)]))

# do the encoding to binary format
bufr.kdata = numpy.array(repl_counts)
print('bufr.kdata = ', bufr.kdata)
bufr.encode_data(values, cvals)

# get an instance of the RawBUFRFile class
bf1 = RawBUFRFile()

output_bufr_file = 'dummy_bufr_file.bfr'
# open the file for writing
bf1.open(output_bufr_file, 'wb')
# write the encoded BUFR message
bf1.write_raw_bufr_msg(bufr.encoded_message)
# close the file
bf1.close()

##############################
# reopen the BUFR file as test
##############################

print('*' * 50)
def create_bufr_file(output_bufr_file, template):
    #  #[ create bufr file
    '''
    and use this BUFR template to create a test BUFR message
    '''
    from pybufr_ecmwf.bufr_interface_ecmwf import BUFRInterfaceECMWF
    bufr = BUFRInterfaceECMWF(verbose=True)

    # fill sections 0, 1, 2 and 3
    num_subsets = 2
    bufr.fill_sections_0123(bufr_code_centre=98, # ECMWF
                            bufr_obstype=3, # sounding
                            bufr_subtype=253, # L2B
                            bufr_table_local_version=1,
                            bufr_table_master=0,
                            bufr_table_master_version=15,
                            bufr_code_subcentre=0, # L2B processing facility
                            num_subsets=num_subsets,
                            bufr_compression_flag=0)
                            # 64=compression/0=no compression

    # determine information from sections 0123 to construct the BUFR table
    # names expected by the ECMWF BUFR library and create symlinks to the
    # default tables if needed
    bufr.setup_tables(table_b_to_use='B'+TABLE_NAME,
                      table_d_to_use='D'+TABLE_NAME)
    bufr.register_and_expand_descriptors(template)

    # activate this one if the encoding crashes without clear cause:
    # bufr.estimated_num_bytes_for_encoding = 25000

    # retrieve the length of the expanded descriptor list
    exp_descr_list_length = bufr.ktdexl
    print "exp_descr_list_length = ", exp_descr_list_length

    # fill the values array with some dummy varying data
    num_values = num_subsets*bufr.max_nr_expanded_descriptors
    values = numpy.zeros(num_values, dtype=numpy.float64) # this is the default

    # note: these two must be identical for now, otherwise the
    # python to fortran interface breaks down. This also ofcourse is the
    # cause of the huge memory use of cvals in case num_values is large.
    num_cvalues = num_values
    cvals = numpy.zeros((num_cvalues, 80), dtype=numpy.character)
    cvals_index = 0

    repl_counts = []

    for subset in range(num_subsets):
        # note that python starts counting with 0, unlike fortran,
        # so there is no need to take (subset-1)

        print 'subset,exp_descr_list_length = ', subset, exp_descr_list_length
        i = subset*exp_descr_list_length

        # fill the message with some dummy data

        # fill year, month, day
        for val in [2014, 3, 19]: # fill the header
            values[i] = val+subset
            i += 1

        # fill prod_name
        txt = 'filename{}.txt'.format(subset)
        cvals[cvals_index, :] = ' '
        for icval, cval in enumerate(txt):
            cvals[cvals_index, icval] = cval

        # values[i] = cvals_index * 1000 + 64 # len(txt)
        values[i] = (cvals_index+1) * 1000 + len(txt)
        i += 1
        cvals_index = cvals_index + 1

        for val in [5.1+0.1*subset, 55.2-0.01*subset, 23., 45., 73., 82.]:
            bufr.verify_in_range(i, val)
            values[i] = val
            i += 1

        if USE_DELAYED_REPLICATION:
            # set actual delayed replication repeats
            num_repl = 3 + 2*subset
            print 'num_repl = ', num_repl
            values[i] = num_repl
            i += 1
            repl_counts.append(num_repl)

            # fill the replicated variable
            for irepl in range(num_repl):
                val = 12.+subset*0.1 + irepl*0.01
                bufr.verify_in_range(i, val)
                values[i] = val
                i += 1

    # do the encoding to binary format
    bufr.kdata = numpy.array(repl_counts)
    print 'bufr.kdata = ', bufr.kdata
    bufr.encode_data(values, cvals)

    print 'DEBUG: values = ', values

    from pybufr_ecmwf.raw_bufr_file import RawBUFRFile
    # get an instance of the RawBUFRFile class
    bf1 = RawBUFRFile()

    # open the file for writing
    bf1.open(output_bufr_file, 'wb')
    # write the encoded BUFR message
    bf1.write_raw_bufr_msg(bufr.encoded_message)
    # close the file
    bf1.close()
Beispiel #13
0
def print_bufr_content3(input_bufr_file, output_fd, separator, max_msg_nr,
                        expand_flags):
    #  #[ implementation 3
    """
    example implementation using the BUFRInterfaceECMWF class
    """
    if expand_flags:
        print('Sorry, expand_flags is not yet implemented ' +
              'for example implementation 3')

    # get an instance of the RawBUFRFile class
    rbf = RawBUFRFile()

    # open the file for reading, count nr of BUFR messages in it
    # and store its content in memory, together with
    # an array of pointers to the start and end of each BUFR message
    rbf.open(input_bufr_file, 'rb')

    # extract the number of BUFR messages from the file
    num_msgs = rbf.get_num_bufr_msgs()

    # print('num_msgs = ',num_msgs)

    for msg_nr in range(1, num_msgs + 1):
        encoded_message, section_sizes, section_start_locations = \
                         rbf.get_raw_bufr_msg(msg_nr)
        bufr_obj = BUFRInterfaceECMWF(encoded_message, section_sizes,
                                      section_start_locations)
        #                              verbose=True)
        bufr_obj.decode_sections_012()
        bufr_obj.setup_tables()
        # print('num_subsets: ', bufr_obj.get_num_subsets())
        # print('num_elements: ',bufr_obj.get_num_elements())
        # bufr_obj.decode_sections_0123()
        # bufr_obj.print_sections_0123_metadata()

        # d = '/home/jos/werk/pybufr_ecmwf_interface/'+\
        #     'BUFR_test_files/radar/bufrtables/'
        # bufr_obj.setup_tables(table_b_to_use = d+'B0000000000085011012.TXT',
        #                      table_d_to_use = d+'D0000000000085011012.TXT')
        # bufr_obj.print_sections_012()
        # bufr_obj.fill_descriptor_list()

        # do the actual decoding
        bufr_obj.decode_data()

        # needed to have the units ready, so autoget_cval will work
        bufr_obj.decode_sections_0123()

        # Create header lines from variable names and units
        if msg_nr == 1:
            list_of_names = []
            list_of_units = []
            for (cname, cunit) in zip(bufr_obj.cnames, bufr_obj.cunits):
                # glue the ndarray of characters together to form strings
                if python3:
                    cname_str = ''.join(c.decode() for c in cname).strip()
                    cunit_str = ''.join(c.decode() for c in cunit).strip()
                else:
                    cname_str = ''.join(cname).strip()
                    cunit_str = ''.join(cunit).strip()

                # cnames is a bit over dimensioned, so check for empty values
                if cname_str.strip() == '':
                    break

                # append the strings to the head list and quote them
                list_of_names.append('"' + cname_str + '"')
                list_of_units.append('"' + cunit_str + '"')

            output_fd.write('"subset nr"' + separator)
            output_fd.write(separator.join(list_of_names) + '\n')

            output_fd.write('""' + separator)
            output_fd.write(separator.join(list_of_units) + '\n')

        nsubsets = bufr_obj.get_num_subsets()
        for subs in range(1, nsubsets + 1):

            # needed to have the units ready, so autoget_cval will work
            bufr_obj.fill_descriptor_list_subset(subs)

            nelements = bufr_obj.get_num_elements()
            data_list = []
            for descr_nr in range(nelements):
                data = bufr_obj.get_value(descr_nr, subs, autoget_cval=True)
                data_list.append(data)
            output_fd.write(
                str(subs) + separator +
                separator.join(str(val) for val in data_list) + "\n")
        print('converted BUFR msg nr. ', msg_nr)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the BUFR file
    rbf.close()
    if num_msgs == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
Beispiel #14
0
def create_bufr_file(output_bufr_file, template):
    #  #[ create bufr file
    '''
    and use this BUFR template to create a test BUFR message
    '''
    from pybufr_ecmwf.bufr_interface_ecmwf import BUFRInterfaceECMWF
    bufr = BUFRInterfaceECMWF(verbose=True)

    # fill sections 0, 1, 2 and 3
    num_subsets = 2
    bufr.fill_sections_0123(bufr_code_centre=98, # ECMWF
                            bufr_obstype=3, # sounding
                            bufr_subtype=253, # L2B
                            bufr_table_local_version=1,
                            bufr_table_master=0,
                            bufr_table_master_version=15,
                            bufr_code_subcentre=0, # L2B processing facility
                            num_subsets=num_subsets,
                            bufr_compression_flag=0)
                            # 64=compression/0=no compression

    # determine information from sections 0123 to construct the BUFR table
    # names expected by the ECMWF BUFR library and create symlinks to the
    # default tables if needed
    bufr.setup_tables(table_b_to_use='B'+TABLE_NAME,
                      table_d_to_use='D'+TABLE_NAME)
    bufr.register_and_expand_descriptors(template)

    # activate this one if the encoding crashes without clear cause:
    # bufr.estimated_num_bytes_for_encoding = 25000

    # retrieve the length of the expanded descriptor list
    exp_descr_list_length = bufr.ktdexl
    print "exp_descr_list_length = ", exp_descr_list_length

    # fill the values array with some dummy varying data
    num_values = num_subsets*bufr.max_nr_expanded_descriptors
    values = numpy.zeros(num_values, dtype=numpy.float64) # this is the default

    # note: these two must be identical for now, otherwise the
    # python to fortran interface breaks down. This also ofcourse is the
    # cause of the huge memory use of cvals in case num_values is large.
    num_cvalues = num_values
    cvals = numpy.zeros((num_cvalues, 80), dtype=numpy.character)
    cvals_index = 0

    repl_counts = []

    for subset in range(num_subsets):
        # note that python starts counting with 0, unlike fortran,
        # so there is no need to take (subset-1)

        print 'subset,exp_descr_list_length = ', subset, exp_descr_list_length
        i = subset*exp_descr_list_length

        # fill the message with some dummy data

        # fill year, month, day
        for val in [2014, 3, 19]: # fill the header
            values[i] = val+subset
            i += 1

        # fill prod_name
        # this is not python2.6 compatible
        #txt = 'filename{}.txt'.format(subset)
        txt = 'filename'+str(subset)+'.txt'
        cvals[cvals_index, :] = ' '
        for icval, cval in enumerate(txt):
            cvals[cvals_index, icval] = cval

        # values[i] = cvals_index * 1000 + 64 # len(txt)
        values[i] = (cvals_index+1) * 1000 + len(txt)
        i += 1
        cvals_index = cvals_index + 1

        for val in [5.1+0.1*subset, 55.2-0.01*subset, 23., 45., 73., 82.]:
            bufr.verify_in_range(i, val)
            values[i] = val
            i += 1

        if USE_DELAYED_REPLICATION:
            # set actual delayed replication repeats
            num_repl = 3 + 2*subset
            print 'num_repl = ', num_repl
            values[i] = num_repl
            i += 1
            repl_counts.append(num_repl)

            # fill the replicated variable
            for irepl in range(num_repl):
                val = 12.+subset*0.1 + irepl*0.01
                bufr.verify_in_range(i, val)
                values[i] = val
                i += 1

    # do the encoding to binary format
    bufr.kdata = numpy.array(repl_counts)
    print 'bufr.kdata = ', bufr.kdata
    bufr.encode_data(values, cvals)

    print 'DEBUG: values = ', values

    from pybufr_ecmwf.raw_bufr_file import RawBUFRFile
    # get an instance of the RawBUFRFile class
    bf1 = RawBUFRFile()

    # open the file for writing
    bf1.open(output_bufr_file, 'wb')
    # write the encoded BUFR message
    bf1.write_raw_bufr_msg(bufr.encoded_message)
    # close the file
    bf1.close()
def select_subsets(input_bufr_file, output_bufr_file):
    #  #[ select on subsets
    """
    select data and write out again
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)

    # open the file for writing
    rbf_out = RawBUFRFile()
    rbf_out.open(output_bufr_file, 'wb')

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        data = bob.get_values_as_2d_array()
        print 'data.shape = ', data.shape

        if data.shape[0]*data.shape[1] == 0:
            print 'NO DATA FOUND! this seems an empty BUFR message !'
            continue

        # select every other subset
        new_data = data[::2, :]

        print 'new_data.shape = ', new_data.shape
        #bob.bufr_obj

        nsub = bob.bufr_obj.get_num_subsets()
        n_exp_descr = len(bob.bufr_obj.values)/nsub
        bob.bufr_obj.fill_descriptor_list(nr_of_expanded_descriptors=
                                          n_exp_descr)
        bob.bufr_obj.ktdlst = bob.bufr_obj.get_descriptor_list()

        delayed_repl_data = bob.bufr_obj.derive_delayed_repl_factors()
        bob.bufr_obj.fill_delayed_repl_data(delayed_repl_data)

        new_nsub = new_data.shape[0]
        bob.bufr_obj.nr_subsets = new_nsub
        btm = BufrTemplate()
        btm.add_descriptors(*bob.bufr_obj.ktdlst)#[:self.ktdlen])
        btm.nr_of_delayed_repl_factors = 1
        btm.del_repl_max_nr_of_repeats_list = list(delayed_repl_data)
        bob.bufr_obj.register_and_expand_descriptors(btm)

        # activate this one if the encoding crashes without clear cause:
        # bob.bufr_obj.estimated_num_bytes_for_encoding = 25000

        bob.bufr_obj.kdate = new_nsub*list(delayed_repl_data)

        print 'bob.bufr_obj.cvals.shape = ', bob.bufr_obj.cvals.shape
        bob.bufr_obj.encode_data(new_data, bob.bufr_obj.cvals[:32, :])
        rbf_out.write_raw_bufr_msg(bob.bufr_obj.encoded_message)

        #for subs in range(len(data[:, 0])):
        #    output_fd.write(str(subs)+separator+
        #                    separator.join(str(val) for val in data[subs, :])+
        #                    "\n")
        print 'converted BUFR msg nr. ', msg_nr


    # close the file
    bob.close()
    if msg_nr == 0:
        print 'no BUFR messages found, are you sure this is a BUFR file?'

    rbf_out.close()
    # in encoding, just one copy should be [rpvided of repeated sequences
    for k in range(1):

        # fill dummy var
        values[i] = 34567.
        i = i+1

# debug
print('values[:25]: ', values[:25].tolist()) #numpy.where(values != 0)]))
print('values[-25:]: ', values[-25:].tolist())

# do the encoding to binary format
bufr.encode_data(values, cvals)

# get an instance of the RawBUFRFile class
bf1 = RawBUFRFile()

output_bufr_file = 'dummy_bufr_file.bfr'
# open the file for writing
bf1.open(output_bufr_file, 'wb')
# write the encoded BUFR message
bf1.write_raw_bufr_msg(bufr.encoded_message)
# close the file
bf1.close()

##############################
# reopen the BUFR file as test
##############################

print('*'*50)
Beispiel #17
0
def encoding_example(output_bufr_file):
    #  #[
    """
    wrap the example in a function to circumvent the pylint
    convention of requiring capitals for constants in the global
    scope (since most of these variables are not constants at all))
    """

    bufr = BUFRInterfaceECMWF(verbose=True)

    # fill sections 0, 1, 2 and 3
    num_subsets = 4
    bufr.fill_sections_0123(
        bufr_code_centre=98,  # ECMWF
        bufr_obstype=3,  # sounding
        bufr_subtype=251,  # L1B
        bufr_table_local_version=1,
        bufr_table_master=0,
        bufr_table_master_version=15,
        bufr_code_subcentre=0,  # L2B processing facility
        num_subsets=num_subsets,
        bufr_compression_flag=0)
    # 64=compression/0=no compression

    # determine information from sections 0123 to construct the BUFR table
    # names expected by the ECMWF BUFR library and create symlinks to the
    # default tables if needed
    bufr.setup_tables()

    # define a descriptor list
    template = BufrTemplate(verbose=True)

    template.add_descriptors(
        DD_D_DATE_YYYYMMDD,  # 0
        DD_D_TIME_HHMM)  # 1

    # delay replication for the next 2 descriptors
    # allow at most 2 delayed replications
    template.add_delayed_replic_descriptors(2, DD_PRESSURE, DD_TEMPERATURE)

    # replicate the next 2 descriptors 3 times
    template.add_replicated_descriptors(3, DD_LATITUDE_HIGH_ACCURACY,
                                        DD_LONGITUDE_HIGH_ACCURACY)

    bufr.register_and_expand_descriptors(template)

    # activate this one if the encoding crashes without clear cause:
    # bufr.estimated_num_bytes_for_encoding = 25000

    # retrieve the length of the expanded descriptor list
    exp_descr_list_length = bufr.ktdexl
    print("exp_descr_list_length = ", exp_descr_list_length)

    # fill the values array with some dummy varying data
    num_values = exp_descr_list_length * num_subsets
    values = np.zeros(num_values, dtype=np.float64)  # this is the default

    # note: these two must be identical for now, otherwise the
    # python to fortran interface breaks down. This also ofcourse is the
    # cause of the huge memory use of cvals in case num_values is large.
    num_cvalues = num_values
    cvals = np.zeros((num_cvalues, 80), dtype=np.character)

    for subset in range(num_subsets):
        # note that python starts counting with 0, unlike fortran,
        # so there is no need to take (subset-1)
        i = subset * exp_descr_list_length

        values[i] = 1999  # year
        i = i + 1
        values[i] = 12  # month
        i = i + 1
        values[i] = 31  # day
        i = i + 1
        values[i] = 23  # hour
        i = i + 1
        values[i] = 59 - subset  # minute
        i = i + 1
        values[i] = 2  # delayed replication factor
        # this delayed replication factor determines the actual number
        # of values to be stored for this particular subset
        # even if it is less then the number given in kdata above !
        for repl in range(2):
            i = i + 1
            values[i] = 1013.e2 - 100.e2 * subset + i + repl  # pressure [pa]
            i = i + 1
            values[i] = 273.15 - 10. * subset + i + repl  # temperature [K]
        for repl in range(3):
            i = i + 1
            values[i] = 51.82 + 0.05 * subset + i + repl  # latitude
            i = i + 1
            values[i] = 5.25 + 0.1 * subset + i + repl  # longitude

    # do the encoding to binary format
    bufr.encode_data(values, cvals)

    # get an instance of the RawBUFRFile class
    bf1 = RawBUFRFile()
    # open the file for writing
    bf1.open(output_bufr_file, 'wb')
    # write the encoded BUFR message
    bf1.write_raw_bufr_msg(bufr.encoded_message)
    # close the file
    bf1.close()
def raw_file_writing_example(output_bufr_file, msg1, msg2):
    #  #[
    """
    example for writing a BUFR message
    """

    # get an instance of the RawBUFRFile class
    bf1 = RawBUFRFile()

    # open the test file for writing
    bf1.open(output_bufr_file, 'wb')

    # write a few raw (encoded) BUFR messages
    bf1.write_raw_bufr_msg(msg1)
    if msg2 is not None:
        bf1.write_raw_bufr_msg(msg2)

    # print the internal data of the class instance
    bf1.print_properties(prefix="RawBUFRFile (opened for writing)")

    # close the file
    bf1.close()

    # delete the class instance
    del bf1
def raw_file_reading_example(input_bufr_file):
    #  #[
    """
    example for reading a BUFR message
    """

    # get an instance of the RawBUFRFile class
    rbf = RawBUFRFile()

    # open the file for reading, count nr of BUFR messages in it
    # and store its content in memory, together with
    # an array of pointers to the start and end of each BUFR message
    rbf.open(input_bufr_file, 'rb')

    # print the internal data of the class instance
    rbf.print_properties(prefix="RawBUFRFile (opened for reading)")

    # print the number of BUFR messages in the file
    num_msgs = rbf.get_num_bufr_msgs()
    print("This file contains: ", num_msgs, " BUFR messages.")

    # sequentially read the raw (undecoded) BUFR messages from the
    # class instance
    msg1 = rbf.get_next_raw_bufr_msg()[0] # should return proper data
    try:
        msg2 = rbf.get_next_raw_bufr_msg()[0] # should return corrupted data
    except EOFError:
        msg2 = None
    try:
        msg3 = rbf.get_next_raw_bufr_msg()[0] # should return corrupted data
    except EOFError:
        msg3 = None

    print("a warning is expected here:")
    # msg4 =
    try:
        rbf.get_next_raw_bufr_msg()[0] # should raise an EOF error
    except EOFError:
        print("Warning: EOF reached !")

    for i in range(1, num_msgs+1):
        # read a selected raw BUFR message from the class instance
        raw_data = rbf.get_raw_bufr_msg(i)[0]
        print("msg ", i, " got ", len(raw_data), " words")

    # close the file
    rbf.close()

    # delete the class instance
    del rbf
    return (msg1, msg2, msg3)
Beispiel #20
0
def print_bufr_content3(input_bufr_file, output_fd, separator,
                        max_msg_nr, expand_flags):
    #  #[ implementation 3
    """
    example implementation using the BUFRInterfaceECMWF class
    """
    if expand_flags:
        print('Sorry, expand_flags is not yet implemented '+
              'for example implementation 3')

    # get an instance of the RawBUFRFile class
    rbf = RawBUFRFile()

    # open the file for reading, count nr of BUFR messages in it
    # and store its content in memory, together with
    # an array of pointers to the start and end of each BUFR message
    rbf.open(input_bufr_file, 'rb')

    # extract the number of BUFR messages from the file
    num_msgs = rbf.get_num_bufr_msgs()

    # print('num_msgs = ',num_msgs)

    for msg_nr in range(1, num_msgs+1):
        encoded_message, section_sizes, section_start_locations = \
                         rbf.get_raw_bufr_msg(msg_nr)
        bufr_obj = BUFRInterfaceECMWF(encoded_message, section_sizes,
                                      section_start_locations)
        #                              verbose=True)
        bufr_obj.decode_sections_012()
        bufr_obj.setup_tables()
        # print('num_subsets: ', bufr_obj.get_num_subsets())
        # print('num_elements: ',bufr_obj.get_num_elements())
        # bufr_obj.decode_sections_0123()
        # bufr_obj.print_sections_0123_metadata()

        # d = '/home/jos/werk/pybufr_ecmwf_interface/'+\
        #     'BUFR_test_files/radar/bufrtables/'
        # bufr_obj.setup_tables(table_b_to_use = d+'B0000000000085011012.TXT',
         #                      table_d_to_use = d+'D0000000000085011012.TXT')
        # bufr_obj.print_sections_012()
        # bufr_obj.fill_descriptor_list()

        # do the actual decoding
        bufr_obj.decode_data()

        # needed to have the units ready, so autoget_cval will work
        bufr_obj.decode_sections_0123()

        # Create header lines from variable names and units
        if msg_nr == 1:
            list_of_names = []
            list_of_units = []
            for (cname, cunit) in zip(bufr_obj.cnames, bufr_obj.cunits):
                # glue the ndarray of characters together to form strings
                if python3:
                    cname_str = ''.join(c.decode() for c in cname).strip()
                    cunit_str = ''.join(c.decode() for c in cunit).strip()
                else:
                    cname_str = ''.join(cname).strip()
                    cunit_str = ''.join(cunit).strip()

                # cnames is a bit over dimensioned, so check for empty values
                if cname_str.strip() == '':
                    break

                # append the strings to the head list and quote them
                list_of_names.append('"'+cname_str+'"')
                list_of_units.append('"'+cunit_str+'"')

            output_fd.write('"subset nr"'+separator)
            output_fd.write(separator.join(list_of_names) + '\n')

            output_fd.write('""'+separator)
            output_fd.write(separator.join(list_of_units) + '\n')

        nsubsets = bufr_obj.get_num_subsets()
        for subs in range(1, nsubsets+1):

            # needed to have the units ready, so autoget_cval will work
            bufr_obj.fill_descriptor_list_subset(subs)

            nelements = bufr_obj.get_num_elements()
            data_list = []
            for descr_nr in range(nelements):
                data = bufr_obj.get_value(descr_nr, subs, autoget_cval=True)
                data_list.append(data)
            output_fd.write(str(subs)+separator+
                            separator.join(str(val) for val in data_list)+
                            "\n")
        print('converted BUFR msg nr. ', msg_nr)
        if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
            print('skipping remainder of this BUFR file')
            break

    # close the BUFR file
    rbf.close()
    if num_msgs == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')
def decoding_example(input_bufr_file):
    #  #[
    """
    wrap the example in a function to circumvent the pylint
    convention of requiring capitals for constants in the global
    scope (since most of these variables are not constants at all))
    """

    rbf = RawBUFRFile()
    rbf.open(input_bufr_file, 'rb')
    words = rbf.get_next_raw_bufr_msg()[0]
    rbf.close()

    # define the needed constants
    max_nr_descriptors = 20 # 300
    max_nr_expanded_descriptors = 140 # 160000
    max_nr_subsets = 361 # 25

    ktdlen = max_nr_descriptors
    # krdlen = max_nr_delayed_replication_factors
    kelem = max_nr_expanded_descriptors
    kvals = max_nr_expanded_descriptors*max_nr_subsets
    # jbufl  = max_bufr_msg_size
    # jsup   = length_ksup

    # handle BUFR tables
    print('------------------------------')

    # define our own location for storing (symlinks to) the BUFR tables
    private_bufr_tables_dir = os.path.abspath("./tmp_BUFR_TABLES")
    if not os.path.exists(private_bufr_tables_dir):
        os.mkdir(private_bufr_tables_dir)

    # make the needed symlinks to bufr tables

    # inspect the location of the ecmwfbufr.so file, and derive
    # from this the location of the BUFR tables that are delivered
    # with the ECMWF BUFR library software
    ecmwfbufr_path = os.path.split(ecmwfbufr.__file__)[0]
    path1 = os.path.join(ecmwfbufr_path, "ecmwf_bufrtables")
    path2 = os.path.join(ecmwfbufr_path, '..', "ecmwf_bufrtables")

    if os.path.exists(path1):
        ecmwf_bufr_tables_dir = path1
    elif os.path.exists(path2):
        ecmwf_bufr_tables_dir = path2
    else:
        print("Error: could not find BUFR tables directory")
        raise IOError

    # make sure the path is absolute, otherwise the ECMWF library
    # might fail when it attempts to use it ...
    ecmwf_bufr_tables_dir = os.path.abspath(ecmwf_bufr_tables_dir)

    needed_b_table = "B0000000000210000001.TXT"
    needed_d_table = "D0000000000210000001.TXT"
    available_b_table = "B0000000000098013001.TXT"
    available_d_table = "D0000000000098013001.TXT"

    source = os.path.join(ecmwf_bufr_tables_dir, available_b_table)
    destination = os.path.join(private_bufr_tables_dir, needed_b_table)
    ensure_symlink_exists(source, destination)

    source = os.path.join(ecmwf_bufr_tables_dir, available_d_table)
    destination = os.path.join(private_bufr_tables_dir, needed_d_table)
    ensure_symlink_exists(source, destination)

    # make sure the BUFR tables can be found
    # also, force a slash at the end, otherwise the library fails
    # to find the tables
    env = os.environ
    env["BUFR_TABLES"] = private_bufr_tables_dir+os.path.sep
    # print('private_bufr_tables_dir+os.path.sep=', \
    #        private_bufr_tables_dir+os.path.sep)

    # redirect all fortran stdout to fileunit 12, so the text
    # will end-up in a file called 'fort.12'
    # This is needed to get reproducible output for my unittests.
    # Without this trick the stdout stream from fortran and the stdout
    # stream from c/python may be mixed in inpredictable ways
    # (depending on the size of the buffer used to store the streams)
    os.environ['STD_OUT'] = '12'

    # suppres the default ECMWF welcome message which
    # is not yet redirected to the above defined fileunit
    os.environ['PRINT_TABLE_NAMES'] = 'FALSE'

    ksup = np.zeros(9, dtype=np.int)
    ksec0 = np.zeros(3, dtype=np.int)
    ksec1 = np.zeros(40, dtype=np.int)
    ksec2 = np.zeros(4096, dtype=np.int)
    kerr = 0

    print("calling: ecmwfbufr.bus012():")
    ecmwfbufr.bus012(words, ksup, ksec0, ksec1, ksec2, kerr)
    check_error_flag('ecmwfbufr.bus012', kerr)

    # convert the arrays to a list before printing, since
    # the standard print for numpy arrays is no longer reproducible
    # (old versions use spaces, new versions use commas as separator)
    print('ksup = ', ksup.tolist())
    print('------------------------------')
    print("printing content of section 0:")
    print("sec0 : ", ksec0.tolist())
    # print("sec0[hex] : ",[hex(i) for i in ksec0])
    ecmwfbufr.buprs0(ksec0)
    print('------------------------------')
    print("printing content of section 1:")
    print("sec1[:25] : ", ksec1[:25].tolist())
    # print("sec1[hex] : ",[hex(i) for i in ksec1])
    ecmwfbufr.buprs1(ksec1)
    key = np.zeros(46, dtype=np.int)
    sec2_len = ksec2[0]
    print('------------------------------')
    print("length of sec2: ", sec2_len)
    if sec2_len > 0:
        # buukey expands local ECMWF information from section 2 to the key array
        print('------------------------------')
        print("calling buukey")
        ecmwfbufr.buukey(ksec1, ksec2, key, ksup, kerr)
        print("sec2[:25] : ", ksec2[:25].tolist())
        print("printing content of section 2:")
        ecmwfbufr.buprs2(ksup, key)
    else:
        print('skipping section 2 [since it seems unused]')

    # these 4 are filled by the BUS012 call above
    # ksup   = np.zeros(         9, dtype = np.int)
    # ksec0  = np.zeros(         3, dtype = np.int)
    # ksec1  = np.zeros(        40, dtype = np.int)
    # ksec2  = np.zeros(      4096, dtype = np.int)

    print('------------------------------')
    ksec3 = np.zeros(4, dtype=np.int)
    ksec4 = np.zeros(2, dtype=np.int)
    cnames = np.zeros((kelem, 64), dtype=np.character)
    cunits = np.zeros((kelem, 24), dtype=np.character)
    values = np.zeros(kvals, dtype=np.float64) # this is the default
    cvals = np.zeros((kvals, 80), dtype=np.character)
    kerr = 0

    print("calling: ecmwfbufr.bufrex():")
    ecmwfbufr.bufrex(words, ksup, ksec0, ksec1, ksec2, ksec3, ksec4,
                     cnames, cunits, values, cvals, kerr)
    check_error_flag('ecmwfbufr.bufrex', kerr)

    # print a selection of the decoded numbers
    print('------------------------------')
    print("Decoded BUFR message:")
    print("ksup : ", ksup.tolist())
    print("sec0 : ", ksec0.tolist())
    print("sec1[:25] : ", ksec1[:25].tolist())
    print("sec2[:25] : ", ksec2[:25].tolist())
    print("sec3 : ", ksec3.tolist())
    print("sec4 : ", ksec4.tolist())
    print("cnames [cunits] : ")
    for (i, cnm) in enumerate(cnames):
        cun = cunits[i]
        if python3:
            txtn = ''.join(c.decode() for c in cnm)
            txtu = ''.join(c.decode() for c in cun)
        else:
            txtn = ''.join(c for c in cnm)
            txtu = ''.join(c for c in cun)

        if txtn.strip() != '':
            print('[%3.3i]:%s [%s]' % (i, txtn, txtu))

    print("values[:25] : ", values[:25].tolist())
    print("values[-25:] : ", values[-25:].tolist())
    txt = ''.join(str(v)+';' for v in values[:20] if v > 0.)
    print("values[:20] : ", txt)

    nsubsets = ksec3[2] # 361 # number of subsets in this BUFR message

    #not yet used:
    #nelements = ksup[4] # 44 # size of one expanded subset

    lat = np.zeros(nsubsets)
    lon = np.zeros(nsubsets)
    for subs in range(nsubsets):
        # index_lat = nelements*(s-1)+24
        # index_lon = nelements*(s-1)+25
        index_lat = max_nr_expanded_descriptors*(subs-1)+24
        index_lon = max_nr_expanded_descriptors*(subs-1)+25
        lat[subs] = values[index_lat]
        lon[subs] = values[index_lon]
        if 30*(subs//30) == subs:
            print("subs = ", subs, "lat = ", lat[subs], " lon = ", lon[subs])
            print("min/max lat", min(lat), max(lat))
            print("min/max lon", min(lon), max(lon))

    print('------------------------------')
    # busel: fill the descriptor list arrays (only needed for printing)

    # warning: this routine has no inputs, and acts on data stored
    #          during previous library calls
    # Therefore it only produces correct results when either bus0123
    # or bufrex have been called previously on the same bufr message.....
    # However, it is not clear to me why it seems to correctly produce
    # the descriptor lists (both bare and expanded), but yet it does
    # not seem to fill the ktdlen and ktdexl values.

    ktdlen = 0
    ktdlst = np.zeros(max_nr_descriptors, dtype=np.int)
    ktdexl = 0
    ktdexp = np.zeros(max_nr_expanded_descriptors, dtype=np.int)
    kerr = 0

    print("calling: ecmwfbufr.busel():")
    ecmwfbufr.busel(ktdlen, # actual number of data descriptors
                    ktdlst, # list of data descriptors
                    ktdexl, # actual number of expanded data descriptors
                    ktdexp, # list of expanded data descriptors
                    kerr)   # error  message
    check_error_flag('ecmwfbufr.busel', kerr)

    print('busel result:')
    print("ktdlen = ", ktdlen)
    print("ktdexl = ", ktdexl)

    selection1 = np.where(ktdlst > 0)
    ktdlen = len(selection1[0])
    selection2 = np.where(ktdexp > 0)
    ktdexl = len(selection2[0])

    print('fixed lengths:')
    print("ktdlen = ", ktdlen)
    print("ktdexl = ", ktdexl)

    print('descriptor lists:')
    print("ktdlst = ", ktdlst[:ktdlen])
    print("ktdexp = ", ktdexp[:ktdexl])

    print('------------------------------')
    print("printing content of section 3:")
    print("sec3 : ", ksec3.tolist())
    ecmwfbufr.buprs3(ksec3, ktdlst, ktdexp, cnames)
def encoding_example(output_bufr_file):
    #  #[
    """
    wrap the example in a function to circumvent the pylint
    convention of requiring capitals for constants in the global
    scope (since most of these variables are not constants at all))
    """

    bufr = BUFRInterfaceECMWF(verbose=True)

    # fill sections 0, 1, 2 and 3
    num_subsets = 4
    bufr.fill_sections_0123(bufr_code_centre=98, # ECMWF
                            bufr_obstype=3, # sounding
                            bufr_subtype=251, # L1B
                            bufr_table_local_version=1,
                            bufr_table_master=0,
                            bufr_table_master_version=15,
                            bufr_code_subcentre=0, # L2B processing facility
                            num_subsets=num_subsets,
                            bufr_compression_flag=0)
    # 64=compression/0=no compression

    # determine information from sections 0123 to construct the BUFR table
    # names expected by the ECMWF BUFR library and create symlinks to the
    # default tables if needed
    bufr.setup_tables()

    # define a descriptor list
    template = BufrTemplate()

    template.add_descriptors(DD_D_DATE_YYYYMMDD, # 0
                             DD_D_TIME_HHMM)     # 1

    # delay replication for the next 2 descriptors
    # allow at most 2 delayed replications
    template.add_delayed_replic_descriptors(2,
                                            DD_PRESSURE,
                                            DD_TEMPERATURE)

    # replicate the next 2 descriptors 3 times
    template.add_replicated_descriptors(3,
                                        DD_LATITUDE_HIGH_ACCURACY,
                                        DD_LONGITUDE_HIGH_ACCURACY)

    bufr.register_and_expand_descriptors(template)

    # activate this one if the encoding crashes without clear cause:
    # bufr.estimated_num_bytes_for_encoding = 25000

    # retrieve the length of the expanded descriptor list
    exp_descr_list_length = bufr.ktdexl
    print "exp_descr_list_length = ", exp_descr_list_length

    # fill the values array with some dummy varying data
    num_values = exp_descr_list_length*num_subsets
    values = np.zeros(num_values, dtype=np.float64) # this is the default

    # note: these two must be identical for now, otherwise the
    # python to fortran interface breaks down. This also ofcourse is the
    # cause of the huge memory use of cvals in case num_values is large.
    num_cvalues = num_values
    cvals = np.zeros((num_cvalues, 80), dtype=np.character)

    for subset in range(num_subsets):
        # note that python starts counting with 0, unlike fortran,
        # so there is no need to take (subset-1)
        i = subset*exp_descr_list_length

        values[i] = 1999 # year
        i = i+1
        values[i] = 12 # month
        i = i+1
        values[i] = 31 # day
        i = i+1
        values[i] = 23 # hour
        i = i+1
        values[i] = 59 - subset # minute
        i = i+1
        values[i] = 2 # delayed replication factor
        # this delayed replication factor determines the actual number
        # of values to be stored for this particular subset
        # even if it is less then the number given in kdata above !
        for repl in range(2):
            i = i+1
            values[i] = 1013.e2 - 100.e2*subset+i+repl # pressure [pa]
            i = i+1
            values[i] = 273.15 - 10.*subset+i+repl # temperature [K]
        for repl in range(3):
            i = i+1
            values[i] = 51.82 + 0.05*subset+i+repl # latitude
            i = i+1
            values[i] = 5.25 + 0.1*subset+i+repl # longitude

    # do the encoding to binary format
    bufr.encode_data(values, cvals)

    # get an instance of the RawBUFRFile class
    bf1 = RawBUFRFile()
    # open the file for writing
    bf1.open(output_bufr_file, 'wb')
    # write the encoded BUFR message
    bf1.write_raw_bufr_msg(bufr.encoded_message)
    # close the file
    bf1.close()
def upgrade_bufr_file(input_bufr_file, output_bufr_file):
    #  #[
    """
    an example routine to demonstrate how to read a BUFR message,
    upgrade it's edition number, and write it to an output BUFR file
    """
    # get 2 instances of the RawBUFRFile class
    rbf_in = RawBUFRFile()
    rbf_out = RawBUFRFile()

    # open the file for reading, count nr of BUFR messages in it
    # and store its content in memory, together with
    # an array of pointers to the start and end of each BUFR message
    rbf_in.open(input_bufr_file, 'rb')

    # open the file for writing
    rbf_out.open(output_bufr_file, 'wb')

    # extract the number of BUFR messages from the file
    num_msgs = rbf_in.get_num_bufr_msgs()

    for msg_nr in range(1, num_msgs+1):
        raw_msg, section_sizes, section_start_locations = \
                 rbf_in.get_raw_bufr_msg(msg_nr)
        bufr_obj = BUFRInterfaceECMWF(raw_msg, section_sizes,
                                      section_start_locations,
                                      verbose=True)

        bufr_obj.decode_sections_012()
        bufr_obj.setup_tables()
        bufr_obj.decode_data()

        nsub = bufr_obj.get_num_subsets()
        n_exp_descr = len(bufr_obj.values)/nsub
        bufr_obj.fill_descriptor_list(nr_of_expanded_descriptors=n_exp_descr)

        bufr_obj.ksec0[3-1] = 4 # set bufr edition to 4
        bufr_obj.ktdlst = bufr_obj.get_descriptor_list()

        # extract delayed replication factors
        delayed_repl_data = bufr_obj.derive_delayed_repl_factors()

        # fill the list of replication factors
        bufr_obj.fill_delayed_repl_data(delayed_repl_data)

        # activate this one if the encoding crashes without clear cause:
        # bufr_obj.estimated_num_bytes_for_encoding = 25000

        # encode the data
        bufr_obj.encode_data(bufr_obj.values, bufr_obj.cvals)
        print 'Encode BUFR msg %i' % msg_nr

        rbf_out.write_raw_bufr_msg(bufr_obj.encoded_message)

    # close the file
    rbf_in.close()
    rbf_out.close()
def raw_file_reading_example(input_bufr_file):
    #  #[
    """
    example for reading a BUFR message
    """

    # get an instance of the RawBUFRFile class
    rbf = RawBUFRFile()

    # open the file for reading, count nr of BUFR messages in it
    # and store its content in memory, together with
    # an array of pointers to the start and end of each BUFR message
    rbf.open(input_bufr_file, 'rb')

    # print the internal data of the class instance
    rbf.print_properties(prefix="RawBUFRFile (opened for reading)")

    # print the number of BUFR messages in the file
    num_msgs = rbf.get_num_bufr_msgs()
    print "This file contains: ", num_msgs, " BUFR messages."

    # sequentially read the raw (undecoded) BUFR messages from the
    # class instance
    msg1 = rbf.get_next_raw_bufr_msg()[0] # should return proper data
    try:
        msg2 = rbf.get_next_raw_bufr_msg()[0] # should return corrupted data
    except EOFError:
        msg2 = None
    try:
        msg3 = rbf.get_next_raw_bufr_msg()[0] # should return corrupted data
    except EOFError:
        msg3 = None

    print "a warning is expected here:"
    # msg4 =
    try:
        rbf.get_next_raw_bufr_msg()[0] # should raise an EOF error
    except EOFError:
        print "Warning: EOF reached !"

    for i in range(1, num_msgs+1):
        # read a selected raw BUFR message from the class instance
        raw_data = rbf.get_raw_bufr_msg(i)[0]
        print "msg ", i, " got ", len(raw_data), " words"

    # close the file
    rbf.close()

    # delete the class instance
    del rbf
    return (msg1, msg2, msg3)
Beispiel #25
0
def select_subsets(input_bufr_file, output_bufr_file):
    #  #[ select on subsets
    """
    select data and write out again
    """

    # get an instance of the BUFR class
    # which automatically opens the file for reading and decodes it
    bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)

    # open the file for writing
    rbf_out = RawBUFRFile()
    rbf_out.open(output_bufr_file, 'wb')

    msg_nr = 0
    while True:
        try:
            bob.get_next_msg()
            msg_nr += 1
        except EOFError:
            break

        data = bob.get_values_as_2d_array()
        print('data.shape = ', data.shape)

        if data.shape[0] * data.shape[1] == 0:
            print('NO DATA FOUND! this seems an empty BUFR message !')
            continue

        # select every other subset
        new_data = data[::2, :]

        print('new_data.shape = ', new_data.shape)
        #bob.bufr_obj

        nsub = bob.bufr_obj.get_num_subsets()
        n_exp_descr = len(bob.bufr_obj.values) / nsub
        bob.bufr_obj.fill_descriptor_list(
            nr_of_expanded_descriptors=n_exp_descr)
        bob.bufr_obj.ktdlst = bob.bufr_obj.get_descriptor_list()

        delayed_repl_data = bob.bufr_obj.derive_delayed_repl_factors()
        bob.bufr_obj.fill_delayed_repl_data(delayed_repl_data)

        new_nsub = new_data.shape[0]
        bob.bufr_obj.nr_subsets = new_nsub
        btm = BufrTemplate()
        btm.add_descriptors(*bob.bufr_obj.ktdlst)  #[:self.ktdlen])
        btm.nr_of_delayed_repl_factors = 1
        btm.del_repl_max_nr_of_repeats_list = list(delayed_repl_data)
        bob.bufr_obj.register_and_expand_descriptors(btm)

        # activate this one if the encoding crashes without clear cause:
        # bob.bufr_obj.estimated_num_bytes_for_encoding = 25000

        bob.bufr_obj.kdate = new_nsub * list(delayed_repl_data)

        print('bob.bufr_obj.cvals.shape = ', bob.bufr_obj.cvals.shape)
        bob.bufr_obj.encode_data(new_data, bob.bufr_obj.cvals[:32, :])
        rbf_out.write_raw_bufr_msg(bob.bufr_obj.encoded_message)

        #for subs in range(len(data[:, 0])):
        #    output_fd.write(str(subs)+separator+
        #                    separator.join(str(val) for val in data[subs, :])+
        #                    "\n")
        print('converted BUFR msg nr. ', msg_nr)

    # close the file
    bob.close()
    if msg_nr == 0:
        print('no BUFR messages found, are you sure this is a BUFR file?')

    rbf_out.close()
def encoding_example(output_bufr_file):
    #  #[
    """
    wrap the example in a function to circumvent the pylint
    convention of requiring capitals for constants in the global
    scope (since most of these variables are not constants at all))
    """

    bufr = BUFRInterfaceECMWF(verbose=True)

    # fill sections 0, 1, 2 and 3
    num_subsets = 4
    bufr.fill_sections_0123(bufr_code_centre=98, # ECMWF
                            bufr_obstype=3, # sounding
                            bufr_subtype=251, # L1B
                            bufr_table_local_version=1,
                            bufr_table_master=0,
                            bufr_table_master_version=15,
                            bufr_code_subcentre=0, # L2B processing facility
                            num_subsets=num_subsets,
                            bufr_compression_flag=0)
    # 64=compression/0=no compression

    # determine information from sections 0123 to construct the BUFR table
    # names expected by the ECMWF BUFR library and create symlinks to the
    # default tables if needed
    # NOTE: these custom BUFR tables have been created by the
    #       create_bufr_tables.py example script
    bufr.setup_tables(table_b_to_use='B_my_test_BUFR_table.txt',
                      table_c_to_use='C_my_test_BUFR_table.txt',
                      table_d_to_use='D_my_test_BUFR_table.txt')

    # define a descriptor list
    template = BufrTemplate(verbose=True)

    template.add_descriptors(DD_B_048001,        # 0
                             DD_B_048002,        # 1
                             DD_D_348001)        # 2

    bufr.register_and_expand_descriptors(template)

    # activate this one if the encoding crashes without clear cause:
    # bufr.estimated_num_bytes_for_encoding = 25000

    # retrieve the length of the expanded descriptor list
    exp_descr_list_length = bufr.ktdexl
    print("exp_descr_list_length = ", exp_descr_list_length)

    # fill the values array with some dummy varying data
    num_values = exp_descr_list_length*num_subsets
    values = np.zeros(num_values, dtype=np.float64) # this is the default

    # note: these two must be identical for now, otherwise the
    # python to fortran interface breaks down. This also ofcourse is the
    # cause of the huge memory use of cvals in case num_values is large.
    num_cvalues = num_values
    cvals = np.zeros((num_cvalues, 80), dtype=np.character)

    for subset in range(num_subsets):
        # note that python starts counting with 0, unlike fortran,
        # so there is no need to take (subset-1)
        i = subset*exp_descr_list_length

        # fill the message with some dummy data
        values[i] = 1.2515 + 0.0011*subset
        i = i+1
        values[i] = (3.4562 + 0.0012*subset)*1.e-9
        i = i+1
        values[i] = 1.2625 + 0.0003*subset
        i = i+1
        values[i] = (3.4561 + 0.0014*subset)*1.e-9

    # do the encoding to binary format
    bufr.encode_data(values, cvals)

    # get an instance of the RawBUFRFile class
    bf1 = RawBUFRFile()
    # open the file for writing
    bf1.open(output_bufr_file, 'wb')
    # write the encoded BUFR message
    bf1.write_raw_bufr_msg(bufr.encoded_message)
    # close the file
    bf1.close()
def raw_file_writing_example(output_bufr_file, msg1, msg2):
    #  #[
    """
    example for writing a BUFR message
    """

    # get an instance of the RawBUFRFile class
    bf1 = RawBUFRFile()

    # open the test file for writing
    bf1.open(output_bufr_file, 'wb')

    # write a few raw (encoded) BUFR messages
    bf1.write_raw_bufr_msg(msg1)
    if msg2 is not None:
        bf1.write_raw_bufr_msg(msg2)

    # print the internal data of the class instance
    bf1.print_properties(prefix="RawBUFRFile (opened for writing)")

    # close the file
    bf1.close()

    # delete the class instance
    del bf1
def encoding_example(output_bufr_file):
    #  #[
    """
    wrap the example in a function to circumvent the pylint
    convention of requiring capitals for constants in the global
    scope (since most of these variables are not constants at all))
    """
    # define the needed constants
    max_nr_descriptors = 20 # 300
    max_nr_expanded_descriptors = 140 # 160000
    max_nr_subsets = 361 # 25

    ktdlen = max_nr_descriptors
    # krdlen = max_nr_delayed_replication_factors
    kelem = max_nr_expanded_descriptors
    kvals = max_nr_expanded_descriptors*max_nr_subsets
    # jbufl  = max_bufr_msg_size
    # jsup   = length_ksup

    #  initialise all arrays
    print '------------------------------'
    print 'reinitialising all arrays...'
    print '------------------------------'
    #ksup   = np.zeros(          9, dtype = np.int)
    ksec0 = np.zeros(3, dtype=np.int)
    ksec1 = np.zeros(40, dtype=np.int)
    ksec2 = np.zeros(4096, dtype=np.int)
    #key    = np.zeros(         52, dtype = np.int)
    ksec3 = np.zeros(4, dtype=np.int)
    ksec4 = np.zeros(2, dtype=np.int)
    cnames = np.zeros((kelem, 64), dtype=np.character)
    cunits = np.zeros((kelem, 24), dtype=np.character)
    values = np.zeros(kvals, dtype=np.float64) # this is the default
    cvals = np.zeros((kvals, 80), dtype=np.character)
    ktdlen = 0
    ktdlst = np.zeros(max_nr_descriptors, dtype=np.int)
    ktdexl = 0
    ktdexp = np.zeros(max_nr_expanded_descriptors, dtype=np.int)
    kerr = 0

    # handle BUFR tables
    print '------------------------------'

    # define our own location for storing (symlinks to) the BUFR tables
    private_bufr_tables_dir = os.path.abspath("./tmp_BUFR_TABLES")
    if not os.path.exists(private_bufr_tables_dir):
        os.mkdir(private_bufr_tables_dir)

    # make the needed symlinks to bufr tables

    # inspect the location of the ecmwfbufr.so file, and derive
    # from this the location of the BUFR tables that are delivered
    # with the ECMWF BUFR library software
    ecmwfbufr_path = os.path.split(ecmwfbufr.__file__)[0]
    path1 = os.path.join(ecmwfbufr_path, "ecmwf_bufrtables")
    path2 = os.path.join(ecmwfbufr_path, '..', "ecmwf_bufrtables")

    if os.path.exists(path1):
        ecmwf_bufr_tables_dir = path1
    elif os.path.exists(path2):
        ecmwf_bufr_tables_dir = path2
    else:
        print "Error: could not find BUFR tables directory"
        raise IOError

    # make sure the path is absolute, otherwise the ECMWF library
    # might fail when it attempts to use it ...
    ecmwf_bufr_tables_dir = os.path.abspath(ecmwf_bufr_tables_dir)

    needed_b_table = "B0000000000098015001.TXT"
    needed_d_table = "D0000000000098015001.TXT"
    available_b_table = "B0000000000098013001.TXT"
    available_d_table = "D0000000000098013001.TXT"

    source = os.path.join(ecmwf_bufr_tables_dir, available_b_table)
    destination = os.path.join(private_bufr_tables_dir, needed_b_table)
    ensure_symlink_exists(source, destination)

    source = os.path.join(ecmwf_bufr_tables_dir, available_d_table)
    destination = os.path.join(private_bufr_tables_dir, needed_d_table)
    ensure_symlink_exists(source, destination)

    # make sure the BUFR tables can be found
    # also, force a slash at the end, otherwise the library fails
    # to find the tables
    envir = os.environ
    envir["BUFR_TABLES"] = private_bufr_tables_dir+os.path.sep

    # redirect all fortran stdout to fileunit 12, so the text
    # will end-up in a file called 'fort.12'
    # This is needed to get reproducible output for my unittests.
    # Without this trick the stdout stream from fortran and the stdout
    # stream from c/python may be mixed in inpredictable ways
    # (depending on the size of the buffer used to store the streams)
    os.environ['STD_OUT'] = '12'

    # suppres the default ECMWF welcome message which
    # is not yet redirected to the above defined fileunit
    os.environ['PRINT_TABLE_NAMES'] = 'FALSE'

    # fill sections 0,1,2 and 3
    bufr_edition = 4
    bufr_code_centre = 98 # ECMWF
    bufr_obstype = 3 # sounding
    bufr_subtype_l1b = 251 # L1B
    bufr_table_local_version = 1
    bufr_table_master = 0
    bufr_table_master_version = 15
    bufr_code_subcentre = 0 # L2B processing facility
    bufr_compression_flag = 0 # 64=compression/0=no compression

    (year, month, day, hour, minute) = time.localtime()[:5]
    #(year, month, day, hour, minute, second,
    # weekday, julianday, isdaylightsavingstime) = time.localtime()

    num_subsets = 4

    # fill section 0
    ksec0[1-1] = 0
    ksec0[2-1] = 0
    ksec0[3-1] = bufr_edition

    # fill section 1
    ksec1[1-1] = 22                       # length sec1 bytes
    #                                        [filled by the encoder]
    # however,a minimum of 22 is obliged here
    ksec1[2-1] = bufr_edition              # bufr edition
    ksec1[3-1] = bufr_code_centre          # originating centre
    ksec1[4-1] = 1                       # update sequence
    ksec1[5-1] = 0                       # (PRESENCE SECT 2)
    #                                        (0/128 = no/yes)
    ksec1[6-1] = bufr_obstype              # message type
    ksec1[7-1] = bufr_subtype_l1b          # subtype
    ksec1[8-1] = bufr_table_local_version  # version of local table
    ksec1[9-1] = (year-2000)               # Without offset year - 2000
    ksec1[10-1] = month                     # month
    ksec1[11-1] = day                       # day
    ksec1[12-1] = hour                      # hour
    ksec1[13-1] = minute                    # minute
    ksec1[14-1] = bufr_table_master         # master table
    ksec1[15-1] = bufr_table_master_version # version of master table
    ksec1[16-1] = bufr_code_subcentre       # originating subcentre
    ksec1[17-1] = 0
    ksec1[18-1] = 0

    # a test for ksec2 is not yet defined

    # fill section 3
    ksec3[1-1] = 0
    ksec3[2-1] = 0
    ksec3[3-1] = num_subsets                # no of data subsets
    ksec3[4-1] = bufr_compression_flag      # compression flag

    # define a descriptor list
    ktdlen = 9 # length of unexpanded descriptor list
    ktdlst = np.zeros(ktdlen, dtype=np.int)

    # define descriptor 1
    dd_d_date_yyyymmdd = 301011 # date
    # this defines the sequence:
    # 004001 ! year
    # 004002 ! month
    # 004003 ! day

    # define descriptor 2
    dd_d_time_hhmm = 301012 # time
    # this defines the sequence:
    # 004004 ! hour
    # 004005 ! minute

    # define descriptor 3
    dd_pressure = int('007004', 10) # pressure [pa]

    # WARNING: filling the descriptor variable with 007004 will fail
    # because python will interpret this as an octal value, and thus
    # automatically convert 007004 to the decimal value 3588

    # define descriptor 4
    dd_temperature = int('012001', 10) # [dry-bulb] temperature [K]

    # define descriptor 5
    dd_latitude_high_accuracy = int('005001', 10)
    # latitude (high accuracy) [degree]

    # define descriptor 6
    dd_longitude_high_accuracy = int('006001', 10)
    # longitude (high accuracy) [degree]

    # define the delayed replication code
    delayed_descr_repl_factor = int('031001', 10)

    def get_replication_code(num_descriptors, num_repeats):
        """
        calculate the replication note needed to describe
        how many times a descriptor sequence is to be repeated
        """
        repl_factor = 100000 + num_descriptors*1000 + num_repeats
        # for example replicating 2 descriptors 25 times will
        # be encoded as: 102025
        # for delayed replication, set num_repeats to 0
        # then add the delayed_descr_repl_factor after this code
        return repl_factor

    ktdlst[0] = dd_d_date_yyyymmdd
    ktdlst[1] = dd_d_time_hhmm

    # delay replication for the next 2 descriptors
    ktdlst[2] = get_replication_code(2, 0)
    ktdlst[3] = delayed_descr_repl_factor # = 031001

    ktdlst[4] = dd_pressure
    ktdlst[5] = dd_temperature

    # replicate the next 2 descriptors 3 times
    ktdlst[6] = get_replication_code(2, 3)

    ktdlst[7] = dd_latitude_high_accuracy
    ktdlst[8] = dd_longitude_high_accuracy

    # call BUXDES
    # buxdes: expand the descriptor list
    #         and fill the array ktdexp and the variable ktdexp
    #         [only usefull when creating a bufr msg with table D entries

    # iprint = 0 # default is to be silent
    iprint = 1
    if iprint == 1:
        print "------------------------"
        print " printing BUFR template "
        print "------------------------"

    # define and fill the list of replication factors
    num_del_repl_factors = 1
    kdata = np.zeros(num_subsets*num_del_repl_factors, dtype=np.int)
    for i in range(num_subsets):
        # Warning: just set the whole array to the maximum you wish to have.
        # Letting this number vary seems not to work with the current
        # ECMWF library. It will allways just look at the first element
        # in the kdata array. (or do I misunderstand the BUFR format here?)
        kdata[i] = 2 # i+1
    print "delayed replication factors: ", kdata

    ecmwfbufr.buxdes(iprint, ksec1, ktdlst, kdata,
                     ktdexl, ktdexp, cnames, cunits, kerr)
    print "ktdlst = ", ktdlst
    selection = np.where(ktdexp > 0)
    #print "selection = ",selection
    print "ktdexp = ", ktdexp[selection]
    print "ktdexl = ", ktdexl # this one seems not to be filled ...?
    if kerr != 0:
        print "kerr = ", kerr
        sys.exit(1)

    # print "cnames = ", cnames
    # print "cunits = ", cunits

    # retrieve the length of the expanded descriptor list
    exp_descr_list_length = len(np.where(ktdexp > 0)[0])
    print "exp_descr_list_length = ", exp_descr_list_length

    # fill the values array with some dummy varying data
    num_values = exp_descr_list_length*num_subsets
    values = np.zeros(num_values, dtype=np.float64) # this is the default

    for subset in range(num_subsets):
        # note that python starts counting with 0, unlike fortran,
        # so there is no need to take (subset-1)
        i = subset*exp_descr_list_length
        values[i] = 1999 # year

        i = i+1
        values[i] = 12 # month

        i = i+1
        values[i] = 31 # day

        i = i+1
        values[i] = 23 # hour

        i = i+1
        values[i] = 59 - subset # minute

        i = i+1
        values[i] = 2 # delayed replication factor
        # this delayed replication factor determines the actual number
        # of values to be stored for this particular subset
        # even if it is less then the number given in kdata above !
        for repl in range(2):
            i = i+1
            values[i] = 1013.e2 - 100.e2*subset+i+repl # pressure [pa]

            i = i+1
            values[i] = 273.15 - 10.*subset+i+repl # temperature [K]

        for repl in range(3):
            i = i+1
            values[i] = 51.82 + 0.05*subset+i+repl # latitude

            i = i+1
            values[i] = 5.25 + 0.1*subset+i+repl # longitude

    # call BUFREN
    #   bufren: encode a bufr message
    #sizewords = 200
    #kbuff = np.zeros(num_values, dtype = np.int)
    cvals = np.zeros((num_values, 80), dtype=np.character)
    # define the output buffer
    num_bytes = 5000
    num_words = num_bytes/4
    words = np.zeros(num_words, dtype=np.int)

    ecmwfbufr.bufren(ksec0, ksec1, ksec2, ksec3, ksec4,
                     ktdlst, kdata, exp_descr_list_length,
                     values, cvals, words, kerr)

    print "bufren call finished"
    if kerr != 0:
        print "kerr = ", kerr
        sys.exit(1)

    print "words="
    print words

    nonzero_locations = np.where(words != 0)
    #print 'nonzero_locations = ',nonzero_locations[0]

    numwords = nonzero_locations[0][-1] + 1
    print "encoded size: ", numwords, " words or ", numwords*4, " bytes"

    encoded_message = words[:numwords]

    # get an instance of the RawBUFRFile class
    bf1 = RawBUFRFile()
    # open the file for writing
    bf1.open(output_bufr_file, 'wb')
    # write the encoded BUFR message
    bf1.write_raw_bufr_msg(encoded_message)
    # close the file
    bf1.close()
Beispiel #29
0
def decoding_example(input_bufr_file):
    #  #[
    """
    wrap the example in a function to circumvent the pylint
    convention of requiring capitals for constants in the global
    scope (since most of these variables are not constants at all))
    """

    rbf = RawBUFRFile()
    rbf.open(input_bufr_file, 'rb')
    words = rbf.get_next_raw_bufr_msg()[0]
    rbf.close()

    # define the needed constants
    max_nr_descriptors = 20  # 300
    max_nr_expanded_descriptors = 140  # 160000
    max_nr_subsets = 361  # 25

    ktdlen = max_nr_descriptors
    # krdlen = max_nr_delayed_replication_factors
    kelem = max_nr_expanded_descriptors
    kvals = max_nr_expanded_descriptors * max_nr_subsets
    # jbufl  = max_bufr_msg_size
    # jsup   = length_ksup

    # handle BUFR tables
    print('------------------------------')

    # define our own location for storing (symlinks to) the BUFR tables
    private_bufr_tables_dir = os.path.abspath("./tmp_BUFR_TABLES")
    if not os.path.exists(private_bufr_tables_dir):
        os.mkdir(private_bufr_tables_dir)

    # make the needed symlinks to bufr tables

    # inspect the location of the ecmwfbufr.so file, and derive
    # from this the location of the BUFR tables that are delivered
    # with the ECMWF BUFR library software
    ecmwfbufr_path = os.path.split(ecmwfbufr.__file__)[0]
    path1 = os.path.join(ecmwfbufr_path, "ecmwf_bufrtables")
    path2 = os.path.join(ecmwfbufr_path, '..', "ecmwf_bufrtables")

    if os.path.exists(path1):
        ecmwf_bufr_tables_dir = path1
    elif os.path.exists(path2):
        ecmwf_bufr_tables_dir = path2
    else:
        print("Error: could not find BUFR tables directory")
        raise IOError

    # make sure the path is absolute, otherwise the ECMWF library
    # might fail when it attempts to use it ...
    ecmwf_bufr_tables_dir = os.path.abspath(ecmwf_bufr_tables_dir)

    needed_b_table = "B0000000000210000001.TXT"
    needed_d_table = "D0000000000210000001.TXT"
    available_b_table = "B0000000000098013001.TXT"
    available_d_table = "D0000000000098013001.TXT"

    source = os.path.join(ecmwf_bufr_tables_dir, available_b_table)
    destination = os.path.join(private_bufr_tables_dir, needed_b_table)
    ensure_symlink_exists(source, destination)

    source = os.path.join(ecmwf_bufr_tables_dir, available_d_table)
    destination = os.path.join(private_bufr_tables_dir, needed_d_table)
    ensure_symlink_exists(source, destination)

    # make sure the BUFR tables can be found
    # also, force a slash at the end, otherwise the library fails
    # to find the tables
    env = os.environ
    env["BUFR_TABLES"] = private_bufr_tables_dir + os.path.sep
    # print('private_bufr_tables_dir+os.path.sep=', \
    #        private_bufr_tables_dir+os.path.sep)

    # redirect all fortran stdout to fileunit 12, so the text
    # will end-up in a file called 'fort.12'
    # This is needed to get reproducible output for my unittests.
    # Without this trick the stdout stream from fortran and the stdout
    # stream from c/python may be mixed in inpredictable ways
    # (depending on the size of the buffer used to store the streams)
    os.environ['STD_OUT'] = '12'

    # suppres the default ECMWF welcome message which
    # is not yet redirected to the above defined fileunit
    os.environ['PRINT_TABLE_NAMES'] = 'FALSE'

    ksup = np.zeros(9, dtype=np.int)
    ksec0 = np.zeros(3, dtype=np.int)
    ksec1 = np.zeros(40, dtype=np.int)
    ksec2 = np.zeros(4096, dtype=np.int)
    kerr = 0

    print("calling: ecmwfbufr.bus012():")
    ecmwfbufr.bus012(words, ksup, ksec0, ksec1, ksec2, kerr)
    check_error_flag('ecmwfbufr.bus012', kerr)

    # convert the arrays to a list before printing, since
    # the standard print for numpy arrays is no longer reproducible
    # (old versions use spaces, new versions use commas as separator)
    print('ksup = ', ksup.tolist())
    print('------------------------------')
    print("printing content of section 0:")
    print("sec0 : ", ksec0.tolist())
    # print("sec0[hex] : ",[hex(i) for i in ksec0])
    ecmwfbufr.buprs0(ksec0)
    print('------------------------------')
    print("printing content of section 1:")
    print("sec1[:25] : ", ksec1[:25].tolist())
    # print("sec1[hex] : ",[hex(i) for i in ksec1])
    ecmwfbufr.buprs1(ksec1)
    key = np.zeros(46, dtype=np.int)
    sec2_len = ksec2[0]
    print('------------------------------')
    print("length of sec2: ", sec2_len)
    if sec2_len > 0:
        # buukey expands local ECMWF information from section 2 to the key array
        print('------------------------------')
        print("calling buukey")
        ecmwfbufr.buukey(ksec1, ksec2, key, ksup, kerr)
        print("sec2[:25] : ", ksec2[:25].tolist())
        print("printing content of section 2:")
        ecmwfbufr.buprs2(ksup, key)
    else:
        print('skipping section 2 [since it seems unused]')

    # these 4 are filled by the BUS012 call above
    # ksup   = np.zeros(         9, dtype = np.int)
    # ksec0  = np.zeros(         3, dtype = np.int)
    # ksec1  = np.zeros(        40, dtype = np.int)
    # ksec2  = np.zeros(      4096, dtype = np.int)

    print('------------------------------')
    ksec3 = np.zeros(4, dtype=np.int)
    ksec4 = np.zeros(2, dtype=np.int)
    cnames = np.zeros((kelem, 64), dtype=np.character)
    cunits = np.zeros((kelem, 24), dtype=np.character)
    values = np.zeros(kvals, dtype=np.float64)  # this is the default
    cvals = np.zeros((kvals, 80), dtype=np.character)
    kerr = 0

    print("calling: ecmwfbufr.bufrex():")
    ecmwfbufr.bufrex(words, ksup, ksec0, ksec1, ksec2, ksec3, ksec4, cnames,
                     cunits, values, cvals, kerr)
    check_error_flag('ecmwfbufr.bufrex', kerr)

    # print a selection of the decoded numbers
    print('------------------------------')
    print("Decoded BUFR message:")
    print("ksup : ", ksup.tolist())
    print("sec0 : ", ksec0.tolist())
    print("sec1[:25] : ", ksec1[:25].tolist())
    print("sec2[:25] : ", ksec2[:25].tolist())
    print("sec3 : ", ksec3.tolist())
    print("sec4 : ", ksec4.tolist())
    print("cnames [cunits] : ")
    for (i, cnm) in enumerate(cnames):
        cun = cunits[i]
        if python3:
            txtn = ''.join(c.decode() for c in cnm)
            txtu = ''.join(c.decode() for c in cun)
        else:
            txtn = ''.join(c for c in cnm)
            txtu = ''.join(c for c in cun)

        if txtn.strip() != '':
            print('[%3.3i]:%s [%s]' % (i, txtn, txtu))

    print("values[:25] : ", values[:25].tolist())
    print("values[-25:] : ", values[-25:].tolist())
    txt = ''.join(str(v) + ';' for v in values[:20] if v > 0.)
    print("values[:20] : ", txt)

    nsubsets = ksec3[2]  # 361 # number of subsets in this BUFR message

    #not yet used:
    #nelements = ksup[4] # 44 # size of one expanded subset

    lat = np.zeros(nsubsets)
    lon = np.zeros(nsubsets)
    for subs in range(nsubsets):
        # index_lat = nelements*(s-1)+24
        # index_lon = nelements*(s-1)+25
        index_lat = max_nr_expanded_descriptors * (subs - 1) + 24
        index_lon = max_nr_expanded_descriptors * (subs - 1) + 25
        lat[subs] = values[index_lat]
        lon[subs] = values[index_lon]
        if 30 * (subs // 30) == subs:
            print("subs = ", subs, "lat = ", lat[subs], " lon = ", lon[subs])
            print("min/max lat", min(lat), max(lat))
            print("min/max lon", min(lon), max(lon))

    print('------------------------------')
    # busel: fill the descriptor list arrays (only needed for printing)

    # warning: this routine has no inputs, and acts on data stored
    #          during previous library calls
    # Therefore it only produces correct results when either bus0123
    # or bufrex have been called previously on the same bufr message.....
    # However, it is not clear to me why it seems to correctly produce
    # the descriptor lists (both bare and expanded), but yet it does
    # not seem to fill the ktdlen and ktdexl values.

    ktdlen = 0
    ktdlst = np.zeros(max_nr_descriptors, dtype=np.int)
    ktdexl = 0
    ktdexp = np.zeros(max_nr_expanded_descriptors, dtype=np.int)
    kerr = 0

    print("calling: ecmwfbufr.busel():")
    ecmwfbufr.busel(
        ktdlen,  # actual number of data descriptors
        ktdlst,  # list of data descriptors
        ktdexl,  # actual number of expanded data descriptors
        ktdexp,  # list of expanded data descriptors
        kerr)  # error  message
    check_error_flag('ecmwfbufr.busel', kerr)

    print('busel result:')
    print("ktdlen = ", ktdlen)
    print("ktdexl = ", ktdexl)

    selection1 = np.where(ktdlst > 0)
    ktdlen = len(selection1[0])
    selection2 = np.where(ktdexp > 0)
    ktdexl = len(selection2[0])

    print('fixed lengths:')
    print("ktdlen = ", ktdlen)
    print("ktdexl = ", ktdexl)

    print('descriptor lists:')
    print("ktdlst = ", ktdlst[:ktdlen])
    print("ktdexp = ", ktdexp[:ktdexl])

    print('------------------------------')
    print("printing content of section 3:")
    print("sec3 : ", ksec3.tolist())
    ecmwfbufr.buprs3(ksec3, ktdlst, ktdexp, cnames)
def encode(output_bufr_file, RadarData, WMOID):

    # Get the data for the wind profilers

    WindComponentData = RadarData.getWindComponents()

    GateSpaceWidths = RadarData.getGateSpace()

    if len(GateSpaceWidths) > 1:

        MaxWidth = 0
        for Width in GateSpaceWidths:

            if Width > MaxWidth:
                MaxWidth = Width
    else:

        MaxWidth = GateSpaces[0]

    print('Using Pulse Mode ' + str(MaxWidth))

    for DateTime in WindComponentData:

        print('Processing ' + str(DateTime))

        print(MaxWidth)
        print(WindComponentData[DateTime].keys())

        if MaxWidth in WindComponentData[DateTime].keys():
            print('Processing high mode only. Pulse ' + str(MaxWidth)
                  + 'ns')

            WindComponents = WindComponentData[DateTime][MaxWidth]

            # Create tge buffer for the hour block of data.

            bufr = BUFRInterfaceECMWF(verbose=True)

            # fill sections 0, 1, 2 and 3 in the BUFR table

            num_subsets = 1
            bufr.fill_sections_0123(  # ECMWF
                                      # wind profiler . Also know as Message Type (Table A)
                                      # Message sub-type
                                      # L2B processing facility
                bufr_code_centre=59,
                bufr_obstype=2,
                bufr_subtype=7,
                bufr_table_local_version=1,
                bufr_table_master=0,
                bufr_table_master_version=3,
                bufr_code_subcentre=0,
                num_subsets=num_subsets,
                bufr_compression_flag=0,
                )

            bufr.setup_tables()

        # define a descriptor list

            template = BufrTemplate()

            print('adding {0} descriptors'.format(10))

            template.add_descriptors(
                WMO_BLOCK_NUM,
                WMO_STATION_NUM,
                DD_LATITUDE_COARSE_ACCURACY,
                DD_LONGITUDE_COARSE_ACCURACY,
                STATION_HEIGHT,
                YEAR,
                MONTH,
                MDAY,
                HOUR,
                MIN,
                TIME_SIGNIFICANCE,
                TIME_PERIOD,
                DD_WIND_SPEED,
                DD_WIND_DIR,
                DD_PRESSURE,
                DD_TEMPERATURE,
                RAINFALL_SWE,
                DD_RELATIVE_HUMD,
                HEIGHT_INCREMENT,
                WIND_PROFILER_SUB_MODE_INFO,
                HEIGHT_INCREMENT,
                HEIGHT_INCREMENT,
                HEIGHT_INCREMENT,
                )

        # delay replication for the next 10 descriptors

            template.add_replicated_descriptors(
                len(WindComponents),
                WIND_PROFILER_MODE_INFO,
                WIND_PROFILER_QC_RESULTS,
                TOTAL_NUMBER,
                WIND_U_COMPONENT,
                WIND_V_COMPONENT,
                STD_DEV_HORIZONTAL_WIND_SPEED,
                TOTAL_NUMBER,
                RADAR_BACK_SCATTER,
                WIND_W_COMPONENT,
                STD_DEV_VERTICAL_SPEED,
                )

            bufr.register_and_expand_descriptors(template)

        # activate this one if the encoding crashes without clear cause:
        # bufr.estimated_num_bytes_for_encoding = 25000

        # retrieve the length of the expanded descriptor list

            exp_descr_list_length = bufr.ktdexl

        # fill the values array with some dummy varying data

            num_values = exp_descr_list_length

            values = np.zeros(num_values, dtype=np.float64)  # this is the default

        # note: these two must be identical for now, otherwise the
        # python to fortran interface breaks down. This also ofcourse is the
        # cause of the huge memory use of cvals in case num_values is large.

            num_cvalues = num_values
            cvals = np.zeros((num_cvalues, 80), dtype=np.character)

        # note that python starts counting with 0, unlike fortran,
            # so there is no need to take (subset-1)
        #     i = subset*exp_descr_list_length

            values[0] = WMOID[0:2]  # WMO Block Number
            values[1] = WMOID[2:5]  # WMO Station #

            values[2] = RadarData.getLatitude()  # Latitude
            values[3] = RadarData.getLongitude()
            values[4] = RadarData.getElev()  # Elevation of Station (meters)
            values[5] = DateTime.timetuple().tm_year  # year
            values[6] = DateTime.timetuple().tm_mon  # month
            values[7] = DateTime.timetuple().tm_mday  # day
            values[8] = DateTime.timetuple().tm_hour  # hour
            values[9] = 0  # minute
            values[10] = 2  # Time Significance
            values[11] = -60  # Time Period
            values[12] = 1  # Wind Speed
            values[13] = 1  # Wind Dir
            values[14] = 1  # Pressure
            values[15] = 1  # Temperature
            values[16] = .2  # Rainfall
            values[17] = 1  # Realative Humidty
            GateSpace = int(MaxWidth) * 1.e-9 * 3.e+8 / 2.
            values[18] = GateSpace  # Height Increment
            values[19] = 0  # Wind Profiler Sub Mode
            values[20] = GateSpace  # Height Increment
            values[21] = GateSpace  # Height Increment
            values[22] = GateSpace  # Height Increment

            print('Number of gates ' + str(len(WindComponents)))

            for i in range(0, len(WindComponents)):

                for t in range(0, 10):

                    # Calulcate the correct index in the BUFR

                    rec = i * 10 + 23 + t

                    if WindComponents[i][0] <= 1000 \
                        and WindComponents[i][1] <= 1000:
                        WindRadians = math.radians(WindComponents[i][1])
                        VelocityU = -WindComponents[i][0] \
                            * math.sin(WindRadians)
                        VelocityV = -WindComponents[i][0] \
                            * math.cos(WindRadians)
                        TotalNumber = WindComponents[i][2]
                        SnrDB = WindComponents[i][3]
                    else:

                        VelocityU = VelocityV = TotalNumbe = SnrDB = -99
                        values[rec] = float('NaN')  # level mode
                        continue

                    if rec % 10 == 3:
                        values[rec] = 1  # level mode

                    if rec % 10 == 4:
                        values[rec] = 0  # Quality Control test

                    if rec % 10 == 5:
                        values[rec] = TotalNumber  # Total Number (with respect to accumlation or average)

                    if rec % 10 == 6:
                        values[rec] = VelocityU  # U-Component

                    if rec % 10 == 7:
                        values[rec] = VelocityV  # V-Component

                    if rec % 10 == 8:
                        values[rec] = 0.000  # Std Deviation of horizontal wind speed

                    if rec % 10 == 9:
                        values[rec] = TotalNumber  # Total Number  (with respect to accumlation or average)

                    if rec % 10 == 0:
                        values[rec] = SnrDB / 100  # Radar Back Scatter (Peak Power)  x100

                    if rec % 10 == 1:
                        values[rec] = 0.000  # W-Component  x100

                    if rec % 10 == 2:
                        values[rec] = 0.000  # Std Deviation of vertical wind

            # do the encoding to binary format

            bufr.encode_data(values, cvals)

            HeaderString = '''

287

IUAK01 PANC %02d%02d00

''' \
                % (DateTime.timetuple().tm_mday,
                   DateTime.timetuple().tm_hour)

            if not os.path.exists(OutputPath):
                os.makedirs(OutputPath)

            OutputFile = \
                '%s/IUPTO2_%s_%02d%02d00_216234297.bufr.%04d%02d%02d%02d' \
                % (
                OutputPath,
                RadarData.getSiteID().upper(),
                DateTime.timetuple().tm_mon,
                DateTime.timetuple().tm_mday,
                DateTime.timetuple().tm_year,
                DateTime.timetuple().tm_mon,
                DateTime.timetuple().tm_mday,
                DateTime.timetuple().tm_hour,
                )

             # Remove file if exsists

            if os.path.exists(OutputFile):
                os.remove(OutputFile)

            bf1 = open(OutputFile, 'ab')
            bf1.write(HeaderString)
            bf1.close()

            # get an instance of the RawBUFRFile class

            bf1 = RawBUFRFile()

            # open the file for writing

            bf1.open(OutputFile, 'ab')

            # write the encoded BUFR message

            bf1.write_raw_bufr_msg(bufr.encoded_message)

            # close the file

            bf1.close()

            #  #]

            print('succesfully written BUFR encoded data to file: ',
                  OutputFile)
Beispiel #31
0
def encode(output_bufr_file, RadarData, WMOID):

    # Get the data for the wind profilers

    WindComponentData = RadarData.getWindComponents()

    GateSpaceWidths = RadarData.getGateSpace()

    if len(GateSpaceWidths) > 1:

        MaxWidth = 0
        for Width in GateSpaceWidths:

            if Width > MaxWidth:
                MaxWidth = Width
    else:

        MaxWidth = GateSpaces[0]

    print('Using Pulse Mode ' + str(MaxWidth))

    for DateTime in WindComponentData:

        print('Processing ' + str(DateTime))

        print(MaxWidth)
        print(WindComponentData[DateTime].keys())

        if MaxWidth in WindComponentData[DateTime].keys():
            print('Processing high mode only. Pulse ' + str(MaxWidth) + 'ns')

            WindComponents = WindComponentData[DateTime][MaxWidth]

            # Create tge buffer for the hour block of data.

            bufr = BUFRInterfaceECMWF(verbose=True)

            # fill sections 0, 1, 2 and 3 in the BUFR table

            num_subsets = 1
            bufr.fill_sections_0123(  # ECMWF
                # wind profiler . Also know as Message Type (Table A)
                # Message sub-type
                # L2B processing facility
                bufr_code_centre=59,
                bufr_obstype=2,
                bufr_subtype=7,
                bufr_table_local_version=1,
                bufr_table_master=0,
                bufr_table_master_version=3,
                bufr_code_subcentre=0,
                num_subsets=num_subsets,
                bufr_compression_flag=0,
            )

            bufr.setup_tables()

            # define a descriptor list

            template = BufrTemplate()

            print('adding {0} descriptors'.format(10))

            template.add_descriptors(
                WMO_BLOCK_NUM,
                WMO_STATION_NUM,
                DD_LATITUDE_COARSE_ACCURACY,
                DD_LONGITUDE_COARSE_ACCURACY,
                STATION_HEIGHT,
                YEAR,
                MONTH,
                MDAY,
                HOUR,
                MIN,
                TIME_SIGNIFICANCE,
                TIME_PERIOD,
                DD_WIND_SPEED,
                DD_WIND_DIR,
                DD_PRESSURE,
                DD_TEMPERATURE,
                RAINFALL_SWE,
                DD_RELATIVE_HUMD,
                HEIGHT_INCREMENT,
                WIND_PROFILER_SUB_MODE_INFO,
                HEIGHT_INCREMENT,
                HEIGHT_INCREMENT,
                HEIGHT_INCREMENT,
            )

            # delay replication for the next 10 descriptors

            template.add_replicated_descriptors(
                len(WindComponents),
                WIND_PROFILER_MODE_INFO,
                WIND_PROFILER_QC_RESULTS,
                TOTAL_NUMBER,
                WIND_U_COMPONENT,
                WIND_V_COMPONENT,
                STD_DEV_HORIZONTAL_WIND_SPEED,
                TOTAL_NUMBER,
                RADAR_BACK_SCATTER,
                WIND_W_COMPONENT,
                STD_DEV_VERTICAL_SPEED,
            )

            bufr.register_and_expand_descriptors(template)

            # activate this one if the encoding crashes without clear cause:
            # bufr.estimated_num_bytes_for_encoding = 25000

            # retrieve the length of the expanded descriptor list

            exp_descr_list_length = bufr.ktdexl

            # fill the values array with some dummy varying data

            num_values = exp_descr_list_length

            values = np.zeros(num_values,
                              dtype=np.float64)  # this is the default

            # note: these two must be identical for now, otherwise the
            # python to fortran interface breaks down. This also ofcourse is the
            # cause of the huge memory use of cvals in case num_values is large.

            num_cvalues = num_values
            cvals = np.zeros((num_cvalues, 80), dtype=np.character)

            # note that python starts counting with 0, unlike fortran,
            # so there is no need to take (subset-1)
            #     i = subset*exp_descr_list_length

            values[0] = WMOID[0:2]  # WMO Block Number
            values[1] = WMOID[2:5]  # WMO Station #

            values[2] = RadarData.getLatitude()  # Latitude
            values[3] = RadarData.getLongitude()
            values[4] = RadarData.getElev()  # Elevation of Station (meters)
            values[5] = DateTime.timetuple().tm_year  # year
            values[6] = DateTime.timetuple().tm_mon  # month
            values[7] = DateTime.timetuple().tm_mday  # day
            values[8] = DateTime.timetuple().tm_hour  # hour
            values[9] = 0  # minute
            values[10] = 2  # Time Significance
            values[11] = -60  # Time Period
            values[12] = 1  # Wind Speed
            values[13] = 1  # Wind Dir
            values[14] = 1  # Pressure
            values[15] = 1  # Temperature
            values[16] = .2  # Rainfall
            values[17] = 1  # Realative Humidty
            GateSpace = int(MaxWidth) * 1.e-9 * 3.e+8 / 2.
            values[18] = GateSpace  # Height Increment
            values[19] = 0  # Wind Profiler Sub Mode
            values[20] = GateSpace  # Height Increment
            values[21] = GateSpace  # Height Increment
            values[22] = GateSpace  # Height Increment

            print('Number of gates ' + str(len(WindComponents)))

            for i in range(0, len(WindComponents)):

                for t in range(0, 10):

                    # Calulcate the correct index in the BUFR

                    rec = i * 10 + 23 + t

                    if WindComponents[i][0] <= 1000 \
                        and WindComponents[i][1] <= 1000:
                        WindRadians = math.radians(WindComponents[i][1])
                        VelocityU = -WindComponents[i][0] \
                            * math.sin(WindRadians)
                        VelocityV = -WindComponents[i][0] \
                            * math.cos(WindRadians)
                        TotalNumber = WindComponents[i][2]
                        SnrDB = WindComponents[i][3]
                    else:

                        VelocityU = VelocityV = TotalNumbe = SnrDB = -99
                        values[rec] = float('NaN')  # level mode
                        continue

                    if rec % 10 == 3:
                        values[rec] = 1  # level mode

                    if rec % 10 == 4:
                        values[rec] = 0  # Quality Control test

                    if rec % 10 == 5:
                        values[
                            rec] = TotalNumber  # Total Number (with respect to accumlation or average)

                    if rec % 10 == 6:
                        values[rec] = VelocityU  # U-Component

                    if rec % 10 == 7:
                        values[rec] = VelocityV  # V-Component

                    if rec % 10 == 8:
                        values[
                            rec] = 0.000  # Std Deviation of horizontal wind speed

                    if rec % 10 == 9:
                        values[
                            rec] = TotalNumber  # Total Number  (with respect to accumlation or average)

                    if rec % 10 == 0:
                        values[
                            rec] = SnrDB / 100  # Radar Back Scatter (Peak Power)  x100

                    if rec % 10 == 1:
                        values[rec] = 0.000  # W-Component  x100

                    if rec % 10 == 2:
                        values[rec] = 0.000  # Std Deviation of vertical wind

            # do the encoding to binary format

            bufr.encode_data(values, cvals)

            HeaderString = '''

287

IUAK01 PANC %02d%02d00

''' \
                % (DateTime.timetuple().tm_mday,
                   DateTime.timetuple().tm_hour)

            if not os.path.exists(OutputPath):
                os.makedirs(OutputPath)

            OutputFile = \
                '%s/IUPTO2_%s_%02d%02d00_216234297.bufr.%04d%02d%02d%02d' \
                % (
                OutputPath,
                RadarData.getSiteID().upper(),
                DateTime.timetuple().tm_mon,
                DateTime.timetuple().tm_mday,
                DateTime.timetuple().tm_year,
                DateTime.timetuple().tm_mon,
                DateTime.timetuple().tm_mday,
                DateTime.timetuple().tm_hour,
                )

            # Remove file if exsists

            if os.path.exists(OutputFile):
                os.remove(OutputFile)

            bf1 = open(OutputFile, 'ab')
            bf1.write(HeaderString)
            bf1.close()

            # get an instance of the RawBUFRFile class

            bf1 = RawBUFRFile()

            # open the file for writing

            bf1.open(OutputFile, 'ab')

            # write the encoded BUFR message

            bf1.write_raw_bufr_msg(bufr.encoded_message)

            # close the file

            bf1.close()

            #  #]

            print('succesfully written BUFR encoded data to file: ',
                  OutputFile)