def raw_file_reading_example(input_bufr_file):
# #[
"""
example for reading a BUFR message
"""
# get an instance of the RawBUFRFile class
rbf = RawBUFRFile()
# open the file for reading, count nr of BUFR messages in it
# and store its content in memory, together with
# an array of pointers to the start and end of each BUFR message
rbf.open(input_bufr_file, 'rb')
# print the internal data of the class instance
rbf.print_properties(prefix="RawBUFRFile (opened for reading)")
# print the number of BUFR messages in the file
num_msgs = rbf.get_num_bufr_msgs()
print "This file contains: ", num_msgs, " BUFR messages."
# sequentially read the raw (undecoded) BUFR messages from the
# class instance
msg1 = rbf.get_next_raw_bufr_msg()[0] # should return proper data
try:
msg2 = rbf.get_next_raw_bufr_msg()[0] # should return corrupted data
except EOFError:
msg2 = None
try:
msg3 = rbf.get_next_raw_bufr_msg()[0] # should return corrupted data
except EOFError:
msg3 = None
print "a warning is expected here:"
# msg4 =
try:
rbf.get_next_raw_bufr_msg()[0] # should raise an EOF error
except EOFError:
print "Warning: EOF reached !"
for i in range(1, num_msgs+1):
# read a selected raw BUFR message from the class instance
raw_data = rbf.get_raw_bufr_msg(i)[0]
print "msg ", i, " got ", len(raw_data), " words"
# close the file
rbf.close()
# delete the class instance
del rbf
return (msg1, msg2, msg3)
def raw_file_reading_example(input_bufr_file):
# #[
"""
example for reading a BUFR message
"""
# get an instance of the RawBUFRFile class
rbf = RawBUFRFile()
# open the file for reading, count nr of BUFR messages in it
# and store its content in memory, together with
# an array of pointers to the start and end of each BUFR message
rbf.open(input_bufr_file, 'rb')
# print the internal data of the class instance
rbf.print_properties(prefix="RawBUFRFile (opened for reading)")
# print the number of BUFR messages in the file
num_msgs = rbf.get_num_bufr_msgs()
print("This file contains: ", num_msgs, " BUFR messages.")
# sequentially read the raw (undecoded) BUFR messages from the
# class instance
msg1 = rbf.get_next_raw_bufr_msg()[0] # should return proper data
try:
msg2 = rbf.get_next_raw_bufr_msg()[0] # should return corrupted data
except EOFError:
msg2 = None
try:
msg3 = rbf.get_next_raw_bufr_msg()[0] # should return corrupted data
except EOFError:
msg3 = None
print("a warning is expected here:")
# msg4 =
try:
rbf.get_next_raw_bufr_msg()[0] # should raise an EOF error
except EOFError:
print("Warning: EOF reached !")
for i in range(1, num_msgs+1):
# read a selected raw BUFR message from the class instance
raw_data = rbf.get_raw_bufr_msg(i)[0]
print("msg ", i, " got ", len(raw_data), " words")
# close the file
rbf.close()
# delete the class instance
del rbf
return (msg1, msg2, msg3)
def decoding_example(input_bufr_file, custom_bufr_tables=None):
# #[
"""
wrap the example in a function to circumvent the pylint
convention of requiring capitals for constants in the global
scope (since most of these variables are not constants at all))
"""
# read the binary data using the BUFRFile class
print('loading testfile: ', input_bufr_file)
rbf = RawBUFRFile(verbose=False)
rbf.open(input_bufr_file, 'rb')
(words, section_sizes, section_start_locations) = \
rbf.get_next_raw_bufr_msg()
rbf.close()
if words is None:
print('No valid BUFR messages found')
sys.exit(0)
print('------------------------------')
bufr = BUFRInterfaceECMWF(encoded_message=words,
section_sizes=section_sizes,
section_start_locations=section_start_locations,
verbose=True)
print("calling: decode_sections_012():")
bufr.decode_sections_012()
print("Metadata for decoded BUFR message:")
bufr.print_sections_012_metadata()
print("calling: setup_tables()")
if custom_bufr_tables:
bufr.setup_tables(table_b_to_use=custom_bufr_tables[0],
table_c_to_use=custom_bufr_tables[1],
table_d_to_use=custom_bufr_tables[2])
else:
bufr.setup_tables()
print("calling: print_sections_012():")
bufr.print_sections_012()
# seems not to work correctly now ...
#bufr.fill_descriptor_list()
#bufr.print_descriptors()
print('------------------------------')
print("calling: bufr.decode_data():")
bufr.decode_data()
bufr.decode_sections_0123()
bufr.fill_descriptor_list_subset(subset=1)
return bufr
def decoding_example(input_bufr_file, custom_bufr_tables=None):
# #[
"""
wrap the example in a function to circumvent the pylint
convention of requiring capitals for constants in the global
scope (since most of these variables are not constants at all))
"""
# read the binary data using the BUFRFile class
print('loading testfile: ', input_bufr_file)
rbf = RawBUFRFile(verbose=False)
rbf.open(input_bufr_file, 'rb')
(words, section_sizes, section_start_locations) = \
rbf.get_next_raw_bufr_msg()
rbf.close()
if words is None:
print('No valid BUFR messages found')
sys.exit(0)
print('------------------------------')
bufr = BUFRInterfaceECMWF(encoded_message=words,
section_sizes=section_sizes,
section_start_locations=section_start_locations,
verbose=True)
print("calling: decode_sections_012():")
bufr.decode_sections_012()
print("Metadata for decoded BUFR message:")
bufr.print_sections_012_metadata()
print("calling: setup_tables()")
if custom_bufr_tables:
bufr.setup_tables(table_b_to_use=custom_bufr_tables[0],
table_c_to_use=custom_bufr_tables[1],
table_d_to_use=custom_bufr_tables[2])
else:
bufr.setup_tables()
print("calling: print_sections_012():")
bufr.print_sections_012()
# seems not to work correctly now ...
#bufr.fill_descriptor_list()
#bufr.print_descriptors()
print('------------------------------')
print("calling: bufr.decode_data():")
bufr.decode_data()
bufr.decode_sections_0123()
bufr.fill_descriptor_list_subset(subset=1)
return bufr
def decoding_example(input_bufr_file):
# #[
"""
wrap the example in a function to circumvent the pylint
convention of requiring capitals for constants in the global
scope (since most of these variables are not constants at all))
"""
rbf = RawBUFRFile()
rbf.open(input_bufr_file, 'rb')
words = rbf.get_next_raw_bufr_msg()[0]
rbf.close()
# define the needed constants
max_nr_descriptors = 20 # 300
max_nr_expanded_descriptors = 140 # 160000
max_nr_subsets = 361 # 25
ktdlen = max_nr_descriptors
# krdlen = max_nr_delayed_replication_factors
kelem = max_nr_expanded_descriptors
kvals = max_nr_expanded_descriptors * max_nr_subsets
# jbufl = max_bufr_msg_size
# jsup = length_ksup
# handle BUFR tables
print('------------------------------')
# define our own location for storing (symlinks to) the BUFR tables
private_bufr_tables_dir = os.path.abspath("./tmp_BUFR_TABLES")
if not os.path.exists(private_bufr_tables_dir):
os.mkdir(private_bufr_tables_dir)
# make the needed symlinks to bufr tables
# inspect the location of the ecmwfbufr.so file, and derive
# from this the location of the BUFR tables that are delivered
# with the ECMWF BUFR library software
ecmwfbufr_path = os.path.split(ecmwfbufr.__file__)[0]
path1 = os.path.join(ecmwfbufr_path, "ecmwf_bufrtables")
path2 = os.path.join(ecmwfbufr_path, '..', "ecmwf_bufrtables")
if os.path.exists(path1):
ecmwf_bufr_tables_dir = path1
elif os.path.exists(path2):
ecmwf_bufr_tables_dir = path2
else:
print("Error: could not find BUFR tables directory")
raise IOError
# make sure the path is absolute, otherwise the ECMWF library
# might fail when it attempts to use it ...
ecmwf_bufr_tables_dir = os.path.abspath(ecmwf_bufr_tables_dir)
needed_b_table = "B0000000000210000001.TXT"
needed_d_table = "D0000000000210000001.TXT"
available_b_table = "B0000000000098013001.TXT"
available_d_table = "D0000000000098013001.TXT"
source = os.path.join(ecmwf_bufr_tables_dir, available_b_table)
destination = os.path.join(private_bufr_tables_dir, needed_b_table)
ensure_symlink_exists(source, destination)
source = os.path.join(ecmwf_bufr_tables_dir, available_d_table)
destination = os.path.join(private_bufr_tables_dir, needed_d_table)
ensure_symlink_exists(source, destination)
# make sure the BUFR tables can be found
# also, force a slash at the end, otherwise the library fails
# to find the tables
env = os.environ
env["BUFR_TABLES"] = private_bufr_tables_dir + os.path.sep
# print('private_bufr_tables_dir+os.path.sep=', \
# private_bufr_tables_dir+os.path.sep)
# redirect all fortran stdout to fileunit 12, so the text
# will end-up in a file called 'fort.12'
# This is needed to get reproducible output for my unittests.
# Without this trick the stdout stream from fortran and the stdout
# stream from c/python may be mixed in inpredictable ways
# (depending on the size of the buffer used to store the streams)
os.environ['STD_OUT'] = '12'
# suppres the default ECMWF welcome message which
# is not yet redirected to the above defined fileunit
os.environ['PRINT_TABLE_NAMES'] = 'FALSE'
ksup = np.zeros(9, dtype=np.int)
ksec0 = np.zeros(3, dtype=np.int)
ksec1 = np.zeros(40, dtype=np.int)
ksec2 = np.zeros(4096, dtype=np.int)
kerr = 0
print("calling: ecmwfbufr.bus012():")
ecmwfbufr.bus012(words, ksup, ksec0, ksec1, ksec2, kerr)
check_error_flag('ecmwfbufr.bus012', kerr)
# convert the arrays to a list before printing, since
# the standard print for numpy arrays is no longer reproducible
# (old versions use spaces, new versions use commas as separator)
print('ksup = ', ksup.tolist())
print('------------------------------')
print("printing content of section 0:")
print("sec0 : ", ksec0.tolist())
# print("sec0[hex] : ",[hex(i) for i in ksec0])
ecmwfbufr.buprs0(ksec0)
print('------------------------------')
print("printing content of section 1:")
print("sec1[:25] : ", ksec1[:25].tolist())
# print("sec1[hex] : ",[hex(i) for i in ksec1])
ecmwfbufr.buprs1(ksec1)
key = np.zeros(46, dtype=np.int)
sec2_len = ksec2[0]
print('------------------------------')
print("length of sec2: ", sec2_len)
if sec2_len > 0:
# buukey expands local ECMWF information from section 2 to the key array
print('------------------------------')
print("calling buukey")
ecmwfbufr.buukey(ksec1, ksec2, key, ksup, kerr)
print("sec2[:25] : ", ksec2[:25].tolist())
print("printing content of section 2:")
ecmwfbufr.buprs2(ksup, key)
else:
print('skipping section 2 [since it seems unused]')
# these 4 are filled by the BUS012 call above
# ksup = np.zeros( 9, dtype = np.int)
# ksec0 = np.zeros( 3, dtype = np.int)
# ksec1 = np.zeros( 40, dtype = np.int)
# ksec2 = np.zeros( 4096, dtype = np.int)
print('------------------------------')
ksec3 = np.zeros(4, dtype=np.int)
ksec4 = np.zeros(2, dtype=np.int)
cnames = np.zeros((kelem, 64), dtype=np.character)
cunits = np.zeros((kelem, 24), dtype=np.character)
values = np.zeros(kvals, dtype=np.float64) # this is the default
cvals = np.zeros((kvals, 80), dtype=np.character)
kerr = 0
print("calling: ecmwfbufr.bufrex():")
ecmwfbufr.bufrex(words, ksup, ksec0, ksec1, ksec2, ksec3, ksec4, cnames,
cunits, values, cvals, kerr)
check_error_flag('ecmwfbufr.bufrex', kerr)
# print a selection of the decoded numbers
print('------------------------------')
print("Decoded BUFR message:")
print("ksup : ", ksup.tolist())
print("sec0 : ", ksec0.tolist())
print("sec1[:25] : ", ksec1[:25].tolist())
print("sec2[:25] : ", ksec2[:25].tolist())
print("sec3 : ", ksec3.tolist())
print("sec4 : ", ksec4.tolist())
print("cnames [cunits] : ")
for (i, cnm) in enumerate(cnames):
cun = cunits[i]
if python3:
txtn = ''.join(c.decode() for c in cnm)
txtu = ''.join(c.decode() for c in cun)
else:
txtn = ''.join(c for c in cnm)
txtu = ''.join(c for c in cun)
if txtn.strip() != '':
print('[%3.3i]:%s [%s]' % (i, txtn, txtu))
print("values[:25] : ", values[:25].tolist())
print("values[-25:] : ", values[-25:].tolist())
txt = ''.join(str(v) + ';' for v in values[:20] if v > 0.)
print("values[:20] : ", txt)
nsubsets = ksec3[2] # 361 # number of subsets in this BUFR message
#not yet used:
#nelements = ksup[4] # 44 # size of one expanded subset
lat = np.zeros(nsubsets)
lon = np.zeros(nsubsets)
for subs in range(nsubsets):
# index_lat = nelements*(s-1)+24
# index_lon = nelements*(s-1)+25
index_lat = max_nr_expanded_descriptors * (subs - 1) + 24
index_lon = max_nr_expanded_descriptors * (subs - 1) + 25
lat[subs] = values[index_lat]
lon[subs] = values[index_lon]
if 30 * (subs // 30) == subs:
print("subs = ", subs, "lat = ", lat[subs], " lon = ", lon[subs])
print("min/max lat", min(lat), max(lat))
print("min/max lon", min(lon), max(lon))
print('------------------------------')
# busel: fill the descriptor list arrays (only needed for printing)
# warning: this routine has no inputs, and acts on data stored
# during previous library calls
# Therefore it only produces correct results when either bus0123
# or bufrex have been called previously on the same bufr message.....
# However, it is not clear to me why it seems to correctly produce
# the descriptor lists (both bare and expanded), but yet it does
# not seem to fill the ktdlen and ktdexl values.
ktdlen = 0
ktdlst = np.zeros(max_nr_descriptors, dtype=np.int)
ktdexl = 0
ktdexp = np.zeros(max_nr_expanded_descriptors, dtype=np.int)
kerr = 0
print("calling: ecmwfbufr.busel():")
ecmwfbufr.busel(
ktdlen, # actual number of data descriptors
ktdlst, # list of data descriptors
ktdexl, # actual number of expanded data descriptors
ktdexp, # list of expanded data descriptors
kerr) # error message
check_error_flag('ecmwfbufr.busel', kerr)
print('busel result:')
print("ktdlen = ", ktdlen)
print("ktdexl = ", ktdexl)
selection1 = np.where(ktdlst > 0)
ktdlen = len(selection1[0])
selection2 = np.where(ktdexp > 0)
ktdexl = len(selection2[0])
print('fixed lengths:')
print("ktdlen = ", ktdlen)
print("ktdexl = ", ktdexl)
print('descriptor lists:')
print("ktdlst = ", ktdlst[:ktdlen])
print("ktdexp = ", ktdexp[:ktdexl])
print('------------------------------')
print("printing content of section 3:")
print("sec3 : ", ksec3.tolist())
ecmwfbufr.buprs3(ksec3, ktdlst, ktdexp, cnames)
def decoding_example(input_bufr_file):
# #[
"""
wrap the example in a function to circumvent the pylint
convention of requiring capitals for constants in the global
scope (since most of these variables are not constants at all))
"""
rbf = RawBUFRFile()
rbf.open(input_bufr_file, 'rb')
words = rbf.get_next_raw_bufr_msg()[0]
rbf.close()
# define the needed constants
max_nr_descriptors = 20 # 300
max_nr_expanded_descriptors = 140 # 160000
max_nr_subsets = 361 # 25
ktdlen = max_nr_descriptors
# krdlen = max_nr_delayed_replication_factors
kelem = max_nr_expanded_descriptors
kvals = max_nr_expanded_descriptors*max_nr_subsets
# jbufl = max_bufr_msg_size
# jsup = length_ksup
# handle BUFR tables
print('------------------------------')
# define our own location for storing (symlinks to) the BUFR tables
private_bufr_tables_dir = os.path.abspath("./tmp_BUFR_TABLES")
if not os.path.exists(private_bufr_tables_dir):
os.mkdir(private_bufr_tables_dir)
# make the needed symlinks to bufr tables
# inspect the location of the ecmwfbufr.so file, and derive
# from this the location of the BUFR tables that are delivered
# with the ECMWF BUFR library software
ecmwfbufr_path = os.path.split(ecmwfbufr.__file__)[0]
path1 = os.path.join(ecmwfbufr_path, "ecmwf_bufrtables")
path2 = os.path.join(ecmwfbufr_path, '..', "ecmwf_bufrtables")
if os.path.exists(path1):
ecmwf_bufr_tables_dir = path1
elif os.path.exists(path2):
ecmwf_bufr_tables_dir = path2
else:
print("Error: could not find BUFR tables directory")
raise IOError
# make sure the path is absolute, otherwise the ECMWF library
# might fail when it attempts to use it ...
ecmwf_bufr_tables_dir = os.path.abspath(ecmwf_bufr_tables_dir)
needed_b_table = "B0000000000210000001.TXT"
needed_d_table = "D0000000000210000001.TXT"
available_b_table = "B0000000000098013001.TXT"
available_d_table = "D0000000000098013001.TXT"
source = os.path.join(ecmwf_bufr_tables_dir, available_b_table)
destination = os.path.join(private_bufr_tables_dir, needed_b_table)
ensure_symlink_exists(source, destination)
source = os.path.join(ecmwf_bufr_tables_dir, available_d_table)
destination = os.path.join(private_bufr_tables_dir, needed_d_table)
ensure_symlink_exists(source, destination)
# make sure the BUFR tables can be found
# also, force a slash at the end, otherwise the library fails
# to find the tables
env = os.environ
env["BUFR_TABLES"] = private_bufr_tables_dir+os.path.sep
# print('private_bufr_tables_dir+os.path.sep=', \
# private_bufr_tables_dir+os.path.sep)
# redirect all fortran stdout to fileunit 12, so the text
# will end-up in a file called 'fort.12'
# This is needed to get reproducible output for my unittests.
# Without this trick the stdout stream from fortran and the stdout
# stream from c/python may be mixed in inpredictable ways
# (depending on the size of the buffer used to store the streams)
os.environ['STD_OUT'] = '12'
# suppres the default ECMWF welcome message which
# is not yet redirected to the above defined fileunit
os.environ['PRINT_TABLE_NAMES'] = 'FALSE'
ksup = np.zeros(9, dtype=np.int)
ksec0 = np.zeros(3, dtype=np.int)
ksec1 = np.zeros(40, dtype=np.int)
ksec2 = np.zeros(4096, dtype=np.int)
kerr = 0
print("calling: ecmwfbufr.bus012():")
ecmwfbufr.bus012(words, ksup, ksec0, ksec1, ksec2, kerr)
check_error_flag('ecmwfbufr.bus012', kerr)
# convert the arrays to a list before printing, since
# the standard print for numpy arrays is no longer reproducible
# (old versions use spaces, new versions use commas as separator)
print('ksup = ', ksup.tolist())
print('------------------------------')
print("printing content of section 0:")
print("sec0 : ", ksec0.tolist())
# print("sec0[hex] : ",[hex(i) for i in ksec0])
ecmwfbufr.buprs0(ksec0)
print('------------------------------')
print("printing content of section 1:")
print("sec1[:25] : ", ksec1[:25].tolist())
# print("sec1[hex] : ",[hex(i) for i in ksec1])
ecmwfbufr.buprs1(ksec1)
key = np.zeros(46, dtype=np.int)
sec2_len = ksec2[0]
print('------------------------------')
print("length of sec2: ", sec2_len)
if sec2_len > 0:
# buukey expands local ECMWF information from section 2 to the key array
print('------------------------------')
print("calling buukey")
ecmwfbufr.buukey(ksec1, ksec2, key, ksup, kerr)
print("sec2[:25] : ", ksec2[:25].tolist())
print("printing content of section 2:")
ecmwfbufr.buprs2(ksup, key)
else:
print('skipping section 2 [since it seems unused]')
# these 4 are filled by the BUS012 call above
# ksup = np.zeros( 9, dtype = np.int)
# ksec0 = np.zeros( 3, dtype = np.int)
# ksec1 = np.zeros( 40, dtype = np.int)
# ksec2 = np.zeros( 4096, dtype = np.int)
print('------------------------------')
ksec3 = np.zeros(4, dtype=np.int)
ksec4 = np.zeros(2, dtype=np.int)
cnames = np.zeros((kelem, 64), dtype=np.character)
cunits = np.zeros((kelem, 24), dtype=np.character)
values = np.zeros(kvals, dtype=np.float64) # this is the default
cvals = np.zeros((kvals, 80), dtype=np.character)
kerr = 0
print("calling: ecmwfbufr.bufrex():")
ecmwfbufr.bufrex(words, ksup, ksec0, ksec1, ksec2, ksec3, ksec4,
cnames, cunits, values, cvals, kerr)
check_error_flag('ecmwfbufr.bufrex', kerr)
# print a selection of the decoded numbers
print('------------------------------')
print("Decoded BUFR message:")
print("ksup : ", ksup.tolist())
print("sec0 : ", ksec0.tolist())
print("sec1[:25] : ", ksec1[:25].tolist())
print("sec2[:25] : ", ksec2[:25].tolist())
print("sec3 : ", ksec3.tolist())
print("sec4 : ", ksec4.tolist())
print("cnames [cunits] : ")
for (i, cnm) in enumerate(cnames):
cun = cunits[i]
if python3:
txtn = ''.join(c.decode() for c in cnm)
txtu = ''.join(c.decode() for c in cun)
else:
txtn = ''.join(c for c in cnm)
txtu = ''.join(c for c in cun)
if txtn.strip() != '':
print('[%3.3i]:%s [%s]' % (i, txtn, txtu))
print("values[:25] : ", values[:25].tolist())
print("values[-25:] : ", values[-25:].tolist())
txt = ''.join(str(v)+';' for v in values[:20] if v > 0.)
print("values[:20] : ", txt)
nsubsets = ksec3[2] # 361 # number of subsets in this BUFR message
#not yet used:
#nelements = ksup[4] # 44 # size of one expanded subset
lat = np.zeros(nsubsets)
lon = np.zeros(nsubsets)
for subs in range(nsubsets):
# index_lat = nelements*(s-1)+24
# index_lon = nelements*(s-1)+25
index_lat = max_nr_expanded_descriptors*(subs-1)+24
index_lon = max_nr_expanded_descriptors*(subs-1)+25
lat[subs] = values[index_lat]
lon[subs] = values[index_lon]
if 30*(subs//30) == subs:
print("subs = ", subs, "lat = ", lat[subs], " lon = ", lon[subs])
print("min/max lat", min(lat), max(lat))
print("min/max lon", min(lon), max(lon))
print('------------------------------')
# busel: fill the descriptor list arrays (only needed for printing)
# warning: this routine has no inputs, and acts on data stored
# during previous library calls
# Therefore it only produces correct results when either bus0123
# or bufrex have been called previously on the same bufr message.....
# However, it is not clear to me why it seems to correctly produce
# the descriptor lists (both bare and expanded), but yet it does
# not seem to fill the ktdlen and ktdexl values.
ktdlen = 0
ktdlst = np.zeros(max_nr_descriptors, dtype=np.int)
ktdexl = 0
ktdexp = np.zeros(max_nr_expanded_descriptors, dtype=np.int)
kerr = 0
print("calling: ecmwfbufr.busel():")
ecmwfbufr.busel(ktdlen, # actual number of data descriptors
ktdlst, # list of data descriptors
ktdexl, # actual number of expanded data descriptors
ktdexp, # list of expanded data descriptors
kerr) # error message
check_error_flag('ecmwfbufr.busel', kerr)
print('busel result:')
print("ktdlen = ", ktdlen)
print("ktdexl = ", ktdexl)
selection1 = np.where(ktdlst > 0)
ktdlen = len(selection1[0])
selection2 = np.where(ktdexp > 0)
ktdexl = len(selection2[0])
print('fixed lengths:')
print("ktdlen = ", ktdlen)
print("ktdexl = ", ktdexl)
print('descriptor lists:')
print("ktdlst = ", ktdlst[:ktdlen])
print("ktdexp = ", ktdexp[:ktdexl])
print('------------------------------')
print("printing content of section 3:")
print("sec3 : ", ksec3.tolist())
ecmwfbufr.buprs3(ksec3, ktdlst, ktdexp, cnames)
