def print_bufr_content1(input_bufr_file, output_fd, separator, max_msg_nr):
# #[ implementation 1
"""
example implementation using the BUFRReader class
combined with the get_values_as_2d_array method
"""
# get an instance of the BUFR class
# which automatically opens the file for reading and decodes it
bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
msg_nr = 0
while True:
try:
bob.get_next_msg()
msg_nr += 1
except EOFError:
break
# add header strings
# print 'DEBUG: bob.msg_loaded ',bob.msg_loaded
if bob.msg_loaded == 1:
list_of_names = []
list_of_units = []
list_of_names.extend(bob.get_names())
list_of_units.extend(bob.get_units())
# print 'DEBUG: ',separator.join(list_of_names)
# print 'DEBUG: ',separator.join(list_of_units)
output_fd.write(separator.join(list_of_names) + "\n")
output_fd.write(separator.join(list_of_units) + "\n")
data = bob.get_values_as_2d_array()
# print 'DEBUG: data.shape = ', data.shape
if data.shape[0] * data.shape[1] == 0:
print 'NO DATA FOUND! this seems an empty BUFR message !'
continue
for subs in range(len(data[:, 0])):
output_fd.write(
str(subs) + separator +
separator.join(str(val) for val in data[subs, :]) + "\n")
print 'converted BUFR msg nr. ', msg_nr
if ((max_msg_nr > 0) and (msg_nr >= max_msg_nr)):
print 'skipping remainder of this BUFR file'
break
# close the file
bob.close()
if msg_nr == 0:
print 'no BUFR messages found, are you sure this is a BUFR file?'
def reopen_bufr_file(input_bufr_file):
# #[ open bufr file
'''
open a bufr file for reading and print its content
'''
print '*'*50
from pybufr_ecmwf.bufr import BUFRReader
bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
bob.setup_tables(table_b_to_use='B'+TABLE_NAME,
table_d_to_use='D'+TABLE_NAME)
bob.get_next_msg()
print 'num_subsets: ', bob.get_num_subsets()
if USE_DELAYED_REPLICATION:
data1 = bob.get_subset_values(0)
print 'data1 = ', data1
data2 = bob.get_subset_values(1)
print 'data2 = ', data2
else:
print 'num_elements: ', bob.get_num_elements()
print bob.get_names()
print bob.get_units()
data = bob.get_values_as_2d_array()
print data.shape
print data
print 'bob.bufr_obj.values = '
print bob.bufr_obj.values
textdata = bob.get_value(3, 0)
print 'textdata(3,0)', textdata
textdata = bob.get_value(3, 0, get_cval=True)
print 'textdata(3,0)', textdata
textdata = bob.get_values(3, get_cval=True)
print 'textdata(3,:)', textdata
bob.close()
def reopen_bufr_file(input_bufr_file):
# #[ open bufr file
'''
open a bufr file for reading and print its content
'''
print '*'*50
from pybufr_ecmwf.bufr import BUFRReader
bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
bob.setup_tables(table_b_to_use='B'+TABLE_NAME,
table_d_to_use='D'+TABLE_NAME)
bob.get_next_msg()
print 'num_subsets: ', bob.get_num_subsets()
if USE_DELAYED_REPLICATION:
data1 = bob.get_subset_values(0)
print 'data1 = ', data1
data2 = bob.get_subset_values(1)
print 'data2 = ', data2
else:
print 'num_elements: ', bob.get_num_elements()
print bob.get_names()
print bob.get_units()
data = bob.get_values_as_2d_array()
print data.shape
print data
print 'bob.bufr_obj.values = '
print bob.bufr_obj.values
textdata = bob.get_value(3, 0)
print 'textdata(3,0)', textdata
textdata = bob.get_value(3, 0, get_cval=True)
print 'textdata(3,0)', textdata
textdata = bob.get_values(3, get_cval=True)
print 'textdata(3,:)', textdata
bob.close()
def print_bufr_content1(input_bufr_file, output_fd, separator,
max_msg_nr, expand_flags):
# #[ implementation 1
"""
example implementation using the BUFRReader class
combined with the get_values_as_2d_array method
"""
# get an instance of the BUFR class
# which automatically opens the file for reading and decodes it
bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False,
expand_flags=expand_flags)
msg_nr = 0
while True:
try:
bob.get_next_msg()
msg_nr += 1
except EOFError:
break
# add header strings
# print('DEBUG: bob.msg_loaded ',bob.msg_loaded)
list_of_names = []
list_of_units = []
list_of_names.extend(bob.get_names())
list_of_units.extend(bob.get_units())
list_of_unexp_descr = bob.get_unexp_descr_list()
#print('list_of_names = ',list_of_names)
#print('list_of_units = ',list_of_units)
if bob.msg_loaded == 1:
output_fd.write('"subset nr"'+separator)
if list_of_names:
for name in list_of_names[:-1]:
output_fd.write('"'+name+'"'+separator)
name = list_of_names[-1]
output_fd.write('"'+name+'"\n')
else:
output_fd.write('"[NO DATA]"\n')
output_fd.write('""'+separator)
if list_of_units:
for unit in list_of_units[:-1]:
output_fd.write('"'+unit+'"'+separator)
unit = list_of_units[-1]
output_fd.write('"'+unit+'"\n')
else:
output_fd.write('"[NO DATA]"\n')
list_of_unexp_descr_first_msg = bob.bufr_obj.py_unexp_descr_list
#print('list_of_unexp_descr_first_msg = ',
# list_of_unexp_descr_first_msg)
data = bob.get_values_as_2d_array()
if list_of_unexp_descr != list_of_unexp_descr_first_msg:
print('\n\n')
print('WARNING: it seems different types of BUFR messages')
print('are mixed in this BUFR file, meaning that the list of')
print('descriptor names and units printed on the first 2 output')
print('lines will not match with all lines of data.')
print('To prevent confusion, therefore decoding is halted')
print('It is recommended to first sort BUFR messages by type')
print('before converting them to ascii or csv.')
print('The example script sort_bufr_msgs.py can be used')
print('to sort a BUFR file.')
print('\n\n')
print('Detailed info:')
print('list_of_unexp_descr != list_of_unexp_descr_first_msg !')
print('list_of_unexp_descr = ',
list_of_unexp_descr)
print('list_of_unexp_descr_first_msg = ',
list_of_unexp_descr_first_msg)
sys.exit(0)
if data.shape[0]*data.shape[1] == 0:
print('NO DATA FOUND! this seems an empty BUFR message !')
continue
for subs in range(len(data[:, 0])):
output_fd.write(str(subs+1)+separator+
separator.join(str(val) for val in data[subs, :])+
"\n")
print('converted BUFR msg nr. ', msg_nr)
if (max_msg_nr > 0) and (msg_nr >= max_msg_nr):
print('skipping remainder of this BUFR file')
break
# close the file
bob.close()
if msg_nr == 0:
print('no BUFR messages found, are you sure this is a BUFR file?')
def select_subsets(input_bufr_file, output_bufr_file):
# #[ select on subsets
"""
select data and write out again
"""
# get an instance of the BUFR class
# which automatically opens the file for reading and decodes it
bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
# open the file for writing
rbf_out = RawBUFRFile()
rbf_out.open(output_bufr_file, 'wb')
msg_nr = 0
while True:
try:
bob.get_next_msg()
msg_nr += 1
except EOFError:
break
data = bob.get_values_as_2d_array()
print 'data.shape = ', data.shape
if data.shape[0]*data.shape[1] == 0:
print 'NO DATA FOUND! this seems an empty BUFR message !'
continue
# select every other subset
new_data = data[::2, :]
print 'new_data.shape = ', new_data.shape
#bob.bufr_obj
nsub = bob.bufr_obj.get_num_subsets()
n_exp_descr = len(bob.bufr_obj.values)/nsub
bob.bufr_obj.fill_descriptor_list(nr_of_expanded_descriptors=
n_exp_descr)
bob.bufr_obj.ktdlst = bob.bufr_obj.get_descriptor_list()
delayed_repl_data = bob.bufr_obj.derive_delayed_repl_factors()
bob.bufr_obj.fill_delayed_repl_data(delayed_repl_data)
new_nsub = new_data.shape[0]
bob.bufr_obj.nr_subsets = new_nsub
btm = BufrTemplate()
btm.add_descriptors(*bob.bufr_obj.ktdlst)#[:self.ktdlen])
btm.nr_of_delayed_repl_factors = 1
btm.del_repl_max_nr_of_repeats_list = list(delayed_repl_data)
bob.bufr_obj.register_and_expand_descriptors(btm)
# activate this one if the encoding crashes without clear cause:
# bob.bufr_obj.estimated_num_bytes_for_encoding = 25000
bob.bufr_obj.kdate = new_nsub*list(delayed_repl_data)
print 'bob.bufr_obj.cvals.shape = ', bob.bufr_obj.cvals.shape
bob.bufr_obj.encode_data(new_data, bob.bufr_obj.cvals[:32, :])
rbf_out.write_raw_bufr_msg(bob.bufr_obj.encoded_message)
#for subs in range(len(data[:, 0])):
# output_fd.write(str(subs)+separator+
# separator.join(str(val) for val in data[subs, :])+
# "\n")
print 'converted BUFR msg nr. ', msg_nr
# close the file
bob.close()
if msg_nr == 0:
print 'no BUFR messages found, are you sure this is a BUFR file?'
rbf_out.close()
def select_subsets(input_bufr_file, output_bufr_file):
# #[ select on subsets
"""
select data and write out again
"""
# get an instance of the BUFR class
# which automatically opens the file for reading and decodes it
bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
# open the file for writing
rbf_out = RawBUFRFile()
rbf_out.open(output_bufr_file, 'wb')
msg_nr = 0
while True:
try:
bob.get_next_msg()
msg_nr += 1
except EOFError:
break
data = bob.get_values_as_2d_array()
print('data.shape = ', data.shape)
if data.shape[0] * data.shape[1] == 0:
print('NO DATA FOUND! this seems an empty BUFR message !')
continue
# select every other subset
new_data = data[::2, :]
print('new_data.shape = ', new_data.shape)
#bob.bufr_obj
nsub = bob.bufr_obj.get_num_subsets()
n_exp_descr = len(bob.bufr_obj.values) / nsub
bob.bufr_obj.fill_descriptor_list(
nr_of_expanded_descriptors=n_exp_descr)
bob.bufr_obj.ktdlst = bob.bufr_obj.get_descriptor_list()
delayed_repl_data = bob.bufr_obj.derive_delayed_repl_factors()
bob.bufr_obj.fill_delayed_repl_data(delayed_repl_data)
new_nsub = new_data.shape[0]
bob.bufr_obj.nr_subsets = new_nsub
btm = BufrTemplate()
btm.add_descriptors(*bob.bufr_obj.ktdlst) #[:self.ktdlen])
btm.nr_of_delayed_repl_factors = 1
btm.del_repl_max_nr_of_repeats_list = list(delayed_repl_data)
bob.bufr_obj.register_and_expand_descriptors(btm)
# activate this one if the encoding crashes without clear cause:
# bob.bufr_obj.estimated_num_bytes_for_encoding = 25000
bob.bufr_obj.kdate = new_nsub * list(delayed_repl_data)
print('bob.bufr_obj.cvals.shape = ', bob.bufr_obj.cvals.shape)
bob.bufr_obj.encode_data(new_data, bob.bufr_obj.cvals[:32, :])
rbf_out.write_raw_bufr_msg(bob.bufr_obj.encoded_message)
#for subs in range(len(data[:, 0])):
# output_fd.write(str(subs)+separator+
# separator.join(str(val) for val in data[subs, :])+
# "\n")
print('converted BUFR msg nr. ', msg_nr)
# close the file
bob.close()
if msg_nr == 0:
print('no BUFR messages found, are you sure this is a BUFR file?')
rbf_out.close()
