def create_bufr_template():
# #[ create the template
'''
now use these definitions to create a BUFR template
'''
from pybufr_ecmwf.bufr_template import BufrTemplate
variable3 = 0
template = BufrTemplate()
template.add_descriptor(D_301192) # 4 items
template.add_descriptor(D_301193) # 6 items
if USE_DELAYED_REPLICATION:
max_nr = 5 # max. 5*1 items
template.add_delayed_replic_descriptors(max_nr, [variable3,])
return template
def create_bufr_template():
# #[ create the template
'''
now use these definitions to create a BUFR template
'''
from pybufr_ecmwf.bufr_template import BufrTemplate
variable3 = 0
template = BufrTemplate(verbose=True)
template.add_descriptor(D_301192) # 4 items
template.add_descriptor(D_301193) # 6 items
if USE_DELAYED_REPLICATION:
max_nr = 5 # max. 5*1 items
template.add_delayed_replic_descriptors(max_nr, [variable3,])
return template
def encode(output_bufr_file, RadarData, WMOID):
# Get the data for the wind profilers
WindComponentData = RadarData.getWindComponents()
GateSpaceWidths = RadarData.getGateSpace()
if len(GateSpaceWidths) > 1:
MaxWidth = 0
for Width in GateSpaceWidths:
if Width > MaxWidth:
MaxWidth = Width
else:
MaxWidth = GateSpaces[0]
print('Using Pulse Mode ' + str(MaxWidth))
for DateTime in WindComponentData:
print('Processing ' + str(DateTime))
print(MaxWidth)
print(WindComponentData[DateTime].keys())
if MaxWidth in WindComponentData[DateTime].keys():
print('Processing high mode only. Pulse ' + str(MaxWidth)
+ 'ns')
WindComponents = WindComponentData[DateTime][MaxWidth]
# Create tge buffer for the hour block of data.
bufr = BUFRInterfaceECMWF(verbose=True)
# fill sections 0, 1, 2 and 3 in the BUFR table
num_subsets = 1
bufr.fill_sections_0123( # ECMWF
# wind profiler . Also know as Message Type (Table A)
# Message sub-type
# L2B processing facility
bufr_code_centre=59,
bufr_obstype=2,
bufr_subtype=7,
bufr_table_local_version=1,
bufr_table_master=0,
bufr_table_master_version=3,
bufr_code_subcentre=0,
num_subsets=num_subsets,
bufr_compression_flag=0,
)
bufr.setup_tables()
# define a descriptor list
template = BufrTemplate()
print('adding {0} descriptors'.format(10))
template.add_descriptors(
WMO_BLOCK_NUM,
WMO_STATION_NUM,
DD_LATITUDE_COARSE_ACCURACY,
DD_LONGITUDE_COARSE_ACCURACY,
STATION_HEIGHT,
YEAR,
MONTH,
MDAY,
HOUR,
MIN,
TIME_SIGNIFICANCE,
TIME_PERIOD,
DD_WIND_SPEED,
DD_WIND_DIR,
DD_PRESSURE,
DD_TEMPERATURE,
RAINFALL_SWE,
DD_RELATIVE_HUMD,
HEIGHT_INCREMENT,
WIND_PROFILER_SUB_MODE_INFO,
HEIGHT_INCREMENT,
HEIGHT_INCREMENT,
HEIGHT_INCREMENT,
)
# delay replication for the next 10 descriptors
template.add_replicated_descriptors(
len(WindComponents),
WIND_PROFILER_MODE_INFO,
WIND_PROFILER_QC_RESULTS,
TOTAL_NUMBER,
WIND_U_COMPONENT,
WIND_V_COMPONENT,
STD_DEV_HORIZONTAL_WIND_SPEED,
TOTAL_NUMBER,
RADAR_BACK_SCATTER,
WIND_W_COMPONENT,
STD_DEV_VERTICAL_SPEED,
)
bufr.register_and_expand_descriptors(template)
# activate this one if the encoding crashes without clear cause:
# bufr.estimated_num_bytes_for_encoding = 25000
# retrieve the length of the expanded descriptor list
exp_descr_list_length = bufr.ktdexl
# fill the values array with some dummy varying data
num_values = exp_descr_list_length
values = np.zeros(num_values, dtype=np.float64) # this is the default
# note: these two must be identical for now, otherwise the
# python to fortran interface breaks down. This also ofcourse is the
# cause of the huge memory use of cvals in case num_values is large.
num_cvalues = num_values
cvals = np.zeros((num_cvalues, 80), dtype=np.character)
# note that python starts counting with 0, unlike fortran,
# so there is no need to take (subset-1)
# i = subset*exp_descr_list_length
values[0] = WMOID[0:2] # WMO Block Number
values[1] = WMOID[2:5] # WMO Station #
values[2] = RadarData.getLatitude() # Latitude
values[3] = RadarData.getLongitude()
values[4] = RadarData.getElev() # Elevation of Station (meters)
values[5] = DateTime.timetuple().tm_year # year
values[6] = DateTime.timetuple().tm_mon # month
values[7] = DateTime.timetuple().tm_mday # day
values[8] = DateTime.timetuple().tm_hour # hour
values[9] = 0 # minute
values[10] = 2 # Time Significance
values[11] = -60 # Time Period
values[12] = 1 # Wind Speed
values[13] = 1 # Wind Dir
values[14] = 1 # Pressure
values[15] = 1 # Temperature
values[16] = .2 # Rainfall
values[17] = 1 # Realative Humidty
GateSpace = int(MaxWidth) * 1.e-9 * 3.e+8 / 2.
values[18] = GateSpace # Height Increment
values[19] = 0 # Wind Profiler Sub Mode
values[20] = GateSpace # Height Increment
values[21] = GateSpace # Height Increment
values[22] = GateSpace # Height Increment
print('Number of gates ' + str(len(WindComponents)))
for i in range(0, len(WindComponents)):
for t in range(0, 10):
# Calulcate the correct index in the BUFR
rec = i * 10 + 23 + t
if WindComponents[i][0] <= 1000 \
and WindComponents[i][1] <= 1000:
WindRadians = math.radians(WindComponents[i][1])
VelocityU = -WindComponents[i][0] \
* math.sin(WindRadians)
VelocityV = -WindComponents[i][0] \
* math.cos(WindRadians)
TotalNumber = WindComponents[i][2]
SnrDB = WindComponents[i][3]
else:
VelocityU = VelocityV = TotalNumbe = SnrDB = -99
values[rec] = float('NaN') # level mode
continue
if rec % 10 == 3:
values[rec] = 1 # level mode
if rec % 10 == 4:
values[rec] = 0 # Quality Control test
if rec % 10 == 5:
values[rec] = TotalNumber # Total Number (with respect to accumlation or average)
if rec % 10 == 6:
values[rec] = VelocityU # U-Component
if rec % 10 == 7:
values[rec] = VelocityV # V-Component
if rec % 10 == 8:
values[rec] = 0.000 # Std Deviation of horizontal wind speed
if rec % 10 == 9:
values[rec] = TotalNumber # Total Number (with respect to accumlation or average)
if rec % 10 == 0:
values[rec] = SnrDB / 100 # Radar Back Scatter (Peak Power) x100
if rec % 10 == 1:
values[rec] = 0.000 # W-Component x100
if rec % 10 == 2:
values[rec] = 0.000 # Std Deviation of vertical wind
# do the encoding to binary format
bufr.encode_data(values, cvals)
HeaderString = '''
287
IUAK01 PANC %02d%02d00
''' \
% (DateTime.timetuple().tm_mday,
DateTime.timetuple().tm_hour)
if not os.path.exists(OutputPath):
os.makedirs(OutputPath)
OutputFile = \
'%s/IUPTO2_%s_%02d%02d00_216234297.bufr.%04d%02d%02d%02d' \
% (
OutputPath,
RadarData.getSiteID().upper(),
DateTime.timetuple().tm_mon,
DateTime.timetuple().tm_mday,
DateTime.timetuple().tm_year,
DateTime.timetuple().tm_mon,
DateTime.timetuple().tm_mday,
DateTime.timetuple().tm_hour,
)
# Remove file if exsists
if os.path.exists(OutputFile):
os.remove(OutputFile)
bf1 = open(OutputFile, 'ab')
bf1.write(HeaderString)
bf1.close()
# get an instance of the RawBUFRFile class
bf1 = RawBUFRFile()
# open the file for writing
bf1.open(OutputFile, 'ab')
# write the encoded BUFR message
bf1.write_raw_bufr_msg(bufr.encoded_message)
# close the file
bf1.close()
# #]
print('succesfully written BUFR encoded data to file: ',
OutputFile)
def encoding_example(output_bufr_file):
# #[
"""
wrap the example in a function to circumvent the pylint
convention of requiring capitals for constants in the global
scope (since most of these variables are not constants at all))
"""
bufr = BUFRInterfaceECMWF(verbose=True)
# fill sections 0, 1, 2 and 3
num_subsets = 4
bufr.fill_sections_0123(bufr_code_centre=98, # ECMWF
bufr_obstype=3, # sounding
bufr_subtype=251, # L1B
bufr_table_local_version=1,
bufr_table_master=0,
bufr_table_master_version=15,
bufr_code_subcentre=0, # L2B processing facility
num_subsets=num_subsets,
bufr_compression_flag=0)
# 64=compression/0=no compression
# determine information from sections 0123 to construct the BUFR table
# names expected by the ECMWF BUFR library and create symlinks to the
# default tables if needed
# NOTE: these custom BUFR tables have been created by the
# create_bufr_tables.py example script
bufr.setup_tables(table_b_to_use='B_my_test_BUFR_table.txt',
table_c_to_use='C_my_test_BUFR_table.txt',
table_d_to_use='D_my_test_BUFR_table.txt')
# define a descriptor list
template = BufrTemplate(verbose=True)
template.add_descriptors(DD_B_048001, # 0
DD_B_048002, # 1
DD_D_348001) # 2
bufr.register_and_expand_descriptors(template)
# activate this one if the encoding crashes without clear cause:
# bufr.estimated_num_bytes_for_encoding = 25000
# retrieve the length of the expanded descriptor list
exp_descr_list_length = bufr.ktdexl
print("exp_descr_list_length = ", exp_descr_list_length)
# fill the values array with some dummy varying data
num_values = exp_descr_list_length*num_subsets
values = np.zeros(num_values, dtype=np.float64) # this is the default
# note: these two must be identical for now, otherwise the
# python to fortran interface breaks down. This also ofcourse is the
# cause of the huge memory use of cvals in case num_values is large.
num_cvalues = num_values
cvals = np.zeros((num_cvalues, 80), dtype=np.character)
for subset in range(num_subsets):
# note that python starts counting with 0, unlike fortran,
# so there is no need to take (subset-1)
i = subset*exp_descr_list_length
# fill the message with some dummy data
values[i] = 1.2515 + 0.0011*subset
i = i+1
values[i] = (3.4562 + 0.0012*subset)*1.e-9
i = i+1
values[i] = 1.2625 + 0.0003*subset
i = i+1
values[i] = (3.4561 + 0.0014*subset)*1.e-9
# do the encoding to binary format
bufr.encode_data(values, cvals)
# get an instance of the RawBUFRFile class
bf1 = RawBUFRFile()
# open the file for writing
bf1.open(output_bufr_file, 'wb')
# write the encoded BUFR message
bf1.write_raw_bufr_msg(bufr.encoded_message)
# close the file
bf1.close()
print("D-table:")
print('='*50)
bufr_table_set.print_D_table()
print('='*50)
# define the table name without preceding 'B' or 'D' character
# (which will be prepended by the below write method)
table_name = '_test_table.txt'
bufr_table_set.write_tables(table_name)
# now use these definitions to create a BUFR template
max_nr_of_repeats = 5
num_subsets = 2
num_repetitions = [3, 5] # actual number to be used
template = BufrTemplate(verbose=True)
template.add_descriptor(D_363192) # 1 item
template.del_repl_max_nr_of_repeats_list = [max_nr_of_repeats,]*num_subsets
# and use this BUFR template to create a test BUFR message
bufr = BUFRInterfaceECMWF(verbose=True)
# fill sections 0, 1, 2 and 3
bufr.fill_sections_0123(bufr_code_centre=0,
bufr_obstype=0,
bufr_subtype=0,
bufr_table_local_version=0,
bufr_table_master=0,
bufr_table_master_version=0,
bufr_code_subcentre=0,
num_subsets=num_subsets,
def set_template(self, *args, **kwargs):
# #[ set the template
self.template = BufrTemplate()
# todo: see if it is possible to also allow
# a bufr_template instance as input
for descr in args:
# inputs may be integer, string or a Descriptor instance
# print('adding descriptor: ', descr, ' of type ', type(descr))
self.template.add_descriptor(descr)
if 'max_repl' in kwargs:
#print('max_repl = ', kwargs['max_repl'])
self.template.del_repl_max_nr_of_repeats_list = kwargs['max_repl']
self._bufr_obj.register_and_expand_descriptors(self.template)
# retrieve the length of the expanded descriptor list
exp_descr_list_length = self._bufr_obj.ktdexl
if self.verbose:
print("exp_descr_list_length = ", exp_descr_list_length)
# ensure zeros at the end are removed, so explicitely
# define the end of the slice
exp_descr_list = self._bufr_obj.ktdexp[:exp_descr_list_length]
if self.verbose:
print("exp_descr_list = ", self._bufr_obj.ktdexp)
self.num_fields = exp_descr_list_length
# ensure all descriptors are instances of bufr_table.Descriptor
self.normalised_descriptor_list = \
self._bufr_obj.bt.normalise_descriptor_list(exp_descr_list)
# allocate the needed values and cvalues arrays
self.num_values = self.num_subsets * self.num_fields
self.values = numpy.zeros(self.num_values, dtype=numpy.float64)
# note: float64 is the default but it doesnt hurt to be explicit
if self.verbose:
print("self.num_values = ", self.num_values)
# note: these two must be identical for now, otherwise the
# python to fortran interface breaks down. This also ofcourse is the
# cause of the huge memory use of cvals in case num_values is large.
self.num_cvalues = self.num_values
self.cvals = numpy.zeros((self.num_cvalues, 80), dtype=numpy.character)
self.cvals_index = 0
# dont use this, it is not compatible to python 2.6:
# from collections import OrderedDict
# since I cannot use an orderddict due to missing compatibility
# to python 2.6, I'll use an additional (ordered) list of keys
# fill an ordered dict with field properties for convenience
self.field_properties = {}
self.field_properties_keys = []
for idx, descr in enumerate(self.normalised_descriptor_list):
if descr.unit == 'CCITTIA5':
(min_allowed_num_chars, max_allowed_num_chars,
dummy_var) = descr.get_min_max_step()
p = {
'index': idx,
'name': descr.name,
'min_allowed_num_chars': min_allowed_num_chars,
'max_allowed_num_chars': max_allowed_num_chars
}
else:
(min_allowed_value, max_allowed_value,
step) = descr.get_min_max_step()
p = {
'index': idx,
'name': descr.name,
'min_allowed_value': min_allowed_value,
'max_allowed_value': max_allowed_value,
'step': step
}
self.field_properties[descr.reference] = p
self.field_properties_keys.append(descr.reference)
class BUFRMessage_W:
# #[ bufr msg class for writing
"""
a class that implements iteration over the data in
a given bufr message for reading
"""
def __init__(self,
parent,
num_subsets=1,
verbose=False,
do_range_check=False):
# #[ initialise a message for writing
self.parent = parent
self.num_subsets = num_subsets
self.verbose = verbose
self.do_range_check = do_range_check
self._bufr_obj = BUFRInterfaceECMWF(verbose=verbose)
# fill sections 0, 1, 2 and 3 with default values
self._bufr_obj.fill_sections_0123(
bufr_code_centre=0, # use official WMO tables
bufr_obstype=3, # sounding
bufr_subtype=253, # L2B
bufr_table_local_version=0, # dont use local tables
bufr_table_master=0,
bufr_table_master_version=26, # use latest WMO version
bufr_code_subcentre=0, # L2B processing facility
num_subsets=num_subsets,
bufr_compression_flag=64)
# 64=compression/0=no compression
#table_name = 'default'
# self._bufr_obj.setup_tables(table_b_to_use='B'+table_name,
# table_d_to_use='D'+table_name)
# use information from sections 0123 to construct the BUFR table
# names expected by the ECMWF BUFR library
self._bufr_obj.setup_tables()
# init to None
self.template = None
self.values = None
self.cvals = None
# #]
def set_template(self, *args, **kwargs):
# #[ set the template
self.template = BufrTemplate()
# todo: see if it is possible to also allow
# a bufr_template instance as input
for descr in args:
# inputs may be integer, string or a Descriptor instance
# print('adding descriptor: ', descr, ' of type ', type(descr))
self.template.add_descriptor(descr)
if 'max_repl' in kwargs:
#print('max_repl = ', kwargs['max_repl'])
self.template.del_repl_max_nr_of_repeats_list = kwargs['max_repl']
self._bufr_obj.register_and_expand_descriptors(self.template)
# retrieve the length of the expanded descriptor list
exp_descr_list_length = self._bufr_obj.ktdexl
if self.verbose:
print("exp_descr_list_length = ", exp_descr_list_length)
# ensure zeros at the end are removed, so explicitely
# define the end of the slice
exp_descr_list = self._bufr_obj.ktdexp[:exp_descr_list_length]
if self.verbose:
print("exp_descr_list = ", self._bufr_obj.ktdexp)
self.num_fields = exp_descr_list_length
# ensure all descriptors are instances of bufr_table.Descriptor
self.normalised_descriptor_list = \
self._bufr_obj.bt.normalise_descriptor_list(exp_descr_list)
# allocate the needed values and cvalues arrays
self.num_values = self.num_subsets * self.num_fields
self.values = numpy.zeros(self.num_values, dtype=numpy.float64)
# note: float64 is the default but it doesnt hurt to be explicit
if self.verbose:
print("self.num_values = ", self.num_values)
# note: these two must be identical for now, otherwise the
# python to fortran interface breaks down. This also ofcourse is the
# cause of the huge memory use of cvals in case num_values is large.
self.num_cvalues = self.num_values
self.cvals = numpy.zeros((self.num_cvalues, 80), dtype=numpy.character)
self.cvals_index = 0
# dont use this, it is not compatible to python 2.6:
# from collections import OrderedDict
# since I cannot use an orderddict due to missing compatibility
# to python 2.6, I'll use an additional (ordered) list of keys
# fill an ordered dict with field properties for convenience
self.field_properties = {}
self.field_properties_keys = []
for idx, descr in enumerate(self.normalised_descriptor_list):
if descr.unit == 'CCITTIA5':
(min_allowed_num_chars, max_allowed_num_chars,
dummy_var) = descr.get_min_max_step()
p = {
'index': idx,
'name': descr.name,
'min_allowed_num_chars': min_allowed_num_chars,
'max_allowed_num_chars': max_allowed_num_chars
}
else:
(min_allowed_value, max_allowed_value,
step) = descr.get_min_max_step()
p = {
'index': idx,
'name': descr.name,
'min_allowed_value': min_allowed_value,
'max_allowed_value': max_allowed_value,
'step': step
}
self.field_properties[descr.reference] = p
self.field_properties_keys.append(descr.reference)
# #]
def copy_template_from_bufr_msg(self, msg):
pass
def get_field_names(self):
# #[ request field names
names = []
for key in self.field_properties_keys:
p = self.field_properties[key]
names.append(p['name'])
return names
# #]
def add_subset_data(self, data):
pass
def write_msg_to_file(self):
# #[ write out the current message
# do the encoding to binary format
self._bufr_obj.encode_data(self.values, self.cvals)
# check if file was properly opened
if not self.parent.is_open:
errtxt = 'please open the bufr file before writing data to it!'
raise IncorrectUsageError(errtxt)
# write the encoded BUFR message
self.parent.raw_bf.write_raw_bufr_msg(self._bufr_obj.encoded_message)
# #]
def str_get_index_to_use(self, this_key):
# #[ convert string input for key to index in exp. descr. list
# see if an index is provided
index = -1
if '[' in this_key:
parts = this_key.split('[')
this_key = parts[0]
index_str = parts[1][:-1]
index = int(index_str)
possible_matches = []
names_of_possible_matches = []
try:
reference = int(this_key)
p = self.field_properties[reference]
descr_name = p['name']
except:
# this appears to be not an integer number, so assume
# (part of) the name is given
descr_name = this_key
for key in self.field_properties_keys:
p = self.field_properties[key]
if descr_name in p['name']:
possible_matches.append(key)
names_of_possible_matches.append(p['name'])
# print('possible matches for key: ', possible_matches)
if len(possible_matches) == 1:
# ok, proper location found
key = possible_matches[0]
p = self.field_properties[key]
index_to_use = p['index']
# print('filling row:', p)
elif len(possible_matches) == 0:
errtxt = ('ERROR: the current BUFRmessage does not contain any ' +
'fields that have [{}] in their name.'.format(this_key))
raise IncorrectUsageError(errtxt)
elif index >= 0:
# ok, proper location found since an index was supplied
try:
key = possible_matches[index]
except:
# invalid index
errtxt = ('ERROR: the index on the requested descriptor ' +
'is out of the possible range. ' +
'Only {0} '.format(len(possible_matches)) +
'possible matches are present in this template. ' +
'while the requested index was {} '.format(index) +
'for key {0}.'.format(this_key))
raise IncorrectUsageError(errtxt)
p = self.field_properties[key]
index_to_use = p['index']
# print('filling row:', p)
else:
errtxt = ('ERROR: the current BUFRmessage has multiple ' +
'fields that have [{}] in their name.'.format(this_key) +
' Please add an index to indicate which ' +
'field should be used. Key [{}] matches with {}.'.format(
this_key, names_of_possible_matches))
raise IncorrectUsageError(errtxt)
return index_to_use, p
# #]
def num_get_index_to_use(self, this_key):
# #[ get properties for direct index
# print('self.field_properties_keys = ', self.field_properties_keys)
# print('self.field_properties = ', self.field_properties)
index_to_use = this_key
reference = self.field_properties_keys[this_key]
p = self.field_properties[reference]
return index_to_use, p
# #]
def fill(self, values):
# #[ fill all elements for all subsets using a 2d array
np_values = numpy.array(values)
# check array shape
nfields_data, nsubsets_data = numpy.shape(np_values)
if ((nsubsets_data != self.num_subsets)
or (nfields_data != self.num_fields)):
errtxt = (
'input values array has wrong shape! ' +
'values shape: {0}:{1} '.format(nsubsets_data, nfields_data) +
'but expected shape is: {0}:{1}'.format(
self.num_subsets, self.num_fields))
raise IncorrectUsageError(errtxt)
# fill the requested row with data
for subset in range(self.num_subsets):
i = subset * self.num_fields
for j in range(self.num_fields):
self.values[i + j] = np_values[j, subset]
# #]
def fill_subset(self, isubset, values):
# #[ fill all elements for a given subset
np_values = numpy.array(values)
# check subset index
nfields_data = len(np_values)
if ((isubset < 0) or (isubset >= self.num_subsets)):
errtxt = ('incorrect subset number: {0} '.format(isubset) +
'The subset index must be between {0} and {1}.'.format(
0, self.num_subsets - 1))
raise IncorrectUsageError(errtxt)
# check array length
if (nfields_data != self.num_fields):
errtxt = ('input values array has wrong length! ' +
'values length: {0} '.format(nfields_data) +
'but expected length is: {0}'.format(self.num_fields))
raise IncorrectUsageError(errtxt)
# fill the requested row with data
i = isubset * self.num_fields
for j in range(self.num_fields):
self.values[i + j] = np_values[j]
# #]
def check_and_assign_val(self, this_value, p, j):
# #[ chack range and assign
if self.do_range_check:
# optional, since this may make the code slower
check_range(p, this_value)
self.values[j] = this_value
# #]
def check_and_assign_ascii_val(self, this_value, p, j):
# #[ check length of input string and assign to cvals array
# no need to check this one I guess
#p['min_allowed_num_chars']
max_len = p['max_allowed_num_chars']
if len(this_value) > max_len:
print('WARNING: string is too long and will be truncated',
file=sys.stderr)
print('during encoding of: [{0}]'.format(this_value),
file=sys.stderr)
print(
'Maximum allowed lenght in the current template is: {}'.format(
max_len),
file=sys.stderr)
print('but this string has length: {}'.format(len(this_value)),
file=sys.stderr)
# truncate string
this_value = this_value[:max_len]
# ensure input string has correct length
# and is left aligned
# (if optional right alignment is needed, change < in >)
this_value = '{0:<{width}s}'.format(this_value, width=max_len)
self.cvals[self.cvals_index, :] = ' ' # init with spaces
for ic, c in enumerate(this_value):
self.cvals[self.cvals_index, ic] = c # copy characters
# store the cvals_index for the cvals array in the values
# array, this is needed so the software can find the the
# text string
self.values[j] = ((self.cvals_index + 1) * 1000 + len(this_value))
self.cvals_index = self.cvals_index + 1
# #]
def __setitem__(self, this_key, this_value):
# #[ allow addition of date with dict like interface
# print('searching for: ', this_key)
if type(this_key) is int:
# a direct index to the expanded list of descriptors
# should be given in this case
index_to_use, p = self.num_get_index_to_use(this_key)
elif type(this_key) is str:
index_to_use, p = self.str_get_index_to_use(this_key)
else:
errtxt = 'key has unknown type: {}'.format(type(this_key))
raise IncorrectUsageError(errtxt)
# check if input value is character string
input_is_ccittia5 = False
if type(this_value) is str:
input_is_ccittia5 = True
n = 1
else:
# check length of input (scalar or array?)
try:
n = len(this_value)
try:
if type(this_value[0]) is str:
input_is_ccittia5 = True
except IndexError:
pass
except TypeError:
n = 1
if n != 1:
if n != self.num_subsets:
errtxt = ('Please provide an array of size num_subsets! ' +
'Current array has size {0} '.format(n) +
'but num_subsets is {0}'.format(self.num_subsets))
raise IncorrectUsageError(errtxt)
# fill the requested row with data
for subset in range(self.num_subsets):
i = subset * self.num_fields
j = i + index_to_use
if not input_is_ccittia5:
if n == 1:
self.check_and_assign_val(this_value, p, j)
else:
self.check_and_assign_val(this_value[subset], p, j)
else:
# special case for character strings
if n == 1:
self.check_and_assign_ascii_val(this_value, p, j)
else:
self.check_and_assign_ascii_val(this_value[subset], p, j)
def encoding_example(output_bufr_file):
# #[
"""
wrap the example in a function to circumvent the pylint
convention of requiring capitals for constants in the global
scope (since most of these variables are not constants at all))
"""
bufr = BUFRInterfaceECMWF(verbose=True)
# fill sections 0, 1, 2 and 3
num_subsets = 4
bufr.fill_sections_0123(bufr_code_centre=98, # ECMWF
bufr_obstype=3, # sounding
bufr_subtype=251, # L1B
bufr_table_local_version=1,
bufr_table_master=0,
bufr_table_master_version=15,
bufr_code_subcentre=0, # L2B processing facility
num_subsets=num_subsets,
bufr_compression_flag=0)
# 64=compression/0=no compression
# determine information from sections 0123 to construct the BUFR table
# names expected by the ECMWF BUFR library and create symlinks to the
# default tables if needed
bufr.setup_tables()
# define a descriptor list
template = BufrTemplate()
template.add_descriptors(DD_D_DATE_YYYYMMDD, # 0
DD_D_TIME_HHMM) # 1
# delay replication for the next 2 descriptors
# allow at most 2 delayed replications
template.add_delayed_replic_descriptors(2,
DD_PRESSURE,
DD_TEMPERATURE)
# replicate the next 2 descriptors 3 times
template.add_replicated_descriptors(3,
DD_LATITUDE_HIGH_ACCURACY,
DD_LONGITUDE_HIGH_ACCURACY)
bufr.register_and_expand_descriptors(template)
# activate this one if the encoding crashes without clear cause:
# bufr.estimated_num_bytes_for_encoding = 25000
# retrieve the length of the expanded descriptor list
exp_descr_list_length = bufr.ktdexl
print "exp_descr_list_length = ", exp_descr_list_length
# fill the values array with some dummy varying data
num_values = exp_descr_list_length*num_subsets
values = np.zeros(num_values, dtype=np.float64) # this is the default
# note: these two must be identical for now, otherwise the
# python to fortran interface breaks down. This also ofcourse is the
# cause of the huge memory use of cvals in case num_values is large.
num_cvalues = num_values
cvals = np.zeros((num_cvalues, 80), dtype=np.character)
for subset in range(num_subsets):
# note that python starts counting with 0, unlike fortran,
# so there is no need to take (subset-1)
i = subset*exp_descr_list_length
values[i] = 1999 # year
i = i+1
values[i] = 12 # month
i = i+1
values[i] = 31 # day
i = i+1
values[i] = 23 # hour
i = i+1
values[i] = 59 - subset # minute
i = i+1
values[i] = 2 # delayed replication factor
# this delayed replication factor determines the actual number
# of values to be stored for this particular subset
# even if it is less then the number given in kdata above !
for repl in range(2):
i = i+1
values[i] = 1013.e2 - 100.e2*subset+i+repl # pressure [pa]
i = i+1
values[i] = 273.15 - 10.*subset+i+repl # temperature [K]
for repl in range(3):
i = i+1
values[i] = 51.82 + 0.05*subset+i+repl # latitude
i = i+1
values[i] = 5.25 + 0.1*subset+i+repl # longitude
# do the encoding to binary format
bufr.encode_data(values, cvals)
# get an instance of the RawBUFRFile class
bf1 = RawBUFRFile()
# open the file for writing
bf1.open(output_bufr_file, 'wb')
# write the encoded BUFR message
bf1.write_raw_bufr_msg(bufr.encoded_message)
# close the file
bf1.close()
print('='*50)
bufr_table_set.print_D_table()
print('='*50)
# define the table name without preceding 'B' or 'D' character
# (which will be prepended by the below write method)
table_name = '_test_table.txt'
bufr_table_set.write_tables(table_name)
# now use these definitions to create a BUFR template
max_nr_of_replications = 3
num_subsets = 3
num_replications1 = [1, 2, 3]
num_replications2 = [[1, ], [2, 2, ], [3, 3, 3]]
template = BufrTemplate(verbose=True)
template.add_descriptor(D_363192) # 1 item
template.del_repl_max_nr_of_repeats_list = ([max_nr_of_replications,]*
max_nr_of_replications*num_subsets)
# and use this BUFR template to create a test BUFR message
bufr = BUFRInterfaceECMWF(verbose=True)
# fill sections 0, 1, 2 and 3
bufr.fill_sections_0123(bufr_code_centre=0,
bufr_obstype=0,
bufr_subtype=0,
bufr_table_local_version=0,
bufr_table_master=0,
bufr_table_master_version=0,
bufr_code_subcentre=0,
# (which will be prepended by the below write method)
table_name = '_test_table.txt'
bufr_table_set.write_tables(table_name)
# now use these definitions to create a BUFR template
max_nr_of_replications = 3
num_subsets = 3
num_replications1 = [1, 2, 3]
num_replications2 = [[
1,
], [
2,
2,
], [3, 3, 3]]
template = BufrTemplate(verbose=True)
template.add_descriptor(D_363192) # 1 item
template.del_repl_max_nr_of_repeats_list = ([
max_nr_of_replications,
] * max_nr_of_replications * num_subsets)
# and use this BUFR template to create a test BUFR message
bufr = BUFRInterfaceECMWF(verbose=True)
# fill sections 0, 1, 2 and 3
bufr.fill_sections_0123(bufr_code_centre=0,
bufr_obstype=0,
bufr_subtype=0,
bufr_table_local_version=0,
bufr_table_master=0,
bufr_table_master_version=0,
print('='*50)
bufr_table_set.print_B_table()
print('='*50)
print("D-table:")
print('='*50)
bufr_table_set.print_D_table()
print('='*50)
# define the table name without preceding 'B' or 'D' character
# (which will be prepended by the below write method)
table_name = '_test_table.txt'
bufr_table_set.write_tables(table_name)
# now use these definitions to create a BUFR template
max_nr_of_repeats = 5
template = BufrTemplate(verbose=True)
template.add_descriptor(var1) # 1 item
template.add_delayed_replic_descriptors(max_nr_of_repeats,
D_363192) # max. 1 + 5*5 items
# and use this BUFR template to create a test BUFR message
bufr = BUFRInterfaceECMWF(verbose=True)
# fill sections 0, 1, 2 and 3
num_subsets = 3
bufr.fill_sections_0123(bufr_code_centre=0,
bufr_obstype=0,
bufr_subtype=0,
bufr_table_local_version=0,
bufr_table_master=0,
bufr_table_master_version=0,
def select_subsets(input_bufr_file, output_bufr_file):
# #[ select on subsets
"""
select data and write out again
"""
# get an instance of the BUFR class
# which automatically opens the file for reading and decodes it
bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
# open the file for writing
rbf_out = RawBUFRFile()
rbf_out.open(output_bufr_file, 'wb')
msg_nr = 0
while True:
try:
bob.get_next_msg()
msg_nr += 1
except EOFError:
break
data = bob.get_values_as_2d_array()
print 'data.shape = ', data.shape
if data.shape[0]*data.shape[1] == 0:
print 'NO DATA FOUND! this seems an empty BUFR message !'
continue
# select every other subset
new_data = data[::2, :]
print 'new_data.shape = ', new_data.shape
#bob.bufr_obj
nsub = bob.bufr_obj.get_num_subsets()
n_exp_descr = len(bob.bufr_obj.values)/nsub
bob.bufr_obj.fill_descriptor_list(nr_of_expanded_descriptors=
n_exp_descr)
bob.bufr_obj.ktdlst = bob.bufr_obj.get_descriptor_list()
delayed_repl_data = bob.bufr_obj.derive_delayed_repl_factors()
bob.bufr_obj.fill_delayed_repl_data(delayed_repl_data)
new_nsub = new_data.shape[0]
bob.bufr_obj.nr_subsets = new_nsub
btm = BufrTemplate()
btm.add_descriptors(*bob.bufr_obj.ktdlst)#[:self.ktdlen])
btm.nr_of_delayed_repl_factors = 1
btm.del_repl_max_nr_of_repeats_list = list(delayed_repl_data)
bob.bufr_obj.register_and_expand_descriptors(btm)
# activate this one if the encoding crashes without clear cause:
# bob.bufr_obj.estimated_num_bytes_for_encoding = 25000
bob.bufr_obj.kdate = new_nsub*list(delayed_repl_data)
print 'bob.bufr_obj.cvals.shape = ', bob.bufr_obj.cvals.shape
bob.bufr_obj.encode_data(new_data, bob.bufr_obj.cvals[:32, :])
rbf_out.write_raw_bufr_msg(bob.bufr_obj.encoded_message)
#for subs in range(len(data[:, 0])):
# output_fd.write(str(subs)+separator+
# separator.join(str(val) for val in data[subs, :])+
# "\n")
print 'converted BUFR msg nr. ', msg_nr
# close the file
bob.close()
if msg_nr == 0:
print 'no BUFR messages found, are you sure this is a BUFR file?'
rbf_out.close()
def select_subsets(input_bufr_file, output_bufr_file):
# #[ select on subsets
"""
select data and write out again
"""
# get an instance of the BUFR class
# which automatically opens the file for reading and decodes it
bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False)
# open the file for writing
rbf_out = RawBUFRFile()
rbf_out.open(output_bufr_file, 'wb')
msg_nr = 0
while True:
try:
bob.get_next_msg()
msg_nr += 1
except EOFError:
break
data = bob.get_values_as_2d_array()
print('data.shape = ', data.shape)
if data.shape[0] * data.shape[1] == 0:
print('NO DATA FOUND! this seems an empty BUFR message !')
continue
# select every other subset
new_data = data[::2, :]
print('new_data.shape = ', new_data.shape)
#bob.bufr_obj
nsub = bob.bufr_obj.get_num_subsets()
n_exp_descr = len(bob.bufr_obj.values) / nsub
bob.bufr_obj.fill_descriptor_list(
nr_of_expanded_descriptors=n_exp_descr)
bob.bufr_obj.ktdlst = bob.bufr_obj.get_descriptor_list()
delayed_repl_data = bob.bufr_obj.derive_delayed_repl_factors()
bob.bufr_obj.fill_delayed_repl_data(delayed_repl_data)
new_nsub = new_data.shape[0]
bob.bufr_obj.nr_subsets = new_nsub
btm = BufrTemplate()
btm.add_descriptors(*bob.bufr_obj.ktdlst) #[:self.ktdlen])
btm.nr_of_delayed_repl_factors = 1
btm.del_repl_max_nr_of_repeats_list = list(delayed_repl_data)
bob.bufr_obj.register_and_expand_descriptors(btm)
# activate this one if the encoding crashes without clear cause:
# bob.bufr_obj.estimated_num_bytes_for_encoding = 25000
bob.bufr_obj.kdate = new_nsub * list(delayed_repl_data)
print('bob.bufr_obj.cvals.shape = ', bob.bufr_obj.cvals.shape)
bob.bufr_obj.encode_data(new_data, bob.bufr_obj.cvals[:32, :])
rbf_out.write_raw_bufr_msg(bob.bufr_obj.encoded_message)
#for subs in range(len(data[:, 0])):
# output_fd.write(str(subs)+separator+
# separator.join(str(val) for val in data[subs, :])+
# "\n")
print('converted BUFR msg nr. ', msg_nr)
# close the file
bob.close()
if msg_nr == 0:
print('no BUFR messages found, are you sure this is a BUFR file?')
rbf_out.close()
def encoding_example(output_bufr_file):
# #[
"""
wrap the example in a function to circumvent the pylint
convention of requiring capitals for constants in the global
scope (since most of these variables are not constants at all))
"""
bufr = BUFRInterfaceECMWF(verbose=True)
# fill sections 0, 1, 2 and 3
num_subsets = 4
bufr.fill_sections_0123(
bufr_code_centre=98, # ECMWF
bufr_obstype=3, # sounding
bufr_subtype=251, # L1B
bufr_table_local_version=1,
bufr_table_master=0,
bufr_table_master_version=15,
bufr_code_subcentre=0, # L2B processing facility
num_subsets=num_subsets,
bufr_compression_flag=0)
# 64=compression/0=no compression
# determine information from sections 0123 to construct the BUFR table
# names expected by the ECMWF BUFR library and create symlinks to the
# default tables if needed
bufr.setup_tables()
# define a descriptor list
template = BufrTemplate(verbose=True)
template.add_descriptors(
DD_D_DATE_YYYYMMDD, # 0
DD_D_TIME_HHMM) # 1
# delay replication for the next 2 descriptors
# allow at most 2 delayed replications
template.add_delayed_replic_descriptors(2, DD_PRESSURE, DD_TEMPERATURE)
# replicate the next 2 descriptors 3 times
template.add_replicated_descriptors(3, DD_LATITUDE_HIGH_ACCURACY,
DD_LONGITUDE_HIGH_ACCURACY)
bufr.register_and_expand_descriptors(template)
# activate this one if the encoding crashes without clear cause:
# bufr.estimated_num_bytes_for_encoding = 25000
# retrieve the length of the expanded descriptor list
exp_descr_list_length = bufr.ktdexl
print("exp_descr_list_length = ", exp_descr_list_length)
# fill the values array with some dummy varying data
num_values = exp_descr_list_length * num_subsets
values = np.zeros(num_values, dtype=np.float64) # this is the default
# note: these two must be identical for now, otherwise the
# python to fortran interface breaks down. This also ofcourse is the
# cause of the huge memory use of cvals in case num_values is large.
num_cvalues = num_values
cvals = np.zeros((num_cvalues, 80), dtype=np.character)
for subset in range(num_subsets):
# note that python starts counting with 0, unlike fortran,
# so there is no need to take (subset-1)
i = subset * exp_descr_list_length
values[i] = 1999 # year
i = i + 1
values[i] = 12 # month
i = i + 1
values[i] = 31 # day
i = i + 1
values[i] = 23 # hour
i = i + 1
values[i] = 59 - subset # minute
i = i + 1
values[i] = 2 # delayed replication factor
# this delayed replication factor determines the actual number
# of values to be stored for this particular subset
# even if it is less then the number given in kdata above !
for repl in range(2):
i = i + 1
values[i] = 1013.e2 - 100.e2 * subset + i + repl # pressure [pa]
i = i + 1
values[i] = 273.15 - 10. * subset + i + repl # temperature [K]
for repl in range(3):
i = i + 1
values[i] = 51.82 + 0.05 * subset + i + repl # latitude
i = i + 1
values[i] = 5.25 + 0.1 * subset + i + repl # longitude
# do the encoding to binary format
bufr.encode_data(values, cvals)
# get an instance of the RawBUFRFile class
bf1 = RawBUFRFile()
# open the file for writing
bf1.open(output_bufr_file, 'wb')
# write the encoded BUFR message
bf1.write_raw_bufr_msg(bufr.encoded_message)
# close the file
bf1.close()
def encode(output_bufr_file, RadarData, WMOID):
# Get the data for the wind profilers
WindComponentData = RadarData.getWindComponents()
GateSpaceWidths = RadarData.getGateSpace()
if len(GateSpaceWidths) > 1:
MaxWidth = 0
for Width in GateSpaceWidths:
if Width > MaxWidth:
MaxWidth = Width
else:
MaxWidth = GateSpaces[0]
print('Using Pulse Mode ' + str(MaxWidth))
for DateTime in WindComponentData:
print('Processing ' + str(DateTime))
print(MaxWidth)
print(WindComponentData[DateTime].keys())
if MaxWidth in WindComponentData[DateTime].keys():
print('Processing high mode only. Pulse ' + str(MaxWidth) + 'ns')
WindComponents = WindComponentData[DateTime][MaxWidth]
# Create tge buffer for the hour block of data.
bufr = BUFRInterfaceECMWF(verbose=True)
# fill sections 0, 1, 2 and 3 in the BUFR table
num_subsets = 1
bufr.fill_sections_0123( # ECMWF
# wind profiler . Also know as Message Type (Table A)
# Message sub-type
# L2B processing facility
bufr_code_centre=59,
bufr_obstype=2,
bufr_subtype=7,
bufr_table_local_version=1,
bufr_table_master=0,
bufr_table_master_version=3,
bufr_code_subcentre=0,
num_subsets=num_subsets,
bufr_compression_flag=0,
)
bufr.setup_tables()
# define a descriptor list
template = BufrTemplate()
print('adding {0} descriptors'.format(10))
template.add_descriptors(
WMO_BLOCK_NUM,
WMO_STATION_NUM,
DD_LATITUDE_COARSE_ACCURACY,
DD_LONGITUDE_COARSE_ACCURACY,
STATION_HEIGHT,
YEAR,
MONTH,
MDAY,
HOUR,
MIN,
TIME_SIGNIFICANCE,
TIME_PERIOD,
DD_WIND_SPEED,
DD_WIND_DIR,
DD_PRESSURE,
DD_TEMPERATURE,
RAINFALL_SWE,
DD_RELATIVE_HUMD,
HEIGHT_INCREMENT,
WIND_PROFILER_SUB_MODE_INFO,
HEIGHT_INCREMENT,
HEIGHT_INCREMENT,
HEIGHT_INCREMENT,
)
# delay replication for the next 10 descriptors
template.add_replicated_descriptors(
len(WindComponents),
WIND_PROFILER_MODE_INFO,
WIND_PROFILER_QC_RESULTS,
TOTAL_NUMBER,
WIND_U_COMPONENT,
WIND_V_COMPONENT,
STD_DEV_HORIZONTAL_WIND_SPEED,
TOTAL_NUMBER,
RADAR_BACK_SCATTER,
WIND_W_COMPONENT,
STD_DEV_VERTICAL_SPEED,
)
bufr.register_and_expand_descriptors(template)
# activate this one if the encoding crashes without clear cause:
# bufr.estimated_num_bytes_for_encoding = 25000
# retrieve the length of the expanded descriptor list
exp_descr_list_length = bufr.ktdexl
# fill the values array with some dummy varying data
num_values = exp_descr_list_length
values = np.zeros(num_values,
dtype=np.float64) # this is the default
# note: these two must be identical for now, otherwise the
# python to fortran interface breaks down. This also ofcourse is the
# cause of the huge memory use of cvals in case num_values is large.
num_cvalues = num_values
cvals = np.zeros((num_cvalues, 80), dtype=np.character)
# note that python starts counting with 0, unlike fortran,
# so there is no need to take (subset-1)
# i = subset*exp_descr_list_length
values[0] = WMOID[0:2] # WMO Block Number
values[1] = WMOID[2:5] # WMO Station #
values[2] = RadarData.getLatitude() # Latitude
values[3] = RadarData.getLongitude()
values[4] = RadarData.getElev() # Elevation of Station (meters)
values[5] = DateTime.timetuple().tm_year # year
values[6] = DateTime.timetuple().tm_mon # month
values[7] = DateTime.timetuple().tm_mday # day
values[8] = DateTime.timetuple().tm_hour # hour
values[9] = 0 # minute
values[10] = 2 # Time Significance
values[11] = -60 # Time Period
values[12] = 1 # Wind Speed
values[13] = 1 # Wind Dir
values[14] = 1 # Pressure
values[15] = 1 # Temperature
values[16] = .2 # Rainfall
values[17] = 1 # Realative Humidty
GateSpace = int(MaxWidth) * 1.e-9 * 3.e+8 / 2.
values[18] = GateSpace # Height Increment
values[19] = 0 # Wind Profiler Sub Mode
values[20] = GateSpace # Height Increment
values[21] = GateSpace # Height Increment
values[22] = GateSpace # Height Increment
print('Number of gates ' + str(len(WindComponents)))
for i in range(0, len(WindComponents)):
for t in range(0, 10):
# Calulcate the correct index in the BUFR
rec = i * 10 + 23 + t
if WindComponents[i][0] <= 1000 \
and WindComponents[i][1] <= 1000:
WindRadians = math.radians(WindComponents[i][1])
VelocityU = -WindComponents[i][0] \
* math.sin(WindRadians)
VelocityV = -WindComponents[i][0] \
* math.cos(WindRadians)
TotalNumber = WindComponents[i][2]
SnrDB = WindComponents[i][3]
else:
VelocityU = VelocityV = TotalNumbe = SnrDB = -99
values[rec] = float('NaN') # level mode
continue
if rec % 10 == 3:
values[rec] = 1 # level mode
if rec % 10 == 4:
values[rec] = 0 # Quality Control test
if rec % 10 == 5:
values[
rec] = TotalNumber # Total Number (with respect to accumlation or average)
if rec % 10 == 6:
values[rec] = VelocityU # U-Component
if rec % 10 == 7:
values[rec] = VelocityV # V-Component
if rec % 10 == 8:
values[
rec] = 0.000 # Std Deviation of horizontal wind speed
if rec % 10 == 9:
values[
rec] = TotalNumber # Total Number (with respect to accumlation or average)
if rec % 10 == 0:
values[
rec] = SnrDB / 100 # Radar Back Scatter (Peak Power) x100
if rec % 10 == 1:
values[rec] = 0.000 # W-Component x100
if rec % 10 == 2:
values[rec] = 0.000 # Std Deviation of vertical wind
# do the encoding to binary format
bufr.encode_data(values, cvals)
HeaderString = '''
287
IUAK01 PANC %02d%02d00
''' \
% (DateTime.timetuple().tm_mday,
DateTime.timetuple().tm_hour)
if not os.path.exists(OutputPath):
os.makedirs(OutputPath)
OutputFile = \
'%s/IUPTO2_%s_%02d%02d00_216234297.bufr.%04d%02d%02d%02d' \
% (
OutputPath,
RadarData.getSiteID().upper(),
DateTime.timetuple().tm_mon,
DateTime.timetuple().tm_mday,
DateTime.timetuple().tm_year,
DateTime.timetuple().tm_mon,
DateTime.timetuple().tm_mday,
DateTime.timetuple().tm_hour,
)
# Remove file if exsists
if os.path.exists(OutputFile):
os.remove(OutputFile)
bf1 = open(OutputFile, 'ab')
bf1.write(HeaderString)
bf1.close()
# get an instance of the RawBUFRFile class
bf1 = RawBUFRFile()
# open the file for writing
bf1.open(OutputFile, 'ab')
# write the encoded BUFR message
bf1.write_raw_bufr_msg(bufr.encoded_message)
# close the file
bf1.close()
# #]
print('succesfully written BUFR encoded data to file: ',
OutputFile)
