def create_bufr_template(): # #[ create the template ''' now use these definitions to create a BUFR template ''' from pybufr_ecmwf.bufr_template import BufrTemplate variable3 = 0 template = BufrTemplate() template.add_descriptor(D_301192) # 4 items template.add_descriptor(D_301193) # 6 items if USE_DELAYED_REPLICATION: max_nr = 5 # max. 5*1 items template.add_delayed_replic_descriptors(max_nr, [variable3,]) return template
def create_bufr_template(): # #[ create the template ''' now use these definitions to create a BUFR template ''' from pybufr_ecmwf.bufr_template import BufrTemplate variable3 = 0 template = BufrTemplate(verbose=True) template.add_descriptor(D_301192) # 4 items template.add_descriptor(D_301193) # 6 items if USE_DELAYED_REPLICATION: max_nr = 5 # max. 5*1 items template.add_delayed_replic_descriptors(max_nr, [variable3,]) return template
def encode(output_bufr_file, RadarData, WMOID): # Get the data for the wind profilers WindComponentData = RadarData.getWindComponents() GateSpaceWidths = RadarData.getGateSpace() if len(GateSpaceWidths) > 1: MaxWidth = 0 for Width in GateSpaceWidths: if Width > MaxWidth: MaxWidth = Width else: MaxWidth = GateSpaces[0] print('Using Pulse Mode ' + str(MaxWidth)) for DateTime in WindComponentData: print('Processing ' + str(DateTime)) print(MaxWidth) print(WindComponentData[DateTime].keys()) if MaxWidth in WindComponentData[DateTime].keys(): print('Processing high mode only. Pulse ' + str(MaxWidth) + 'ns') WindComponents = WindComponentData[DateTime][MaxWidth] # Create tge buffer for the hour block of data. bufr = BUFRInterfaceECMWF(verbose=True) # fill sections 0, 1, 2 and 3 in the BUFR table num_subsets = 1 bufr.fill_sections_0123( # ECMWF # wind profiler . Also know as Message Type (Table A) # Message sub-type # L2B processing facility bufr_code_centre=59, bufr_obstype=2, bufr_subtype=7, bufr_table_local_version=1, bufr_table_master=0, bufr_table_master_version=3, bufr_code_subcentre=0, num_subsets=num_subsets, bufr_compression_flag=0, ) bufr.setup_tables() # define a descriptor list template = BufrTemplate() print('adding {0} descriptors'.format(10)) template.add_descriptors( WMO_BLOCK_NUM, WMO_STATION_NUM, DD_LATITUDE_COARSE_ACCURACY, DD_LONGITUDE_COARSE_ACCURACY, STATION_HEIGHT, YEAR, MONTH, MDAY, HOUR, MIN, TIME_SIGNIFICANCE, TIME_PERIOD, DD_WIND_SPEED, DD_WIND_DIR, DD_PRESSURE, DD_TEMPERATURE, RAINFALL_SWE, DD_RELATIVE_HUMD, HEIGHT_INCREMENT, WIND_PROFILER_SUB_MODE_INFO, HEIGHT_INCREMENT, HEIGHT_INCREMENT, HEIGHT_INCREMENT, ) # delay replication for the next 10 descriptors template.add_replicated_descriptors( len(WindComponents), WIND_PROFILER_MODE_INFO, WIND_PROFILER_QC_RESULTS, TOTAL_NUMBER, WIND_U_COMPONENT, WIND_V_COMPONENT, STD_DEV_HORIZONTAL_WIND_SPEED, TOTAL_NUMBER, RADAR_BACK_SCATTER, WIND_W_COMPONENT, STD_DEV_VERTICAL_SPEED, ) bufr.register_and_expand_descriptors(template) # activate this one if the encoding crashes without clear cause: # bufr.estimated_num_bytes_for_encoding = 25000 # retrieve the length of the expanded descriptor list exp_descr_list_length = bufr.ktdexl # fill the values array with some dummy varying data num_values = exp_descr_list_length values = np.zeros(num_values, dtype=np.float64) # this is the default # note: these two must be identical for now, otherwise the # python to fortran interface breaks down. This also ofcourse is the # cause of the huge memory use of cvals in case num_values is large. num_cvalues = num_values cvals = np.zeros((num_cvalues, 80), dtype=np.character) # note that python starts counting with 0, unlike fortran, # so there is no need to take (subset-1) # i = subset*exp_descr_list_length values[0] = WMOID[0:2] # WMO Block Number values[1] = WMOID[2:5] # WMO Station # values[2] = RadarData.getLatitude() # Latitude values[3] = RadarData.getLongitude() values[4] = RadarData.getElev() # Elevation of Station (meters) values[5] = DateTime.timetuple().tm_year # year values[6] = DateTime.timetuple().tm_mon # month values[7] = DateTime.timetuple().tm_mday # day values[8] = DateTime.timetuple().tm_hour # hour values[9] = 0 # minute values[10] = 2 # Time Significance values[11] = -60 # Time Period values[12] = 1 # Wind Speed values[13] = 1 # Wind Dir values[14] = 1 # Pressure values[15] = 1 # Temperature values[16] = .2 # Rainfall values[17] = 1 # Realative Humidty GateSpace = int(MaxWidth) * 1.e-9 * 3.e+8 / 2. values[18] = GateSpace # Height Increment values[19] = 0 # Wind Profiler Sub Mode values[20] = GateSpace # Height Increment values[21] = GateSpace # Height Increment values[22] = GateSpace # Height Increment print('Number of gates ' + str(len(WindComponents))) for i in range(0, len(WindComponents)): for t in range(0, 10): # Calulcate the correct index in the BUFR rec = i * 10 + 23 + t if WindComponents[i][0] <= 1000 \ and WindComponents[i][1] <= 1000: WindRadians = math.radians(WindComponents[i][1]) VelocityU = -WindComponents[i][0] \ * math.sin(WindRadians) VelocityV = -WindComponents[i][0] \ * math.cos(WindRadians) TotalNumber = WindComponents[i][2] SnrDB = WindComponents[i][3] else: VelocityU = VelocityV = TotalNumbe = SnrDB = -99 values[rec] = float('NaN') # level mode continue if rec % 10 == 3: values[rec] = 1 # level mode if rec % 10 == 4: values[rec] = 0 # Quality Control test if rec % 10 == 5: values[rec] = TotalNumber # Total Number (with respect to accumlation or average) if rec % 10 == 6: values[rec] = VelocityU # U-Component if rec % 10 == 7: values[rec] = VelocityV # V-Component if rec % 10 == 8: values[rec] = 0.000 # Std Deviation of horizontal wind speed if rec % 10 == 9: values[rec] = TotalNumber # Total Number (with respect to accumlation or average) if rec % 10 == 0: values[rec] = SnrDB / 100 # Radar Back Scatter (Peak Power) x100 if rec % 10 == 1: values[rec] = 0.000 # W-Component x100 if rec % 10 == 2: values[rec] = 0.000 # Std Deviation of vertical wind # do the encoding to binary format bufr.encode_data(values, cvals) HeaderString = ''' 287 IUAK01 PANC %02d%02d00 ''' \ % (DateTime.timetuple().tm_mday, DateTime.timetuple().tm_hour) if not os.path.exists(OutputPath): os.makedirs(OutputPath) OutputFile = \ '%s/IUPTO2_%s_%02d%02d00_216234297.bufr.%04d%02d%02d%02d' \ % ( OutputPath, RadarData.getSiteID().upper(), DateTime.timetuple().tm_mon, DateTime.timetuple().tm_mday, DateTime.timetuple().tm_year, DateTime.timetuple().tm_mon, DateTime.timetuple().tm_mday, DateTime.timetuple().tm_hour, ) # Remove file if exsists if os.path.exists(OutputFile): os.remove(OutputFile) bf1 = open(OutputFile, 'ab') bf1.write(HeaderString) bf1.close() # get an instance of the RawBUFRFile class bf1 = RawBUFRFile() # open the file for writing bf1.open(OutputFile, 'ab') # write the encoded BUFR message bf1.write_raw_bufr_msg(bufr.encoded_message) # close the file bf1.close() # #] print('succesfully written BUFR encoded data to file: ', OutputFile)
def encoding_example(output_bufr_file): # #[ """ wrap the example in a function to circumvent the pylint convention of requiring capitals for constants in the global scope (since most of these variables are not constants at all)) """ bufr = BUFRInterfaceECMWF(verbose=True) # fill sections 0, 1, 2 and 3 num_subsets = 4 bufr.fill_sections_0123(bufr_code_centre=98, # ECMWF bufr_obstype=3, # sounding bufr_subtype=251, # L1B bufr_table_local_version=1, bufr_table_master=0, bufr_table_master_version=15, bufr_code_subcentre=0, # L2B processing facility num_subsets=num_subsets, bufr_compression_flag=0) # 64=compression/0=no compression # determine information from sections 0123 to construct the BUFR table # names expected by the ECMWF BUFR library and create symlinks to the # default tables if needed # NOTE: these custom BUFR tables have been created by the # create_bufr_tables.py example script bufr.setup_tables(table_b_to_use='B_my_test_BUFR_table.txt', table_c_to_use='C_my_test_BUFR_table.txt', table_d_to_use='D_my_test_BUFR_table.txt') # define a descriptor list template = BufrTemplate(verbose=True) template.add_descriptors(DD_B_048001, # 0 DD_B_048002, # 1 DD_D_348001) # 2 bufr.register_and_expand_descriptors(template) # activate this one if the encoding crashes without clear cause: # bufr.estimated_num_bytes_for_encoding = 25000 # retrieve the length of the expanded descriptor list exp_descr_list_length = bufr.ktdexl print("exp_descr_list_length = ", exp_descr_list_length) # fill the values array with some dummy varying data num_values = exp_descr_list_length*num_subsets values = np.zeros(num_values, dtype=np.float64) # this is the default # note: these two must be identical for now, otherwise the # python to fortran interface breaks down. This also ofcourse is the # cause of the huge memory use of cvals in case num_values is large. num_cvalues = num_values cvals = np.zeros((num_cvalues, 80), dtype=np.character) for subset in range(num_subsets): # note that python starts counting with 0, unlike fortran, # so there is no need to take (subset-1) i = subset*exp_descr_list_length # fill the message with some dummy data values[i] = 1.2515 + 0.0011*subset i = i+1 values[i] = (3.4562 + 0.0012*subset)*1.e-9 i = i+1 values[i] = 1.2625 + 0.0003*subset i = i+1 values[i] = (3.4561 + 0.0014*subset)*1.e-9 # do the encoding to binary format bufr.encode_data(values, cvals) # get an instance of the RawBUFRFile class bf1 = RawBUFRFile() # open the file for writing bf1.open(output_bufr_file, 'wb') # write the encoded BUFR message bf1.write_raw_bufr_msg(bufr.encoded_message) # close the file bf1.close()
print("D-table:") print('='*50) bufr_table_set.print_D_table() print('='*50) # define the table name without preceding 'B' or 'D' character # (which will be prepended by the below write method) table_name = '_test_table.txt' bufr_table_set.write_tables(table_name) # now use these definitions to create a BUFR template max_nr_of_repeats = 5 num_subsets = 2 num_repetitions = [3, 5] # actual number to be used template = BufrTemplate(verbose=True) template.add_descriptor(D_363192) # 1 item template.del_repl_max_nr_of_repeats_list = [max_nr_of_repeats,]*num_subsets # and use this BUFR template to create a test BUFR message bufr = BUFRInterfaceECMWF(verbose=True) # fill sections 0, 1, 2 and 3 bufr.fill_sections_0123(bufr_code_centre=0, bufr_obstype=0, bufr_subtype=0, bufr_table_local_version=0, bufr_table_master=0, bufr_table_master_version=0, bufr_code_subcentre=0, num_subsets=num_subsets,
def set_template(self, *args, **kwargs): # #[ set the template self.template = BufrTemplate() # todo: see if it is possible to also allow # a bufr_template instance as input for descr in args: # inputs may be integer, string or a Descriptor instance # print('adding descriptor: ', descr, ' of type ', type(descr)) self.template.add_descriptor(descr) if 'max_repl' in kwargs: #print('max_repl = ', kwargs['max_repl']) self.template.del_repl_max_nr_of_repeats_list = kwargs['max_repl'] self._bufr_obj.register_and_expand_descriptors(self.template) # retrieve the length of the expanded descriptor list exp_descr_list_length = self._bufr_obj.ktdexl if self.verbose: print("exp_descr_list_length = ", exp_descr_list_length) # ensure zeros at the end are removed, so explicitely # define the end of the slice exp_descr_list = self._bufr_obj.ktdexp[:exp_descr_list_length] if self.verbose: print("exp_descr_list = ", self._bufr_obj.ktdexp) self.num_fields = exp_descr_list_length # ensure all descriptors are instances of bufr_table.Descriptor self.normalised_descriptor_list = \ self._bufr_obj.bt.normalise_descriptor_list(exp_descr_list) # allocate the needed values and cvalues arrays self.num_values = self.num_subsets * self.num_fields self.values = numpy.zeros(self.num_values, dtype=numpy.float64) # note: float64 is the default but it doesnt hurt to be explicit if self.verbose: print("self.num_values = ", self.num_values) # note: these two must be identical for now, otherwise the # python to fortran interface breaks down. This also ofcourse is the # cause of the huge memory use of cvals in case num_values is large. self.num_cvalues = self.num_values self.cvals = numpy.zeros((self.num_cvalues, 80), dtype=numpy.character) self.cvals_index = 0 # dont use this, it is not compatible to python 2.6: # from collections import OrderedDict # since I cannot use an orderddict due to missing compatibility # to python 2.6, I'll use an additional (ordered) list of keys # fill an ordered dict with field properties for convenience self.field_properties = {} self.field_properties_keys = [] for idx, descr in enumerate(self.normalised_descriptor_list): if descr.unit == 'CCITTIA5': (min_allowed_num_chars, max_allowed_num_chars, dummy_var) = descr.get_min_max_step() p = { 'index': idx, 'name': descr.name, 'min_allowed_num_chars': min_allowed_num_chars, 'max_allowed_num_chars': max_allowed_num_chars } else: (min_allowed_value, max_allowed_value, step) = descr.get_min_max_step() p = { 'index': idx, 'name': descr.name, 'min_allowed_value': min_allowed_value, 'max_allowed_value': max_allowed_value, 'step': step } self.field_properties[descr.reference] = p self.field_properties_keys.append(descr.reference)
class BUFRMessage_W: # #[ bufr msg class for writing """ a class that implements iteration over the data in a given bufr message for reading """ def __init__(self, parent, num_subsets=1, verbose=False, do_range_check=False): # #[ initialise a message for writing self.parent = parent self.num_subsets = num_subsets self.verbose = verbose self.do_range_check = do_range_check self._bufr_obj = BUFRInterfaceECMWF(verbose=verbose) # fill sections 0, 1, 2 and 3 with default values self._bufr_obj.fill_sections_0123( bufr_code_centre=0, # use official WMO tables bufr_obstype=3, # sounding bufr_subtype=253, # L2B bufr_table_local_version=0, # dont use local tables bufr_table_master=0, bufr_table_master_version=26, # use latest WMO version bufr_code_subcentre=0, # L2B processing facility num_subsets=num_subsets, bufr_compression_flag=64) # 64=compression/0=no compression #table_name = 'default' # self._bufr_obj.setup_tables(table_b_to_use='B'+table_name, # table_d_to_use='D'+table_name) # use information from sections 0123 to construct the BUFR table # names expected by the ECMWF BUFR library self._bufr_obj.setup_tables() # init to None self.template = None self.values = None self.cvals = None # #] def set_template(self, *args, **kwargs): # #[ set the template self.template = BufrTemplate() # todo: see if it is possible to also allow # a bufr_template instance as input for descr in args: # inputs may be integer, string or a Descriptor instance # print('adding descriptor: ', descr, ' of type ', type(descr)) self.template.add_descriptor(descr) if 'max_repl' in kwargs: #print('max_repl = ', kwargs['max_repl']) self.template.del_repl_max_nr_of_repeats_list = kwargs['max_repl'] self._bufr_obj.register_and_expand_descriptors(self.template) # retrieve the length of the expanded descriptor list exp_descr_list_length = self._bufr_obj.ktdexl if self.verbose: print("exp_descr_list_length = ", exp_descr_list_length) # ensure zeros at the end are removed, so explicitely # define the end of the slice exp_descr_list = self._bufr_obj.ktdexp[:exp_descr_list_length] if self.verbose: print("exp_descr_list = ", self._bufr_obj.ktdexp) self.num_fields = exp_descr_list_length # ensure all descriptors are instances of bufr_table.Descriptor self.normalised_descriptor_list = \ self._bufr_obj.bt.normalise_descriptor_list(exp_descr_list) # allocate the needed values and cvalues arrays self.num_values = self.num_subsets * self.num_fields self.values = numpy.zeros(self.num_values, dtype=numpy.float64) # note: float64 is the default but it doesnt hurt to be explicit if self.verbose: print("self.num_values = ", self.num_values) # note: these two must be identical for now, otherwise the # python to fortran interface breaks down. This also ofcourse is the # cause of the huge memory use of cvals in case num_values is large. self.num_cvalues = self.num_values self.cvals = numpy.zeros((self.num_cvalues, 80), dtype=numpy.character) self.cvals_index = 0 # dont use this, it is not compatible to python 2.6: # from collections import OrderedDict # since I cannot use an orderddict due to missing compatibility # to python 2.6, I'll use an additional (ordered) list of keys # fill an ordered dict with field properties for convenience self.field_properties = {} self.field_properties_keys = [] for idx, descr in enumerate(self.normalised_descriptor_list): if descr.unit == 'CCITTIA5': (min_allowed_num_chars, max_allowed_num_chars, dummy_var) = descr.get_min_max_step() p = { 'index': idx, 'name': descr.name, 'min_allowed_num_chars': min_allowed_num_chars, 'max_allowed_num_chars': max_allowed_num_chars } else: (min_allowed_value, max_allowed_value, step) = descr.get_min_max_step() p = { 'index': idx, 'name': descr.name, 'min_allowed_value': min_allowed_value, 'max_allowed_value': max_allowed_value, 'step': step } self.field_properties[descr.reference] = p self.field_properties_keys.append(descr.reference) # #] def copy_template_from_bufr_msg(self, msg): pass def get_field_names(self): # #[ request field names names = [] for key in self.field_properties_keys: p = self.field_properties[key] names.append(p['name']) return names # #] def add_subset_data(self, data): pass def write_msg_to_file(self): # #[ write out the current message # do the encoding to binary format self._bufr_obj.encode_data(self.values, self.cvals) # check if file was properly opened if not self.parent.is_open: errtxt = 'please open the bufr file before writing data to it!' raise IncorrectUsageError(errtxt) # write the encoded BUFR message self.parent.raw_bf.write_raw_bufr_msg(self._bufr_obj.encoded_message) # #] def str_get_index_to_use(self, this_key): # #[ convert string input for key to index in exp. descr. list # see if an index is provided index = -1 if '[' in this_key: parts = this_key.split('[') this_key = parts[0] index_str = parts[1][:-1] index = int(index_str) possible_matches = [] names_of_possible_matches = [] try: reference = int(this_key) p = self.field_properties[reference] descr_name = p['name'] except: # this appears to be not an integer number, so assume # (part of) the name is given descr_name = this_key for key in self.field_properties_keys: p = self.field_properties[key] if descr_name in p['name']: possible_matches.append(key) names_of_possible_matches.append(p['name']) # print('possible matches for key: ', possible_matches) if len(possible_matches) == 1: # ok, proper location found key = possible_matches[0] p = self.field_properties[key] index_to_use = p['index'] # print('filling row:', p) elif len(possible_matches) == 0: errtxt = ('ERROR: the current BUFRmessage does not contain any ' + 'fields that have [{}] in their name.'.format(this_key)) raise IncorrectUsageError(errtxt) elif index >= 0: # ok, proper location found since an index was supplied try: key = possible_matches[index] except: # invalid index errtxt = ('ERROR: the index on the requested descriptor ' + 'is out of the possible range. ' + 'Only {0} '.format(len(possible_matches)) + 'possible matches are present in this template. ' + 'while the requested index was {} '.format(index) + 'for key {0}.'.format(this_key)) raise IncorrectUsageError(errtxt) p = self.field_properties[key] index_to_use = p['index'] # print('filling row:', p) else: errtxt = ('ERROR: the current BUFRmessage has multiple ' + 'fields that have [{}] in their name.'.format(this_key) + ' Please add an index to indicate which ' + 'field should be used. Key [{}] matches with {}.'.format( this_key, names_of_possible_matches)) raise IncorrectUsageError(errtxt) return index_to_use, p # #] def num_get_index_to_use(self, this_key): # #[ get properties for direct index # print('self.field_properties_keys = ', self.field_properties_keys) # print('self.field_properties = ', self.field_properties) index_to_use = this_key reference = self.field_properties_keys[this_key] p = self.field_properties[reference] return index_to_use, p # #] def fill(self, values): # #[ fill all elements for all subsets using a 2d array np_values = numpy.array(values) # check array shape nfields_data, nsubsets_data = numpy.shape(np_values) if ((nsubsets_data != self.num_subsets) or (nfields_data != self.num_fields)): errtxt = ( 'input values array has wrong shape! ' + 'values shape: {0}:{1} '.format(nsubsets_data, nfields_data) + 'but expected shape is: {0}:{1}'.format( self.num_subsets, self.num_fields)) raise IncorrectUsageError(errtxt) # fill the requested row with data for subset in range(self.num_subsets): i = subset * self.num_fields for j in range(self.num_fields): self.values[i + j] = np_values[j, subset] # #] def fill_subset(self, isubset, values): # #[ fill all elements for a given subset np_values = numpy.array(values) # check subset index nfields_data = len(np_values) if ((isubset < 0) or (isubset >= self.num_subsets)): errtxt = ('incorrect subset number: {0} '.format(isubset) + 'The subset index must be between {0} and {1}.'.format( 0, self.num_subsets - 1)) raise IncorrectUsageError(errtxt) # check array length if (nfields_data != self.num_fields): errtxt = ('input values array has wrong length! ' + 'values length: {0} '.format(nfields_data) + 'but expected length is: {0}'.format(self.num_fields)) raise IncorrectUsageError(errtxt) # fill the requested row with data i = isubset * self.num_fields for j in range(self.num_fields): self.values[i + j] = np_values[j] # #] def check_and_assign_val(self, this_value, p, j): # #[ chack range and assign if self.do_range_check: # optional, since this may make the code slower check_range(p, this_value) self.values[j] = this_value # #] def check_and_assign_ascii_val(self, this_value, p, j): # #[ check length of input string and assign to cvals array # no need to check this one I guess #p['min_allowed_num_chars'] max_len = p['max_allowed_num_chars'] if len(this_value) > max_len: print('WARNING: string is too long and will be truncated', file=sys.stderr) print('during encoding of: [{0}]'.format(this_value), file=sys.stderr) print( 'Maximum allowed lenght in the current template is: {}'.format( max_len), file=sys.stderr) print('but this string has length: {}'.format(len(this_value)), file=sys.stderr) # truncate string this_value = this_value[:max_len] # ensure input string has correct length # and is left aligned # (if optional right alignment is needed, change < in >) this_value = '{0:<{width}s}'.format(this_value, width=max_len) self.cvals[self.cvals_index, :] = ' ' # init with spaces for ic, c in enumerate(this_value): self.cvals[self.cvals_index, ic] = c # copy characters # store the cvals_index for the cvals array in the values # array, this is needed so the software can find the the # text string self.values[j] = ((self.cvals_index + 1) * 1000 + len(this_value)) self.cvals_index = self.cvals_index + 1 # #] def __setitem__(self, this_key, this_value): # #[ allow addition of date with dict like interface # print('searching for: ', this_key) if type(this_key) is int: # a direct index to the expanded list of descriptors # should be given in this case index_to_use, p = self.num_get_index_to_use(this_key) elif type(this_key) is str: index_to_use, p = self.str_get_index_to_use(this_key) else: errtxt = 'key has unknown type: {}'.format(type(this_key)) raise IncorrectUsageError(errtxt) # check if input value is character string input_is_ccittia5 = False if type(this_value) is str: input_is_ccittia5 = True n = 1 else: # check length of input (scalar or array?) try: n = len(this_value) try: if type(this_value[0]) is str: input_is_ccittia5 = True except IndexError: pass except TypeError: n = 1 if n != 1: if n != self.num_subsets: errtxt = ('Please provide an array of size num_subsets! ' + 'Current array has size {0} '.format(n) + 'but num_subsets is {0}'.format(self.num_subsets)) raise IncorrectUsageError(errtxt) # fill the requested row with data for subset in range(self.num_subsets): i = subset * self.num_fields j = i + index_to_use if not input_is_ccittia5: if n == 1: self.check_and_assign_val(this_value, p, j) else: self.check_and_assign_val(this_value[subset], p, j) else: # special case for character strings if n == 1: self.check_and_assign_ascii_val(this_value, p, j) else: self.check_and_assign_ascii_val(this_value[subset], p, j)
def encoding_example(output_bufr_file): # #[ """ wrap the example in a function to circumvent the pylint convention of requiring capitals for constants in the global scope (since most of these variables are not constants at all)) """ bufr = BUFRInterfaceECMWF(verbose=True) # fill sections 0, 1, 2 and 3 num_subsets = 4 bufr.fill_sections_0123(bufr_code_centre=98, # ECMWF bufr_obstype=3, # sounding bufr_subtype=251, # L1B bufr_table_local_version=1, bufr_table_master=0, bufr_table_master_version=15, bufr_code_subcentre=0, # L2B processing facility num_subsets=num_subsets, bufr_compression_flag=0) # 64=compression/0=no compression # determine information from sections 0123 to construct the BUFR table # names expected by the ECMWF BUFR library and create symlinks to the # default tables if needed bufr.setup_tables() # define a descriptor list template = BufrTemplate() template.add_descriptors(DD_D_DATE_YYYYMMDD, # 0 DD_D_TIME_HHMM) # 1 # delay replication for the next 2 descriptors # allow at most 2 delayed replications template.add_delayed_replic_descriptors(2, DD_PRESSURE, DD_TEMPERATURE) # replicate the next 2 descriptors 3 times template.add_replicated_descriptors(3, DD_LATITUDE_HIGH_ACCURACY, DD_LONGITUDE_HIGH_ACCURACY) bufr.register_and_expand_descriptors(template) # activate this one if the encoding crashes without clear cause: # bufr.estimated_num_bytes_for_encoding = 25000 # retrieve the length of the expanded descriptor list exp_descr_list_length = bufr.ktdexl print "exp_descr_list_length = ", exp_descr_list_length # fill the values array with some dummy varying data num_values = exp_descr_list_length*num_subsets values = np.zeros(num_values, dtype=np.float64) # this is the default # note: these two must be identical for now, otherwise the # python to fortran interface breaks down. This also ofcourse is the # cause of the huge memory use of cvals in case num_values is large. num_cvalues = num_values cvals = np.zeros((num_cvalues, 80), dtype=np.character) for subset in range(num_subsets): # note that python starts counting with 0, unlike fortran, # so there is no need to take (subset-1) i = subset*exp_descr_list_length values[i] = 1999 # year i = i+1 values[i] = 12 # month i = i+1 values[i] = 31 # day i = i+1 values[i] = 23 # hour i = i+1 values[i] = 59 - subset # minute i = i+1 values[i] = 2 # delayed replication factor # this delayed replication factor determines the actual number # of values to be stored for this particular subset # even if it is less then the number given in kdata above ! for repl in range(2): i = i+1 values[i] = 1013.e2 - 100.e2*subset+i+repl # pressure [pa] i = i+1 values[i] = 273.15 - 10.*subset+i+repl # temperature [K] for repl in range(3): i = i+1 values[i] = 51.82 + 0.05*subset+i+repl # latitude i = i+1 values[i] = 5.25 + 0.1*subset+i+repl # longitude # do the encoding to binary format bufr.encode_data(values, cvals) # get an instance of the RawBUFRFile class bf1 = RawBUFRFile() # open the file for writing bf1.open(output_bufr_file, 'wb') # write the encoded BUFR message bf1.write_raw_bufr_msg(bufr.encoded_message) # close the file bf1.close()
print('='*50) bufr_table_set.print_D_table() print('='*50) # define the table name without preceding 'B' or 'D' character # (which will be prepended by the below write method) table_name = '_test_table.txt' bufr_table_set.write_tables(table_name) # now use these definitions to create a BUFR template max_nr_of_replications = 3 num_subsets = 3 num_replications1 = [1, 2, 3] num_replications2 = [[1, ], [2, 2, ], [3, 3, 3]] template = BufrTemplate(verbose=True) template.add_descriptor(D_363192) # 1 item template.del_repl_max_nr_of_repeats_list = ([max_nr_of_replications,]* max_nr_of_replications*num_subsets) # and use this BUFR template to create a test BUFR message bufr = BUFRInterfaceECMWF(verbose=True) # fill sections 0, 1, 2 and 3 bufr.fill_sections_0123(bufr_code_centre=0, bufr_obstype=0, bufr_subtype=0, bufr_table_local_version=0, bufr_table_master=0, bufr_table_master_version=0, bufr_code_subcentre=0,
# (which will be prepended by the below write method) table_name = '_test_table.txt' bufr_table_set.write_tables(table_name) # now use these definitions to create a BUFR template max_nr_of_replications = 3 num_subsets = 3 num_replications1 = [1, 2, 3] num_replications2 = [[ 1, ], [ 2, 2, ], [3, 3, 3]] template = BufrTemplate(verbose=True) template.add_descriptor(D_363192) # 1 item template.del_repl_max_nr_of_repeats_list = ([ max_nr_of_replications, ] * max_nr_of_replications * num_subsets) # and use this BUFR template to create a test BUFR message bufr = BUFRInterfaceECMWF(verbose=True) # fill sections 0, 1, 2 and 3 bufr.fill_sections_0123(bufr_code_centre=0, bufr_obstype=0, bufr_subtype=0, bufr_table_local_version=0, bufr_table_master=0, bufr_table_master_version=0,
print('='*50) bufr_table_set.print_B_table() print('='*50) print("D-table:") print('='*50) bufr_table_set.print_D_table() print('='*50) # define the table name without preceding 'B' or 'D' character # (which will be prepended by the below write method) table_name = '_test_table.txt' bufr_table_set.write_tables(table_name) # now use these definitions to create a BUFR template max_nr_of_repeats = 5 template = BufrTemplate(verbose=True) template.add_descriptor(var1) # 1 item template.add_delayed_replic_descriptors(max_nr_of_repeats, D_363192) # max. 1 + 5*5 items # and use this BUFR template to create a test BUFR message bufr = BUFRInterfaceECMWF(verbose=True) # fill sections 0, 1, 2 and 3 num_subsets = 3 bufr.fill_sections_0123(bufr_code_centre=0, bufr_obstype=0, bufr_subtype=0, bufr_table_local_version=0, bufr_table_master=0, bufr_table_master_version=0,
def select_subsets(input_bufr_file, output_bufr_file): # #[ select on subsets """ select data and write out again """ # get an instance of the BUFR class # which automatically opens the file for reading and decodes it bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False) # open the file for writing rbf_out = RawBUFRFile() rbf_out.open(output_bufr_file, 'wb') msg_nr = 0 while True: try: bob.get_next_msg() msg_nr += 1 except EOFError: break data = bob.get_values_as_2d_array() print 'data.shape = ', data.shape if data.shape[0]*data.shape[1] == 0: print 'NO DATA FOUND! this seems an empty BUFR message !' continue # select every other subset new_data = data[::2, :] print 'new_data.shape = ', new_data.shape #bob.bufr_obj nsub = bob.bufr_obj.get_num_subsets() n_exp_descr = len(bob.bufr_obj.values)/nsub bob.bufr_obj.fill_descriptor_list(nr_of_expanded_descriptors= n_exp_descr) bob.bufr_obj.ktdlst = bob.bufr_obj.get_descriptor_list() delayed_repl_data = bob.bufr_obj.derive_delayed_repl_factors() bob.bufr_obj.fill_delayed_repl_data(delayed_repl_data) new_nsub = new_data.shape[0] bob.bufr_obj.nr_subsets = new_nsub btm = BufrTemplate() btm.add_descriptors(*bob.bufr_obj.ktdlst)#[:self.ktdlen]) btm.nr_of_delayed_repl_factors = 1 btm.del_repl_max_nr_of_repeats_list = list(delayed_repl_data) bob.bufr_obj.register_and_expand_descriptors(btm) # activate this one if the encoding crashes without clear cause: # bob.bufr_obj.estimated_num_bytes_for_encoding = 25000 bob.bufr_obj.kdate = new_nsub*list(delayed_repl_data) print 'bob.bufr_obj.cvals.shape = ', bob.bufr_obj.cvals.shape bob.bufr_obj.encode_data(new_data, bob.bufr_obj.cvals[:32, :]) rbf_out.write_raw_bufr_msg(bob.bufr_obj.encoded_message) #for subs in range(len(data[:, 0])): # output_fd.write(str(subs)+separator+ # separator.join(str(val) for val in data[subs, :])+ # "\n") print 'converted BUFR msg nr. ', msg_nr # close the file bob.close() if msg_nr == 0: print 'no BUFR messages found, are you sure this is a BUFR file?' rbf_out.close()
def select_subsets(input_bufr_file, output_bufr_file): # #[ select on subsets """ select data and write out again """ # get an instance of the BUFR class # which automatically opens the file for reading and decodes it bob = BUFRReader(input_bufr_file, warn_about_bufr_size=False) # open the file for writing rbf_out = RawBUFRFile() rbf_out.open(output_bufr_file, 'wb') msg_nr = 0 while True: try: bob.get_next_msg() msg_nr += 1 except EOFError: break data = bob.get_values_as_2d_array() print('data.shape = ', data.shape) if data.shape[0] * data.shape[1] == 0: print('NO DATA FOUND! this seems an empty BUFR message !') continue # select every other subset new_data = data[::2, :] print('new_data.shape = ', new_data.shape) #bob.bufr_obj nsub = bob.bufr_obj.get_num_subsets() n_exp_descr = len(bob.bufr_obj.values) / nsub bob.bufr_obj.fill_descriptor_list( nr_of_expanded_descriptors=n_exp_descr) bob.bufr_obj.ktdlst = bob.bufr_obj.get_descriptor_list() delayed_repl_data = bob.bufr_obj.derive_delayed_repl_factors() bob.bufr_obj.fill_delayed_repl_data(delayed_repl_data) new_nsub = new_data.shape[0] bob.bufr_obj.nr_subsets = new_nsub btm = BufrTemplate() btm.add_descriptors(*bob.bufr_obj.ktdlst) #[:self.ktdlen]) btm.nr_of_delayed_repl_factors = 1 btm.del_repl_max_nr_of_repeats_list = list(delayed_repl_data) bob.bufr_obj.register_and_expand_descriptors(btm) # activate this one if the encoding crashes without clear cause: # bob.bufr_obj.estimated_num_bytes_for_encoding = 25000 bob.bufr_obj.kdate = new_nsub * list(delayed_repl_data) print('bob.bufr_obj.cvals.shape = ', bob.bufr_obj.cvals.shape) bob.bufr_obj.encode_data(new_data, bob.bufr_obj.cvals[:32, :]) rbf_out.write_raw_bufr_msg(bob.bufr_obj.encoded_message) #for subs in range(len(data[:, 0])): # output_fd.write(str(subs)+separator+ # separator.join(str(val) for val in data[subs, :])+ # "\n") print('converted BUFR msg nr. ', msg_nr) # close the file bob.close() if msg_nr == 0: print('no BUFR messages found, are you sure this is a BUFR file?') rbf_out.close()
def encoding_example(output_bufr_file): # #[ """ wrap the example in a function to circumvent the pylint convention of requiring capitals for constants in the global scope (since most of these variables are not constants at all)) """ bufr = BUFRInterfaceECMWF(verbose=True) # fill sections 0, 1, 2 and 3 num_subsets = 4 bufr.fill_sections_0123( bufr_code_centre=98, # ECMWF bufr_obstype=3, # sounding bufr_subtype=251, # L1B bufr_table_local_version=1, bufr_table_master=0, bufr_table_master_version=15, bufr_code_subcentre=0, # L2B processing facility num_subsets=num_subsets, bufr_compression_flag=0) # 64=compression/0=no compression # determine information from sections 0123 to construct the BUFR table # names expected by the ECMWF BUFR library and create symlinks to the # default tables if needed bufr.setup_tables() # define a descriptor list template = BufrTemplate(verbose=True) template.add_descriptors( DD_D_DATE_YYYYMMDD, # 0 DD_D_TIME_HHMM) # 1 # delay replication for the next 2 descriptors # allow at most 2 delayed replications template.add_delayed_replic_descriptors(2, DD_PRESSURE, DD_TEMPERATURE) # replicate the next 2 descriptors 3 times template.add_replicated_descriptors(3, DD_LATITUDE_HIGH_ACCURACY, DD_LONGITUDE_HIGH_ACCURACY) bufr.register_and_expand_descriptors(template) # activate this one if the encoding crashes without clear cause: # bufr.estimated_num_bytes_for_encoding = 25000 # retrieve the length of the expanded descriptor list exp_descr_list_length = bufr.ktdexl print("exp_descr_list_length = ", exp_descr_list_length) # fill the values array with some dummy varying data num_values = exp_descr_list_length * num_subsets values = np.zeros(num_values, dtype=np.float64) # this is the default # note: these two must be identical for now, otherwise the # python to fortran interface breaks down. This also ofcourse is the # cause of the huge memory use of cvals in case num_values is large. num_cvalues = num_values cvals = np.zeros((num_cvalues, 80), dtype=np.character) for subset in range(num_subsets): # note that python starts counting with 0, unlike fortran, # so there is no need to take (subset-1) i = subset * exp_descr_list_length values[i] = 1999 # year i = i + 1 values[i] = 12 # month i = i + 1 values[i] = 31 # day i = i + 1 values[i] = 23 # hour i = i + 1 values[i] = 59 - subset # minute i = i + 1 values[i] = 2 # delayed replication factor # this delayed replication factor determines the actual number # of values to be stored for this particular subset # even if it is less then the number given in kdata above ! for repl in range(2): i = i + 1 values[i] = 1013.e2 - 100.e2 * subset + i + repl # pressure [pa] i = i + 1 values[i] = 273.15 - 10. * subset + i + repl # temperature [K] for repl in range(3): i = i + 1 values[i] = 51.82 + 0.05 * subset + i + repl # latitude i = i + 1 values[i] = 5.25 + 0.1 * subset + i + repl # longitude # do the encoding to binary format bufr.encode_data(values, cvals) # get an instance of the RawBUFRFile class bf1 = RawBUFRFile() # open the file for writing bf1.open(output_bufr_file, 'wb') # write the encoded BUFR message bf1.write_raw_bufr_msg(bufr.encoded_message) # close the file bf1.close()
def encode(output_bufr_file, RadarData, WMOID): # Get the data for the wind profilers WindComponentData = RadarData.getWindComponents() GateSpaceWidths = RadarData.getGateSpace() if len(GateSpaceWidths) > 1: MaxWidth = 0 for Width in GateSpaceWidths: if Width > MaxWidth: MaxWidth = Width else: MaxWidth = GateSpaces[0] print('Using Pulse Mode ' + str(MaxWidth)) for DateTime in WindComponentData: print('Processing ' + str(DateTime)) print(MaxWidth) print(WindComponentData[DateTime].keys()) if MaxWidth in WindComponentData[DateTime].keys(): print('Processing high mode only. Pulse ' + str(MaxWidth) + 'ns') WindComponents = WindComponentData[DateTime][MaxWidth] # Create tge buffer for the hour block of data. bufr = BUFRInterfaceECMWF(verbose=True) # fill sections 0, 1, 2 and 3 in the BUFR table num_subsets = 1 bufr.fill_sections_0123( # ECMWF # wind profiler . Also know as Message Type (Table A) # Message sub-type # L2B processing facility bufr_code_centre=59, bufr_obstype=2, bufr_subtype=7, bufr_table_local_version=1, bufr_table_master=0, bufr_table_master_version=3, bufr_code_subcentre=0, num_subsets=num_subsets, bufr_compression_flag=0, ) bufr.setup_tables() # define a descriptor list template = BufrTemplate() print('adding {0} descriptors'.format(10)) template.add_descriptors( WMO_BLOCK_NUM, WMO_STATION_NUM, DD_LATITUDE_COARSE_ACCURACY, DD_LONGITUDE_COARSE_ACCURACY, STATION_HEIGHT, YEAR, MONTH, MDAY, HOUR, MIN, TIME_SIGNIFICANCE, TIME_PERIOD, DD_WIND_SPEED, DD_WIND_DIR, DD_PRESSURE, DD_TEMPERATURE, RAINFALL_SWE, DD_RELATIVE_HUMD, HEIGHT_INCREMENT, WIND_PROFILER_SUB_MODE_INFO, HEIGHT_INCREMENT, HEIGHT_INCREMENT, HEIGHT_INCREMENT, ) # delay replication for the next 10 descriptors template.add_replicated_descriptors( len(WindComponents), WIND_PROFILER_MODE_INFO, WIND_PROFILER_QC_RESULTS, TOTAL_NUMBER, WIND_U_COMPONENT, WIND_V_COMPONENT, STD_DEV_HORIZONTAL_WIND_SPEED, TOTAL_NUMBER, RADAR_BACK_SCATTER, WIND_W_COMPONENT, STD_DEV_VERTICAL_SPEED, ) bufr.register_and_expand_descriptors(template) # activate this one if the encoding crashes without clear cause: # bufr.estimated_num_bytes_for_encoding = 25000 # retrieve the length of the expanded descriptor list exp_descr_list_length = bufr.ktdexl # fill the values array with some dummy varying data num_values = exp_descr_list_length values = np.zeros(num_values, dtype=np.float64) # this is the default # note: these two must be identical for now, otherwise the # python to fortran interface breaks down. This also ofcourse is the # cause of the huge memory use of cvals in case num_values is large. num_cvalues = num_values cvals = np.zeros((num_cvalues, 80), dtype=np.character) # note that python starts counting with 0, unlike fortran, # so there is no need to take (subset-1) # i = subset*exp_descr_list_length values[0] = WMOID[0:2] # WMO Block Number values[1] = WMOID[2:5] # WMO Station # values[2] = RadarData.getLatitude() # Latitude values[3] = RadarData.getLongitude() values[4] = RadarData.getElev() # Elevation of Station (meters) values[5] = DateTime.timetuple().tm_year # year values[6] = DateTime.timetuple().tm_mon # month values[7] = DateTime.timetuple().tm_mday # day values[8] = DateTime.timetuple().tm_hour # hour values[9] = 0 # minute values[10] = 2 # Time Significance values[11] = -60 # Time Period values[12] = 1 # Wind Speed values[13] = 1 # Wind Dir values[14] = 1 # Pressure values[15] = 1 # Temperature values[16] = .2 # Rainfall values[17] = 1 # Realative Humidty GateSpace = int(MaxWidth) * 1.e-9 * 3.e+8 / 2. values[18] = GateSpace # Height Increment values[19] = 0 # Wind Profiler Sub Mode values[20] = GateSpace # Height Increment values[21] = GateSpace # Height Increment values[22] = GateSpace # Height Increment print('Number of gates ' + str(len(WindComponents))) for i in range(0, len(WindComponents)): for t in range(0, 10): # Calulcate the correct index in the BUFR rec = i * 10 + 23 + t if WindComponents[i][0] <= 1000 \ and WindComponents[i][1] <= 1000: WindRadians = math.radians(WindComponents[i][1]) VelocityU = -WindComponents[i][0] \ * math.sin(WindRadians) VelocityV = -WindComponents[i][0] \ * math.cos(WindRadians) TotalNumber = WindComponents[i][2] SnrDB = WindComponents[i][3] else: VelocityU = VelocityV = TotalNumbe = SnrDB = -99 values[rec] = float('NaN') # level mode continue if rec % 10 == 3: values[rec] = 1 # level mode if rec % 10 == 4: values[rec] = 0 # Quality Control test if rec % 10 == 5: values[ rec] = TotalNumber # Total Number (with respect to accumlation or average) if rec % 10 == 6: values[rec] = VelocityU # U-Component if rec % 10 == 7: values[rec] = VelocityV # V-Component if rec % 10 == 8: values[ rec] = 0.000 # Std Deviation of horizontal wind speed if rec % 10 == 9: values[ rec] = TotalNumber # Total Number (with respect to accumlation or average) if rec % 10 == 0: values[ rec] = SnrDB / 100 # Radar Back Scatter (Peak Power) x100 if rec % 10 == 1: values[rec] = 0.000 # W-Component x100 if rec % 10 == 2: values[rec] = 0.000 # Std Deviation of vertical wind # do the encoding to binary format bufr.encode_data(values, cvals) HeaderString = ''' 287 IUAK01 PANC %02d%02d00 ''' \ % (DateTime.timetuple().tm_mday, DateTime.timetuple().tm_hour) if not os.path.exists(OutputPath): os.makedirs(OutputPath) OutputFile = \ '%s/IUPTO2_%s_%02d%02d00_216234297.bufr.%04d%02d%02d%02d' \ % ( OutputPath, RadarData.getSiteID().upper(), DateTime.timetuple().tm_mon, DateTime.timetuple().tm_mday, DateTime.timetuple().tm_year, DateTime.timetuple().tm_mon, DateTime.timetuple().tm_mday, DateTime.timetuple().tm_hour, ) # Remove file if exsists if os.path.exists(OutputFile): os.remove(OutputFile) bf1 = open(OutputFile, 'ab') bf1.write(HeaderString) bf1.close() # get an instance of the RawBUFRFile class bf1 = RawBUFRFile() # open the file for writing bf1.open(OutputFile, 'ab') # write the encoded BUFR message bf1.write_raw_bufr_msg(bufr.encoded_message) # close the file bf1.close() # #] print('succesfully written BUFR encoded data to file: ', OutputFile)