def _create_scaled_int16_vector(height, standard_name=None, original_name=None, long_name=None, scale_factor=0.01): default_array = DefaultData.create_default_vector(height, np.float32) variable = Variable(["y"], default_array) tu.add_encoding(variable, np.int16, DefaultData.get_default_fill_value(np.int16), scale_factor) HIRS._set_name_attributes(long_name, original_name, standard_name, variable) return variable
def _create_int32_vector(height, standard_name=None, long_name=None, orig_name=None): default_array = DefaultData.create_default_vector(height, np.int32) variable = Variable(["y"], default_array) tu.add_fill_value(variable, DefaultData.get_default_fill_value(np.int32)) HIRS._set_name_attributes(long_name, orig_name, standard_name, variable) return variable
def add_full_fcdr_variables(dataset, height): # u_btemps variable = AMSUB_MHS._create_3d_float_variable(height) variable.attrs[ "long_name"] = "total uncertainty of brightness temperature" tu.add_units(variable, "K") dataset["u_btemps"] = variable # u_syst_btemps variable = AMSUB_MHS._create_3d_float_variable(height) variable.attrs[ "long_name"] = "systematic uncertainty of brightness temperature" tu.add_units(variable, "K") dataset["u_syst_btemps"] = variable # u_random_btemps variable = AMSUB_MHS._create_3d_float_variable(height) variable.attrs["long_name"] = "noise on brightness temperature" tu.add_units(variable, "K") dataset["u_random_btemps"] = variable # u_instrtemp default_array = DefaultData.create_default_vector(height, np.float32, fill_value=np.NaN) variable = Variable(["y"], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["long_name"] = "uncertainty of instrument temperature" tu.add_units(variable, "K") dataset["u_instrtemp"] = variable # u_latitude variable = AMSUB_MHS.create_angle_uncertainty_variable( "latitude", height) dataset["u_latitude"] = variable # u_longitude variable = AMSUB_MHS.create_angle_uncertainty_variable( "longitude", height) dataset["u_longitude"] = variable # u_satellite_azimuth_angle variable = AMSUB_MHS.create_angle_uncertainty_variable( "satellite azimuth angle", height) dataset["u_satellite_azimuth_angle"] = variable # u_satellite_zenith_angle variable = AMSUB_MHS.create_angle_uncertainty_variable( "satellite zenith angle", height) dataset["u_satellite_zenith_angle"] = variable # u_solar_azimuth_angle variable = AMSUB_MHS.create_angle_uncertainty_variable( "solar azimuth angle", height) dataset["u_solar_azimuth_angle"] = variable # u_solar_zenith_angle variable = AMSUB_MHS.create_angle_uncertainty_variable( "solar zenith angle", height) dataset["u_solar_zenith_angle"] = variable
def _create_counts_uncertainty_vector_uint32(height, standard_name): default_array = DefaultData.create_default_vector(height, np.float32) variable = Variable(["y"], default_array) tu.add_encoding(variable, np.uint32, DefaultData.get_default_fill_value(np.uint32), 0.01) variable.attrs["standard_name"] = standard_name tu.add_units(variable, "count") return variable
def _create_angle_variable_int(scale_factor, standard_name=None, long_name=None, unsigned=False, fill_value=None): default_array = DefaultData.create_default_array(TIE_SIZE, TIE_SIZE, np.float32, fill_value=np.NaN) variable = Variable(["y_tie", "x_tie"], default_array) if unsigned is True: data_type = np.uint16 else: data_type = np.int16 if fill_value is None: fill_value = DefaultData.get_default_fill_value(data_type) if standard_name is not None: variable.attrs["standard_name"] = standard_name if long_name is not None: variable.attrs["long_name"] = long_name tu.add_units(variable, "degree") variable.attrs["tie_points"] = "true" tu.add_encoding(variable, data_type, fill_value, scale_factor, chunksizes=CHUNKSIZES) return variable
def test_add_geolocation_attribute(self): default_array = DefaultData.create_default_array_3d( 8, 6, 4, np.float32, np.NaN) variable = Variable(["channel", "y", "x"], default_array) TemplateUtil.add_geolocation_attribute(variable) self.assertEqual("longitude latitude", variable.attrs["coordinates"])
def create_angle_uncertainty_variable(angle_name, height): variable = tu.create_float_variable(SWATH_WIDTH, height, long_name="uncertainty of " + angle_name, fill_value=np.NaN) tu.add_units(variable, "degree") return variable
def _create_temperature_array_3d(height, long_name, orig_name, dim_names): default_array = DefaultData.create_default_array_3d(PRT_READING, height, PRT_NUMBER, np.float32, fill_value=np.NaN, dims_names=dim_names) variable = Variable(["prt_number", "y", "prt_reading"], default_array) tu.add_fill_value(variable, np.NaN) tu.add_units(variable, "K") variable.attrs["long_name"] = long_name variable.attrs["orig_name"] = orig_name return variable
def _create_time_ranges_variable(height, width, description): default_array = DefaultData.create_default_array_3d( width, height, 2, np.int32, fill_value=4294967295) variable = Variable(["bounds", "y", "x"], default_array) tu.add_fill_value(variable, 4294967295) tu.add_units(variable, "s") variable.attrs["description"] = description variable.attrs["coordinates"] = "lon lat" return variable
def _create_3d_float_variable(height): default_array = DefaultData.create_default_array_3d(SWATH_WIDTH, height, NUM_CHANNELS, np.float32, fill_value=np.NaN) variable = Variable(["channel", "y", "x"], default_array) tu.add_fill_value(variable, np.NaN) return variable
def _create_geo_angle_uncertainty_variable(standard_name, height, fill_value, orig_name=None): default_array = DefaultData.create_default_array(SWATH_WIDTH, height, np.float32, fill_value=fill_value) variable = Variable(["y", "x"], default_array) tu.add_encoding(variable, np.uint16, fill_value, scale_factor=0.01) variable.attrs["standard_name"] = standard_name if orig_name is not None: variable.attrs["orig_name"] = orig_name tu.add_units(variable, "degree") return variable
def create_angle_variable(height, standard_name): default_array = DefaultData.create_default_array(SWATH_WIDTH, height, np.float32, fill_value=np.NaN) variable = Variable(["y", "x"], default_array) variable.attrs["standard_name"] = standard_name tu.add_units(variable, "degree") tu.add_encoding(variable, np.int32, -999999, scale_factor=0.01) return variable
def _create_overpass_counts_variable(height, width, description): fill_value = DefaultData.get_default_fill_value(np.uint8) default_array = DefaultData.create_default_array(width, height, np.uint8, fill_value=fill_value) variable = Variable(["y", "x"], default_array) tu.add_fill_value(variable, fill_value) variable.attrs["description"] = description variable.attrs["coordinates"] = "lon lat" return variable
def _create_float32_vector(fill_value, height, long_name, orig_name): default_array = DefaultData.create_default_vector(height, np.float32, fill_value=fill_value) variable = Variable(["y"], default_array) if fill_value is None: tu.add_fill_value(variable, DefaultData.get_default_fill_value(np.float32)) else: tu.add_fill_value(variable, fill_value) variable.attrs["long_name"] = long_name if orig_name is not None: variable.attrs["orig_name"] = orig_name return variable
def _create_uth_variable(width, height, description=None): default_array = DefaultData.create_default_array(width, height, np.float32, fill_value=np.NaN) variable = Variable(["y", "x"], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["coordinates"] = "lon lat" variable.attrs["long_name"] = "upper_tropospheric_humidity" tu.add_units(variable, "%") if description is not None: variable.attrs["description"] = description return variable
def test_add_coordinates(self): ds = xr.Dataset() TemplateUtil.add_geolocation_variables(ds, 13, 108) TemplateUtil.add_coordinates(ds) x = ds["x"] self.assertEqual((13, ), x.shape) self.assertEqual(np.uint16, x.dtype) y = ds["y"] self.assertEqual((108, ), y.shape) self.assertEqual(np.uint16, y.dtype)
def test_add_quality_flags(self): ds = xr.Dataset() TemplateUtil.add_quality_flags(ds, 9, 11) quality = ds.variables["quality_pixel_bitmask"] self.assertEqual((11, 9), quality.shape) self.assertEqual(0, quality.data[5, 5]) self.assertEqual(np.uint8, quality.dtype) self.assertEqual("status_flag", quality.attrs["standard_name"]) self.assertEqual("1, 2, 4, 8, 16, 32, 64, 128", quality.attrs["flag_masks"]) self.assertEqual( "invalid use_with_caution invalid_input invalid_geoloc invalid_time sensor_error padded_data incomplete_channel_data", quality.attrs["flag_meanings"])
def _create_refl_uncertainty_variable(height, long_name=None, structured=False): default_array = DefaultData.create_default_array(SWATH_WIDTH, height, np.float32, fill_value=np.NaN) variable = Variable(["y", "x"], default_array) tu.add_units(variable, "percent") tu.add_geolocation_attribute(variable) variable.attrs["long_name"] = long_name if structured: tu.add_encoding(variable, np.int16, DefaultData.get_default_fill_value(np.int16), 0.01, chunksizes=CHUNKS_2D) variable.attrs["valid_min"] = 3 variable.attrs["valid_max"] = 5 else: tu.add_encoding(variable, np.int16, DefaultData.get_default_fill_value(np.int16), 0.00001, chunksizes=CHUNKS_2D) variable.attrs["valid_max"] = 1000 variable.attrs["valid_min"] = 10 return variable
def add_easy_fcdr_variables(dataset, height, corr_dx=None, corr_dy=None, lut_size=None): default_array = DefaultData.create_default_array_3d(SWATH_WIDTH, height, NUM_CHANNELS, np.float32, fill_value=np.NaN) variable = Variable(["channel", "y", "x"], default_array) tu.add_fill_value(variable, np.NaN) tu.add_units(variable, "K") variable.attrs["long_name"] = "independent uncertainty per pixel" dataset["u_independent_tb"] = variable default_array = DefaultData.create_default_array_3d(SWATH_WIDTH, height, NUM_CHANNELS, np.float32, fill_value=np.NaN) variable = Variable(["channel", "y", "x"], default_array) tu.add_fill_value(variable, np.NaN) tu.add_units(variable, "K") variable.attrs["long_name"] = "structured uncertainty per pixel" dataset["u_structured_tb"] = variable
def _create_bt_variable(width, height, description=None): default_array = DefaultData.create_default_array(width, height, np.float32, fill_value=np.NaN) variable = Variable(["y", "x"], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["coordinates"] = "lon lat" variable.attrs["standard_name"] = "toa_brightness_temperature" tu.add_units(variable, "K") if description is not None: variable.attrs["description"] = description return variable
def add_easy_fcdr_variables(dataset, height, srf_size=None, corr_dx=None, corr_dy=None, lut_size=None): # u_independent_btemps variable = AMSUB_MHS._create_3d_float_variable(height) tu.add_units(variable, "K") variable.attrs["long_name"] = "independent uncertainty per pixel" dataset["u_independent_btemps"] = variable # u_structured_btemps variable = AMSUB_MHS._create_3d_float_variable(height) tu.add_units(variable, "K") variable.attrs["long_name"] = "structured uncertainty per pixel" dataset["u_structured_btemps"] = variable
def _create_counts_variable(height, long_name): default_array = DefaultData.create_default_array( SWATH_WIDTH, height, np.int32) variable = Variable(["y", "x"], default_array) tu.add_fill_value(variable, DefaultData.get_default_fill_value(np.int32)) variable.attrs["long_name"] = long_name tu.add_units(variable, "count") tu.add_geolocation_attribute(variable) tu.add_chunking(variable, CHUNKS_2D) return variable
def add_variables(dataset, width, height, num_samples=10): tu.add_geolocation_variables(dataset, width, height, chunksizes=CHUNKING) tu.add_quality_flags(dataset, width, height, chunksizes=CHUNKING) default_array = DefaultData.create_default_vector(height, np.int32, fill_value=4294967295) variable = Variable(["y"], default_array) tu.add_fill_value(variable, 4294967295) variable.attrs["standard_name"] = "time" variable.attrs["long_name"] = "Acquisition time in seconds since 1970-01-01 00:00:00" tu.add_units(variable, "s") dataset["time"] = variable default_array = DefaultData.create_default_array_3d(width, height, num_samples, np.float32, fill_value=np.NaN) variable = Variable(["samples","y", "x"], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["standard_name"] = "sea_surface_temperature" variable.attrs["units"] = "K" variable.attrs["coordinates"] = "longitude latitude" dataset["sst"] = variable
def _create_counts_uncertainty_variable(height, long_name): variable = tu.create_float_variable(SWATH_WIDTH, height, long_name=long_name, fill_value=np.NaN) tu.add_units(variable, "count") tu.add_geolocation_attribute(variable) tu.add_chunking(variable, CHUNKS_2D) return variable
def test_add_coordinates_with_channel(self): ds = xr.Dataset() TemplateUtil.add_geolocation_variables(ds, 8, 11) default_array = DefaultData.create_default_array_3d( 8, 11, 4, np.float32, np.NaN) ds["three_d"] = Variable(["channel", "y", "x"], default_array) TemplateUtil.add_coordinates(ds) x = ds["x"] self.assertEqual((8, ), x.shape) self.assertEqual(np.uint16, x.dtype) y = ds["y"] self.assertEqual((11, ), y.shape) self.assertEqual(np.uint16, y.dtype) channel = ds["channel"] self.assertEqual((4, ), channel.shape) self.assertEqual(np.uint16, channel.dtype)
def _create_angle_uncertainty_variable(angle_name, height): variable = tu.create_float_variable(SWATH_WIDTH, height, long_name="uncertainty of " + angle_name, fill_value=np.NaN) tu.add_units(variable, "degree") tu.add_geolocation_attribute(variable) tu.add_chunking(variable, CHUNKS_2D) return variable
def test_add_geolocation_variables(self): ds = xr.Dataset() TemplateUtil.add_geolocation_variables(ds, 8, 10) latitude = ds.variables["latitude"] self.assertEqual((10, 8), latitude.shape) self.assertTrue(np.isnan(latitude.data[4, 4])) self.assertEqual("latitude", latitude.attrs["standard_name"]) self.assertEqual("degrees_north", latitude.attrs["units"]) self.assertEqual(np.int16, latitude.encoding['dtype']) self.assertEqual(-32768, latitude.encoding['_FillValue']) self.assertEqual(0.0027466658, latitude.encoding['scale_factor']) self.assertEqual(0.0, latitude.encoding['add_offset']) longitude = ds.variables["longitude"] self.assertEqual((10, 8), longitude.shape) self.assertTrue(np.isnan(longitude.data[5, 6])) self.assertEqual("longitude", longitude.attrs["standard_name"]) self.assertEqual("degrees_east", longitude.attrs["units"]) self.assertEqual(np.int16, longitude.encoding['dtype']) self.assertEqual(-32768, longitude.encoding['_FillValue']) self.assertEqual(0.0054933317, longitude.encoding['scale_factor']) self.assertEqual(0.0, longitude.encoding['add_offset'])
def test_add_gridded_geolocation_variables(self): ds = xr.Dataset() TemplateUtil.add_gridded_geolocation_variables(ds, 9, 11) lat = ds.variables["lat"] self.assertEqual((11, ), lat.shape) self.assertTrue(np.isnan(lat.data[5])) self.assertTrue(np.isnan(lat.attrs['_FillValue'])) self.assertEqual("latitude", lat.attrs["standard_name"]) self.assertEqual("latitude", lat.attrs["long_name"]) self.assertEqual("degrees_north", lat.attrs["units"]) self.assertEqual("lat_bnds", lat.attrs["bounds"]) lat_bnds = ds.variables["lat_bnds"] self.assertEqual((11, 2), lat_bnds.shape) self.assertTrue(np.isnan(lat_bnds.data[6, 0])) self.assertTrue(np.isnan(lat_bnds.attrs['_FillValue'])) self.assertEqual("latitude cell boundaries", lat_bnds.attrs["long_name"]) self.assertEqual("degrees_north", lat_bnds.attrs["units"]) lon = ds.variables["lon"] self.assertEqual((9, ), lon.shape) self.assertTrue(np.isnan(lon.data[6])) self.assertTrue(np.isnan(lon.attrs['_FillValue'])) self.assertEqual("longitude", lon.attrs["standard_name"]) self.assertEqual("longitude", lon.attrs["long_name"]) self.assertEqual("degrees_east", lon.attrs["units"]) self.assertEqual("lon_bnds", lon.attrs["bounds"]) lon_bnds = ds.variables["lon_bnds"] self.assertEqual((9, 2), lon_bnds.shape) self.assertTrue(np.isnan(lon_bnds.data[7, 1])) self.assertTrue(np.isnan(lon_bnds.attrs['_FillValue'])) self.assertEqual("longitude cell boundaries", lon_bnds.attrs["long_name"]) self.assertEqual("degrees_east", lon_bnds.attrs["units"])
def _create_easy_fcdr_variable(height, long_name): default_array = DefaultData.create_default_array_3d(SWATH_WIDTH, height, NUM_CHANNELS, np.float32, np.NaN) variable = Variable(["channel", "y", "x"], default_array) tu.add_encoding(variable, np.uint16, DefaultData.get_default_fill_value(np.uint16), 0.001, chunksizes=CHUNKING_BT) variable.attrs["long_name"] = long_name tu.add_units(variable, "K") tu.add_geolocation_attribute(variable) variable.attrs["valid_min"] = 1 variable.attrs["valid_max"] = 65534 return variable
def _create_geo_angle_variable(standard_name, height, orig_name=None, chunking=None): default_array = DefaultData.create_default_array(SWATH_WIDTH, height, np.float32, fill_value=np.NaN) variable = Variable(["y", "x"], default_array) variable.attrs["standard_name"] = standard_name if orig_name is not None: variable.attrs["orig_name"] = orig_name tu.add_units(variable, "degree") tu.add_geolocation_attribute(variable) tu.add_encoding(variable, np.uint16, DefaultData.get_default_fill_value(np.uint16), 0.01, -180.0, chunking) return variable