def _create_scaled_int16_vector(height, standard_name=None, original_name=None, long_name=None, scale_factor=0.01):
default_array = DefaultData.create_default_vector(height, np.float32)
variable = Variable(["y"], default_array)
tu.add_encoding(variable, np.int16, DefaultData.get_default_fill_value(np.int16), scale_factor)
HIRS._set_name_attributes(long_name, original_name, standard_name, variable)
return variable
def _create_int32_vector(height, standard_name=None, long_name=None, orig_name=None):
default_array = DefaultData.create_default_vector(height, np.int32)
variable = Variable(["y"], default_array)
tu.add_fill_value(variable, DefaultData.get_default_fill_value(np.int32))
HIRS._set_name_attributes(long_name, orig_name, standard_name, variable)
return variable
def add_full_fcdr_variables(dataset, height):
# u_btemps
variable = AMSUB_MHS._create_3d_float_variable(height)
variable.attrs[
"long_name"] = "total uncertainty of brightness temperature"
tu.add_units(variable, "K")
dataset["u_btemps"] = variable
# u_syst_btemps
variable = AMSUB_MHS._create_3d_float_variable(height)
variable.attrs[
"long_name"] = "systematic uncertainty of brightness temperature"
tu.add_units(variable, "K")
dataset["u_syst_btemps"] = variable
# u_random_btemps
variable = AMSUB_MHS._create_3d_float_variable(height)
variable.attrs["long_name"] = "noise on brightness temperature"
tu.add_units(variable, "K")
dataset["u_random_btemps"] = variable
# u_instrtemp
default_array = DefaultData.create_default_vector(height,
np.float32,
fill_value=np.NaN)
variable = Variable(["y"], default_array)
tu.add_fill_value(variable, np.NaN)
variable.attrs["long_name"] = "uncertainty of instrument temperature"
tu.add_units(variable, "K")
dataset["u_instrtemp"] = variable
# u_latitude
variable = AMSUB_MHS.create_angle_uncertainty_variable(
"latitude", height)
dataset["u_latitude"] = variable
# u_longitude
variable = AMSUB_MHS.create_angle_uncertainty_variable(
"longitude", height)
dataset["u_longitude"] = variable
# u_satellite_azimuth_angle
variable = AMSUB_MHS.create_angle_uncertainty_variable(
"satellite azimuth angle", height)
dataset["u_satellite_azimuth_angle"] = variable
# u_satellite_zenith_angle
variable = AMSUB_MHS.create_angle_uncertainty_variable(
"satellite zenith angle", height)
dataset["u_satellite_zenith_angle"] = variable
# u_solar_azimuth_angle
variable = AMSUB_MHS.create_angle_uncertainty_variable(
"solar azimuth angle", height)
dataset["u_solar_azimuth_angle"] = variable
# u_solar_zenith_angle
variable = AMSUB_MHS.create_angle_uncertainty_variable(
"solar zenith angle", height)
dataset["u_solar_zenith_angle"] = variable
def _create_counts_uncertainty_vector_uint32(height, standard_name):
default_array = DefaultData.create_default_vector(height, np.float32)
variable = Variable(["y"], default_array)
tu.add_encoding(variable, np.uint32, DefaultData.get_default_fill_value(np.uint32), 0.01)
variable.attrs["standard_name"] = standard_name
tu.add_units(variable, "count")
return variable
def _create_angle_variable_int(scale_factor,
standard_name=None,
long_name=None,
unsigned=False,
fill_value=None):
default_array = DefaultData.create_default_array(TIE_SIZE,
TIE_SIZE,
np.float32,
fill_value=np.NaN)
variable = Variable(["y_tie", "x_tie"], default_array)
if unsigned is True:
data_type = np.uint16
else:
data_type = np.int16
if fill_value is None:
fill_value = DefaultData.get_default_fill_value(data_type)
if standard_name is not None:
variable.attrs["standard_name"] = standard_name
if long_name is not None:
variable.attrs["long_name"] = long_name
tu.add_units(variable, "degree")
variable.attrs["tie_points"] = "true"
tu.add_encoding(variable,
data_type,
fill_value,
scale_factor,
chunksizes=CHUNKSIZES)
return variable
def test_add_geolocation_attribute(self):
default_array = DefaultData.create_default_array_3d(
8, 6, 4, np.float32, np.NaN)
variable = Variable(["channel", "y", "x"], default_array)
TemplateUtil.add_geolocation_attribute(variable)
self.assertEqual("longitude latitude", variable.attrs["coordinates"])
def create_angle_uncertainty_variable(angle_name, height):
variable = tu.create_float_variable(SWATH_WIDTH,
height,
long_name="uncertainty of " +
angle_name,
fill_value=np.NaN)
tu.add_units(variable, "degree")
return variable
def _create_temperature_array_3d(height, long_name, orig_name, dim_names):
default_array = DefaultData.create_default_array_3d(PRT_READING, height, PRT_NUMBER, np.float32, fill_value=np.NaN, dims_names=dim_names)
variable = Variable(["prt_number", "y", "prt_reading"], default_array)
tu.add_fill_value(variable, np.NaN)
tu.add_units(variable, "K")
variable.attrs["long_name"] = long_name
variable.attrs["orig_name"] = orig_name
return variable
def _create_time_ranges_variable(height, width, description):
default_array = DefaultData.create_default_array_3d(
width, height, 2, np.int32, fill_value=4294967295)
variable = Variable(["bounds", "y", "x"], default_array)
tu.add_fill_value(variable, 4294967295)
tu.add_units(variable, "s")
variable.attrs["description"] = description
variable.attrs["coordinates"] = "lon lat"
return variable
def _create_3d_float_variable(height):
default_array = DefaultData.create_default_array_3d(SWATH_WIDTH,
height,
NUM_CHANNELS,
np.float32,
fill_value=np.NaN)
variable = Variable(["channel", "y", "x"], default_array)
tu.add_fill_value(variable, np.NaN)
return variable
def _create_geo_angle_uncertainty_variable(standard_name, height, fill_value, orig_name=None):
default_array = DefaultData.create_default_array(SWATH_WIDTH, height, np.float32, fill_value=fill_value)
variable = Variable(["y", "x"], default_array)
tu.add_encoding(variable, np.uint16, fill_value, scale_factor=0.01)
variable.attrs["standard_name"] = standard_name
if orig_name is not None:
variable.attrs["orig_name"] = orig_name
tu.add_units(variable, "degree")
return variable
def create_angle_variable(height, standard_name):
default_array = DefaultData.create_default_array(SWATH_WIDTH,
height,
np.float32,
fill_value=np.NaN)
variable = Variable(["y", "x"], default_array)
variable.attrs["standard_name"] = standard_name
tu.add_units(variable, "degree")
tu.add_encoding(variable, np.int32, -999999, scale_factor=0.01)
return variable
def _create_overpass_counts_variable(height, width, description):
fill_value = DefaultData.get_default_fill_value(np.uint8)
default_array = DefaultData.create_default_array(width,
height,
np.uint8,
fill_value=fill_value)
variable = Variable(["y", "x"], default_array)
tu.add_fill_value(variable, fill_value)
variable.attrs["description"] = description
variable.attrs["coordinates"] = "lon lat"
return variable
def _create_float32_vector(fill_value, height, long_name, orig_name):
default_array = DefaultData.create_default_vector(height, np.float32, fill_value=fill_value)
variable = Variable(["y"], default_array)
if fill_value is None:
tu.add_fill_value(variable, DefaultData.get_default_fill_value(np.float32))
else:
tu.add_fill_value(variable, fill_value)
variable.attrs["long_name"] = long_name
if orig_name is not None:
variable.attrs["orig_name"] = orig_name
return variable
def _create_uth_variable(width, height, description=None):
default_array = DefaultData.create_default_array(width,
height,
np.float32,
fill_value=np.NaN)
variable = Variable(["y", "x"], default_array)
tu.add_fill_value(variable, np.NaN)
variable.attrs["coordinates"] = "lon lat"
variable.attrs["long_name"] = "upper_tropospheric_humidity"
tu.add_units(variable, "%")
if description is not None:
variable.attrs["description"] = description
return variable
def test_add_coordinates(self):
ds = xr.Dataset()
TemplateUtil.add_geolocation_variables(ds, 13, 108)
TemplateUtil.add_coordinates(ds)
x = ds["x"]
self.assertEqual((13, ), x.shape)
self.assertEqual(np.uint16, x.dtype)
y = ds["y"]
self.assertEqual((108, ), y.shape)
self.assertEqual(np.uint16, y.dtype)
def test_add_quality_flags(self):
ds = xr.Dataset()
TemplateUtil.add_quality_flags(ds, 9, 11)
quality = ds.variables["quality_pixel_bitmask"]
self.assertEqual((11, 9), quality.shape)
self.assertEqual(0, quality.data[5, 5])
self.assertEqual(np.uint8, quality.dtype)
self.assertEqual("status_flag", quality.attrs["standard_name"])
self.assertEqual("1, 2, 4, 8, 16, 32, 64, 128",
quality.attrs["flag_masks"])
self.assertEqual(
"invalid use_with_caution invalid_input invalid_geoloc invalid_time sensor_error padded_data incomplete_channel_data",
quality.attrs["flag_meanings"])
def _create_refl_uncertainty_variable(height,
long_name=None,
structured=False):
default_array = DefaultData.create_default_array(SWATH_WIDTH,
height,
np.float32,
fill_value=np.NaN)
variable = Variable(["y", "x"], default_array)
tu.add_units(variable, "percent")
tu.add_geolocation_attribute(variable)
variable.attrs["long_name"] = long_name
if structured:
tu.add_encoding(variable,
np.int16,
DefaultData.get_default_fill_value(np.int16),
0.01,
chunksizes=CHUNKS_2D)
variable.attrs["valid_min"] = 3
variable.attrs["valid_max"] = 5
else:
tu.add_encoding(variable,
np.int16,
DefaultData.get_default_fill_value(np.int16),
0.00001,
chunksizes=CHUNKS_2D)
variable.attrs["valid_max"] = 1000
variable.attrs["valid_min"] = 10
return variable
def add_easy_fcdr_variables(dataset,
height,
corr_dx=None,
corr_dy=None,
lut_size=None):
default_array = DefaultData.create_default_array_3d(SWATH_WIDTH,
height,
NUM_CHANNELS,
np.float32,
fill_value=np.NaN)
variable = Variable(["channel", "y", "x"], default_array)
tu.add_fill_value(variable, np.NaN)
tu.add_units(variable, "K")
variable.attrs["long_name"] = "independent uncertainty per pixel"
dataset["u_independent_tb"] = variable
default_array = DefaultData.create_default_array_3d(SWATH_WIDTH,
height,
NUM_CHANNELS,
np.float32,
fill_value=np.NaN)
variable = Variable(["channel", "y", "x"], default_array)
tu.add_fill_value(variable, np.NaN)
tu.add_units(variable, "K")
variable.attrs["long_name"] = "structured uncertainty per pixel"
dataset["u_structured_tb"] = variable
def _create_bt_variable(width, height, description=None):
default_array = DefaultData.create_default_array(width,
height,
np.float32,
fill_value=np.NaN)
variable = Variable(["y", "x"], default_array)
tu.add_fill_value(variable, np.NaN)
variable.attrs["coordinates"] = "lon lat"
variable.attrs["standard_name"] = "toa_brightness_temperature"
tu.add_units(variable, "K")
if description is not None:
variable.attrs["description"] = description
return variable
def add_easy_fcdr_variables(dataset,
height,
srf_size=None,
corr_dx=None,
corr_dy=None,
lut_size=None):
# u_independent_btemps
variable = AMSUB_MHS._create_3d_float_variable(height)
tu.add_units(variable, "K")
variable.attrs["long_name"] = "independent uncertainty per pixel"
dataset["u_independent_btemps"] = variable
# u_structured_btemps
variable = AMSUB_MHS._create_3d_float_variable(height)
tu.add_units(variable, "K")
variable.attrs["long_name"] = "structured uncertainty per pixel"
dataset["u_structured_btemps"] = variable
def _create_counts_variable(height, long_name):
default_array = DefaultData.create_default_array(
SWATH_WIDTH, height, np.int32)
variable = Variable(["y", "x"], default_array)
tu.add_fill_value(variable,
DefaultData.get_default_fill_value(np.int32))
variable.attrs["long_name"] = long_name
tu.add_units(variable, "count")
tu.add_geolocation_attribute(variable)
tu.add_chunking(variable, CHUNKS_2D)
return variable
def add_variables(dataset, width, height, num_samples=10):
tu.add_geolocation_variables(dataset, width, height, chunksizes=CHUNKING)
tu.add_quality_flags(dataset, width, height, chunksizes=CHUNKING)
default_array = DefaultData.create_default_vector(height, np.int32, fill_value=4294967295)
variable = Variable(["y"], default_array)
tu.add_fill_value(variable, 4294967295)
variable.attrs["standard_name"] = "time"
variable.attrs["long_name"] = "Acquisition time in seconds since 1970-01-01 00:00:00"
tu.add_units(variable, "s")
dataset["time"] = variable
default_array = DefaultData.create_default_array_3d(width, height, num_samples, np.float32, fill_value=np.NaN)
variable = Variable(["samples","y", "x"], default_array)
tu.add_fill_value(variable, np.NaN)
variable.attrs["standard_name"] = "sea_surface_temperature"
variable.attrs["units"] = "K"
variable.attrs["coordinates"] = "longitude latitude"
dataset["sst"] = variable
def _create_counts_uncertainty_variable(height, long_name):
variable = tu.create_float_variable(SWATH_WIDTH,
height,
long_name=long_name,
fill_value=np.NaN)
tu.add_units(variable, "count")
tu.add_geolocation_attribute(variable)
tu.add_chunking(variable, CHUNKS_2D)
return variable
def test_add_coordinates_with_channel(self):
ds = xr.Dataset()
TemplateUtil.add_geolocation_variables(ds, 8, 11)
default_array = DefaultData.create_default_array_3d(
8, 11, 4, np.float32, np.NaN)
ds["three_d"] = Variable(["channel", "y", "x"], default_array)
TemplateUtil.add_coordinates(ds)
x = ds["x"]
self.assertEqual((8, ), x.shape)
self.assertEqual(np.uint16, x.dtype)
y = ds["y"]
self.assertEqual((11, ), y.shape)
self.assertEqual(np.uint16, y.dtype)
channel = ds["channel"]
self.assertEqual((4, ), channel.shape)
self.assertEqual(np.uint16, channel.dtype)
def _create_angle_uncertainty_variable(angle_name, height):
variable = tu.create_float_variable(SWATH_WIDTH,
height,
long_name="uncertainty of " +
angle_name,
fill_value=np.NaN)
tu.add_units(variable, "degree")
tu.add_geolocation_attribute(variable)
tu.add_chunking(variable, CHUNKS_2D)
return variable
def test_add_geolocation_variables(self):
ds = xr.Dataset()
TemplateUtil.add_geolocation_variables(ds, 8, 10)
latitude = ds.variables["latitude"]
self.assertEqual((10, 8), latitude.shape)
self.assertTrue(np.isnan(latitude.data[4, 4]))
self.assertEqual("latitude", latitude.attrs["standard_name"])
self.assertEqual("degrees_north", latitude.attrs["units"])
self.assertEqual(np.int16, latitude.encoding['dtype'])
self.assertEqual(-32768, latitude.encoding['_FillValue'])
self.assertEqual(0.0027466658, latitude.encoding['scale_factor'])
self.assertEqual(0.0, latitude.encoding['add_offset'])
longitude = ds.variables["longitude"]
self.assertEqual((10, 8), longitude.shape)
self.assertTrue(np.isnan(longitude.data[5, 6]))
self.assertEqual("longitude", longitude.attrs["standard_name"])
self.assertEqual("degrees_east", longitude.attrs["units"])
self.assertEqual(np.int16, longitude.encoding['dtype'])
self.assertEqual(-32768, longitude.encoding['_FillValue'])
self.assertEqual(0.0054933317, longitude.encoding['scale_factor'])
self.assertEqual(0.0, longitude.encoding['add_offset'])
def test_add_gridded_geolocation_variables(self):
ds = xr.Dataset()
TemplateUtil.add_gridded_geolocation_variables(ds, 9, 11)
lat = ds.variables["lat"]
self.assertEqual((11, ), lat.shape)
self.assertTrue(np.isnan(lat.data[5]))
self.assertTrue(np.isnan(lat.attrs['_FillValue']))
self.assertEqual("latitude", lat.attrs["standard_name"])
self.assertEqual("latitude", lat.attrs["long_name"])
self.assertEqual("degrees_north", lat.attrs["units"])
self.assertEqual("lat_bnds", lat.attrs["bounds"])
lat_bnds = ds.variables["lat_bnds"]
self.assertEqual((11, 2), lat_bnds.shape)
self.assertTrue(np.isnan(lat_bnds.data[6, 0]))
self.assertTrue(np.isnan(lat_bnds.attrs['_FillValue']))
self.assertEqual("latitude cell boundaries",
lat_bnds.attrs["long_name"])
self.assertEqual("degrees_north", lat_bnds.attrs["units"])
lon = ds.variables["lon"]
self.assertEqual((9, ), lon.shape)
self.assertTrue(np.isnan(lon.data[6]))
self.assertTrue(np.isnan(lon.attrs['_FillValue']))
self.assertEqual("longitude", lon.attrs["standard_name"])
self.assertEqual("longitude", lon.attrs["long_name"])
self.assertEqual("degrees_east", lon.attrs["units"])
self.assertEqual("lon_bnds", lon.attrs["bounds"])
lon_bnds = ds.variables["lon_bnds"]
self.assertEqual((9, 2), lon_bnds.shape)
self.assertTrue(np.isnan(lon_bnds.data[7, 1]))
self.assertTrue(np.isnan(lon_bnds.attrs['_FillValue']))
self.assertEqual("longitude cell boundaries",
lon_bnds.attrs["long_name"])
self.assertEqual("degrees_east", lon_bnds.attrs["units"])
def _create_easy_fcdr_variable(height, long_name):
default_array = DefaultData.create_default_array_3d(SWATH_WIDTH, height, NUM_CHANNELS, np.float32, np.NaN)
variable = Variable(["channel", "y", "x"], default_array)
tu.add_encoding(variable, np.uint16, DefaultData.get_default_fill_value(np.uint16), 0.001, chunksizes=CHUNKING_BT)
variable.attrs["long_name"] = long_name
tu.add_units(variable, "K")
tu.add_geolocation_attribute(variable)
variable.attrs["valid_min"] = 1
variable.attrs["valid_max"] = 65534
return variable
def _create_geo_angle_variable(standard_name, height, orig_name=None, chunking=None):
default_array = DefaultData.create_default_array(SWATH_WIDTH, height, np.float32, fill_value=np.NaN)
variable = Variable(["y", "x"], default_array)
variable.attrs["standard_name"] = standard_name
if orig_name is not None:
variable.attrs["orig_name"] = orig_name
tu.add_units(variable, "degree")
tu.add_geolocation_attribute(variable)
tu.add_encoding(variable, np.uint16, DefaultData.get_default_fill_value(np.uint16), 0.01, -180.0, chunking)
return variable
