def test_add_fill_value_attributes_scaling_brightness_temperature(self):
variable = self._create_variable(np.int16)
variable.attrs["OFFSET"] = 327.68
variable.attrs["SCALE_FACTOR"] = 0.01
MmdReader._add_fill_value_attributes(
variable, "amsre.brightness_temperature23V")
self.assertAlmostEqual(0.0, variable.attrs["_FillValue"], 8)
def test_scale_data_no_scaling(self):
array = DefaultData.create_default_array(6, 12, np.int16)
array[0, 0] = 22
array[1, 0] = 23
variable = Variable(["y", "x"], array)
MmdReader._scale_data(variable)
self.assertEqual(22, variable.data[0, 0])
self.assertEqual(23, variable.data[1, 0])
def test_scale_data_only_offset(self):
array = DefaultData.create_default_array(6, 10, np.int16)
array[4, 0] = 22
array[5, 0] = 23
variable = Variable(["y", "x"], array)
variable.attrs["OFFSET"] = -10
MmdReader._scale_data(variable)
self.assertAlmostEqual(12, variable.data[4, 0], 8)
self.assertAlmostEqual(13, variable.data[5, 0], 8)
def test_scale_data_only_scaling(self):
array = DefaultData.create_default_array(6, 11, np.int16)
array[2, 0] = 22
array[3, 0] = 23
variable = Variable(["y", "x"], array)
variable.attrs["SCALE_FACTOR"] = 0.2
MmdReader._scale_data(variable)
self.assertAlmostEqual(4.4, variable.data[2, 0], 8)
self.assertAlmostEqual(4.6, variable.data[3, 0], 8)
def test_scale_data_scaling_and_offset(self):
array = DefaultData.create_default_array(6, 9, np.int16)
array[0, 1] = 22
array[1, 1] = 23
variable = Variable(["y", "x"], array)
variable.attrs["SCALE_FACTOR"] = 0.2
variable.attrs["OFFSET"] = 1.6
MmdReader._scale_data(variable)
self.assertAlmostEqual(6.0, variable.data[0, 1], 8)
self.assertAlmostEqual(6.2, variable.data[1, 1], 8)
def run(self, args):
self.parse_cmd_line(args)
print("reading input file: " + self.input_file)
mmd_reader = MmdReader()
mmd_data = mmd_reader.read(self.input_file)
print("... success")
start_time = time.time()
matchup_count = mmd_data.dims["matchup_count"]
print(matchup_count, " matches")
flag_coding = FlagCoding(matchup_count)
print("running preprocessing ...")
preprocessor = Preprocessor()
pre_proc_mmd_data = preprocessor.run(mmd_data, flag_coding)
print("... success")
print("running input data QA ...")
qa_processor = QaProcessor()
qa_processor.run_qa(pre_proc_mmd_data, flag_coding)
print("... success")
print("running input bias correction ...")
bt_bias_correction = BtBiasCorrection()
bt_bias_correction.run(pre_proc_mmd_data)
print("... success")
print("preparing output file ...")
results = self._create_result_structure(matchup_count, MAX_ITERATIONS,
NUM_BT)
print("... success")
print("running retrieval ...")
retrieval = Retrieval()
results = retrieval.run(pre_proc_mmd_data, results, flag_coding)
print("... success")
print("writing ")
self.add_flags_variable(flag_coding, results)
print("--- %s seconds ---" % (time.time() - start_time))
self._write_result_data(self.output_directory, self.input_file,
results)
mmd_reader.close()
print("... success")
def test_add_fill_value_attributes_no_scaling_fill_value_untouched(self):
variable = self._create_variable(np.int8)
variable.attrs["_FillValue"] = -127
MmdReader._add_fill_value_attributes(variable, "don't care")
self.assertEqual(-127, variable.attrs["_FillValue"])
def test_add_fill_value_attributes_no_scaling_no_fill_value_float32(self):
variable = self._create_variable(np.float32)
MmdReader._add_fill_value_attributes(variable, "don't care")
self.assertEqual(DefaultData.get_default_fill_value(np.float32),
variable.attrs["_FillValue"])
def test_get_insitu_sensor_unsupported(self):
dataset = self._create_dataset_with_variable("unsupported-sensor")
with self.assertRaises(IOError):
MmdReader._get_insitu_sensor(dataset)
def test_get_insitu_sensor_radiometer(self):
dataset = self._create_dataset_with_variable("radiometer-sst_blubb")
insitu_sensor = MmdReader._get_insitu_sensor(dataset)
self.assertEqual("radiometer-sst", insitu_sensor)
def test_get_insitu_sensor_animal(self):
dataset = self._create_dataset_with_variable("animal-sst_bla")
insitu_sensor = MmdReader._get_insitu_sensor(dataset)
self.assertEqual("animal-sst", insitu_sensor)
def test_add_fill_value_attributes_insitu_data(self):
variable = self._create_variable(np.int16)
MmdReader._add_fill_value_attributes(variable, "insitu.lon")
self.assertAlmostEqual(-32768, variable.attrs["_FillValue"], 8)
def test_add_fill_value_attributes_grib_data(self):
variable = self._create_variable(np.int16)
variable.attrs["source"] = "GRIB data"
MmdReader._add_fill_value_attributes(variable, "don't care")
self.assertAlmostEqual(2e20, variable.attrs["_FillValue"], 8)
def test_add_fill_value_attributes_scaling_no_fill_value_int16(self):
variable = self._create_variable(np.int16)
variable.attrs["OFFSET"] = 0.0
variable.attrs["SCALE_FACTOR"] = 0.01
MmdReader._add_fill_value_attributes(variable, "don't care")
self.assertAlmostEqual(-327.67, variable.attrs["_FillValue"], 8)
def test_read(self):
test_mmd = TestDataUtils.get_test_mmd()
self.assertTrue(os.path.isfile(test_mmd))
reader = MmdReader()
input_data = reader.read(test_mmd)
self.assertIsNotNone(input_data)
self.assertIsNotNone(input_data.variables["amsre.latitude"])
self.assertIsNotNone(input_data.variables["amsre.longitude"])
# IS NOT USED self.assertIsNotNone(input_data.variables["amsre.time"])
self.assertIsNotNone(input_data.variables["amsre.solar_zenith_angle"])
self.assertIsNotNone(
input_data.variables["amsre.satellite_zenith_angle"])
self.assertIsNotNone(
input_data.variables["amsre.satellite_azimuth_angle"])
# IS NOT USED self.assertIsNotNone(input_data.variables["amsre.land_ocean_flag_6"])
self.assertIsNotNone(
input_data.variables["amsre.brightness_temperature6V"])
self.assertIsNotNone(
input_data.variables["amsre.brightness_temperature6H"])
self.assertIsNotNone(
input_data.variables["amsre.brightness_temperature10V"])
self.assertIsNotNone(
input_data.variables["amsre.brightness_temperature10H"])
self.assertIsNotNone(
input_data.variables["amsre.brightness_temperature18V"])
self.assertIsNotNone(
input_data.variables["amsre.brightness_temperature18H"])
self.assertIsNotNone(
input_data.variables["amsre.brightness_temperature23V"])
self.assertIsNotNone(
input_data.variables["amsre.brightness_temperature23H"])
self.assertIsNotNone(
input_data.variables["amsre.brightness_temperature36V"])
self.assertIsNotNone(
input_data.variables["amsre.brightness_temperature36H"])
self.assertIsNotNone(
input_data.variables["amsre.pixel_data_quality6V"])
self.assertIsNotNone(
input_data.variables["amsre.pixel_data_quality6H"])
self.assertIsNotNone(
input_data.variables["amsre.pixel_data_quality10V"])
self.assertIsNotNone(
input_data.variables["amsre.pixel_data_quality10H"])
self.assertIsNotNone(
input_data.variables["amsre.pixel_data_quality18V"])
self.assertIsNotNone(
input_data.variables["amsre.pixel_data_quality18H"])
self.assertIsNotNone(
input_data.variables["amsre.pixel_data_quality23V"])
self.assertIsNotNone(
input_data.variables["amsre.pixel_data_quality23H"])
self.assertIsNotNone(
input_data.variables["amsre.pixel_data_quality36V"])
self.assertIsNotNone(
input_data.variables["amsre.pixel_data_quality36H"])
self.assertIsNotNone(input_data.variables["amsre.scan_data_quality"])
self.assertIsNotNone(
input_data.variables["amsre.Geostationary_Reflection_Latitude"])
self.assertIsNotNone(
input_data.variables["amsre.Geostationary_Reflection_Longitude"])
# IS NOT USED self.assertIsNotNone(input_data.variables["amsre.nwp.seaice_fraction"])
self.assertIsNotNone(
input_data.variables["amsre.nwp.sea_surface_temperature"])
self.assertIsNotNone(
input_data.variables["amsre.nwp.10m_east_wind_component"])
self.assertIsNotNone(
input_data.variables["amsre.nwp.10m_north_wind_component"])
self.assertIsNotNone(
input_data.variables["amsre.nwp.skin_temperature"])
self.assertIsNotNone(
input_data.variables["amsre.nwp.log_surface_pressure"])
self.assertIsNotNone(
input_data.variables["amsre.nwp.cloud_liquid_water"])
self.assertIsNotNone(
input_data.variables["amsre.nwp.total_column_water_vapour"])
self.assertIsNotNone(input_data.variables["amsre.nwp.total_precip"])
self.assertIsNotNone(
input_data.variables["insitu.time"]
) # @todo 3 tb/tb check if this is ever used 2017-10-27
self.assertIsNotNone(
input_data.variables["insitu.lat"]
) # @todo 3 tb/tb check if this is ever used 2017-10-27
self.assertIsNotNone(
input_data.variables["insitu.lon"]
) # @todo 3 tb/tb check if this is ever used 2017-10-27
self.assertIsNotNone(
input_data.variables["insitu.sea_surface_temperature"])
self.assertIsNotNone(
input_data.variables["insitu.sst_depth"]
) # @todo 3 tb/tb check if this is ever used 2017-10-27
self.assertIsNotNone(input_data.variables["insitu.sst_qc_flag"])
self.assertIsNotNone(input_data.variables["insitu.sst_track_flag"])
# verify that the following variables are converted to the appropriate target data type:
self.assertAlmostEqual(
55.125,
input_data.variables["amsre.satellite_zenith_angle"].data[0, 0,
0], 8)
self.assertAlmostEqual(
44.049999,
input_data.variables["amsre.satellite_azimuth_angle"].data[1, 0,
0], 6)
self.assertAlmostEqual(
175.59999,
input_data.variables["amsre.brightness_temperature6V"].data[1, 1,
0], 5)
self.assertAlmostEqual(
90.459991,
input_data.variables["amsre.brightness_temperature6H"].data[1, 1,
1], 6)
self.assertAlmostEqual(
180.05,
input_data.variables["amsre.brightness_temperature10V"].data[2, 1,
1], 5)
self.assertAlmostEqual(
93.819992,
input_data.variables["amsre.brightness_temperature10H"].data[2, 2,
1], 6)
self.assertAlmostEqual(
200.73999,
input_data.variables["amsre.brightness_temperature18V"].data[2, 2,
2], 6)
self.assertAlmostEqual(
127.59,
input_data.variables["amsre.brightness_temperature18H"].data[3, 2,
2], 5)
self.assertAlmostEqual(
229.01999,
input_data.variables["amsre.brightness_temperature23V"].data[3, 3,
2], 5)
self.assertAlmostEqual(
176.75999,
input_data.variables["amsre.brightness_temperature23H"].data[3, 3,
3], 5)
self.assertAlmostEqual(
215.17,
input_data.variables["amsre.brightness_temperature36V"].data[4, 3,
3], 5)
self.assertAlmostEqual(
152.19,
input_data.variables["amsre.brightness_temperature36H"].data[4, 4,
3], 5)
self.assertAlmostEqual(
-6.7799997, input_data.
variables["amsre.Geostationary_Reflection_Latitude"].data[4, 4,
4], 7)
self.assertAlmostEqual(
-22.959999, input_data.
variables["amsre.Geostationary_Reflection_Longitude"].data[5, 4,
4], 6)