def test_run_stddev_variables_too_many_fill_values(self): data = DefaultData.create_default_array_3d(21, 21, 5, np.float32) for x in range(0, 21): for y in range(0, 21): data[:, y, x] = x + y data[1, :, 1] = -81 data[1, :, 2] = -81 data[1, :, 5] = -81 variable = Variable(["matchup_count", "ny", "nx"], data) variable.attrs["_FillValue"] = -81 self.dataset["amsre.brightness_temperature36V"] = variable flag_coding = FlagCoding(5) prep_data = self.preprocessor.run(self.dataset, flag_coding=flag_coding) variable = prep_data.variables[ "amsre.brightness_temperature36V_stddev"] self.assertEqual((5, ), variable.shape) self.assertAlmostEqual(8.563489, variable.data[0], 7) self.assertAlmostEqual(-81, variable.data[1], 7) self.assertAlmostEqual(8.563489, variable.data[2], 7) flags = flag_coding.get_flags() self.assertEqual(1, flags[1])
def test_run_stddev_variables_masks_fill_value(self): data = DefaultData.create_default_array_3d(21, 21, 5, np.float32) for x in range(0, 21): for y in range(0, 21): data[:, y, x] = (x + y) * 1.0 data[:, 1, 1] = -80 data[:, 10, 10] = -80 data[:, 15, 15] = -80 data[:, 20, 20] = -80 variable = Variable(["matchup_count", "ny", "nx"], data) variable.attrs["_FillValue"] = -80 self.dataset["amsre.brightness_temperature23H"] = variable flag_coding = FlagCoding(5) prep_data = self.preprocessor.run(self.dataset, flag_coding=flag_coding) variable = prep_data.variables[ "amsre.brightness_temperature23H_stddev"] self.assertEqual((5, ), variable.shape) self.assertAlmostEqual(8.4922457, variable.data[1], 7) flags = flag_coding.get_flags() self.assertEqual(0, flags[1])
def test_extract_ascending_descending_corrupt_filename(self): data = [ "AMSR_E_L2A_BrightnessTemperatures_V12_200807110947_D.hdf", "AMSR_E_ohlala_something_is_wrong_here.hdf", "AMSR_E_L2A_BrightnessTemperatures_V12_200807112208_A.hdf", "AMSR_E_L2A_BrightnessTemperatures_V12_200807111215_A.hdf", "AMSR_E_L2A_BrightnessTemperatures_V12_200807112258_D.hdf" ] self.dataset["amsre.l2a_filename"] = Variable(["matchup_count"], data) flag_coding = FlagCoding(5) prep_data = self.preprocessor.run(self.dataset, flag_coding=flag_coding) variable = prep_data.variables["amsre.ascending"] self.assertFalse(variable.data[0]) self.assertFalse(variable.data[1]) self.assertTrue(variable.data[2]) self.assertTrue(variable.data[3]) self.assertFalse(variable.data[4]) flags = flag_coding.get_flags() self.assertEqual(0, flags[0]) self.assertEqual(256, flags[1]) self.assertEqual(0, flags[2]) self.assertEqual(0, flags[3]) self.assertEqual(0, flags[4])
def test_run_average_variables_masks_invalid(self): fill_value = -79 data = DefaultData.create_default_array_3d(9, 9, 4, np.float32) data[0, 2, :] = 104 data[0, 3, :] = 105 data[0, 4, :] = 106 data[0, 5, :] = 107 data[0, 6, :] = 108 data[0, 3, 4] = fill_value data[0, 4, 6] = fill_value variable = Variable(["matchup_count", "ny", "nx"], data) variable.attrs["_FillValue"] = fill_value self.dataset["amsre.brightness_temperature6H"] = variable flag_coding = FlagCoding(4) prep_data = self.preprocessor.run(self.dataset, flag_coding=flag_coding) variable = prep_data.variables["amsre.brightness_temperature6H"] self.assertEqual((4, ), variable.shape) self.assertAlmostEqual(106.0, variable.data[0], 5) flags = flag_coding.get_flags() self.assertEqual(0, flags[0])
def test_find_rfi(self): sat_lon = [ 128.6643, -171.7969, -80.4683, -19.5940, -19.2279, 166.5237, -2.9735, -174.1847, 174.9277, 171.3845, 171.3845, 6.0 ] self.dataset["amsre.longitude"] = Variable(["matchup_count"], sat_lon) sat_lat = [ 24.1321, 36.5892, -18.7240, -25.0698, -13.3672, -52.1502, 69.3439, 11.3338, -13.4490, -16.0562, -16.0562, 64.0 ] self.dataset["amsre.latitude"] = Variable(["matchup_count"], sat_lat) geo_refl_lon = [ 75.910, -111.240, -111.920, -53.200, -10.180, -104.260, -117.880, 170.480, 125.520, -170.290, 236.0, -170.290 ] self.dataset["amsre.Geostationary_Reflection_Longitude"] = Variable( ["matchup_count"], geo_refl_lon) geo_refl_lat = [ 23.0600, 77.3500, -61.4200, -68.8300, 33.7200, -56.7100, 53.7500, 57.8800, -26.3100, 28.3900, -20.3, 28.3900 ] self.dataset["amsre.Geostationary_Reflection_Latitude"] = Variable( ["matchup_count"], geo_refl_lat) i_asc = [1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0] self.dataset["amsre.ascending"] = Variable(["matchup_count"], i_asc) flag_coding = FlagCoding(12) rfi_processor = RfiProcessor() rfi_processor.find_rfi(self.dataset, flag_coding) flags = flag_coding.get_flags() for i in range(0, 9): self.assertEqual(0, flags[i]) self.assertEqual(512, flags[10]) self.assertEqual(512, flags[11])
def test_run_average_variables(self): data = DefaultData.create_default_array_3d(7, 7, 5, np.float32) data[0, 1, :] = 102 data[0, 2, :] = 101 data[0, 3, :] = 103 data[0, 4, :] = 105 data[0, 5, :] = 104 variable = Variable(["matchup_count", "ny", "nx"], data) variable.attrs["_FillValue"] = -78 self.dataset["amsre.brightness_temperature6V"] = variable flag_coding = FlagCoding(5) prep_data = self.preprocessor.run(self.dataset, flag_coding=flag_coding) variable = prep_data.variables["amsre.brightness_temperature6V"] self.assertEqual((5, ), variable.shape) self.assertEqual(103, variable.data[0]) flags = flag_coding.get_flags() self.assertEqual(0, flags[0])
def test_run_stddev_variables(self): data = DefaultData.create_default_array_3d(21, 21, 5, np.float32) for x in range(0, 21): for y in range(0, 21): data[:, y, x] = (x + y) * 0.23 variable = Variable(["matchup_count", "ny", "nx"], data) variable.attrs["_FillValue"] = -79 self.dataset["amsre.brightness_temperature23V"] = variable flag_coding = FlagCoding(5) prep_data = self.preprocessor.run(self.dataset, flag_coding=flag_coding) variable = prep_data.variables[ "amsre.brightness_temperature23V_stddev"] self.assertEqual((5, ), variable.shape) self.assertAlmostEqual(1.9696023, variable.data[0], 7) flags = flag_coding.get_flags() self.assertEqual(0, flags[0])
def run(self, args): self.parse_cmd_line(args) print("reading input file: " + self.input_file) mmd_reader = MmdReader() mmd_data = mmd_reader.read(self.input_file) print("... success") start_time = time.time() matchup_count = mmd_data.dims["matchup_count"] print(matchup_count, " matches") flag_coding = FlagCoding(matchup_count) print("running preprocessing ...") preprocessor = Preprocessor() pre_proc_mmd_data = preprocessor.run(mmd_data, flag_coding) print("... success") print("running input data QA ...") qa_processor = QaProcessor() qa_processor.run_qa(pre_proc_mmd_data, flag_coding) print("... success") print("running input bias correction ...") bt_bias_correction = BtBiasCorrection() bt_bias_correction.run(pre_proc_mmd_data) print("... success") print("preparing output file ...") results = self._create_result_structure(matchup_count, MAX_ITERATIONS, NUM_BT) print("... success") print("running retrieval ...") retrieval = Retrieval() results = retrieval.run(pre_proc_mmd_data, results, flag_coding) print("... success") print("writing ") self.add_flags_variable(flag_coding, results) print("--- %s seconds ---" % (time.time() - start_time)) self._write_result_data(self.output_directory, self.input_file, results) mmd_reader.close() print("... success")
def test_run(self): dataset = xr.Dataset() dataset["amsre.brightness_temperature6V"] = Variable( ["matchup_count"], np.array([176.4254, 169.4794, 169.3118], dtype=np.float64)) dataset["amsre.brightness_temperature6H"] = Variable( ["matchup_count"], np.array([90.9698, 86.6282, 82.611], dtype=np.float64)) dataset["amsre.brightness_temperature10V"] = Variable( ["matchup_count"], np.array([181.49, 174.48, 173.5316], dtype=np.float64)) dataset["amsre.brightness_temperature10H"] = Variable( ["matchup_count"], np.array([98.8989, 93.5897, 86.9333], dtype=np.float64)) dataset["amsre.brightness_temperature18V"] = Variable( ["matchup_count"], np.array([212.9936, 199.0224, 192.7028], dtype=np.float64)) dataset["amsre.brightness_temperature18H"] = Variable( ["matchup_count"], np.array([153.7084, 133.4428, 115.918], dtype=np.float64)) dataset["amsre.brightness_temperature23V"] = Variable( ["matchup_count"], np.array([250.0724, 226.9596, 215.15], dtype=np.float64)) dataset["amsre.brightness_temperature23H"] = Variable( ["matchup_count"], np.array([219.4987, 184.0727, 153.9983], dtype=np.float64)) dataset["amsre.brightness_temperature36V"] = Variable( ["matchup_count"], np.array([232.129, 220.8774, 217.5726], dtype=np.float64)) dataset["amsre.brightness_temperature36H"] = Variable( ["matchup_count"], np.array([183.7511, 167.1675, 151.3251], dtype=np.float64)) dataset["amsre.nwp.abs_wind_speed"] = Variable( ["matchup_count"], np.array([6.00405503911681, 9.20655470261406, 5.10105703955726], dtype=np.float64)) dataset["amsre.nwp.total_column_water_vapour"] = Variable( ["matchup_count"], np.array([59.2188758850098, 26.8414897918701, 14.3991870880127], dtype=np.float64)) dataset["amsre.nwp.total_column_liquid_water"] = Variable( ["matchup_count"], np.array( [0.153278715134895, 0.210529339037996, 0.00125914319133341], dtype=np.float64)) dataset["amsre.nwp.sea_surface_temperature"] = Variable( ["matchup_count"], np.array([301.679042997567, 291.330501737802, 295.058376493661], dtype=np.float64) - 273.15) dataset["amsre.satellite_zenith_angle"] = Variable( ["matchup_count"], np.array([55.19, 55.23, 55.2], dtype=np.float64)) dataset["relative_angle"] = Variable( ["matchup_count"], np.array([78.1866134486559, 359.01513331183, 26.1734235301682], dtype=np.float64)) result = MwOeSstProcessor._create_result_structure(3, 10, 10) self.retrieval.run(dataset, result, FlagCoding(3)) self.assertEqual((3, 10), result.j.shape) self.assertAlmostEqual(7538.919, result.j.data[0, 0], 3) self.assertAlmostEqual(7.849407, result.j.data[1, 1], 6) self.assertAlmostEqual(17.829288, result.j.data[2, 2], 6) self.assertEqual((3, 10), result.tb_rmse_ite.shape) self.assertAlmostEqual(0.89614433, result.tb_rmse_ite.data[0, 3], 7) self.assertTrue(np.isnan(result.tb_rmse_ite.data[1, 4])) self.assertTrue(np.isnan(result.tb_rmse_ite.data[2, 5])) self.assertEqual((3, 10), result.tb_chi_ite.shape) self.assertTrue(np.isnan(result.tb_chi_ite.data[0, 6])) self.assertTrue(np.isnan(result.tb_chi_ite.data[1, 7])) self.assertTrue(np.isnan(result.tb_chi_ite.data[2, 8])) self.assertEqual((3, ), result.convergence_passed_flag.shape) self.assertEqual(1, result.convergence_passed_flag.data[0]) self.assertEqual(1, result.convergence_passed_flag.data[1]) self.assertEqual(1, result.convergence_passed_flag.data[2]) self.assertEqual((3, 10), result.di2.shape) self.assertTrue(np.isnan(result.di2.data[0, 9])) self.assertAlmostEqual(4071.0098, result.di2.data[1, 0], 4) self.assertAlmostEqual(184.93216, result.di2.data[2, 1], 4) self.assertEqual((3, ), result.i.shape) self.assertEqual(4, result.i.data[0]) self.assertEqual(3, result.i.data[1]) self.assertEqual(3, result.i.data[2]) self.assertEqual((3, ), result.tb_rmse_ite0.shape) self.assertAlmostEqual(5.0651402, result.tb_rmse_ite0.data[0], 6) self.assertAlmostEqual(2.7727253, result.tb_rmse_ite0.data[1], 7) self.assertAlmostEqual(9.249838, result.tb_rmse_ite0.data[2], 6) self.assertEqual((3, 10), result.dtb_ite0.shape) self.assertAlmostEqual(-6.1169634, result.dtb_ite0.data[0, 3], 6) self.assertAlmostEqual(-0.18351583, result.dtb_ite0.data[1, 4], 7) self.assertAlmostEqual(-10.374541, result.dtb_ite0.data[2, 5], 6) self.assertEqual((3, 10), result.TA0_ite0.shape) self.assertAlmostEqual(256.65015, result.TA0_ite0.data[0, 6], 5) self.assertAlmostEqual(181.64264, result.TA0_ite0.data[1, 7], 5) self.assertAlmostEqual(209.93489, result.TA0_ite0.data[2, 8], 5) self.assertEqual((3, ), result.j_ite0.shape) self.assertAlmostEqual(24122.828, result.j_ite0.data[0], 3) self.assertAlmostEqual(4573.326, result.j_ite0.data[1], 3) self.assertAlmostEqual(291431.88, result.j_ite0.data[2], 1) self.assertEqual((3, 4), result.A.shape) self.assertAlmostEqual(0.9962128, result.A.data[0, 0], 7) self.assertAlmostEqual(0.910339, result.A.data[1, 1], 7) self.assertAlmostEqual(0.99998724, result.A.data[2, 2], 7) self.assertEqual((3, ), result.chisq.shape) self.assertAlmostEqual(2.7899823, result.chisq.data[0], 7) self.assertAlmostEqual(1.5961539, result.chisq.data[1], 7) self.assertAlmostEqual(0.710269, result.chisq.data[2], 7) self.assertEqual((3, ), result.mu_sst.shape) self.assertAlmostEqual(0.49287936, result.mu_sst.data[0], 7) self.assertAlmostEqual(0.23875031, result.mu_sst.data[1], 7) self.assertAlmostEqual(0.18248218, result.mu_sst.data[2], 8) self.assertEqual((3, 4), result.x.shape) self.assertAlmostEqual(13.069875, result.x.data[0, 0], 6) self.assertAlmostEqual(27.391582, result.x.data[1, 1], 5) self.assertAlmostEqual(0.120024666, result.x.data[2, 2], 8) self.assertEqual((3, 4), result.p0.shape) self.assertAlmostEqual(26.84149, result.p0.data[1, 1], 6) self.assertAlmostEqual(0.0012591432, result.p0.data[2, 2], 5) self.assertAlmostEqual(301.67905, result.p0.data[0, 3], 5) self.assertEqual((3, 4), result.S.shape) self.assertAlmostEqual(0.123080365, result.S.data[0, 0], 8) self.assertAlmostEqual(0.26949105, result.S.data[1, 1], 8) self.assertAlmostEqual(0.0035749427, result.S.data[2, 2], 8) self.assertEqual((3, 10), result.F.shape) self.assertAlmostEqual(98.216354, result.F.data[0, 3], 5) self.assertAlmostEqual(199.18938, result.F.data[1, 4], 5) self.assertAlmostEqual(115.8715, result.F.data[2, 5], 5) self.assertEqual((3, 10), result.y.shape) self.assertAlmostEqual(98.8989, result.y.data[0, 3], 5) self.assertAlmostEqual(199.0224, result.y.data[1, 4], 5) self.assertAlmostEqual(115.918, result.y.data[2, 5], 5) self.assertEqual((3, 10), result.dtb.shape) self.assertAlmostEqual(0.93957216, result.dtb.data[0, 6], 8) self.assertAlmostEqual(-0.40983388, result.dtb.data[1, 7], 8) self.assertAlmostEqual(-0.48253775, result.dtb.data[2, 8], 8) self.assertEqual((3, ), result.ds.shape) self.assertAlmostEqual(3.337153, result.ds.data[0], 7) self.assertAlmostEqual(3.3807921, result.ds.data[1], 7) self.assertAlmostEqual(3.4556863, result.ds.data[2], 7) self.assertEqual((3, ), result.dn.shape) self.assertAlmostEqual(0.662847, result.dn.data[0], 7) self.assertAlmostEqual(0.6192079, result.dn.data[1], 7) self.assertAlmostEqual(0.54431367, result.dn.data[2], 8) self.assertEqual((3, 10), result.K4.shape) self.assertAlmostEqual(0.05651569, result.K4.data[0, 9], 8) self.assertAlmostEqual(0.55655426, result.K4.data[1, 0], 7) self.assertAlmostEqual(0.1891833, result.K4.data[2, 1], 7) self.assertEqual((3, ), result.ite_index.shape) self.assertEqual(4, result.ite_index.data[0]) self.assertEqual(3, result.ite_index.data[1]) self.assertEqual(3, result.ite_index.data[2])
class FlagCodingTest(unittest.TestCase): flag_coding = None def setUp(self): self.flag_coding = FlagCoding(NUM_SAMPLES) def test_get_flag_masks(self): masks = FlagCoding.get_flag_masks() self.assertEqual("1,2,4,8,16,32,64,128,256,512,1024,2048,4096", masks) def test_get_flag_meanings(self): masks = FlagCoding.get_flag_meanings() self.assertEqual( "avg_inv_thresh amsre_flag bt_out_of_range ws_out_of_range inv_geolocation sza_out_of_range sst_out_of_range bt_pol_test_failed inv_file_name rfi_possible diurnal_warming rain_possible std_dev_too_high", masks) def test_get_flags_initial(self): flags = self.flag_coding.get_flags() self.assertIsNotNone(flags) for i in range(0, NUM_SAMPLES): self.assertEqual(0, flags[i]) def test_add_avg_inv_thresh(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[5] = True tags[19] = True self.flag_coding.add_avg_inv_thresh(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[0]) self.assertEqual(0, flags[1]) self.assertEqual(1, flags[5]) self.assertEqual(1, flags[19]) def test_add_amsre_flag(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[6] = True tags[20] = True self.flag_coding.add_amsre_flag(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[1]) self.assertEqual(0, flags[2]) self.assertEqual(2, flags[6]) self.assertEqual(2, flags[20]) def test_add_bt_out_of_range(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[7] = True tags[21] = True self.flag_coding.add_bt_out_of_range(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[2]) self.assertEqual(0, flags[3]) self.assertEqual(4, flags[7]) self.assertEqual(4, flags[21]) def test_add_ws_out_of_range(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[8] = True tags[22] = True self.flag_coding.add_ws_out_of_range(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[3]) self.assertEqual(0, flags[4]) self.assertEqual(8, flags[8]) self.assertEqual(8, flags[22]) def test_add_invald_geolocation(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[9] = True tags[23] = True self.flag_coding.add_invalid_geolocation(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[4]) self.assertEqual(0, flags[5]) self.assertEqual(16, flags[9]) self.assertEqual(16, flags[23]) def test_add_sza_out_of_range(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[10] = True tags[24] = True self.flag_coding.add_sza_out_of_range(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[4]) self.assertEqual(0, flags[5]) self.assertEqual(32, flags[10]) self.assertEqual(32, flags[24]) def test_add_sst_out_of_range(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[11] = True tags[25] = True self.flag_coding.add_sst_out_of_range(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[5]) self.assertEqual(0, flags[6]) self.assertEqual(64, flags[11]) self.assertEqual(64, flags[25]) def test_add_bt_pol_test_failed(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[12] = True tags[26] = True self.flag_coding.add_bt_pol_test_failed(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[6]) self.assertEqual(0, flags[7]) self.assertEqual(128, flags[12]) self.assertEqual(128, flags[26]) def test_add_inv_filename(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[13] = True tags[27] = True self.flag_coding.add_inv_filename(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[8]) self.assertEqual(0, flags[9]) self.assertEqual(256, flags[13]) self.assertEqual(256, flags[27]) def test_add_rfi_possible(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[14] = True tags[28] = True self.flag_coding.add_rfi_possible(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[9]) self.assertEqual(0, flags[10]) self.assertEqual(512, flags[14]) self.assertEqual(512, flags[28]) def test_add_diurnal_warming(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[15] = True tags[29] = True self.flag_coding.add_diurnal_warming(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[10]) self.assertEqual(0, flags[11]) self.assertEqual(1024, flags[15]) self.assertEqual(1024, flags[29]) def test_add_rain_possible(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[16] = True tags[30] = True self.flag_coding.add_rain_possible(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[11]) self.assertEqual(0, flags[12]) self.assertEqual(2048, flags[16]) self.assertEqual(2048, flags[30]) def test_add_stddev_too_high(self): tags = np.zeros(NUM_SAMPLES, dtype=np.bool) tags[17] = True tags[31] = True self.flag_coding.add_stddev_too_high(tags) flags = self.flag_coding.get_flags() self.assertEqual(0, flags[12]) self.assertEqual(0, flags[13]) self.assertEqual(4096, flags[17]) self.assertEqual(4096, flags[31])
def test_get_flag_meanings(self): masks = FlagCoding.get_flag_meanings() self.assertEqual( "avg_inv_thresh amsre_flag bt_out_of_range ws_out_of_range inv_geolocation sza_out_of_range sst_out_of_range bt_pol_test_failed inv_file_name rfi_possible diurnal_warming rain_possible std_dev_too_high", masks)
def test_get_flag_masks(self): masks = FlagCoding.get_flag_masks() self.assertEqual("1,2,4,8,16,32,64,128,256,512,1024,2048,4096", masks)
def setUp(self): self.flag_coding = FlagCoding(NUM_SAMPLES)