def test_create_file_name_FCDR_easy(self):
start = datetime.datetime(2015, 8, 23, 14, 24, 52)
end = datetime.datetime(2015, 8, 23, 15, 25, 53)
self.assertEqual(
"FIDUCEO_FCDR_L1C_MVIRI_MET7-0.00_20150823142452_20150823152553_EASY_v02.3_fv2.0.0.nc",
FCDRWriter.create_file_name_FCDR_easy("MVIRI", "MET7-0.00", start,
end, "02.3"))
start = datetime.datetime(2014, 7, 22, 13, 23, 51)
end = datetime.datetime(2014, 7, 22, 14, 24, 52)
self.assertEqual(
"FIDUCEO_FCDR_L1C_HIRS3_NOAA15_20140722132351_20140722142452_EASY_v03.4_fv2.0.0.nc",
FCDRWriter.create_file_name_FCDR_easy("HIRS3", "NOAA15", start,
end, "03.4"))
self.assertEqual(
"FIDUCEO_FCDR_L1C_HIRS3_NOAA15_20140722132351_20140722142452_EASY_v03.4_fv2.0.0.nc",
FCDRWriter.create_file_name_FCDR_easy("hirs3", "noaa15", start,
end, "03.4"))
start = datetime.datetime(2013, 6, 21, 12, 23, 50)
end = datetime.datetime(2013, 6, 21, 13, 23, 51)
self.assertEqual(
"FIDUCEO_FCDR_L1C_HIRS4_METOPA_20130621122350_20130621132351_EASY_v04.5_fv2.0.0.nc",
FCDRWriter.create_file_name_FCDR_easy("HIRS4", "METOPA", start,
end, "04.5"))
start = datetime.datetime(2012, 5, 20, 11, 22, 49)
end = datetime.datetime(2012, 5, 20, 12, 22, 50)
self.assertEqual(
"FIDUCEO_FCDR_L1C_AMSUB_NOAA17_20120520112249_20120520122250_EASY_v05.6_fv2.0.0.nc",
FCDRWriter.create_file_name_FCDR_easy("AMSUB", "NOAA17", start,
end, "05.6"))
def test_write_overwrite_true(self):
testFile = os.path.join(self.testDir, 'delete_me.nc')
FCDRWriter.write(self.dataset, testFile, overwrite=True)
self.assertTrue(os.path.isfile(testFile))
FCDRWriter.write(self.dataset, testFile, overwrite=True)
self.assertTrue(os.path.isfile(testFile))
def test_write_overwrite_false(self):
testFile = os.path.join(self.testDir, 'delete_me.nc')
FCDRWriter.write(self.dataset, testFile)
self.assertTrue(os.path.isfile(testFile))
try:
FCDRWriter.write(self.dataset, testFile, overwrite=False)
self.fail("IOError expected")
except IOError:
pass
def test_create_file_name_FCDR_full(self):
start = datetime.datetime(2015, 8, 7, 14, 24, 52)
end = datetime.datetime(2015, 8, 7, 15, 25, 53)
self.assertEqual(
"FIDUCEO_FCDR_L1C_MVIRI_MET7-0.00_20150807142452_20150807152553_FULL_v02.3_fv2.0.0.nc",
FCDRWriter.create_file_name_FCDR_full("MVIRI", "MET7-0.00", start,
end, "02.3"))
start = datetime.datetime(2014, 7, 21, 13, 23, 51)
end = datetime.datetime(2014, 7, 21, 14, 24, 52)
self.assertEqual(
"FIDUCEO_FCDR_L1C_HIRS3_NOAA15_20140721132351_20140721142452_FULL_v03.4_fv2.0.0.nc",
FCDRWriter.create_file_name_FCDR_full("HIRS3", "NOAA15", start,
end, "03.4"))
def test_write_easy(self):
mviri_easy = self.create_easy_dataset()
start = datetime(2011, 9, 12, 13, 24, 52)
end = datetime(2011, 9, 12, 13, 27, 51)
file_name = FCDRWriter.create_file_name_FCDR_easy("MVIRI", "Meteosat8", start, end, "1.0")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.now()
FCDRWriter.write(mviri_easy, self.target_path)
end_time = datetime.now()
elapsed_time = end_time - start_time
print("MVIRI EASY write time: " + str(elapsed_time.seconds) + "." + str(round(elapsed_time.microseconds / 1000)))
self.assertTrue(os.path.isfile(self.target_path))
target_data = xr.open_dataset(self.target_path)
try:
self.assert_global_flags(target_data)
self.assert_sensor_variables(target_data)
variable = target_data["toa_bidirectional_reflectance_vis"]
self.assertAlmostEqual(0.75, variable.data[25, 25], 5)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_independent_toa_bidirectional_reflectance"]
self.assertAlmostEqual(1.0400092904, variable.data[26, 26], 4)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_structured_toa_bidirectional_reflectance"]
self.assertAlmostEqual(1.3500070712000001, variable.data[27, 27], 4)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["sub_satellite_latitude_start"]
self.assertAlmostEqual(25.25, variable.data, 8)
variable = target_data["sub_satellite_longitude_start"]
self.assertAlmostEqual(26.26, variable.data, 8)
variable = target_data["sub_satellite_latitude_end"]
self.assertAlmostEqual(27.27, variable.data, 8)
variable = target_data["sub_satellite_longitude_end"]
self.assertAlmostEqual(28.28, variable.data, 8)
finally:
target_data.close()
def testCreateTemplateFull_AMSUB(self):
ds = FCDRWriter.createTemplateFull('AMSUB', 2562)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(27, len(ds.data_vars))
# geolocation + flags
self._verify_geolocation_variables(ds)
self._verify_quality_flags(ds)
# sensor specific
self._verify_amsub_specific_variables(ds)
# full FCDR variables
self.assertIsNotNone(ds.variables["u_btemps"])
self.assertIsNotNone(ds.variables["u_syst_btemps"])
self.assertIsNotNone(ds.variables["u_random_btemps"])
self.assertIsNotNone(ds.variables["u_instrtemp"])
self.assertIsNotNone(ds.variables["u_latitude"])
self.assertIsNotNone(ds.variables["u_longitude"])
self.assertIsNotNone(ds.variables["u_satellite_azimuth_angle"])
self.assertIsNotNone(ds.variables["u_satellite_zenith_angle"])
self.assertIsNotNone(ds.variables["u_solar_azimuth_angle"])
self.assertIsNotNone(ds.variables["u_solar_zenith_angle"])
def create_easy_dataset(self):
avhrr_easy = FCDRWriter.createTemplateEasy("AVHRR", PRODUCT_HEIGHT)
self.add_global_attributes(avhrr_easy)
self.add_geolocation_data(avhrr_easy)
self.add_global_flags(avhrr_easy)
self.add_sensor_data(avhrr_easy)
self.add_srf_data(avhrr_easy)
for x in range(0, PRODUCT_WIDTH):
avhrr_easy["u_independent_Ch1"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.013
avhrr_easy["u_independent_Ch2"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.014
avhrr_easy["u_independent_Ch3a"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.015
avhrr_easy["u_independent_Ch4"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.016
avhrr_easy["u_independent_Ch5"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.017
avhrr_easy["u_structured_Ch1"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.018
avhrr_easy["u_structured_Ch2"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.019
avhrr_easy["u_structured_Ch3a"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.020
avhrr_easy["u_structured_Ch4"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.021
avhrr_easy["u_structured_Ch5"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.022
avhrr_easy["u_common_Ch1"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.003
avhrr_easy["u_common_Ch2"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.004
avhrr_easy["u_common_Ch3a"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.005
avhrr_easy["u_common_Ch4"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.006
avhrr_easy["u_common_Ch5"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.007
for x in range(0, 6):
avhrr_easy["quality_channel_bitmask"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int8) * x
avhrr_easy["quality_scanline_bitmask"].data[:] = np.ones(PRODUCT_HEIGHT, np.int8)
return avhrr_easy
def testCreateTemplateEasy_HIRS4(self):
ds = FCDRWriter.createTemplateEasy('HIRS4', 211)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(22, len(ds.data_vars))
# geolocation + flags
self._verify_geolocation_variables(ds)
self._verify_quality_flags(ds)
# sensor specific
self.assertIsNotNone(ds.variables["bt"])
self.assertIsNotNone(ds.variables["satellite_zenith_angle"])
self.assertIsNotNone(ds.variables["satellite_azimuth_angle"])
self.assertIsNotNone(ds.variables["solar_zenith_angle"])
self.assertIsNotNone(ds.variables["solar_azimuth_angle"])
self.assertIsNotNone(ds.variables["scanline"])
self.assertIsNotNone(ds.variables["time"])
self.assertIsNotNone(ds.variables["data_quality_bitmask"])
self.assertIsNotNone(ds.variables["quality_scanline_bitmask"])
self.assertIsNotNone(ds.variables["quality_channel_bitmask"])
self.assertIsNotNone(ds.variables["SRF_weights"])
self.assertIsNotNone(ds.variables["SRF_wavelengths"])
self.assertIsNotNone(ds.variables["scanline_map_to_origl1bfile"])
self.assertIsNotNone(ds.variables["scanline_origl1b"])
self._assert_HIRS_easy_uncertainties(ds)
def create_full_dataset(self):
mviri_full = FCDRWriter.createTemplateFull("MVIRI", 5000)
self.add_global_attributes(mviri_full)
self.add_global_flags(mviri_full)
self.add_sensor_data(mviri_full)
for x in range(0, 5000):
mviri_full["count_vis"].data[:, x] = np.ones((5000), np.uint8) * x
for x in range(0, 500):
mviri_full["u_latitude"].data[:, x] = np.ones((500), np.float32) * x * 0.1
mviri_full["u_longitude"].data[:, x] = np.ones((500), np.float32) * x * 0.2
mviri_full["u_satellite_zenith_angle"].data[:, x] = np.ones((500), np.float32) * x * 0.3
mviri_full["u_satellite_azimuth_angle"].data[:, x] = np.ones((500), np.float32) * x * 0.4
mviri_full["u_solar_zenith_angle"].data[:, x] = np.ones((500), np.float32) * x * 0.5
mviri_full["u_solar_azimuth_angle"].data[:, x] = np.ones((500), np.float32) * x * 0.6
mviri_full["u_time"].data[:] = np.ones((2500), np.float32) * 0.4
mviri_full["covariance_a_vis"].data = np.ones((3, 3), np.float32) * 0.56
mviri_full["a0_vis"].data = 7.7
mviri_full["a1_vis"].data = 8.8
mviri_full["mean_count_space_vis"].data = 9.9
mviri_full["u_a0_vis"].data = 10.1
mviri_full["u_a1_vis"].data = 11.11
mviri_full["u_a2_vis"].data = 12.12
mviri_full["u_zero_vis"].data = 13.13
mviri_full["u_electronics_counts_vis"].data = 13.13
mviri_full["u_digitization_counts_vis"].data = 14.14
mviri_full["allan_deviation_counts_space_vis"].data = 15.15
return mviri_full
def testCreateTemplateEasy_MVIRI(self):
ds = FCDRWriter.createTemplateEasy('MVIRI', 5000)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self._verify_quality_flags(ds)
self.assertEqual(40, len(ds.data_vars))
# sensor specific
self.assertIsNotNone(ds.variables["time"])
self.assertIsNotNone(ds.variables["SRF_weights"])
self.assertIsNotNone(ds.variables["SRF_frequencies"])
self.assertIsNotNone(
ds.variables["covariance_spectral_response_function_vis"])
self.assertIsNotNone(ds.variables["u_spectral_response_function_ir"])
self.assertIsNotNone(ds.variables["u_spectral_response_function_wv"])
self.assertIsNotNone(ds.variables["solar_zenith_angle"])
self.assertIsNotNone(ds.variables["solar_azimuth_angle"])
self.assertIsNotNone(ds.variables["satellite_azimuth_angle"])
self.assertIsNotNone(ds.variables["satellite_zenith_angle"])
self.assertIsNotNone(ds.variables["count_ir"])
self.assertIsNotNone(ds.variables["count_wv"])
self.assertIsNotNone(ds.variables["data_quality_bitmask"])
self.assertIsNotNone(ds.variables["distance_sun_earth"])
self.assertIsNotNone(ds.variables["solar_irradiance_vis"])
self.assertIsNotNone(ds.variables["u_solar_irradiance_vis"])
self.assertIsNotNone(ds.variables["a_ir"])
self.assertIsNotNone(ds.variables["b_ir"])
self.assertIsNotNone(ds.variables["u_a_ir"])
self.assertIsNotNone(ds.variables["u_b_ir"])
self.assertIsNotNone(ds.variables["a_wv"])
self.assertIsNotNone(ds.variables["b_wv"])
self.assertIsNotNone(ds.variables["u_a_wv"])
self.assertIsNotNone(ds.variables["u_b_wv"])
self.assertIsNotNone(ds.variables["bt_a_ir"])
self.assertIsNotNone(ds.variables["bt_b_ir"])
self.assertIsNotNone(ds.variables["bt_a_wv"])
self.assertIsNotNone(ds.variables["bt_b_wv"])
self.assertIsNotNone(ds.variables["years_since_launch"])
# easy FCDR uncertainties
self.assertIsNotNone(ds.variables["toa_bidirectional_reflectance_vis"])
self.assertIsNotNone(
ds.variables["u_independent_toa_bidirectional_reflectance"])
self.assertIsNotNone(
ds.variables["u_structured_toa_bidirectional_reflectance"])
self.assertIsNotNone(
ds.variables["u_common_toa_bidirectional_reflectance"])
self.assertIsNotNone(ds.variables["sub_satellite_latitude_start"])
self.assertIsNotNone(ds.variables["sub_satellite_longitude_start"])
self.assertIsNotNone(ds.variables["sub_satellite_latitude_end"])
self.assertIsNotNone(ds.variables["sub_satellite_longitude_end"])
self.assertIsNotNone(
ds.variables["channel_correlation_matrix_independent"])
self.assertIsNotNone(
ds.variables["channel_correlation_matrix_structured"])
def create_easy_dataset(self, type):
hirs_easy = FCDRWriter.createTemplateEasy(type, 944, SRF_SIZE)
hirs_easy.attrs["institution"] = "test"
hirs_easy.attrs["title"] = "sir"
hirs_easy.attrs["source"] = "invention"
hirs_easy.attrs["history"] = "new"
hirs_easy.attrs["references"] = "myself"
hirs_easy.attrs["comment"] = "should define a test version of this set"
for x in range(0, 56):
hirs_easy["bt"].data[:, :, x] = np.ones((944), np.int16) * x * 0.01
hirs_easy["latitude"].data[:, x] = np.ones(
(944), np.int16) * x * 0.02
hirs_easy["longitude"].data[:, x] = np.ones(
(944), np.int16) * x * 0.03
hirs_easy["data_quality_bitmask"].data[:, x] = np.ones(
(944), np.int8) * x
hirs_easy["quality_pixel_bitmask"].data[:, x] = np.ones(
(944), np.int8) * x
if type != "HIRS2":
hirs_easy["satellite_zenith_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.04
hirs_easy["satellite_azimuth_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.05
hirs_easy["solar_zenith_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.06
hirs_easy["solar_azimuth_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.05
hirs_easy["u_independent"].data[:, :, x] = np.ones(
(944), np.int16) * x * 0.06
hirs_easy["u_structured"].data[:, :, x] = np.ones(
(944), np.int16) * x * 0.07
hirs_easy["u_common"].data[:, :, x] = np.ones(
(944), np.int16) * x * 0.08
if type != "HIRS2":
for x in range(0, 19):
hirs_easy["quality_channel_bitmask"].data[:, x] = np.ones(
(944), np.int32) * x * 3
hirs_easy["quality_scanline_bitmask"].data[:] = np.ones((944), np.int8)
hirs_easy["scanline"].data[:] = np.ones((944), np.int8) * 3
hirs_easy["time"].data[:] = np.ones((944), np.int32) * 4
for x in range(0, SRF_SIZE):
hirs_easy["SRF_weights"].data[:, x] = np.ones(
(NUM_CHANNELS), np.float32) * x * 0.04
hirs_easy["SRF_wavelengths"].data[:, x] = np.ones(
(NUM_CHANNELS), np.float32) * x * 0.05
if type == "HIRS2":
hirs_easy["satellite_zenith_angle"].data[:] = np.ones(
(944), np.int8) * 2
return hirs_easy
def testCreateTemplateFull_HIRS3(self):
ds = FCDRWriter.createTemplateFull('HIRS3', 209)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(98, len(ds.data_vars))
# geolocation + flags
self._verify_geolocation_variables(ds)
self._verify_quality_flags(ds)
self._assert_HIRS_common_variables(ds)
self.assertIsNotNone(ds.variables["satellite_azimuth_angle"])
self.assertIsNotNone(ds.variables["solar_zenith_angle"])
self.assertIsNotNone(ds.variables["scanline_map_to_origl1bfile"])
self.assertIsNotNone(ds.variables["scanline_origl1b"])
self.assertIsNotNone(ds.variables["data_quality_bitmask"])
self.assertIsNotNone(ds.variables["c_earth"])
self.assertIsNotNone(ds.variables["L_earth"])
self.assertIsNotNone(ds.variables["u_L_earth_independent"])
self.assertIsNotNone(ds.variables["navigation_status"])
self.assertIsNotNone(ds.variables["platform_altitude"])
self.assertIsNotNone(ds.variables["platform_pitch_angle"])
self.assertIsNotNone(ds.variables["platform_roll_angle"])
self.assertIsNotNone(ds.variables["platform_yaw_angle"])
self.assertIsNotNone(ds.variables["quality_flags"])
self.assertIsNotNone(ds.variables["scan_angles"])
self.assertIsNotNone(ds.variables["l1b_scanline_number"])
self.assertIsNotNone(ds.variables["scanline_position"])
self.assertIsNotNone(
ds.variables["l1b_second_original_calibration_coefficients"])
self.assertIsNotNone(ds.variables["u_c_earth"])
self.assertIsNotNone(ds.variables["u_c_earth_chan_corr"])
self.assertIsNotNone(ds.variables["u_c_space"])
self.assertIsNotNone(ds.variables["u_c_space_chan_corr"])
self.assertIsNotNone(ds.variables["u_Earthshine"])
self.assertIsNotNone(ds.variables["u_O_Re"])
self.assertIsNotNone(ds.variables["u_O_TIWCT"])
self.assertIsNotNone(ds.variables["u_O_TPRT"])
self.assertIsNotNone(ds.variables["u_d_PRT"])
self.assertIsNotNone(ds.variables["u_electronics"])
self.assertIsNotNone(ds.variables["u_nonlinearity"])
self.assertIsNotNone(ds.variables["temp_corr_slope"])
self.assertIsNotNone(ds.variables["temp_corr_offset"])
self.assertIsNotNone(ds.variables["emissivity"])
self.assertIsNotNone(ds.variables["mnfrqualflags"])
def testCreateTemplateEasy_AVHRR(self):
ds = FCDRWriter.createTemplateEasy('AVHRR',
12198,
srf_size=13,
corr_dx=14,
corr_dy=15)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(42, len(ds.data_vars))
# geolocation + flags
self._verify_geolocation_variables(ds)
self._verify_quality_flags(ds)
# sensor specific
self._verify_avhrr_specific_variables(ds)
# easy FCDR variables
self.assertIsNotNone(ds.variables["u_independent_Ch1"])
self.assertIsNotNone(ds.variables["u_structured_Ch1"])
self.assertIsNotNone(ds.variables["u_common_Ch1"])
self.assertIsNotNone(ds.variables["u_independent_Ch2"])
self.assertIsNotNone(ds.variables["u_structured_Ch2"])
self.assertIsNotNone(ds.variables["u_common_Ch2"])
self.assertIsNotNone(ds.variables["u_independent_Ch3a"])
self.assertIsNotNone(ds.variables["u_structured_Ch3a"])
self.assertIsNotNone(ds.variables["u_common_Ch3a"])
self.assertIsNotNone(ds.variables["u_independent_Ch3b"])
self.assertIsNotNone(ds.variables["u_structured_Ch3b"])
self.assertIsNotNone(ds.variables["u_common_Ch3b"])
self.assertIsNotNone(ds.variables["u_independent_Ch4"])
self.assertIsNotNone(ds.variables["u_structured_Ch4"])
self.assertIsNotNone(ds.variables["u_common_Ch4"])
self.assertIsNotNone(ds.variables["u_independent_Ch5"])
self.assertIsNotNone(ds.variables["u_structured_Ch5"])
self.assertIsNotNone(ds.variables["u_common_Ch5"])
self.assertIsNotNone(
ds.variables["channel_correlation_matrix_independent"])
self.assertIsNotNone(
ds.variables["channel_correlation_matrix_structured"])
self.assertIsNotNone(
ds.variables["cross_element_correlation_coefficients"])
self.assertIsNotNone(
ds.variables["cross_line_correlation_coefficients"])
def create_easy_dataset(self):
mviri_easy = FCDRWriter.createTemplateEasy("MVIRI", 5000)
self.add_global_attributes(mviri_easy)
self.add_global_flags(mviri_easy)
self.add_sensor_data(mviri_easy)
for x in range(0, 5000):
mviri_easy["toa_bidirectional_reflectance_vis"].data[:, x] = np.ones((5000), np.uint16) * x * 0.03
mviri_easy["u_independent_toa_bidirectional_reflectance"].data[:, x] = np.ones((5000), np.uint16) * x * 0.04
mviri_easy["u_structured_toa_bidirectional_reflectance"].data[:, x] = np.ones((5000), np.uint16) * x * 0.05
mviri_easy["sub_satellite_latitude_start"].data = 25.25
mviri_easy["sub_satellite_longitude_start"].data = 26.26
mviri_easy["sub_satellite_latitude_end"].data = 27.27
mviri_easy["sub_satellite_longitude_end"].data = 28.28
return mviri_easy
def testCreateTemplateEasy_SSMT2(self):
ds = FCDRWriter.createTemplateEasy('SSMT2', 722)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(15, len(ds.data_vars))
# geolocation + flags
self._verify_geolocation_variables(ds)
self._verify_quality_flags(ds)
# sensor specific
self.verify_SSMT2_specific_variables(ds)
# easy FCDR variables
self.assertIsNotNone(ds.variables["u_independent_tb"])
self.assertIsNotNone(ds.variables["u_structured_tb"])
def testCreateTemplateEasy_AMSUB(self):
ds = FCDRWriter.createTemplateEasy('AMSUB', 2561)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(19, len(ds.data_vars))
# geolocation + flags
self._verify_geolocation_variables(ds)
self._verify_quality_flags(ds)
# sensor specific
self._verify_amsub_specific_variables(ds)
# easy FCDR variables
self.assertIsNotNone(ds.variables["u_independent_btemps"])
self.assertIsNotNone(ds.variables["u_structured_btemps"])
def main():
writer = FCDRWriter()
# get a template for sensor name in FULL format, supply product height
# The scan-width is set automatically
# ---------------------------------------------------------------------
dataset = writer.createTemplateFull("AVHRR", 128)
# set some mandatory global attributes (CF standards). Writing will fail if not all of them are filled
# automatically set: CF version and FIDUCEO license
# ----------------------------------------------------------------------------------------------------
dataset.attrs["institution"] = "Brockmann Consult GmbH"
dataset.attrs["title"] = "FIDUCEO test dataset"
dataset.attrs["source"] = "arbitray stuff"
dataset.attrs["history"] = "none"
dataset.attrs["references"] = "CDR_FCDR sensor reference documentation"
dataset.attrs[
"comment"] = "just to show how things are intended to be used"
# write real data to the variables. All variables initially contain "_FillValue".
# Not writing to the whole array is completely OK
# -------------------------------------------------------------------------------
Time = dataset.variables["Time"]
Time.data[44] = 0.456
Time.data[45] = 0.457
raa = dataset.variables["relative_azimuth_angle"]
raa.data[3, 0] = 0.567
raa.data[3, 1] = 0.568
# ensure not to generate over/underflows
# --------------------------------------
DataUtility.check_scaling_ranges(raa)
# create a standardized file name
# -------------------------------
start = datetime.datetime(2006, 8, 23, 14, 24, 52)
end = datetime.datetime(2006, 8, 23, 15, 25, 53)
file_name = writer.create_file_name_FCDR_full("AVHRR", "NOAA12", start,
end, "01.2")
# dump it to disk, netcdf4, medium compression
# overwrite existing file
# --------------------------------------------
writer.write(dataset,
"D:\\Satellite\\DELETE\\" + file_name,
overwrite=True)
def testCreateTemplateFull_SSMT2(self):
ds = FCDRWriter.createTemplateFull('SSMT2', 722)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(21, len(ds.data_vars))
# geolocation + flags
self._verify_geolocation_variables(ds)
self._verify_quality_flags(ds)
# sensor specific
self.verify_SSMT2_specific_variables(ds)
# easy FCDR variables
self.assertIsNotNone(ds.variables["u_Temperature_misc_housekeeping"])
self.assertIsNotNone(ds.variables["u_cold_counts"])
self.assertIsNotNone(ds.variables["u_counts_to_tb_gain"])
self.assertIsNotNone(ds.variables["u_counts_to_tb_offset"])
self.assertIsNotNone(ds.variables["u_gain_control"])
self.assertIsNotNone(ds.variables["u_tb"])
self.assertIsNotNone(ds.variables["u_thermal_reference"])
self.assertIsNotNone(ds.variables["u_warm_counts"])
def test_write_easy(self):
avhrr_easy = self.create_easy_dataset()
start = datetime.datetime(2016, 11, 22, 13, 24, 52)
end = datetime.datetime(2016, 11, 22, 14, 25, 53)
file_name = FCDRWriter.create_file_name_FCDR_easy("AVHRR", "NOAA18", start, end, "1.0")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.datetime.now()
FCDRWriter.write(avhrr_easy, self.target_path)
end_time = datetime.datetime.now()
elapsed_time = end_time - start_time
print("AVHRR EASY write time: " + str(elapsed_time.seconds) + "." + str(round(elapsed_time.microseconds / 1000)))
self.assertTrue(os.path.isfile(self.target_path))
# target_data = xr.open_dataset(self.target_path, chunks=128)
target_data = xr.open_dataset(self.target_path)
try:
self.assert_geolocation_variables(target_data)
self.assert_global_flags(target_data)
self.assert_sensor_variables(target_data)
variable = target_data["relative_azimuth_angle"]
self.assertAlmostEqual(0.11, variable.data[11, 11], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_independent_Ch1"]
self.assertAlmostEqual(0.182, variable.data[14, 14], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_independent_Ch2"]
self.assertAlmostEqual(0.21, variable.data[15, 15], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_independent_Ch3a"]
self.assertAlmostEqual(0.24, variable.data[16, 16], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_independent_Ch4"]
self.assertAlmostEqual(0.272, variable.data[17, 17], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_independent_Ch5"]
self.assertAlmostEqual(0.306, variable.data[18, 18], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_structured_Ch1"]
self.assertAlmostEqual(-0.31336, variable.data[19, 19], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_structured_Ch2"]
self.assertAlmostEqual(-0.27536, variable.data[20, 20], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_structured_Ch3a"]
self.assertAlmostEqual(-0.23536, variable.data[21, 21], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_structured_Ch4"]
self.assertAlmostEqual(0.462, variable.data[22, 22], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_structured_Ch5"]
self.assertAlmostEqual(0.506, variable.data[23, 23], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_common_Ch1"]
self.assertAlmostEqual(0.072, variable.data[24, 24], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_common_Ch2"]
self.assertAlmostEqual(0.1, variable.data[25, 25], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_common_Ch3a"]
self.assertAlmostEqual(0.13, variable.data[26, 26], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_common_Ch4"]
self.assertAlmostEqual(0.162, variable.data[27, 27], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_common_Ch5"]
self.assertAlmostEqual(0.196, variable.data[28, 28], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
self.assert_srf_data(target_data)
finally:
target_data.close()
def test_write_full(self):
avhrr_full = self.create_full_dataset()
start = datetime.datetime(2016, 11, 22, 14, 25, 53)
end = datetime.datetime(2016, 11, 22, 15, 26, 54)
file_name = FCDRWriter.create_file_name_FCDR_full("AVHRR", "NOAA19", start, end, "1.0")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.datetime.now()
FCDRWriter.write(avhrr_full, self.target_path)
end_time = datetime.datetime.now()
elapsed_time = end_time - start_time
print("AVHRR FULL write time: " + str(elapsed_time.seconds) + "." + str(round(elapsed_time.microseconds / 1000)))
self.assertTrue(os.path.isfile(self.target_path))
target_data = xr.open_dataset(self.target_path)
try:
self.assert_geolocation_variables(target_data)
self.assert_global_flags(target_data)
self.assert_sensor_variables(target_data)
variable = target_data["u_latitude"]
self.assertAlmostEqual(3.36, variable.data[24, 24], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_longitude"]
self.assertAlmostEqual(3.75, variable.data[25, 25], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_time"]
self.assertAlmostEqual(0.16, variable.data[256], 8)
variable = target_data["u_satellite_azimuth_angle"]
self.assertAlmostEqual(4.32, variable.data[27, 27], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_satellite_zenith_angle"]
self.assertAlmostEqual(4.76, variable.data[28, 28], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_solar_azimuth_angle"]
self.assertAlmostEqual(5.22, variable.data[29, 29], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_solar_zenith_angle"]
self.assertAlmostEqual(5.7, variable.data[30, 30], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["PRT_C"]
self.assertEqual(0, variable.data[31, 0])
self.assertEqual((13198, 3), variable.encoding["chunksizes"])
variable = target_data["u_prt"]
self.assertAlmostEqual(0.2, variable.data[32, 1], 8)
self.assertEqual((13198, 3), variable.encoding["chunksizes"])
variable = target_data["R_ICT"]
self.assertAlmostEqual(0.42, variable.data[33, 2], 7)
self.assertEqual((13198, 3), variable.encoding["chunksizes"])
variable = target_data["T_instr"]
self.assertAlmostEqual(0.17, variable.data[257], 8)
variable = target_data["Ch1_Csp"]
self.assertEqual(68, variable.data[34, 34])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch2_Csp"]
self.assertEqual(105, variable.data[35, 35])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3a_Csp"]
self.assertEqual(144, variable.data[36, 36])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_Csp"]
self.assertEqual(185, variable.data[37, 37])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_Csp"]
self.assertEqual(228, variable.data[38, 38])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_Csp"]
self.assertEqual(273, variable.data[39, 39])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_Cict"]
self.assertEqual(320, variable.data[40, 40])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_Cict"]
self.assertEqual(369, variable.data[41, 41])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_Cict"]
self.assertEqual(420, variable.data[42, 42])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch1_Ce"]
self.assertEqual(473, variable.data[43, 43])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch2_Ce"]
self.assertEqual(528, variable.data[44, 44])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3a_Ce"]
self.assertEqual(585, variable.data[45, 45])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_Ce"]
self.assertEqual(644, variable.data[46, 46])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_Ce"]
self.assertEqual(705, variable.data[47, 47])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_Ce"]
self.assertEqual(768, variable.data[48, 48])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch1_u_Csp"]
self.assertAlmostEqual(8.33, variable.data[49, 49], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch2_u_Csp"]
self.assertAlmostEqual(9.0, variable.data[50, 50], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3a_u_Csp"]
self.assertAlmostEqual(9.69, variable.data[51, 51], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_u_Csp"]
self.assertAlmostEqual(10.4, variable.data[52, 52], 5)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_u_Csp"]
self.assertAlmostEqual(11.13, variable.data[53, 53], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_u_Csp"]
self.assertAlmostEqual(11.88, variable.data[54, 54], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_u_Cict"]
self.assertAlmostEqual(12.65, variable.data[55, 55], 5)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_u_Cict"]
self.assertAlmostEqual(13.44, variable.data[56, 56], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_u_Cict"]
self.assertAlmostEqual(14.25, variable.data[57, 57], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch1_u_Ce"]
self.assertAlmostEqual(15.08, variable.data[58, 58], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch2_u_Ce"]
self.assertAlmostEqual(15.93, variable.data[59, 59], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3a_u_Ce"]
self.assertAlmostEqual(16.8, variable.data[60, 60], 5)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_u_Ce"]
self.assertAlmostEqual(17.69, variable.data[61, 61], 5)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_u_Ce"]
self.assertAlmostEqual(18.6, variable.data[62, 62], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_u_Ce"]
self.assertAlmostEqual(19.53, variable.data[63, 63], 5)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch1_u_Refl"]
self.assertAlmostEqual(20.48, variable.data[64, 64], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch2_u_Refl"]
self.assertAlmostEqual(21.45, variable.data[65, 65], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3a_u_Refl"]
self.assertAlmostEqual(22.44, variable.data[66, 66], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_u_Bt"]
self.assertAlmostEqual(23.45, variable.data[67, 67], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_u_Bt"]
self.assertAlmostEqual(24.48, variable.data[68, 68], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_u_Bt"]
self.assertAlmostEqual(25.53, variable.data[69, 69], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_ur_Bt"]
self.assertAlmostEqual(26.6, variable.data[70, 70], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_ur_Bt"]
self.assertAlmostEqual(27.69, variable.data[71, 71], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_ur_Bt"]
self.assertAlmostEqual(28.8, variable.data[72, 72], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_us_Bt"]
self.assertAlmostEqual(29.93, variable.data[73, 73], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_us_Bt"]
self.assertAlmostEqual(31.08, variable.data[74, 74], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_us_Bt"]
self.assertAlmostEqual(32.25, variable.data[75, 75], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
finally:
target_data.close()
def create_easy_dataset(self, type):
hirs_easy = FCDRWriter.createTemplateEasy(type, 944, SRF_SIZE)
hirs_easy.attrs["institution"] = "test"
hirs_easy.attrs["title"] = "sir"
hirs_easy.attrs["source"] = "invention"
hirs_easy.attrs["history"] = "new"
hirs_easy.attrs["references"] = "myself"
hirs_easy.attrs["project"] = "test-project"
hirs_easy.attrs["creator_url"] = "test-url"
hirs_easy.attrs["creator_name"] = "test-name"
hirs_easy.attrs["creator_email"] = "*****@*****.**"
hirs_easy.attrs["fcdr_software_version"] = "12"
hirs_easy.attrs["data_version"] = "13"
hirs_easy.attrs["time_coverage_start"] = "now"
hirs_easy.attrs["time_coverage_end"] = "later"
hirs_easy.attrs["time_coverage_duration"] = "some moment"
hirs_easy.attrs["platform"] = "yo"
hirs_easy.attrs["comment"] = "what?"
for x in range(0, 56):
hirs_easy["bt"].data[:, :, x] = np.ones((944), np.int16) * x * 0.01
hirs_easy["latitude"].data[:, x] = np.ones(
(944), np.int16) * x * 0.02
hirs_easy["longitude"].data[:, x] = np.ones(
(944), np.int16) * x * 0.03
hirs_easy["data_quality_bitmask"].data[:, x] = np.ones(
(944), np.int8) * x
hirs_easy["quality_pixel_bitmask"].data[:, x] = np.ones(
(944), np.int8) * x
hirs_easy["satellite_zenith_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.04
hirs_easy["satellite_azimuth_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.05
hirs_easy["solar_zenith_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.06
hirs_easy["solar_azimuth_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.05
hirs_easy["u_independent"].data[:, :, x] = np.ones(
(944), np.int16) * x * 0.06
hirs_easy["u_structured"].data[:, :, x] = np.ones(
(944), np.int16) * x * 0.07
hirs_easy["u_common"].data[:, :, x] = np.ones(
(944), np.int16) * x * 0.08
if type != "HIRS2":
for x in range(0, 19):
hirs_easy["quality_channel_bitmask"].data[:, x] = np.ones(
(944), np.int32) * x * 3
hirs_easy["quality_scanline_bitmask"].data[:] = np.ones((944), np.int8)
hirs_easy["scanline"].data[:] = np.ones((944), np.int8) * 3
hirs_easy["time"].data[:] = np.ones((944), np.int32) * 400000000000000
for x in range(0, SRF_SIZE):
hirs_easy["SRF_weights"].data[:, x] = np.ones(
(NUM_CHANNELS), np.float32) * x * 0.04
hirs_easy["SRF_wavelengths"].data[:, x] = np.ones(
(NUM_CHANNELS), np.float32) * x * 0.05
return hirs_easy
def test_write_HIRS2(self):
hirs_easy = self.create_easy_dataset("HIRS2")
start = datetime(2015, 10, 21, 13, 24, 52)
end = datetime(2015, 10, 21, 14, 25, 53)
file_name = FCDRWriter.create_file_name_FCDR_easy(
"HIRS2", "NOAA12", start, end, "1.0")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.now()
FCDRWriter.write(hirs_easy, self.target_path)
end_time = datetime.now()
elapsed_time = end_time - start_time
print("HIRS/2 EASY write time: " + "%d.%03d" %
(elapsed_time.seconds,
int(round(elapsed_time.microseconds / 1000))))
self.assertTrue(os.path.isfile(self.target_path))
target_data = xr.open_dataset(self.target_path)
try:
variable = target_data["bt"]
self.assertAlmostEqual(0.0, variable.data[0, 0, 0], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
variable = target_data["latitude"]
self.assertAlmostEqual(0.0192266606, variable.data[1, 1], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["longitude"]
self.assertAlmostEqual(0.0604266487, variable.data[2, 2], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["quality_pixel_bitmask"]
self.assertEqual(3, variable.data[3, 3])
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["data_quality_bitmask"]
self.assertEqual(2, variable.data[9, 2])
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["quality_scanline_bitmask"]
self.assertEqual(1, variable.data[4])
self.assertEqual((944, ), variable.encoding["chunksizes"])
variable = target_data["satellite_zenith_angle"]
self.assertAlmostEqual(0.2, variable.data[5, 5], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["satellite_azimuth_angle"]
self.assertAlmostEqual(0.30, variable.data[6, 6], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["scanline"]
self.assertEqual(3, variable.data[6])
self.assertEqual((944, ), variable.encoding["chunksizes"])
variable = target_data["solar_azimuth_angle"]
self.assertAlmostEqual(0.35, variable.data[7, 7], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["solar_azimuth_angle"]
self.assertAlmostEqual(0.4, variable.data[8, 8], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["solar_zenith_angle"]
self.assertAlmostEqual(0.54, variable.data[9, 9], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["time"]
self.assertEqual(np.datetime64('1970-01-05T15:06:40.000000000'),
variable.data[8])
self.assertEqual((944, ), variable.encoding["chunksizes"])
variable = target_data["u_independent"]
self.assertAlmostEqual(0.54, variable.data[9, 9, 9], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
variable = target_data["u_structured"]
self.assertAlmostEqual(0.7, variable.data[10, 10, 10], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
variable = target_data["u_common"]
self.assertAlmostEqual(0.88, variable.data[11, 11, 11], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
self.assert_srf(target_data)
finally:
target_data.close()
def setUp(self):
self.fcdr_reader = FCDRReader()
fcdr_writer = FCDRWriter()
self.dataset = fcdr_writer.createTemplateFull("MVIRI", 5000)
def test_write_full(self):
mviri_full = self.create_full_dataset()
start = datetime(2010, 8, 11, 13, 24, 52)
end = datetime(2010, 8, 11, 13, 27, 51)
file_name = FCDRWriter.create_file_name_FCDR_full("MVIRI", "Meteosat7", start, end, "1.0")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.now()
FCDRWriter.write(mviri_full, self.target_path)
end_time = datetime.now()
elapsed_time = end_time - start_time
print("MVIRI FULL write time: " + str(elapsed_time.seconds) + "." + str(round(elapsed_time.microseconds / 1000)))
self.assertTrue(os.path.isfile(self.target_path))
target_data = xr.open_dataset(self.target_path)
try:
self.assert_global_flags(target_data)
self.assert_sensor_variables(target_data)
variable = target_data["count_vis"]
self.assertEqual(11, variable.data[11, 11])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_latitude"]
self.assertAlmostEqual(0.21696, variable.data[12, 12], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_longitude"]
self.assertAlmostEqual(0.633915, variable.data[13, 13], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_time"]
self.assertAlmostEqual(0.402832012, variable.data[14], 8)
self.assertEqual((2500,), variable.encoding["chunksizes"])
variable = target_data["u_satellite_zenith_angle"]
self.assertAlmostEqual(4.4999668098, variable.data[15, 15], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_satellite_azimuth_angle"]
self.assertAlmostEqual(1.399993065, variable.data[16, 16], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_solar_zenith_angle"]
self.assertAlmostEqual(3.4999826625, variable.data[17, 17], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_solar_azimuth_angle"]
self.assertAlmostEqual(0.8000178354, variable.data[18, 18], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["a0_vis"]
self.assertAlmostEqual(7.7, variable.data, 8)
variable = target_data["a1_vis"]
self.assertAlmostEqual(8.8, variable.data, 8)
variable = target_data["mean_count_space_vis"]
self.assertAlmostEqual(9.9, variable.data, 8)
variable = target_data["u_a0_vis"]
self.assertAlmostEqual(10.1, variable.data, 8)
variable = target_data["u_a1_vis"]
self.assertAlmostEqual(11.11, variable.data, 8)
variable = target_data["u_a2_vis"]
self.assertAlmostEqual(12.12, variable.data, 8)
variable = target_data["u_zero_vis"]
self.assertAlmostEqual(13.13, variable.data, 8)
variable = target_data["covariance_a_vis"]
self.assertEqual((3,3), variable.shape)
self.assertAlmostEqual(0.56, variable.data[1, 2], 8)
variable = target_data["u_electronics_counts_vis"]
self.assertAlmostEqual(13.13, variable.data, 8)
variable = target_data["u_digitization_counts_vis"]
self.assertAlmostEqual(14.14, variable.data, 8)
variable = target_data["allan_deviation_counts_space_vis"]
self.assertAlmostEqual(15.15, variable.data, 8)
finally:
target_data.close()
def testCreateTemplateFull_MVIRI(self):
ds = FCDRWriter.createTemplateFull('MVIRI', 5000)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(58, len(ds.data_vars))
self._verify_quality_flags(ds)
# sensor specific
self.assertIsNotNone(ds.variables["count_vis"])
self.assertIsNotNone(ds.variables["time"])
self.assertIsNotNone(ds.variables["solar_zenith_angle"])
self.assertIsNotNone(ds.variables["solar_azimuth_angle"])
self.assertIsNotNone(ds.variables["satellite_azimuth_angle"])
self.assertIsNotNone(ds.variables["satellite_zenith_angle"])
self.assertIsNotNone(ds.variables["count_ir"])
self.assertIsNotNone(ds.variables["count_wv"])
self.assertIsNotNone(ds.variables["count_vis"])
self.assertIsNotNone(ds.variables["data_quality_bitmask"])
self.assertIsNotNone(ds.variables["SRF_weights"])
self.assertIsNotNone(ds.variables["SRF_frequencies"])
self.assertIsNotNone(
ds.variables["covariance_spectral_response_function_vis"])
self.assertIsNotNone(ds.variables["u_spectral_response_function_ir"])
self.assertIsNotNone(ds.variables["u_spectral_response_function_wv"])
self.assertIsNotNone(ds.variables["a0_vis"])
self.assertIsNotNone(ds.variables["a1_vis"])
self.assertIsNotNone(ds.variables["a2_vis"])
self.assertIsNotNone(ds.variables["solar_irradiance_vis"])
self.assertIsNotNone(ds.variables["u_solar_irradiance_vis"])
self.assertIsNotNone(ds.variables["distance_sun_earth"])
self.assertIsNotNone(ds.variables["a_ir"])
self.assertIsNotNone(ds.variables["b_ir"])
self.assertIsNotNone(ds.variables["u_a_ir"])
self.assertIsNotNone(ds.variables["u_b_ir"])
self.assertIsNotNone(ds.variables["a_wv"])
self.assertIsNotNone(ds.variables["b_wv"])
self.assertIsNotNone(ds.variables["u_a_wv"])
self.assertIsNotNone(ds.variables["u_b_wv"])
self.assertIsNotNone(ds.variables["bt_a_ir"])
self.assertIsNotNone(ds.variables["bt_b_ir"])
self.assertIsNotNone(ds.variables["bt_a_wv"])
self.assertIsNotNone(ds.variables["bt_b_wv"])
self.assertIsNotNone(ds.variables["years_since_launch"])
# full FCDR uncertainties
self.assertIsNotNone(ds.variables["u_latitude"])
self.assertIsNotNone(ds.variables["u_longitude"])
self.assertIsNotNone(ds.variables["u_time"])
self.assertIsNotNone(ds.variables["u_a0_vis"])
self.assertIsNotNone(ds.variables["u_a1_vis"])
self.assertIsNotNone(ds.variables["u_a2_vis"])
self.assertIsNotNone(ds.variables["u_zero_vis"])
self.assertIsNotNone(ds.variables["covariance_a_vis"])
self.assertIsNotNone(ds.variables["u_electronics_counts_vis"])
self.assertIsNotNone(ds.variables["u_digitization_counts_vis"])
self.assertIsNotNone(ds.variables["allan_deviation_counts_space_vis"])
self.assertIsNotNone(ds.variables["u_mean_counts_space_vis"])
self.assertIsNotNone(ds.variables["u_solar_zenith_angle"])
self.assertIsNotNone(ds.variables["u_solar_azimuth_angle"])
self.assertIsNotNone(ds.variables["u_satellite_zenith_angle"])
self.assertIsNotNone(ds.variables["u_satellite_azimuth_angle"])
self.assertIsNotNone(ds.variables["effect_correlation_matrix"])
# virtual variables
self.assertIsNotNone(ds.variables["sensitivity_solar_irradiance_vis"])
self.assertIsNotNone(ds.variables["sensitivity_count_vis"])
self.assertIsNotNone(ds.variables["sensitivity_count_space"])
self.assertIsNotNone(ds.variables["sensitivity_a0_vis"])
self.assertIsNotNone(ds.variables["sensitivity_a1_vis"])
self.assertIsNotNone(ds.variables["sensitivity_a2_vis"])
def testCreate_MVIRI_STATIC(self):
ds = FCDRWriter.createTemplateEasy('MVIRI_STATIC', 5000)
self.assertIsNotNone(ds.variables["latitude_vis"])
self.assertIsNotNone(ds.variables["longitude_vis"])
self.assertIsNotNone(ds.variables["latitude_ir_wv"])
self.assertIsNotNone(ds.variables["longitude_ir_wv"])
def create_full_dataset(self):
avhrr_full = FCDRWriter.createTemplateFull("AVHRR", PRODUCT_HEIGHT)
self.add_global_attributes(avhrr_full)
self.add_geolocation_data(avhrr_full)
self.add_global_flags(avhrr_full)
self.add_sensor_data(avhrr_full)
for x in range(0, PRODUCT_WIDTH):
avhrr_full["u_latitude"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.14
avhrr_full["u_longitude"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.15
avhrr_full["u_satellite_azimuth_angle"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.16
avhrr_full["u_satellite_zenith_angle"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.17
avhrr_full["u_solar_azimuth_angle"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.18
avhrr_full["u_solar_zenith_angle"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.19
avhrr_full["Ch1_Csp"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 2
avhrr_full["Ch2_Csp"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 3
avhrr_full["Ch3a_Csp"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 4
avhrr_full["Ch3b_Csp"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 5
avhrr_full["Ch4_Csp"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 6
avhrr_full["Ch5_Csp"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 7
avhrr_full["Ch3b_Cict"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 8
avhrr_full["Ch4_Cict"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 9
avhrr_full["Ch5_Cict"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 10
avhrr_full["Ch1_Ce"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 11
avhrr_full["Ch2_Ce"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 12
avhrr_full["Ch3a_Ce"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 13
avhrr_full["Ch3b_Ce"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 14
avhrr_full["Ch4_Ce"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 15
avhrr_full["Ch5_Ce"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int32) * x * 16
avhrr_full["Ch1_u_Csp"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.17
avhrr_full["Ch2_u_Csp"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.18
avhrr_full["Ch3a_u_Csp"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.19
avhrr_full["Ch3b_u_Csp"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.2
avhrr_full["Ch4_u_Csp"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.21
avhrr_full["Ch5_u_Csp"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.22
avhrr_full["Ch3b_u_Cict"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.23
avhrr_full["Ch4_u_Cict"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.24
avhrr_full["Ch5_u_Cict"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.25
avhrr_full["Ch1_u_Ce"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.26
avhrr_full["Ch2_u_Ce"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.27
avhrr_full["Ch3a_u_Ce"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.28
avhrr_full["Ch3b_u_Ce"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.29
avhrr_full["Ch4_u_Ce"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.3
avhrr_full["Ch5_u_Ce"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.31
avhrr_full["Ch1_u_Refl"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.32
avhrr_full["Ch2_u_Refl"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.33
avhrr_full["Ch3a_u_Refl"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.34
avhrr_full["Ch3b_u_Bt"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.35
avhrr_full["Ch4_u_Bt"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.36
avhrr_full["Ch5_u_Bt"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.37
avhrr_full["Ch3b_ur_Bt"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.38
avhrr_full["Ch4_ur_Bt"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.39
avhrr_full["Ch5_ur_Bt"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.4
avhrr_full["Ch3b_us_Bt"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.41
avhrr_full["Ch4_us_Bt"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.42
avhrr_full["Ch5_us_Bt"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.43
for x in range(0, N_PRT):
avhrr_full["PRT_C"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x
avhrr_full["u_prt"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.2
avhrr_full["R_ICT"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.float32) * x * 0.21
avhrr_full["u_time"].data[:] = np.ones(PRODUCT_HEIGHT, np.float64) * 0.16
avhrr_full["T_instr"].data[:] = np.ones(PRODUCT_HEIGHT, np.float32) * 0.17
for x in range(0, 6):
avhrr_full["quality_channel_bitmask"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int8) * x
avhrr_full["quality_scanline_bitmask"].data[:] = np.ones(PRODUCT_HEIGHT, np.int8)
return avhrr_full
def testCreateTemplateFull_AVHRR(self):
ds = FCDRWriter.createTemplateFull('AVHRR', 13667)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(73, len(ds.data_vars))
# geolocation + flags
self._verify_geolocation_variables(ds)
self._verify_quality_flags(ds)
# sensor specific
self._verify_avhrr_specific_variables(ds)
# variables of full FCDR
self.assertIsNotNone(ds.variables["u_latitude"])
self.assertIsNotNone(ds.variables["u_longitude"])
self.assertIsNotNone(ds.variables["u_time"])
self.assertIsNotNone(ds.variables["u_satellite_azimuth_angle"])
self.assertIsNotNone(ds.variables["u_satellite_zenith_angle"])
self.assertIsNotNone(ds.variables["u_solar_azimuth_angle"])
self.assertIsNotNone(ds.variables["u_solar_zenith_angle"])
self.assertIsNotNone(ds.variables["PRT_C"])
self.assertIsNotNone(ds.variables["u_prt"]) # geolocation
self._verify_geolocation_variables(ds)
self.assertIsNotNone(ds.variables["R_ICT"])
self.assertIsNotNone(ds.variables["T_instr"])
self.assertIsNotNone(ds.variables["Ch1_Csp"])
self.assertIsNotNone(ds.variables["Ch2_Csp"])
self.assertIsNotNone(ds.variables["Ch3a_Csp"])
self.assertIsNotNone(ds.variables["Ch3b_Csp"])
self.assertIsNotNone(ds.variables["Ch4_Csp"])
self.assertIsNotNone(ds.variables["Ch5_Csp"])
self.assertIsNotNone(ds.variables["Ch3b_Cict"])
self.assertIsNotNone(ds.variables["Ch4_Cict"])
self.assertIsNotNone(ds.variables["Ch5_Cict"])
self.assertIsNotNone(ds.variables["Ch1_Ce"])
self.assertIsNotNone(ds.variables["Ch2_Ce"])
self.assertIsNotNone(ds.variables["Ch3a_Ce"])
self.assertIsNotNone(ds.variables["Ch3b_Ce"])
self.assertIsNotNone(ds.variables["Ch4_Ce"])
self.assertIsNotNone(ds.variables["Ch5_Ce"])
self.assertIsNotNone(ds.variables["Ch1_u_Csp"])
self.assertIsNotNone(ds.variables["Ch2_u_Csp"])
self.assertIsNotNone(ds.variables["Ch3a_u_Csp"])
self.assertIsNotNone(ds.variables["Ch3b_u_Csp"])
self.assertIsNotNone(ds.variables["Ch4_u_Csp"])
self.assertIsNotNone(ds.variables["Ch5_u_Csp"])
self.assertIsNotNone(ds.variables["Ch3b_u_Cict"])
self.assertIsNotNone(ds.variables["Ch4_u_Cict"])
self.assertIsNotNone(ds.variables["Ch5_u_Cict"])
self.assertIsNotNone(ds.variables["Ch1_u_Ce"])
self.assertIsNotNone(ds.variables["Ch2_u_Ce"])
self.assertIsNotNone(ds.variables["Ch3a_u_Ce"])
self.assertIsNotNone(ds.variables["Ch3b_u_Ce"])
self.assertIsNotNone(ds.variables["Ch4_u_Ce"])
self.assertIsNotNone(ds.variables["Ch5_u_Ce"])
self.assertIsNotNone(ds.variables["Ch1_u_Refl"])
self.assertIsNotNone(ds.variables["Ch2_u_Refl"])
self.assertIsNotNone(ds.variables["Ch3a_u_Refl"])
self.assertIsNotNone(ds.variables["Ch3b_u_Bt"])
self.assertIsNotNone(ds.variables["Ch4_u_Bt"])
self.assertIsNotNone(ds.variables["Ch5_u_Bt"])
self.assertIsNotNone(ds.variables["Ch3b_ur_Bt"])
self.assertIsNotNone(ds.variables["Ch4_ur_Bt"])
self.assertIsNotNone(ds.variables["Ch5_ur_Bt"])
self.assertIsNotNone(ds.variables["Ch3b_us_Bt"])
self.assertIsNotNone(ds.variables["Ch4_us_Bt"])
self.assertIsNotNone(ds.variables["Ch5_us_Bt"])
def main(file_in, fileout='None'):
data = read_netcdf(file_in)
writer = FCDRWriter()
# get a template for sensor name in EASY format, supply product height
# The scan-width is set automatically
dataset = writer.createTemplateEasy("AVHRR", data.ny)
# set some mandatory global attributes. Writing will fail if not all of them are filled
dataset.attrs["institution"] = "University of Reading"
dataset.attrs[
"title"] = "pre-B version of AVHRR Fundamental Climate Data Records"
dataset.attrs["source"] = "FIDUCEO"
dataset.attrs["history"] = ""
dataset.attrs["references"] = "CDF_FCDR_File Spec"
dataset.attrs[
"comment"] = "This version is a pre-B one and aims at showing the kind of uncertainties we are aiming to deliver within FIDUCEO. The values are not final ones and should not be used for science purposes."
dataset.attrs['sensor'] = "AVHRR"
dataset.attrs['platform'] = data.noaa_string
dataset.attrs['software_version'] = data.version
# write real data to the variables. All variables initially contain "_FillValue".
# Not writing to the whole array is completely OK
dataset.variables["latitude"].data = data.lat
dataset.variables["longitude"].data = data.lon
dataset.variables["Time"].data = data.time
dataset.variables["satellite_zenith_angle"].data = data.satza
dataset.variables["solar_zenith_angle"].data = data.solza
dataset.variables["relative_azimuth_angle"].data = data.relaz
dataset.variables["Ch1_Ref"].data = data.ch1
dataset.variables["Ch2_Ref"].data = data.ch2
if data.ch3a_there:
dataset.variables["Ch3a_Ref"].data = data.ch3a
dataset.variables["Ch3b_Bt"].data = data.ch3b
dataset.variables["Ch4_Bt"].data = data.ch4
if data.ch5_there:
dataset.variables["Ch5_Bt"].data = data.ch5
dataset.variables["u_random_Ch1"].data = data.u_random_ch1
dataset.variables["u_random_Ch2"].data = data.u_random_ch2
if data.ch3a_there:
dataset.variables["u_random_Ch3a"].data = data.u_random_ch3a
dataset.variables["u_random_Ch3b"].data = data.u_random_ch3b
dataset.variables["u_random_Ch4"].data = data.u_random_ch4
if data.ch5_there:
dataset.variables["u_random_Ch5"].data = data.u_random_ch5
dataset.variables["u_non_random_Ch1"].data = data.u_non_random_ch1
dataset.variables["u_non_random_Ch2"].data = data.u_non_random_ch2
if data.ch3a_there:
dataset.variables["u_non_random_Ch3a"].data = data.u_non_random_ch3a
dataset.variables["u_non_random_Ch3b"].data = data.u_non_random_ch3b
dataset.variables["u_non_random_Ch4"].data = data.u_non_random_ch4
if data.ch5_there:
dataset.variables["u_non_random_Ch5"].data = data.u_non_random_ch5
dataset.variables["quality_scanline_bitmask"].data = data.scan_qual
dataset.variables["quality_channel_bitmask"].data = data.chan_qual
#avhrr.AVHRR._create_channel_refl_variable(12835, "Channel 6 Reflectance")
# dump it to disk, netcdf4, medium compression
# writing will fail when the target file already exists
if 'None' == fileout:
file_out = writer.create_file_name_FCDR_easy('AVHRR',data.noaa_string,\
data.date_time[0],\
data.date_time[-1],\
data.version)
else:
file_out = fileout
writer.write(dataset, file_out)
def test_write_HIRS4(self):
hirs_easy = self.create_easy_dataset("HIRS4")
start = datetime(2013, 8, 19, 13, 24, 52)
end = datetime(2013, 8, 19, 14, 25, 53)
file_name = FCDRWriter.create_file_name_FCDR_easy(
"HIRS4", "NOAA18", start, end, "1.0")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.now()
FCDRWriter.write(hirs_easy, self.target_path)
end_time = datetime.now()
elapsed_time = end_time - start_time
print("HIRS/4 EASY write time: " + "%d.%03d" %
(elapsed_time.seconds,
int(round(elapsed_time.microseconds / 1000))))
self.assertTrue(os.path.isfile(self.target_path))
target_data = xr.open_dataset(self.target_path)
try:
variable = target_data["bt"]
self.assertAlmostEqual(0.02, variable.data[2, 2, 2], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
variable = target_data["latitude"]
self.assertAlmostEqual(0.0604266476, variable.data[3, 3], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["longitude"]
self.assertAlmostEqual(0.1208532974, variable.data[4, 4], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["quality_pixel_bitmask"]
self.assertEqual(7, variable.data[5, 5])
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["data_quality_bitmask"]
self.assertEqual(4, variable.data[11, 4])
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["quality_scanline_bitmask"]
self.assertEqual(1, variable.data[6])
self.assertEqual((944, ), variable.encoding["chunksizes"])
variable = target_data["quality_channel_bitmask"]
self.assertEqual(21, variable.data[7, 7])
self.assertEqual((944, 19), variable.encoding["chunksizes"])
variable = target_data["satellite_zenith_angle"]
self.assertAlmostEqual(0.28, variable.data[7, 7], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["satellite_azimuth_angle"]
self.assertAlmostEqual(0.4, variable.data[8, 8], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["scanline"]
self.assertEqual(3, variable.data[8])
self.assertEqual((944, ), variable.encoding["chunksizes"])
variable = target_data["solar_azimuth_angle"]
self.assertAlmostEqual(0.45, variable.data[9, 9], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["solar_zenith_angle"]
self.assertAlmostEqual(0.6, variable.data[10, 10], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["time"]
self.assertEqual(4, variable.data[10])
self.assertEqual((944, ), variable.encoding["chunksizes"])
variable = target_data["u_independent"]
self.assertAlmostEqual(0.66, variable.data[11, 11, 11], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
variable = target_data["u_structured"]
self.assertAlmostEqual(0.84, variable.data[12, 12, 12], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
variable = target_data["u_common"]
self.assertAlmostEqual(1.04, variable.data[13, 13, 13], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
self.assert_srf(target_data)
finally:
target_data.close()
