def create_easy_dataset(self):
avhrr_easy = FCDRWriter.createTemplateEasy("AVHRR", PRODUCT_HEIGHT)
self.add_global_attributes(avhrr_easy)
self.add_geolocation_data(avhrr_easy)
self.add_global_flags(avhrr_easy)
self.add_sensor_data(avhrr_easy)
self.add_srf_data(avhrr_easy)
for x in range(0, PRODUCT_WIDTH):
avhrr_easy["u_independent_Ch1"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.013
avhrr_easy["u_independent_Ch2"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.014
avhrr_easy["u_independent_Ch3a"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.015
avhrr_easy["u_independent_Ch4"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.016
avhrr_easy["u_independent_Ch5"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.017
avhrr_easy["u_structured_Ch1"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.018
avhrr_easy["u_structured_Ch2"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.019
avhrr_easy["u_structured_Ch3a"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.020
avhrr_easy["u_structured_Ch4"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.021
avhrr_easy["u_structured_Ch5"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.022
avhrr_easy["u_common_Ch1"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.003
avhrr_easy["u_common_Ch2"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.004
avhrr_easy["u_common_Ch3a"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.005
avhrr_easy["u_common_Ch4"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.006
avhrr_easy["u_common_Ch5"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.007
for x in range(0, 6):
avhrr_easy["quality_channel_bitmask"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int8) * x
avhrr_easy["quality_scanline_bitmask"].data[:] = np.ones(PRODUCT_HEIGHT, np.int8)
return avhrr_easy
def testCreateTemplateEasy_HIRS4(self):
ds = FCDRWriter.createTemplateEasy('HIRS4', 211)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(22, len(ds.data_vars))
# geolocation + flags
self._verify_geolocation_variables(ds)
self._verify_quality_flags(ds)
# sensor specific
self.assertIsNotNone(ds.variables["bt"])
self.assertIsNotNone(ds.variables["satellite_zenith_angle"])
self.assertIsNotNone(ds.variables["satellite_azimuth_angle"])
self.assertIsNotNone(ds.variables["solar_zenith_angle"])
self.assertIsNotNone(ds.variables["solar_azimuth_angle"])
self.assertIsNotNone(ds.variables["scanline"])
self.assertIsNotNone(ds.variables["time"])
self.assertIsNotNone(ds.variables["data_quality_bitmask"])
self.assertIsNotNone(ds.variables["quality_scanline_bitmask"])
self.assertIsNotNone(ds.variables["quality_channel_bitmask"])
self.assertIsNotNone(ds.variables["SRF_weights"])
self.assertIsNotNone(ds.variables["SRF_wavelengths"])
self.assertIsNotNone(ds.variables["scanline_map_to_origl1bfile"])
self.assertIsNotNone(ds.variables["scanline_origl1b"])
self._assert_HIRS_easy_uncertainties(ds)
def testCreateTemplateEasy_MVIRI(self):
ds = FCDRWriter.createTemplateEasy('MVIRI', 5000)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self._verify_quality_flags(ds)
self.assertEqual(40, len(ds.data_vars))
# sensor specific
self.assertIsNotNone(ds.variables["time"])
self.assertIsNotNone(ds.variables["SRF_weights"])
self.assertIsNotNone(ds.variables["SRF_frequencies"])
self.assertIsNotNone(
ds.variables["covariance_spectral_response_function_vis"])
self.assertIsNotNone(ds.variables["u_spectral_response_function_ir"])
self.assertIsNotNone(ds.variables["u_spectral_response_function_wv"])
self.assertIsNotNone(ds.variables["solar_zenith_angle"])
self.assertIsNotNone(ds.variables["solar_azimuth_angle"])
self.assertIsNotNone(ds.variables["satellite_azimuth_angle"])
self.assertIsNotNone(ds.variables["satellite_zenith_angle"])
self.assertIsNotNone(ds.variables["count_ir"])
self.assertIsNotNone(ds.variables["count_wv"])
self.assertIsNotNone(ds.variables["data_quality_bitmask"])
self.assertIsNotNone(ds.variables["distance_sun_earth"])
self.assertIsNotNone(ds.variables["solar_irradiance_vis"])
self.assertIsNotNone(ds.variables["u_solar_irradiance_vis"])
self.assertIsNotNone(ds.variables["a_ir"])
self.assertIsNotNone(ds.variables["b_ir"])
self.assertIsNotNone(ds.variables["u_a_ir"])
self.assertIsNotNone(ds.variables["u_b_ir"])
self.assertIsNotNone(ds.variables["a_wv"])
self.assertIsNotNone(ds.variables["b_wv"])
self.assertIsNotNone(ds.variables["u_a_wv"])
self.assertIsNotNone(ds.variables["u_b_wv"])
self.assertIsNotNone(ds.variables["bt_a_ir"])
self.assertIsNotNone(ds.variables["bt_b_ir"])
self.assertIsNotNone(ds.variables["bt_a_wv"])
self.assertIsNotNone(ds.variables["bt_b_wv"])
self.assertIsNotNone(ds.variables["years_since_launch"])
# easy FCDR uncertainties
self.assertIsNotNone(ds.variables["toa_bidirectional_reflectance_vis"])
self.assertIsNotNone(
ds.variables["u_independent_toa_bidirectional_reflectance"])
self.assertIsNotNone(
ds.variables["u_structured_toa_bidirectional_reflectance"])
self.assertIsNotNone(
ds.variables["u_common_toa_bidirectional_reflectance"])
self.assertIsNotNone(ds.variables["sub_satellite_latitude_start"])
self.assertIsNotNone(ds.variables["sub_satellite_longitude_start"])
self.assertIsNotNone(ds.variables["sub_satellite_latitude_end"])
self.assertIsNotNone(ds.variables["sub_satellite_longitude_end"])
self.assertIsNotNone(
ds.variables["channel_correlation_matrix_independent"])
self.assertIsNotNone(
ds.variables["channel_correlation_matrix_structured"])
def create_easy_dataset(self, type):
hirs_easy = FCDRWriter.createTemplateEasy(type, 944, SRF_SIZE)
hirs_easy.attrs["institution"] = "test"
hirs_easy.attrs["title"] = "sir"
hirs_easy.attrs["source"] = "invention"
hirs_easy.attrs["history"] = "new"
hirs_easy.attrs["references"] = "myself"
hirs_easy.attrs["comment"] = "should define a test version of this set"
for x in range(0, 56):
hirs_easy["bt"].data[:, :, x] = np.ones((944), np.int16) * x * 0.01
hirs_easy["latitude"].data[:, x] = np.ones(
(944), np.int16) * x * 0.02
hirs_easy["longitude"].data[:, x] = np.ones(
(944), np.int16) * x * 0.03
hirs_easy["data_quality_bitmask"].data[:, x] = np.ones(
(944), np.int8) * x
hirs_easy["quality_pixel_bitmask"].data[:, x] = np.ones(
(944), np.int8) * x
if type != "HIRS2":
hirs_easy["satellite_zenith_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.04
hirs_easy["satellite_azimuth_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.05
hirs_easy["solar_zenith_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.06
hirs_easy["solar_azimuth_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.05
hirs_easy["u_independent"].data[:, :, x] = np.ones(
(944), np.int16) * x * 0.06
hirs_easy["u_structured"].data[:, :, x] = np.ones(
(944), np.int16) * x * 0.07
hirs_easy["u_common"].data[:, :, x] = np.ones(
(944), np.int16) * x * 0.08
if type != "HIRS2":
for x in range(0, 19):
hirs_easy["quality_channel_bitmask"].data[:, x] = np.ones(
(944), np.int32) * x * 3
hirs_easy["quality_scanline_bitmask"].data[:] = np.ones((944), np.int8)
hirs_easy["scanline"].data[:] = np.ones((944), np.int8) * 3
hirs_easy["time"].data[:] = np.ones((944), np.int32) * 4
for x in range(0, SRF_SIZE):
hirs_easy["SRF_weights"].data[:, x] = np.ones(
(NUM_CHANNELS), np.float32) * x * 0.04
hirs_easy["SRF_wavelengths"].data[:, x] = np.ones(
(NUM_CHANNELS), np.float32) * x * 0.05
if type == "HIRS2":
hirs_easy["satellite_zenith_angle"].data[:] = np.ones(
(944), np.int8) * 2
return hirs_easy
def testCreateTemplateEasy_AVHRR(self):
ds = FCDRWriter.createTemplateEasy('AVHRR',
12198,
srf_size=13,
corr_dx=14,
corr_dy=15)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(42, len(ds.data_vars))
# geolocation + flags
self._verify_geolocation_variables(ds)
self._verify_quality_flags(ds)
# sensor specific
self._verify_avhrr_specific_variables(ds)
# easy FCDR variables
self.assertIsNotNone(ds.variables["u_independent_Ch1"])
self.assertIsNotNone(ds.variables["u_structured_Ch1"])
self.assertIsNotNone(ds.variables["u_common_Ch1"])
self.assertIsNotNone(ds.variables["u_independent_Ch2"])
self.assertIsNotNone(ds.variables["u_structured_Ch2"])
self.assertIsNotNone(ds.variables["u_common_Ch2"])
self.assertIsNotNone(ds.variables["u_independent_Ch3a"])
self.assertIsNotNone(ds.variables["u_structured_Ch3a"])
self.assertIsNotNone(ds.variables["u_common_Ch3a"])
self.assertIsNotNone(ds.variables["u_independent_Ch3b"])
self.assertIsNotNone(ds.variables["u_structured_Ch3b"])
self.assertIsNotNone(ds.variables["u_common_Ch3b"])
self.assertIsNotNone(ds.variables["u_independent_Ch4"])
self.assertIsNotNone(ds.variables["u_structured_Ch4"])
self.assertIsNotNone(ds.variables["u_common_Ch4"])
self.assertIsNotNone(ds.variables["u_independent_Ch5"])
self.assertIsNotNone(ds.variables["u_structured_Ch5"])
self.assertIsNotNone(ds.variables["u_common_Ch5"])
self.assertIsNotNone(
ds.variables["channel_correlation_matrix_independent"])
self.assertIsNotNone(
ds.variables["channel_correlation_matrix_structured"])
self.assertIsNotNone(
ds.variables["cross_element_correlation_coefficients"])
self.assertIsNotNone(
ds.variables["cross_line_correlation_coefficients"])
def create_easy_dataset(self):
mviri_easy = FCDRWriter.createTemplateEasy("MVIRI", 5000)
self.add_global_attributes(mviri_easy)
self.add_global_flags(mviri_easy)
self.add_sensor_data(mviri_easy)
for x in range(0, 5000):
mviri_easy["toa_bidirectional_reflectance_vis"].data[:, x] = np.ones((5000), np.uint16) * x * 0.03
mviri_easy["u_independent_toa_bidirectional_reflectance"].data[:, x] = np.ones((5000), np.uint16) * x * 0.04
mviri_easy["u_structured_toa_bidirectional_reflectance"].data[:, x] = np.ones((5000), np.uint16) * x * 0.05
mviri_easy["sub_satellite_latitude_start"].data = 25.25
mviri_easy["sub_satellite_longitude_start"].data = 26.26
mviri_easy["sub_satellite_latitude_end"].data = 27.27
mviri_easy["sub_satellite_longitude_end"].data = 28.28
return mviri_easy
def testCreateTemplateEasy_AMSUB(self):
ds = FCDRWriter.createTemplateEasy('AMSUB', 2561)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(19, len(ds.data_vars))
# geolocation + flags
self._verify_geolocation_variables(ds)
self._verify_quality_flags(ds)
# sensor specific
self._verify_amsub_specific_variables(ds)
# easy FCDR variables
self.assertIsNotNone(ds.variables["u_independent_btemps"])
self.assertIsNotNone(ds.variables["u_structured_btemps"])
def testCreateTemplateEasy_SSMT2(self):
ds = FCDRWriter.createTemplateEasy('SSMT2', 722)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
self.assertEqual(15, len(ds.data_vars))
# geolocation + flags
self._verify_geolocation_variables(ds)
self._verify_quality_flags(ds)
# sensor specific
self.verify_SSMT2_specific_variables(ds)
# easy FCDR variables
self.assertIsNotNone(ds.variables["u_independent_tb"])
self.assertIsNotNone(ds.variables["u_structured_tb"])
def main(file_in, fileout='None'):
data = read_netcdf(file_in)
writer = FCDRWriter()
# get a template for sensor name in EASY format, supply product height
# The scan-width is set automatically
dataset = writer.createTemplateEasy("AVHRR", data.ny)
# set some mandatory global attributes. Writing will fail if not all of them are filled
dataset.attrs["institution"] = "University of Reading"
dataset.attrs[
"title"] = "pre-B version of AVHRR Fundamental Climate Data Records"
dataset.attrs["source"] = "FIDUCEO"
dataset.attrs["history"] = ""
dataset.attrs["references"] = "CDF_FCDR_File Spec"
dataset.attrs[
"comment"] = "This version is a pre-B one and aims at showing the kind of uncertainties we are aiming to deliver within FIDUCEO. The values are not final ones and should not be used for science purposes."
dataset.attrs['sensor'] = "AVHRR"
dataset.attrs['platform'] = data.noaa_string
dataset.attrs['software_version'] = data.version
# write real data to the variables. All variables initially contain "_FillValue".
# Not writing to the whole array is completely OK
dataset.variables["latitude"].data = data.lat
dataset.variables["longitude"].data = data.lon
dataset.variables["Time"].data = data.time
dataset.variables["satellite_zenith_angle"].data = data.satza
dataset.variables["solar_zenith_angle"].data = data.solza
dataset.variables["relative_azimuth_angle"].data = data.relaz
dataset.variables["Ch1_Ref"].data = data.ch1
dataset.variables["Ch2_Ref"].data = data.ch2
if data.ch3a_there:
dataset.variables["Ch3a_Ref"].data = data.ch3a
dataset.variables["Ch3b_Bt"].data = data.ch3b
dataset.variables["Ch4_Bt"].data = data.ch4
if data.ch5_there:
dataset.variables["Ch5_Bt"].data = data.ch5
dataset.variables["u_random_Ch1"].data = data.u_random_ch1
dataset.variables["u_random_Ch2"].data = data.u_random_ch2
if data.ch3a_there:
dataset.variables["u_random_Ch3a"].data = data.u_random_ch3a
dataset.variables["u_random_Ch3b"].data = data.u_random_ch3b
dataset.variables["u_random_Ch4"].data = data.u_random_ch4
if data.ch5_there:
dataset.variables["u_random_Ch5"].data = data.u_random_ch5
dataset.variables["u_non_random_Ch1"].data = data.u_non_random_ch1
dataset.variables["u_non_random_Ch2"].data = data.u_non_random_ch2
if data.ch3a_there:
dataset.variables["u_non_random_Ch3a"].data = data.u_non_random_ch3a
dataset.variables["u_non_random_Ch3b"].data = data.u_non_random_ch3b
dataset.variables["u_non_random_Ch4"].data = data.u_non_random_ch4
if data.ch5_there:
dataset.variables["u_non_random_Ch5"].data = data.u_non_random_ch5
dataset.variables["quality_scanline_bitmask"].data = data.scan_qual
dataset.variables["quality_channel_bitmask"].data = data.chan_qual
#avhrr.AVHRR._create_channel_refl_variable(12835, "Channel 6 Reflectance")
# dump it to disk, netcdf4, medium compression
# writing will fail when the target file already exists
if 'None' == fileout:
file_out = writer.create_file_name_FCDR_easy('AVHRR',data.noaa_string,\
data.date_time[0],\
data.date_time[-1],\
data.version)
else:
file_out = fileout
writer.write(dataset, file_out)
def create_easy_dataset(self, type):
hirs_easy = FCDRWriter.createTemplateEasy(type, 944, SRF_SIZE)
hirs_easy.attrs["institution"] = "test"
hirs_easy.attrs["title"] = "sir"
hirs_easy.attrs["source"] = "invention"
hirs_easy.attrs["history"] = "new"
hirs_easy.attrs["references"] = "myself"
hirs_easy.attrs["project"] = "test-project"
hirs_easy.attrs["creator_url"] = "test-url"
hirs_easy.attrs["creator_name"] = "test-name"
hirs_easy.attrs["creator_email"] = "*****@*****.**"
hirs_easy.attrs["fcdr_software_version"] = "12"
hirs_easy.attrs["data_version"] = "13"
hirs_easy.attrs["time_coverage_start"] = "now"
hirs_easy.attrs["time_coverage_end"] = "later"
hirs_easy.attrs["time_coverage_duration"] = "some moment"
hirs_easy.attrs["platform"] = "yo"
hirs_easy.attrs["comment"] = "what?"
for x in range(0, 56):
hirs_easy["bt"].data[:, :, x] = np.ones((944), np.int16) * x * 0.01
hirs_easy["latitude"].data[:, x] = np.ones(
(944), np.int16) * x * 0.02
hirs_easy["longitude"].data[:, x] = np.ones(
(944), np.int16) * x * 0.03
hirs_easy["data_quality_bitmask"].data[:, x] = np.ones(
(944), np.int8) * x
hirs_easy["quality_pixel_bitmask"].data[:, x] = np.ones(
(944), np.int8) * x
hirs_easy["satellite_zenith_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.04
hirs_easy["satellite_azimuth_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.05
hirs_easy["solar_zenith_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.06
hirs_easy["solar_azimuth_angle"].data[:, x] = np.ones(
(944), np.int16) * x * 0.05
hirs_easy["u_independent"].data[:, :, x] = np.ones(
(944), np.int16) * x * 0.06
hirs_easy["u_structured"].data[:, :, x] = np.ones(
(944), np.int16) * x * 0.07
hirs_easy["u_common"].data[:, :, x] = np.ones(
(944), np.int16) * x * 0.08
if type != "HIRS2":
for x in range(0, 19):
hirs_easy["quality_channel_bitmask"].data[:, x] = np.ones(
(944), np.int32) * x * 3
hirs_easy["quality_scanline_bitmask"].data[:] = np.ones((944), np.int8)
hirs_easy["scanline"].data[:] = np.ones((944), np.int8) * 3
hirs_easy["time"].data[:] = np.ones((944), np.int32) * 400000000000000
for x in range(0, SRF_SIZE):
hirs_easy["SRF_weights"].data[:, x] = np.ones(
(NUM_CHANNELS), np.float32) * x * 0.04
hirs_easy["SRF_wavelengths"].data[:, x] = np.ones(
(NUM_CHANNELS), np.float32) * x * 0.05
return hirs_easy
def testCreate_MVIRI_STATIC(self):
ds = FCDRWriter.createTemplateEasy('MVIRI_STATIC', 5000)
self.assertIsNotNone(ds.variables["latitude_vis"])
self.assertIsNotNone(ds.variables["longitude_vis"])
self.assertIsNotNone(ds.variables["latitude_ir_wv"])
self.assertIsNotNone(ds.variables["longitude_ir_wv"])