def create_easy_dataset(self): avhrr_easy = FCDRWriter.createTemplateEasy("AVHRR", PRODUCT_HEIGHT) self.add_global_attributes(avhrr_easy) self.add_geolocation_data(avhrr_easy) self.add_global_flags(avhrr_easy) self.add_sensor_data(avhrr_easy) self.add_srf_data(avhrr_easy) for x in range(0, PRODUCT_WIDTH): avhrr_easy["u_independent_Ch1"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.013 avhrr_easy["u_independent_Ch2"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.014 avhrr_easy["u_independent_Ch3a"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.015 avhrr_easy["u_independent_Ch4"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.016 avhrr_easy["u_independent_Ch5"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.017 avhrr_easy["u_structured_Ch1"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.018 avhrr_easy["u_structured_Ch2"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.019 avhrr_easy["u_structured_Ch3a"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.020 avhrr_easy["u_structured_Ch4"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.021 avhrr_easy["u_structured_Ch5"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.022 avhrr_easy["u_common_Ch1"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.003 avhrr_easy["u_common_Ch2"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.004 avhrr_easy["u_common_Ch3a"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.005 avhrr_easy["u_common_Ch4"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.006 avhrr_easy["u_common_Ch5"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int16) * x * 0.007 for x in range(0, 6): avhrr_easy["quality_channel_bitmask"].data[:, x] = np.ones(PRODUCT_HEIGHT, np.int8) * x avhrr_easy["quality_scanline_bitmask"].data[:] = np.ones(PRODUCT_HEIGHT, np.int8) return avhrr_easy
def testCreateTemplateEasy_HIRS4(self): ds = FCDRWriter.createTemplateEasy('HIRS4', 211) self.assertIsNotNone(ds) Assertions.assert_global_attributes(self, ds.attrs) self.assertEqual(22, len(ds.data_vars)) # geolocation + flags self._verify_geolocation_variables(ds) self._verify_quality_flags(ds) # sensor specific self.assertIsNotNone(ds.variables["bt"]) self.assertIsNotNone(ds.variables["satellite_zenith_angle"]) self.assertIsNotNone(ds.variables["satellite_azimuth_angle"]) self.assertIsNotNone(ds.variables["solar_zenith_angle"]) self.assertIsNotNone(ds.variables["solar_azimuth_angle"]) self.assertIsNotNone(ds.variables["scanline"]) self.assertIsNotNone(ds.variables["time"]) self.assertIsNotNone(ds.variables["data_quality_bitmask"]) self.assertIsNotNone(ds.variables["quality_scanline_bitmask"]) self.assertIsNotNone(ds.variables["quality_channel_bitmask"]) self.assertIsNotNone(ds.variables["SRF_weights"]) self.assertIsNotNone(ds.variables["SRF_wavelengths"]) self.assertIsNotNone(ds.variables["scanline_map_to_origl1bfile"]) self.assertIsNotNone(ds.variables["scanline_origl1b"]) self._assert_HIRS_easy_uncertainties(ds)
def testCreateTemplateEasy_MVIRI(self): ds = FCDRWriter.createTemplateEasy('MVIRI', 5000) self.assertIsNotNone(ds) Assertions.assert_global_attributes(self, ds.attrs) self._verify_quality_flags(ds) self.assertEqual(40, len(ds.data_vars)) # sensor specific self.assertIsNotNone(ds.variables["time"]) self.assertIsNotNone(ds.variables["SRF_weights"]) self.assertIsNotNone(ds.variables["SRF_frequencies"]) self.assertIsNotNone( ds.variables["covariance_spectral_response_function_vis"]) self.assertIsNotNone(ds.variables["u_spectral_response_function_ir"]) self.assertIsNotNone(ds.variables["u_spectral_response_function_wv"]) self.assertIsNotNone(ds.variables["solar_zenith_angle"]) self.assertIsNotNone(ds.variables["solar_azimuth_angle"]) self.assertIsNotNone(ds.variables["satellite_azimuth_angle"]) self.assertIsNotNone(ds.variables["satellite_zenith_angle"]) self.assertIsNotNone(ds.variables["count_ir"]) self.assertIsNotNone(ds.variables["count_wv"]) self.assertIsNotNone(ds.variables["data_quality_bitmask"]) self.assertIsNotNone(ds.variables["distance_sun_earth"]) self.assertIsNotNone(ds.variables["solar_irradiance_vis"]) self.assertIsNotNone(ds.variables["u_solar_irradiance_vis"]) self.assertIsNotNone(ds.variables["a_ir"]) self.assertIsNotNone(ds.variables["b_ir"]) self.assertIsNotNone(ds.variables["u_a_ir"]) self.assertIsNotNone(ds.variables["u_b_ir"]) self.assertIsNotNone(ds.variables["a_wv"]) self.assertIsNotNone(ds.variables["b_wv"]) self.assertIsNotNone(ds.variables["u_a_wv"]) self.assertIsNotNone(ds.variables["u_b_wv"]) self.assertIsNotNone(ds.variables["bt_a_ir"]) self.assertIsNotNone(ds.variables["bt_b_ir"]) self.assertIsNotNone(ds.variables["bt_a_wv"]) self.assertIsNotNone(ds.variables["bt_b_wv"]) self.assertIsNotNone(ds.variables["years_since_launch"]) # easy FCDR uncertainties self.assertIsNotNone(ds.variables["toa_bidirectional_reflectance_vis"]) self.assertIsNotNone( ds.variables["u_independent_toa_bidirectional_reflectance"]) self.assertIsNotNone( ds.variables["u_structured_toa_bidirectional_reflectance"]) self.assertIsNotNone( ds.variables["u_common_toa_bidirectional_reflectance"]) self.assertIsNotNone(ds.variables["sub_satellite_latitude_start"]) self.assertIsNotNone(ds.variables["sub_satellite_longitude_start"]) self.assertIsNotNone(ds.variables["sub_satellite_latitude_end"]) self.assertIsNotNone(ds.variables["sub_satellite_longitude_end"]) self.assertIsNotNone( ds.variables["channel_correlation_matrix_independent"]) self.assertIsNotNone( ds.variables["channel_correlation_matrix_structured"])
def create_easy_dataset(self, type): hirs_easy = FCDRWriter.createTemplateEasy(type, 944, SRF_SIZE) hirs_easy.attrs["institution"] = "test" hirs_easy.attrs["title"] = "sir" hirs_easy.attrs["source"] = "invention" hirs_easy.attrs["history"] = "new" hirs_easy.attrs["references"] = "myself" hirs_easy.attrs["comment"] = "should define a test version of this set" for x in range(0, 56): hirs_easy["bt"].data[:, :, x] = np.ones((944), np.int16) * x * 0.01 hirs_easy["latitude"].data[:, x] = np.ones( (944), np.int16) * x * 0.02 hirs_easy["longitude"].data[:, x] = np.ones( (944), np.int16) * x * 0.03 hirs_easy["data_quality_bitmask"].data[:, x] = np.ones( (944), np.int8) * x hirs_easy["quality_pixel_bitmask"].data[:, x] = np.ones( (944), np.int8) * x if type != "HIRS2": hirs_easy["satellite_zenith_angle"].data[:, x] = np.ones( (944), np.int16) * x * 0.04 hirs_easy["satellite_azimuth_angle"].data[:, x] = np.ones( (944), np.int16) * x * 0.05 hirs_easy["solar_zenith_angle"].data[:, x] = np.ones( (944), np.int16) * x * 0.06 hirs_easy["solar_azimuth_angle"].data[:, x] = np.ones( (944), np.int16) * x * 0.05 hirs_easy["u_independent"].data[:, :, x] = np.ones( (944), np.int16) * x * 0.06 hirs_easy["u_structured"].data[:, :, x] = np.ones( (944), np.int16) * x * 0.07 hirs_easy["u_common"].data[:, :, x] = np.ones( (944), np.int16) * x * 0.08 if type != "HIRS2": for x in range(0, 19): hirs_easy["quality_channel_bitmask"].data[:, x] = np.ones( (944), np.int32) * x * 3 hirs_easy["quality_scanline_bitmask"].data[:] = np.ones((944), np.int8) hirs_easy["scanline"].data[:] = np.ones((944), np.int8) * 3 hirs_easy["time"].data[:] = np.ones((944), np.int32) * 4 for x in range(0, SRF_SIZE): hirs_easy["SRF_weights"].data[:, x] = np.ones( (NUM_CHANNELS), np.float32) * x * 0.04 hirs_easy["SRF_wavelengths"].data[:, x] = np.ones( (NUM_CHANNELS), np.float32) * x * 0.05 if type == "HIRS2": hirs_easy["satellite_zenith_angle"].data[:] = np.ones( (944), np.int8) * 2 return hirs_easy
def testCreateTemplateEasy_AVHRR(self): ds = FCDRWriter.createTemplateEasy('AVHRR', 12198, srf_size=13, corr_dx=14, corr_dy=15) self.assertIsNotNone(ds) Assertions.assert_global_attributes(self, ds.attrs) self.assertEqual(42, len(ds.data_vars)) # geolocation + flags self._verify_geolocation_variables(ds) self._verify_quality_flags(ds) # sensor specific self._verify_avhrr_specific_variables(ds) # easy FCDR variables self.assertIsNotNone(ds.variables["u_independent_Ch1"]) self.assertIsNotNone(ds.variables["u_structured_Ch1"]) self.assertIsNotNone(ds.variables["u_common_Ch1"]) self.assertIsNotNone(ds.variables["u_independent_Ch2"]) self.assertIsNotNone(ds.variables["u_structured_Ch2"]) self.assertIsNotNone(ds.variables["u_common_Ch2"]) self.assertIsNotNone(ds.variables["u_independent_Ch3a"]) self.assertIsNotNone(ds.variables["u_structured_Ch3a"]) self.assertIsNotNone(ds.variables["u_common_Ch3a"]) self.assertIsNotNone(ds.variables["u_independent_Ch3b"]) self.assertIsNotNone(ds.variables["u_structured_Ch3b"]) self.assertIsNotNone(ds.variables["u_common_Ch3b"]) self.assertIsNotNone(ds.variables["u_independent_Ch4"]) self.assertIsNotNone(ds.variables["u_structured_Ch4"]) self.assertIsNotNone(ds.variables["u_common_Ch4"]) self.assertIsNotNone(ds.variables["u_independent_Ch5"]) self.assertIsNotNone(ds.variables["u_structured_Ch5"]) self.assertIsNotNone(ds.variables["u_common_Ch5"]) self.assertIsNotNone( ds.variables["channel_correlation_matrix_independent"]) self.assertIsNotNone( ds.variables["channel_correlation_matrix_structured"]) self.assertIsNotNone( ds.variables["cross_element_correlation_coefficients"]) self.assertIsNotNone( ds.variables["cross_line_correlation_coefficients"])
def create_easy_dataset(self): mviri_easy = FCDRWriter.createTemplateEasy("MVIRI", 5000) self.add_global_attributes(mviri_easy) self.add_global_flags(mviri_easy) self.add_sensor_data(mviri_easy) for x in range(0, 5000): mviri_easy["toa_bidirectional_reflectance_vis"].data[:, x] = np.ones((5000), np.uint16) * x * 0.03 mviri_easy["u_independent_toa_bidirectional_reflectance"].data[:, x] = np.ones((5000), np.uint16) * x * 0.04 mviri_easy["u_structured_toa_bidirectional_reflectance"].data[:, x] = np.ones((5000), np.uint16) * x * 0.05 mviri_easy["sub_satellite_latitude_start"].data = 25.25 mviri_easy["sub_satellite_longitude_start"].data = 26.26 mviri_easy["sub_satellite_latitude_end"].data = 27.27 mviri_easy["sub_satellite_longitude_end"].data = 28.28 return mviri_easy
def testCreateTemplateEasy_AMSUB(self): ds = FCDRWriter.createTemplateEasy('AMSUB', 2561) self.assertIsNotNone(ds) Assertions.assert_global_attributes(self, ds.attrs) self.assertEqual(19, len(ds.data_vars)) # geolocation + flags self._verify_geolocation_variables(ds) self._verify_quality_flags(ds) # sensor specific self._verify_amsub_specific_variables(ds) # easy FCDR variables self.assertIsNotNone(ds.variables["u_independent_btemps"]) self.assertIsNotNone(ds.variables["u_structured_btemps"])
def testCreateTemplateEasy_SSMT2(self): ds = FCDRWriter.createTemplateEasy('SSMT2', 722) self.assertIsNotNone(ds) Assertions.assert_global_attributes(self, ds.attrs) self.assertEqual(15, len(ds.data_vars)) # geolocation + flags self._verify_geolocation_variables(ds) self._verify_quality_flags(ds) # sensor specific self.verify_SSMT2_specific_variables(ds) # easy FCDR variables self.assertIsNotNone(ds.variables["u_independent_tb"]) self.assertIsNotNone(ds.variables["u_structured_tb"])
def main(file_in, fileout='None'): data = read_netcdf(file_in) writer = FCDRWriter() # get a template for sensor name in EASY format, supply product height # The scan-width is set automatically dataset = writer.createTemplateEasy("AVHRR", data.ny) # set some mandatory global attributes. Writing will fail if not all of them are filled dataset.attrs["institution"] = "University of Reading" dataset.attrs[ "title"] = "pre-B version of AVHRR Fundamental Climate Data Records" dataset.attrs["source"] = "FIDUCEO" dataset.attrs["history"] = "" dataset.attrs["references"] = "CDF_FCDR_File Spec" dataset.attrs[ "comment"] = "This version is a pre-B one and aims at showing the kind of uncertainties we are aiming to deliver within FIDUCEO. The values are not final ones and should not be used for science purposes." dataset.attrs['sensor'] = "AVHRR" dataset.attrs['platform'] = data.noaa_string dataset.attrs['software_version'] = data.version # write real data to the variables. All variables initially contain "_FillValue". # Not writing to the whole array is completely OK dataset.variables["latitude"].data = data.lat dataset.variables["longitude"].data = data.lon dataset.variables["Time"].data = data.time dataset.variables["satellite_zenith_angle"].data = data.satza dataset.variables["solar_zenith_angle"].data = data.solza dataset.variables["relative_azimuth_angle"].data = data.relaz dataset.variables["Ch1_Ref"].data = data.ch1 dataset.variables["Ch2_Ref"].data = data.ch2 if data.ch3a_there: dataset.variables["Ch3a_Ref"].data = data.ch3a dataset.variables["Ch3b_Bt"].data = data.ch3b dataset.variables["Ch4_Bt"].data = data.ch4 if data.ch5_there: dataset.variables["Ch5_Bt"].data = data.ch5 dataset.variables["u_random_Ch1"].data = data.u_random_ch1 dataset.variables["u_random_Ch2"].data = data.u_random_ch2 if data.ch3a_there: dataset.variables["u_random_Ch3a"].data = data.u_random_ch3a dataset.variables["u_random_Ch3b"].data = data.u_random_ch3b dataset.variables["u_random_Ch4"].data = data.u_random_ch4 if data.ch5_there: dataset.variables["u_random_Ch5"].data = data.u_random_ch5 dataset.variables["u_non_random_Ch1"].data = data.u_non_random_ch1 dataset.variables["u_non_random_Ch2"].data = data.u_non_random_ch2 if data.ch3a_there: dataset.variables["u_non_random_Ch3a"].data = data.u_non_random_ch3a dataset.variables["u_non_random_Ch3b"].data = data.u_non_random_ch3b dataset.variables["u_non_random_Ch4"].data = data.u_non_random_ch4 if data.ch5_there: dataset.variables["u_non_random_Ch5"].data = data.u_non_random_ch5 dataset.variables["quality_scanline_bitmask"].data = data.scan_qual dataset.variables["quality_channel_bitmask"].data = data.chan_qual #avhrr.AVHRR._create_channel_refl_variable(12835, "Channel 6 Reflectance") # dump it to disk, netcdf4, medium compression # writing will fail when the target file already exists if 'None' == fileout: file_out = writer.create_file_name_FCDR_easy('AVHRR',data.noaa_string,\ data.date_time[0],\ data.date_time[-1],\ data.version) else: file_out = fileout writer.write(dataset, file_out)
def create_easy_dataset(self, type): hirs_easy = FCDRWriter.createTemplateEasy(type, 944, SRF_SIZE) hirs_easy.attrs["institution"] = "test" hirs_easy.attrs["title"] = "sir" hirs_easy.attrs["source"] = "invention" hirs_easy.attrs["history"] = "new" hirs_easy.attrs["references"] = "myself" hirs_easy.attrs["project"] = "test-project" hirs_easy.attrs["creator_url"] = "test-url" hirs_easy.attrs["creator_name"] = "test-name" hirs_easy.attrs["creator_email"] = "*****@*****.**" hirs_easy.attrs["fcdr_software_version"] = "12" hirs_easy.attrs["data_version"] = "13" hirs_easy.attrs["time_coverage_start"] = "now" hirs_easy.attrs["time_coverage_end"] = "later" hirs_easy.attrs["time_coverage_duration"] = "some moment" hirs_easy.attrs["platform"] = "yo" hirs_easy.attrs["comment"] = "what?" for x in range(0, 56): hirs_easy["bt"].data[:, :, x] = np.ones((944), np.int16) * x * 0.01 hirs_easy["latitude"].data[:, x] = np.ones( (944), np.int16) * x * 0.02 hirs_easy["longitude"].data[:, x] = np.ones( (944), np.int16) * x * 0.03 hirs_easy["data_quality_bitmask"].data[:, x] = np.ones( (944), np.int8) * x hirs_easy["quality_pixel_bitmask"].data[:, x] = np.ones( (944), np.int8) * x hirs_easy["satellite_zenith_angle"].data[:, x] = np.ones( (944), np.int16) * x * 0.04 hirs_easy["satellite_azimuth_angle"].data[:, x] = np.ones( (944), np.int16) * x * 0.05 hirs_easy["solar_zenith_angle"].data[:, x] = np.ones( (944), np.int16) * x * 0.06 hirs_easy["solar_azimuth_angle"].data[:, x] = np.ones( (944), np.int16) * x * 0.05 hirs_easy["u_independent"].data[:, :, x] = np.ones( (944), np.int16) * x * 0.06 hirs_easy["u_structured"].data[:, :, x] = np.ones( (944), np.int16) * x * 0.07 hirs_easy["u_common"].data[:, :, x] = np.ones( (944), np.int16) * x * 0.08 if type != "HIRS2": for x in range(0, 19): hirs_easy["quality_channel_bitmask"].data[:, x] = np.ones( (944), np.int32) * x * 3 hirs_easy["quality_scanline_bitmask"].data[:] = np.ones((944), np.int8) hirs_easy["scanline"].data[:] = np.ones((944), np.int8) * 3 hirs_easy["time"].data[:] = np.ones((944), np.int32) * 400000000000000 for x in range(0, SRF_SIZE): hirs_easy["SRF_weights"].data[:, x] = np.ones( (NUM_CHANNELS), np.float32) * x * 0.04 hirs_easy["SRF_wavelengths"].data[:, x] = np.ones( (NUM_CHANNELS), np.float32) * x * 0.05 return hirs_easy
def testCreate_MVIRI_STATIC(self): ds = FCDRWriter.createTemplateEasy('MVIRI_STATIC', 5000) self.assertIsNotNone(ds.variables["latitude_vis"]) self.assertIsNotNone(ds.variables["longitude_vis"]) self.assertIsNotNone(ds.variables["latitude_ir_wv"]) self.assertIsNotNone(ds.variables["longitude_ir_wv"])