def add_variables(dataset, width, height): tu.add_geolocation_variables(dataset, width, height, chunksizes=CHUNKING) tu.add_quality_flags(dataset, width, height, chunksizes=CHUNKING) default_array = DefaultData.create_default_vector( height, np.int32, fill_value=4294967295) variable = Variable(["y"], default_array) tu.add_fill_value(variable, 4294967295) variable.attrs["standard_name"] = "time" variable.attrs[ "long_name"] = "Acquisition time in seconds since 1970-01-01 00:00:00" tu.add_units(variable, "s") dataset["time"] = variable default_array = DefaultData.create_default_array(width, height, np.float32, fill_value=np.NaN) variable = Variable(["y", "x"], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["coordinates"] = "longitude latitude" dataset["aot"] = variable dataset["u_independent_aot"] = tu.create_CDR_uncertainty( width, height, "Uncertainty of aot due to independent effects") dataset["u_structured_aot"] = tu.create_CDR_uncertainty( width, height, "Uncertainty of aot due to structured effects") dataset["u_common_aot"] = tu.create_CDR_uncertainty( width, height, "Uncertainty of aot due to common effects")
def test_add_coordinates(self): ds = xr.Dataset() TemplateUtil.add_geolocation_variables(ds, 13, 108) TemplateUtil.add_coordinates(ds) x = ds["x"] self.assertEqual((13, ), x.shape) self.assertEqual(np.uint16, x.dtype) y = ds["y"] self.assertEqual((108, ), y.shape) self.assertEqual(np.uint16, y.dtype)
def add_variables(dataset, width, height, num_samples=10): tu.add_geolocation_variables(dataset, width, height, chunksizes=CHUNKING) tu.add_quality_flags(dataset, width, height, chunksizes=CHUNKING) default_array = DefaultData.create_default_vector(height, np.int32, fill_value=4294967295) variable = Variable(["y"], default_array) tu.add_fill_value(variable, 4294967295) variable.attrs["standard_name"] = "time" variable.attrs["long_name"] = "Acquisition time in seconds since 1970-01-01 00:00:00" tu.add_units(variable, "s") dataset["time"] = variable default_array = DefaultData.create_default_array_3d(width, height, num_samples, np.float32, fill_value=np.NaN) variable = Variable(["samples","y", "x"], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["standard_name"] = "sea_surface_temperature" variable.attrs["units"] = "K" variable.attrs["coordinates"] = "longitude latitude" dataset["sst"] = variable
def test_add_coordinates_with_channel(self): ds = xr.Dataset() TemplateUtil.add_geolocation_variables(ds, 8, 11) default_array = DefaultData.create_default_array_3d( 8, 11, 4, np.float32, np.NaN) ds["three_d"] = Variable(["channel", "y", "x"], default_array) TemplateUtil.add_coordinates(ds) x = ds["x"] self.assertEqual((8, ), x.shape) self.assertEqual(np.uint16, x.dtype) y = ds["y"] self.assertEqual((11, ), y.shape) self.assertEqual(np.uint16, y.dtype) channel = ds["channel"] self.assertEqual((4, ), channel.shape) self.assertEqual(np.uint16, channel.dtype)
def test_add_geolocation_variables(self): ds = xr.Dataset() TemplateUtil.add_geolocation_variables(ds, 8, 10) latitude = ds.variables["latitude"] self.assertEqual((10, 8), latitude.shape) self.assertTrue(np.isnan(latitude.data[4, 4])) self.assertEqual("latitude", latitude.attrs["standard_name"]) self.assertEqual("degrees_north", latitude.attrs["units"]) self.assertEqual(np.int16, latitude.encoding['dtype']) self.assertEqual(-32768, latitude.encoding['_FillValue']) self.assertEqual(0.0027466658, latitude.encoding['scale_factor']) self.assertEqual(0.0, latitude.encoding['add_offset']) longitude = ds.variables["longitude"] self.assertEqual((10, 8), longitude.shape) self.assertTrue(np.isnan(longitude.data[5, 6])) self.assertEqual("longitude", longitude.attrs["standard_name"]) self.assertEqual("degrees_east", longitude.attrs["units"]) self.assertEqual(np.int16, longitude.encoding['dtype']) self.assertEqual(-32768, longitude.encoding['_FillValue']) self.assertEqual(0.0054933317, longitude.encoding['scale_factor']) self.assertEqual(0.0, longitude.encoding['add_offset'])
def add_original_variables(dataset, height, srf_size=None, corr_dx=None, corr_dy=None, lut_size=None): tu.add_geolocation_variables(dataset, SWATH_WIDTH, height) tu.add_quality_flags(dataset, SWATH_WIDTH, height) # btemps default_array = DefaultData.create_default_array_3d( SWATH_WIDTH, height, NUM_CHANNELS, np.float32, np.NaN) variable = Variable(["channel", "y", "x"], default_array) variable.attrs["standard_name"] = "toa_brightness_temperature" tu.add_encoding(variable, np.int32, -999999, scale_factor=0.01) tu.add_units(variable, "K") variable.attrs["ancillary_variables"] = "chanqual qualind scanqual" dataset["btemps"] = variable # chanqual default_array = DefaultData.create_default_array( height, NUM_CHANNELS, np.int32, dims_names=["channel", "y"], fill_value=0) variable = Variable(["channel", "y"], default_array) variable.attrs["standard_name"] = "status_flag" variable.attrs["flag_masks"] = "1, 2, 4, 8, 16, 32" variable.attrs[ "flag_meanings"] = "some_bad_prt_temps some_bad_space_view_counts some_bad_bb_counts no_good_prt_temps no_good_space_view_counts no_good_bb_counts" dataset["chanqual"] = variable # instrtemp default_array = DefaultData.create_default_vector(height, np.float32, fill_value=np.NaN) variable = Variable(["y"], default_array) tu.add_units(variable, "K") tu.add_encoding(variable, np.int32, DefaultData.get_default_fill_value(np.int32), scale_factor=0.01) variable.attrs["long_name"] = "instrument_temperature" dataset["instrtemp"] = variable # qualind default_array = DefaultData.create_default_vector(height, np.int32, fill_value=0) variable = Variable(["y"], default_array) variable.attrs["standard_name"] = "status_flag" variable.attrs[ "flag_masks"] = "33554432, 67108864, 134217728, 268435456, 536870912, 1073741824, 2147483648" variable.attrs[ "flag_meanings"] = "instr_status_changed first_good_clock_update no_earth_loc no_calib data_gap_precedes time_seq_error not_use_scan" dataset["qualind"] = variable # scanqual default_array = DefaultData.create_default_vector(height, np.int32, fill_value=0) variable = Variable(["y"], default_array) variable.attrs["standard_name"] = "status_flag" variable.attrs[ "flag_masks"] = "8, 16, 32, 64, 128, 1024, 2048, 4096, 8192, 16384, 32768, 1048576, 2097152, 4194304, 8388608" variable.attrs[ "flag_meanings"] = "earth_loc_quest_ant_pos earth_loc_quest_reas earth_loc_quest_margin earth_loc_quest_time no_earth_loc_time uncalib_instr_mode uncalib_channels calib_marg_prt uncalib_bad_prt calib_few_scans uncalib_bad_time repeat_scan_times inconsistent_time time_field_bad time_field_inferred" dataset["scanqual"] = variable # scnlin default_array = DefaultData.create_default_vector(height, np.int32) variable = Variable(["y"], default_array) tu.add_fill_value(variable, DefaultData.get_default_fill_value(np.int32)) variable.attrs["long_name"] = "scanline" dataset["scnlin"] = variable # scnlindy default_array = DefaultData.create_default_vector(height, np.int32) variable = Variable(["y"], default_array) tu.add_fill_value(variable, DefaultData.get_default_fill_value(np.int32)) variable.attrs["long_name"] = "Acquisition day of year of scan" dataset["scnlindy"] = variable # scnlintime default_array = DefaultData.create_default_vector(height, np.int32) variable = Variable(["y"], default_array) tu.add_fill_value(variable, DefaultData.get_default_fill_value(np.int32)) variable.attrs[ "long_name"] = "Acquisition time of scan in milliseconds since beginning of the day" tu.add_units(variable, "ms") dataset["scnlintime"] = variable # scnlinyr default_array = DefaultData.create_default_vector(height, np.int32) variable = Variable(["y"], default_array) tu.add_fill_value(variable, DefaultData.get_default_fill_value(np.int32)) variable.attrs["long_name"] = "Acquisition year of scan" dataset["scnlinyr"] = variable # satellite_azimuth_angle variable = AMSUB_MHS.create_angle_variable(height, "sensor_azimuth_angle") dataset["satellite_azimuth_angle"] = variable # satellite_zenith_angle variable = AMSUB_MHS.create_angle_variable(height, "sensor_zenith_angle") dataset["satellite_zenith_angle"] = variable # solar_azimuth_angle variable = AMSUB_MHS.create_angle_variable(height, "solar_azimuth_angle") dataset["solar_azimuth_angle"] = variable # solar_zenith_angle variable = AMSUB_MHS.create_angle_variable(height, "solar_zenith_angle") dataset["solar_zenith_angle"] = variable # acquisition_time default_array = DefaultData.create_default_vector(height, np.int32) variable = Variable(["y"], default_array) tu.add_fill_value(variable, DefaultData.get_default_fill_value(np.int32)) variable.attrs["standard_name"] = "time" variable.attrs[ "long_name"] = "Acquisition time in seconds since 1970-01-01 00:00:00" tu.add_units(variable, "s") dataset["acquisition_time"] = variable
def add_original_variables(dataset, height, srf_size=None): tu.add_geolocation_variables(dataset, SWATH_WIDTH, height, chunksizes=CHUNKS_2D) tu.add_quality_flags(dataset, SWATH_WIDTH, height, chunksizes=CHUNKS_2D) # Time default_array = DefaultData.create_default_vector(height, np.float64, fill_value=np.NaN) variable = Variable(["y"], default_array) tu.add_fill_value(variable, np.NaN) tu.add_units(variable, "s") variable.attrs["standard_name"] = "time" variable.attrs[ "long_name"] = "Acquisition time in seconds since 1970-01-01 00:00:00" dataset["Time"] = variable # relative_azimuth_angle default_array = DefaultData.create_default_array(SWATH_WIDTH, height, np.float32, fill_value=np.NaN) variable = Variable(["y", "x"], default_array) variable.attrs["standard_name"] = "relative_azimuth_angle" tu.add_units(variable, "degree") tu.add_encoding(variable, np.int16, DefaultData.get_default_fill_value(np.int16), 0.01, chunksizes=CHUNKS_2D) variable.attrs["valid_max"] = 18000 variable.attrs["valid_min"] = -18000 tu.add_geolocation_attribute(variable) dataset["relative_azimuth_angle"] = variable # satellite_zenith_angle default_array = DefaultData.create_default_array(SWATH_WIDTH, height, np.float32, fill_value=np.NaN) variable = Variable(["y", "x"], default_array) variable.attrs["standard_name"] = "sensor_zenith_angle" tu.add_units(variable, "degree") tu.add_encoding(variable, np.int16, DefaultData.get_default_fill_value(np.int16), 0.01, chunksizes=CHUNKS_2D) variable.attrs["valid_max"] = 9000 variable.attrs["valid_min"] = 0 tu.add_geolocation_attribute(variable) dataset["satellite_zenith_angle"] = variable # solar_zenith_angle default_array = DefaultData.create_default_array(SWATH_WIDTH, height, np.float32, fill_value=np.NaN) variable = Variable(["y", "x"], default_array) variable.attrs["standard_name"] = "solar_zenith_angle" tu.add_units(variable, "degree") tu.add_encoding(variable, np.int16, DefaultData.get_default_fill_value(np.int16), 0.01, chunksizes=CHUNKS_2D) variable.attrs["valid_max"] = 18000 variable.attrs["valid_min"] = 0 tu.add_geolocation_attribute(variable) dataset["solar_zenith_angle"] = variable dataset["Ch1"] = AVHRR._create_channel_refl_variable( height, "Channel 1 Reflectance") dataset["Ch2"] = AVHRR._create_channel_refl_variable( height, "Channel 2 Reflectance") dataset["Ch3a"] = AVHRR._create_channel_refl_variable( height, "Channel 3a Reflectance") dataset["Ch3b"] = AVHRR._create_channel_bt_variable( height, "Channel 3b Brightness Temperature") dataset["Ch4"] = AVHRR._create_channel_bt_variable( height, "Channel 4 Brightness Temperature") dataset["Ch5"] = AVHRR._create_channel_bt_variable( height, "Channel 5 Brightness Temperature") # data_quality_bitmask default_array = DefaultData.create_default_array(SWATH_WIDTH, height, np.uint8, fill_value=0) variable = Variable(["y", "x"], default_array) variable.attrs["standard_name"] = 'status_flag' variable.attrs["long_name"] = 'bitmask for quality per pixel' variable.attrs["flag_masks"] = '1,2' variable.attrs[ 'flag_meanings'] = 'bad_geolocation_timing_err bad_calibration_radiometer_err' tu.add_chunking(variable, CHUNKS_2D) tu.add_geolocation_attribute(variable) dataset['data_quality_bitmask'] = variable default_array = DefaultData.create_default_vector(height, np.uint8, fill_value=0) variable = Variable(["y"], default_array) variable.attrs["long_name"] = 'bitmask for quality per scanline' variable.attrs["standard_name"] = 'status_flag' variable.attrs["flag_masks"] = '1,2,4,8,16,32,64' variable.attrs[ 'flag_meanings'] = 'do_not_use bad_time bad_navigation bad_calibration channel3a_present solar_contamination_failure solar_contamination' dataset['quality_scanline_bitmask'] = variable default_array = DefaultData.create_default_array(N_CHANS, height, np.uint8, fill_value=0) variable = Variable(["y", "channel"], default_array) variable.attrs["long_name"] = 'bitmask for quality per channel' variable.attrs["standard_name"] = 'status_flag' variable.attrs["flag_masks"] = '1,2' variable.attrs[ 'flag_meanings'] = 'bad_channel some_pixels_not_detected_2sigma' dataset['quality_channel_bitmask'] = variable if srf_size is None: srf_size = MAX_SRF_SIZE default_array = DefaultData.create_default_array(srf_size, N_CHANS, np.float32, fill_value=np.NaN) variable = Variable(["channel", "n_frequencies"], default_array) variable.attrs["long_name"] = 'Spectral Response Function weights' variable.attrs[ "description"] = 'Per channel: weights for the relative spectral response function' tu.add_encoding(variable, np.int16, -32768, 0.000033) dataset['SRF_weights'] = variable default_array = DefaultData.create_default_array(srf_size, N_CHANS, np.float32, fill_value=np.NaN) variable = Variable(["channel", "n_frequencies"], default_array) variable.attrs["long_name"] = 'Spectral Response Function wavelengths' variable.attrs[ "description"] = 'Per channel: wavelengths for the relative spectral response function' tu.add_encoding(variable, np.int32, -2147483648, 0.0001) tu.add_units(variable, "um") dataset['SRF_wavelengths'] = variable default_vector = DefaultData.create_default_vector(height, np.uint8, fill_value=255) variable = Variable(["y"], default_vector) tu.add_fill_value(variable, 255) variable.attrs["long_name"] = 'Indicator of original file' variable.attrs[ "description"] = "Indicator for mapping each line to its corresponding original level 1b file. See global attribute 'source' for the filenames. 0 corresponds to 1st listed file, 1 to 2nd file." dataset["scanline_map_to_origl1bfile"] = variable default_vector = DefaultData.create_default_vector( height, np.int16, fill_value=DefaultData.get_default_fill_value(np.int16)) variable = Variable(["y"], default_vector) tu.add_fill_value(variable, DefaultData.get_default_fill_value(np.int16)) variable.attrs["long_name"] = 'Original_Scan_line_number' variable.attrs[ "description"] = 'Original scan line numbers from corresponding l1b records' dataset["scanline_origl1b"] = variable tu.add_coordinates(dataset, ["Ch1", "Ch2", "Ch3a", "Ch3b", "Ch4", "Ch5"])
def add_geolocation_variables(dataset, height): tu.add_geolocation_variables(dataset, SWATH_WIDTH, height, chunksizes=CHUNKING_2D)
def add_original_variables(dataset, height, srf_size=None): tu.add_geolocation_variables(dataset, SWATH_WIDTH, height) tu.add_quality_flags(dataset, SWATH_WIDTH, height) # Temperature_misc_housekeeping default_array = DefaultData.create_default_array( height, NUM_THERMISTORS, np.float32, dims_names=["housekeeping", "y"], fill_value=np.NaN) variable = Variable(["housekeeping", "y"], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["long_name"] = "TODO" variable.attrs["units"] = "TODO" dataset["Temperature_misc_housekeeping"] = variable # ancil_data default_array = DefaultData.create_default_array( height, ANCIL_VAL, np.float64, dims_names=["ancil_val", "y"], fill_value=np.NaN) variable = Variable(["ancil_val", "y"], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["long_name"] = "Additional per scan information: year, day_of_year, secs_of_day, sat_lat, " \ "sat_long, sat_alt, sat_heading, year, day_of_year, secs_of_day" dataset["ancil_data"] = variable # channel_quality_flag default_array = DefaultData.create_default_array_3d( SWATH_WIDTH, height, NUM_CHANNELS, np.float32, np.NaN) variable = Variable(["channel", "y", "x"], default_array) tu.add_fill_value(variable, np.NaN) dataset["channel_quality_flag"] = variable # cold_counts default_array = DefaultData.create_default_array_3d( SWATH_WIDTH, height, CALIB_NUMBER, np.float32, np.NaN) variable = Variable(["calib_number", "y", "x"], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["long_name"] = "TODO" dataset["cold_counts"] = variable # counts_to_tb_gain default_array = DefaultData.create_default_array( height, NUM_CHANNELS, np.float32, dims_names=["channel", "y"], fill_value=np.NaN) variable = Variable([ "channel", "y", ], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["long_name"] = "TODO" dataset["counts_to_tb_gain"] = variable # counts_to_tb_offset default_array = DefaultData.create_default_array( height, NUM_CHANNELS, np.float32, dims_names=["channel", "y"], fill_value=np.NaN) variable = Variable([ "channel", "y", ], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["long_name"] = "TODO" dataset["counts_to_tb_offset"] = variable # gain_control default_array = DefaultData.create_default_array( height, NUM_CHANNELS, np.float32, dims_names=["channel", "y"], fill_value=np.NaN) variable = Variable([ "channel", "y", ], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["long_name"] = "TODO" dataset["gain_control"] = variable # tb default_array = DefaultData.create_default_array_3d( SWATH_WIDTH, height, NUM_CHANNELS, np.float32, np.NaN) variable = Variable(["channel", "y", "x"], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["long_name"] = "TODO" variable.attrs["standard_name"] = "toa_brightness_temperature" tu.add_units(variable, "K") dataset["tb"] = variable # thermal_reference default_array = DefaultData.create_default_vector( height, np.float32, np.NaN) variable = Variable(["y"], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["long_name"] = "TODO" tu.add_units(variable, "TODO") dataset["thermal_reference"] = variable # warm_counts default_array = DefaultData.create_default_array_3d( SWATH_WIDTH, height, CALIB_NUMBER, np.float32, np.NaN) variable = Variable(["calib_number", "y", "x"], default_array) tu.add_fill_value(variable, np.NaN) variable.attrs["long_name"] = "TODO" dataset["warm_counts"] = variable
def add_geolocation_variables(dataset, height): chunking_2d = HIRS._ensure_chunking_2d(height) tu.add_geolocation_variables(dataset, SWATH_WIDTH, height, chunksizes=chunking_2d)