def test_write_overwrite_true(self):
testFile = os.path.join(self.testDir, 'delete_me.nc')
FCDRWriter.write(self.dataset, testFile, overwrite=True)
self.assertTrue(os.path.isfile(testFile))
FCDRWriter.write(self.dataset, testFile, overwrite=True)
self.assertTrue(os.path.isfile(testFile))
def test_write_overwrite_false(self):
testFile = os.path.join(self.testDir, 'delete_me.nc')
FCDRWriter.write(self.dataset, testFile)
self.assertTrue(os.path.isfile(testFile))
try:
FCDRWriter.write(self.dataset, testFile, overwrite=False)
self.fail("IOError expected")
except IOError:
pass
def test_write_easy(self):
mviri_easy = self.create_easy_dataset()
start = datetime(2011, 9, 12, 13, 24, 52)
end = datetime(2011, 9, 12, 13, 27, 51)
file_name = FCDRWriter.create_file_name_FCDR_easy("MVIRI", "Meteosat8", start, end, "1.0")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.now()
FCDRWriter.write(mviri_easy, self.target_path)
end_time = datetime.now()
elapsed_time = end_time - start_time
print("MVIRI EASY write time: " + str(elapsed_time.seconds) + "." + str(round(elapsed_time.microseconds / 1000)))
self.assertTrue(os.path.isfile(self.target_path))
target_data = xr.open_dataset(self.target_path)
try:
self.assert_global_flags(target_data)
self.assert_sensor_variables(target_data)
variable = target_data["toa_bidirectional_reflectance_vis"]
self.assertAlmostEqual(0.75, variable.data[25, 25], 5)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_independent_toa_bidirectional_reflectance"]
self.assertAlmostEqual(1.0400092904, variable.data[26, 26], 4)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_structured_toa_bidirectional_reflectance"]
self.assertAlmostEqual(1.3500070712000001, variable.data[27, 27], 4)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["sub_satellite_latitude_start"]
self.assertAlmostEqual(25.25, variable.data, 8)
variable = target_data["sub_satellite_longitude_start"]
self.assertAlmostEqual(26.26, variable.data, 8)
variable = target_data["sub_satellite_latitude_end"]
self.assertAlmostEqual(27.27, variable.data, 8)
variable = target_data["sub_satellite_longitude_end"]
self.assertAlmostEqual(28.28, variable.data, 8)
finally:
target_data.close()
def main():
writer = FCDRWriter()
# get a template for sensor name in FULL format, supply product height
# The scan-width is set automatically
# ---------------------------------------------------------------------
dataset = writer.createTemplateFull("AVHRR", 128)
# set some mandatory global attributes (CF standards). Writing will fail if not all of them are filled
# automatically set: CF version and FIDUCEO license
# ----------------------------------------------------------------------------------------------------
dataset.attrs["institution"] = "Brockmann Consult GmbH"
dataset.attrs["title"] = "FIDUCEO test dataset"
dataset.attrs["source"] = "arbitray stuff"
dataset.attrs["history"] = "none"
dataset.attrs["references"] = "CDR_FCDR sensor reference documentation"
dataset.attrs[
"comment"] = "just to show how things are intended to be used"
# write real data to the variables. All variables initially contain "_FillValue".
# Not writing to the whole array is completely OK
# -------------------------------------------------------------------------------
Time = dataset.variables["Time"]
Time.data[44] = 0.456
Time.data[45] = 0.457
raa = dataset.variables["relative_azimuth_angle"]
raa.data[3, 0] = 0.567
raa.data[3, 1] = 0.568
# ensure not to generate over/underflows
# --------------------------------------
DataUtility.check_scaling_ranges(raa)
# create a standardized file name
# -------------------------------
start = datetime.datetime(2006, 8, 23, 14, 24, 52)
end = datetime.datetime(2006, 8, 23, 15, 25, 53)
file_name = writer.create_file_name_FCDR_full("AVHRR", "NOAA12", start,
end, "01.2")
# dump it to disk, netcdf4, medium compression
# overwrite existing file
# --------------------------------------------
writer.write(dataset,
"D:\\Satellite\\DELETE\\" + file_name,
overwrite=True)
def main(file_in, fileout='None'):
data = read_netcdf(file_in)
writer = FCDRWriter()
# get a template for sensor name in EASY format, supply product height
# The scan-width is set automatically
dataset = writer.createTemplateEasy("AVHRR", data.ny)
# set some mandatory global attributes. Writing will fail if not all of them are filled
dataset.attrs["institution"] = "University of Reading"
dataset.attrs[
"title"] = "pre-B version of AVHRR Fundamental Climate Data Records"
dataset.attrs["source"] = "FIDUCEO"
dataset.attrs["history"] = ""
dataset.attrs["references"] = "CDF_FCDR_File Spec"
dataset.attrs[
"comment"] = "This version is a pre-B one and aims at showing the kind of uncertainties we are aiming to deliver within FIDUCEO. The values are not final ones and should not be used for science purposes."
dataset.attrs['sensor'] = "AVHRR"
dataset.attrs['platform'] = data.noaa_string
dataset.attrs['software_version'] = data.version
# write real data to the variables. All variables initially contain "_FillValue".
# Not writing to the whole array is completely OK
dataset.variables["latitude"].data = data.lat
dataset.variables["longitude"].data = data.lon
dataset.variables["Time"].data = data.time
dataset.variables["satellite_zenith_angle"].data = data.satza
dataset.variables["solar_zenith_angle"].data = data.solza
dataset.variables["relative_azimuth_angle"].data = data.relaz
dataset.variables["Ch1_Ref"].data = data.ch1
dataset.variables["Ch2_Ref"].data = data.ch2
if data.ch3a_there:
dataset.variables["Ch3a_Ref"].data = data.ch3a
dataset.variables["Ch3b_Bt"].data = data.ch3b
dataset.variables["Ch4_Bt"].data = data.ch4
if data.ch5_there:
dataset.variables["Ch5_Bt"].data = data.ch5
dataset.variables["u_random_Ch1"].data = data.u_random_ch1
dataset.variables["u_random_Ch2"].data = data.u_random_ch2
if data.ch3a_there:
dataset.variables["u_random_Ch3a"].data = data.u_random_ch3a
dataset.variables["u_random_Ch3b"].data = data.u_random_ch3b
dataset.variables["u_random_Ch4"].data = data.u_random_ch4
if data.ch5_there:
dataset.variables["u_random_Ch5"].data = data.u_random_ch5
dataset.variables["u_non_random_Ch1"].data = data.u_non_random_ch1
dataset.variables["u_non_random_Ch2"].data = data.u_non_random_ch2
if data.ch3a_there:
dataset.variables["u_non_random_Ch3a"].data = data.u_non_random_ch3a
dataset.variables["u_non_random_Ch3b"].data = data.u_non_random_ch3b
dataset.variables["u_non_random_Ch4"].data = data.u_non_random_ch4
if data.ch5_there:
dataset.variables["u_non_random_Ch5"].data = data.u_non_random_ch5
dataset.variables["quality_scanline_bitmask"].data = data.scan_qual
dataset.variables["quality_channel_bitmask"].data = data.chan_qual
#avhrr.AVHRR._create_channel_refl_variable(12835, "Channel 6 Reflectance")
# dump it to disk, netcdf4, medium compression
# writing will fail when the target file already exists
if 'None' == fileout:
file_out = writer.create_file_name_FCDR_easy('AVHRR',data.noaa_string,\
data.date_time[0],\
data.date_time[-1],\
data.version)
else:
file_out = fileout
writer.write(dataset, file_out)
def test_write_easy(self):
avhrr_easy = self.create_easy_dataset()
start = datetime.datetime(2016, 11, 22, 13, 24, 52)
end = datetime.datetime(2016, 11, 22, 14, 25, 53)
file_name = FCDRWriter.create_file_name_FCDR_easy("AVHRR", "NOAA18", start, end, "1.0")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.datetime.now()
FCDRWriter.write(avhrr_easy, self.target_path)
end_time = datetime.datetime.now()
elapsed_time = end_time - start_time
print("AVHRR EASY write time: " + str(elapsed_time.seconds) + "." + str(round(elapsed_time.microseconds / 1000)))
self.assertTrue(os.path.isfile(self.target_path))
# target_data = xr.open_dataset(self.target_path, chunks=128)
target_data = xr.open_dataset(self.target_path)
try:
self.assert_geolocation_variables(target_data)
self.assert_global_flags(target_data)
self.assert_sensor_variables(target_data)
variable = target_data["relative_azimuth_angle"]
self.assertAlmostEqual(0.11, variable.data[11, 11], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_independent_Ch1"]
self.assertAlmostEqual(0.182, variable.data[14, 14], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_independent_Ch2"]
self.assertAlmostEqual(0.21, variable.data[15, 15], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_independent_Ch3a"]
self.assertAlmostEqual(0.24, variable.data[16, 16], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_independent_Ch4"]
self.assertAlmostEqual(0.272, variable.data[17, 17], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_independent_Ch5"]
self.assertAlmostEqual(0.306, variable.data[18, 18], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_structured_Ch1"]
self.assertAlmostEqual(-0.31336, variable.data[19, 19], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_structured_Ch2"]
self.assertAlmostEqual(-0.27536, variable.data[20, 20], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_structured_Ch3a"]
self.assertAlmostEqual(-0.23536, variable.data[21, 21], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_structured_Ch4"]
self.assertAlmostEqual(0.462, variable.data[22, 22], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_structured_Ch5"]
self.assertAlmostEqual(0.506, variable.data[23, 23], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_common_Ch1"]
self.assertAlmostEqual(0.072, variable.data[24, 24], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_common_Ch2"]
self.assertAlmostEqual(0.1, variable.data[25, 25], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_common_Ch3a"]
self.assertAlmostEqual(0.13, variable.data[26, 26], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_common_Ch4"]
self.assertAlmostEqual(0.162, variable.data[27, 27], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_common_Ch5"]
self.assertAlmostEqual(0.196, variable.data[28, 28], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
self.assert_srf_data(target_data)
finally:
target_data.close()
def test_write_full(self):
avhrr_full = self.create_full_dataset()
start = datetime.datetime(2016, 11, 22, 14, 25, 53)
end = datetime.datetime(2016, 11, 22, 15, 26, 54)
file_name = FCDRWriter.create_file_name_FCDR_full("AVHRR", "NOAA19", start, end, "1.0")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.datetime.now()
FCDRWriter.write(avhrr_full, self.target_path)
end_time = datetime.datetime.now()
elapsed_time = end_time - start_time
print("AVHRR FULL write time: " + str(elapsed_time.seconds) + "." + str(round(elapsed_time.microseconds / 1000)))
self.assertTrue(os.path.isfile(self.target_path))
target_data = xr.open_dataset(self.target_path)
try:
self.assert_geolocation_variables(target_data)
self.assert_global_flags(target_data)
self.assert_sensor_variables(target_data)
variable = target_data["u_latitude"]
self.assertAlmostEqual(3.36, variable.data[24, 24], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_longitude"]
self.assertAlmostEqual(3.75, variable.data[25, 25], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_time"]
self.assertAlmostEqual(0.16, variable.data[256], 8)
variable = target_data["u_satellite_azimuth_angle"]
self.assertAlmostEqual(4.32, variable.data[27, 27], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_satellite_zenith_angle"]
self.assertAlmostEqual(4.76, variable.data[28, 28], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_solar_azimuth_angle"]
self.assertAlmostEqual(5.22, variable.data[29, 29], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_solar_zenith_angle"]
self.assertAlmostEqual(5.7, variable.data[30, 30], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["PRT_C"]
self.assertEqual(0, variable.data[31, 0])
self.assertEqual((13198, 3), variable.encoding["chunksizes"])
variable = target_data["u_prt"]
self.assertAlmostEqual(0.2, variable.data[32, 1], 8)
self.assertEqual((13198, 3), variable.encoding["chunksizes"])
variable = target_data["R_ICT"]
self.assertAlmostEqual(0.42, variable.data[33, 2], 7)
self.assertEqual((13198, 3), variable.encoding["chunksizes"])
variable = target_data["T_instr"]
self.assertAlmostEqual(0.17, variable.data[257], 8)
variable = target_data["Ch1_Csp"]
self.assertEqual(68, variable.data[34, 34])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch2_Csp"]
self.assertEqual(105, variable.data[35, 35])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3a_Csp"]
self.assertEqual(144, variable.data[36, 36])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_Csp"]
self.assertEqual(185, variable.data[37, 37])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_Csp"]
self.assertEqual(228, variable.data[38, 38])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_Csp"]
self.assertEqual(273, variable.data[39, 39])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_Cict"]
self.assertEqual(320, variable.data[40, 40])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_Cict"]
self.assertEqual(369, variable.data[41, 41])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_Cict"]
self.assertEqual(420, variable.data[42, 42])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch1_Ce"]
self.assertEqual(473, variable.data[43, 43])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch2_Ce"]
self.assertEqual(528, variable.data[44, 44])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3a_Ce"]
self.assertEqual(585, variable.data[45, 45])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_Ce"]
self.assertEqual(644, variable.data[46, 46])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_Ce"]
self.assertEqual(705, variable.data[47, 47])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_Ce"]
self.assertEqual(768, variable.data[48, 48])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch1_u_Csp"]
self.assertAlmostEqual(8.33, variable.data[49, 49], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch2_u_Csp"]
self.assertAlmostEqual(9.0, variable.data[50, 50], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3a_u_Csp"]
self.assertAlmostEqual(9.69, variable.data[51, 51], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_u_Csp"]
self.assertAlmostEqual(10.4, variable.data[52, 52], 5)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_u_Csp"]
self.assertAlmostEqual(11.13, variable.data[53, 53], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_u_Csp"]
self.assertAlmostEqual(11.88, variable.data[54, 54], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_u_Cict"]
self.assertAlmostEqual(12.65, variable.data[55, 55], 5)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_u_Cict"]
self.assertAlmostEqual(13.44, variable.data[56, 56], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_u_Cict"]
self.assertAlmostEqual(14.25, variable.data[57, 57], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch1_u_Ce"]
self.assertAlmostEqual(15.08, variable.data[58, 58], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch2_u_Ce"]
self.assertAlmostEqual(15.93, variable.data[59, 59], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3a_u_Ce"]
self.assertAlmostEqual(16.8, variable.data[60, 60], 5)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_u_Ce"]
self.assertAlmostEqual(17.69, variable.data[61, 61], 5)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_u_Ce"]
self.assertAlmostEqual(18.6, variable.data[62, 62], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_u_Ce"]
self.assertAlmostEqual(19.53, variable.data[63, 63], 5)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch1_u_Refl"]
self.assertAlmostEqual(20.48, variable.data[64, 64], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch2_u_Refl"]
self.assertAlmostEqual(21.45, variable.data[65, 65], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3a_u_Refl"]
self.assertAlmostEqual(22.44, variable.data[66, 66], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_u_Bt"]
self.assertAlmostEqual(23.45, variable.data[67, 67], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_u_Bt"]
self.assertAlmostEqual(24.48, variable.data[68, 68], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_u_Bt"]
self.assertAlmostEqual(25.53, variable.data[69, 69], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_ur_Bt"]
self.assertAlmostEqual(26.6, variable.data[70, 70], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_ur_Bt"]
self.assertAlmostEqual(27.69, variable.data[71, 71], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_ur_Bt"]
self.assertAlmostEqual(28.8, variable.data[72, 72], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch3b_us_Bt"]
self.assertAlmostEqual(29.93, variable.data[73, 73], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch4_us_Bt"]
self.assertAlmostEqual(31.08, variable.data[74, 74], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["Ch5_us_Bt"]
self.assertAlmostEqual(32.25, variable.data[75, 75], 6)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
finally:
target_data.close()
def test_write_HIRS2(self):
hirs_easy = self.create_easy_dataset("HIRS2")
start = datetime(2015, 10, 21, 13, 24, 52)
end = datetime(2015, 10, 21, 14, 25, 53)
file_name = FCDRWriter.create_file_name_FCDR_easy(
"HIRS2", "NOAA12", start, end, "1.0")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.now()
FCDRWriter.write(hirs_easy, self.target_path)
end_time = datetime.now()
elapsed_time = end_time - start_time
print("HIRS/2 EASY write time: " + "%d.%03d" %
(elapsed_time.seconds,
int(round(elapsed_time.microseconds / 1000))))
self.assertTrue(os.path.isfile(self.target_path))
target_data = xr.open_dataset(self.target_path)
try:
variable = target_data["bt"]
self.assertAlmostEqual(0.0, variable.data[0, 0, 0], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
variable = target_data["latitude"]
self.assertAlmostEqual(0.0192266606, variable.data[1, 1], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["longitude"]
self.assertAlmostEqual(0.0604266487, variable.data[2, 2], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["quality_pixel_bitmask"]
self.assertEqual(3, variable.data[3, 3])
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["data_quality_bitmask"]
self.assertEqual(2, variable.data[9, 2])
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["quality_scanline_bitmask"]
self.assertEqual(1, variable.data[4])
self.assertEqual((944, ), variable.encoding["chunksizes"])
variable = target_data["satellite_zenith_angle"]
self.assertAlmostEqual(0.2, variable.data[5, 5], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["satellite_azimuth_angle"]
self.assertAlmostEqual(0.30, variable.data[6, 6], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["scanline"]
self.assertEqual(3, variable.data[6])
self.assertEqual((944, ), variable.encoding["chunksizes"])
variable = target_data["solar_azimuth_angle"]
self.assertAlmostEqual(0.35, variable.data[7, 7], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["solar_azimuth_angle"]
self.assertAlmostEqual(0.4, variable.data[8, 8], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["solar_zenith_angle"]
self.assertAlmostEqual(0.54, variable.data[9, 9], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["time"]
self.assertEqual(np.datetime64('1970-01-05T15:06:40.000000000'),
variable.data[8])
self.assertEqual((944, ), variable.encoding["chunksizes"])
variable = target_data["u_independent"]
self.assertAlmostEqual(0.54, variable.data[9, 9, 9], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
variable = target_data["u_structured"]
self.assertAlmostEqual(0.7, variable.data[10, 10, 10], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
variable = target_data["u_common"]
self.assertAlmostEqual(0.88, variable.data[11, 11, 11], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
self.assert_srf(target_data)
finally:
target_data.close()
def test_write_full(self):
mviri_full = self.create_full_dataset()
start = datetime(2010, 8, 11, 13, 24, 52)
end = datetime(2010, 8, 11, 13, 27, 51)
file_name = FCDRWriter.create_file_name_FCDR_full("MVIRI", "Meteosat7", start, end, "1.0")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.now()
FCDRWriter.write(mviri_full, self.target_path)
end_time = datetime.now()
elapsed_time = end_time - start_time
print("MVIRI FULL write time: " + str(elapsed_time.seconds) + "." + str(round(elapsed_time.microseconds / 1000)))
self.assertTrue(os.path.isfile(self.target_path))
target_data = xr.open_dataset(self.target_path)
try:
self.assert_global_flags(target_data)
self.assert_sensor_variables(target_data)
variable = target_data["count_vis"]
self.assertEqual(11, variable.data[11, 11])
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_latitude"]
self.assertAlmostEqual(0.21696, variable.data[12, 12], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_longitude"]
self.assertAlmostEqual(0.633915, variable.data[13, 13], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_time"]
self.assertAlmostEqual(0.402832012, variable.data[14], 8)
self.assertEqual((2500,), variable.encoding["chunksizes"])
variable = target_data["u_satellite_zenith_angle"]
self.assertAlmostEqual(4.4999668098, variable.data[15, 15], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_satellite_azimuth_angle"]
self.assertAlmostEqual(1.399993065, variable.data[16, 16], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_solar_zenith_angle"]
self.assertAlmostEqual(3.4999826625, variable.data[17, 17], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["u_solar_azimuth_angle"]
self.assertAlmostEqual(0.8000178354, variable.data[18, 18], 8)
self.assertEqual(EXPECTED_CHUNKING, variable.encoding["chunksizes"])
variable = target_data["a0_vis"]
self.assertAlmostEqual(7.7, variable.data, 8)
variable = target_data["a1_vis"]
self.assertAlmostEqual(8.8, variable.data, 8)
variable = target_data["mean_count_space_vis"]
self.assertAlmostEqual(9.9, variable.data, 8)
variable = target_data["u_a0_vis"]
self.assertAlmostEqual(10.1, variable.data, 8)
variable = target_data["u_a1_vis"]
self.assertAlmostEqual(11.11, variable.data, 8)
variable = target_data["u_a2_vis"]
self.assertAlmostEqual(12.12, variable.data, 8)
variable = target_data["u_zero_vis"]
self.assertAlmostEqual(13.13, variable.data, 8)
variable = target_data["covariance_a_vis"]
self.assertEqual((3,3), variable.shape)
self.assertAlmostEqual(0.56, variable.data[1, 2], 8)
variable = target_data["u_electronics_counts_vis"]
self.assertAlmostEqual(13.13, variable.data, 8)
variable = target_data["u_digitization_counts_vis"]
self.assertAlmostEqual(14.14, variable.data, 8)
variable = target_data["allan_deviation_counts_space_vis"]
self.assertAlmostEqual(15.15, variable.data, 8)
finally:
target_data.close()
def test_write_HIRS4(self):
hirs_easy = self.create_easy_dataset("HIRS4")
start = datetime(2013, 8, 19, 13, 24, 52)
end = datetime(2013, 8, 19, 14, 25, 53)
file_name = FCDRWriter.create_file_name_FCDR_easy(
"HIRS4", "NOAA18", start, end, "1.0")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.now()
FCDRWriter.write(hirs_easy, self.target_path)
end_time = datetime.now()
elapsed_time = end_time - start_time
print("HIRS/4 EASY write time: " + "%d.%03d" %
(elapsed_time.seconds,
int(round(elapsed_time.microseconds / 1000))))
self.assertTrue(os.path.isfile(self.target_path))
target_data = xr.open_dataset(self.target_path)
try:
variable = target_data["bt"]
self.assertAlmostEqual(0.02, variable.data[2, 2, 2], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
variable = target_data["latitude"]
self.assertAlmostEqual(0.0604266476, variable.data[3, 3], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["longitude"]
self.assertAlmostEqual(0.1208532974, variable.data[4, 4], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["quality_pixel_bitmask"]
self.assertEqual(7, variable.data[5, 5])
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["data_quality_bitmask"]
self.assertEqual(4, variable.data[11, 4])
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["quality_scanline_bitmask"]
self.assertEqual(1, variable.data[6])
self.assertEqual((944, ), variable.encoding["chunksizes"])
variable = target_data["quality_channel_bitmask"]
self.assertEqual(21, variable.data[7, 7])
self.assertEqual((944, 19), variable.encoding["chunksizes"])
variable = target_data["satellite_zenith_angle"]
self.assertAlmostEqual(0.28, variable.data[7, 7], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["satellite_azimuth_angle"]
self.assertAlmostEqual(0.4, variable.data[8, 8], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["scanline"]
self.assertEqual(3, variable.data[8])
self.assertEqual((944, ), variable.encoding["chunksizes"])
variable = target_data["solar_azimuth_angle"]
self.assertAlmostEqual(0.45, variable.data[9, 9], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["solar_zenith_angle"]
self.assertAlmostEqual(0.6, variable.data[10, 10], 8)
self.assertEqual(EXPECTED_CHUNKING_2D,
variable.encoding["chunksizes"])
variable = target_data["time"]
self.assertEqual(4, variable.data[10])
self.assertEqual((944, ), variable.encoding["chunksizes"])
variable = target_data["u_independent"]
self.assertAlmostEqual(0.66, variable.data[11, 11, 11], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
variable = target_data["u_structured"]
self.assertAlmostEqual(0.84, variable.data[12, 12, 12], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
variable = target_data["u_common"]
self.assertAlmostEqual(1.04, variable.data[13, 13, 13], 8)
self.assertEqual(EXPECTED_CHUNKING_3D,
variable.encoding["chunksizes"])
self.assert_srf(target_data)
finally:
target_data.close()