def test_create_ds_template_context_no_db(self, mock_pnu,
mock_create_template_dataset):
context = setup_test_context()
dim_sizes_dict = {"n_w": 271, "n_s": 10}
variables_dict_defs = {"def1": "vars1", "def2": "vars2"}
metadata_defs = {"def1": {}, "def2": {}}
hdsb = HypernetsDSBuilder(context=context,
variables_dict_defs=variables_dict_defs,
metadata_defs=metadata_defs)
ds = hdsb.create_ds_template(dim_sizes_dict,
ds_format="def1",
propagate_ds="propagate_ds")
# test calls to create_template_dataset
mock_pnu.return_value.create_product_name.assert_called_once_with(
"def1")
mock_create_template_dataset.assert_called_once_with(
"vars1",
dim_sizes_dict,
metadata={
"product_name":
mock_pnu.return_value.create_product_name.return_value
},
propagate_ds="propagate_ds",
metadata_db=context.metadata_db,
metadata_db_query=None)
def test_get_target_sequences_1sequence(self):
tmpdir = "tmp_" + "".join(random.choices(string.ascii_lowercase, k=6))
context = setup_test_context(
raw_data_directory=os.path.join(tmpdir, "data"),
archive_directory=os.path.join(tmpdir, "out"),
metadata_db_url="sqlite:///" + tmpdir + "/metadata.db",
anomaly_db_url="sqlite:///" + tmpdir + "/anomaly.db",
archive_db_url="sqlite:///" + tmpdir + "/archive.db",
create_directories=True,
create_dbs=True)
context.set_config_value(
"raw_data_directory",
os.path.join(context.get_config_value("raw_data_directory"),
"SEQ20200311T112230"),
)
expected_sequences = [
os.path.join(tmpdir, "data", "SEQ20200311T112230")
]
sequences = get_target_sequences(context, False)
self.assertCountEqual(expected_sequences, sequences)
shutil.rmtree(tmpdir)
def test_create_ds_template_context(self, mock_pnu,
mock_create_template_dataset):
temp_name = ''.join(random.choices(string.ascii_lowercase, k=6))
context = setup_test_context(
anomaly_db_url="sqlite:///anomaly_" + temp_name + ".db",
metadata_db_url="sqlite:///metadata_" + temp_name + ".db",
create_dbs=True)
dim_sizes_dict = {"n_w": 271, "n_s": 10}
variables_dict_defs = {"def1": "vars1", "def2": "vars2"}
metadata_defs = {"def1": {}, "def2": {}}
hdsb = HypernetsDSBuilder(context=context,
variables_dict_defs=variables_dict_defs,
metadata_defs=metadata_defs)
ds = hdsb.create_ds_template(dim_sizes_dict,
ds_format="def1",
propagate_ds="propagate_ds")
# test calls to create_template_dataset
mock_pnu.return_value.create_product_name.assert_called_once_with(
"def1")
mock_create_template_dataset.assert_called_once_with(
"vars1",
dim_sizes_dict,
metadata={
"product_name":
mock_pnu.return_value.create_product_name.return_value
},
propagate_ds="propagate_ds",
metadata_db=context.metadata_db,
metadata_db_query={})
teardown_test_context(context, remove_dbs=True)
def test_run_mock(self, mock_RhymerHypstar):
context = setup_test_context()
hp = HypernetsProcessor()
hp.context = context
hp.run()
# Here you can check what RhymerHypstar was called with to see that that works correctly
mock_RhymerHypstar.assert_called_once_with()
def test_run(self):
context = setup_test_context()
hp = HypernetsProcessor()
hp.context = context
hp.run()
self.assertTrue(True)
def test_create_product_name_context(self):
with freeze_time("2020/1/8 9:45:30"):
context = setup_test_context()
context.set_config_value("time", datetime(2018, 4, 3, 11, 0, 0))
pnu = ProductNameUtil(context=context)
pname = pnu.create_product_name("L_L1A_RAD")
self.assertEqual(
"HYPERNETS_L_TEST_L1A_RAD_201804031100_202001080945_v0.0",
pname)
cov_filament = punpy.convert_corr_to_cov(
np.ones((len(gains[:, 2]), len(gains[:, 2]))),
gains[:, 2] * (gains[:, 6]**2)**0.5 / 100)
calibration_data["corr_systematic_indep_gains"].values[i_cal] = \
punpy.correlation_from_covariance(cov_diag+cov_other+cov_full+cov_filament)
calibration_data["u_systematic_corr_rad_irr_gains"].values[
i_cal] = gains[:, 2] * (gains[:, 4]**2 + gains[:, 5]**2 +
gains[:, 18]**2)**0.5 / 100
cov_other = punpy.convert_corr_to_cov(
np.eye(len(gains[:, 2])), gains[:, 2] *
(gains[:, 4]**2 + gains[:, 18]**2)**0.5 / 100)
cov_filament = punpy.convert_corr_to_cov(
np.ones((len(gains[:, 2]), len(gains[:, 2]))),
gains[:, 2] * (gains[:, 5]**2)**0.5 / 100)
calibration_data["corr_systematic_corr_rad_irr_gains"].values[i_cal] = \
punpy.correlation_from_covariance(cov_other+cov_filament)
i_cal += 1
return calibration_data
if __name__ == '__main__':
context = setup_test_context()
calcov = CalibrationConverter(context)
calcov.convert_all_calibration_data()
teardown_test_context(context)
def test_end_to_end(self):
this_directory_path = os.path.abspath(os.path.dirname(__file__))
this_directory_path = os.path.join(this_directory_path,"..\\")
tmpdir = "tmp_"+"".join(random.choices(string.ascii_lowercase,k=6))
#context = setup_test_context()
context = setup_test_context(
raw_data_directory = os.path.join(tmpdir,"data"),
archive_directory = os.path.join(tmpdir,"out"),
metadata_db_url = "sqlite:///"+tmpdir+"/metadata.db",
anomaly_db_url = "sqlite:///"+tmpdir+"/anomoly.db",
archive_db_url = "sqlite:///"+tmpdir+"/archive.db",
create_directories = True,
create_dbs = False )
context.set_config_value('network', 'l')
context.set_config_value('measurement_function_calibrate', 'StandardMeasurementFunction')
context.set_config_value('measurement_function_interpolate', 'LandNetworkInterpolationIrradianceLinear')
context.set_config_value('measurement_function_surface_reflectance', 'LandNetworkProtocol')
context.set_config_value("processor_directory",this_directory_path)
context.set_config_value("calibration_directory",os.path.join(this_directory_path,"..\\calibration_files\\HYPSTAR_cal\\"))
context.set_config_value("archive_directory", os.path.join(tmpdir,"out"))
context.set_config_value('version','0.1')
context.set_config_value('site_abbr','test')
context.set_config_value('product_format','netcdf4')
context.set_config_value('hypstar_cal_number','220241')
context.set_config_value('cal_date','200728')
context.set_config_value('outliers',7)
context.set_config_value('write_l2a',True)
context.set_config_value('write_l1a',True)
context.set_config_value('write_l1b',True)
context.set_config_value('plot_l2a',True)
context.set_config_value('plot_l1a',True)
context.set_config_value('plot_l1b',True)
context.set_config_value('plot_diff',True)
context.set_config_value('plotting_directory',os.path.join(tmpdir,"out/plots/"))
context.set_config_value('plotting_format',"png")
cal = Calibrate(context,MCsteps=100)
avg = Average(context)
intp = Interpolate(context,MCsteps=1000)
surf = SurfaceReflectance(context,MCsteps=1000)
test_l0_rad,test_l0_irr,test_l0_bla,test_l1a_rad,test_l1a_irr,test_l1b_rad,test_l1b_irr,test_l2a,test_l2a_avg=setup_test_files(context)
L1a_rad = cal.calibrate_l1a("radiance",test_l0_rad,test_l0_bla)
L1a_irr = cal.calibrate_l1a("irradiance",test_l0_irr,test_l0_bla)
L1b_rad = avg.average_l1b("radiance",L1a_rad)
L1b_irr = avg.average_l1b("irradiance",L1a_irr)
L1c = intp.interpolate_l1c(L1b_rad,L1b_irr)
L2a = surf.process_l2(L1c)
# np.testing.assert_allclose(test_l1a_rad["radiance"].values[:,0],L1a_rad["radiance"].values[:,0],rtol=0.12,equal_nan=True)
# np.testing.assert_allclose(test_l1b_rad["radiance"].values,L1b_rad["radiance"].values,rtol=0.12,equal_nan=True)
#np.testing.assert_allclose(np.nansum(test_l1b_rad["radiance"].values),np.nansum(L1b_rad["radiance"].values),rtol=0.05,equal_nan=True)
#np.testing.assert_allclose(test_l1b_rad["radiance"].values,L1b_rad["radiance"].values,rtol=0.03,equal_nan=True)
#np.testing.assert_allclose(L1b_irr["irradiance"].values,test_l1b_irr["irradiance"].values,rtol=0.04,equal_nan=True)
np.testing.assert_allclose(test_l2a["reflectance"].values,L2a["reflectance"].values,rtol=0.19,equal_nan=True)
np.testing.assert_allclose(np.nansum(test_l2a["reflectance"].values),np.nansum(L2a["reflectance"].values),rtol=0.05,equal_nan=True)
np.testing.assert_allclose(np.nansum(test_l2a_avg["reflectance"].values),np.nansum(L2a["reflectance"].values),rtol=0.001,equal_nan=True)
shutil.rmtree(tmpdir)