def test_fill_ds(self):
ds = Dataset()
ds["array_variable1"] = DatasetUtil.create_variable([7, 8], np.float32)
ds["array_variable2"] = DatasetUtil.create_variable([7, 8], np.float32)
ds["array_variable1"][2, 3] = np.nan
ds["array_variable2"][2, 3] = np.nan
HypernetsWriter.fill_ds(ds)
self.assertTrue(np.all(ds["array_variable1"] == 9.96921E36))
self.assertTrue(np.all(ds["array_variable2"] == 9.96921E36))
def test__get_flag_encoding_not_flag_var(self):
ds = Dataset()
ds["array_variable"] = DatasetUtil.create_variable(
[7, 8, 3], np.int8, attributes={"standard_name": "std"})
self.assertRaises(KeyError, DatasetUtil._get_flag_encoding,
ds["array_variable"])
def test_create_variable_1D_int(self):
vector_variable = DatasetUtil.create_variable([5], np.int8)
self.assertIsNotNone(vector_variable)
self.assertEqual(Variable, type(vector_variable))
self.assertEqual((5, ), vector_variable.shape)
self.assertEqual(np.int8, vector_variable.dtype)
self.assertEqual(-127, vector_variable[2])
def test_create_variable_3D_int(self):
array_variable = DatasetUtil.create_variable([7, 8, 3], np.int8)
self.assertIsNotNone(array_variable)
self.assertEqual(Variable, type(array_variable))
self.assertEqual((7, 8, 3), array_variable.shape)
self.assertEqual(np.int8, array_variable.dtype)
self.assertEqual(-127, array_variable[2, 4, 2])
def test_create_variable_3D_int_attributes(self):
array_variable = DatasetUtil.create_variable(
[7, 8, 3], np.int8, attributes={"standard_name": "std"})
self.assertIsNotNone(array_variable)
self.assertEqual(Variable, type(array_variable))
self.assertEqual((7, 8, 3), array_variable.shape)
self.assertEqual(np.int8, array_variable.dtype)
self.assertEqual(-127, array_variable[2, 4, 2])
self.assertEqual("std", array_variable.attrs["standard_name"])
def test_create_variable_1D_int_attributes(self):
vector_variable = DatasetUtil.create_variable(
[5], np.int8, attributes={"standard_name": "std"})
self.assertIsNotNone(vector_variable)
self.assertEqual(Variable, type(vector_variable))
self.assertEqual((5, ), vector_variable.shape)
self.assertEqual(np.int8, vector_variable.dtype)
self.assertEqual(-127, vector_variable[2])
self.assertEqual("std", vector_variable.attrs["standard_name"])
def test_create_variable_1D_int_dims(self):
vector_variable = DatasetUtil.create_variable([5],
np.int8,
dim_names=["dim1"])
self.assertIsNotNone(vector_variable)
self.assertEqual(Variable, type(vector_variable))
self.assertEqual((5, ), vector_variable.shape)
self.assertEqual(np.int8, vector_variable.dtype)
self.assertEqual(-127, vector_variable[2])
self.assertEqual(("dim1", ), vector_variable.dims)
def add_variables(ds, variables_dict, dim_sizes_dict):
"""
Adds defined variables dataset
:type ds: xarray.Dataset
:param ds: dataset
:type variables_dict: dict
:type variables_dict: dictionary defining variables
:type dim_sizes_dict: dict
:param dim_sizes_dict: entry per dataset dimension with value of size as int
:return: dataset with defined variables
:rtype: xarray.Dataset
"""
du = DatasetUtil()
for variable_name in variables_dict.keys():
variable_attrs = variables_dict[variable_name]
# Check variable definition
TemplateUtil._check_variable_definition(variable_name, variable_attrs)
# Unpack variable attributes
dtype = variable_attrs["dtype"]
dim_names = variable_attrs["dim"]
attributes = deepcopy(variable_attrs["attributes"]) if "attributes" in variable_attrs else None
# Determine variable shape from dims
try:
dim_sizes = TemplateUtil._return_variable_shape(dim_names, dim_sizes_dict)
except KeyError:
raise KeyError("Dim Name Error - Variable " + variable_name + " defined with dim not in dim_sizes_dict")
# Create variable and add to dataset
if dtype == "flag":
flag_meanings = attributes.pop("flag_meanings")
variable = du.create_flags_variable(dim_sizes, meanings=flag_meanings,
dim_names=dim_names, attributes=attributes)
else:
variable = du.create_variable(dim_sizes, dim_names=dim_names,
dtype=dtype, attributes=attributes)
if "encoding" in variable_attrs:
du.add_encoding(variable, **variable_attrs["encoding"])
ds[variable_name] = variable
return ds
def test_create_variable_3D_int_dims(self):
array_variable = DatasetUtil.create_variable(
[7, 8, 3], np.int8, dim_names=["dim1", "dim2", "dim3"])
self.assertIsNotNone(array_variable)
self.assertEqual(Variable, type(array_variable))
self.assertEqual((7, 8, 3), array_variable.shape)
self.assertEqual(np.int8, array_variable.dtype)
self.assertEqual(-127, array_variable[2, 4, 2])
self.assertEqual((
"dim1",
"dim2",
"dim3",
), array_variable.dims)
def test_add_encoding(self):
vector_variable = DatasetUtil.create_variable([5], np.int8)
DatasetUtil.add_encoding(vector_variable,
np.int32,
scale_factor=10,
offset=23,
fill_value=11,
chunksizes=12)
self.assertIsNotNone(vector_variable)
self.assertEqual(Variable, type(vector_variable))
self.assertEqual((5, ), vector_variable.shape)
self.assertEqual(np.int8, vector_variable.dtype)
self.assertEqual(-127, vector_variable[2])
self.assertEqual(np.int32, vector_variable.encoding["dtype"])
self.assertEqual(10, vector_variable.encoding["scale_factor"])
self.assertEqual(23, vector_variable.encoding["add_offset"])
self.assertEqual(11, vector_variable.encoding["_FillValue"])
self.assertEqual(12, vector_variable.encoding["chunksizes"])
def preprocess_l0(self, datasetl0, datasetl0_bla, dataset_calib):
"""
Identifies and removes faulty measurements (e.g. due to cloud cover).
:param dataset_l0:
:type dataset_l0:
:return:
:rtype:
"""
wavs = dataset_calib["wavelength"].values
wavpix = dataset_calib["wavpix"].values
datasetl0 = datasetl0.isel(wavelength=slice(int(wavpix[0]),
int(wavpix[-1]) + 1))
datasetl0_bla = datasetl0_bla.isel(
wavelength=slice(int(wavpix[0]),
int(wavpix[-1]) + 1))
mask = self.clip_and_mask(datasetl0, datasetl0_bla)
datasetl0 = datasetl0.assign_coords(wavelength=wavs)
datasetl0_bla = datasetl0_bla.assign_coords(wavelength=wavs)
datasetl0["quality_flag"][np.where(mask == 1)] = DatasetUtil.set_flag(
datasetl0["quality_flag"][np.where(mask == 1)],
"outliers") #for i in range(len(mask))]
DN_rand = DatasetUtil.create_variable(
[len(datasetl0["wavelength"]),
len(datasetl0["scan"])],
dim_names=["wavelength", "scan"],
dtype=np.uint32,
fill_value=0)
datasetl0["u_random_digital_number"] = DN_rand
rand = np.zeros_like(DN_rand.values)
series_ids = np.unique(datasetl0['series_id'])
for i in range(len(series_ids)):
ids = np.where(datasetl0['series_id'] == series_ids[i])[0]
ids_masked = np.where((datasetl0['series_id'] == series_ids[i])
& (mask == 0))[0]
dark_signals = np.zeros_like(
datasetl0['digital_number'].values[:, ids_masked])
for ii, id in enumerate(ids_masked):
dark_signals[:, ii] = self.find_nearest_black(
datasetl0_bla, datasetl0['acquisition_time'].values[id],
datasetl0['integration_time'].values[id])
std = np.std((datasetl0['digital_number'].values[:, ids_masked] -
dark_signals),
axis=1)
for ii, id in enumerate(ids):
rand[:, id] = std
datasetl0["u_random_digital_number"].values = rand
DN_dark = DatasetUtil.create_variable(
[len(datasetl0["wavelength"]),
len(datasetl0["scan"])],
dim_names=["wavelength", "scan"],
dtype=np.uint32,
fill_value=0)
datasetl0["dark_signal"] = DN_dark
dark_signals = []
acqui = datasetl0['acquisition_time'].values
inttimes = datasetl0['integration_time'].values
for i in range(len(acqui)):
dark_signals.append(
self.find_nearest_black(datasetl0_bla, acqui[i], inttimes[i]))
datasetl0["dark_signal"].values = np.array(dark_signals).T
return datasetl0