def test_mask_set_with_flag_values(self):
s2l2a_slc_meanings = [
'no_data', 'saturated_or_defective', 'dark_area_pixels',
'cloud_shadows', 'vegetation', 'bare_soils', 'water',
'clouds_low_probability_or_unclassified',
'clouds_medium_probability', 'clouds_high_probability', 'cirrus',
'snow_or_ice'
]
data = np.array([[1, 2, 8, 3], [7, 6, 0, 4], [9, 5, 11, 10]],
dtype=np.uint8)
flag_var = xr.DataArray(
data,
dims=('y', 'x'),
name='SLC',
attrs=dict(
long_name="Scene classification flags",
flag_values=','.join(f'{i}'
for i in range(len(s2l2a_slc_meanings))),
flag_meanings=' '.join(s2l2a_slc_meanings),
))
mask_set = MaskSet(flag_var)
self.assertEqual(
'SLC(no_data=(None, 0), saturated_or_defective=(None, 1), dark_area_pixels=(None, 2), '
'cloud_shadows=(None, 3), vegetation=(None, 4), bare_soils=(None, 5), water=(None, 6), '
'clouds_low_probability_or_unclassified=(None, 7), clouds_medium_probability=(None, 8), '
'clouds_high_probability=(None, 9), cirrus=(None, 10), snow_or_ice=(None, 11))',
str(mask_set))
validation_data = ((0, 'no_data', mask_set.no_data,
np.array(
[[0, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 0]],
dtype=np.uint8)),
(4, 'vegetation', mask_set.vegetation,
np.array(
[[0, 0, 0, 0], [0, 0, 0, 1], [0, 0, 0, 0]],
dtype=np.uint8)),
(10, 'cirrus', mask_set.cirrus,
np.array(
[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 1]],
dtype=np.uint8)),
(6, 'water', mask_set.water,
np.array(
[[0, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0]],
dtype=np.uint8)))
for index, name, mask, data in validation_data:
msg = f'index={index}, name={name!r}, data={data!r}'
self.assertIs(mask, mask_set[index], msg=msg)
self.assertIs(mask, mask_set[name], msg=msg)
assert_array_almost_equal(mask.values, data, err_msg=msg)
self.assertEqual(set(s2l2a_slc_meanings), set(dir(mask_set)))
html = mask_set._repr_html_()
self.assertTrue(html.startswith('<html>'))
self.assertTrue(html.endswith('</html>'))
def test_mask_set_with_flag_mask_int_array(self):
flag_var = create_c2rcc_flag_var().chunk(dict(x=2, y=2))
mask_set = MaskSet(flag_var)
self.assertEqual(
'c2rcc_flags(F1=(1, None), F2=(2, None), F3=(4, None), F4=(8, None))',
str(mask_set))
expected_chunks = ((2, 1), (2, 2))
self.assertMaskOk(mask_set, 'F1', expected_chunks)
self.assertMaskOk(mask_set, 'F2', expected_chunks)
self.assertMaskOk(mask_set, 'F3', expected_chunks)
self.assertMaskOk(mask_set, 'F4', expected_chunks)
validation_data = ((0, 'F1', mask_set.F1,
np.array(
[[1, 1, 1, 1], [1, 0, 1, 0], [0, 1, 1, 1]],
dtype=np.uint8)),
(1, 'F2', mask_set.F2,
np.array(
[[0, 0, 0, 0], [0, 0, 0, 1], [0, 0, 0, 0]],
dtype=np.uint8)),
(2, 'F3', mask_set.F3,
np.array(
[[0, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0]],
dtype=np.uint8)),
(3, 'F4', mask_set.F4,
np.array(
[[0, 0, 0, 0], [0, 0, 0, 0], [1, 0, 0, 0]],
dtype=np.uint8)))
for index, name, mask, data in validation_data:
self.assertIs(mask, mask_set[index])
self.assertIs(mask, mask_set[name])
assert_array_almost_equal(mask.values, data)
def test_get_mask_sets(self):
dataset = create_highroc_dataset()
mask_sets = MaskSet.get_mask_sets(dataset)
self.assertIsNotNone(mask_sets)
self.assertEqual(len(mask_sets), 1)
self.assertIn('c2rcc_flags', mask_sets)
mask_set = mask_sets['c2rcc_flags']
self.assertIsInstance(mask_set, MaskSet)
def test_mask_set_with_flag_mask_str(self):
flag_var = create_cmems_sst_flag_var()
mask_set = MaskSet(flag_var)
self.assertEqual(
'mask(sea=(1, None), land=(2, None), lake=(4, None), ice=(8, None))',
str(mask_set))
mask_f1 = mask_set.sea
self.assertIs(mask_f1, mask_set.sea)
mask_f2 = mask_set.land
self.assertIs(mask_f2, mask_set.land)
mask_f3 = mask_set.lake
self.assertIs(mask_f3, mask_set.lake)
mask_f4 = mask_set.ice
self.assertIs(mask_f4, mask_set.ice)
validation_data = ((0, 'sea', mask_f1,
np.array(
[[[1, 0, 0, 0], [1, 1, 0, 0], [1, 1, 1, 0]]],
dtype=np.uint8)),
(1, 'land', mask_f2,
np.array(
[[[0, 1, 0, 0], [0, 0, 1, 1], [0, 0, 0, 1]]],
dtype=np.uint8)),
(2, 'lake', mask_f3,
np.array(
[[[0, 0, 1, 1], [0, 0, 0, 0], [0, 0, 0, 0]]],
dtype=np.uint8)),
(3, 'ice', mask_f4,
np.array(
[[[1, 1, 1, 0], [1, 0, 0, 0], [0, 0, 0, 0]]],
dtype=np.uint8)))
for index, name, mask, data in validation_data:
self.assertIs(mask, mask_set[index])
self.assertIs(mask, mask_set[name])
assert_array_almost_equal(mask.values,
data,
err_msg=f'{index}, {name}, {mask.name}')
def test_mask_set_with_flag_mask_int_array(self):
flag_var = create_c2rcc_flag_var()
mask_set = MaskSet(flag_var)
self.assertEqual(
'c2rcc_flags(F1=(1, None), F2=(2, None), F3=(4, None), F4=(8, None))',
str(mask_set))
mask_f1 = mask_set.F1
self.assertIs(mask_f1, mask_set.F1)
mask_f2 = mask_set.F2
self.assertIs(mask_f2, mask_set.F2)
mask_f3 = mask_set.F3
self.assertIs(mask_f3, mask_set.F3)
mask_f4 = mask_set.F4
self.assertIs(mask_f4, mask_set.F4)
validation_data = ((0, 'F1', mask_f1,
np.array(
[[1, 1, 1, 1], [1, 0, 1, 0], [0, 1, 1, 1]],
dtype=np.uint8)),
(1, 'F2', mask_f2,
np.array(
[[0, 0, 0, 0], [0, 0, 0, 1], [0, 0, 0, 0]],
dtype=np.uint8)),
(2, 'F3', mask_f3,
np.array(
[[0, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0]],
dtype=np.uint8)),
(3, 'F4', mask_f4,
np.array(
[[0, 0, 0, 0], [0, 0, 0, 0], [1, 0, 0, 0]],
dtype=np.uint8)))
for index, name, mask, data in validation_data:
self.assertIs(mask, mask_set[index])
self.assertIs(mask, mask_set[name])
assert_array_almost_equal(mask.values, data)
def test_mask_set_with_flag_mask_str(self):
flag_var = create_cmems_sst_flag_var().chunk(dict(lon=2, lat=2))
mask_set = MaskSet(flag_var)
self.assertEqual(
'mask(sea=(1, None), land=(2, None), lake=(4, None), ice=(8, None))',
str(mask_set))
expected_chunks = ((1, ), (2, 1), (2, 2))
self.assertMaskOk(mask_set, 'sea', expected_chunks)
self.assertMaskOk(mask_set, 'land', expected_chunks)
self.assertMaskOk(mask_set, 'lake', expected_chunks)
self.assertMaskOk(mask_set, 'ice', expected_chunks)
validation_data = ((0, 'sea', mask_set.sea,
np.array(
[[[1, 0, 0, 0], [1, 1, 0, 0], [1, 1, 1, 0]]],
dtype=np.uint8)),
(1, 'land', mask_set.land,
np.array(
[[[0, 1, 0, 0], [0, 0, 1, 1], [0, 0, 0, 1]]],
dtype=np.uint8)),
(2, 'lake', mask_set.lake,
np.array(
[[[0, 0, 1, 1], [0, 0, 0, 0], [0, 0, 0, 0]]],
dtype=np.uint8)),
(3, 'ice', mask_set.ice,
np.array(
[[[1, 1, 1, 0], [1, 0, 0, 0], [0, 0, 0, 0]]],
dtype=np.uint8)))
for index, name, mask, data in validation_data:
self.assertIs(mask, mask_set[index])
self.assertIs(mask, mask_set[name])
assert_array_almost_equal(mask.values,
data,
err_msg=f'{index}, {name}, {mask.name}')
else:
data = "2020"
date_x = dates[2]
date_y = dates[3]
IPython.display.GeoJSON(shapely.geometry.box(*aoi).__geo_interface__)
cube_con = CubeConfig(
dataset_name=dataset,
band_names=band_names,
tile_size=[512, 512],
geometry=aoi, # area of interest
spatial_res=spatial_res, #0.00009
time_range=[date_x, date_y],
time_period=time_period) # what is time tolerance
cube = open_cube(cube_con, **hc)
scl = MaskSet(cube.SCL)
#cube = cube.where((scl.clouds_high_probability) == 0) #shows weird errors
cube = cube.where(
(scl.clouds_high_probability + scl.clouds_medium_probability +
scl.clouds_low_probability_or_unclassified + scl.cirrus) == 0)
date = dates[0]
t1 = cube.sel(time=cube.time[0])
B02 = t1.B02
B03 = t1.B03
B04 = t1.B04
B08 = t1.B08
B11 = t1.B11
# Compute a roads mask using band ratios and thresholds
IPython.display.GeoJSON(shapely.geometry.box(*aoi).__geo_interface__)
## Do Calculation for one time stamp
# Get Sentinel-2 L2A data
cube_con = CubeConfig(dataset_name=dataset,
band_names=band_names,
tile_size=[512, 512],
geometry=aoi,
spatial_res=spatial_res,
time_range=[dates[0], dates[1]],
time_period=time_period)
cube = open_cube(cube_con)
scl = MaskSet(cube.SCL)
cube = cube.where((scl.clouds_high_probability) == 0)
#date = dates[-1] # do exmaple calculation for last timestamp of dates
date = dates[0]
#timestamp1 = cube.sel(time = cube.time[-1])
timestamp1 = cube.sel(time=cube.time[0])
timestamp1.B02.plot.imshow(vmin=0, vmax=0.2, figsize=[16, 8])
# Compute a roads mask using band ratios and thresholds
B02 = timestamp1.B02
B03 = timestamp1.B03
B04 = timestamp1.B04
B08 = timestamp1.B08
B11 = timestamp1.B11
def evaluate_dataset(dataset: xr.Dataset,
processed_variables: NameDictPairList = None,
errors: str = 'raise') -> xr.Dataset:
"""
Compute a dataset from another dataset by evaluating expressions provided as variable attributes.
New variables are computed according to the value of an ``expression`` attribute which, if given,
must by a valid Python expression that can reference any other preceding variables by name.
The expression can also reference any flags defined by another variable according the their CF
attributes ``flag_meaning`` and ``flag_values``.
Invalid values may be masked out using the value of an
optional ``valid_pixel_expression`` attribute that forms a boolean Python expression.
The value of the ``_FillValue`` attribute or NaN will be used in the new variable where the
expression returns zero or false.
Other attributes will be stored as variable metadata as-is.
:param dataset: A dataset.
:param processed_variables: Optional list of variables that will be loaded or computed in the order given.
Each variable is either identified by name or by a name to variable attributes mapping.
:param errors: How to deal with errors while evaluating expressions.
May be be one of "raise", "warn", or "ignore".
:return: new dataset with computed variables
"""
if processed_variables:
processed_variables = to_resolved_name_dict_pairs(processed_variables, dataset, keep=True)
else:
var_names = list(dataset.data_vars)
var_names = sorted(var_names, key=functools.partial(_get_var_sort_key, dataset))
processed_variables = [(var_name, None) for var_name in var_names]
# Initialize namespace with some constants and modules
namespace = dict(NaN=np.nan, PI=math.pi, np=np, xr=xr)
# Now add all mask sets and variables
for var_name in dataset.data_vars:
var = dataset[var_name]
if MaskSet.is_flag_var(var):
namespace[var_name] = MaskSet(var)
else:
namespace[var_name] = var
for var_name, var_props in processed_variables:
if var_name in dataset.data_vars:
# Existing variable
var = dataset[var_name]
if var_props:
var_props_temp = var_props
var_props = dict(var.attrs)
var_props.update(var_props_temp)
else:
var_props = dict(var.attrs)
else:
# Computed variable
var = None
if var_props is None:
var_props = dict()
do_load = var_props.get('load', False)
expression = var_props.get('expression')
if expression:
# Compute new variable
computed_array = compute_array_expr(expression,
namespace=namespace,
result_name=f'{var_name!r}',
errors=errors)
if computed_array is not None:
if hasattr(computed_array, 'attrs'):
var = computed_array
var.attrs.update(var_props)
if do_load:
computed_array.load()
namespace[var_name] = computed_array
valid_pixel_expression = var_props.get('valid_pixel_expression')
if valid_pixel_expression:
# Compute new mask for existing variable
if var is None:
raise ValueError(f'undefined variable {var_name!r}')
valid_mask = compute_array_expr(valid_pixel_expression,
namespace=namespace,
result_name=f'valid mask for {var_name!r}',
errors=errors)
if valid_mask is not None:
masked_var = var.where(valid_mask)
if hasattr(masked_var, 'attrs'):
masked_var.attrs.update(var_props)
if do_load:
masked_var.load()
namespace[var_name] = masked_var
computed_dataset = dataset.copy()
for name, value in namespace.items():
if isinstance(value, xr.DataArray):
computed_dataset[name] = value
return computed_dataset
def test_mask_set_with_missing_meanings_attr(self):
flag_var = create_c2rcc_flag_var().chunk(dict(x=2, y=2))
flag_var.attrs.pop('flag_meanings', None)
with self.assertRaises(ValueError):
MaskSet(flag_var)
def evaluate_dataset(dataset: xr.Dataset,
processed_variables: NameDictPairList = None,
errors: str = 'raise') -> xr.Dataset:
"""
Compute new variables or mask existing variables in *dataset*
by the evaluation of Python expressions, that may refer to other
existing or new variables.
Returns a new dataset that contains the old and new variables,
where both may bew now masked.
Expressions may be given by attributes of existing variables in
*dataset* or passed a via the *processed_variables* argument
which is a sequence of variable name / attributes tuples.
Two types of expression attributes are recognized in the attributes:
1. The attribute ``expression`` generates
a new variable computed from its attribute value.
2. The attribute ``valid_pixel_expression`` masks out
invalid variable values.
In both cases the attribuite value must be a string that forms
a valid Python expression that can reference any other preceding
variables by name.
The expression can also reference any flags defined by another
variable according the their CF attributes ``flag_meaning``
and ``flag_values``.
Invalid variable values may be masked out using the value the
``valid_pixel_expression`` attribute whose value should form
a Boolean Python expression. In case, the expression
returns zero or false, the value of the ``_FillValue`` attribute
or NaN will be used in the new variable.
Other attributes will be stored as variable metadata as-is.
:param dataset: A dataset.
:param processed_variables: Optional list of variable
name-attributes pairs that will processed in the given order.
:param errors: How to deal with errors while evaluating expressions.
May be be one of "raise", "warn", or "ignore".
:return: new dataset with computed variables
"""
if processed_variables:
processed_variables = to_resolved_name_dict_pairs(processed_variables,
dataset,
keep=True)
else:
var_names = list(dataset.data_vars)
var_names = sorted(var_names,
key=functools.partial(_get_var_sort_key, dataset))
processed_variables = [(var_name, None) for var_name in var_names]
# Initialize namespace with some constants and modules
namespace = dict(NaN=np.nan, PI=math.pi, np=np, xr=xr)
# Now add all mask sets and variables
for var_name in dataset.data_vars:
var = dataset[var_name]
if MaskSet.is_flag_var(var):
namespace[var_name] = MaskSet(var)
else:
namespace[var_name] = var
for var_name, var_props in processed_variables:
if var_name in dataset.data_vars:
# Existing variable
var = dataset[var_name]
if var_props:
var_props_temp = var_props
var_props = dict(var.attrs)
var_props.update(var_props_temp)
else:
var_props = dict(var.attrs)
else:
# Computed variable
var = None
if var_props is None:
var_props = dict()
do_load = var_props.get('load', False)
expression = var_props.get('expression')
if expression:
# Compute new variable
computed_array = compute_array_expr(expression,
namespace=namespace,
result_name=f'{var_name!r}',
errors=errors)
if computed_array is not None:
if hasattr(computed_array, 'attrs'):
var = computed_array
var.attrs.update(var_props)
if do_load:
computed_array.load()
namespace[var_name] = computed_array
valid_pixel_expression = var_props.get('valid_pixel_expression')
if valid_pixel_expression:
# Compute new mask for existing variable
if var is None:
raise ValueError(f'undefined variable {var_name!r}')
valid_mask = compute_array_expr(
valid_pixel_expression,
namespace=namespace,
result_name=f'valid mask for {var_name!r}',
errors=errors)
if valid_mask is not None:
masked_var = var.where(valid_mask)
if hasattr(masked_var, 'attrs'):
masked_var.attrs.update(var_props)
if do_load:
masked_var.load()
namespace[var_name] = masked_var
computed_dataset = dataset.copy()
for name, value in namespace.items():
if isinstance(value, xr.DataArray):
computed_dataset[name] = value
return computed_dataset