print('\nData type =', type(data))
print('data =', data)
# We can see how the quality control data is stored and what assessments,
# or test descriptions are set. Some of the tests have also added attributes to
# store the test limit values.
qc_varialbe = ds_object[result['qc_variable_name']]
print('\nQC Variable =', qc_varialbe)
# The test numbers are not the flag_masks numbers. The flag masks numbers
# are bit-paked numbers used to store what bit is set. To see the test
# numbers we can unpack the bits.
print('\nmask : test')
print('-' * 11)
for mask in qc_varialbe.attrs['flag_masks']:
print(mask, ' : ', parse_bit(mask))
# We can also just use the get_masked_data() method to get data
# the same as using ".values" method on the xarray dataset. If we don't
# request any tests or assessments to mask the returned masked array
# will not have any mask set. The returned value is a numpy masked array
# where the raw numpy array is accessable with .data property.
data = ds_object.qcfilter.get_masked_data(var_name)
print('\nNormal numpy array data values:', data.data)
print('Mask associated with values:', data.mask)
# We can use the get_masked_data() method to return a masked array
# where the test is set in the quality control varialbe, and use the
# masked array method to see if any of the values have the test set.
data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=3)
print('\nAt least one less than test set =', data.mask.any())
def test_qcfilter():
ds_object = read_netcdf(EXAMPLE_IRT25m20s)
var_name = 'inst_up_long_dome_resist'
expected_qc_var_name = 'qc_' + var_name
ds_object.qcfilter.check_for_ancillary_qc(var_name,
add_if_missing=True,
cleanup=False,
flag_type=False)
assert expected_qc_var_name in list(ds_object.keys())
del ds_object[expected_qc_var_name]
# Perform adding of quality control variables to object
result = ds_object.qcfilter.add_test(var_name, test_meaning='Birds!')
assert isinstance(result, dict)
qc_var_name = result['qc_variable_name']
assert qc_var_name == expected_qc_var_name
# Check that new linking and describing attributes are set
assert ds_object[qc_var_name].attrs['standard_name'] == 'quality_flag'
assert ds_object[var_name].attrs['ancillary_variables'] == qc_var_name
# Check that CF attributes are set including new flag_assessments
assert 'flag_masks' in ds_object[qc_var_name].attrs.keys()
assert 'flag_meanings' in ds_object[qc_var_name].attrs.keys()
assert 'flag_assessments' in ds_object[qc_var_name].attrs.keys()
# Check that the values of the attributes are set correctly
assert ds_object[qc_var_name].attrs['flag_assessments'][0] == 'Bad'
assert ds_object[qc_var_name].attrs['flag_meanings'][0] == 'Birds!'
assert ds_object[qc_var_name].attrs['flag_masks'][0] == 1
# Set some test values
index = [0, 1, 2, 30]
ds_object.qcfilter.set_test(var_name,
index=index,
test_number=result['test_number'])
# Add a new test and set values
index2 = [6, 7, 8, 50]
ds_object.qcfilter.add_test(var_name,
index=index2,
test_number=9,
test_meaning='testing high number',
test_assessment='Suspect')
# Retrieve data from object as numpy masked array. Count number of masked
# elements and ensure equal to size of index array.
data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments='Bad')
assert np.ma.count_masked(data) == len(index)
data = ds_object.qcfilter.get_masked_data(var_name,
rm_assessments='Suspect',
return_nan_array=True)
assert np.sum(np.isnan(data)) == len(index2)
data = ds_object.qcfilter.get_masked_data(
var_name, rm_assessments=['Bad', 'Suspect'], ma_fill_value=np.nan)
assert np.ma.count_masked(data) == len(index + index2)
# Test internal function for returning the index array of where the
# tests are set.
assert np.sum(
ds_object.qcfilter.get_qc_test_mask(
var_name, result['test_number'], return_index=True) -
np.array(index, dtype=int)) == 0
# Unset a test
ds_object.qcfilter.unset_test(var_name,
index=0,
test_number=result['test_number'])
# Remove the test
ds_object.qcfilter.remove_test(var_name, test_number=33)
ds_object.qcfilter.remove_test(var_name, test_number=result['test_number'])
pytest.raises(ValueError, ds_object.qcfilter.add_test, var_name)
pytest.raises(ValueError, ds_object.qcfilter.remove_test, var_name)
ds_object.close()
assert np.all(parse_bit([257]) == np.array([1, 9], dtype=np.int32))
pytest.raises(ValueError, parse_bit, [1, 2])
pytest.raises(ValueError, parse_bit, -1)
assert set_bit(0, 16) == 32768
data = range(0, 4)
assert isinstance(set_bit(list(data), 2), list)
assert isinstance(set_bit(tuple(data), 2), tuple)
assert isinstance(unset_bit(list(data), 2), list)
assert isinstance(unset_bit(tuple(data), 2), tuple)
# Fill in missing tests
ds_object = read_netcdf(EXAMPLE_IRT25m20s)
del ds_object[var_name].attrs['long_name']
# Test creating a qc variable
ds_object.qcfilter.create_qc_variable(var_name)
# Test creating a second qc variable and of flag type
ds_object.qcfilter.create_qc_variable(var_name, flag_type=True)
result = ds_object.qcfilter.add_test(var_name,
index=[1, 2, 3],
test_number=9,
test_meaning='testing high number',
flag_value=True)
ds_object.qcfilter.set_test(var_name,
index=5,
test_number=9,
flag_value=True)
data = ds_object.qcfilter.get_masked_data(var_name)
assert np.isclose(np.sum(data), 42674.766, 0.01)
data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments='Bad')
assert np.isclose(np.sum(data), 42643.195, 0.01)
ds_object.qcfilter.unset_test(var_name, test_number=9, flag_value=True)
ds_object.qcfilter.unset_test(var_name,
index=1,
test_number=9,
flag_value=True)
assert ds_object.qcfilter.available_bit(result['qc_variable_name']) == 10
assert ds_object.qcfilter.available_bit(result['qc_variable_name'],
recycle=True) == 1
ds_object.qcfilter.remove_test(var_name, test_number=9, flag_value=True)
ds_object.qcfilter.update_ancillary_variable(var_name)
# Test updating ancillary variable if does not exist
ds_object.qcfilter.update_ancillary_variable('not_a_variable_name')
# Change ancillary_variables attribute to test if add correct qc variable correctly
ds_object[var_name].attrs['ancillary_variables'] = 'a_different_name'
ds_object.qcfilter.update_ancillary_variable(
var_name, qc_var_name=expected_qc_var_name)
assert (expected_qc_var_name
in ds_object[var_name].attrs['ancillary_variables'])
# Test flag QC
var_name = 'inst_sfc_ir_temp'
qc_var_name = 'qc_' + var_name
ds_object.qcfilter.create_qc_variable(var_name, flag_type=True)
assert qc_var_name in list(ds_object.data_vars)
assert 'flag_values' in ds_object[qc_var_name].attrs.keys()
assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys()
del ds_object[qc_var_name]
qc_var_name = ds_object.qcfilter.check_for_ancillary_qc(
var_name, add_if_missing=True, cleanup=False, flag_type=True)
assert qc_var_name in list(ds_object.data_vars)
assert 'flag_values' in ds_object[qc_var_name].attrs.keys()
assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys()
del ds_object[qc_var_name]
ds_object.qcfilter.add_missing_value_test(var_name,
flag_value=True,
prepend_text='arm')
ds_object.qcfilter.add_test(var_name,
index=list(range(0, 20)),
test_number=2,
test_meaning='Testing flag',
flag_value=True,
test_assessment='Suspect')
assert qc_var_name in list(ds_object.data_vars)
assert 'flag_values' in ds_object[qc_var_name].attrs.keys()
assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys()
assert 'standard_name' in ds_object[qc_var_name].attrs.keys()
assert ds_object[qc_var_name].attrs['flag_values'] == [1, 2]
assert ds_object[qc_var_name].attrs['flag_assessments'] == [
'Bad', 'Suspect'
]
ds_object.close()
def clean_arm_qc(self,
override_cf_flag=True,
clean_units_string=True,
correct_valid_min_max=True,
remove_unset_global_tests=True,
**kwargs):
"""
Method to clean up xarray object QC variables.
Parameters
----------
override_cf_flag : bool
Option to overwrite CF flag_masks, flag_meanings, flag_values
if exists.
clean_units_string : bool
Option to clean up units string from 'unitless'
to udunits compliant '1'.
correct_valid_min_max : bool
Option to correct use of valid_min and valid_max with QC variables
by moving from data variable to QC varible, renaming to fail_min,
fail_max and fail_detla if the valid_min, valid_max or valid_delta
is listed in bit discription attribute. If not listed as
used with QC will assume is being used correctly.
remove_unset_global_tests : bool
Option to look for globaly defined tests that are not set at the
variable level and remove from quality control variable.
"""
global_qc = self.get_attr_info()
for qc_var in self.matched_qc_variables:
# Clean up units attribute from unitless to udunits '1'
try:
if clean_units_string and self._obj[qc_var].attrs[
'units'] == 'unitless':
self._obj[qc_var].attrs['units'] = '1'
except KeyError:
pass
qc_attributes = self.get_attr_info(variable=qc_var)
if qc_attributes is None:
qc_attributes = global_qc
# Add new attributes to variable
for attr in [
'flag_masks',
'flag_meanings',
'flag_assessments',
'flag_values',
'flag_comments',
]:
if qc_attributes is not None and len(qc_attributes[attr]) > 0:
# Only add if attribute does not exists
if attr in self._obj[qc_var].attrs.keys() is False:
self._obj[qc_var].attrs[attr] = copy.copy(
qc_attributes[attr])
# If flag is set add attribure even if already exists
elif override_cf_flag:
self._obj[qc_var].attrs[attr] = copy.copy(
qc_attributes[attr])
# Remove replaced attributes
if qc_attributes is not None:
arm_attributes = qc_attributes['arm_attributes']
if 'description' not in arm_attributes:
arm_attributes.append('description')
if 'flag_method' not in arm_attributes:
arm_attributes.append('flag_method')
for attr in arm_attributes:
try:
del self._obj[qc_var].attrs[attr]
except KeyError:
pass
# Check for use of valid_min and valid_max as QC limits and fix
if correct_valid_min_max:
self._obj.clean.correct_valid_minmax(qc_var)
# Clean up global attributes
if global_qc is not None:
global_attributes = global_qc['arm_attributes']
global_attributes.extend(['qc_bit_comment'])
for attr in global_attributes:
try:
del self._obj.attrs[attr]
except KeyError:
pass
# If requested remove tests at variable level that were set from global level descriptions.
# This is assuming the test was only performed if the limit value is listed with the variable
# even if the global level describes the test.
if remove_unset_global_tests and global_qc is not None:
limit_name_list = ['fail_min', 'fail_max', 'fail_delta']
for qc_var_name in self.matched_qc_variables:
flag_meanings = self._obj[qc_var_name].attrs['flag_meanings']
flag_masks = self._obj[qc_var_name].attrs['flag_masks']
tests_to_remove = []
for ii, flag_meaning in enumerate(flag_meanings):
# Loop over usual test attribute names looking to see if they
# are listed in test description. If so use that name for look up.
test_attribute_limit_name = None
for name in limit_name_list:
if name in flag_meaning:
test_attribute_limit_name = name
break
if test_attribute_limit_name is None:
continue
remove_test = True
test_number = int(parse_bit(flag_masks[ii]))
for attr_name in self._obj[qc_var_name].attrs:
if test_attribute_limit_name == attr_name:
remove_test = False
break
index = self._obj.qcfilter.get_qc_test_mask(
qc_var_name=qc_var_name, test_number=test_number)
if np.any(index):
remove_test = False
break
if remove_test:
tests_to_remove.append(test_number)
if len(tests_to_remove) > 0:
for test_to_remove in tests_to_remove:
self._obj.qcfilter.remove_test(
qc_var_name=qc_var_name,
test_number=test_to_remove)