print('\nData type =', type(data)) print('data =', data) # We can see how the quality control data is stored and what assessments, # or test descriptions are set. Some of the tests have also added attributes to # store the test limit values. qc_varialbe = ds_object[result['qc_variable_name']] print('\nQC Variable =', qc_varialbe) # The test numbers are not the flag_masks numbers. The flag masks numbers # are bit-paked numbers used to store what bit is set. To see the test # numbers we can unpack the bits. print('\nmask : test') print('-' * 11) for mask in qc_varialbe.attrs['flag_masks']: print(mask, ' : ', parse_bit(mask)) # We can also just use the get_masked_data() method to get data # the same as using ".values" method on the xarray dataset. If we don't # request any tests or assessments to mask the returned masked array # will not have any mask set. The returned value is a numpy masked array # where the raw numpy array is accessable with .data property. data = ds_object.qcfilter.get_masked_data(var_name) print('\nNormal numpy array data values:', data.data) print('Mask associated with values:', data.mask) # We can use the get_masked_data() method to return a masked array # where the test is set in the quality control varialbe, and use the # masked array method to see if any of the values have the test set. data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=3) print('\nAt least one less than test set =', data.mask.any())
def test_qcfilter(): ds_object = read_netcdf(EXAMPLE_IRT25m20s) var_name = 'inst_up_long_dome_resist' expected_qc_var_name = 'qc_' + var_name ds_object.qcfilter.check_for_ancillary_qc(var_name, add_if_missing=True, cleanup=False, flag_type=False) assert expected_qc_var_name in list(ds_object.keys()) del ds_object[expected_qc_var_name] # Perform adding of quality control variables to object result = ds_object.qcfilter.add_test(var_name, test_meaning='Birds!') assert isinstance(result, dict) qc_var_name = result['qc_variable_name'] assert qc_var_name == expected_qc_var_name # Check that new linking and describing attributes are set assert ds_object[qc_var_name].attrs['standard_name'] == 'quality_flag' assert ds_object[var_name].attrs['ancillary_variables'] == qc_var_name # Check that CF attributes are set including new flag_assessments assert 'flag_masks' in ds_object[qc_var_name].attrs.keys() assert 'flag_meanings' in ds_object[qc_var_name].attrs.keys() assert 'flag_assessments' in ds_object[qc_var_name].attrs.keys() # Check that the values of the attributes are set correctly assert ds_object[qc_var_name].attrs['flag_assessments'][0] == 'Bad' assert ds_object[qc_var_name].attrs['flag_meanings'][0] == 'Birds!' assert ds_object[qc_var_name].attrs['flag_masks'][0] == 1 # Set some test values index = [0, 1, 2, 30] ds_object.qcfilter.set_test(var_name, index=index, test_number=result['test_number']) # Add a new test and set values index2 = [6, 7, 8, 50] ds_object.qcfilter.add_test(var_name, index=index2, test_number=9, test_meaning='testing high number', test_assessment='Suspect') # Retrieve data from object as numpy masked array. Count number of masked # elements and ensure equal to size of index array. data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments='Bad') assert np.ma.count_masked(data) == len(index) data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments='Suspect', return_nan_array=True) assert np.sum(np.isnan(data)) == len(index2) data = ds_object.qcfilter.get_masked_data( var_name, rm_assessments=['Bad', 'Suspect'], ma_fill_value=np.nan) assert np.ma.count_masked(data) == len(index + index2) # Test internal function for returning the index array of where the # tests are set. assert np.sum( ds_object.qcfilter.get_qc_test_mask( var_name, result['test_number'], return_index=True) - np.array(index, dtype=int)) == 0 # Unset a test ds_object.qcfilter.unset_test(var_name, index=0, test_number=result['test_number']) # Remove the test ds_object.qcfilter.remove_test(var_name, test_number=33) ds_object.qcfilter.remove_test(var_name, test_number=result['test_number']) pytest.raises(ValueError, ds_object.qcfilter.add_test, var_name) pytest.raises(ValueError, ds_object.qcfilter.remove_test, var_name) ds_object.close() assert np.all(parse_bit([257]) == np.array([1, 9], dtype=np.int32)) pytest.raises(ValueError, parse_bit, [1, 2]) pytest.raises(ValueError, parse_bit, -1) assert set_bit(0, 16) == 32768 data = range(0, 4) assert isinstance(set_bit(list(data), 2), list) assert isinstance(set_bit(tuple(data), 2), tuple) assert isinstance(unset_bit(list(data), 2), list) assert isinstance(unset_bit(tuple(data), 2), tuple) # Fill in missing tests ds_object = read_netcdf(EXAMPLE_IRT25m20s) del ds_object[var_name].attrs['long_name'] # Test creating a qc variable ds_object.qcfilter.create_qc_variable(var_name) # Test creating a second qc variable and of flag type ds_object.qcfilter.create_qc_variable(var_name, flag_type=True) result = ds_object.qcfilter.add_test(var_name, index=[1, 2, 3], test_number=9, test_meaning='testing high number', flag_value=True) ds_object.qcfilter.set_test(var_name, index=5, test_number=9, flag_value=True) data = ds_object.qcfilter.get_masked_data(var_name) assert np.isclose(np.sum(data), 42674.766, 0.01) data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments='Bad') assert np.isclose(np.sum(data), 42643.195, 0.01) ds_object.qcfilter.unset_test(var_name, test_number=9, flag_value=True) ds_object.qcfilter.unset_test(var_name, index=1, test_number=9, flag_value=True) assert ds_object.qcfilter.available_bit(result['qc_variable_name']) == 10 assert ds_object.qcfilter.available_bit(result['qc_variable_name'], recycle=True) == 1 ds_object.qcfilter.remove_test(var_name, test_number=9, flag_value=True) ds_object.qcfilter.update_ancillary_variable(var_name) # Test updating ancillary variable if does not exist ds_object.qcfilter.update_ancillary_variable('not_a_variable_name') # Change ancillary_variables attribute to test if add correct qc variable correctly ds_object[var_name].attrs['ancillary_variables'] = 'a_different_name' ds_object.qcfilter.update_ancillary_variable( var_name, qc_var_name=expected_qc_var_name) assert (expected_qc_var_name in ds_object[var_name].attrs['ancillary_variables']) # Test flag QC var_name = 'inst_sfc_ir_temp' qc_var_name = 'qc_' + var_name ds_object.qcfilter.create_qc_variable(var_name, flag_type=True) assert qc_var_name in list(ds_object.data_vars) assert 'flag_values' in ds_object[qc_var_name].attrs.keys() assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys() del ds_object[qc_var_name] qc_var_name = ds_object.qcfilter.check_for_ancillary_qc( var_name, add_if_missing=True, cleanup=False, flag_type=True) assert qc_var_name in list(ds_object.data_vars) assert 'flag_values' in ds_object[qc_var_name].attrs.keys() assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys() del ds_object[qc_var_name] ds_object.qcfilter.add_missing_value_test(var_name, flag_value=True, prepend_text='arm') ds_object.qcfilter.add_test(var_name, index=list(range(0, 20)), test_number=2, test_meaning='Testing flag', flag_value=True, test_assessment='Suspect') assert qc_var_name in list(ds_object.data_vars) assert 'flag_values' in ds_object[qc_var_name].attrs.keys() assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys() assert 'standard_name' in ds_object[qc_var_name].attrs.keys() assert ds_object[qc_var_name].attrs['flag_values'] == [1, 2] assert ds_object[qc_var_name].attrs['flag_assessments'] == [ 'Bad', 'Suspect' ] ds_object.close()
def clean_arm_qc(self, override_cf_flag=True, clean_units_string=True, correct_valid_min_max=True, remove_unset_global_tests=True, **kwargs): """ Method to clean up xarray object QC variables. Parameters ---------- override_cf_flag : bool Option to overwrite CF flag_masks, flag_meanings, flag_values if exists. clean_units_string : bool Option to clean up units string from 'unitless' to udunits compliant '1'. correct_valid_min_max : bool Option to correct use of valid_min and valid_max with QC variables by moving from data variable to QC varible, renaming to fail_min, fail_max and fail_detla if the valid_min, valid_max or valid_delta is listed in bit discription attribute. If not listed as used with QC will assume is being used correctly. remove_unset_global_tests : bool Option to look for globaly defined tests that are not set at the variable level and remove from quality control variable. """ global_qc = self.get_attr_info() for qc_var in self.matched_qc_variables: # Clean up units attribute from unitless to udunits '1' try: if clean_units_string and self._obj[qc_var].attrs[ 'units'] == 'unitless': self._obj[qc_var].attrs['units'] = '1' except KeyError: pass qc_attributes = self.get_attr_info(variable=qc_var) if qc_attributes is None: qc_attributes = global_qc # Add new attributes to variable for attr in [ 'flag_masks', 'flag_meanings', 'flag_assessments', 'flag_values', 'flag_comments', ]: if qc_attributes is not None and len(qc_attributes[attr]) > 0: # Only add if attribute does not exists if attr in self._obj[qc_var].attrs.keys() is False: self._obj[qc_var].attrs[attr] = copy.copy( qc_attributes[attr]) # If flag is set add attribure even if already exists elif override_cf_flag: self._obj[qc_var].attrs[attr] = copy.copy( qc_attributes[attr]) # Remove replaced attributes if qc_attributes is not None: arm_attributes = qc_attributes['arm_attributes'] if 'description' not in arm_attributes: arm_attributes.append('description') if 'flag_method' not in arm_attributes: arm_attributes.append('flag_method') for attr in arm_attributes: try: del self._obj[qc_var].attrs[attr] except KeyError: pass # Check for use of valid_min and valid_max as QC limits and fix if correct_valid_min_max: self._obj.clean.correct_valid_minmax(qc_var) # Clean up global attributes if global_qc is not None: global_attributes = global_qc['arm_attributes'] global_attributes.extend(['qc_bit_comment']) for attr in global_attributes: try: del self._obj.attrs[attr] except KeyError: pass # If requested remove tests at variable level that were set from global level descriptions. # This is assuming the test was only performed if the limit value is listed with the variable # even if the global level describes the test. if remove_unset_global_tests and global_qc is not None: limit_name_list = ['fail_min', 'fail_max', 'fail_delta'] for qc_var_name in self.matched_qc_variables: flag_meanings = self._obj[qc_var_name].attrs['flag_meanings'] flag_masks = self._obj[qc_var_name].attrs['flag_masks'] tests_to_remove = [] for ii, flag_meaning in enumerate(flag_meanings): # Loop over usual test attribute names looking to see if they # are listed in test description. If so use that name for look up. test_attribute_limit_name = None for name in limit_name_list: if name in flag_meaning: test_attribute_limit_name = name break if test_attribute_limit_name is None: continue remove_test = True test_number = int(parse_bit(flag_masks[ii])) for attr_name in self._obj[qc_var_name].attrs: if test_attribute_limit_name == attr_name: remove_test = False break index = self._obj.qcfilter.get_qc_test_mask( qc_var_name=qc_var_name, test_number=test_number) if np.any(index): remove_test = False break if remove_test: tests_to_remove.append(test_number) if len(tests_to_remove) > 0: for test_to_remove in tests_to_remove: self._obj.qcfilter.remove_test( qc_var_name=qc_var_name, test_number=test_to_remove)