def test_colorbar_labels(): variable = 'cloud_phase_hsrl' obj = arm.read_netcdf(sample_files.EXAMPLE_CLOUDPHASE) obj.clean.clean_arm_state_variables(variable) display = TimeSeriesDisplay(obj, figsize=(12, 8), subplot_shape=(1,)) y_axis_labels = {} flag_colors = ['white', 'green', 'blue', 'red', 'cyan', 'orange', 'yellow', 'black', 'gray'] for value, meaning, color in zip( obj[variable].attrs['flag_values'], obj[variable].attrs['flag_meanings'], flag_colors ): y_axis_labels[value] = {'text': meaning, 'color': color} display.plot(variable, subplot_index=(0,), colorbar_labels=y_axis_labels, cbar_h_adjust=0) display.fig.subplots_adjust(left=0.08, right=0.88, bottom=0.1, top=0.94) return display.fig
def test_geoplot(): sonde_ds = arm.read_netcdf(sample_files.EXAMPLE_SONDE1) try: geodisplay = GeographicPlotDisplay({'sgpsondewnpnC1.b1': sonde_ds}) geodisplay.geoplot('tdry', marker='.', cartopy_feature=[ 'STATES', 'LAND', 'OCEAN', 'COASTLINE', 'BORDERS', 'LAKES', 'RIVERS' ], text={'Ponca City': [-97.0725, 36.7125]}) try: return geodisplay.fig finally: matplotlib.pyplot.close(geodisplay.fig) except Exception: pass sonde_ds.close()
def test_assessment_overplot(): var_name = 'temp_mean' files = sample_files.EXAMPLE_MET1 ds = arm.read_netcdf(files) ds.load() ds.clean.cleanup() ds.qcfilter.set_test(var_name, index=np.arange(100, 300, dtype=int), test_number=2) ds.qcfilter.set_test(var_name, index=np.arange(420, 422, dtype=int), test_number=3) ds.qcfilter.set_test(var_name, index=np.arange(500, 800, dtype=int), test_number=4) ds.qcfilter.set_test(var_name, index=np.arange(900, 901, dtype=int), test_number=4) # Plot data display = TimeSeriesDisplay(ds, subplot_shape=(1, ), figsize=(10, 6)) display.plot(var_name, day_night_background=True, assessment_overplot=True) ds.close() return display.fig
def test_wind_rose(): sonde_ds = arm.read_netcdf(sample_files.EXAMPLE_TWP_SONDE_WILDCARD) WindDisplay = WindRoseDisplay(sonde_ds, figsize=(10, 10)) WindDisplay.plot('deg', 'wspd', spd_bins=np.linspace(0, 20, 10), num_dirs=30, tick_interval=2, cmap='viridis') WindDisplay.set_thetarng(trng=(0., 360.)) WindDisplay.set_rrng((0., 14)) sonde_ds.close() try: return WindDisplay.fig finally: matplotlib.pyplot.close(WindDisplay.fig)
def test_datafilter(): from act.io.armfiles import read_netcdf from act.tests import EXAMPLE_MET1 ds = read_netcdf(EXAMPLE_MET1) ds.clean.cleanup() var_name = 'atmos_pressure' ds_1 = ds.mean() ds.qcfilter.add_less_test(var_name, 99, test_assessment='Bad') ds.qcfilter.datafilter(rm_assessments='Bad') ds_2 = ds.mean() assert np.isclose(ds_1[var_name].values, 98.86, atol=0.01) assert np.isclose(ds_2[var_name].values, 99.15, atol=0.01) ds.close()
def test_qc_flag_block_plot(): obj = arm.read_netcdf(sample_files.EXAMPLE_SURFSPECALB1MLAWER) display = TimeSeriesDisplay(obj, subplot_shape=(2, ), figsize=(8, 2 * 4)) display.plot('surface_albedo_mfr_narrowband_10m', force_line_plot=True, labels=True) display.qc_flag_block_plot('surface_albedo_mfr_narrowband_10m', subplot_index=(1, )) obj.close() del obj try: return display.fig finally: matplotlib.pyplot.close(display.fig)
def arm_read_netcdf(full_directory_filebase, sdate, edate): '''Read a set of maraos files and append them together : param directory: The directory in which to scan for the .nc/.cdf files relative to '/Users/qingn/Desktop/NQ' : param filebase: File specification potentially including wild cards : returns: A list of <xarray.Dataset>''' files = glob.glob(full_directory_filebase) files.sort() files = [ f for f in files if f.split('.')[-3] >= sdate and f.split('.')[-3] <= edate ] # file_dir = str(HOME_DIR + directory_filebase) file_ori = arm.read_netcdf(files) _, index1 = np.unique(file_ori['time'], return_index=True) file_ori = file_ori.isel(time=index1) # file = file_ori.resample(time='10s').mean() file = file_ori.resample(time='1h').nearest() return file
def test_qc_bar_plot(): ds_object = arm.read_netcdf(sample_files.EXAMPLE_MET1) ds_object.clean.cleanup() var_name = 'temp_mean' ds_object.qcfilter.set_test(var_name, index=range(100, 600), test_number=2) # Testing out when the assessment is not listed ds_object.qcfilter.set_test(var_name, index=range(500, 800), test_number=4) ds_object['qc_' + var_name].attrs['flag_assessments'][3] = 'Wonky' display = TimeSeriesDisplay({'sgpmetE13.b1': ds_object}, subplot_shape=(2, ), figsize=(7, 4)) display.plot(var_name, subplot_index=(0, ), assessment_overplot=True) display.day_night_background('sgpmetE13.b1', subplot_index=(0, )) display.qc_flag_block_plot(var_name, subplot_index=(1, )) ds_object.close() return display.fig
def test_plot(): # Process MET data to get simple LCL files = sample_files.EXAMPLE_SONDE_WILDCARD met = arm.read_netcdf(files) met_temp = met.temp_mean met_rh = met.rh_mean met_lcl = (20. + met_temp / 5.) * (100. - met_rh) / 1000. met['met_lcl'] = met_lcl * 1000. met['met_lcl'].attrs['units'] = 'm' met['met_lcl'].attrs['long_name'] = 'LCL Calculated from SGP MET E13' # Plot data # Plot data display = TimeSeriesDisplay(met) display.add_subplots((3, ), figsize=(15, 10)) display.plot('wspd_vec_mean', subplot_index=(0, )) display.plot('temp_mean', subplot_index=(1, )) display.plot('rh_mean', subplot_index=(2, )) return display.fig
def test_assessment_overplot_multi(): var_name1, var_name2 = 'wspd_arith_mean', 'wspd_vec_mean' files = sample_files.EXAMPLE_MET1 ds = arm.read_netcdf(files) ds.load() ds.clean.cleanup() ds.qcfilter.set_test(var_name1, index=np.arange(100, 200, dtype=int), test_number=2) ds.qcfilter.set_test(var_name1, index=np.arange(500, 600, dtype=int), test_number=4) ds.qcfilter.set_test(var_name2, index=np.arange(300, 400, dtype=int), test_number=4) # Plot data display = TimeSeriesDisplay(ds, subplot_shape=(1, ), figsize=(10, 6)) display.plot(var_name1, label=var_name1, assessment_overplot=True, overplot_behind=True) display.plot(var_name2, day_night_background=True, color='green', label=var_name2, assessment_overplot=True) ds.close() return display.fig
def test_contour2(): files = glob.glob(sample_files.EXAMPLE_MET_CONTOUR) time = '2019-05-08T04:00:00.000000000' data = {} fields = {} wind_fields = {} station_fields = {} for f in files: obj = arm.read_netcdf(f) data.update({f: obj}) fields.update({f: ['lon', 'lat', 'temp_mean']}) wind_fields.update({f: ['lon', 'lat', 'wspd_vec_mean', 'wdir_vec_mean']}) station_fields.update({f: ['lon', 'lat', 'atmos_pressure']}) display = ContourDisplay(data, figsize=(8, 8)) display.create_contour(fields=fields, time=time, levels=50, contour='contour', cmap='viridis') display.plot_vectors_from_spd_dir(fields=wind_fields, time=time, mesh=False, grid_delta=(0.1, 0.1)) display.plot_station(fields=station_fields, time=time, markersize=7, color='pink') return display.fig
def test_arm_qc(): # Test DQR Webservice using known DQR variable = 'wspd_vec_mean' qc_variable = 'qc_' + variable obj = read_netcdf(EXAMPLE_METE40) # DQR webservice does go down, so ensure it # properly runs first before testing try: obj = add_dqr_to_qc(obj, variable=variable) ran = True except ValueError: ran = False if ran: assert qc_variable in obj dqr = [ True for d in obj[qc_variable].attrs['flag_meanings'] if 'D190529.4' in d ] assert dqr[0] is True
def test_clean(): ceil_ds = read_netcdf([EXAMPLE_CEIL1]) ceil_ds.clean.cleanup(clean_arm_state_vars=['detection_status']) # Check that global attribures are removed global_attributes = [ 'qc_bit_comment', 'qc_bit_1_description', 'qc_bit_1_assessment', 'qc_bit_2_description', 'qc_bit_2_assessment' 'qc_bit_3_description', 'qc_bit_3_assessment', 'qc_bit_4_description', 'qc_bit_4_assessment' ] for glb_att in global_attributes: assert glb_att not in ceil_ds.attrs.keys() # Check that CF attributes are set including new flag_assessments assert 'flag_masks' in ceil_ds['qc_first_cbh'].attrs.keys() assert 'flag_meanings' in ceil_ds['qc_first_cbh'].attrs.keys() assert 'flag_assessments' in ceil_ds['qc_first_cbh'].attrs.keys() # Check the value of flag_assessments is as expected assert (all([ ii == 'Bad' for ii in ceil_ds['qc_first_cbh'].attrs['flag_assessments'] ])) # Check the type is correct assert (type(ceil_ds['qc_first_cbh'].attrs['flag_masks'])) == list # Check that ancillary varibles is being added assert ('qc_first_cbh' in ceil_ds['first_cbh'].attrs['ancillary_variables'].split()) # Check that state field is updated to CF assert 'flag_values' in ceil_ds['detection_status'].attrs.keys() assert 'flag_meanings' in ceil_ds['detection_status'].attrs.keys() assert 'flag_0_description' not in ceil_ds['detection_status'].attrs.keys() assert ('detection_status' in ceil_ds['first_cbh'].attrs['ancillary_variables'].split()) ceil_ds.close()
def test_qc_data_type(): drop_vars = [ 'base_time', 'time_offset', 'inst_up_long_case_resist', 'inst_up_long_hemisp_tp', 'inst_up_short_hemisp_tp', 'inst_sfc_ir_temp', 'lat', 'lon', 'alt' ] ds_object = read_netcdf(EXAMPLE_IRT25m20s, drop_variables=drop_vars) var_name = 'inst_up_long_dome_resist' expected_qc_var_name = 'qc_' + var_name ds_object.qcfilter.check_for_ancillary_qc(var_name, add_if_missing=True) del ds_object[expected_qc_var_name].attrs['flag_meanings'] del ds_object[expected_qc_var_name].attrs['flag_assessments'] ds_object[expected_qc_var_name] = ds_object[expected_qc_var_name].astype( np.int8) ds_object.qcfilter.add_test(var_name, index=[1], test_number=9, test_meaning='First test') assert ds_object[expected_qc_var_name].attrs['flag_masks'][ 0].dtype == np.uint32 assert ds_object[expected_qc_var_name].dtype == np.int16 ds_object.qcfilter.add_test(var_name, index=[1], test_number=17, test_meaning='Second test') assert ds_object[expected_qc_var_name].dtype == np.int32 ds_object.qcfilter.add_test(var_name, index=[1], test_number=33, test_meaning='Third test') assert ds_object[expected_qc_var_name].dtype == np.int64 assert ds_object[expected_qc_var_name].attrs['flag_masks'][ 0].dtype == np.uint64 ds_object.qcfilter.add_test(var_name, index=[1], test_meaning='Fourth test', recycle=True)
def plot_wacr_reflectivity(i): plt.close("all") ''' No.i files in wacr ''' wacr = arm.read_netcdf(files[i]) #%% Plotting fig = plt.figure(figsize=(12, 3)) ax = plt.gca() plt.contourf(wacr.time.values, wacr.height.values, wacr.cloud_mask_95ghz_kollias.T ) #,label = 'Micro-pulse lidar cloud mask') ec = plt.contourf(wacr.time.values, wacr.height.values, wacr['reflectivity_best_estimate'].T, cmap='jet') #,label = 'Hydrometeor-only 95GHz') plt.plot(wacr.time.values, wacr.cloud_base_best_estimate, 'w.', alpha=0.3, label='cloud_base', markersize=3) fig.autofmt_xdate() plt.xlabel('Date') plt.ylabel('Height(m)') plt.yscale('log') plt.legend() #plt.show() cbar = plt.colorbar(ec) cbar.set_label('reflectivity(dBZ)') #%% fig.tight_layout() fsave = "Full_cloud_mask_" + str(wacr.reflectivity.time[0].values)[:13] fig.savefig(f"{os.path.join(figpath, fsave)}.png", dpi=300, fmt="png")
def test_xsection_plot_map(): radar_ds = arm.read_netcdf(sample_files.EXAMPLE_VISST, combine='nested', concat_dim='time') try: xsection = XSectionDisplay(radar_ds, figsize=(15, 8)) xsection.plot_xsection_map(None, 'ir_temperature', vmin=220, vmax=300, cmap='Greys', x='longitude', y='latitude', isel_kwargs={'time': 0}) radar_ds.close() try: return xsection.fig finally: matplotlib.pyplot.close(xsection.fig) except Exception: pass
def test_qc_flag_description(): """ This will check if the cleanup() method will correctly convert convert flag_#_description to CF flag_masks and flag_meanings. """ ds = read_netcdf(EXAMPLE_CO2FLX4M) ds.clean.cleanup() qc_var_name = ds.qcfilter.check_for_ancillary_qc('momentum_flux', add_if_missing=False, cleanup=False) assert isinstance(ds[qc_var_name].attrs['flag_masks'], list) assert isinstance(ds[qc_var_name].attrs['flag_meanings'], list) assert isinstance(ds[qc_var_name].attrs['flag_assessments'], list) assert ds[qc_var_name].attrs['standard_name'] == 'quality_flag' assert len(ds[qc_var_name].attrs['flag_masks']) == 9 unique_flag_assessments = list(set(['Acceptable', 'Indeterminate', 'Bad'])) assert list(set( ds[qc_var_name].attrs['flag_assessments'])) == unique_flag_assessments
def test_qctests_dos(): ds_object = read_netcdf(EXAMPLE_IRT25m20s) var_name = 'inst_up_long_dome_resist' # persistence test data = ds_object[var_name].values data[1000:2500] = data[1000] ds_object[var_name].values = data ds_object.qcfilter.add_persistence_test(var_name) qc_var_name = ds_object.qcfilter.check_for_ancillary_qc( var_name, add_if_missing=False, cleanup=False, flag_type=False) test_meaning = ('Data failing persistence test. Standard Deviation over a ' 'window of 10 values less than 0.0001.') assert ds_object[qc_var_name].attrs['flag_meanings'][-1] == test_meaning assert np.sum(ds_object[qc_var_name].values) == 1500 ds_object.qcfilter.add_persistence_test(var_name, window=10000, prepend_text='DQO') test_meaning = ( 'DQO: Data failing persistence test. Standard Deviation over a window of ' '4320 values less than 0.0001.') assert ds_object[qc_var_name].attrs['flag_meanings'][-1] == test_meaning
def test_multi_skewt_plot(): files = glob.glob(sample_files.EXAMPLE_TWP_SONDE_20060121) test = {} for f in files: time = f.split('.')[-3] sonde_ds = arm.read_netcdf(f) test.update({time: sonde_ds}) if METPY: skewt = SkewTDisplay(test, subplot_shape=(2, 2)) i = 0 j = 0 for f in files: time = f.split('.')[-3] skewt.plot_from_spd_and_dir('wspd', 'deg', 'pres', 'tdry', 'dp', subplot_index=(j, i), dsname=time, p_levels_to_plot=np.arange(10., 1000., 25)) if j == 1: i += 1 j = 0 elif j == 0: j += 1 return skewt.fig
def test_qctests_dos(): ds_object = read_netcdf(EXAMPLE_IRT25m20s) var_name = 'inst_up_long_dome_resist' # persistence test data = ds_object[var_name].values data[1000:2500] = data[1000] ds_object[var_name].values = data result = ds_object.qcfilter.add_persistence_test(var_name) qc_var_name = result['qc_variable_name'] test_meaning = ('Data failing persistence test. Standard Deviation over a ' 'window of 10 values less than 0.0001.') assert ds_object[qc_var_name].attrs['flag_meanings'][-1] == test_meaning # There is a precision issue with GitHub testing that makes the number of tests # tripped off by 1. This isclose() option is to account for that. assert np.isclose(np.sum(ds_object[qc_var_name].values), 1500, atol=1) ds_object.qcfilter.add_persistence_test(var_name, window=10000, prepend_text='DQO') test_meaning = ( 'DQO: Data failing persistence test. Standard Deviation over a window of ' '4320 values less than 0.0001.') assert ds_object[qc_var_name].attrs['flag_meanings'][-1] == test_meaning
def test_qc_bar_plot(): ds_object = arm.read_netcdf(sample_files.EXAMPLE_MET1) ds_object.clean.cleanup() var_name = 'temp_mean' ds_object.qcfilter.set_test(var_name, index=range(100, 600), test_number=2) # Testing out when the assessment is not listed ds_object.qcfilter.set_test(var_name, index=range(500, 800), test_number=4) ds_object['qc_' + var_name].attrs['flag_assessments'][3] = 'Wonky' display = TimeSeriesDisplay({'sgpmetE13.b1': ds_object}, subplot_shape=(2, ), figsize=(7, 4)) display.plot(var_name, subplot_index=(0, ), assessment_overplot=True) display.day_night_background('sgpmetE13.b1', subplot_index=(0, )) color_lookup = {'Bad': 'red', 'Incorrect': 'red', 'Indeterminate': 'orange', 'Suspect': 'orange', 'Missing': 'darkgray', 'Not Failing': 'green', 'Acceptable': 'green'} display.qc_flag_block_plot(var_name, subplot_index=(1, ), assessment_color=color_lookup) ds_object.close() return display.fig
FIGWIDTH = 6 FIGHEIGHT = 4 FONTSIZE = 18 LABELSIZE = 18 plt.rcParams['figure.figsize'] = (FIGWIDTH, FIGHEIGHT) plt.rcParams['font.size'] = FONTSIZE plt.rcParams['xtick.labelsize'] = FONTSIZE plt.rcParams['ytick.labelsize'] = FONTSIZE matplotlib.rc('xtick', labelsize=LABELSIZE) matplotlib.rc('ytick', labelsize=LABELSIZE) # %% Read in data use arm act function # CPC cpc = arm.read_netcdf('/Users/qingn/Desktop/NQ/maraoscpc/maraoscpcfM1.b1.2018*') _, index1 = np.unique(cpc['time'], return_index = True) cpc = cpc.isel(time = index1) cpc_con = cpc['concentration'] qc_cpc = cpc['qc_concentration'] time_cpc = cpc['time'].values # EXHAUST_ID exhaust_id = netCDF4.Dataset('/Users/qingn/Desktop/NQ/exhaust_id/AAS_4292_ExhaustID_201718_AA_MARCUS.nc') exhaust = exhaust_id['exhaust_4mad02thresh'] time_id = np.array(exhaust_id['time']) time_id_date = pd.to_datetime(time_id, unit ='s', origin = pd.Timestamp('2017-10-18 23:45:06')) # UHSAS path_uhsas = '/Users/qingn/Desktop/NQ/maraosuhsas/maraosuhsasM1.a1.2018*.nc' uhsas = arm.read_netcdf(path_uhsas)
import os from act.io.armfiles import read_netcdf import datetime import matplotlib import sys import matplotlib.dates as mdates #import act import matplotlib.pyplot as plt import mpl_toolkits #import mpl_toolkits.basemap as bm from mpl_toolkits.basemap import Basemap, cm import act import act.io.armfiles as arm import act.plotting.plot as armplot from sklearn.ensemble import RandomForestClassifier matplotlib.rc('xtick', labelsize=26) matplotlib.rc('ytick', labelsize=26) # %% cpc = arm.read_netcdf('/Users/qingn/Desktop/NQ/maraoscpc/maraos*.nc') #exhaust_id = arm.read_netcdf('/Users/qingn/Desktop/NQ/exhaust_id/AAS*.nc') co = arm.read_netcdf('/Users/qingn/Desktop/NQ/maraosco/mar*.nc') exhaust_id = netCDF4.Dataset( '/Users/qingn/Desktop/NQ/exhaust_id/AAS_4292_ExhaustID_201718_AA_MARCUS.nc' ) time_cpc = cpc['time'].values time_id = exhaust_id['time'].values time_co = co['time'].values #%%
def test_qctests(): ds_object = read_netcdf(EXAMPLE_IRT25m20s) var_name = 'inst_up_long_dome_resist' # Add in one missing value and test for that missing value data = ds_object[var_name].values data[0] = np.nan ds_object[var_name].values = data result = ds_object.qcfilter.add_missing_value_test(var_name) data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert data.mask[0] # less than min test limit_value = 6.8 result = ds_object.qcfilter.add_less_test(var_name, limit_value, prepend_text='arm') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 54 assert 'fail_min' in ds_object[result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['fail_min'].dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_min'], limit_value) result = ds_object.qcfilter.add_less_test(var_name, limit_value, test_assessment='Suspect') assert 'warn_min' in ds_object[result['qc_variable_name']].attrs.keys() # greator than max test limit_value = 12.7 result = ds_object.qcfilter.add_greater_test(var_name, limit_value, prepend_text='arm') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 61 assert 'fail_max' in ds_object[result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['fail_max'].dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_max'], limit_value) result = ds_object.qcfilter.add_greater_test(var_name, limit_value, test_assessment='Suspect') assert 'warn_max' in ds_object[result['qc_variable_name']].attrs.keys() # less than or equal test limit_value = 6.9 result = ds_object.qcfilter.add_less_equal_test(var_name, limit_value, test_assessment='Suspect', prepend_text='arm') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 149 assert 'warn_min' in ds_object[result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['warn_min'].dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose(ds_object[result['qc_variable_name']].attrs['warn_min'], limit_value) result = ds_object.qcfilter.add_less_equal_test(var_name, limit_value) assert 'fail_min' in ds_object[result['qc_variable_name']].attrs.keys() # greater than or equal test limit_value = 12 result = ds_object.qcfilter.add_greater_equal_test( var_name, limit_value, test_assessment='Suspect', prepend_text='arm') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 606 assert 'warn_max' in ds_object[result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['warn_max'].dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose(ds_object[result['qc_variable_name']].attrs['warn_max'], limit_value) result = ds_object.qcfilter.add_greater_equal_test(var_name, limit_value) assert 'fail_max' in ds_object[result['qc_variable_name']].attrs.keys() # equal to test limit_value = 7.6705 result = ds_object.qcfilter.add_equal_to_test(var_name, limit_value, prepend_text='arm') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 2 assert 'fail_equal_to' in ds_object[ result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['fail_equal_to'].dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['fail_equal_to'], limit_value) result = ds_object.qcfilter.add_equal_to_test( var_name, limit_value, test_assessment='Indeterminate') assert 'warn_equal_to' in ds_object[ result['qc_variable_name']].attrs.keys() # not equal to test limit_value = 7.6705 result = ds_object.qcfilter.add_not_equal_to_test( var_name, limit_value, test_assessment='Indeterminate', prepend_text='arm') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 4318 assert 'warn_not_equal_to' in ds_object[ result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['warn_not_equal_to']. dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['warn_not_equal_to'], limit_value) result = ds_object.qcfilter.add_not_equal_to_test(var_name, limit_value) assert 'fail_not_equal_to' in ds_object[ result['qc_variable_name']].attrs.keys() # outside range test limit_value1 = 6.8 limit_value2 = 12.7 result = ds_object.qcfilter.add_outside_test(var_name, limit_value1, limit_value2, prepend_text='arm') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 115 assert 'fail_lower_range' in ds_object[ result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['fail_lower_range']. dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['fail_lower_range'], limit_value1) assert 'fail_upper_range' in ds_object[ result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['fail_upper_range']. dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['fail_upper_range'], limit_value2) result = ds_object.qcfilter.add_outside_test( var_name, limit_value1, limit_value2, test_assessment='Indeterminate') assert 'warn_lower_range' in ds_object[ result['qc_variable_name']].attrs.keys() assert 'warn_upper_range' in ds_object[ result['qc_variable_name']].attrs.keys() # inside range test limit_value1 = 7 limit_value2 = 8 result = ds_object.qcfilter.add_inside_test(var_name, limit_value1, limit_value2, prepend_text='arm') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 479 assert 'fail_lower_range_inner' in ds_object[ result['qc_variable_name']].attrs.keys() assert ( ds_object[result['qc_variable_name']].attrs['fail_lower_range_inner']. dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['fail_lower_range_inner'], limit_value1) assert 'fail_upper_range_inner' in ds_object[ result['qc_variable_name']].attrs.keys() assert ( ds_object[result['qc_variable_name']].attrs['fail_upper_range_inner']. dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['fail_upper_range_inner'], limit_value2) result = ds_object.qcfilter.add_inside_test( var_name, limit_value1, limit_value2, test_assessment='Indeterminate') assert 'warn_lower_range_inner' in ds_object[ result['qc_variable_name']].attrs.keys() assert 'warn_upper_range_inner' in ds_object[ result['qc_variable_name']].attrs.keys() # delta test test_limit = 0.05 result = ds_object.qcfilter.add_delta_test(var_name, test_limit, prepend_text='arm') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 175 assert 'warn_delta' in ds_object[result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['warn_delta'].dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['warn_delta'], test_limit) data = ds_object.qcfilter.get_masked_data( var_name, rm_assessments=['Suspect', 'Bad']) assert np.ma.count_masked(data) == 4320 result = ds_object.qcfilter.add_delta_test(var_name, test_limit, test_assessment='Bad') assert 'fail_delta' in ds_object[result['qc_variable_name']].attrs.keys() comp_object = read_netcdf(EXAMPLE_IRT25m20s) result = ds_object.qcfilter.add_difference_test( var_name, {comp_object.attrs['datastream']: comp_object}, var_name, diff_limit=1, prepend_text='arm') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert not (data.mask).all() comp_object.close() ds_object.close()
def test_fft_shading_test(): obj = read_netcdf(EXAMPLE_MFRSR) obj.clean.cleanup() obj = fft_shading_test(obj) qc_data = obj['qc_diffuse_hemisp_narrowband_filter4'] assert np.nansum(qc_data.values) == 456
import os import json import matplotlib import sys import numpy as np import xarray as xr import pandas as pd import datetime from sklearn.ensemble import RandomForestClassifier from sklearn import preprocessing co_path = '/Users/qingn/Desktop/NQ/maraosco/maraoscoM1.b1.2017111[3-4]*' o3_path = '/Users/qingn/Desktop/NQ/maraoso3/maraoso3M1.b1.2017111[3-4]*.custom.nc' cpc_path = '/Users/qingn/Desktop/NQ/maraoscpc/maraoscpcfM1.b1.2017111[3-4]*' co = arm.read_netcdf(co_path) o3 = arm.read_netcdf(o3_path) cpc = arm.read_netcdf(cpc_path) #%% cpc_con = cpc['concentration'] #.resample(time='1min').mean() cpc_con = cpc_con.resample(time='1s').nearest() # obj = arm.read_netcdf(files,variables=var) co = arm.read_netcdf(co_path) o3 = arm.read_netcdf(o3_path) co_con = co['co_dry'].where(co['qc_co_dry'] < 16384) co_con = co_con.resample(time='1s').nearest() co_diff = co_con.diff('time', n=1) o3_con = o3['o3'].where(o3['qc_o3'] < 262144)
import os import json import matplotlib import sys import numpy as np import xarray as xr import pandas as pd import datetime from sklearn.ensemble import RandomForestClassifier from sklearn import preprocessing co_path = '/Users/qingn/Desktop/NQ/maraosco/maraoscoM1.b1.2017111[3-4]*' o3_path = '/Users/qingn/Desktop/NQ/maraoso3/maraoso3M1.b1.2017111[3-4]*.custom.nc' cpc_path = '/Users/qingn/Desktop/NQ/maraoscpc/maraoscpcfM1.b1.2017111[3-4]*' co = arm.read_netcdf(co_path) o3 = arm.read_netcdf(o3_path) cpc = arm.read_netcdf(cpc_path) #%% cpc_con = cpc['concentration'] #.resample(time='1min').mean() cpc_con = cpc_con.resample(time='1s').nearest() # obj = arm.read_netcdf(files,variables=var) co = arm.read_netcdf(co_path) o3 = arm.read_netcdf(o3_path) co_con = co['co_dry'].where(co['qc_co_dry'] < 16384) co_con = co_con.resample(time='1s').nearest() co_diff = co_con.diff('time', n=1) o3_con = o3['o3'].where(o3['qc_o3'] < 262144)
def test_clean(): # Read test data ceil_ds = read_netcdf([EXAMPLE_CEIL1]) # Cleanup QC data ceil_ds.clean.cleanup(clean_arm_state_vars=['detection_status']) # Check that global attribures are removed global_attributes = [ 'qc_bit_comment', 'qc_bit_1_description', 'qc_bit_1_assessment', 'qc_bit_2_description', 'qc_bit_2_assessment' 'qc_bit_3_description', 'qc_bit_3_assessment' ] for glb_att in global_attributes: assert glb_att not in ceil_ds.attrs.keys() # Check that CF attributes are set including new flag_assessments var_name = 'qc_first_cbh' for attr_name in ['flag_masks', 'flag_meanings', 'flag_assessments']: assert attr_name in ceil_ds[var_name].attrs.keys() assert isinstance(ceil_ds[var_name].attrs[attr_name], list) # Check that the flag_mask values are set correctly assert ceil_ds['qc_first_cbh'].attrs['flag_masks'] == [1, 2, 4] # Check that the flag_meanings values are set correctly assert (ceil_ds['qc_first_cbh'].attrs['flag_meanings'] == [ 'Value is equal to missing_value.', 'Value is less than the fail_min.', 'Value is greater than the fail_max.' ]) # Check the value of flag_assessments is as expected assert ceil_ds['qc_first_cbh'].attrs['flag_assessments'] == [ 'Bad', 'Bad', 'Bad' ] # Check that ancillary varibles is being added assert 'qc_first_cbh' in ceil_ds['first_cbh'].attrs[ 'ancillary_variables'].split() # Check that state field is updated to CF assert 'flag_values' in ceil_ds['detection_status'].attrs.keys() assert isinstance(ceil_ds['detection_status'].attrs['flag_values'], list) assert ceil_ds['detection_status'].attrs['flag_values'] == [ 0, 1, 2, 3, 4, 5 ] assert 'flag_meanings' in ceil_ds['detection_status'].attrs.keys() assert isinstance(ceil_ds['detection_status'].attrs['flag_meanings'], list) assert (ceil_ds['detection_status'].attrs['flag_meanings'] == [ 'No significant backscatter', 'One cloud base detected', 'Two cloud bases detected', 'Three cloud bases detected', 'Full obscuration determined but no cloud base detected', 'Some obscuration detected but determined to be transparent' ]) assert 'flag_0_description' not in ceil_ds['detection_status'].attrs.keys() assert ('detection_status' in ceil_ds['first_cbh'].attrs['ancillary_variables'].split()) ceil_ds.close()
def test_qcfilter(): ds_object = read_netcdf(EXAMPLE_IRT25m20s) var_name = 'inst_up_long_dome_resist' expected_qc_var_name = 'qc_' + var_name # Perform adding of quality control variables to object result = ds_object.qcfilter.add_test(var_name, test_meaning='Birds!') assert isinstance(result, dict) qc_var_name = result['qc_variable_name'] assert qc_var_name == expected_qc_var_name # Check that new linking and describing attributes are set assert ds_object[qc_var_name].attrs['standard_name'] == 'quality_flag' assert ds_object[var_name].attrs['ancillary_variables'] == qc_var_name # Check that CF attributes are set including new flag_assessments assert 'flag_masks' in ds_object[qc_var_name].attrs.keys() assert 'flag_meanings' in ds_object[qc_var_name].attrs.keys() assert 'flag_assessments' in ds_object[qc_var_name].attrs.keys() # Check that the values of the attributes are set correctly assert ds_object[qc_var_name].attrs['flag_assessments'][0] == 'Bad' assert ds_object[qc_var_name].attrs['flag_meanings'][0] == 'Birds!' assert ds_object[qc_var_name].attrs['flag_masks'][0] == 1 # Set some test values index = [0, 1, 2, 30] ds_object.qcfilter.set_test(var_name, index=index, test_number=result['test_number']) # Add a new test and set values index2 = [6, 7, 8, 50] ds_object.qcfilter.add_test(var_name, index=index2, test_number=9, test_meaning='testing high number', test_assessment='Suspect') # Retrieve data from object as numpy masked array. Count number of masked # elements and ensure equal to size of index array. data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments='Bad') assert np.ma.count_masked(data) == len(index) data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments='Suspect', return_nan_array=True) assert np.sum(np.isnan(data)) == len(index2) data = ds_object.qcfilter.get_masked_data( var_name, rm_assessments=['Bad', 'Suspect'], ma_fill_value=np.nan) assert np.ma.count_masked(data) == len(index + index2) # Test internal function for returning the index array of where the # tests are set. assert np.sum( ds_object.qcfilter.get_qc_test_mask( var_name, result['test_number'], return_index=True) - np.array(index, dtype=np.int)) == 0 # Unset a test ds_object.qcfilter.unset_test(var_name, index=0, test_number=result['test_number']) # Remove the test ds_object.qcfilter.remove_test(var_name, test_number=result['test_number']) pytest.raises(ValueError, ds_object.qcfilter.add_test, var_name) pytest.raises(ValueError, ds_object.qcfilter.remove_test, var_name) ds_object.close() pytest.raises(ValueError, parse_bit, [1, 2]) pytest.raises(ValueError, parse_bit, -1) assert set_bit(0, 16) == 32768 data = range(0, 4) assert isinstance(set_bit(list(data), 2), list) assert isinstance(set_bit(tuple(data), 2), tuple) assert isinstance(unset_bit(list(data), 2), list) assert isinstance(unset_bit(tuple(data), 2), tuple) # Fill in missing tests ds_object = read_netcdf(EXAMPLE_IRT25m20s) del ds_object[var_name].attrs['long_name'] # Test creating a qc variable ds_object.qcfilter.create_qc_variable(var_name) # Test creating a second qc variable and of flag type ds_object.qcfilter.create_qc_variable(var_name, flag_type=True) result = ds_object.qcfilter.add_test(var_name, index=[1, 2, 3], test_number=9, test_meaning='testing high number', flag_value=True) ds_object.qcfilter.set_test(var_name, index=5, test_number=9, flag_value=True) data = ds_object.qcfilter.get_masked_data(var_name) assert np.isclose(np.sum(data), 42674.766, 0.01) data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments='Bad') assert np.isclose(np.sum(data), 42643.195, 0.01) ds_object.qcfilter.unset_test(var_name, test_number=9, flag_value=True) ds_object.qcfilter.unset_test(var_name, index=1, test_number=9, flag_value=True) assert ds_object.qcfilter.available_bit(result['qc_variable_name']) == 10 assert ds_object.qcfilter.available_bit(result['qc_variable_name'], recycle=True) == 1 ds_object.qcfilter.remove_test(var_name, test_number=9, flag_value=True) ds_object.qcfilter.update_ancillary_variable(var_name) # Test updating ancillary variable if does not exist ds_object.qcfilter.update_ancillary_variable('not_a_variable_name') # Change ancillary_variables attribute to test if add correct qc variable correctly ds_object[var_name].attrs['ancillary_variables'] = 'a_different_name' ds_object.qcfilter.update_ancillary_variable( var_name, qc_var_name=expected_qc_var_name) assert (expected_qc_var_name in ds_object[var_name].attrs['ancillary_variables']) # Test flag QC var_name = 'inst_sfc_ir_temp' qc_var_name = 'qc_' + var_name ds_object.qcfilter.create_qc_variable(var_name, flag_type=True) assert qc_var_name in list(ds_object.data_vars) assert 'flag_values' in ds_object[qc_var_name].attrs.keys() assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys() del ds_object[qc_var_name] qc_var_name = ds_object.qcfilter.check_for_ancillary_qc( var_name, add_if_missing=True, cleanup=False, flag_type=True) assert qc_var_name in list(ds_object.data_vars) assert 'flag_values' in ds_object[qc_var_name].attrs.keys() assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys() del ds_object[qc_var_name] ds_object.qcfilter.add_missing_value_test(var_name, flag_value=True, prepend_text='arm') ds_object.qcfilter.add_test(var_name, index=list(range(0, 20)), test_number=2, test_meaning='Testing flag', flag_value=True, test_assessment='Suspect') assert qc_var_name in list(ds_object.data_vars) assert 'flag_values' in ds_object[qc_var_name].attrs.keys() assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys() assert 'standard_name' in ds_object[qc_var_name].attrs.keys() assert ds_object[qc_var_name].attrs['flag_values'] == [1, 2] assert ds_object[qc_var_name].attrs['flag_assessments'] == [ 'Bad', 'Suspect' ] ds_object.close()
def test_qctests(): ds_object = read_netcdf(EXAMPLE_IRT25m20s) var_name = 'inst_up_long_dome_resist' # Add in one missing value and test for that missing value data = ds_object[var_name].values data[0] = np.nan ds_object[var_name].data = da.from_array(data) result = ds_object.qcfilter.add_missing_value_test(var_name) data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert data.mask[0] result = ds_object.qcfilter.add_missing_value_test(var_name, use_dask=True) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert data == np.array([0]) ds_object.qcfilter.remove_test(var_name, test_number=result['test_number']) # less than min test limit_value = 6.8 result = ds_object.qcfilter.add_less_test(var_name, limit_value, prepend_text='arm', limit_attr_name='fail_min') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 54 assert 'fail_min' in ds_object[result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['fail_min'].dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_min'], limit_value) result = ds_object.qcfilter.add_less_test(var_name, limit_value, test_assessment='Suspect') assert 'warn_min' in ds_object[result['qc_variable_name']].attrs.keys() limit_value = 8 result = ds_object.qcfilter.add_less_test(var_name, limit_value) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 2911939 result = ds_object.qcfilter.add_less_test(var_name, limit_value, use_dask=True) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 2911939 # greator than max test limit_value = 12.7 result = ds_object.qcfilter.add_greater_test(var_name, limit_value, prepend_text='arm', limit_attr_name='fail_max') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 61 assert 'fail_max' in ds_object[result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['fail_max'].dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_max'], limit_value) result = ds_object.qcfilter.add_greater_test(var_name, limit_value, test_assessment='Suspect') assert 'warn_max' in ds_object[result['qc_variable_name']].attrs.keys() result = ds_object.qcfilter.add_greater_test(var_name, limit_value, use_dask=True) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 125458 result = ds_object.qcfilter.add_greater_test(var_name, limit_value) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 125458 # less than or equal test limit_value = 6.9 result = ds_object.qcfilter.add_less_equal_test(var_name, limit_value, test_assessment='Suspect', prepend_text='arm', limit_attr_name='warn_min') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 149 assert 'warn_min' in ds_object[result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['warn_min'].dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose(ds_object[result['qc_variable_name']].attrs['warn_min'], limit_value) result = ds_object.qcfilter.add_less_equal_test(var_name, limit_value) assert 'fail_min' in ds_object[result['qc_variable_name']].attrs.keys() result = ds_object.qcfilter.add_less_equal_test(var_name, limit_value, use_dask=True) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 601581 result = ds_object.qcfilter.add_less_equal_test(var_name, limit_value) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 601581 # greater than or equal test result = ds_object.qcfilter.add_greater_equal_test(var_name, None) limit_value = 12 result = ds_object.qcfilter.add_greater_equal_test( var_name, limit_value, test_assessment='Suspect', prepend_text='arm', limit_attr_name='warn_max') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 606 assert 'warn_max' in ds_object[result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['warn_max'].dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose(ds_object[result['qc_variable_name']].attrs['warn_max'], limit_value) result = ds_object.qcfilter.add_greater_equal_test(var_name, limit_value) assert 'fail_max' in ds_object[result['qc_variable_name']].attrs.keys() result = ds_object.qcfilter.add_greater_equal_test(var_name, limit_value, use_dask=True) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 1189873 result = ds_object.qcfilter.add_greater_equal_test(var_name, limit_value) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 1189873 # equal to test limit_value = 7.6705 result = ds_object.qcfilter.add_equal_to_test( var_name, limit_value, prepend_text='arm', limit_attr_name='fail_equal_to') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 2 assert 'fail_equal_to' in ds_object[ result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['fail_equal_to'].dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['fail_equal_to'], limit_value) result = ds_object.qcfilter.add_equal_to_test( var_name, limit_value, test_assessment='Indeterminate') assert 'warn_equal_to' in ds_object[ result['qc_variable_name']].attrs.keys() result = ds_object.qcfilter.add_equal_to_test(var_name, limit_value, use_dask=True) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 8631 result = ds_object.qcfilter.add_equal_to_test(var_name, limit_value) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 8631 # not equal to test limit_value = 7.6705 result = ds_object.qcfilter.add_not_equal_to_test( var_name, limit_value, test_assessment='Indeterminate', prepend_text='arm', limit_attr_name='warn_not_equal_to') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 4318 assert 'warn_not_equal_to' in ds_object[ result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['warn_not_equal_to']. dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['warn_not_equal_to'], limit_value) result = ds_object.qcfilter.add_not_equal_to_test(var_name, limit_value) assert 'fail_not_equal_to' in ds_object[ result['qc_variable_name']].attrs.keys() result = ds_object.qcfilter.add_not_equal_to_test(var_name, limit_value, use_dask=True) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 9320409 result = ds_object.qcfilter.add_not_equal_to_test(var_name, limit_value) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 9320409 # outside range test limit_value1 = 6.8 limit_value2 = 12.7 result = ds_object.qcfilter.add_outside_test( var_name, limit_value1, limit_value2, prepend_text='arm', limit_attr_names=['fail_lower_range', 'fail_upper_range']) data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 115 assert 'fail_lower_range' in ds_object[ result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['fail_lower_range']. dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['fail_lower_range'], limit_value1) assert 'fail_upper_range' in ds_object[ result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['fail_upper_range']. dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['fail_upper_range'], limit_value2) result = ds_object.qcfilter.add_outside_test( var_name, limit_value1, limit_value2, test_assessment='Indeterminate') assert 'warn_lower_range' in ds_object[ result['qc_variable_name']].attrs.keys() assert 'warn_upper_range' in ds_object[ result['qc_variable_name']].attrs.keys() result = ds_object.qcfilter.add_outside_test(var_name, limit_value1, limit_value2, use_dask=True) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 342254 result = ds_object.qcfilter.add_outside_test( var_name, limit_value1, limit_value2, ) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 342254 # Starting to run out of space for tests. Remove some tests. for ii in range(16, 30): ds_object.qcfilter.remove_test(var_name, test_number=ii) # inside range test limit_value1 = 7 limit_value2 = 8 result = ds_object.qcfilter.add_inside_test( var_name, limit_value1, limit_value2, prepend_text='arm', limit_attr_names=['fail_lower_range_inner', 'fail_upper_range_inner']) data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 479 assert 'fail_lower_range_inner' in ds_object[ result['qc_variable_name']].attrs.keys() assert ( ds_object[result['qc_variable_name']].attrs['fail_lower_range_inner']. dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['fail_lower_range_inner'], limit_value1) assert 'fail_upper_range_inner' in ds_object[ result['qc_variable_name']].attrs.keys() assert ( ds_object[result['qc_variable_name']].attrs['fail_upper_range_inner']. dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['fail_upper_range_inner'], limit_value2) result = ds_object.qcfilter.add_inside_test( var_name, limit_value1, limit_value2, test_assessment='Indeterminate') assert 'warn_lower_range_inner' in ds_object[ result['qc_variable_name']].attrs.keys() assert 'warn_upper_range_inner' in ds_object[ result['qc_variable_name']].attrs.keys() result = ds_object.qcfilter.add_inside_test(var_name, limit_value1, limit_value2, use_dask=True) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 1820693 result = ds_object.qcfilter.add_inside_test( var_name, limit_value1, limit_value2, ) data = ds_object.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 1820693 # delta test test_limit = 0.05 result = ds_object.qcfilter.add_delta_test(var_name, test_limit, prepend_text='arm', limit_attr_name='warn_delta') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert np.ma.count_masked(data) == 175 assert 'warn_delta' in ds_object[result['qc_variable_name']].attrs.keys() assert (ds_object[result['qc_variable_name']].attrs['warn_delta'].dtype == ds_object[result['variable_name']].values.dtype) assert np.isclose( ds_object[result['qc_variable_name']].attrs['warn_delta'], test_limit) data = ds_object.qcfilter.get_masked_data( var_name, rm_assessments=['Suspect', 'Bad']) assert np.ma.count_masked(data) == 1355 result = ds_object.qcfilter.add_delta_test(var_name, test_limit, test_assessment='Bad') assert 'fail_delta' in ds_object[result['qc_variable_name']].attrs.keys() comp_object = read_netcdf(EXAMPLE_IRT25m20s) with np.testing.assert_raises(ValueError): result = ds_object.qcfilter.add_difference_test(var_name, 'test') with np.testing.assert_raises(ValueError): result = ds_object.qcfilter.add_difference_test( var_name, {comp_object.attrs['datastream']: comp_object}, var_name, diff_limit=None) assert ds_object.qcfilter.add_difference_test( var_name, set_test_regardless=False) is None result = ds_object.qcfilter.add_difference_test( var_name, {comp_object.attrs['datastream']: comp_object}, var_name, diff_limit=1, prepend_text='arm') data = ds_object.qcfilter.get_masked_data(var_name, rm_tests=result['test_number']) assert 'arm' in result['test_meaning'] assert not (data.mask).all() comp_object.close() ds_object.close()