def test_colorbar_labels():
variable = 'cloud_phase_hsrl'
obj = arm.read_netcdf(sample_files.EXAMPLE_CLOUDPHASE)
obj.clean.clean_arm_state_variables(variable)
display = TimeSeriesDisplay(obj, figsize=(12, 8), subplot_shape=(1,))
y_axis_labels = {}
flag_colors = ['white', 'green', 'blue', 'red', 'cyan', 'orange', 'yellow', 'black', 'gray']
for value, meaning, color in zip(
obj[variable].attrs['flag_values'], obj[variable].attrs['flag_meanings'], flag_colors
):
y_axis_labels[value] = {'text': meaning, 'color': color}
display.plot(variable, subplot_index=(0,), colorbar_labels=y_axis_labels, cbar_h_adjust=0)
display.fig.subplots_adjust(left=0.08, right=0.88, bottom=0.1, top=0.94)
return display.fig
def test_geoplot():
sonde_ds = arm.read_netcdf(sample_files.EXAMPLE_SONDE1)
try:
geodisplay = GeographicPlotDisplay({'sgpsondewnpnC1.b1': sonde_ds})
geodisplay.geoplot('tdry',
marker='.',
cartopy_feature=[
'STATES', 'LAND', 'OCEAN', 'COASTLINE',
'BORDERS', 'LAKES', 'RIVERS'
],
text={'Ponca City': [-97.0725, 36.7125]})
try:
return geodisplay.fig
finally:
matplotlib.pyplot.close(geodisplay.fig)
except Exception:
pass
sonde_ds.close()
def test_assessment_overplot():
var_name = 'temp_mean'
files = sample_files.EXAMPLE_MET1
ds = arm.read_netcdf(files)
ds.load()
ds.clean.cleanup()
ds.qcfilter.set_test(var_name, index=np.arange(100, 300, dtype=int), test_number=2)
ds.qcfilter.set_test(var_name, index=np.arange(420, 422, dtype=int), test_number=3)
ds.qcfilter.set_test(var_name, index=np.arange(500, 800, dtype=int), test_number=4)
ds.qcfilter.set_test(var_name, index=np.arange(900, 901, dtype=int), test_number=4)
# Plot data
display = TimeSeriesDisplay(ds, subplot_shape=(1, ), figsize=(10, 6))
display.plot(var_name, day_night_background=True, assessment_overplot=True)
ds.close()
return display.fig
def test_wind_rose():
sonde_ds = arm.read_netcdf(sample_files.EXAMPLE_TWP_SONDE_WILDCARD)
WindDisplay = WindRoseDisplay(sonde_ds, figsize=(10, 10))
WindDisplay.plot('deg',
'wspd',
spd_bins=np.linspace(0, 20, 10),
num_dirs=30,
tick_interval=2,
cmap='viridis')
WindDisplay.set_thetarng(trng=(0., 360.))
WindDisplay.set_rrng((0., 14))
sonde_ds.close()
try:
return WindDisplay.fig
finally:
matplotlib.pyplot.close(WindDisplay.fig)
def test_datafilter():
from act.io.armfiles import read_netcdf
from act.tests import EXAMPLE_MET1
ds = read_netcdf(EXAMPLE_MET1)
ds.clean.cleanup()
var_name = 'atmos_pressure'
ds_1 = ds.mean()
ds.qcfilter.add_less_test(var_name, 99, test_assessment='Bad')
ds.qcfilter.datafilter(rm_assessments='Bad')
ds_2 = ds.mean()
assert np.isclose(ds_1[var_name].values, 98.86, atol=0.01)
assert np.isclose(ds_2[var_name].values, 99.15, atol=0.01)
ds.close()
def test_qc_flag_block_plot():
obj = arm.read_netcdf(sample_files.EXAMPLE_SURFSPECALB1MLAWER)
display = TimeSeriesDisplay(obj, subplot_shape=(2, ), figsize=(8, 2 * 4))
display.plot('surface_albedo_mfr_narrowband_10m',
force_line_plot=True,
labels=True)
display.qc_flag_block_plot('surface_albedo_mfr_narrowband_10m',
subplot_index=(1, ))
obj.close()
del obj
try:
return display.fig
finally:
matplotlib.pyplot.close(display.fig)
def arm_read_netcdf(full_directory_filebase, sdate, edate):
'''Read a set of maraos files and append them together
: param directory: The directory in which to scan for the .nc/.cdf files
relative to '/Users/qingn/Desktop/NQ'
: param filebase: File specification potentially including wild cards
: returns: A list of <xarray.Dataset>'''
files = glob.glob(full_directory_filebase)
files.sort()
files = [
f for f in files
if f.split('.')[-3] >= sdate and f.split('.')[-3] <= edate
]
# file_dir = str(HOME_DIR + directory_filebase)
file_ori = arm.read_netcdf(files)
_, index1 = np.unique(file_ori['time'], return_index=True)
file_ori = file_ori.isel(time=index1)
# file = file_ori.resample(time='10s').mean()
file = file_ori.resample(time='1h').nearest()
return file
def test_qc_bar_plot():
ds_object = arm.read_netcdf(sample_files.EXAMPLE_MET1)
ds_object.clean.cleanup()
var_name = 'temp_mean'
ds_object.qcfilter.set_test(var_name, index=range(100, 600), test_number=2)
# Testing out when the assessment is not listed
ds_object.qcfilter.set_test(var_name, index=range(500, 800), test_number=4)
ds_object['qc_' + var_name].attrs['flag_assessments'][3] = 'Wonky'
display = TimeSeriesDisplay({'sgpmetE13.b1': ds_object},
subplot_shape=(2, ),
figsize=(7, 4))
display.plot(var_name, subplot_index=(0, ), assessment_overplot=True)
display.day_night_background('sgpmetE13.b1', subplot_index=(0, ))
display.qc_flag_block_plot(var_name, subplot_index=(1, ))
ds_object.close()
return display.fig
def test_plot():
# Process MET data to get simple LCL
files = sample_files.EXAMPLE_SONDE_WILDCARD
met = arm.read_netcdf(files)
met_temp = met.temp_mean
met_rh = met.rh_mean
met_lcl = (20. + met_temp / 5.) * (100. - met_rh) / 1000.
met['met_lcl'] = met_lcl * 1000.
met['met_lcl'].attrs['units'] = 'm'
met['met_lcl'].attrs['long_name'] = 'LCL Calculated from SGP MET E13'
# Plot data
# Plot data
display = TimeSeriesDisplay(met)
display.add_subplots((3, ), figsize=(15, 10))
display.plot('wspd_vec_mean', subplot_index=(0, ))
display.plot('temp_mean', subplot_index=(1, ))
display.plot('rh_mean', subplot_index=(2, ))
return display.fig
def test_assessment_overplot_multi():
var_name1, var_name2 = 'wspd_arith_mean', 'wspd_vec_mean'
files = sample_files.EXAMPLE_MET1
ds = arm.read_netcdf(files)
ds.load()
ds.clean.cleanup()
ds.qcfilter.set_test(var_name1, index=np.arange(100, 200, dtype=int), test_number=2)
ds.qcfilter.set_test(var_name1, index=np.arange(500, 600, dtype=int), test_number=4)
ds.qcfilter.set_test(var_name2, index=np.arange(300, 400, dtype=int), test_number=4)
# Plot data
display = TimeSeriesDisplay(ds, subplot_shape=(1, ), figsize=(10, 6))
display.plot(var_name1, label=var_name1,
assessment_overplot=True, overplot_behind=True)
display.plot(var_name2, day_night_background=True, color='green',
label=var_name2, assessment_overplot=True)
ds.close()
return display.fig
def test_contour2():
files = glob.glob(sample_files.EXAMPLE_MET_CONTOUR)
time = '2019-05-08T04:00:00.000000000'
data = {}
fields = {}
wind_fields = {}
station_fields = {}
for f in files:
obj = arm.read_netcdf(f)
data.update({f: obj})
fields.update({f: ['lon', 'lat', 'temp_mean']})
wind_fields.update({f: ['lon', 'lat', 'wspd_vec_mean', 'wdir_vec_mean']})
station_fields.update({f: ['lon', 'lat', 'atmos_pressure']})
display = ContourDisplay(data, figsize=(8, 8))
display.create_contour(fields=fields, time=time, levels=50,
contour='contour', cmap='viridis')
display.plot_vectors_from_spd_dir(fields=wind_fields, time=time, mesh=False, grid_delta=(0.1, 0.1))
display.plot_station(fields=station_fields, time=time, markersize=7, color='pink')
return display.fig
def test_arm_qc():
# Test DQR Webservice using known DQR
variable = 'wspd_vec_mean'
qc_variable = 'qc_' + variable
obj = read_netcdf(EXAMPLE_METE40)
# DQR webservice does go down, so ensure it
# properly runs first before testing
try:
obj = add_dqr_to_qc(obj, variable=variable)
ran = True
except ValueError:
ran = False
if ran:
assert qc_variable in obj
dqr = [
True for d in obj[qc_variable].attrs['flag_meanings']
if 'D190529.4' in d
]
assert dqr[0] is True
def test_clean():
ceil_ds = read_netcdf([EXAMPLE_CEIL1])
ceil_ds.clean.cleanup(clean_arm_state_vars=['detection_status'])
# Check that global attribures are removed
global_attributes = [
'qc_bit_comment', 'qc_bit_1_description', 'qc_bit_1_assessment',
'qc_bit_2_description', 'qc_bit_2_assessment'
'qc_bit_3_description', 'qc_bit_3_assessment', 'qc_bit_4_description',
'qc_bit_4_assessment'
]
for glb_att in global_attributes:
assert glb_att not in ceil_ds.attrs.keys()
# Check that CF attributes are set including new flag_assessments
assert 'flag_masks' in ceil_ds['qc_first_cbh'].attrs.keys()
assert 'flag_meanings' in ceil_ds['qc_first_cbh'].attrs.keys()
assert 'flag_assessments' in ceil_ds['qc_first_cbh'].attrs.keys()
# Check the value of flag_assessments is as expected
assert (all([
ii == 'Bad' for ii in ceil_ds['qc_first_cbh'].attrs['flag_assessments']
]))
# Check the type is correct
assert (type(ceil_ds['qc_first_cbh'].attrs['flag_masks'])) == list
# Check that ancillary varibles is being added
assert ('qc_first_cbh'
in ceil_ds['first_cbh'].attrs['ancillary_variables'].split())
# Check that state field is updated to CF
assert 'flag_values' in ceil_ds['detection_status'].attrs.keys()
assert 'flag_meanings' in ceil_ds['detection_status'].attrs.keys()
assert 'flag_0_description' not in ceil_ds['detection_status'].attrs.keys()
assert ('detection_status'
in ceil_ds['first_cbh'].attrs['ancillary_variables'].split())
ceil_ds.close()
def test_qc_data_type():
drop_vars = [
'base_time', 'time_offset', 'inst_up_long_case_resist',
'inst_up_long_hemisp_tp', 'inst_up_short_hemisp_tp',
'inst_sfc_ir_temp', 'lat', 'lon', 'alt'
]
ds_object = read_netcdf(EXAMPLE_IRT25m20s, drop_variables=drop_vars)
var_name = 'inst_up_long_dome_resist'
expected_qc_var_name = 'qc_' + var_name
ds_object.qcfilter.check_for_ancillary_qc(var_name, add_if_missing=True)
del ds_object[expected_qc_var_name].attrs['flag_meanings']
del ds_object[expected_qc_var_name].attrs['flag_assessments']
ds_object[expected_qc_var_name] = ds_object[expected_qc_var_name].astype(
np.int8)
ds_object.qcfilter.add_test(var_name,
index=[1],
test_number=9,
test_meaning='First test')
assert ds_object[expected_qc_var_name].attrs['flag_masks'][
0].dtype == np.uint32
assert ds_object[expected_qc_var_name].dtype == np.int16
ds_object.qcfilter.add_test(var_name,
index=[1],
test_number=17,
test_meaning='Second test')
assert ds_object[expected_qc_var_name].dtype == np.int32
ds_object.qcfilter.add_test(var_name,
index=[1],
test_number=33,
test_meaning='Third test')
assert ds_object[expected_qc_var_name].dtype == np.int64
assert ds_object[expected_qc_var_name].attrs['flag_masks'][
0].dtype == np.uint64
ds_object.qcfilter.add_test(var_name,
index=[1],
test_meaning='Fourth test',
recycle=True)
def plot_wacr_reflectivity(i):
plt.close("all")
''' No.i files in wacr
'''
wacr = arm.read_netcdf(files[i])
#%% Plotting
fig = plt.figure(figsize=(12, 3))
ax = plt.gca()
plt.contourf(wacr.time.values, wacr.height.values,
wacr.cloud_mask_95ghz_kollias.T
) #,label = 'Micro-pulse lidar cloud mask')
ec = plt.contourf(wacr.time.values,
wacr.height.values,
wacr['reflectivity_best_estimate'].T,
cmap='jet') #,label = 'Hydrometeor-only 95GHz')
plt.plot(wacr.time.values,
wacr.cloud_base_best_estimate,
'w.',
alpha=0.3,
label='cloud_base',
markersize=3)
fig.autofmt_xdate()
plt.xlabel('Date')
plt.ylabel('Height(m)')
plt.yscale('log')
plt.legend()
#plt.show()
cbar = plt.colorbar(ec)
cbar.set_label('reflectivity(dBZ)')
#%%
fig.tight_layout()
fsave = "Full_cloud_mask_" + str(wacr.reflectivity.time[0].values)[:13]
fig.savefig(f"{os.path.join(figpath, fsave)}.png", dpi=300, fmt="png")
def test_xsection_plot_map():
radar_ds = arm.read_netcdf(sample_files.EXAMPLE_VISST,
combine='nested',
concat_dim='time')
try:
xsection = XSectionDisplay(radar_ds, figsize=(15, 8))
xsection.plot_xsection_map(None,
'ir_temperature',
vmin=220,
vmax=300,
cmap='Greys',
x='longitude',
y='latitude',
isel_kwargs={'time': 0})
radar_ds.close()
try:
return xsection.fig
finally:
matplotlib.pyplot.close(xsection.fig)
except Exception:
pass
def test_qc_flag_description():
"""
This will check if the cleanup() method will correctly convert convert
flag_#_description to CF flag_masks and flag_meanings.
"""
ds = read_netcdf(EXAMPLE_CO2FLX4M)
ds.clean.cleanup()
qc_var_name = ds.qcfilter.check_for_ancillary_qc('momentum_flux',
add_if_missing=False,
cleanup=False)
assert isinstance(ds[qc_var_name].attrs['flag_masks'], list)
assert isinstance(ds[qc_var_name].attrs['flag_meanings'], list)
assert isinstance(ds[qc_var_name].attrs['flag_assessments'], list)
assert ds[qc_var_name].attrs['standard_name'] == 'quality_flag'
assert len(ds[qc_var_name].attrs['flag_masks']) == 9
unique_flag_assessments = list(set(['Acceptable', 'Indeterminate', 'Bad']))
assert list(set(
ds[qc_var_name].attrs['flag_assessments'])) == unique_flag_assessments
def test_qctests_dos():
ds_object = read_netcdf(EXAMPLE_IRT25m20s)
var_name = 'inst_up_long_dome_resist'
# persistence test
data = ds_object[var_name].values
data[1000:2500] = data[1000]
ds_object[var_name].values = data
ds_object.qcfilter.add_persistence_test(var_name)
qc_var_name = ds_object.qcfilter.check_for_ancillary_qc(
var_name, add_if_missing=False, cleanup=False, flag_type=False)
test_meaning = ('Data failing persistence test. Standard Deviation over a '
'window of 10 values less than 0.0001.')
assert ds_object[qc_var_name].attrs['flag_meanings'][-1] == test_meaning
assert np.sum(ds_object[qc_var_name].values) == 1500
ds_object.qcfilter.add_persistence_test(var_name,
window=10000,
prepend_text='DQO')
test_meaning = (
'DQO: Data failing persistence test. Standard Deviation over a window of '
'4320 values less than 0.0001.')
assert ds_object[qc_var_name].attrs['flag_meanings'][-1] == test_meaning
def test_multi_skewt_plot():
files = glob.glob(sample_files.EXAMPLE_TWP_SONDE_20060121)
test = {}
for f in files:
time = f.split('.')[-3]
sonde_ds = arm.read_netcdf(f)
test.update({time: sonde_ds})
if METPY:
skewt = SkewTDisplay(test, subplot_shape=(2, 2))
i = 0
j = 0
for f in files:
time = f.split('.')[-3]
skewt.plot_from_spd_and_dir('wspd', 'deg', 'pres', 'tdry', 'dp',
subplot_index=(j, i), dsname=time,
p_levels_to_plot=np.arange(10., 1000., 25))
if j == 1:
i += 1
j = 0
elif j == 0:
j += 1
return skewt.fig
def test_qctests_dos():
ds_object = read_netcdf(EXAMPLE_IRT25m20s)
var_name = 'inst_up_long_dome_resist'
# persistence test
data = ds_object[var_name].values
data[1000:2500] = data[1000]
ds_object[var_name].values = data
result = ds_object.qcfilter.add_persistence_test(var_name)
qc_var_name = result['qc_variable_name']
test_meaning = ('Data failing persistence test. Standard Deviation over a '
'window of 10 values less than 0.0001.')
assert ds_object[qc_var_name].attrs['flag_meanings'][-1] == test_meaning
# There is a precision issue with GitHub testing that makes the number of tests
# tripped off by 1. This isclose() option is to account for that.
assert np.isclose(np.sum(ds_object[qc_var_name].values), 1500, atol=1)
ds_object.qcfilter.add_persistence_test(var_name,
window=10000,
prepend_text='DQO')
test_meaning = (
'DQO: Data failing persistence test. Standard Deviation over a window of '
'4320 values less than 0.0001.')
assert ds_object[qc_var_name].attrs['flag_meanings'][-1] == test_meaning
def test_qc_bar_plot():
ds_object = arm.read_netcdf(sample_files.EXAMPLE_MET1)
ds_object.clean.cleanup()
var_name = 'temp_mean'
ds_object.qcfilter.set_test(var_name, index=range(100, 600), test_number=2)
# Testing out when the assessment is not listed
ds_object.qcfilter.set_test(var_name, index=range(500, 800), test_number=4)
ds_object['qc_' + var_name].attrs['flag_assessments'][3] = 'Wonky'
display = TimeSeriesDisplay({'sgpmetE13.b1': ds_object},
subplot_shape=(2, ), figsize=(7, 4))
display.plot(var_name, subplot_index=(0, ), assessment_overplot=True)
display.day_night_background('sgpmetE13.b1', subplot_index=(0, ))
color_lookup = {'Bad': 'red', 'Incorrect': 'red',
'Indeterminate': 'orange', 'Suspect': 'orange',
'Missing': 'darkgray', 'Not Failing': 'green',
'Acceptable': 'green'}
display.qc_flag_block_plot(var_name, subplot_index=(1, ),
assessment_color=color_lookup)
ds_object.close()
return display.fig
FIGWIDTH = 6
FIGHEIGHT = 4
FONTSIZE = 18
LABELSIZE = 18
plt.rcParams['figure.figsize'] = (FIGWIDTH, FIGHEIGHT)
plt.rcParams['font.size'] = FONTSIZE
plt.rcParams['xtick.labelsize'] = FONTSIZE
plt.rcParams['ytick.labelsize'] = FONTSIZE
matplotlib.rc('xtick', labelsize=LABELSIZE)
matplotlib.rc('ytick', labelsize=LABELSIZE)
# %% Read in data use arm act function
# CPC
cpc = arm.read_netcdf('/Users/qingn/Desktop/NQ/maraoscpc/maraoscpcfM1.b1.2018*')
_, index1 = np.unique(cpc['time'], return_index = True)
cpc = cpc.isel(time = index1)
cpc_con = cpc['concentration']
qc_cpc = cpc['qc_concentration']
time_cpc = cpc['time'].values
# EXHAUST_ID
exhaust_id = netCDF4.Dataset('/Users/qingn/Desktop/NQ/exhaust_id/AAS_4292_ExhaustID_201718_AA_MARCUS.nc')
exhaust = exhaust_id['exhaust_4mad02thresh']
time_id = np.array(exhaust_id['time'])
time_id_date = pd.to_datetime(time_id, unit ='s', origin = pd.Timestamp('2017-10-18 23:45:06'))
# UHSAS
path_uhsas = '/Users/qingn/Desktop/NQ/maraosuhsas/maraosuhsasM1.a1.2018*.nc'
uhsas = arm.read_netcdf(path_uhsas)
import os
from act.io.armfiles import read_netcdf
import datetime
import matplotlib
import sys
import matplotlib.dates as mdates
#import act
import matplotlib.pyplot as plt
import mpl_toolkits
#import mpl_toolkits.basemap as bm
from mpl_toolkits.basemap import Basemap, cm
import act
import act.io.armfiles as arm
import act.plotting.plot as armplot
from sklearn.ensemble import RandomForestClassifier
matplotlib.rc('xtick', labelsize=26)
matplotlib.rc('ytick', labelsize=26)
# %%
cpc = arm.read_netcdf('/Users/qingn/Desktop/NQ/maraoscpc/maraos*.nc')
#exhaust_id = arm.read_netcdf('/Users/qingn/Desktop/NQ/exhaust_id/AAS*.nc')
co = arm.read_netcdf('/Users/qingn/Desktop/NQ/maraosco/mar*.nc')
exhaust_id = netCDF4.Dataset(
'/Users/qingn/Desktop/NQ/exhaust_id/AAS_4292_ExhaustID_201718_AA_MARCUS.nc'
)
time_cpc = cpc['time'].values
time_id = exhaust_id['time'].values
time_co = co['time'].values
#%%
def test_qctests():
ds_object = read_netcdf(EXAMPLE_IRT25m20s)
var_name = 'inst_up_long_dome_resist'
# Add in one missing value and test for that missing value
data = ds_object[var_name].values
data[0] = np.nan
ds_object[var_name].values = data
result = ds_object.qcfilter.add_missing_value_test(var_name)
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert data.mask[0]
# less than min test
limit_value = 6.8
result = ds_object.qcfilter.add_less_test(var_name,
limit_value,
prepend_text='arm')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 54
assert 'fail_min' in ds_object[result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['fail_min'].dtype ==
ds_object[result['variable_name']].values.dtype)
assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_min'],
limit_value)
result = ds_object.qcfilter.add_less_test(var_name,
limit_value,
test_assessment='Suspect')
assert 'warn_min' in ds_object[result['qc_variable_name']].attrs.keys()
# greator than max test
limit_value = 12.7
result = ds_object.qcfilter.add_greater_test(var_name,
limit_value,
prepend_text='arm')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 61
assert 'fail_max' in ds_object[result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['fail_max'].dtype ==
ds_object[result['variable_name']].values.dtype)
assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_max'],
limit_value)
result = ds_object.qcfilter.add_greater_test(var_name,
limit_value,
test_assessment='Suspect')
assert 'warn_max' in ds_object[result['qc_variable_name']].attrs.keys()
# less than or equal test
limit_value = 6.9
result = ds_object.qcfilter.add_less_equal_test(var_name,
limit_value,
test_assessment='Suspect',
prepend_text='arm')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 149
assert 'warn_min' in ds_object[result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['warn_min'].dtype ==
ds_object[result['variable_name']].values.dtype)
assert np.isclose(ds_object[result['qc_variable_name']].attrs['warn_min'],
limit_value)
result = ds_object.qcfilter.add_less_equal_test(var_name, limit_value)
assert 'fail_min' in ds_object[result['qc_variable_name']].attrs.keys()
# greater than or equal test
limit_value = 12
result = ds_object.qcfilter.add_greater_equal_test(
var_name, limit_value, test_assessment='Suspect', prepend_text='arm')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 606
assert 'warn_max' in ds_object[result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['warn_max'].dtype ==
ds_object[result['variable_name']].values.dtype)
assert np.isclose(ds_object[result['qc_variable_name']].attrs['warn_max'],
limit_value)
result = ds_object.qcfilter.add_greater_equal_test(var_name, limit_value)
assert 'fail_max' in ds_object[result['qc_variable_name']].attrs.keys()
# equal to test
limit_value = 7.6705
result = ds_object.qcfilter.add_equal_to_test(var_name,
limit_value,
prepend_text='arm')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 2
assert 'fail_equal_to' in ds_object[
result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['fail_equal_to'].dtype
== ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['fail_equal_to'],
limit_value)
result = ds_object.qcfilter.add_equal_to_test(
var_name, limit_value, test_assessment='Indeterminate')
assert 'warn_equal_to' in ds_object[
result['qc_variable_name']].attrs.keys()
# not equal to test
limit_value = 7.6705
result = ds_object.qcfilter.add_not_equal_to_test(
var_name,
limit_value,
test_assessment='Indeterminate',
prepend_text='arm')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 4318
assert 'warn_not_equal_to' in ds_object[
result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['warn_not_equal_to'].
dtype == ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['warn_not_equal_to'],
limit_value)
result = ds_object.qcfilter.add_not_equal_to_test(var_name, limit_value)
assert 'fail_not_equal_to' in ds_object[
result['qc_variable_name']].attrs.keys()
# outside range test
limit_value1 = 6.8
limit_value2 = 12.7
result = ds_object.qcfilter.add_outside_test(var_name,
limit_value1,
limit_value2,
prepend_text='arm')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 115
assert 'fail_lower_range' in ds_object[
result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['fail_lower_range'].
dtype == ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['fail_lower_range'],
limit_value1)
assert 'fail_upper_range' in ds_object[
result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['fail_upper_range'].
dtype == ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['fail_upper_range'],
limit_value2)
result = ds_object.qcfilter.add_outside_test(
var_name, limit_value1, limit_value2, test_assessment='Indeterminate')
assert 'warn_lower_range' in ds_object[
result['qc_variable_name']].attrs.keys()
assert 'warn_upper_range' in ds_object[
result['qc_variable_name']].attrs.keys()
# inside range test
limit_value1 = 7
limit_value2 = 8
result = ds_object.qcfilter.add_inside_test(var_name,
limit_value1,
limit_value2,
prepend_text='arm')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 479
assert 'fail_lower_range_inner' in ds_object[
result['qc_variable_name']].attrs.keys()
assert (
ds_object[result['qc_variable_name']].attrs['fail_lower_range_inner'].
dtype == ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['fail_lower_range_inner'],
limit_value1)
assert 'fail_upper_range_inner' in ds_object[
result['qc_variable_name']].attrs.keys()
assert (
ds_object[result['qc_variable_name']].attrs['fail_upper_range_inner'].
dtype == ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['fail_upper_range_inner'],
limit_value2)
result = ds_object.qcfilter.add_inside_test(
var_name, limit_value1, limit_value2, test_assessment='Indeterminate')
assert 'warn_lower_range_inner' in ds_object[
result['qc_variable_name']].attrs.keys()
assert 'warn_upper_range_inner' in ds_object[
result['qc_variable_name']].attrs.keys()
# delta test
test_limit = 0.05
result = ds_object.qcfilter.add_delta_test(var_name,
test_limit,
prepend_text='arm')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 175
assert 'warn_delta' in ds_object[result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['warn_delta'].dtype ==
ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['warn_delta'], test_limit)
data = ds_object.qcfilter.get_masked_data(
var_name, rm_assessments=['Suspect', 'Bad'])
assert np.ma.count_masked(data) == 4320
result = ds_object.qcfilter.add_delta_test(var_name,
test_limit,
test_assessment='Bad')
assert 'fail_delta' in ds_object[result['qc_variable_name']].attrs.keys()
comp_object = read_netcdf(EXAMPLE_IRT25m20s)
result = ds_object.qcfilter.add_difference_test(
var_name, {comp_object.attrs['datastream']: comp_object},
var_name,
diff_limit=1,
prepend_text='arm')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert not (data.mask).all()
comp_object.close()
ds_object.close()
def test_fft_shading_test():
obj = read_netcdf(EXAMPLE_MFRSR)
obj.clean.cleanup()
obj = fft_shading_test(obj)
qc_data = obj['qc_diffuse_hemisp_narrowband_filter4']
assert np.nansum(qc_data.values) == 456
import os
import json
import matplotlib
import sys
import numpy as np
import xarray as xr
import pandas as pd
import datetime
from sklearn.ensemble import RandomForestClassifier
from sklearn import preprocessing
co_path = '/Users/qingn/Desktop/NQ/maraosco/maraoscoM1.b1.2017111[3-4]*'
o3_path = '/Users/qingn/Desktop/NQ/maraoso3/maraoso3M1.b1.2017111[3-4]*.custom.nc'
cpc_path = '/Users/qingn/Desktop/NQ/maraoscpc/maraoscpcfM1.b1.2017111[3-4]*'
co = arm.read_netcdf(co_path)
o3 = arm.read_netcdf(o3_path)
cpc = arm.read_netcdf(cpc_path)
#%%
cpc_con = cpc['concentration'] #.resample(time='1min').mean()
cpc_con = cpc_con.resample(time='1s').nearest()
# obj = arm.read_netcdf(files,variables=var)
co = arm.read_netcdf(co_path)
o3 = arm.read_netcdf(o3_path)
co_con = co['co_dry'].where(co['qc_co_dry'] < 16384)
co_con = co_con.resample(time='1s').nearest()
co_diff = co_con.diff('time', n=1)
o3_con = o3['o3'].where(o3['qc_o3'] < 262144)
import os
import json
import matplotlib
import sys
import numpy as np
import xarray as xr
import pandas as pd
import datetime
from sklearn.ensemble import RandomForestClassifier
from sklearn import preprocessing
co_path = '/Users/qingn/Desktop/NQ/maraosco/maraoscoM1.b1.2017111[3-4]*'
o3_path = '/Users/qingn/Desktop/NQ/maraoso3/maraoso3M1.b1.2017111[3-4]*.custom.nc'
cpc_path = '/Users/qingn/Desktop/NQ/maraoscpc/maraoscpcfM1.b1.2017111[3-4]*'
co = arm.read_netcdf(co_path)
o3 = arm.read_netcdf(o3_path)
cpc = arm.read_netcdf(cpc_path)
#%%
cpc_con = cpc['concentration'] #.resample(time='1min').mean()
cpc_con = cpc_con.resample(time='1s').nearest()
# obj = arm.read_netcdf(files,variables=var)
co = arm.read_netcdf(co_path)
o3 = arm.read_netcdf(o3_path)
co_con = co['co_dry'].where(co['qc_co_dry'] < 16384)
co_con = co_con.resample(time='1s').nearest()
co_diff = co_con.diff('time', n=1)
o3_con = o3['o3'].where(o3['qc_o3'] < 262144)
def test_clean():
# Read test data
ceil_ds = read_netcdf([EXAMPLE_CEIL1])
# Cleanup QC data
ceil_ds.clean.cleanup(clean_arm_state_vars=['detection_status'])
# Check that global attribures are removed
global_attributes = [
'qc_bit_comment', 'qc_bit_1_description', 'qc_bit_1_assessment',
'qc_bit_2_description', 'qc_bit_2_assessment'
'qc_bit_3_description', 'qc_bit_3_assessment'
]
for glb_att in global_attributes:
assert glb_att not in ceil_ds.attrs.keys()
# Check that CF attributes are set including new flag_assessments
var_name = 'qc_first_cbh'
for attr_name in ['flag_masks', 'flag_meanings', 'flag_assessments']:
assert attr_name in ceil_ds[var_name].attrs.keys()
assert isinstance(ceil_ds[var_name].attrs[attr_name], list)
# Check that the flag_mask values are set correctly
assert ceil_ds['qc_first_cbh'].attrs['flag_masks'] == [1, 2, 4]
# Check that the flag_meanings values are set correctly
assert (ceil_ds['qc_first_cbh'].attrs['flag_meanings'] == [
'Value is equal to missing_value.', 'Value is less than the fail_min.',
'Value is greater than the fail_max.'
])
# Check the value of flag_assessments is as expected
assert ceil_ds['qc_first_cbh'].attrs['flag_assessments'] == [
'Bad', 'Bad', 'Bad'
]
# Check that ancillary varibles is being added
assert 'qc_first_cbh' in ceil_ds['first_cbh'].attrs[
'ancillary_variables'].split()
# Check that state field is updated to CF
assert 'flag_values' in ceil_ds['detection_status'].attrs.keys()
assert isinstance(ceil_ds['detection_status'].attrs['flag_values'], list)
assert ceil_ds['detection_status'].attrs['flag_values'] == [
0, 1, 2, 3, 4, 5
]
assert 'flag_meanings' in ceil_ds['detection_status'].attrs.keys()
assert isinstance(ceil_ds['detection_status'].attrs['flag_meanings'], list)
assert (ceil_ds['detection_status'].attrs['flag_meanings'] == [
'No significant backscatter', 'One cloud base detected',
'Two cloud bases detected', 'Three cloud bases detected',
'Full obscuration determined but no cloud base detected',
'Some obscuration detected but determined to be transparent'
])
assert 'flag_0_description' not in ceil_ds['detection_status'].attrs.keys()
assert ('detection_status'
in ceil_ds['first_cbh'].attrs['ancillary_variables'].split())
ceil_ds.close()
def test_qcfilter():
ds_object = read_netcdf(EXAMPLE_IRT25m20s)
var_name = 'inst_up_long_dome_resist'
expected_qc_var_name = 'qc_' + var_name
# Perform adding of quality control variables to object
result = ds_object.qcfilter.add_test(var_name, test_meaning='Birds!')
assert isinstance(result, dict)
qc_var_name = result['qc_variable_name']
assert qc_var_name == expected_qc_var_name
# Check that new linking and describing attributes are set
assert ds_object[qc_var_name].attrs['standard_name'] == 'quality_flag'
assert ds_object[var_name].attrs['ancillary_variables'] == qc_var_name
# Check that CF attributes are set including new flag_assessments
assert 'flag_masks' in ds_object[qc_var_name].attrs.keys()
assert 'flag_meanings' in ds_object[qc_var_name].attrs.keys()
assert 'flag_assessments' in ds_object[qc_var_name].attrs.keys()
# Check that the values of the attributes are set correctly
assert ds_object[qc_var_name].attrs['flag_assessments'][0] == 'Bad'
assert ds_object[qc_var_name].attrs['flag_meanings'][0] == 'Birds!'
assert ds_object[qc_var_name].attrs['flag_masks'][0] == 1
# Set some test values
index = [0, 1, 2, 30]
ds_object.qcfilter.set_test(var_name,
index=index,
test_number=result['test_number'])
# Add a new test and set values
index2 = [6, 7, 8, 50]
ds_object.qcfilter.add_test(var_name,
index=index2,
test_number=9,
test_meaning='testing high number',
test_assessment='Suspect')
# Retrieve data from object as numpy masked array. Count number of masked
# elements and ensure equal to size of index array.
data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments='Bad')
assert np.ma.count_masked(data) == len(index)
data = ds_object.qcfilter.get_masked_data(var_name,
rm_assessments='Suspect',
return_nan_array=True)
assert np.sum(np.isnan(data)) == len(index2)
data = ds_object.qcfilter.get_masked_data(
var_name, rm_assessments=['Bad', 'Suspect'], ma_fill_value=np.nan)
assert np.ma.count_masked(data) == len(index + index2)
# Test internal function for returning the index array of where the
# tests are set.
assert np.sum(
ds_object.qcfilter.get_qc_test_mask(
var_name, result['test_number'], return_index=True) -
np.array(index, dtype=np.int)) == 0
# Unset a test
ds_object.qcfilter.unset_test(var_name,
index=0,
test_number=result['test_number'])
# Remove the test
ds_object.qcfilter.remove_test(var_name, test_number=result['test_number'])
pytest.raises(ValueError, ds_object.qcfilter.add_test, var_name)
pytest.raises(ValueError, ds_object.qcfilter.remove_test, var_name)
ds_object.close()
pytest.raises(ValueError, parse_bit, [1, 2])
pytest.raises(ValueError, parse_bit, -1)
assert set_bit(0, 16) == 32768
data = range(0, 4)
assert isinstance(set_bit(list(data), 2), list)
assert isinstance(set_bit(tuple(data), 2), tuple)
assert isinstance(unset_bit(list(data), 2), list)
assert isinstance(unset_bit(tuple(data), 2), tuple)
# Fill in missing tests
ds_object = read_netcdf(EXAMPLE_IRT25m20s)
del ds_object[var_name].attrs['long_name']
# Test creating a qc variable
ds_object.qcfilter.create_qc_variable(var_name)
# Test creating a second qc variable and of flag type
ds_object.qcfilter.create_qc_variable(var_name, flag_type=True)
result = ds_object.qcfilter.add_test(var_name,
index=[1, 2, 3],
test_number=9,
test_meaning='testing high number',
flag_value=True)
ds_object.qcfilter.set_test(var_name,
index=5,
test_number=9,
flag_value=True)
data = ds_object.qcfilter.get_masked_data(var_name)
assert np.isclose(np.sum(data), 42674.766, 0.01)
data = ds_object.qcfilter.get_masked_data(var_name, rm_assessments='Bad')
assert np.isclose(np.sum(data), 42643.195, 0.01)
ds_object.qcfilter.unset_test(var_name, test_number=9, flag_value=True)
ds_object.qcfilter.unset_test(var_name,
index=1,
test_number=9,
flag_value=True)
assert ds_object.qcfilter.available_bit(result['qc_variable_name']) == 10
assert ds_object.qcfilter.available_bit(result['qc_variable_name'],
recycle=True) == 1
ds_object.qcfilter.remove_test(var_name, test_number=9, flag_value=True)
ds_object.qcfilter.update_ancillary_variable(var_name)
# Test updating ancillary variable if does not exist
ds_object.qcfilter.update_ancillary_variable('not_a_variable_name')
# Change ancillary_variables attribute to test if add correct qc variable correctly
ds_object[var_name].attrs['ancillary_variables'] = 'a_different_name'
ds_object.qcfilter.update_ancillary_variable(
var_name, qc_var_name=expected_qc_var_name)
assert (expected_qc_var_name
in ds_object[var_name].attrs['ancillary_variables'])
# Test flag QC
var_name = 'inst_sfc_ir_temp'
qc_var_name = 'qc_' + var_name
ds_object.qcfilter.create_qc_variable(var_name, flag_type=True)
assert qc_var_name in list(ds_object.data_vars)
assert 'flag_values' in ds_object[qc_var_name].attrs.keys()
assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys()
del ds_object[qc_var_name]
qc_var_name = ds_object.qcfilter.check_for_ancillary_qc(
var_name, add_if_missing=True, cleanup=False, flag_type=True)
assert qc_var_name in list(ds_object.data_vars)
assert 'flag_values' in ds_object[qc_var_name].attrs.keys()
assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys()
del ds_object[qc_var_name]
ds_object.qcfilter.add_missing_value_test(var_name,
flag_value=True,
prepend_text='arm')
ds_object.qcfilter.add_test(var_name,
index=list(range(0, 20)),
test_number=2,
test_meaning='Testing flag',
flag_value=True,
test_assessment='Suspect')
assert qc_var_name in list(ds_object.data_vars)
assert 'flag_values' in ds_object[qc_var_name].attrs.keys()
assert 'flag_masks' not in ds_object[qc_var_name].attrs.keys()
assert 'standard_name' in ds_object[qc_var_name].attrs.keys()
assert ds_object[qc_var_name].attrs['flag_values'] == [1, 2]
assert ds_object[qc_var_name].attrs['flag_assessments'] == [
'Bad', 'Suspect'
]
ds_object.close()
def test_qctests():
ds_object = read_netcdf(EXAMPLE_IRT25m20s)
var_name = 'inst_up_long_dome_resist'
# Add in one missing value and test for that missing value
data = ds_object[var_name].values
data[0] = np.nan
ds_object[var_name].data = da.from_array(data)
result = ds_object.qcfilter.add_missing_value_test(var_name)
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert data.mask[0]
result = ds_object.qcfilter.add_missing_value_test(var_name, use_dask=True)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert data == np.array([0])
ds_object.qcfilter.remove_test(var_name, test_number=result['test_number'])
# less than min test
limit_value = 6.8
result = ds_object.qcfilter.add_less_test(var_name,
limit_value,
prepend_text='arm',
limit_attr_name='fail_min')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 54
assert 'fail_min' in ds_object[result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['fail_min'].dtype ==
ds_object[result['variable_name']].values.dtype)
assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_min'],
limit_value)
result = ds_object.qcfilter.add_less_test(var_name,
limit_value,
test_assessment='Suspect')
assert 'warn_min' in ds_object[result['qc_variable_name']].attrs.keys()
limit_value = 8
result = ds_object.qcfilter.add_less_test(var_name, limit_value)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 2911939
result = ds_object.qcfilter.add_less_test(var_name,
limit_value,
use_dask=True)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 2911939
# greator than max test
limit_value = 12.7
result = ds_object.qcfilter.add_greater_test(var_name,
limit_value,
prepend_text='arm',
limit_attr_name='fail_max')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 61
assert 'fail_max' in ds_object[result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['fail_max'].dtype ==
ds_object[result['variable_name']].values.dtype)
assert np.isclose(ds_object[result['qc_variable_name']].attrs['fail_max'],
limit_value)
result = ds_object.qcfilter.add_greater_test(var_name,
limit_value,
test_assessment='Suspect')
assert 'warn_max' in ds_object[result['qc_variable_name']].attrs.keys()
result = ds_object.qcfilter.add_greater_test(var_name,
limit_value,
use_dask=True)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 125458
result = ds_object.qcfilter.add_greater_test(var_name, limit_value)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 125458
# less than or equal test
limit_value = 6.9
result = ds_object.qcfilter.add_less_equal_test(var_name,
limit_value,
test_assessment='Suspect',
prepend_text='arm',
limit_attr_name='warn_min')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 149
assert 'warn_min' in ds_object[result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['warn_min'].dtype ==
ds_object[result['variable_name']].values.dtype)
assert np.isclose(ds_object[result['qc_variable_name']].attrs['warn_min'],
limit_value)
result = ds_object.qcfilter.add_less_equal_test(var_name, limit_value)
assert 'fail_min' in ds_object[result['qc_variable_name']].attrs.keys()
result = ds_object.qcfilter.add_less_equal_test(var_name,
limit_value,
use_dask=True)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 601581
result = ds_object.qcfilter.add_less_equal_test(var_name, limit_value)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 601581
# greater than or equal test
result = ds_object.qcfilter.add_greater_equal_test(var_name, None)
limit_value = 12
result = ds_object.qcfilter.add_greater_equal_test(
var_name,
limit_value,
test_assessment='Suspect',
prepend_text='arm',
limit_attr_name='warn_max')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 606
assert 'warn_max' in ds_object[result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['warn_max'].dtype ==
ds_object[result['variable_name']].values.dtype)
assert np.isclose(ds_object[result['qc_variable_name']].attrs['warn_max'],
limit_value)
result = ds_object.qcfilter.add_greater_equal_test(var_name, limit_value)
assert 'fail_max' in ds_object[result['qc_variable_name']].attrs.keys()
result = ds_object.qcfilter.add_greater_equal_test(var_name,
limit_value,
use_dask=True)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 1189873
result = ds_object.qcfilter.add_greater_equal_test(var_name, limit_value)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 1189873
# equal to test
limit_value = 7.6705
result = ds_object.qcfilter.add_equal_to_test(
var_name,
limit_value,
prepend_text='arm',
limit_attr_name='fail_equal_to')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 2
assert 'fail_equal_to' in ds_object[
result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['fail_equal_to'].dtype
== ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['fail_equal_to'],
limit_value)
result = ds_object.qcfilter.add_equal_to_test(
var_name, limit_value, test_assessment='Indeterminate')
assert 'warn_equal_to' in ds_object[
result['qc_variable_name']].attrs.keys()
result = ds_object.qcfilter.add_equal_to_test(var_name,
limit_value,
use_dask=True)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 8631
result = ds_object.qcfilter.add_equal_to_test(var_name, limit_value)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 8631
# not equal to test
limit_value = 7.6705
result = ds_object.qcfilter.add_not_equal_to_test(
var_name,
limit_value,
test_assessment='Indeterminate',
prepend_text='arm',
limit_attr_name='warn_not_equal_to')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 4318
assert 'warn_not_equal_to' in ds_object[
result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['warn_not_equal_to'].
dtype == ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['warn_not_equal_to'],
limit_value)
result = ds_object.qcfilter.add_not_equal_to_test(var_name, limit_value)
assert 'fail_not_equal_to' in ds_object[
result['qc_variable_name']].attrs.keys()
result = ds_object.qcfilter.add_not_equal_to_test(var_name,
limit_value,
use_dask=True)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 9320409
result = ds_object.qcfilter.add_not_equal_to_test(var_name, limit_value)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 9320409
# outside range test
limit_value1 = 6.8
limit_value2 = 12.7
result = ds_object.qcfilter.add_outside_test(
var_name,
limit_value1,
limit_value2,
prepend_text='arm',
limit_attr_names=['fail_lower_range', 'fail_upper_range'])
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 115
assert 'fail_lower_range' in ds_object[
result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['fail_lower_range'].
dtype == ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['fail_lower_range'],
limit_value1)
assert 'fail_upper_range' in ds_object[
result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['fail_upper_range'].
dtype == ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['fail_upper_range'],
limit_value2)
result = ds_object.qcfilter.add_outside_test(
var_name, limit_value1, limit_value2, test_assessment='Indeterminate')
assert 'warn_lower_range' in ds_object[
result['qc_variable_name']].attrs.keys()
assert 'warn_upper_range' in ds_object[
result['qc_variable_name']].attrs.keys()
result = ds_object.qcfilter.add_outside_test(var_name,
limit_value1,
limit_value2,
use_dask=True)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 342254
result = ds_object.qcfilter.add_outside_test(
var_name,
limit_value1,
limit_value2,
)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 342254
# Starting to run out of space for tests. Remove some tests.
for ii in range(16, 30):
ds_object.qcfilter.remove_test(var_name, test_number=ii)
# inside range test
limit_value1 = 7
limit_value2 = 8
result = ds_object.qcfilter.add_inside_test(
var_name,
limit_value1,
limit_value2,
prepend_text='arm',
limit_attr_names=['fail_lower_range_inner', 'fail_upper_range_inner'])
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 479
assert 'fail_lower_range_inner' in ds_object[
result['qc_variable_name']].attrs.keys()
assert (
ds_object[result['qc_variable_name']].attrs['fail_lower_range_inner'].
dtype == ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['fail_lower_range_inner'],
limit_value1)
assert 'fail_upper_range_inner' in ds_object[
result['qc_variable_name']].attrs.keys()
assert (
ds_object[result['qc_variable_name']].attrs['fail_upper_range_inner'].
dtype == ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['fail_upper_range_inner'],
limit_value2)
result = ds_object.qcfilter.add_inside_test(
var_name, limit_value1, limit_value2, test_assessment='Indeterminate')
assert 'warn_lower_range_inner' in ds_object[
result['qc_variable_name']].attrs.keys()
assert 'warn_upper_range_inner' in ds_object[
result['qc_variable_name']].attrs.keys()
result = ds_object.qcfilter.add_inside_test(var_name,
limit_value1,
limit_value2,
use_dask=True)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 1820693
result = ds_object.qcfilter.add_inside_test(
var_name,
limit_value1,
limit_value2,
)
data = ds_object.qcfilter.get_qc_test_mask(var_name,
result['test_number'],
return_index=True)
assert np.sum(data) == 1820693
# delta test
test_limit = 0.05
result = ds_object.qcfilter.add_delta_test(var_name,
test_limit,
prepend_text='arm',
limit_attr_name='warn_delta')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert np.ma.count_masked(data) == 175
assert 'warn_delta' in ds_object[result['qc_variable_name']].attrs.keys()
assert (ds_object[result['qc_variable_name']].attrs['warn_delta'].dtype ==
ds_object[result['variable_name']].values.dtype)
assert np.isclose(
ds_object[result['qc_variable_name']].attrs['warn_delta'], test_limit)
data = ds_object.qcfilter.get_masked_data(
var_name, rm_assessments=['Suspect', 'Bad'])
assert np.ma.count_masked(data) == 1355
result = ds_object.qcfilter.add_delta_test(var_name,
test_limit,
test_assessment='Bad')
assert 'fail_delta' in ds_object[result['qc_variable_name']].attrs.keys()
comp_object = read_netcdf(EXAMPLE_IRT25m20s)
with np.testing.assert_raises(ValueError):
result = ds_object.qcfilter.add_difference_test(var_name, 'test')
with np.testing.assert_raises(ValueError):
result = ds_object.qcfilter.add_difference_test(
var_name, {comp_object.attrs['datastream']: comp_object},
var_name,
diff_limit=None)
assert ds_object.qcfilter.add_difference_test(
var_name, set_test_regardless=False) is None
result = ds_object.qcfilter.add_difference_test(
var_name, {comp_object.attrs['datastream']: comp_object},
var_name,
diff_limit=1,
prepend_text='arm')
data = ds_object.qcfilter.get_masked_data(var_name,
rm_tests=result['test_number'])
assert 'arm' in result['test_meaning']
assert not (data.mask).all()
comp_object.close()
ds_object.close()
