def test_running_save_nc(self):
c = NcQcConfig(self.fp)
ncresults = c.run(self.fp)
c.save_to_netcdf(self.fp, ncresults)
with nc4.Dataset(self.fp) as ncd:
assert 'data1' in ncd.variables
assert 'qc1' in ncd.variables
qcv = ncd.variables['qc1']
datav = ncd.variables['data1']
assert datav.ancillary_variables == 'qc1'
assert datav.standard_name == 'air_temperature'
npt.assert_array_equal(
datav[:],
self.data
)
assert qcv.standard_name == 'status_flag'
assert qcv.ioos_qc_module == 'qartod'
assert qcv.ioos_qc_test == 'gross_range_test'
assert qcv.ioos_qc_target == 'data1'
assert qcv.ioos_qc_config == json.dumps(self.config)
npt.assert_array_equal(
qcv[:],
self.expected
)
def test_running_climatology_save_netcdf(self):
qc = NcQcConfig(self.config)
ncresults = qc.run(
self.fp,
data1={
'tinp': self.times,
'zinp': self.depths
}
)
npt.assert_array_equal(
ncresults['data1']['qartod']['climatology_test'],
self.climate_expected
)
npt.assert_array_equal(
ncresults['data1']['qartod']['gross_range_test'],
self.gross_expected
)
qc.save_to_netcdf(self.fp, ncresults)
with nc4.Dataset(self.fp) as ncd:
assert 'data1' in ncd.variables
assert 'qc1' in ncd.variables
assert 'qc2' in ncd.variables
datav = ncd.variables['data1']
assert sorted(datav.ancillary_variables.split(' ')) == ['qc1', 'qc2']
assert datav.standard_name == 'air_temperature'
npt.assert_array_equal(
datav[:],
self.values
)
qc1 = ncd.variables['qc1']
assert qc1.standard_name == 'status_flag'
assert qc1.ioos_qc_module == 'qartod'
assert qc1.ioos_qc_test == 'gross_range_test'
assert qc1.ioos_qc_target == 'data1'
assert qc1.ioos_qc_config == json.dumps(self.gross_config, cls=GeoNumpyDateEncoder)
npt.assert_array_equal(
qc1[:],
self.gross_expected
)
qc2 = ncd.variables['qc2']
assert qc2.standard_name == 'status_flag'
assert qc2.ioos_qc_module == 'qartod'
assert qc2.ioos_qc_test == 'climatology_test'
assert qc2.ioos_qc_target == 'data1'
assert qc2.ioos_qc_config == json.dumps(self.climatology_config, cls=GeoNumpyDateEncoder)
npt.assert_array_equal(
qc2[:],
self.climate_expected
)
def test_comparing_nc_and_qc_from_nc(self):
c = NcQcConfig(self.fp)
ncresults = c.run(self.fp)
qcr = QcConfig(c.config['data1'])
result = qcr.run(inp=list(range(13)))
npt.assert_array_equal(
ncresults['data1']['qartod']['gross_range_test'],
result['qartod']['gross_range_test'], self.expected)
def test_load_climatology_from_netcdf(self):
qc = NcQcConfig(self.fp, tinp='time', zinp='depth')
ncresults = qc.run(self.fp)
npt.assert_array_equal(
ncresults['data1']['qartod']['climatology_test'],
self.climate_expected)
npt.assert_array_equal(
ncresults['data1']['qartod']['gross_range_test'],
self.gross_expected)
def test_comparing_nc_and_qc_config(self):
# Compare results from QcConfig to those from NcQcConfig
nc_config = NcQcConfig(self.config)
nc_results = nc_config.run(self.fp)
qc_config = QcConfig(self.config['data1'])
qc_results = qc_config.run(inp=self.data)
npt.assert_array_equal(
nc_results['data1']['qartod']['gross_range_test'],
qc_results['qartod']['gross_range_test'], self.expected)
def test_run_and_save_to_netcdf(self):
# Config is defined as a dict externally, and passed to NcQcConfig
c = NcQcConfig(self.config)
# Run tests against the input file
nc_results = c.run(self.fp)
npt.assert_array_equal(
nc_results['data1']['qartod']['gross_range_test'], self.expected)
# Save results to netcdf file
c.save_to_netcdf(self.fp, nc_results)
with nc4.Dataset(self.fp, 'r') as ncd:
assert 'data1' in list(ncd.variables.keys())
assert 'data1_qartod_gross_range_test' in list(
ncd.variables.keys())
qcv = ncd.variables['data1_qartod_gross_range_test']
datav = ncd.variables['data1']
assert datav.standard_name == 'air_temperature'
npt.assert_array_equal(datav[:], self.data)
assert datav.ancillary_variables == 'data1_qartod_gross_range_test'
assert qcv.standard_name == 'gross_range_test_quality_flag'
assert qcv.ioos_qc_module == 'qartod'
assert qcv.ioos_qc_test == 'gross_range_test'
assert qcv.ioos_qc_target == 'data1'
assert qcv.ioos_qc_config == json.dumps(
self.config['data1']['qartod']['gross_range_test'])
npt.assert_array_equal(qcv[:], self.expected)
# Now we can update data and run again with the config in the file
# Update data
with nc4.Dataset(self.fp, 'r+') as ncd_upd:
data1_upd = ncd_upd.variables['data1']
data1_upd[:] = np.append(data1_upd[:], 13)
assert len(datav) == 14
assert len(ncd_upd['data1_qartod_gross_range_test']) == 14
assert np.ma.is_masked(
ncd_upd['data1_qartod_gross_range_test'][13])
# Run tests again. This will re-use the config saved to the netcdf
c_upd = NcQcConfig(self.fp)
nc_results_upd = c_upd.run(self.fp)
npt.assert_array_equal(
nc_results_upd['data1']['qartod']['gross_range_test'],
self.expected + [4])
c_upd.save_to_netcdf(self.fp, nc_results_upd)
with nc4.Dataset(self.fp, 'r') as ncd_final:
assert ncd_final['data1_qartod_gross_range_test'][13] == 4
def test_load_climatology_from_netcdf(self):
qc = NcQcConfig(self.fp)
ncresults = qc.run(self.fp,
data1={
'tinp': np.array(self.times),
'zinp': self.depths
})
npt.assert_array_equal(
ncresults['data1']['qartod']['climatology_test'],
self.climate_expected)
npt.assert_array_equal(
ncresults['data1']['qartod']['gross_range_test'],
self.gross_expected)
def test_loading_dict(self):
c = NcQcConfig(
{'data1': {
'qartod': {
'gross_range_test': self.config
}
}})
assert 'data1' in c.config
assert c.config['data1']['qartod']['gross_range_test'] == self.config
def process_data(row, dest_dir='.', config=None):
""" Apply standard processing method and QAQC to the CTD time series."""
if row['Link to Raw Data'] is None:
return
file_output = dest_dir + row['file_name']
# Read Seabird CNV
print('Read ' + row['raw_file_path'])
c = fCNV(row['raw_file_path'])
# Save to NetCDF
print('Save to ' + row['file_name'] + '_L0.nc')
l0_file = path.join(dest_dir, row['file_name'] + '_L0.nc')
l1_file = path.join(dest_dir, row['file_name'] + '_L1.nc')
cnv2nc(c, l0_file)
# Add Metadata to NetCDF
# + Metadata Variables
# + Global Attributes
nc = netCDF4.Dataset(l0_file, 'a')
# Latitude
latitude = nc.createVariable('latitude', float)
latitude.units = 'degrees_north'
latitude[:] = row['Latitude']
# Longitude
longitude = nc.createVariable('longitude', float)
longitude.units = 'degrees_east'
longitude[:] = row['Longitude']
# Station
station = nc.createVariable('station', str)
station[0] = row['Site']
# File name (timeseries_id)
# TODO this is temporary I would prefer having access to the instrument serial number instead
file_name = nc.createVariable('file_id', str)
file_name[0] = row['file_name']
# TODO Add timeseries_id and cdm_data_type info but it may be better to rely on the
# seabird tool for doing that
# timeseries_id Can be generated by the tool itself
# Add extra metadata as global attributes
for key, value in row.drop(['Latitude', 'Longitude']).items():
if value:
if type(value) is not (float, int, str):
value = str(value) # Likely datetime
nc.setncattr(key.split('(')[0].strip(),
value) # Keep anything before (
# Find Crop data to keep in water only
ds = xr.open_dataset(l0_file)
start_end_results = process.detect_start_end(ds,
'time',
'PRESPR01',
figure_path=file_output +
'_crop.png')
# Output Cropped time series a L1
ds = ds.loc[dict(time=slice(start_end_results['first_good_record_time'],
start_end_results['last_good_record_time']))]
ds.to_netcdf(l1_file)
# Run QARTOD on the NetCDF file
# Retrieve Hakai QARTOD Tests
if not config:
config = get_ctd_qc_config()
# Use deprecated NcQcConfig
qc = NcQcConfig(config, tinp='time')
qartod_results = qc.run(l1_file)
# Upload QARTOD Flags to NetCDF
qc.save_to_netcdf(l1_file, qartod_results)
# Move away from the NcQcConfig method temporary (hopefully we'll use the streams method soon.
# ds = process.run_qartod(ds, config)
# ds.to_netcdf(l1_file)
return {'l0': l0_file, 'l1': l1_file}
def test_load_path_object(self):
qc = NcQcConfig(Path(self.yamlfile))
assert qc.config == self.expected_dict
def test_load_file_path(self):
qc = NcQcConfig(self.yamlfile)
assert qc.config == self.expected_dict
def test_load_yaml_dict_object(self):
with open(self.yamlfile) as f:
y = yaml.load(f.read(), Loader=yaml.Loader)
qc = NcQcConfig(y)
assert qc.config == self.expected_dict
def test_loading_netcdf(self):
c = NcQcConfig(self.fp)
assert 'data1' in c.config
assert c.config['data1']['qartod']['gross_range_test'] == self.config
def test_run(self):
# setup data
config = {
'data1': {
'qartod': {
'gross_range_test': {
'suspect_span': [2, 10],
'fail_span': [0, 12],
},
'flat_line_test': {
'suspect_threshold': 10,
'fail_threshold': 100,
},
'location_test': {
'bbox': [-80, 40, -70, 60]
},
'aggregate': {}
}
}
}
time_vals = [1, 2, 3, 4, 5]
data_vals = [1, 1, 2, 3, 4]
lon_vals = [80, -78, -71, -79, 500]
lat_vals = [50, 50, 59, 10, -60]
gross_range_expected = [3, 3, 1, 1, 1]
flat_line_expected = [1, 1, 1, 1, 1]
location_expected = [4, 1, 1, 4, 4]
aggregate_expected = [4, 3, 1, 4, 4]
with nc4.Dataset(self.fp, 'w') as ncd:
ncd.createDimension('time')
ncd.createDimension('lon')
ncd.createDimension('lat')
time = ncd.createVariable('time', 'i4', ('time', ))
time[:] = time_vals
lon = ncd.createVariable('lon', 'f8', ('time', ))
lon[:] = lon_vals
lat = ncd.createVariable('lat', 'f8', ('time', ))
lat[:] = lat_vals
data1 = ncd.createVariable('data1', 'f8', ('time', ))
data1.standard_name = 'air_temperature'
data1[:] = data_vals
# run tests
c = NcQcConfig(config, tinp='time', lon='lon', lat='lat')
nc_results = c.run(self.fp)
npt.assert_array_equal(
nc_results['data1']['qartod']['gross_range_test'],
gross_range_expected)
npt.assert_array_equal(nc_results['data1']['qartod']['flat_line_test'],
flat_line_expected)
npt.assert_array_equal(nc_results['data1']['qartod']['location_test'],
location_expected)
npt.assert_array_equal(nc_results['data1']['qartod']['aggregate'],
aggregate_expected)
# Save results to netcdf file
c.save_to_netcdf(self.fp, nc_results)
with nc4.Dataset(self.fp, 'r') as ncd:
assert 'data1' in ncd.variables
assert 'data1_qartod_gross_range_test' in ncd.variables
assert 'data1_qartod_flat_line_test' in ncd.variables
assert 'data1_qartod_location_test' in ncd.variables
assert 'data1_qartod_aggregate' in ncd.variables
datav = ncd.variables['data1']
av = datav.ancillary_variables
assert set(av.split(' ')) == set([
'data1_qartod_gross_range_test', 'data1_qartod_flat_line_test',
'data1_qartod_location_test', 'data1_qartod_aggregate'
])
qc_grt = ncd.variables['data1_qartod_gross_range_test']
assert qc_grt.standard_name == 'gross_range_test_quality_flag'
npt.assert_array_equal(qc_grt[:], gross_range_expected)
qc_flt = ncd.variables['data1_qartod_flat_line_test']
assert qc_flt.standard_name == 'flat_line_test_quality_flag'
npt.assert_array_equal(qc_flt[:], flat_line_expected)
qc_loc = ncd.variables['data1_qartod_location_test']
assert qc_loc.standard_name == 'location_test_quality_flag'
npt.assert_array_equal(qc_loc[:], location_expected)
qc_agg = ncd.variables['data1_qartod_aggregate']
assert qc_agg.standard_name == 'aggregate_quality_flag'
npt.assert_array_equal(qc_agg[:], aggregate_expected)
