def test_run_and_save_to_netcdf(self): # Config is defined as a dict externally, and passed to NcQcConfig c = NcQcConfig(self.config) # Run tests against the input file nc_results = c.run(self.fp) npt.assert_array_equal( nc_results['data1']['qartod']['gross_range_test'], self.expected) # Save results to netcdf file c.save_to_netcdf(self.fp, nc_results) with nc4.Dataset(self.fp, 'r') as ncd: assert 'data1' in list(ncd.variables.keys()) assert 'data1_qartod_gross_range_test' in list( ncd.variables.keys()) qcv = ncd.variables['data1_qartod_gross_range_test'] datav = ncd.variables['data1'] assert datav.standard_name == 'air_temperature' npt.assert_array_equal(datav[:], self.data) assert datav.ancillary_variables == 'data1_qartod_gross_range_test' assert qcv.standard_name == 'gross_range_test_quality_flag' assert qcv.ioos_qc_module == 'qartod' assert qcv.ioos_qc_test == 'gross_range_test' assert qcv.ioos_qc_target == 'data1' assert qcv.ioos_qc_config == json.dumps( self.config['data1']['qartod']['gross_range_test']) npt.assert_array_equal(qcv[:], self.expected) # Now we can update data and run again with the config in the file # Update data with nc4.Dataset(self.fp, 'r+') as ncd_upd: data1_upd = ncd_upd.variables['data1'] data1_upd[:] = np.append(data1_upd[:], 13) assert len(datav) == 14 assert len(ncd_upd['data1_qartod_gross_range_test']) == 14 assert np.ma.is_masked( ncd_upd['data1_qartod_gross_range_test'][13]) # Run tests again. This will re-use the config saved to the netcdf c_upd = NcQcConfig(self.fp) nc_results_upd = c_upd.run(self.fp) npt.assert_array_equal( nc_results_upd['data1']['qartod']['gross_range_test'], self.expected + [4]) c_upd.save_to_netcdf(self.fp, nc_results_upd) with nc4.Dataset(self.fp, 'r') as ncd_final: assert ncd_final['data1_qartod_gross_range_test'][13] == 4
def test_running_save_nc(self): c = NcQcConfig(self.fp) ncresults = c.run(self.fp) c.save_to_netcdf(self.fp, ncresults) with nc4.Dataset(self.fp) as ncd: assert 'data1' in ncd.variables assert 'qc1' in ncd.variables qcv = ncd.variables['qc1'] datav = ncd.variables['data1'] assert datav.ancillary_variables == 'qc1' assert datav.standard_name == 'air_temperature' npt.assert_array_equal( datav[:], self.data ) assert qcv.standard_name == 'status_flag' assert qcv.ioos_qc_module == 'qartod' assert qcv.ioos_qc_test == 'gross_range_test' assert qcv.ioos_qc_target == 'data1' assert qcv.ioos_qc_config == json.dumps(self.config) npt.assert_array_equal( qcv[:], self.expected )
def test_running_climatology_save_netcdf(self): qc = NcQcConfig(self.config) ncresults = qc.run( self.fp, data1={ 'tinp': self.times, 'zinp': self.depths } ) npt.assert_array_equal( ncresults['data1']['qartod']['climatology_test'], self.climate_expected ) npt.assert_array_equal( ncresults['data1']['qartod']['gross_range_test'], self.gross_expected ) qc.save_to_netcdf(self.fp, ncresults) with nc4.Dataset(self.fp) as ncd: assert 'data1' in ncd.variables assert 'qc1' in ncd.variables assert 'qc2' in ncd.variables datav = ncd.variables['data1'] assert sorted(datav.ancillary_variables.split(' ')) == ['qc1', 'qc2'] assert datav.standard_name == 'air_temperature' npt.assert_array_equal( datav[:], self.values ) qc1 = ncd.variables['qc1'] assert qc1.standard_name == 'status_flag' assert qc1.ioos_qc_module == 'qartod' assert qc1.ioos_qc_test == 'gross_range_test' assert qc1.ioos_qc_target == 'data1' assert qc1.ioos_qc_config == json.dumps(self.gross_config, cls=GeoNumpyDateEncoder) npt.assert_array_equal( qc1[:], self.gross_expected ) qc2 = ncd.variables['qc2'] assert qc2.standard_name == 'status_flag' assert qc2.ioos_qc_module == 'qartod' assert qc2.ioos_qc_test == 'climatology_test' assert qc2.ioos_qc_target == 'data1' assert qc2.ioos_qc_config == json.dumps(self.climatology_config, cls=GeoNumpyDateEncoder) npt.assert_array_equal( qc2[:], self.climate_expected )
def test_comparing_nc_and_qc_from_nc(self): c = NcQcConfig(self.fp) ncresults = c.run(self.fp) qcr = QcConfig(c.config['data1']) result = qcr.run(inp=list(range(13))) npt.assert_array_equal( ncresults['data1']['qartod']['gross_range_test'], result['qartod']['gross_range_test'], self.expected)
def test_load_climatology_from_netcdf(self): qc = NcQcConfig(self.fp, tinp='time', zinp='depth') ncresults = qc.run(self.fp) npt.assert_array_equal( ncresults['data1']['qartod']['climatology_test'], self.climate_expected) npt.assert_array_equal( ncresults['data1']['qartod']['gross_range_test'], self.gross_expected)
def test_comparing_nc_and_qc_config(self): # Compare results from QcConfig to those from NcQcConfig nc_config = NcQcConfig(self.config) nc_results = nc_config.run(self.fp) qc_config = QcConfig(self.config['data1']) qc_results = qc_config.run(inp=self.data) npt.assert_array_equal( nc_results['data1']['qartod']['gross_range_test'], qc_results['qartod']['gross_range_test'], self.expected)
def test_load_climatology_from_netcdf(self): qc = NcQcConfig(self.fp) ncresults = qc.run(self.fp, data1={ 'tinp': np.array(self.times), 'zinp': self.depths }) npt.assert_array_equal( ncresults['data1']['qartod']['climatology_test'], self.climate_expected) npt.assert_array_equal( ncresults['data1']['qartod']['gross_range_test'], self.gross_expected)
def process_data(row, dest_dir='.', config=None): """ Apply standard processing method and QAQC to the CTD time series.""" if row['Link to Raw Data'] is None: return file_output = dest_dir + row['file_name'] # Read Seabird CNV print('Read ' + row['raw_file_path']) c = fCNV(row['raw_file_path']) # Save to NetCDF print('Save to ' + row['file_name'] + '_L0.nc') l0_file = path.join(dest_dir, row['file_name'] + '_L0.nc') l1_file = path.join(dest_dir, row['file_name'] + '_L1.nc') cnv2nc(c, l0_file) # Add Metadata to NetCDF # + Metadata Variables # + Global Attributes nc = netCDF4.Dataset(l0_file, 'a') # Latitude latitude = nc.createVariable('latitude', float) latitude.units = 'degrees_north' latitude[:] = row['Latitude'] # Longitude longitude = nc.createVariable('longitude', float) longitude.units = 'degrees_east' longitude[:] = row['Longitude'] # Station station = nc.createVariable('station', str) station[0] = row['Site'] # File name (timeseries_id) # TODO this is temporary I would prefer having access to the instrument serial number instead file_name = nc.createVariable('file_id', str) file_name[0] = row['file_name'] # TODO Add timeseries_id and cdm_data_type info but it may be better to rely on the # seabird tool for doing that # timeseries_id Can be generated by the tool itself # Add extra metadata as global attributes for key, value in row.drop(['Latitude', 'Longitude']).items(): if value: if type(value) is not (float, int, str): value = str(value) # Likely datetime nc.setncattr(key.split('(')[0].strip(), value) # Keep anything before ( # Find Crop data to keep in water only ds = xr.open_dataset(l0_file) start_end_results = process.detect_start_end(ds, 'time', 'PRESPR01', figure_path=file_output + '_crop.png') # Output Cropped time series a L1 ds = ds.loc[dict(time=slice(start_end_results['first_good_record_time'], start_end_results['last_good_record_time']))] ds.to_netcdf(l1_file) # Run QARTOD on the NetCDF file # Retrieve Hakai QARTOD Tests if not config: config = get_ctd_qc_config() # Use deprecated NcQcConfig qc = NcQcConfig(config, tinp='time') qartod_results = qc.run(l1_file) # Upload QARTOD Flags to NetCDF qc.save_to_netcdf(l1_file, qartod_results) # Move away from the NcQcConfig method temporary (hopefully we'll use the streams method soon. # ds = process.run_qartod(ds, config) # ds.to_netcdf(l1_file) return {'l0': l0_file, 'l1': l1_file}
def test_run(self): # setup data config = { 'data1': { 'qartod': { 'gross_range_test': { 'suspect_span': [2, 10], 'fail_span': [0, 12], }, 'flat_line_test': { 'suspect_threshold': 10, 'fail_threshold': 100, }, 'location_test': { 'bbox': [-80, 40, -70, 60] }, 'aggregate': {} } } } time_vals = [1, 2, 3, 4, 5] data_vals = [1, 1, 2, 3, 4] lon_vals = [80, -78, -71, -79, 500] lat_vals = [50, 50, 59, 10, -60] gross_range_expected = [3, 3, 1, 1, 1] flat_line_expected = [1, 1, 1, 1, 1] location_expected = [4, 1, 1, 4, 4] aggregate_expected = [4, 3, 1, 4, 4] with nc4.Dataset(self.fp, 'w') as ncd: ncd.createDimension('time') ncd.createDimension('lon') ncd.createDimension('lat') time = ncd.createVariable('time', 'i4', ('time', )) time[:] = time_vals lon = ncd.createVariable('lon', 'f8', ('time', )) lon[:] = lon_vals lat = ncd.createVariable('lat', 'f8', ('time', )) lat[:] = lat_vals data1 = ncd.createVariable('data1', 'f8', ('time', )) data1.standard_name = 'air_temperature' data1[:] = data_vals # run tests c = NcQcConfig(config, tinp='time', lon='lon', lat='lat') nc_results = c.run(self.fp) npt.assert_array_equal( nc_results['data1']['qartod']['gross_range_test'], gross_range_expected) npt.assert_array_equal(nc_results['data1']['qartod']['flat_line_test'], flat_line_expected) npt.assert_array_equal(nc_results['data1']['qartod']['location_test'], location_expected) npt.assert_array_equal(nc_results['data1']['qartod']['aggregate'], aggregate_expected) # Save results to netcdf file c.save_to_netcdf(self.fp, nc_results) with nc4.Dataset(self.fp, 'r') as ncd: assert 'data1' in ncd.variables assert 'data1_qartod_gross_range_test' in ncd.variables assert 'data1_qartod_flat_line_test' in ncd.variables assert 'data1_qartod_location_test' in ncd.variables assert 'data1_qartod_aggregate' in ncd.variables datav = ncd.variables['data1'] av = datav.ancillary_variables assert set(av.split(' ')) == set([ 'data1_qartod_gross_range_test', 'data1_qartod_flat_line_test', 'data1_qartod_location_test', 'data1_qartod_aggregate' ]) qc_grt = ncd.variables['data1_qartod_gross_range_test'] assert qc_grt.standard_name == 'gross_range_test_quality_flag' npt.assert_array_equal(qc_grt[:], gross_range_expected) qc_flt = ncd.variables['data1_qartod_flat_line_test'] assert qc_flt.standard_name == 'flat_line_test_quality_flag' npt.assert_array_equal(qc_flt[:], flat_line_expected) qc_loc = ncd.variables['data1_qartod_location_test'] assert qc_loc.standard_name == 'location_test_quality_flag' npt.assert_array_equal(qc_loc[:], location_expected) qc_agg = ncd.variables['data1_qartod_aggregate'] assert qc_agg.standard_name == 'aggregate_quality_flag' npt.assert_array_equal(qc_agg[:], aggregate_expected)