def test_get_unmasked(self):
with Dataset(STATIC_FILES['leorgn'], 'r') as nc:
qc = DatasetQC(nc, self.config_path)
variable = nc.variables['blue_green_algae']
times, values, mask = qc.get_unmasked(variable)
np.testing.assert_allclose(
values[:4], np.array([0.13, 0.12, 0.13, 0.08], dtype='f'))
def test_loading_config(self):
qc = DatasetQC(None, self.config_path)
assert qc.config is not None
df = qc.config
variables = df[df['station_id'] == 'leorgn'].variable
assert u'blue_green_algae' in variables.tolist()
def test_loading_config(self):
config_path = 'tests/data/GLOS-Climatologies.xlsx'
qc = DatasetQC(None, config_path)
assert qc.config is not None
df = qc.config
variables = df[df['station_id'] == 'leorgn'].variable
assert u'blue_green_algae' in variables.tolist()
def run_qc(config, ncfile):
'''
Runs QC on a netCDF file
'''
qc = DatasetQC(ncfile, config)
for varname in qc.find_geophysical_variables():
get_logger().info("Applying QC to %s", varname)
ncvar = ncfile.variables[varname]
for qcvarname in qc.create_qc_variables(ncvar):
qcvar = ncfile.variables[qcvarname]
get_logger().info(qcvarname)
qc.apply_qc(qcvar)
get_logger().info("Primary QC")
qc.apply_primary_qc(ncvar)
def test_apply_qc(self):
self.addCleanup(self.reload_file)
with Dataset(STATIC_FILES['leorgn'], 'r+') as nc:
qc = DatasetQC(nc, self.config_path)
variable = nc.variables['blue_green_algae']
for qcvarname in qc.create_qc_variables(variable):
qcvar = nc.variables[qcvarname]
qc.apply_qc(qcvar)
qc.apply_primary_qc(variable)
def run_qc(config, ncfile, qc_extension='ncq'):
'''
Runs QC on a netCDF file
Takes a path to an Excel file to be read or pandas DataFrame containing
QC configuration and applies the QC to the file at `filepath`.
Creates a file with the same base name as filepath, but with the file
extension specified by `qc_extension`.
:param config: str or pandas.DataFrame
:param ncfile: netCDF4.Dataset
:param qc_extension: str
'''
fname_base = ncfile.filepath().rsplit('.', 1)[0]
qc_filename = "{}.{}".format(fname_base, qc_extension)
# Look for existing qc_file or return new one
qc_file = create_or_open_qc_file(qc_filename, ncfile.dimensions)
# load NcML aggregation if it exists
ncml_filename = fname_base + '.ncml'
qc = DatasetQC(ncfile, qc_file, ncml_filename, config)
# zero length times will throw an IndexError in the netCDF interface,
# and won't result in any QC being applied anyways, so skip them if present
if qc.ncfile.variables['time'].size > 0:
for varname in qc.find_geophysical_variables():
get_logger().info("Applying QC to %s", varname)
ncvar = ncfile.variables[varname]
for qcvarname in qc.create_qc_variables(ncvar):
qcvar = qc_file.variables[qcvarname]
get_logger().info(qcvarname)
qc.apply_qc(qcvar)
get_logger().info("Primary QC")
qc.apply_primary_qc(ncvar)
# if there were changes in the ncml file, write them
if qc.ncml_write_flag:
with open(ncml_filename, 'w') as ncml_file:
ncml_file.write(etree.tostring(qc.ncml))
qc_file.close()
def test_get_config(self):
with Dataset(STATIC_FILES['leorgn'], 'r') as nc:
qc = DatasetQC(nc, self.config_path)
config = qc.get_config('blue_green_algae')
assert config.units == u'rfu'
assert config['gross_range.sensor_min'] == -1
def test_find_geophysical_variables(self):
with Dataset(STATIC_FILES['leorgn'], 'r') as nc:
qc = DatasetQC(nc, self.config_path)
variables = qc.find_geophysical_variables()
assert u'blue_green_algae' in variables
def test_find_station_name(self):
with Dataset(STATIC_FILES['leorgn'], 'r') as nc:
qc = DatasetQC(nc, self.config_path)
station_name = qc.find_station_name()
assert station_name == "urn:ioos:station:glos:leorgn"