def test_apply_qc(self):
self.addCleanup(self.reload_file)
with Dataset(STATIC_FILES['leorgn'], 'r+') as nc:
qc = DatasetQC(nc, self.config_path)
variable = nc.variables['blue_green_algae']
for qcvarname in qc.create_qc_variables(variable):
qcvar = nc.variables[qcvarname]
qc.apply_qc(qcvar)
qc.apply_primary_qc(variable)
def run_qc(config, ncfile):
'''
Runs QC on a netCDF file
'''
qc = DatasetQC(ncfile, config)
for varname in qc.find_geophysical_variables():
get_logger().info("Applying QC to %s", varname)
ncvar = ncfile.variables[varname]
for qcvarname in qc.create_qc_variables(ncvar):
qcvar = ncfile.variables[qcvarname]
get_logger().info(qcvarname)
qc.apply_qc(qcvar)
get_logger().info("Primary QC")
qc.apply_primary_qc(ncvar)
def run_qc(config, ncfile, qc_extension='ncq'):
'''
Runs QC on a netCDF file
Takes a path to an Excel file to be read or pandas DataFrame containing
QC configuration and applies the QC to the file at `filepath`.
Creates a file with the same base name as filepath, but with the file
extension specified by `qc_extension`.
:param config: str or pandas.DataFrame
:param ncfile: netCDF4.Dataset
:param qc_extension: str
'''
fname_base = ncfile.filepath().rsplit('.', 1)[0]
qc_filename = "{}.{}".format(fname_base, qc_extension)
# Look for existing qc_file or return new one
qc_file = create_or_open_qc_file(qc_filename, ncfile.dimensions)
# load NcML aggregation if it exists
ncml_filename = fname_base + '.ncml'
qc = DatasetQC(ncfile, qc_file, ncml_filename, config)
# zero length times will throw an IndexError in the netCDF interface,
# and won't result in any QC being applied anyways, so skip them if present
if qc.ncfile.variables['time'].size > 0:
for varname in qc.find_geophysical_variables():
get_logger().info("Applying QC to %s", varname)
ncvar = ncfile.variables[varname]
for qcvarname in qc.create_qc_variables(ncvar):
qcvar = qc_file.variables[qcvarname]
get_logger().info(qcvarname)
qc.apply_qc(qcvar)
get_logger().info("Primary QC")
qc.apply_primary_qc(ncvar)
# if there were changes in the ncml file, write them
if qc.ncml_write_flag:
with open(ncml_filename, 'w') as ncml_file:
ncml_file.write(etree.tostring(qc.ncml))
qc_file.close()