def test(p): '''Return quality control decisions. ''' # The test is run on re-ordered data. nlevels, z, t = ICDC.reordered_data(p) # Define default QC. defaultqc = np.zeros(p.n_levels(), dtype=bool) # No check for the Caspian Sea or Great Lakes. lat = p.latitude() lon = p.longitude() if ((lat >= 35.0 and lat <= 45.0 and lon >= 45.0 and lon <= 60.0) or (lat >= 40.0 and lat <= 50.0 and lon >= -95.0 and lon <= -75.0)): return defaultqc # Get range. ranges = get_climatology_range(nlevels, z, lat, lon, p.month()) if ranges is None: return defaultqc # Perform the QC. tmin, tmax = ranges qc = ((t < tmin) | (t > tmax)) & (tmin != fillValue) & (tmax != fillValue) return ICDC.revert_qc_order(p, qc)
def test(p, parameters): '''Return quality control decisions. ''' # The test is run on re-ordered data. nlevels, z, t = ICDC.reordered_data(p) # Define default QC. defaultqc = np.zeros(p.n_levels(), dtype=bool) # No check for the Caspian Sea or Great Lakes. lat = p.latitude() lon = p.longitude() if ((lat >= 35.0 and lat <= 45.0 and lon >= 45.0 and lon <= 60.0) or (lat >= 40.0 and lat <= 50.0 and lon >= -95.0 and lon <= -75.0)): return defaultqc # parameters nc = Dataset('data/climatological_t_median_and_amd_for_aqc.nc', 'r') # Get range. ranges = get_climatology_range(nlevels, z, lat, lon, p.month(), nc) if ranges is None: return defaultqc # Perform the QC. tmin, tmax = ranges qc = ((t < tmin) | (t > tmax)) & (tmin != nc.fillValue) & (tmax != nc.fillValue) return ICDC.revert_qc_order(p, qc)
def test(p, parameters): '''Return quality control decisions. ''' # The test is run on re-ordered data. nlevels, z, t = ICDC.reordered_data(p) qc = np.zeros( nlevels, dtype=bool) # Reordered data may be a subset of available levels. defaultqc = np.zeros( p.n_levels(), dtype=bool) # Default QC flags for full set of levels. if nlevels < 3: return defaultqc # Not enough levels to check. # Ignore any levels outside of limits. parminover = -2.3 parmaxover = 33.0 use = (t > parminover) & (t < parmaxover) nuse = np.count_nonzero(use) if nuse < 3: return defaultqc zuse = z[use] tuse = t[use] origlevels = (np.arange(nlevels))[use] # Extract sections of the arrays. We are QCing the values # in the z2 and v3 arrays. z1 = zuse[0:-2] z2 = zuse[1:-1] z3 = zuse[2:] v1 = tuse[0:-2] v2 = tuse[1:-1] v3 = tuse[2:] ol = origlevels[1:-1] # Calculate the level of 'spike'. z13 = z3 - z1 z12 = z2 - z1 z23 = z3 - z2 a = 0.5 * (v1 + v3) q1 = np.abs(v2 - a) q2 = np.abs(0.5 * (v3 - v1)) spike = q1 - q2 # Define the threshold at each level. spikemax = np.ndarray(nuse - 2) spikemax[:] = 4.0 spikemax[z2 > 1000.0] = 3.0 spikemax[z2 > 2000.0] = 2.0 # Set QC flags. qc[ol[spike > spikemax]] = True return ICDC.revert_qc_order(p, qc)
def test(p, parameters): '''Return quality control decisions. ''' # The test is run on re-ordered data. nlevels, z, t = ICDC.reordered_data(p) qc = np.zeros(nlevels, dtype=bool) # Reordered data may be a subset of available levels. defaultqc = np.zeros(p.n_levels(), dtype=bool) # Default QC flags for full set of levels. if nlevels < 3: return defaultqc # Not enough levels to check. # Ignore any levels outside of limits. parminover = -2.3 parmaxover = 33.0 use = (t > parminover) & (t < parmaxover) nuse = np.count_nonzero(use) if nuse < 3: return defaultqc zuse = z[use] tuse = t[use] origlevels = (np.arange(nlevels))[use] # Extract sections of the arrays. We are QCing the values # in the z2 and v3 arrays. z1 = zuse[0:-2] z2 = zuse[1:-1] z3 = zuse[2:] v1 = tuse[0:-2] v2 = tuse[1:-1] v3 = tuse[2:] ol = origlevels[1:-1] # Calculate the level of 'spike'. z13 = z3 - z1 z12 = z2 - z1 z23 = z3 - z2 a = 0.5 * (v1 + v3) q1 = np.abs(v2 - a) q2 = np.abs(0.5 * (v3 - v1)) spike = q1 - q2 # Define the threshold at each level. spikemax = np.ndarray(nuse - 2) spikemax[:] = 4.0 spikemax[z2 > 1000.0] = 3.0 spikemax[z2 > 2000.0] = 2.0 # Set QC flags. qc[ol[spike > spikemax]] = True return ICDC.revert_qc_order(p, qc)
def test(p): '''Return quality control decisions. ''' # Default set of QC flags to return. qc = np.zeros(p.n_levels(), dtype=bool) # Set minimum allowed levels. db = wod_database(p) if db == 'OSD': minlevs = 7 elif db == 'CTD': minlevs = 50 elif db == 'PFL': minlevs = 20 elif db == 'APB': minlevs = 20 elif db == 'MBT': minlevs = 7 elif db == 'XBT': minlevs = 20 else: return qc # Do not have the information to QC other types. # Check that we have the levels we need. nlevels, z, t = ICDC.reordered_data(p) if nlevels <= minlevs: return qc # Count stuck values. n = np.ones(nlevels, dtype=int) for i in range(nlevels - minlevs): for j in range(i + 1, nlevels): diff = np.abs(t[i] - t[j]) if diff > 0.0001: break n[i] += 1 # Find the largest stuck value range. i = np.argmax(n) if n[i] < minlevs: return qc thick = z[i + n[i] - 1] - z[i] if thick >= 200.0: # If setting the QC flags we need to be careful about level order. qclo = qc[0:nlevels] qclo[i:i + n[i]] = True qc = ICDC.revert_qc_order(p, qclo) return qc
def test(p, parameters): '''Return quality control decisions. ''' # Default set of QC flags to return. qc = np.zeros(p.n_levels(), dtype=bool) # Set minimum allowed levels. db = wod_database(p) if db == 'OSD': minlevs = 7 elif db == 'CTD': minlevs = 50 elif db == 'PFL': minlevs = 20 elif db == 'APB': minlevs = 20 elif db == 'MBT': minlevs = 7 elif db == 'XBT': minlevs = 20 else: return qc # Do not have the information to QC other types. # Check that we have the levels we need. nlevels, z, t = ICDC.reordered_data(p) if nlevels <= minlevs: return qc # Count stuck values. n = np.ones(nlevels, dtype=int) for i in range(nlevels - minlevs): for j in range(i + 1, nlevels): diff = np.abs(t[i] - t[j]) if diff > 0.0001: break n[i] += 1 # Find the largest stuck value range. i = np.argmax(n) if n[i] < minlevs: return qc thick = z[i + n[i] - 1] - z[i] if thick >= 200.0: # If setting the QC flags we need to be careful about level order. qclo = qc[0:nlevels] qclo[i:i+n[i]] = True qc = ICDC.revert_qc_order(p, qclo) return qc
def test(p, parameters): '''Return a set of QC decisions. ''' nlevels, z, t = ICDC.reordered_data(p) qc = (t < parminover) | (t > parmaxover) for i, tval in enumerate(t): if qc[i]: continue # Already rejected. zval = z[i] if np.any((tval >= tcrude1) & (tval <= tcrude2) & (zval <= zcrude1) & (zval >= zcrude2)): qc[i] = True return ICDC.revert_qc_order(p, qc)
def test(p): '''Return a set of QC decisions. ''' nlevels, z, t = ICDC.reordered_data(p) qc = (t < parminover) | (t > parmaxover) for i, tval in enumerate(t): if qc[i]: continue # Already rejected. zval = z[i] if np.any((tval >= tcrude1) & (tval <= tcrude2) & (zval <= zcrude1) & (zval >= zcrude2)): qc[i] = True return ICDC.revert_qc_order(p, qc)
def test(p, parameters): '''Return quality control decisions. ''' # Global ranges - data outside these bounds are ignored. parminover = -2.3 parmaxover = 33.0 # The test is run on re-ordered data. nlevels, z, t = ICDC.reordered_data(p) qc = np.zeros(nlevels, dtype=bool) # Calculate gradients and thresholds. z0 = z[0:-1] z1 = z[1:] t0 = t[0:-1] t1 = t[1:] gradients = (t1 - t0) / (z1 - z0) zmean = 0.5 * (z0 + z1) zmean[zmean < 1.0] = 1.0 gradmin = -150.0 / zmean - 0.010 gradmin[gradmin < -4.0] = -4.0 gradmax = 100.0 / zmean + 0.015 gradmax[gradmax > 1.5] = 1.5 # Find where the gradients and outside the thresholds. result = np.where(((gradients < gradmin) | (gradients > gradmax)) & (t0 > parminover) & (t1 > parminover) & (t0 < parmaxover) & (t1 < parmaxover))[0] # Both levels that form the gradient have to be rejected. if len(result) > 0: qc[result] = True qc[result + 1] = True return ICDC.revert_qc_order(p, qc)
def test(p, parameters): '''Return quality control decisions. ''' # Initialise data. qc = np.zeros(p.n_levels(), dtype=bool) parminover = -2.3 parmaxover = 33.0 levminext = 6 deltaext = 0.5 maxextre = 4 # Check that we have the levels we need. nlevels, z, t = ICDC.reordered_data(p) if nlevels <= levminext: return qc # Exclude data outside allowed range. use = (t > parminover) & (t <= parmaxover) nuse = np.count_nonzero(use) if nuse < levminext: return qc z = z[use] t = t[use] # Find and count the extrema. ima = 0 for i in range(1, nuse - 1): pcent = t[i] pa = np.abs(pcent - t[i - 1]) pb = np.abs(pcent - t[i + 1]) pmin = min(pa, pb) if pcent > t[i - 1] and pcent > t[i + 1] and pmin > deltaext: ima += 1 if pcent < t[i - 1] and pcent < t[i + 1] and pmin > deltaext: ima += 1 if ima > maxextre: qc[:] = True return qc
def test(p): '''Return quality control decisions. ''' # Initialise data. qc = np.zeros(p.n_levels(), dtype=bool) parminover = -2.3 parmaxover = 33.0 levminext = 6 deltaext = 0.5 maxextre = 4 # Check that we have the levels we need. nlevels, z, t = ICDC.reordered_data(p) if nlevels <= levminext: return qc # Exclude data outside allowed range. use = (t > parminover) & (t <= parmaxover) nuse = np.count_nonzero(use) if nuse < levminext: return qc z = z[use] t = t[use] # Find and count the extrema. ima = 0 for i in range(1, nuse - 1): pcent = t[i] pa = np.abs(pcent - t[i - 1]) pb = np.abs(pcent - t[i + 1]) pmin = min(pa, pb) if pcent > t[i - 1] and pcent > t[i + 1] and pmin > deltaext: ima += 1 if pcent < t[i - 1] and pcent < t[i + 1] and pmin > deltaext: ima += 1 if ima > maxextre: qc[:] = True return qc