def referenceResults_example_test(self):
'''
make sure main.referenceResults is extacting the correct references from data/example.dat
'''
profile = main.extractProfiles(['data/example.dat'])[0]
current = ''
p, current, f = main.profileData(profile, current, None)
ref = main.referenceResults([p])
assert ref[0][0] == False, 'incorrect extraction of overall reference result for data/example.dat'
assert numpy.array_equal(ref[1][0], [False, False, False, False] ), 'incorrect extraction of verbose reference results for data/example.dat'
def catchFlags_example_test(self):
'''
make sure main.catchFlags is flagging temperatures of 99.9 as missing,
using the artificial data generated in temp.dat above.
'''
profile = main.extractProfiles(['temp.dat'])[0]
current = ''
p, current, f = main.profileData(profile, current, None)
main.catchFlags(p)
assert p.profile_data[0]['variables'][0]['Missing'], 'failed to flag a temperature of 99.9 as a missing value'
def catchFlags_example_test(self):
'''
make sure main.catchFlags is flagging temperatures of 99.9 as missing,
using the artificial data generated in temp.dat above.
'''
profile = main.extractProfiles(['temp.dat'])[0]
current = ''
p, current, f = main.profileData(profile, current, None)
main.catchFlags(p)
assert p.profile_data[0]['variables'][0][
'Missing'], 'failed to flag a temperature of 99.9 as a missing value'
def profileData_example_test(self):
'''
continue examining data extracted from pp 137 of
http://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf
'''
profile = main.extractProfiles(['data/example.dat'])[0]
current = ''
p, current, f = main.profileData(profile, current, None)
assert numpy.array_equal(p.z(), [0.0, 10.0, 25.0, 50.0])
assert numpy.array_equal(p.z_level_qc(), [0,0,0,0])
assert numpy.array_equal(p.t(), [8.960, 8.950, 0.900, -1.230])
assert numpy.array_equal(p.t_level_qc(), [0,0,0,0])
assert numpy.array_equal(p.s(), [30.900, 30.900, 31.910, 32.410])
assert numpy.array_equal(p.s_level_qc(), [0,0,0,0])
def referenceResults_example_test(self):
'''
make sure main.referenceResults is extacting the correct references from data/example.dat
'''
profile = main.extractProfiles(['data/example.dat'])[0]
current = ''
p, current, f = main.profileData(profile, current, None)
ref = main.referenceResults([p])
assert ref[0][
0] == False, 'incorrect extraction of overall reference result for data/example.dat'
assert numpy.array_equal(
ref[1][0], [False, False, False, False]
), 'incorrect extraction of verbose reference results for data/example.dat'
def profileData_example_test(self):
'''
continue examining data extracted from pp 137 of
http://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf
'''
profile = main.extractProfiles(['data/example.dat'])[0]
current = ''
p, current, f = main.profileData(profile, current, None)
assert numpy.array_equal(p.z(), [0.0, 10.0, 25.0, 50.0])
assert numpy.array_equal(p.z_level_qc(), [0, 0, 0, 0])
assert numpy.array_equal(p.t(), [8.960, 8.950, 0.900, -1.230])
assert numpy.array_equal(p.t_level_qc(), [0, 0, 0, 0])
assert numpy.array_equal(p.s(), [30.900, 30.900, 31.910, 32.410])
assert numpy.array_equal(p.s_level_qc(), [0, 0, 0, 0])
def processFile(fName):
# run each test on each profile, and record its summary & verbose performance
testResults = []
testVerbose = []
trueResults = []
trueVerbose = []
profileIDs = []
firstProfile = True
currentFile = ''
f = None
# keep a list of only the profiles in this thread
data.ds.threadProfiles = main.extractProfiles([fName])
data.ds.threadFile = fName
for iprofile, pinfo in enumerate(data.ds.threadProfiles):
# Load the profile data.
p, currentFile, f = main.profileData(pinfo, currentFile, f)
# Check that there are temperature data in the profile, otherwise skip.
if p.var_index() is None:
continue
main.catchFlags(p)
if np.sum(p.t().mask == False) == 0:
continue
# Run each test.
for itest, test in enumerate(testNames):
result = run(test, [p])
if firstProfile:
testResults.append(result[0])
testVerbose.append(result[1])
else:
testResults[itest].append(result[0][0])
testVerbose[itest].append(result[1][0])
firstProfile = False
# Read the reference result.
truth = main.referenceResults([p])
trueResults.append(truth[0][0])
trueVerbose.append(truth[1][0])
profileIDs.append(p.uid())
# testResults[i][j] now contains a flag indicating the exception raised by test i on profile j
return trueResults, testResults, profileIDs
def processFile(fName):
# run each test on each profile, and record its summary & verbose performance
testResults = []
testVerbose = []
trueResults = []
trueVerbose = []
profileIDs = []
firstProfile = True
currentFile = ''
f = None
# keep a list of only the profiles in this thread
data.ds.threadProfiles = main.extractProfiles([fName])
for iprofile, pinfo in enumerate(data.ds.threadProfiles):
# Load the profile data.
p, currentFile, f = main.profileData(pinfo, currentFile, f)
# Check that there are temperature data in the profile, otherwise skip.
if p.var_index() is None:
continue
main.catchFlags(p)
if np.sum(p.t().mask == False) == 0:
continue
# Run each test.
for itest, test in enumerate(testNames):
result = run(test, [p])
if firstProfile:
testResults.append(result[0])
testVerbose.append(result[1])
else:
testResults[itest].append(result[0][0])
testVerbose[itest].append(result[1][0])
firstProfile = False
# Read the reference result.
truth = main.referenceResults([p])
trueResults.append(truth[0][0])
trueVerbose.append(truth[1][0])
profileIDs.append(p.uid())
# testResults[i][j] now contains a flag indicating the exception raised by test i on profile j
return trueResults, testResults, profileIDs
def test(p):
"""
Runs the quality control check on profile p and returns a numpy array
of quality control decisions with False where the data value has
passed the check and True where it failed.
"""
# Define an array to hold results.
qc = np.zeros(p.n_levels(), dtype=bool)
# Obtain the obs minus background differences on standard levels.
result = stdLevelData(p)
if result is None: return qc
# Unpack the results.
levels, origLevels, assocLevels = result
# Retrieve the background and observation error variances.
bgev = EN_background_check.bgevStdLevels
obev = EN_background_check.auxParam['obev']
#find initial pge
pgeData = determine_pge(levels, bgev, obev, p)
# Find buddy.
profiles = data.ds.profiles
minDist = 1000000000.0
iMinDist = None
for iProfile, profile in enumerate(profiles):
pDist = assessBuddyDistance(p, profile)
if pDist is not None and pDist < minDist:
minDist = pDist
iMinDist = iProfile
# Check if we have found a buddy and process if so.
if minDist <= 400000:
fid = None
pBuddy, currentFile, fid = main.profileData(profiles[iMinDist], '',
fid)
fid.close()
# buddy vetos
Fail = False
if pBuddy.var_index() is None:
Fail = True
if Fail == False:
main.catchFlags(pBuddy)
if np.sum(pBuddy.t().mask == False) == 0:
Fail = True
if Fail == False:
result = stdLevelData(pBuddy)
if result is not None:
levelsBuddy, origLevelsBuddy, assocLevelsBuddy = result
bgevBuddy = EN_background_check.bgevStdLevels
pgeBuddy = determine_pge(levels, bgevBuddy, obev, pBuddy)
pgeData = update_pgeData(pgeData, pgeBuddy, levels,
levelsBuddy, minDist, p, pBuddy, obev,
bgev, bgevBuddy)
# Assign the QC flags.
for i, pge in enumerate(pgeData):
if pgeData.mask[i]: continue
if pge < 0.5: continue
for j, assocLevel in enumerate(assocLevels):
if assocLevel == i:
origLevel = origLevels[j]
qc[origLevel] = True
return qc
