def referenceResults_example_test(self): ''' make sure main.referenceResults is extacting the correct references from data/example.dat ''' profile = main.extractProfiles(['data/example.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) ref = main.referenceResults([p]) assert ref[0][0] == False, 'incorrect extraction of overall reference result for data/example.dat' assert numpy.array_equal(ref[1][0], [False, False, False, False] ), 'incorrect extraction of verbose reference results for data/example.dat'
def catchFlags_example_test(self): ''' make sure main.catchFlags is flagging temperatures of 99.9 as missing, using the artificial data generated in temp.dat above. ''' profile = main.extractProfiles(['temp.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) main.catchFlags(p) assert p.profile_data[0]['variables'][0]['Missing'], 'failed to flag a temperature of 99.9 as a missing value'
def catchFlags_example_test(self): ''' make sure main.catchFlags is flagging temperatures of 99.9 as missing, using the artificial data generated in temp.dat above. ''' profile = main.extractProfiles(['temp.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) main.catchFlags(p) assert p.profile_data[0]['variables'][0][ 'Missing'], 'failed to flag a temperature of 99.9 as a missing value'
def profileData_example_test(self): ''' continue examining data extracted from pp 137 of http://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf ''' profile = main.extractProfiles(['data/example.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) assert numpy.array_equal(p.z(), [0.0, 10.0, 25.0, 50.0]) assert numpy.array_equal(p.z_level_qc(), [0,0,0,0]) assert numpy.array_equal(p.t(), [8.960, 8.950, 0.900, -1.230]) assert numpy.array_equal(p.t_level_qc(), [0,0,0,0]) assert numpy.array_equal(p.s(), [30.900, 30.900, 31.910, 32.410]) assert numpy.array_equal(p.s_level_qc(), [0,0,0,0])
def referenceResults_example_test(self): ''' make sure main.referenceResults is extacting the correct references from data/example.dat ''' profile = main.extractProfiles(['data/example.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) ref = main.referenceResults([p]) assert ref[0][ 0] == False, 'incorrect extraction of overall reference result for data/example.dat' assert numpy.array_equal( ref[1][0], [False, False, False, False] ), 'incorrect extraction of verbose reference results for data/example.dat'
def profileData_example_test(self): ''' continue examining data extracted from pp 137 of http://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf ''' profile = main.extractProfiles(['data/example.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) assert numpy.array_equal(p.z(), [0.0, 10.0, 25.0, 50.0]) assert numpy.array_equal(p.z_level_qc(), [0, 0, 0, 0]) assert numpy.array_equal(p.t(), [8.960, 8.950, 0.900, -1.230]) assert numpy.array_equal(p.t_level_qc(), [0, 0, 0, 0]) assert numpy.array_equal(p.s(), [30.900, 30.900, 31.910, 32.410]) assert numpy.array_equal(p.s_level_qc(), [0, 0, 0, 0])
def processFile(fName): # run each test on each profile, and record its summary & verbose performance testResults = [] testVerbose = [] trueResults = [] trueVerbose = [] profileIDs = [] firstProfile = True currentFile = '' f = None # keep a list of only the profiles in this thread data.ds.threadProfiles = main.extractProfiles([fName]) data.ds.threadFile = fName for iprofile, pinfo in enumerate(data.ds.threadProfiles): # Load the profile data. p, currentFile, f = main.profileData(pinfo, currentFile, f) # Check that there are temperature data in the profile, otherwise skip. if p.var_index() is None: continue main.catchFlags(p) if np.sum(p.t().mask == False) == 0: continue # Run each test. for itest, test in enumerate(testNames): result = run(test, [p]) if firstProfile: testResults.append(result[0]) testVerbose.append(result[1]) else: testResults[itest].append(result[0][0]) testVerbose[itest].append(result[1][0]) firstProfile = False # Read the reference result. truth = main.referenceResults([p]) trueResults.append(truth[0][0]) trueVerbose.append(truth[1][0]) profileIDs.append(p.uid()) # testResults[i][j] now contains a flag indicating the exception raised by test i on profile j return trueResults, testResults, profileIDs
def processFile(fName): # run each test on each profile, and record its summary & verbose performance testResults = [] testVerbose = [] trueResults = [] trueVerbose = [] profileIDs = [] firstProfile = True currentFile = '' f = None # keep a list of only the profiles in this thread data.ds.threadProfiles = main.extractProfiles([fName]) for iprofile, pinfo in enumerate(data.ds.threadProfiles): # Load the profile data. p, currentFile, f = main.profileData(pinfo, currentFile, f) # Check that there are temperature data in the profile, otherwise skip. if p.var_index() is None: continue main.catchFlags(p) if np.sum(p.t().mask == False) == 0: continue # Run each test. for itest, test in enumerate(testNames): result = run(test, [p]) if firstProfile: testResults.append(result[0]) testVerbose.append(result[1]) else: testResults[itest].append(result[0][0]) testVerbose[itest].append(result[1][0]) firstProfile = False # Read the reference result. truth = main.referenceResults([p]) trueResults.append(truth[0][0]) trueVerbose.append(truth[1][0]) profileIDs.append(p.uid()) # testResults[i][j] now contains a flag indicating the exception raised by test i on profile j return trueResults, testResults, profileIDs
def test(p): """ Runs the quality control check on profile p and returns a numpy array of quality control decisions with False where the data value has passed the check and True where it failed. """ # Define an array to hold results. qc = np.zeros(p.n_levels(), dtype=bool) # Obtain the obs minus background differences on standard levels. result = stdLevelData(p) if result is None: return qc # Unpack the results. levels, origLevels, assocLevels = result # Retrieve the background and observation error variances. bgev = EN_background_check.bgevStdLevels obev = EN_background_check.auxParam['obev'] #find initial pge pgeData = determine_pge(levels, bgev, obev, p) # Find buddy. profiles = data.ds.profiles minDist = 1000000000.0 iMinDist = None for iProfile, profile in enumerate(profiles): pDist = assessBuddyDistance(p, profile) if pDist is not None and pDist < minDist: minDist = pDist iMinDist = iProfile # Check if we have found a buddy and process if so. if minDist <= 400000: fid = None pBuddy, currentFile, fid = main.profileData(profiles[iMinDist], '', fid) fid.close() # buddy vetos Fail = False if pBuddy.var_index() is None: Fail = True if Fail == False: main.catchFlags(pBuddy) if np.sum(pBuddy.t().mask == False) == 0: Fail = True if Fail == False: result = stdLevelData(pBuddy) if result is not None: levelsBuddy, origLevelsBuddy, assocLevelsBuddy = result bgevBuddy = EN_background_check.bgevStdLevels pgeBuddy = determine_pge(levels, bgevBuddy, obev, pBuddy) pgeData = update_pgeData(pgeData, pgeBuddy, levels, levelsBuddy, minDist, p, pBuddy, obev, bgev, bgevBuddy) # Assign the QC flags. for i, pge in enumerate(pgeData): if pgeData.mask[i]: continue if pge < 0.5: continue for j, assocLevel in enumerate(assocLevels): if assocLevel == i: origLevel = origLevels[j] qc[origLevel] = True return qc