def process_row(uid, logdir): '''run all tests on the indicated database row''' # reroute stdout, stderr to separate files for each profile to preserve logs sys.stdout = open(logdir + "/" + str(uid) + ".stdout", "w") sys.stderr = open(logdir + "/" + str(uid) + ".stderr", "w") # extract profile profile = main.get_profile_from_db(uid) # mask out error codes in temperature data main.catchFlags(profile) # run tests for itest, test in enumerate(testNames): try: result = run(test, [profile], parameterStore)[0] except: print test, 'exception', sys.exc_info() result = np.zeros(1, dtype=bool) try: query = "UPDATE " + sys.argv[1] + " SET " + test + "=? WHERE uid=" + str(profile.uid()) + ";" main.dbinteract(query, [main.pack_array(result)]) except: print 'db exception', sys.exc_info()
def process_row(uid, logdir): '''run all tests on the indicated database row''' # reroute stdout, stderr to separate files for each profile to preserve logs sys.stdout = open(logdir + "/" + str(uid) + ".stdout", "w") sys.stderr = open(logdir + "/" + str(uid) + ".stderr", "w") # extract profile profile = main.get_profile_from_db(uid) # mask out error codes in temperature data main.catchFlags(profile) # run tests for itest, test in enumerate(testNames): try: result = run(test, [profile], parameterStore)[0] except: print test, 'exception', sys.exc_info() result = np.zeros(1, dtype=bool) try: query = "UPDATE " + sys.argv[ 1] + " SET " + test + "=? WHERE uid=" + str( profile.uid()) + ";" main.dbinteract(query, [main.pack_array(result)]) except: print 'db exception', sys.exc_info()
def catchFlags_example_test(self): ''' make sure main.catchFlags is flagging temperatures of 99.9 as missing, using the artificial data generated in temp.dat above. ''' profile = main.extractProfiles(['temp.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) main.catchFlags(p) assert p.profile_data[0]['variables'][0]['Missing'], 'failed to flag a temperature of 99.9 as a missing value'
def catchFlags_example_test(self): ''' make sure main.catchFlags is flagging temperatures of 99.9 as missing, using the artificial data generated in temp.dat above. ''' profile = main.extractProfiles(['temp.dat'])[0] current = '' p, current, f = main.profileData(profile, current, None) main.catchFlags(p) assert p.profile_data[0]['variables'][0][ 'Missing'], 'failed to flag a temperature of 99.9 as a missing value'
def processFile(fName): # run each test on each profile, and record its summary & verbose performance testResults = [] testVerbose = [] trueResults = [] trueVerbose = [] profileIDs = [] firstProfile = True currentFile = '' f = None # keep a list of only the profiles in this thread data.ds.threadProfiles = main.extractProfiles([fName]) data.ds.threadFile = fName for iprofile, pinfo in enumerate(data.ds.threadProfiles): # Load the profile data. p, currentFile, f = main.profileData(pinfo, currentFile, f) # Check that there are temperature data in the profile, otherwise skip. if p.var_index() is None: continue main.catchFlags(p) if np.sum(p.t().mask == False) == 0: continue # Run each test. for itest, test in enumerate(testNames): result = run(test, [p]) if firstProfile: testResults.append(result[0]) testVerbose.append(result[1]) else: testResults[itest].append(result[0][0]) testVerbose[itest].append(result[1][0]) firstProfile = False # Read the reference result. truth = main.referenceResults([p]) trueResults.append(truth[0][0]) trueVerbose.append(truth[1][0]) profileIDs.append(p.uid()) # testResults[i][j] now contains a flag indicating the exception raised by test i on profile j return trueResults, testResults, profileIDs
def processFile(fName): # run each test on each profile, and record its summary & verbose performance testResults = [] testVerbose = [] trueResults = [] trueVerbose = [] profileIDs = [] firstProfile = True currentFile = '' f = None # keep a list of only the profiles in this thread data.ds.threadProfiles = main.extractProfiles([fName]) for iprofile, pinfo in enumerate(data.ds.threadProfiles): # Load the profile data. p, currentFile, f = main.profileData(pinfo, currentFile, f) # Check that there are temperature data in the profile, otherwise skip. if p.var_index() is None: continue main.catchFlags(p) if np.sum(p.t().mask == False) == 0: continue # Run each test. for itest, test in enumerate(testNames): result = run(test, [p]) if firstProfile: testResults.append(result[0]) testVerbose.append(result[1]) else: testResults[itest].append(result[0][0]) testVerbose[itest].append(result[1][0]) firstProfile = False # Read the reference result. truth = main.referenceResults([p]) trueResults.append(truth[0][0]) trueVerbose.append(truth[1][0]) profileIDs.append(p.uid()) # testResults[i][j] now contains a flag indicating the exception raised by test i on profile j return trueResults, testResults, profileIDs
def test(p, parameters, allow_level_reinstating=True): """ Runs the quality control check on profile p and returns a numpy array of quality control decisions with False where the data value has passed the check and True where it failed. If allow_level_reinstating is set to True then rejected levels can be reprieved by comparing with levels above and below. NB this is done by default in EN processing. """ # Define an array to hold results. qc = np.zeros(p.n_levels(), dtype=bool) # Obtain the obs minus background differences on standard levels. result = stdLevelData(p, parameters) if result is None: return qc # Unpack the results. levels, origLevels, assocLevels = result # Retrieve the background and observation error variances and # the background values. query = 'SELECT bgstdlevels, bgevstdlevels FROM enbackground WHERE uid = ' + str( p.uid()) enbackground_pars = main.dbinteract(query) enbackground_pars = main.unpack_row(enbackground_pars[0]) bgsl = enbackground_pars[0] slev = parameters['enbackground']['depth'] bgev = enbackground_pars[1] obev = parameters['enbackground']['obev'] #find initial pge pgeData = determine_pge(levels, bgev, obev, p) # Find buddy. profiles = get_profile_info(parameters) minDist = 1000000000.0 iMinDist = None for iProfile, profile in enumerate(profiles): pDist = assessBuddyDistance(p, profile) if pDist is not None and pDist < minDist: minDist = pDist iMinDist = iProfile # Check if we have found a buddy and process if so. if minDist <= 400000: pBuddy = main.get_profile_from_db(profiles[iMinDist][0]) # buddy vetos Fail = False if pBuddy.var_index() is None: Fail = True if Fail == False: main.catchFlags(pBuddy) if np.sum(pBuddy.t().mask == False) == 0: Fail = True if Fail == False: result = stdLevelData(pBuddy, parameters) query = 'SELECT bgevstdlevels FROM enbackground WHERE uid = ' + str( pBuddy.uid()) buddy_pars = main.dbinteract(query) buddy_pars = main.unpack_row(buddy_pars[0]) if result is not None: levelsBuddy, origLevelsBuddy, assocLevelsBuddy = result bgevBuddy = buddy_pars[0] pgeBuddy = determine_pge(levels, bgevBuddy, obev, pBuddy) pgeData = update_pgeData(pgeData, pgeBuddy, levels, levelsBuddy, minDist, p, pBuddy, obev, bgev, bgevBuddy) # Check if levels should be reinstated. if allow_level_reinstating: if np.abs(p.latitude()) < 20.0: depthTol = 300.0 else: depthTol = 200.0 stdLevelFlags = pgeData >= 0.5 for i, slflag in enumerate(stdLevelFlags): if slflag: # Check for non rejected surrounding levels. okbelow = False if i > 0: if stdLevelFlags[i - 1] == False and levels.mask[ i - 1] == False and bgsl.mask[i - 1] == False: okbelow = True okabove = False nsl = len(stdLevelFlags) if i < nsl - 1: if stdLevelFlags[i + 1] == False and levels.mask[ i + 1] == False and bgsl.mask[i + 1] == False: okabove = True # Work out tolerances. if slev[i] > depthTol + 100: tolFactor = 0.5 elif slev[i] > depthTol: tolFactor = 1.0 - 0.005 * (slev[i] - depthTol) else: tolFactor = 1.0 ttol = 0.5 * tolFactor if okbelow == True and okabove == True: xmax = levels[i - 1] + bgsl[i - 1] + ttol xmin = levels[i + 1] + bgsl[i + 1] - ttol elif okbelow == True: xmax = levels[i - 1] + bgsl[i - 1] + ttol xmin = levels[i - 1] + bgsl[i - 1] - ttol elif okabove == True: xmax = levels[i + 1] + bgsl[i + 1] + ttol xmin = levels[i + 1] + bgsl[i + 1] - ttol else: continue # Reassign PGE if level is within the tolerances. if levels[i] + bgsl[i] >= xmin and levels[i] + bgsl[i] <= xmax: pgeData[i] = 0.49 # Assign the QC flags to original levels. for i, pge in enumerate(pgeData): if pgeData.mask[i]: continue if pge < 0.5: continue for j, assocLevel in enumerate(assocLevels): if assocLevel == i: origLevel = origLevels[j] qc[origLevel] = True return qc
def test(p, parameters, allow_level_reinstating=True): """ Runs the quality control check on profile p and returns a numpy array of quality control decisions with False where the data value has passed the check and True where it failed. If allow_level_reinstating is set to True then rejected levels can be reprieved by comparing with levels above and below. NB this is done by default in EN processing. """ # Define an array to hold results. qc = np.zeros(p.n_levels(), dtype=bool) # Obtain the obs minus background differences on standard levels. result = stdLevelData(p, parameters) if result is None: return qc # Unpack the results. levels, origLevels, assocLevels = result # Retrieve the background and observation error variances and # the background values. query = 'SELECT bgstdlevels, bgevstdlevels FROM enbackground WHERE uid = ' + str(p.uid()) enbackground_pars = main.dbinteract(query) enbackground_pars = main.unpack_row(enbackground_pars[0]) bgsl = enbackground_pars[0] slev = parameters['enbackground']['depth'] bgev = enbackground_pars[1] obev = parameters['enbackground']['obev'] #find initial pge pgeData = determine_pge(levels, bgev, obev, p) # Find buddy. profiles = get_profile_info(parameters) minDist = 1000000000.0 iMinDist = None for iProfile, profile in enumerate(profiles): pDist = assessBuddyDistance(p, profile) if pDist is not None and pDist < minDist: minDist = pDist iMinDist = iProfile # Check if we have found a buddy and process if so. if minDist <= 400000: pBuddy = main.get_profile_from_db(profiles[iMinDist][0]) # buddy vetos Fail = False if pBuddy.var_index() is None: Fail = True if Fail == False: main.catchFlags(pBuddy) if np.sum(pBuddy.t().mask == False) == 0: Fail = True if Fail == False: result = stdLevelData(pBuddy, parameters) query = 'SELECT bgevstdlevels FROM enbackground WHERE uid = ' + str(pBuddy.uid()) buddy_pars = main.dbinteract(query) buddy_pars = main.unpack_row(buddy_pars[0]) if result is not None: levelsBuddy, origLevelsBuddy, assocLevelsBuddy = result bgevBuddy = buddy_pars[0] pgeBuddy = determine_pge(levels, bgevBuddy, obev, pBuddy) pgeData = update_pgeData(pgeData, pgeBuddy, levels, levelsBuddy, minDist, p, pBuddy, obev, bgev, bgevBuddy) # Check if levels should be reinstated. if allow_level_reinstating: if np.abs(p.latitude()) < 20.0: depthTol = 300.0 else: depthTol = 200.0 stdLevelFlags = pgeData >= 0.5 for i, slflag in enumerate(stdLevelFlags): if slflag: # Check for non rejected surrounding levels. okbelow = False if i > 0: if stdLevelFlags[i - 1] == False and levels.mask[i - 1] == False and bgsl.mask[i - 1] == False: okbelow = True okabove = False nsl = len(stdLevelFlags) if i < nsl - 1: if stdLevelFlags[i + 1] == False and levels.mask[i + 1] == False and bgsl.mask[i + 1] == False: okabove = True # Work out tolerances. if slev[i] > depthTol + 100: tolFactor = 0.5 elif slev[i] > depthTol: tolFactor = 1.0 - 0.005 * (slev[i] - depthTol) else: tolFactor = 1.0 ttol = 0.5 * tolFactor if okbelow == True and okabove == True: xmax = levels[i - 1] + bgsl[i - 1] + ttol xmin = levels[i + 1] + bgsl[i + 1] - ttol elif okbelow == True: xmax = levels[i - 1] + bgsl[i - 1] + ttol xmin = levels[i - 1] + bgsl[i - 1] - ttol elif okabove == True: xmax = levels[i + 1] + bgsl[i + 1] + ttol xmin = levels[i + 1] + bgsl[i + 1] - ttol else: continue # Reassign PGE if level is within the tolerances. if levels[i] + bgsl[i] >= xmin and levels[i] + bgsl[i] <= xmax: pgeData[i] = 0.49 # Assign the QC flags to original levels. for i, pge in enumerate(pgeData): if pgeData.mask[i]: continue if pge < 0.5: continue for j, assocLevel in enumerate(assocLevels): if assocLevel == i: origLevel = origLevels[j] qc[origLevel] = True return qc
def test(p): """ Runs the quality control check on profile p and returns a numpy array of quality control decisions with False where the data value has passed the check and True where it failed. """ # Define an array to hold results. qc = np.zeros(p.n_levels(), dtype=bool) # Obtain the obs minus background differences on standard levels. result = stdLevelData(p) if result is None: return qc # Unpack the results. levels, origLevels, assocLevels = result # Retrieve the background and observation error variances. bgev = EN_background_check.bgevStdLevels obev = EN_background_check.auxParam['obev'] #find initial pge pgeData = determine_pge(levels, bgev, obev, p) # Find buddy. profiles = data.ds.profiles minDist = 1000000000.0 iMinDist = None for iProfile, profile in enumerate(profiles): pDist = assessBuddyDistance(p, profile) if pDist is not None and pDist < minDist: minDist = pDist iMinDist = iProfile # Check if we have found a buddy and process if so. if minDist <= 400000: fid = None pBuddy, currentFile, fid = main.profileData(profiles[iMinDist], '', fid) fid.close() # buddy vetos Fail = False if pBuddy.var_index() is None: Fail = True if Fail == False: main.catchFlags(pBuddy) if np.sum(pBuddy.t().mask == False) == 0: Fail = True if Fail == False: result = stdLevelData(pBuddy) if result is not None: levelsBuddy, origLevelsBuddy, assocLevelsBuddy = result bgevBuddy = EN_background_check.bgevStdLevels pgeBuddy = determine_pge(levels, bgevBuddy, obev, pBuddy) pgeData = update_pgeData(pgeData, pgeBuddy, levels, levelsBuddy, minDist, p, pBuddy, obev, bgev, bgevBuddy) # Assign the QC flags. for i, pge in enumerate(pgeData): if pgeData.mask[i]: continue if pge < 0.5: continue for j, assocLevel in enumerate(assocLevels): if assocLevel == i: origLevel = origLevels[j] qc[origLevel] = True return qc