Example #1
0
def process_row(uid, logdir):
  '''run all tests on the indicated database row'''
  
  # reroute stdout, stderr to separate files for each profile to preserve logs
  sys.stdout = open(logdir + "/" + str(uid) + ".stdout", "w")
  sys.stderr = open(logdir + "/" + str(uid) + ".stderr", "w")

  # extract profile
  profile = main.get_profile_from_db(uid)
  
  # mask out error codes in temperature data
  main.catchFlags(profile)

  # run tests
  for itest, test in enumerate(testNames):
    try:
      result = run(test, [profile], parameterStore)[0]
    except:
      print test, 'exception', sys.exc_info()
      result = np.zeros(1, dtype=bool)

    try:
      query = "UPDATE " + sys.argv[1] + " SET " + test + "=? WHERE uid=" + str(profile.uid()) + ";"
      main.dbinteract(query, [main.pack_array(result)])
    except:
      print 'db exception', sys.exc_info()
Example #2
0
def process_row(uid, logdir):
    '''run all tests on the indicated database row'''

    # reroute stdout, stderr to separate files for each profile to preserve logs
    sys.stdout = open(logdir + "/" + str(uid) + ".stdout", "w")
    sys.stderr = open(logdir + "/" + str(uid) + ".stderr", "w")

    # extract profile
    profile = main.get_profile_from_db(uid)

    # mask out error codes in temperature data
    main.catchFlags(profile)

    # run tests
    for itest, test in enumerate(testNames):
        try:
            result = run(test, [profile], parameterStore)[0]
        except:
            print test, 'exception', sys.exc_info()
            result = np.zeros(1, dtype=bool)

        try:
            query = "UPDATE " + sys.argv[
                1] + " SET " + test + "=? WHERE uid=" + str(
                    profile.uid()) + ";"
            main.dbinteract(query, [main.pack_array(result)])
        except:
            print 'db exception', sys.exc_info()
Example #3
0
    def catchFlags_example_test(self):
        '''
        make sure main.catchFlags is flagging temperatures of 99.9 as missing,
        using the artificial data generated in temp.dat above.
        '''

        profile = main.extractProfiles(['temp.dat'])[0]
        current = ''
        p, current, f = main.profileData(profile, current, None)

        main.catchFlags(p)

        assert p.profile_data[0]['variables'][0]['Missing'], 'failed to flag a temperature of 99.9 as a missing value'
Example #4
0
    def catchFlags_example_test(self):
        '''
        make sure main.catchFlags is flagging temperatures of 99.9 as missing,
        using the artificial data generated in temp.dat above.
        '''

        profile = main.extractProfiles(['temp.dat'])[0]
        current = ''
        p, current, f = main.profileData(profile, current, None)

        main.catchFlags(p)

        assert p.profile_data[0]['variables'][0][
            'Missing'], 'failed to flag a temperature of 99.9 as a missing value'
Example #5
0
def processFile(fName):
  # run each test on each profile, and record its summary & verbose performance
  testResults  = []
  testVerbose  = []
  trueResults  = []
  trueVerbose  = []
  profileIDs   = []
  firstProfile = True
  currentFile  = ''
  f = None

  # keep a list of only the profiles in this thread
  data.ds.threadProfiles = main.extractProfiles([fName])
  data.ds.threadFile     = fName

  for iprofile, pinfo in enumerate(data.ds.threadProfiles):
    # Load the profile data.
    p, currentFile, f = main.profileData(pinfo, currentFile, f)
    # Check that there are temperature data in the profile, otherwise skip.
    if p.var_index() is None:
      continue
    main.catchFlags(p)
    if np.sum(p.t().mask == False) == 0:
      continue
    # Run each test.    
    for itest, test in enumerate(testNames):
      result = run(test, [p])
      if firstProfile:
        testResults.append(result[0])
        testVerbose.append(result[1])
      else:
        testResults[itest].append(result[0][0])
        testVerbose[itest].append(result[1][0])
    firstProfile = False
    # Read the reference result.
    truth = main.referenceResults([p])
    trueResults.append(truth[0][0])
    trueVerbose.append(truth[1][0])
    profileIDs.append(p.uid())
  # testResults[i][j] now contains a flag indicating the exception raised by test i on profile j

  return trueResults, testResults, profileIDs
Example #6
0
def processFile(fName):
    # run each test on each profile, and record its summary & verbose performance
    testResults = []
    testVerbose = []
    trueResults = []
    trueVerbose = []
    profileIDs = []
    firstProfile = True
    currentFile = ''
    f = None

    # keep a list of only the profiles in this thread
    data.ds.threadProfiles = main.extractProfiles([fName])

    for iprofile, pinfo in enumerate(data.ds.threadProfiles):
        # Load the profile data.
        p, currentFile, f = main.profileData(pinfo, currentFile, f)
        # Check that there are temperature data in the profile, otherwise skip.
        if p.var_index() is None:
            continue
        main.catchFlags(p)
        if np.sum(p.t().mask == False) == 0:
            continue
        # Run each test.
        for itest, test in enumerate(testNames):
            result = run(test, [p])
            if firstProfile:
                testResults.append(result[0])
                testVerbose.append(result[1])
            else:
                testResults[itest].append(result[0][0])
                testVerbose[itest].append(result[1][0])
        firstProfile = False
        # Read the reference result.
        truth = main.referenceResults([p])
        trueResults.append(truth[0][0])
        trueVerbose.append(truth[1][0])
        profileIDs.append(p.uid())
    # testResults[i][j] now contains a flag indicating the exception raised by test i on profile j

    return trueResults, testResults, profileIDs
Example #7
0
def test(p, parameters, allow_level_reinstating=True):
    """ 
    Runs the quality control check on profile p and returns a numpy array 
    of quality control decisions with False where the data value has 
    passed the check and True where it failed. 

    If allow_level_reinstating is set to True then rejected levels can be
    reprieved by comparing with levels above and below. NB this is done by
    default in EN processing.
    """

    # Define an array to hold results.
    qc = np.zeros(p.n_levels(), dtype=bool)

    # Obtain the obs minus background differences on standard levels.
    result = stdLevelData(p, parameters)
    if result is None:
        return qc

    # Unpack the results.
    levels, origLevels, assocLevels = result
    # Retrieve the background and observation error variances and
    # the background values.
    query = 'SELECT bgstdlevels, bgevstdlevels FROM enbackground WHERE uid = ' + str(
        p.uid())
    enbackground_pars = main.dbinteract(query)
    enbackground_pars = main.unpack_row(enbackground_pars[0])
    bgsl = enbackground_pars[0]
    slev = parameters['enbackground']['depth']
    bgev = enbackground_pars[1]
    obev = parameters['enbackground']['obev']

    #find initial pge
    pgeData = determine_pge(levels, bgev, obev, p)

    # Find buddy.
    profiles = get_profile_info(parameters)
    minDist = 1000000000.0
    iMinDist = None
    for iProfile, profile in enumerate(profiles):
        pDist = assessBuddyDistance(p, profile)
        if pDist is not None and pDist < minDist:
            minDist = pDist
            iMinDist = iProfile

    # Check if we have found a buddy and process if so.
    if minDist <= 400000:
        pBuddy = main.get_profile_from_db(profiles[iMinDist][0])

        # buddy vetos
        Fail = False
        if pBuddy.var_index() is None:
            Fail = True
        if Fail == False:
            main.catchFlags(pBuddy)
            if np.sum(pBuddy.t().mask == False) == 0:
                Fail = True

        if Fail == False:

            result = stdLevelData(pBuddy, parameters)

            query = 'SELECT bgevstdlevels FROM enbackground WHERE uid = ' + str(
                pBuddy.uid())
            buddy_pars = main.dbinteract(query)

            buddy_pars = main.unpack_row(buddy_pars[0])

            if result is not None:
                levelsBuddy, origLevelsBuddy, assocLevelsBuddy = result
                bgevBuddy = buddy_pars[0]
                pgeBuddy = determine_pge(levels, bgevBuddy, obev, pBuddy)
                pgeData = update_pgeData(pgeData, pgeBuddy, levels,
                                         levelsBuddy, minDist, p, pBuddy, obev,
                                         bgev, bgevBuddy)

    # Check if levels should be reinstated.
    if allow_level_reinstating:
        if np.abs(p.latitude()) < 20.0:
            depthTol = 300.0
        else:
            depthTol = 200.0
        stdLevelFlags = pgeData >= 0.5
        for i, slflag in enumerate(stdLevelFlags):
            if slflag:
                # Check for non rejected surrounding levels.
                okbelow = False
                if i > 0:
                    if stdLevelFlags[i - 1] == False and levels.mask[
                            i - 1] == False and bgsl.mask[i - 1] == False:
                        okbelow = True
                okabove = False
                nsl = len(stdLevelFlags)
                if i < nsl - 1:
                    if stdLevelFlags[i + 1] == False and levels.mask[
                            i + 1] == False and bgsl.mask[i + 1] == False:
                        okabove = True
                # Work out tolerances.
                if slev[i] > depthTol + 100:
                    tolFactor = 0.5
                elif slev[i] > depthTol:
                    tolFactor = 1.0 - 0.005 * (slev[i] - depthTol)
                else:
                    tolFactor = 1.0
                ttol = 0.5 * tolFactor
                if okbelow == True and okabove == True:
                    xmax = levels[i - 1] + bgsl[i - 1] + ttol
                    xmin = levels[i + 1] + bgsl[i + 1] - ttol
                elif okbelow == True:
                    xmax = levels[i - 1] + bgsl[i - 1] + ttol
                    xmin = levels[i - 1] + bgsl[i - 1] - ttol
                elif okabove == True:
                    xmax = levels[i + 1] + bgsl[i + 1] + ttol
                    xmin = levels[i + 1] + bgsl[i + 1] - ttol
                else:
                    continue
                # Reassign PGE if level is within the tolerances.
                if levels[i] + bgsl[i] >= xmin and levels[i] + bgsl[i] <= xmax:
                    pgeData[i] = 0.49

    # Assign the QC flags to original levels.
    for i, pge in enumerate(pgeData):
        if pgeData.mask[i]: continue
        if pge < 0.5: continue
        for j, assocLevel in enumerate(assocLevels):
            if assocLevel == i:
                origLevel = origLevels[j]
                qc[origLevel] = True

    return qc
def test(p, parameters, allow_level_reinstating=True):
    """ 
    Runs the quality control check on profile p and returns a numpy array 
    of quality control decisions with False where the data value has 
    passed the check and True where it failed. 

    If allow_level_reinstating is set to True then rejected levels can be
    reprieved by comparing with levels above and below. NB this is done by
    default in EN processing.
    """

    # Define an array to hold results.
    qc = np.zeros(p.n_levels(), dtype=bool)

    # Obtain the obs minus background differences on standard levels.
    result = stdLevelData(p, parameters)
    if result is None:
        return qc
    
    # Unpack the results.
    levels, origLevels, assocLevels = result
    # Retrieve the background and observation error variances and
    # the background values.
    query = 'SELECT bgstdlevels, bgevstdlevels FROM enbackground WHERE uid = ' + str(p.uid())
    enbackground_pars = main.dbinteract(query)
    enbackground_pars = main.unpack_row(enbackground_pars[0])
    bgsl = enbackground_pars[0]
    slev = parameters['enbackground']['depth']
    bgev = enbackground_pars[1]
    obev = parameters['enbackground']['obev']

    #find initial pge
    pgeData = determine_pge(levels, bgev, obev, p)

    # Find buddy.
    profiles = get_profile_info(parameters)
    minDist  = 1000000000.0
    iMinDist = None
    for iProfile, profile in enumerate(profiles):
        pDist = assessBuddyDistance(p, profile)
        if pDist is not None and pDist < minDist:
            minDist  = pDist
            iMinDist = iProfile

    # Check if we have found a buddy and process if so.
    if minDist <= 400000:
        pBuddy = main.get_profile_from_db(profiles[iMinDist][0])

        # buddy vetos
        Fail = False
        if pBuddy.var_index() is None:
            Fail = True
        if Fail == False:
            main.catchFlags(pBuddy)
            if np.sum(pBuddy.t().mask == False) == 0:
                Fail = True

        if Fail == False:

          result = stdLevelData(pBuddy, parameters)

          query = 'SELECT bgevstdlevels FROM enbackground WHERE uid = ' + str(pBuddy.uid())
          buddy_pars = main.dbinteract(query)

          buddy_pars = main.unpack_row(buddy_pars[0])

          if result is not None: 
            levelsBuddy, origLevelsBuddy, assocLevelsBuddy = result
            bgevBuddy = buddy_pars[0]
            pgeBuddy  = determine_pge(levels, bgevBuddy, obev, pBuddy)
            pgeData   = update_pgeData(pgeData, pgeBuddy, levels, levelsBuddy, minDist, p, pBuddy, obev, bgev, bgevBuddy)

    # Check if levels should be reinstated.
    if allow_level_reinstating:
        if np.abs(p.latitude()) < 20.0:
            depthTol = 300.0
        else:
            depthTol = 200.0
        stdLevelFlags = pgeData >= 0.5
        for i, slflag in enumerate(stdLevelFlags):
            if slflag:
                # Check for non rejected surrounding levels.
                okbelow = False
                if i > 0:
                    if stdLevelFlags[i - 1] == False and levels.mask[i - 1] == False and bgsl.mask[i - 1] == False:
                        okbelow = True
                okabove = False
                nsl = len(stdLevelFlags)
                if i < nsl - 1:
                    if stdLevelFlags[i + 1] == False and levels.mask[i + 1] == False and bgsl.mask[i + 1] == False:
                        okabove = True
                # Work out tolerances.
                if slev[i] > depthTol + 100: 
                    tolFactor = 0.5
                elif slev[i] > depthTol:
                    tolFactor = 1.0 - 0.005 * (slev[i] - depthTol)
                else:
                    tolFactor = 1.0
                ttol = 0.5 * tolFactor 
                if okbelow == True and okabove == True:
                    xmax = levels[i - 1] + bgsl[i - 1] + ttol
                    xmin = levels[i + 1] + bgsl[i + 1] - ttol
                elif okbelow == True:
                    xmax = levels[i - 1] + bgsl[i - 1] + ttol
                    xmin = levels[i - 1] + bgsl[i - 1] - ttol
                elif okabove == True:
                    xmax = levels[i + 1] + bgsl[i + 1] + ttol
                    xmin = levels[i + 1] + bgsl[i + 1] - ttol
                else:
                    continue
                # Reassign PGE if level is within the tolerances.
                if levels[i] + bgsl[i] >= xmin and levels[i] + bgsl[i] <= xmax:
                    pgeData[i] = 0.49      

    # Assign the QC flags to original levels.
    for i, pge in enumerate(pgeData):
        if pgeData.mask[i]: continue
        if pge < 0.5: continue
        for j, assocLevel in enumerate(assocLevels):
            if assocLevel == i:
                origLevel = origLevels[j]        
                qc[origLevel] = True

    return qc
Example #9
0
def test(p):
    """ 
    Runs the quality control check on profile p and returns a numpy array 
    of quality control decisions with False where the data value has 
    passed the check and True where it failed. 
    """

    # Define an array to hold results.
    qc = np.zeros(p.n_levels(), dtype=bool)

    # Obtain the obs minus background differences on standard levels.
    result = stdLevelData(p)
    if result is None: return qc

    # Unpack the results.
    levels, origLevels, assocLevels = result
    # Retrieve the background and observation error variances.
    bgev = EN_background_check.bgevStdLevels
    obev = EN_background_check.auxParam['obev']

    #find initial pge
    pgeData = determine_pge(levels, bgev, obev, p)

    # Find buddy.
    profiles = data.ds.profiles
    minDist = 1000000000.0
    iMinDist = None
    for iProfile, profile in enumerate(profiles):
        pDist = assessBuddyDistance(p, profile)
        if pDist is not None and pDist < minDist:
            minDist = pDist
            iMinDist = iProfile

    # Check if we have found a buddy and process if so.
    if minDist <= 400000:
        fid = None
        pBuddy, currentFile, fid = main.profileData(profiles[iMinDist], '',
                                                    fid)
        fid.close()

        # buddy vetos
        Fail = False
        if pBuddy.var_index() is None:
            Fail = True
        if Fail == False:
            main.catchFlags(pBuddy)
            if np.sum(pBuddy.t().mask == False) == 0:
                Fail = True

        if Fail == False:
            result = stdLevelData(pBuddy)
            if result is not None:
                levelsBuddy, origLevelsBuddy, assocLevelsBuddy = result
                bgevBuddy = EN_background_check.bgevStdLevels
                pgeBuddy = determine_pge(levels, bgevBuddy, obev, pBuddy)
                pgeData = update_pgeData(pgeData, pgeBuddy, levels,
                                         levelsBuddy, minDist, p, pBuddy, obev,
                                         bgev, bgevBuddy)

    # Assign the QC flags.
    for i, pge in enumerate(pgeData):
        if pgeData.mask[i]: continue
        if pge < 0.5: continue
        for j, assocLevel in enumerate(assocLevels):
            if assocLevel == i:
                origLevel = origLevels[j]
                qc[origLevel] = True

    return qc