def test(p, parameters): """ Runs the quality control check on profile p and returns a numpy array of quality control decisions with False where the data value has passed the check and True where it failed. """ country = p.primary_header['Country code'] cruise = p.cruise() originator_cruise = p.originator_cruise() uid = p.uid() # don't bother if this has already been analyzed command = 'SELECT en_track_check FROM ' + parameters["table"] + ' WHERE uid = ' + str(uid) + ';' en_track_result = main.dbinteract(command) if en_track_result[0][0] is not None: en_track_result = main.unpack_row(en_track_result[0])[0] result = np.zeros(1, dtype=bool) result[0] = np.any(en_track_result) return result # make sure this profile makes sense in the track check if not assess_usability(p): return np.zeros(1, dtype=bool) # fetch all profiles on track, sorted chronologically, earliest first (None sorted as highest), then by uid command = 'SELECT uid, year, month, day, time, lat, long, probe, raw FROM ' + parameters["table"] + ' WHERE cruise = ' + str(cruise) + ' and country = "' + str(country) + '" and ocruise = "' + str(originator_cruise) + '" and year is not null and month is not null and day is not null and time is not null ORDER BY year, month, day, time, uid ASC;' track_rows = main.dbinteract(command) # avoid inappropriate profiles track_rows = [tr for tr in track_rows if assess_usability_raw(tr[8][1:-1])] # start all as passing by default EN_track_results = {} for i in range(len(track_rows)): EN_track_results[track_rows[i][0]] = np.zeros(1, dtype=bool) # copy the list of headers; # remove entries as they are flagged. passed_rows = copy.deepcopy(track_rows) rejects = findOutlier(passed_rows, EN_track_results) while rejects != []: passed_index = [x for x in range(len(passed_rows)) if x not in rejects ] passed_rows = [passed_rows[index] for index in passed_index ] rejects = findOutlier(passed_rows, EN_track_results) # if more than half got rejected, reject everyone if len(passed_rows) < len(track_rows) / 2: for i in range(len(track_rows)): EN_track_results[track_rows[i][0]][0] = True # write all to db result = [] for i in range(len(track_rows)): result.append((main.pack_array(EN_track_results[track_rows[i][0]]), track_rows[i][0])) query = "UPDATE " + sys.argv[1] + " SET en_track_check=? WHERE uid=?" main.interact_many(query, result) return EN_track_results[uid]
def test(p, parameters): """ Runs the quality control check on profile p and returns a numpy array of quality control decisions with False where the data value has passed the check and True where it failed. """ cruise = p.cruise() uid = p.uid() # don't bother if cruise == 0 or None, or if timestamp is corrupt if (cruise in [0, None]) or (None in [p.year(), p.month(), p.day(), p.time()]): return np.zeros(1, dtype=bool) # don't bother if this has already been analyzed command = 'SELECT en_track_check FROM ' + parameters["table"] + ' WHERE uid = ' + str(uid) + ';' en_track_result = main.dbinteract(command) if en_track_result[0][0] is not None: en_track_result = main.unpack_row(en_track_result[0])[0] result = np.zeros(1, dtype=bool) result[0] = np.any(en_track_result) return result # some detector types cannot be assessed by this test; do not raise flag. if p.probe_type() in [None]: return np.zeros(1, dtype=bool) # fetch all profiles on track, sorted chronologically, earliest first (None sorted as highest) command = 'SELECT uid, year, month, day, time, lat, long, probe FROM ' + parameters["table"] + ' WHERE cruise = ' + str(cruise) + ' and year is not null and month is not null and day is not null and time is not null ORDER BY year, month, day, time, uid ASC;' track_rows = main.dbinteract(command) # start all as passing by default: EN_track_results = {} for i in range(len(track_rows)): EN_track_results[track_rows[i][0]] = np.zeros(1, dtype=bool) # copy the list of headers; # remove entries as they are flagged. passed_rows = copy.deepcopy(track_rows) rejects = findOutlier(passed_rows, EN_track_results) while rejects != []: passed_index = [x for x in range(len(passed_rows)) if x not in rejects ] passed_rows = [passed_rows[index] for index in passed_index ] rejects = findOutlier(passed_rows, EN_track_results) # if more than half got rejected, reject everyone if len(passed_rows) < len(track_rows) / 2: for i in range(len(track_rows)): EN_track_results[track_rows[i][0]][0] = True # write all to db result = [] for i in range(len(track_rows)): result.append((main.pack_array(EN_track_results[track_rows[i][0]]), track_rows[i][0])) query = "UPDATE " + sys.argv[1] + " SET en_track_check=? WHERE uid=?" main.interact_many(query, result) return EN_track_results[uid]
def test(p, parameters): """ Runs the quality control check on profile p and returns a numpy array of quality control decisions with False where the data value has passed the check and True where it failed. """ country = p.primary_header['Country code'] cruise = p.cruise() originator_cruise = p.originator_cruise() uid = p.uid() # don't bother if this has already been analyzed command = 'SELECT en_track_check FROM ' + parameters["table"] + ' WHERE uid = ' + str(uid) + ';' en_track_result = main.dbinteract(command) if en_track_result[0][0] is not None: en_track_result = main.unpack_row(en_track_result[0])[0] result = np.zeros(1, dtype=bool) result[0] = np.any(en_track_result) return result # make sure this profile makes sense in the track check if not assess_usability(p): return np.zeros(1, dtype=bool) # fetch all profiles on track, sorted chronologically, earliest first (None sorted as highest), then by uid command = 'SELECT uid, year, month, day, time, lat, long, probe, raw FROM ' + parameters["table"] + ' WHERE cruise = ' + str(cruise) + ' and country = "' + str(country) + '" and ocruise = "' + str(originator_cruise) + '" and year is not null and month is not null and day is not null and time is not null ORDER BY year, month, day, time, uid ASC;' track_rows = main.dbinteract(command) # avoid inappropriate profiles track_rows = [tr for tr in track_rows if assess_usability_raw(tr[8][1:-1])] # start all as passing by default EN_track_results = {} for i in range(len(track_rows)): EN_track_results[track_rows[i][0]] = np.zeros(1, dtype=bool) # copy the list of headers; # remove entries as they are flagged. passed_rows = copy.deepcopy(track_rows) rejects = findOutlier(passed_rows, EN_track_results) while rejects != []: passed_index = [x for x in range(len(passed_rows)) if x not in rejects ] passed_rows = [passed_rows[index] for index in passed_index ] rejects = findOutlier(passed_rows, EN_track_results) # if more than half got rejected, reject everyone if len(passed_rows) < len(track_rows) / 2: for i in range(len(track_rows)): EN_track_results[track_rows[i][0]][0] = True # write all to db result = [] for i in range(len(track_rows)): result.append((main.pack_array(EN_track_results[track_rows[i][0]]), track_rows[i][0])) query = "UPDATE " + sys.argv[1] + " SET en_track_check=? WHERE uid=?" main.interact_many(query, result) return EN_track_results[uid]
def process_row(uid, logdir): '''run all tests on the indicated database row''' # reroute stdout, stderr to separate files for each profile to preserve logs sys.stdout = open(logdir + "/" + str(uid) + ".stdout", "w") sys.stderr = open(logdir + "/" + str(uid) + ".stderr", "w") # extract profile profile = main.get_profile_from_db(uid) # mask out error codes in temperature data main.catchFlags(profile) # run tests for itest, test in enumerate(testNames): try: result = run(test, [profile], parameterStore)[0] except: print test, 'exception', sys.exc_info() result = np.zeros(1, dtype=bool) try: query = "UPDATE " + sys.argv[ 1] + " SET " + test + "=? WHERE uid=" + str( profile.uid()) + ";" main.dbinteract(query, [main.pack_array(result)]) except: print 'db exception', sys.exc_info()
def level_order(p, parameters): '''Reorders data into depth order and rejects levels with negative depth. ''' # check if the relevant info is already in the db query = 'SELECT nlevels, origlevels, zr, tr, qc FROM icdclevelorder WHERE uid = ' + str( p.uid()) precomputed = main.dbinteract(query, targetdb=parameters["db"]) if len(precomputed) > 0: nlevels = precomputed[0][0] origlevels = pickle.load(io.BytesIO(precomputed[0][1])) zr = pickle.load(io.BytesIO(precomputed[0][2])) tr = pickle.load(io.BytesIO(precomputed[0][3])) qc = pickle.load(io.BytesIO(precomputed[0][4])) return p.uid(), nlevels, origlevels, zr, tr, qc # Extract data and define the index for each level. z = p.z() t = p.t() origlevels = np.arange(p.n_levels()) # Implement the QC. For this test we only reject negative depths. qc = z < 0 # Remove occurrences of no data at a level and rejected obs. use = (z.mask == False) & (t.mask == False) & (qc == False) z = z[use] t = t[use] origlevels = origlevels[use] nlevels = np.count_nonzero(use) if nlevels > 1: # Sort the data. Using mergesort keeps levels with the same depth # in the same order. isort = np.argsort(z, kind='mergesort') zr = z[isort] tr = t[isort] origlevels = origlevels[isort] else: zr = z tr = t # register pre-computed arrays in db for reuse origlevels_p = pickle.dumps(origlevels, -1) zr_p = pickle.dumps(zr, -1) tr_p = pickle.dumps(tr, -1) qc_p = pickle.dumps(qc, -1) query = "REPLACE INTO icdclevelorder VALUES(?,?,?,?,?,?)" main.dbinteract(query, [ p.uid(), nlevels, sqlite3.Binary(origlevels_p), sqlite3.Binary(zr_p), sqlite3.Binary(tr_p), sqlite3.Binary(qc_p) ], targetdb=parameters["db"]) return p.uid(), nlevels, origlevels, zr, tr, qc
def process_row(uid, logdir): '''run all tests on the indicated database row''' # reroute stdout, stderr to separate files for each profile to preserve logs sys.stdout = open(logdir + "/" + str(uid) + ".stdout", "w") sys.stderr = open(logdir + "/" + str(uid) + ".stderr", "w") # extract profile profile = main.get_profile_from_db(uid) # mask out error codes in temperature data main.catchFlags(profile) # run tests for itest, test in enumerate(testNames): try: result = run(test, [profile], parameterStore)[0] except: print test, 'exception', sys.exc_info() result = np.zeros(1, dtype=bool) try: query = "UPDATE " + sys.argv[1] + " SET " + test + "=? WHERE uid=" + str(profile.uid()) + ";" main.dbinteract(query, [main.pack_array(result)]) except: print 'db exception', sys.exc_info()
def loadParameters(parameterStore): main.dbinteract("DROP TABLE IF EXISTS enbackground") main.dbinteract( "CREATE TABLE IF NOT EXISTS enbackground (uid INTEGER PRIMARY KEY, bgstdlevels BLOB, bgevstdlevels BLOB, origlevels BLOB, ptlevels BLOB, bglevels BLOB)" ) parameterStore['enbackground'] = readENBackgroundCheckAux()
def loadParameters(parameterStore): main.dbinteract("DROP TABLE IF EXISTS enspikeandstep", targetdb=parameterStore["db"]) main.dbinteract( "CREATE TABLE IF NOT EXISTS enspikeandstep (uid INTEGER PRIMARY KEY, suspect BLOB)", targetdb=parameterStore["db"])
def loadParameters(parameterStore): main.dbinteract("DROP TABLE IF EXISTS icdclevelorder", targetdb=parameterStore["db"]) main.dbinteract( "CREATE TABLE IF NOT EXISTS icdclevelorder (uid INTEGER PRIMARY KEY, nlevels INTEGER, origlevels BLOB, zr BLOB, tr BLOB, qc BLOB)", targetdb=parameterStore["db"])
def record_parameters(profile, bgStdLevels, bgevStdLevels, origLevels, ptLevels, bgLevels): # pack the parameter arrays into the enbackground table # for consumption by the buddy check bgstdlevels = main.pack_array(bgStdLevels) bgevstdlevels = main.pack_array(bgevStdLevels) origlevels = main.pack_array(origLevels) ptlevels = main.pack_array(ptLevels) bglevels = main.pack_array(bgLevels) query = "REPLACE INTO enbackground VALUES(?,?,?,?,?,?);" main.dbinteract(query, [ profile.uid(), bgstdlevels, bgevstdlevels, origlevels, ptlevels, bglevels ])
def stdLevelData(p, parameters): """ Combines data that have passed other QC checks to create a set of observation minus background data on standard levels. """ # Combine other QC results. preQC = (EN_background_check.test(p, parameters) | EN_constant_value_check.test(p, parameters) | EN_increasing_depth_check.test(p, parameters) | EN_range_check.test(p, parameters) | EN_spike_and_step_check.test(p, parameters) | EN_stability_check.test(p, parameters)) # Get the data stored by the EN background check. # As it was run above we know that the data is available in the db. query = 'SELECT origlevels, ptlevels, bglevels FROM enbackground WHERE uid = ' + str(p.uid()) enbackground_pars = main.dbinteract(query) enbackground_pars = main.unpack_row(enbackground_pars[0]) origlevels = enbackground_pars[0] ptlevels = enbackground_pars[1] bglevels = enbackground_pars[2] origLevels = np.array(origlevels) diffLevels = (np.array(ptlevels) - np.array(bglevels)) nLevels = len(origLevels) if nLevels == 0: return None # Nothing more to do. # Remove any levels that failed previous QC. nLevels, origLevels, diffLevels = filterLevels(preQC, origLevels, diffLevels) if nLevels == 0: return None levels, assocLevs = meanDifferencesAtStandardLevels(origLevels, diffLevels, p.z(), parameters) return levels, origLevels, assocLevs
def test_determine_pge(self): ''' totally ridiculous differences between observation and background should give pge == 1 ''' p = util.testingProfile.fakeProfile([1.8, 1.8, 1.8, 7.1], [0.0, 2.5, 5.0, 7.5], latitude=55.6, longitude=12.9, date=[1900, 01, 15, 0], probe_type=7, uid=8888) levels = numpy.ma.array([1000, 1000, 1000, 1000]) levels.mask = False #bgev = qctests.EN_background_check.bgevStdLevels qctests.EN_background_check.test( p, self.parameters ) #need to populate the enbackground db with profile specific info query = 'SELECT bgevstdlevels FROM enbackground WHERE uid = 8888' enbackground_pars = main.dbinteract(query) bgev = pickle.load(StringIO.StringIO(enbackground_pars[0][0])) obev = self.parameters['enbackground']['obev'] expected = [1.0, 1.0, 1.0, 1.0] assert numpy.array_equal( qctests.EN_std_lev_bkg_and_buddy_check.determine_pge( levels, bgev, obev, p), expected ), 'PGE of extreme departures from background not flagged as 1.0'
def test(p, parameters): """ Runs the quality control check on profile p and returns a numpy array of quality control decisions with False where the data value has passed the check and True where it failed. """ # Check if the QC of this profile was already done and if not # run the QC. query = 'SELECT en_constant_value_check FROM ' + parameters["table"] + ' WHERE uid = ' + str(p.uid()) + ';' qc_log = main.dbinteract(query) qc_log = main.unpack_row(qc_log[0]) if qc_log[0] is not None: return qc_log[0] return run_qc(p, parameters)
def test(p, parameters): """ Runs the quality control check on profile p and returns a numpy array of quality control decisions with False where the data value has passed the check and True where it failed. """ # Check if the QC of this profile was already done and if not # run the QC. query = 'SELECT en_increasing_depth_check FROM ' + parameters["table"] + ' WHERE uid = ' + str(p.uid()) + ';' qc_log = main.dbinteract(query) qc_log = main.unpack_row(qc_log[0]) if qc_log[0] is not None: return qc_log[0] return run_qc(p, parameters)
def test_determine_pge(self): ''' totally ridiculous differences between observation and background should give pge == 1 ''' p = util.testingProfile.fakeProfile([1.8, 1.8, 1.8, 7.1], [0.0, 2.5, 5.0, 7.5], latitude=55.6, longitude=12.9, date=[1900, 01, 15, 0], probe_type=7, uid=8888) levels = numpy.ma.array([1000,1000,1000,1000]) levels.mask = False #bgev = qctests.EN_background_check.bgevStdLevels qctests.EN_background_check.test(p, self.parameters) #need to populate the enbackground db with profile specific info query = 'SELECT bgevstdlevels FROM enbackground WHERE uid = 8888' enbackground_pars = main.dbinteract(query) bgev = pickle.load(StringIO.StringIO(enbackground_pars[0][0])) obev = self.parameters['enbackground']['obev'] expected = [1.0, 1.0, 1.0, 1.0] assert numpy.array_equal(qctests.EN_std_lev_bkg_and_buddy_check.determine_pge(levels, bgev, obev, p), expected), 'PGE of extreme departures from background not flagged as 1.0'
def stdLevelData(p, parameters): """ Combines data that have passed other QC checks to create a set of observation minus background data on standard levels. """ # Combine other QC results. preQC = (EN_background_check.test(p, parameters) | EN_constant_value_check.test(p, parameters) | EN_increasing_depth_check.test(p, parameters) | EN_range_check.test(p, parameters) | EN_spike_and_step_check.test(p, parameters) | EN_stability_check.test(p, parameters)) # Get the data stored by the EN background check. # As it was run above we know that the data is available in the db. query = 'SELECT origlevels, ptlevels, bglevels FROM enbackground WHERE uid = ' + str( p.uid()) enbackground_pars = main.dbinteract(query) enbackground_pars = main.unpack_row(enbackground_pars[0]) origlevels = enbackground_pars[0] ptlevels = enbackground_pars[1] bglevels = enbackground_pars[2] origLevels = np.array(origlevels) diffLevels = (np.array(ptlevels) - np.array(bglevels)) nLevels = len(origLevels) if nLevels == 0: return None # Nothing more to do. # Remove any levels that failed previous QC. nLevels, origLevels, diffLevels = filterLevels(preQC, origLevels, diffLevels) if nLevels == 0: return None levels, assocLevs = meanDifferencesAtStandardLevels( origLevels, diffLevels, p.z(), parameters) return levels, origLevels, assocLevs
def find_roc(table, costratio=[2.5, 1.0], filter_on_wire_break_test=False, filter_from_file_spec=True, enforce_types_of_check=True, n_profiles_to_analyse=np.iinfo(np.int32).max, n_combination_iterations=1, with_reverses=False, effectiveness_ratio=2.0, improve_threshold=1.0, verbose=True, plot_roc=True, write_roc=True): ''' Generates a ROC curve from the database data in table by maximising the gradient of the ROC curve. It will combine different tests together and invert the results of tests if requested. costratio - two element iterable that defines how the ROC curve is developed. Higher numbers gives a ROC curve with lower false rates; the two elements allows control over the shape of the ROC curve near the start and end. E.g. [2.5, 1.0]. filter_on_wire_break_test - filter out the impact of XBT wire breaks from results. filter_from_file_spec - use specification from file to choose filtering. enforce_types_of_check - use specification from file on particular types of checks to use. n_profiles_to_analyse - restrict the number of profiles extracted from the database. n_combination_iterations - AND tests together; restricted to max of 2 as otherwise number of tests gets very large. with_reverses - if True, a copy of each test with inverted results is made. effectiveness_ratio - will give a warning if TPR / FPR is less than this value. improve_threshold - ignores tests if they do not results in a change in true positive rate (in %) of at least this amount. verbose - if True, will print a lot of messages to screen. plot_roc - if True, will save an image of the ROC to roc.png. write_roc - if True, will save the ROC data to roc.json. ''' # Read QC test specifications if required. groupdefinition = {} if filter_from_file_spec or enforce_types_of_check: groupdefinition = read_qc_groups() # Read data from database into a pandas data frame. df = dbutils.db_to_df(sys.argv[1], filter_on_wire_break_test = filter_on_wire_break_test, filter_on_tests = groupdefinition, n_to_extract = n_profiles_to_analyse) # Drop nondiscriminating tests nondiscrim = [] cols = list(df.columns) for c in cols: if len(pandas.unique(df[c])) == 1: nondiscrim.append(c) if verbose: print c + ' is nondiscriminating and will be removed' cols = [t for t in cols if t not in nondiscrim] df = df[cols] print list(df) testNames = df.columns[2:].values.tolist() if verbose: print 'Number of profiles is: ', len(df.index) print 'Number of quality checks to process is: ', len(testNames) # mark chosen profiles as part of the training set all_uids = main.dbinteract('SELECT uid from ' + sys.argv[1] + ';') for uid in all_uids: uid = uid[0] is_training = int(uid in df['uid'].astype(int).as_matrix()) query = "UPDATE " + sys.argv[1] + " SET training=" + str(is_training) + " WHERE uid=" + str(uid) + ";" main.dbinteract(query) # Convert to numpy structures and make inverse versions of tests if required. # Any test with true positive rate of zero is discarded. truth = df['Truth'].as_matrix() tests = [] names = [] tprs = [] fprs = [] if with_reverses: reverselist = [False, True] else: reverselist = [False] for i, testname in enumerate(testNames): for reversal in reverselist: results = df[testname].as_matrix() != reversal tpr, fpr, fnr, tnr = main.calcRates(results, truth) if tpr > 0.0: tests.append(results) if reversal: addtext = 'r' else: addtext = '' names.append(addtext + testname) tprs.append(tpr) fprs.append(fpr) del df # No further need for the data frame. if verbose: print 'Number of quality checks after adding reverses and removing zero TPR was: ', len(names) # Create storage to hold the roc curve. cumulative = truth.copy() cumulative[:] = False currenttpr = 0.0 currentfpr = 0.0 r_fprs = [] # The false positive rate for each ROC point. r_tprs = [] # True positive rate for each ROC point. testcomb = [] # The QC test that was added at each ROC point. groupsel = [] # Set to True if the ROC point was from an enforced group. # Pre-select some tests if required. if enforce_types_of_check: if verbose: print 'Enforcing types of checks' while len(groupdefinition['At least one from group']) > 0: bestchoice = '' bestgroup = '' bestdist = np.sqrt(100.0**2 + 100.0**2) besti = -1 for key in groupdefinition['At least one from group']: for testname in groupdefinition['At least one from group'][key]: # Need to check that the test exists - it may have been removed # if it was non-discriminating. if testname in names: for itest, name in enumerate(names): if name == testname: cumulativenew = np.logical_or(cumulative, tests[itest]) tpr, fpr, fnr, tnr = main.calcRates(cumulativenew, truth) newdist = return_cost(costratio, tpr, fpr) print ' ', tpr, fpr, newdist, bestdist, testname if newdist == bestdist: if verbose: print ' ' + bestchoice + ' and ' + testname + ' have the same results and the first is kept' elif newdist < bestdist: bestchoice = testname bestdist = newdist besti = itest bestgroup = key else: if verbose: print ' ' + testname + ' not found and so was skipped' #assert bestchoice != '', ' Error, did not make a choice in group ' + key if verbose: print ' ' + bestchoice + ' was selected from group ' + bestgroup if fprs[besti] > 0: if tprs[besti] / fprs[besti] < effectiveness_ratio: print 'WARNING - ' + bestchoice + ' TPR / FPR is below the effectiveness ratio limit: ', tprs[besti] / fprs[besti], effectiveness_ratio cumulative = np.logical_or(cumulative, tests[besti]) currenttpr, currentfpr, fnr, tnr = main.calcRates(cumulative, truth) testcomb.append(names[besti]) r_fprs.append(currentfpr) r_tprs.append(currenttpr) groupsel.append(True) # Once a test has been added, it can be deleted so that it is not considered again. del names[besti] del tests[besti] del fprs[besti] del tprs[besti] del groupdefinition['At least one from group'][bestgroup] print 'ROC point from enforced group: ', currenttpr, currentfpr, testcomb[-1], bestgroup # Make combinations of the single checks and store. assert n_combination_iterations <= 2, 'Setting n_combination_iterations > 2 results in a very large number of combinations' if verbose: print 'Starting construction of combinations with number of iterations: ', n_combination_iterations for its in range(n_combination_iterations): ntests = len(names) for i in range(ntests - 1): if verbose: print 'Processing iteration ', its + 1, ' out of ', n_combination_iterations, ' step ', i + 1, ' out of ', ntests - 1, ' with number of tests now ', len(names) for j in range(i + 1, ntests): # Create the name for this combination. newname = ('&').join(sorted((names[i] + '&' + names[j]).split('&'))) if newname in names: continue # Do not keep multiple copies of the same combination. results = np.logical_and(tests[i], tests[j]) tpr, fpr, fnr, tnr = main.calcRates(results, truth) if tpr > 0.0: tests.append(results) tprs.append(tpr) fprs.append(fpr) names.append(newname) if verbose: print 'Completed generation of tests, now constructing roc from number of tests: ', len(names) # Create roc. used = np.zeros(len(names), dtype=bool) overallbest = return_cost(costratio, tpr, fpr) keepgoing = True while keepgoing: keepgoing = False besti = -1 bestcost = overallbest bestncomb = 100000 bestdtpr = 0 bestdfpr = 100000 for i in range(len(names)): if used[i]: continue cumulativenew = np.logical_or(cumulative, tests[i]) tpr, fpr, fnr, tnr = main.calcRates(cumulativenew, truth) dtpr = tpr - currenttpr dfpr = fpr - currentfpr newcost = return_cost(costratio, tpr, fpr) newbest = False if newcost <= bestcost and dtpr >= improve_threshold and dtpr > 0.0: # If cost is better than found previously, use it else if it is # the same then decide if to use it or not. if newcost < bestcost: newbest = True elif dtpr >= bestdtpr: if dtpr > bestdtpr: newbest = True elif len(names[i].split('&')) < bestncomb: newbest = True if newbest: besti = i bestcost = newcost bestncomb = len(names[i].split('&')) bestdtpr = dtpr bestdfpr = dfpr if besti >= 0: keepgoing = True used[besti] = True overallbest = bestcost cumulative = np.logical_or(cumulative, tests[besti]) currenttpr, currentfpr, fnr, tnr = main.calcRates(cumulative, truth) testcomb.append(names[besti]) r_fprs.append(currentfpr) r_tprs.append(currenttpr) groupsel.append(False) print 'ROC point: ', currenttpr, currentfpr, names[besti], overallbest if plot_roc: plt.plot(r_fprs, r_tprs, 'k') for i in range(len(r_fprs)): if groupsel[i]: colour = 'r' else: colour = 'b' plt.plot(r_fprs[i], r_tprs[i], colour + 'o') plt.xlim(0, 100) plt.ylim(0, 100) plt.xlabel('False positive rate (%)') plt.ylabel('True positive rate (%)') plt.savefig('roc.png') plt.close() if write_roc: f = open('roc.json', 'w') r = {} r['tpr'] = r_tprs r['fpr'] = r_fprs r['tests'] = testcomb r['groupsel'] = groupsel json.dump(r, f) f.close()
def run_qc(p, suspect): # check for pre-registered suspect tabulation, if that's what we want: if suspect: query = 'SELECT suspect FROM enspikeandstep WHERE uid = ' + str(p.uid()) + ';' susp = main.dbinteract(query) if len(susp) > 0: return main.unpack_row(susp[0])[0] # Define tolerances used. tolD = np.array([0, 200, 300, 500, 600]) tolDTrop = np.array([0, 300, 400, 500, 600]) tolT = np.array([5.0, 5.0, 2.5, 2.0, 1.5]) # Define an array to hold results. qc = np.zeros(p.n_levels(), dtype=bool) # Get depth and temperature values from the profile. z = p.z() t = p.t() # Find which levels have data. isTemperature = (t.mask==False) isDepth = (z.mask==False) isData = isTemperature & isDepth # Array to hold temperature differences between levels and gradients. dt, gt = composeDT(t, z, p.n_levels()) # Spikes and steps detection. for i in range(1, p.n_levels()): if i >= 2: if (isData[i-2] and isData[i-1] and isData[i]) == False: continue if z[i] - z[i-2] >= 5.0: wt1 = (z[i-1] - z[i-2]) / (z[i] - z[i-2]) else: wt1 = 0.5 else: if (isData[i-1] and isData[i]) == False: continue wt1 = 0.5 dTTol = determineDepthTolerance(z[i-1], np.abs(p.latitude())) gTTol = 0.05 # Check for low temperatures in the Tropics. # This might be more appropriate to appear in a separate EN regional # range check but is included here for now for consistency with the # original code. if (np.abs(p.latitude()) < 20.0 and z[i-1] < 1000.0 and t[i-1] < 1.0): dt[i] = np.ma.masked if suspect == True: qc[i-1] = True continue qc, dt = conditionA(dt, dTTol, qc, wt1, i, suspect) qc, dt = conditionB(dt, dTTol, gTTol, qc, gt, i, suspect) qc = conditionC(dt, dTTol, z, qc, t, i, suspect) # End of loop over levels. # Step or 0.0 at the bottom of a profile. if isData[-1] and dt.mask[-1] == False: dTTol = determineDepthTolerance(z[-1], np.abs(p.latitude())) if np.abs(dt[-1]) > dTTol: if suspect == True: qc[-1] = True if isTemperature[-1]: if t[-1] == 0.0: if suspect == True: qc[-1] = True # If 4 levels or more than half the profile is rejected then reject all. if suspect == False: nRejects = np.count_nonzero(qc) if nRejects >= 4 or nRejects > p.n_levels()/2: qc[:] = True # register suspects, if computed, to db if suspect: query = "REPLACE INTO enspikeandstep VALUES(?,?);" main.dbinteract(query, [p.uid(), main.pack_array(qc)] ) return qc
def test(p, parameters, allow_level_reinstating=True): """ Runs the quality control check on profile p and returns a numpy array of quality control decisions with False where the data value has passed the check and True where it failed. If allow_level_reinstating is set to True then rejected levels can be reprieved by comparing with levels above and below. NB this is done by default in EN processing. """ # Define an array to hold results. qc = np.zeros(p.n_levels(), dtype=bool) # Obtain the obs minus background differences on standard levels. result = stdLevelData(p, parameters) if result is None: return qc # Unpack the results. levels, origLevels, assocLevels = result # Retrieve the background and observation error variances and # the background values. query = 'SELECT bgstdlevels, bgevstdlevels FROM enbackground WHERE uid = ' + str( p.uid()) enbackground_pars = main.dbinteract(query) enbackground_pars = main.unpack_row(enbackground_pars[0]) bgsl = enbackground_pars[0] slev = parameters['enbackground']['depth'] bgev = enbackground_pars[1] obev = parameters['enbackground']['obev'] #find initial pge pgeData = determine_pge(levels, bgev, obev, p) # Find buddy. profiles = get_profile_info(parameters) minDist = 1000000000.0 iMinDist = None for iProfile, profile in enumerate(profiles): pDist = assessBuddyDistance(p, profile) if pDist is not None and pDist < minDist: minDist = pDist iMinDist = iProfile # Check if we have found a buddy and process if so. if minDist <= 400000: pBuddy = main.get_profile_from_db(profiles[iMinDist][0]) # buddy vetos Fail = False if pBuddy.var_index() is None: Fail = True if Fail == False: main.catchFlags(pBuddy) if np.sum(pBuddy.t().mask == False) == 0: Fail = True if Fail == False: result = stdLevelData(pBuddy, parameters) query = 'SELECT bgevstdlevels FROM enbackground WHERE uid = ' + str( pBuddy.uid()) buddy_pars = main.dbinteract(query) buddy_pars = main.unpack_row(buddy_pars[0]) if result is not None: levelsBuddy, origLevelsBuddy, assocLevelsBuddy = result bgevBuddy = buddy_pars[0] pgeBuddy = determine_pge(levels, bgevBuddy, obev, pBuddy) pgeData = update_pgeData(pgeData, pgeBuddy, levels, levelsBuddy, minDist, p, pBuddy, obev, bgev, bgevBuddy) # Check if levels should be reinstated. if allow_level_reinstating: if np.abs(p.latitude()) < 20.0: depthTol = 300.0 else: depthTol = 200.0 stdLevelFlags = pgeData >= 0.5 for i, slflag in enumerate(stdLevelFlags): if slflag: # Check for non rejected surrounding levels. okbelow = False if i > 0: if stdLevelFlags[i - 1] == False and levels.mask[ i - 1] == False and bgsl.mask[i - 1] == False: okbelow = True okabove = False nsl = len(stdLevelFlags) if i < nsl - 1: if stdLevelFlags[i + 1] == False and levels.mask[ i + 1] == False and bgsl.mask[i + 1] == False: okabove = True # Work out tolerances. if slev[i] > depthTol + 100: tolFactor = 0.5 elif slev[i] > depthTol: tolFactor = 1.0 - 0.005 * (slev[i] - depthTol) else: tolFactor = 1.0 ttol = 0.5 * tolFactor if okbelow == True and okabove == True: xmax = levels[i - 1] + bgsl[i - 1] + ttol xmin = levels[i + 1] + bgsl[i + 1] - ttol elif okbelow == True: xmax = levels[i - 1] + bgsl[i - 1] + ttol xmin = levels[i - 1] + bgsl[i - 1] - ttol elif okabove == True: xmax = levels[i + 1] + bgsl[i + 1] + ttol xmin = levels[i + 1] + bgsl[i + 1] - ttol else: continue # Reassign PGE if level is within the tolerances. if levels[i] + bgsl[i] >= xmin and levels[i] + bgsl[i] <= xmax: pgeData[i] = 0.49 # Assign the QC flags to original levels. for i, pge in enumerate(pgeData): if pgeData.mask[i]: continue if pge < 0.5: continue for j, assocLevel in enumerate(assocLevels): if assocLevel == i: origLevel = origLevels[j] qc[origLevel] = True return qc
def run_qc(p, suspect, parameters): # check for pre-registered suspect tabulation, if that's what we want: if suspect: query = 'SELECT suspect FROM enspikeandstep WHERE uid = ' + str( p.uid()) + ';' susp = main.dbinteract(query, targetdb=parameters["db"]) if len(susp) > 0: return main.unpack_row(susp[0])[0] # Define tolerances used. tolD = np.array([0, 200, 300, 500, 600]) tolDTrop = np.array([0, 300, 400, 500, 600]) tolT = np.array([5.0, 5.0, 2.5, 2.0, 1.5]) # Define an array to hold results. qc = np.zeros(p.n_levels(), dtype=bool) # Get depth and temperature values from the profile. z = p.z() t = p.t() # Find which levels have data. isTemperature = (t.mask == False) isDepth = (z.mask == False) isData = isTemperature & isDepth # Array to hold temperature differences between levels and gradients. dt, gt = composeDT(t, z, p.n_levels()) # Spikes and steps detection. for i in range(1, p.n_levels()): if i >= 2: if (isData[i - 2] and isData[i - 1] and isData[i]) == False: continue if z[i] - z[i - 2] >= 5.0: wt1 = (z[i - 1] - z[i - 2]) / (z[i] - z[i - 2]) else: wt1 = 0.5 else: if (isData[i - 1] and isData[i]) == False: continue wt1 = 0.5 dTTol = determineDepthTolerance(z[i - 1], np.abs(p.latitude())) gTTol = 0.05 # Check for low temperatures in the Tropics. # This might be more appropriate to appear in a separate EN regional # range check but is included here for now for consistency with the # original code. if (np.abs(p.latitude()) < 20.0 and z[i - 1] < 1000.0 and t[i - 1] < 1.0): dt[i] = np.ma.masked if suspect == True: qc[i - 1] = True continue qc, dt = conditionA(dt, dTTol, qc, wt1, i, suspect) qc, dt = conditionB(dt, dTTol, gTTol, qc, gt, i, suspect) qc = conditionC(dt, dTTol, z, qc, t, i, suspect) # End of loop over levels. # Step or 0.0 at the bottom of a profile. if isData[-1] and dt.mask[-1] == False: dTTol = determineDepthTolerance(z[-1], np.abs(p.latitude())) if np.abs(dt[-1]) > dTTol: if suspect == True: qc[-1] = True if isTemperature[-1]: if t[-1] == 0.0: if suspect == True: qc[-1] = True # If 4 levels or more than half the profile is rejected then reject all. if suspect == False: nRejects = np.count_nonzero(qc) if nRejects >= 4 or nRejects > p.n_levels() / 2: qc[:] = True # register suspects, if computed, to db if suspect: query = "REPLACE INTO enspikeandstep VALUES(?,?);" main.dbinteract(query, [p.uid(), main.pack_array(qc)], targetdb=parameters["db"]) return qc
def loadParameters(parameterStore): main.dbinteract("DROP TABLE IF EXISTS enspikeandstep") main.dbinteract("CREATE TABLE IF NOT EXISTS enspikeandstep (uid INTEGER PRIMARY KEY, suspect BLOB)")
# Read data from database into a pandas data frame. df = dbutils.db_to_df(sys.argv[1], filter_on_wire_break_test = False, filter_on_tests = groupdefinition, n_to_extract = sys.argv[2]) testNames = df.columns[2:].values.tolist() # declare some downstream constructs accepted = [] unflagged = [] fprs = [] bad = df.loc[df['Truth']] bad.reset_index(inplace=True, drop=True) # mark chosen profiles as part of the training set all_uids = main.dbinteract('SELECT uid from ' + sys.argv[1] + ';') for uid in all_uids: uid = uid[0] is_training = int(uid in df['uid'].astype(int).as_matrix()) query = "UPDATE " + sys.argv[1] + " SET training=" + str(is_training) + " WHERE uid=" + str(uid) + ";" main.dbinteract(query) # algo. step 0: # demand individual QC tests have TPR/FPR > some threshold perf_thresh = 2 drop_tests = [] for test in testNames: tpr, fpr, fnr, tnr = main.calcRates(df[test].tolist(), df['Truth'].tolist()) if fpr > 0 and tpr / fpr < perf_thresh: print('dropping', test, '; tpr/fpr = ', tpr/fpr) df.drop([test], axis=1)
def tearDown(self): main.dbinteract('DROP TABLE icdclevelorder;')
def get_profile_info(parameters): # Gets information about the profiles from the database. query = 'SELECT uid,year,month,cruise,lat,long FROM ' + parameters['table'] return main.dbinteract(query)
def find_roc(table, costratio=[2.5, 1.0], filter_on_wire_break_test=False, filter_from_file_spec=True, enforce_types_of_check=True, n_profiles_to_analyse=np.iinfo(np.int32).max, n_combination_iterations=1, with_reverses=False, effectiveness_ratio=2.0, improve_threshold=1.0, verbose=True, plot_roc=True, write_roc=True): ''' Generates a ROC curve from the database data in table by maximising the gradient of the ROC curve. It will combine different tests together and invert the results of tests if requested. costratio - two element iterable that defines how the ROC curve is developed. Higher numbers gives a ROC curve with lower false rates; the two elements allows control over the shape of the ROC curve near the start and end. E.g. [2.5, 1.0]. filter_on_wire_break_test - filter out the impact of XBT wire breaks from results. filter_from_file_spec - use specification from file to choose filtering. enforce_types_of_check - use specification from file on particular types of checks to use. n_profiles_to_analyse - restrict the number of profiles extracted from the database. n_combination_iterations - AND tests together; restricted to max of 2 as otherwise number of tests gets very large. with_reverses - if True, a copy of each test with inverted results is made. effectiveness_ratio - will give a warning if TPR / FPR is less than this value. improve_threshold - ignores tests if they do not results in a change in true positive rate (in %) of at least this amount. verbose - if True, will print a lot of messages to screen. plot_roc - if True, will save an image of the ROC to roc.png. write_roc - if True, will save the ROC data to roc.json. ''' # Read QC test specifications if required. groupdefinition = {} if filter_from_file_spec or enforce_types_of_check: groupdefinition = read_qc_groups() # Read data from database into a pandas data frame. df = dbutils.db_to_df(sys.argv[1], filter_on_wire_break_test=filter_on_wire_break_test, filter_on_tests=groupdefinition, n_to_extract=n_profiles_to_analyse) # Drop nondiscriminating tests nondiscrim = [] cols = list(df.columns) for c in cols: if len(pandas.unique(df[c])) == 1: nondiscrim.append(c) if verbose: print(c + ' is nondiscriminating and will be removed') cols = [t for t in cols if t not in nondiscrim] df = df[cols] print(list(df)) testNames = df.columns[2:].values.tolist() if verbose: print('Number of profiles is: ', len(df.index)) print('Number of quality checks to process is: ', len(testNames)) # mark chosen profiles as part of the training set all_uids = main.dbinteract('SELECT uid from ' + sys.argv[1] + ';') for uid in all_uids: uid = uid[0] is_training = int(uid in df['uid'].astype(int).as_matrix()) query = "UPDATE " + sys.argv[1] + " SET training=" + str( is_training) + " WHERE uid=" + str(uid) + ";" main.dbinteract(query) # Convert to numpy structures and make inverse versions of tests if required. # Any test with true positive rate of zero is discarded. truth = df['Truth'].as_matrix() tests = [] names = [] tprs = [] fprs = [] if with_reverses: reverselist = [False, True] else: reverselist = [False] for i, testname in enumerate(testNames): for reversal in reverselist: results = df[testname].as_matrix() != reversal tpr, fpr, fnr, tnr = main.calcRates(results, truth) if tpr > 0.0: tests.append(results) if reversal: addtext = 'r' else: addtext = '' names.append(addtext + testname) tprs.append(tpr) fprs.append(fpr) del df # No further need for the data frame. if verbose: print( 'Number of quality checks after adding reverses and removing zero TPR was: ', len(names)) # Create storage to hold the roc curve. cumulative = truth.copy() cumulative[:] = False currenttpr = 0.0 currentfpr = 0.0 r_fprs = [] # The false positive rate for each ROC point. r_tprs = [] # True positive rate for each ROC point. testcomb = [] # The QC test that was added at each ROC point. groupsel = [] # Set to True if the ROC point was from an enforced group. # Pre-select some tests if required. if enforce_types_of_check: if verbose: print('Enforcing types of checks') while len(groupdefinition['At least one from group']) > 0: bestchoice = '' bestgroup = '' bestdist = np.sqrt(100.0**2 + 100.0**2) besti = -1 for key in groupdefinition['At least one from group']: for testname in groupdefinition['At least one from group'][ key]: # Need to check that the test exists - it may have been removed # if it was non-discriminating. if testname in names: for itest, name in enumerate(names): if name == testname: cumulativenew = np.logical_or( cumulative, tests[itest]) tpr, fpr, fnr, tnr = main.calcRates( cumulativenew, truth) newdist = return_cost(costratio, tpr, fpr) print(' ', tpr, fpr, newdist, bestdist, testname) if newdist == bestdist: if verbose: print( ' ' + bestchoice + ' and ' + testname + ' have the same results and the first is kept' ) elif newdist < bestdist: bestchoice = testname bestdist = newdist besti = itest bestgroup = key else: if verbose: print(' ' + testname + ' not found and so was skipped') #assert bestchoice != '', ' Error, did not make a choice in group ' + key if verbose: print(' ' + bestchoice + ' was selected from group ' + bestgroup) if fprs[besti] > 0: if tprs[besti] / fprs[besti] < effectiveness_ratio: print( 'WARNING - ' + bestchoice + ' TPR / FPR is below the effectiveness ratio limit: ', tprs[besti] / fprs[besti], effectiveness_ratio) cumulative = np.logical_or(cumulative, tests[besti]) currenttpr, currentfpr, fnr, tnr = main.calcRates( cumulative, truth) testcomb.append(names[besti]) r_fprs.append(currentfpr) r_tprs.append(currenttpr) groupsel.append(True) # Once a test has been added, it can be deleted so that it is not considered again. del names[besti] del tests[besti] del fprs[besti] del tprs[besti] del groupdefinition['At least one from group'][bestgroup] print('ROC point from enforced group: ', currenttpr, currentfpr, testcomb[-1], bestgroup) # Make combinations of the single checks and store. assert n_combination_iterations <= 2, 'Setting n_combination_iterations > 2 results in a very large number of combinations' if verbose: print( 'Starting construction of combinations with number of iterations: ', n_combination_iterations) for its in range(n_combination_iterations): ntests = len(names) for i in range(ntests - 1): if verbose: print('Processing iteration ', its + 1, ' out of ', n_combination_iterations, ' step ', i + 1, ' out of ', ntests - 1, ' with number of tests now ', len(names)) for j in range(i + 1, ntests): # Create the name for this combination. newname = ('&').join( sorted((names[i] + '&' + names[j]).split('&'))) if newname in names: continue # Do not keep multiple copies of the same combination. results = np.logical_and(tests[i], tests[j]) tpr, fpr, fnr, tnr = main.calcRates(results, truth) if tpr > 0.0: tests.append(results) tprs.append(tpr) fprs.append(fpr) names.append(newname) if verbose: print( 'Completed generation of tests, now constructing roc from number of tests: ', len(names)) # Create roc. used = np.zeros(len(names), dtype=bool) overallbest = return_cost(costratio, tpr, fpr) keepgoing = True while keepgoing: keepgoing = False besti = -1 bestcost = overallbest bestncomb = 100000 bestdtpr = 0 bestdfpr = 100000 for i in range(len(names)): if used[i]: continue cumulativenew = np.logical_or(cumulative, tests[i]) tpr, fpr, fnr, tnr = main.calcRates(cumulativenew, truth) dtpr = tpr - currenttpr dfpr = fpr - currentfpr newcost = return_cost(costratio, tpr, fpr) newbest = False if newcost <= bestcost and dtpr >= improve_threshold and dtpr > 0.0: # If cost is better than found previously, use it else if it is # the same then decide if to use it or not. if newcost < bestcost: newbest = True elif dtpr >= bestdtpr: if dtpr > bestdtpr: newbest = True elif len(names[i].split('&')) < bestncomb: newbest = True if newbest: besti = i bestcost = newcost bestncomb = len(names[i].split('&')) bestdtpr = dtpr bestdfpr = dfpr if besti >= 0: keepgoing = True used[besti] = True overallbest = bestcost cumulative = np.logical_or(cumulative, tests[besti]) currenttpr, currentfpr, fnr, tnr = main.calcRates( cumulative, truth) testcomb.append(names[besti]) r_fprs.append(currentfpr) r_tprs.append(currenttpr) groupsel.append(False) print('ROC point: ', currenttpr, currentfpr, names[besti], overallbest) if plot_roc: plt.plot(r_fprs, r_tprs, 'k') for i in range(len(r_fprs)): if groupsel[i]: colour = 'r' else: colour = 'b' plt.plot(r_fprs[i], r_tprs[i], colour + 'o') plt.xlim(0, 100) plt.ylim(0, 100) plt.xlabel('False positive rate (%)') plt.ylabel('True positive rate (%)') plt.savefig('roc.png') plt.close() if write_roc: f = open('roc.json', 'w') r = {} r['tpr'] = r_tprs r['fpr'] = r_fprs r['tests'] = testcomb r['groupsel'] = groupsel json.dump(r, f) f.close()
def test(p, parameters, allow_level_reinstating=True): """ Runs the quality control check on profile p and returns a numpy array of quality control decisions with False where the data value has passed the check and True where it failed. If allow_level_reinstating is set to True then rejected levels can be reprieved by comparing with levels above and below. NB this is done by default in EN processing. """ # Define an array to hold results. qc = np.zeros(p.n_levels(), dtype=bool) # Obtain the obs minus background differences on standard levels. result = stdLevelData(p, parameters) if result is None: return qc # Unpack the results. levels, origLevels, assocLevels = result # Retrieve the background and observation error variances and # the background values. query = 'SELECT bgstdlevels, bgevstdlevels FROM enbackground WHERE uid = ' + str(p.uid()) enbackground_pars = main.dbinteract(query) enbackground_pars = main.unpack_row(enbackground_pars[0]) bgsl = enbackground_pars[0] slev = parameters['enbackground']['depth'] bgev = enbackground_pars[1] obev = parameters['enbackground']['obev'] #find initial pge pgeData = determine_pge(levels, bgev, obev, p) # Find buddy. profiles = get_profile_info(parameters) minDist = 1000000000.0 iMinDist = None for iProfile, profile in enumerate(profiles): pDist = assessBuddyDistance(p, profile) if pDist is not None and pDist < minDist: minDist = pDist iMinDist = iProfile # Check if we have found a buddy and process if so. if minDist <= 400000: pBuddy = main.get_profile_from_db(profiles[iMinDist][0]) # buddy vetos Fail = False if pBuddy.var_index() is None: Fail = True if Fail == False: main.catchFlags(pBuddy) if np.sum(pBuddy.t().mask == False) == 0: Fail = True if Fail == False: result = stdLevelData(pBuddy, parameters) query = 'SELECT bgevstdlevels FROM enbackground WHERE uid = ' + str(pBuddy.uid()) buddy_pars = main.dbinteract(query) buddy_pars = main.unpack_row(buddy_pars[0]) if result is not None: levelsBuddy, origLevelsBuddy, assocLevelsBuddy = result bgevBuddy = buddy_pars[0] pgeBuddy = determine_pge(levels, bgevBuddy, obev, pBuddy) pgeData = update_pgeData(pgeData, pgeBuddy, levels, levelsBuddy, minDist, p, pBuddy, obev, bgev, bgevBuddy) # Check if levels should be reinstated. if allow_level_reinstating: if np.abs(p.latitude()) < 20.0: depthTol = 300.0 else: depthTol = 200.0 stdLevelFlags = pgeData >= 0.5 for i, slflag in enumerate(stdLevelFlags): if slflag: # Check for non rejected surrounding levels. okbelow = False if i > 0: if stdLevelFlags[i - 1] == False and levels.mask[i - 1] == False and bgsl.mask[i - 1] == False: okbelow = True okabove = False nsl = len(stdLevelFlags) if i < nsl - 1: if stdLevelFlags[i + 1] == False and levels.mask[i + 1] == False and bgsl.mask[i + 1] == False: okabove = True # Work out tolerances. if slev[i] > depthTol + 100: tolFactor = 0.5 elif slev[i] > depthTol: tolFactor = 1.0 - 0.005 * (slev[i] - depthTol) else: tolFactor = 1.0 ttol = 0.5 * tolFactor if okbelow == True and okabove == True: xmax = levels[i - 1] + bgsl[i - 1] + ttol xmin = levels[i + 1] + bgsl[i + 1] - ttol elif okbelow == True: xmax = levels[i - 1] + bgsl[i - 1] + ttol xmin = levels[i - 1] + bgsl[i - 1] - ttol elif okabove == True: xmax = levels[i + 1] + bgsl[i + 1] + ttol xmin = levels[i + 1] + bgsl[i + 1] - ttol else: continue # Reassign PGE if level is within the tolerances. if levels[i] + bgsl[i] >= xmin and levels[i] + bgsl[i] <= xmax: pgeData[i] = 0.49 # Assign the QC flags to original levels. for i, pge in enumerate(pgeData): if pgeData.mask[i]: continue if pge < 0.5: continue for j, assocLevel in enumerate(assocLevels): if assocLevel == i: origLevel = origLevels[j] qc[origLevel] = True return qc
def get_profile_info(parameters): # Gets information about the profiles from the database. query = 'SELECT uid,year,month,cruise,lat,long FROM ' + parameters['table'] return main.dbinteract(query)
def tearDown(self): main.dbinteract('DROP TABLE unit;')
# Read data from database into a pandas data frame. df = dbutils.db_to_df(sys.argv[1], filter_on_wire_break_test = False, filter_on_tests = groupdefinition, n_to_extract = sys.argv[2]) testNames = df.columns[2:].values.tolist() # declare some downstream constructs accepted = [] unflagged = [] fprs = [] bad = df.loc[df['Truth']] bad.reset_index(inplace=True, drop=True) # mark chosen profiles as part of the training set all_uids = main.dbinteract('SELECT uid from ' + sys.argv[1] + ';') for uid in all_uids: uid = uid[0] is_training = int(uid in df['uid'].astype(int).as_matrix()) query = "UPDATE " + sys.argv[1] + " SET training=" + str(is_training) + " WHERE uid=" + str(uid) + ";" main.dbinteract(query) # algo. step 0: # demand individual QC tests have TPR/FPR > some threshold perf_thresh = 2 drop_tests = [] for test in testNames: tpr, fpr, fnr, tnr = main.calcRates(df[test].tolist(), df['Truth'].tolist()) if fpr > 0 and tpr / fpr < perf_thresh: print 'dropping', test, '; tpr/fpr = ', tpr/fpr df.drop([test], axis=1)
# keep tabs on how many good and how many bad profiles have been added to db # nowire == index of first wire break level wireqc = qctests.CSIRO_wire_break.test(profile, {}) try: nowire = list(wireqc).index(True) except: nowire = len(truth) # flag only counts if its before the wire break: flagged = dbutils.summarize_truth(truth[0:nowire]) if flagged: bad += 1 else: good += 1 query = "INSERT INTO " + sys.argv[2] + " (raw, truth, uid, year, month, day, time, lat, long, cruise, probe) values (?,?,?,?,?,?,?,?,?,?,?);" values = (p['raw'], p['truth'], p['uid'], p['year'], p['month'], p['day'], p['time'], p['latitude'], p['longitude'], p['cruise'], p['probe_type']) main.dbinteract(query, values) if profile.is_last_profile_in_file(fid) == True: break conn.commit() print 'number of clean profiles written:', good print 'number of flagged profiles written:', bad else: print 'Usage: python build-db.py inputdatafile databasetable'
def tearDown(self): main.dbinteract('DROP TABLE icdclevelorder;')
# set up global parmaeter store parameterStore = { "table": dbtable, "db": targetdb } for test in testNames: exec('from qctests import ' + test) try: exec(test + '.loadParameters(parameterStore)') except: print('No parameters to load for', test) # connect to database & fetch list of all uids query = 'SELECT uid FROM ' + dbtable + ' ORDER BY uid;' uids = main.dbinteract(query, targetdb=targetdb) # launch async processes if batchnumber is not None and nperbatch is not None: batchnumber = int(batchnumber) nperbatch = int(nperbatch) startindex = batchnumber*nperbatch endindex = min((batchnumber+1)*nperbatch,len(uids)) else: startindex = 0 endindex = len(uids) pool = Pool(processes=int(cores)) for i in range(startindex, endindex): pool.apply_async(process_row, (uids[i][0], logdir, dbtable, targetdb)) pool.close() pool.join()
def tearDown(self): main.dbinteract('DROP TABLE unit;') main.catchFlags = realcatchflagsfunc main.get_profile_from_db = realgetproffunc
def find_roc(table, filter_on_wire_break_test=True, n_profiles_to_analyse=np.iinfo(np.int32).max, n_combination_iterations=2, with_reverses=False, improve_threshold=1.0, verbose=True, plot_roc=True, write_roc=True): ''' Generates a ROC curve from the database data in table by maximising the gradient of the ROC curve. It will combine different tests together and invert the results of tests if requested. filter_on_wire_break_test - filter out the impact of XBT wire breaks from results. n_profiles_to_analyse - restrict the number of profiles extracted from the database. n_combination_iterations - AND tests together; restricted to max of 2 as otherwise number of tests gets very large. with_reverses - if True, a copy of each test with inverted results is made. improve_threshold - ignores tests if they do not results in a change in true positive rate (in %) of at least this amount. verbose - if True, will print a lot of messages to screen. plot_roc - if True, will save an image of the ROC to roc.png. write_roc - if True, will save the ROC data to roc.json. ''' # Read data from database into a pandas data frame. df = dbutils.db_to_df(sys.argv[1], filter_on_wire_break_test=filter_on_wire_break_test, n_to_extract=n_profiles_to_analyse) # mark chosen profiles as part of the training set all_uids = main.dbinteract('SELECT uid from ' + sys.argv[1] + ';') for uid in all_uids: uid = uid[0] is_training = int(uid in df['uid'].astype(int).as_matrix()) query = "UPDATE " + sys.argv[1] + " SET training=" + str(is_training) + " WHERE uid=" + str(uid) + ";" main.dbinteract(query) # drop nondiscriminating tests nondiscrim = [] cols = list(df.columns) for c in cols: if len(pandas.unique(df[c])) == 1: nondiscrim.append(c) cols = [t for t in cols if t not in nondiscrim] df = df[cols] print list(df) testNames = df.columns[2:].values.tolist() if verbose: print 'Number of profiles from database was: ', len(df.index) print 'Number of quality checks from database was: ', len(testNames) # Convert to numpy structures and make inverse versions of tests if required. # Any test with true positive rate of zero is discarded. truth = df['Truth'].as_matrix() tests = [] names = [] tprs = [] fprs = [] if with_reverses: reverselist = [False, True] else: reverselist = [False] for i, testname in enumerate(testNames): for reversal in reverselist: results = df[testname].as_matrix() != reversal tpr, fpr, fnr, tnr = main.calcRates(results, truth) if tpr > 0.0: tests.append(results) if reversal: addtext = 'r' else: addtext = '' names.append(addtext + testname) tprs.append(tpr) fprs.append(fpr) del df # No further need for the data frame. if verbose: print 'Number of quality checks after reverses and removing zero TPR was: ', len(names) # Make combinations of the single checks and store. assert n_combination_iterations <= 2, 'Setting n_combination_iterations > 2 results in a very large number of combinations' if verbose: print 'Starting construction of combinations with number of iterations: ', n_combination_iterations for its in range(n_combination_iterations): ntests = len(names) for i in range(ntests - 1): if verbose: print 'Processing iteration ', its + 1, ' out of ', n_combination_iterations, ' step ', i + 1, ' out of ', ntests - 1, ' with number of tests now ', len(names) for j in range(i + 1, ntests): # Create the name for this combination. newname = ('&').join(sorted((names[i] + '&' + names[j]).split('&'))) if newname in names: continue # Do not keep multiple copies of the same combination. results = np.logical_and(tests[i], tests[j]) tpr, fpr, fnr, tnr = main.calcRates(results, truth) if tpr > 0.0: tests.append(results) tprs.append(tpr) fprs.append(fpr) names.append(newname) if verbose: print 'Completed generation of tests, now constructing roc from number of tests: ', len(names) # Create storage to hold the roc curve. cumulative = truth.copy() cumulative[:] = False currenttpr = 0.0 currentfpr = 0.0 used = np.zeros(len(names), dtype=bool) r_fprs = [] r_tprs = [] # Create roc by keep adding tests in order of ratio of tpr/fpr change to get the highest # gradient in the roc curve. keepgoing = True used = np.zeros(len(names), dtype=bool) testcomb = [] while keepgoing: keepgoing = False besti = -1 bestratio = 0.0 bestncomb = 100000 bestdtpr = 0 bestdfpr = 100000 for i in range(len(names)): if used[i]: continue cumulativenew = np.logical_or(cumulative, tests[i]) tpr, fpr, fnr, tnr = main.calcRates(cumulativenew, truth) dtpr = tpr - currenttpr dfpr = max(fpr - currentfpr, 0.1 / len(cumulative)) # In case of 0 change in fpr. ratio = dtpr / dfpr newbest = False if ratio >= bestratio and dtpr >= improve_threshold and dtpr > 0.0: # If ration is better than found previously, use it else if it is # the same then decide if to use it or not. if ratio > bestratio: newbest = True elif dtpr >= bestdtpr: if dtpr > bestdtpr: newbest = True elif len(names[i].split('&')) < bestncomb: newbest = True if newbest: besti = i bestratio = ratio bestncomb = len(names[i].split('&')) bestdtpr = dtpr bestdfpr = dfpr if besti >= 0: keepgoing = True used[besti] = True cumulative = np.logical_or(cumulative, tests[besti]) currenttpr, currentfpr, fnr, tnr = main.calcRates(cumulative, truth) testcomb.append(names[besti]) r_fprs.append(currentfpr) r_tprs.append(currenttpr) print 'ROC point: ', currenttpr, currentfpr, names[besti] if plot_roc: plt.plot(r_fprs, r_tprs) plt.xlim(0, 100) plt.ylim(0, 100) plt.xlabel('False positive rate (%)') plt.ylabel('True positive rate (%)') plt.savefig('roc.png') plt.close() if write_roc: f = open('roc.json', 'w') r = {} r['tpr'] = r_tprs r['fpr'] = r_fprs r['tests'] = testcomb json.dump(r, f) f.close()
def builddb(infile, check_originator_flag_type=True, months_to_use=range(1, 13), outfile='iquod.db', dbtable='iquod'): conn = sqlite3.connect(outfile, isolation_level=None) cur = conn.cursor() # Identify tests testNames = main.importQC('qctests') testNames.sort() # set up our table query = "CREATE TABLE IF NOT EXISTS " + dbtable + """( raw text, truth BLOB, uid integer PRIMARY KEY, year integer, month integer, day integer, time real, lat real, long real, country text, cruise integer, ocruise text, probe integer, training integer, flagged integer, """ for i in range(len(testNames)): query += testNames[i].lower() + ' BLOB' if i < len(testNames) - 1: query += ',' else: query += ');' cur.execute(query) # populate table from wod-ascii data fid = open(infile) uids = [] good = 0 bad = 0 while True: # extract profile as wodpy object and raw text start = fid.tell() profile = wod.WodProfile(fid) end = fid.tell() fid.seek(start) raw = fid.read(end - start) fid.seek(end) # set up dictionary for populating query string p = profile.npdict() p['raw'] = "'" + raw + "'" # check for duplicate profiles in raw data if p['uid'] in uids: if profile.is_last_profile_in_file(fid) == True: break else: continue uids.append(p['uid']) # skip pathological profiles isgood = assessProfile(profile, check_originator_flag_type, months_to_use) if not isgood and profile.is_last_profile_in_file(fid) == True: break elif not isgood: continue # encode temperature error codes into truth array truth = encodeTruth(profile) p['truth'] = main.pack_array(truth) # extract country code country = profile.primary_header['Country code'] # originator cruise orig_cruise = profile.originator_cruise() # keep tabs on how many good and how many bad profiles have been added to db # nowire == index of first wire break level wireqc = qctests.CSIRO_wire_break.test(profile, {}) try: nowire = list(wireqc).index(True) except: nowire = len(truth) # flag only counts if its before the wire break: flagged = dbutils.summarize_truth(truth[0:nowire]) if flagged: bad += 1 else: good += 1 query = "INSERT INTO " + dbtable + " (raw, truth, uid, year, month, day, time, lat, long, country, cruise, ocruise, probe, flagged) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?);" values = (p['raw'], p['truth'], p['uid'], p['year'], p['month'], p['day'], p['time'], p['latitude'], p['longitude'], country, p['cruise'], orig_cruise, p['probe_type'], int(flagged)) main.dbinteract(query, values, targetdb=outfile) if profile.is_last_profile_in_file(fid) == True: break conn.commit() print('number of clean profiles written:', good) print('number of flagged profiles written:', bad) print('total number of profiles written:', good + bad)
def tearDown(self): main.dbinteract('DROP TABLE unit;') main.catchFlags = realcatchflagsfunc main.get_profile_from_db = realgetproffunc
# Parallel processing. print('\nPlease wait while QC is performed\n') # set up global parmaeter store parameterStore = {"table": sys.argv[1]} for test in testNames: exec('from qctests import ' + test) try: exec(test + '.loadParameters(parameterStore)') except: print 'No parameters to load for', test # connect to database & fetch list of all uids query = 'SELECT uid FROM ' + sys.argv[1] + ' ORDER BY uid;' uids = main.dbinteract(query) # launch async processes if len(sys.argv) > 4: batchnumber = int(sys.argv[3]) nperbatch = int(sys.argv[4]) startindex = batchnumber * nperbatch endindex = min((batchnumber + 1) * nperbatch, len(uids)) else: startindex = 0 endindex = len(uids) pool = Pool(processes=int(sys.argv[2])) for i in range(startindex, endindex): pool.apply_async(process_row, (uids[i][0], logdir)) pool.close() pool.join()
df = dbutils.db_to_df(table=dbtable, targetdb=targetdb, filter_on_wire_break_test=False, filter_on_tests=groupdefinition, n_to_extract=samplesize) testNames = df.columns[2:].values.tolist() # declare some downstream constructs accepted = [] unflagged = [] fprs = [] bad = df.loc[df['Truth']] bad.reset_index(inplace=True, drop=True) # mark chosen profiles as part of the training set all_uids = main.dbinteract('SELECT uid from ' + dbtable + ';', targetdb=targetdb) for uid in all_uids: uid = uid[0] is_training = int(uid in df['uid'].astype(int).as_matrix()) query = "UPDATE " + dbtable + " SET training=" + str( is_training) + " WHERE uid=" + str(uid) + ";" main.dbinteract(query, targetdb=targetdb) # algo. step 0: # demand individual QC tests have TPR/FPR > some threshold perf_thresh = 2 drop_tests = [] for test in testNames: tpr, fpr, fnr, tnr = main.calcRates(df[test].tolist(), df['Truth'].tolist()) if fpr > 0 and tpr / fpr < perf_thresh:
def tearDown(self): main.dbinteract('DROP TABLE unit;')
# Parallel processing. print('\nPlease wait while QC is performed\n') # set up global parmaeter store parameterStore = { "table": sys.argv[1] } for test in testNames: exec('from qctests import ' + test) try: exec(test + '.loadParameters(parameterStore)') except: print 'No parameters to load for', test # connect to database & fetch list of all uids query = 'SELECT uid FROM ' + sys.argv[1] + ' ORDER BY uid;' uids = main.dbinteract(query) # launch async processes pool = Pool(processes=int(sys.argv[2])) for i in range(len(uids)): pool.apply_async(process_row, (uids[i][0], logdir)) pool.close() pool.join() else: print 'Please add command line arguments to name your output file and set parallelization:' print 'python AutoQC <database results table> <number of processes>' print 'will use <database results table> to log QC results in the database, and run the calculation parallelized over <number of processes>.'