def append_category(tests, tablename, database='iquod.db'):
'''
tests: list of test names or combinations of tests joind by &
tablename: sql table to extract from
database: filename of database file
add a column 'category' to a qc table that indicates whether a profile is correctly or incorrectly flagged by
the tests array from an analyze-results or catchall output, encoded as:
0 == true positive
1 == true negative
2 == false positive
3 == false negative
'''
# extract dataframe and apply discriminator
#df = query2df(['truth', 'uid'], '', tablename, database)
df = dbutils.db_to_df(tablename,
filter_on_tests={
'Remove rejected levels':
['CSIRO_depth', 'CSIRO_wire_break']
})
discriminator = construct_discrim(tests)
df['qc'] = df.apply(discriminator, axis=1)
# create empty 'category' column in db
conn = sqlite3.connect(database, isolation_level=None)
cur = conn.cursor()
#query = 'ALTER TABLE ' + tablename + ' ADD category integer;'
#cur.execute(query)
# write results to database
def update_db(row):
if row['qc'] and row['Truth']:
category = 0
elif not row['qc'] and not row['Truth']:
category = 1
elif row['qc'] and not row['Truth']:
category = 2
elif not row['qc'] and row['Truth']:
category = 3
query = 'UPDATE ' + tablename + ' SET category = ' + str(
category) + ' WHERE uid=' + str(row['uid'])
cur.execute(query)
df.apply(update_db, axis=1)
for test in combo[1:]:
decision = decision & df[test]
name = '&'.join(combo)
return df.assign(xx=decision).rename(index=str, columns={'xx': name})
print('==============')
print(sys.argv[1])
print('==============')
# Read QC test specifications if required.
groupdefinition = ar.read_qc_groups()
# Read data from database into a pandas data frame.
df = dbutils.db_to_df(sys.argv[1],
filter_on_wire_break_test = False,
filter_on_tests = groupdefinition,
n_to_extract = sys.argv[2])
testNames = df.columns[2:].values.tolist()
# declare some downstream constructs
accepted = []
unflagged = []
fprs = []
bad = df.loc[df['Truth']]
bad.reset_index(inplace=True, drop=True)
# mark chosen profiles as part of the training set
all_uids = main.dbinteract('SELECT uid from ' + sys.argv[1] + ';')
for uid in all_uids:
uid = uid[0]
is_training = int(uid in df['uid'].astype(int).as_matrix())
def find_roc(table,
costratio=[2.5, 1.0],
filter_on_wire_break_test=False,
filter_from_file_spec=True,
enforce_types_of_check=True,
n_profiles_to_analyse=np.iinfo(np.int32).max,
n_combination_iterations=1,
with_reverses=False,
effectiveness_ratio=2.0,
improve_threshold=1.0,
verbose=True,
plot_roc=True,
write_roc=True):
'''
Generates a ROC curve from the database data in table by maximising the gradient
of the ROC curve. It will combine different tests together and invert the results
of tests if requested.
costratio - two element iterable that defines how the ROC curve is developed. Higher
numbers gives a ROC curve with lower false rates; the two elements allows
control over the shape of the ROC curve near the start and end. E.g. [2.5, 1.0].
filter_on_wire_break_test - filter out the impact of XBT wire breaks from results.
filter_from_file_spec - use specification from file to choose filtering.
enforce_types_of_check - use specification from file on particular types of checks to use.
n_profiles_to_analyse - restrict the number of profiles extracted from the database.
n_combination_iterations - AND tests together; restricted to max of 2 as otherwise
number of tests gets very large.
with_reverses - if True, a copy of each test with inverted results is made.
effectiveness_ratio - will give a warning if TPR / FPR is less than this value.
improve_threshold - ignores tests if they do not results in a change in true positive
rate (in %) of at least this amount.
verbose - if True, will print a lot of messages to screen.
plot_roc - if True, will save an image of the ROC to roc.png.
write_roc - if True, will save the ROC data to roc.json.
'''
# Read QC test specifications if required.
groupdefinition = {}
if filter_from_file_spec or enforce_types_of_check:
groupdefinition = read_qc_groups()
# Read data from database into a pandas data frame.
df = dbutils.db_to_df(sys.argv[1],
filter_on_wire_break_test=filter_on_wire_break_test,
filter_on_tests=groupdefinition,
n_to_extract=n_profiles_to_analyse)
# Drop nondiscriminating tests
nondiscrim = []
cols = list(df.columns)
for c in cols:
if len(pandas.unique(df[c])) == 1:
nondiscrim.append(c)
if verbose: print(c + ' is nondiscriminating and will be removed')
cols = [t for t in cols if t not in nondiscrim]
df = df[cols]
print(list(df))
testNames = df.columns[2:].values.tolist()
if verbose:
print('Number of profiles is: ', len(df.index))
print('Number of quality checks to process is: ', len(testNames))
# mark chosen profiles as part of the training set
all_uids = main.dbinteract('SELECT uid from ' + sys.argv[1] + ';')
for uid in all_uids:
uid = uid[0]
is_training = int(uid in df['uid'].astype(int).as_matrix())
query = "UPDATE " + sys.argv[1] + " SET training=" + str(
is_training) + " WHERE uid=" + str(uid) + ";"
main.dbinteract(query)
# Convert to numpy structures and make inverse versions of tests if required.
# Any test with true positive rate of zero is discarded.
truth = df['Truth'].as_matrix()
tests = []
names = []
tprs = []
fprs = []
if with_reverses:
reverselist = [False, True]
else:
reverselist = [False]
for i, testname in enumerate(testNames):
for reversal in reverselist:
results = df[testname].as_matrix() != reversal
tpr, fpr, fnr, tnr = main.calcRates(results, truth)
if tpr > 0.0:
tests.append(results)
if reversal:
addtext = 'r'
else:
addtext = ''
names.append(addtext + testname)
tprs.append(tpr)
fprs.append(fpr)
del df # No further need for the data frame.
if verbose:
print(
'Number of quality checks after adding reverses and removing zero TPR was: ',
len(names))
# Create storage to hold the roc curve.
cumulative = truth.copy()
cumulative[:] = False
currenttpr = 0.0
currentfpr = 0.0
r_fprs = [] # The false positive rate for each ROC point.
r_tprs = [] # True positive rate for each ROC point.
testcomb = [] # The QC test that was added at each ROC point.
groupsel = [] # Set to True if the ROC point was from an enforced group.
# Pre-select some tests if required.
if enforce_types_of_check:
if verbose: print('Enforcing types of checks')
while len(groupdefinition['At least one from group']) > 0:
bestchoice = ''
bestgroup = ''
bestdist = np.sqrt(100.0**2 + 100.0**2)
besti = -1
for key in groupdefinition['At least one from group']:
for testname in groupdefinition['At least one from group'][
key]:
# Need to check that the test exists - it may have been removed
# if it was non-discriminating.
if testname in names:
for itest, name in enumerate(names):
if name == testname:
cumulativenew = np.logical_or(
cumulative, tests[itest])
tpr, fpr, fnr, tnr = main.calcRates(
cumulativenew, truth)
newdist = return_cost(costratio, tpr, fpr)
print(' ', tpr, fpr, newdist, bestdist,
testname)
if newdist == bestdist:
if verbose:
print(
' ' + bestchoice + ' and ' +
testname +
' have the same results and the first is kept'
)
elif newdist < bestdist:
bestchoice = testname
bestdist = newdist
besti = itest
bestgroup = key
else:
if verbose:
print(' ' + testname +
' not found and so was skipped')
#assert bestchoice != '', ' Error, did not make a choice in group ' + key
if verbose:
print(' ' + bestchoice + ' was selected from group ' +
bestgroup)
if fprs[besti] > 0:
if tprs[besti] / fprs[besti] < effectiveness_ratio:
print(
'WARNING - ' + bestchoice +
' TPR / FPR is below the effectiveness ratio limit: ',
tprs[besti] / fprs[besti], effectiveness_ratio)
cumulative = np.logical_or(cumulative, tests[besti])
currenttpr, currentfpr, fnr, tnr = main.calcRates(
cumulative, truth)
testcomb.append(names[besti])
r_fprs.append(currentfpr)
r_tprs.append(currenttpr)
groupsel.append(True)
# Once a test has been added, it can be deleted so that it is not considered again.
del names[besti]
del tests[besti]
del fprs[besti]
del tprs[besti]
del groupdefinition['At least one from group'][bestgroup]
print('ROC point from enforced group: ', currenttpr, currentfpr,
testcomb[-1], bestgroup)
# Make combinations of the single checks and store.
assert n_combination_iterations <= 2, 'Setting n_combination_iterations > 2 results in a very large number of combinations'
if verbose:
print(
'Starting construction of combinations with number of iterations: ',
n_combination_iterations)
for its in range(n_combination_iterations):
ntests = len(names)
for i in range(ntests - 1):
if verbose:
print('Processing iteration ', its + 1, ' out of ',
n_combination_iterations, ' step ', i + 1, ' out of ',
ntests - 1, ' with number of tests now ', len(names))
for j in range(i + 1, ntests):
# Create the name for this combination.
newname = ('&').join(
sorted((names[i] + '&' + names[j]).split('&')))
if newname in names:
continue # Do not keep multiple copies of the same combination.
results = np.logical_and(tests[i], tests[j])
tpr, fpr, fnr, tnr = main.calcRates(results, truth)
if tpr > 0.0:
tests.append(results)
tprs.append(tpr)
fprs.append(fpr)
names.append(newname)
if verbose:
print(
'Completed generation of tests, now constructing roc from number of tests: ',
len(names))
# Create roc.
used = np.zeros(len(names), dtype=bool)
overallbest = return_cost(costratio, tpr, fpr)
keepgoing = True
while keepgoing:
keepgoing = False
besti = -1
bestcost = overallbest
bestncomb = 100000
bestdtpr = 0
bestdfpr = 100000
for i in range(len(names)):
if used[i]: continue
cumulativenew = np.logical_or(cumulative, tests[i])
tpr, fpr, fnr, tnr = main.calcRates(cumulativenew, truth)
dtpr = tpr - currenttpr
dfpr = fpr - currentfpr
newcost = return_cost(costratio, tpr, fpr)
newbest = False
if newcost <= bestcost and dtpr >= improve_threshold and dtpr > 0.0:
# If cost is better than found previously, use it else if it is
# the same then decide if to use it or not.
if newcost < bestcost:
newbest = True
elif dtpr >= bestdtpr:
if dtpr > bestdtpr:
newbest = True
elif len(names[i].split('&')) < bestncomb:
newbest = True
if newbest:
besti = i
bestcost = newcost
bestncomb = len(names[i].split('&'))
bestdtpr = dtpr
bestdfpr = dfpr
if besti >= 0:
keepgoing = True
used[besti] = True
overallbest = bestcost
cumulative = np.logical_or(cumulative, tests[besti])
currenttpr, currentfpr, fnr, tnr = main.calcRates(
cumulative, truth)
testcomb.append(names[besti])
r_fprs.append(currentfpr)
r_tprs.append(currenttpr)
groupsel.append(False)
print('ROC point: ', currenttpr, currentfpr, names[besti],
overallbest)
if plot_roc:
plt.plot(r_fprs, r_tprs, 'k')
for i in range(len(r_fprs)):
if groupsel[i]:
colour = 'r'
else:
colour = 'b'
plt.plot(r_fprs[i], r_tprs[i], colour + 'o')
plt.xlim(0, 100)
plt.ylim(0, 100)
plt.xlabel('False positive rate (%)')
plt.ylabel('True positive rate (%)')
plt.savefig('roc.png')
plt.close()
if write_roc:
f = open('roc.json', 'w')
r = {}
r['tpr'] = r_tprs
r['fpr'] = r_fprs
r['tests'] = testcomb
r['groupsel'] = groupsel
json.dump(r, f)
f.close()
def find_roc(table,
filter_on_wire_break_test=True,
n_profiles_to_analyse=np.iinfo(np.int32).max,
n_combination_iterations=2,
with_reverses=False,
improve_threshold=1.0,
verbose=True,
plot_roc=True,
write_roc=True):
'''
Generates a ROC curve from the database data in table by maximising the gradient
of the ROC curve. It will combine different tests together and invert the results
of tests if requested.
filter_on_wire_break_test - filter out the impact of XBT wire breaks from results.
n_profiles_to_analyse - restrict the number of profiles extracted from the database.
n_combination_iterations - AND tests together; restricted to max of 2 as otherwise
number of tests gets very large.
with_reverses - if True, a copy of each test with inverted results is made.
improve_threshold - ignores tests if they do not results in a change in true positive
rate (in %) of at least this amount.
verbose - if True, will print a lot of messages to screen.
plot_roc - if True, will save an image of the ROC to roc.png.
write_roc - if True, will save the ROC data to roc.json.
'''
# Read data from database into a pandas data frame.
df = dbutils.db_to_df(sys.argv[1],
filter_on_wire_break_test=filter_on_wire_break_test,
n_to_extract=n_profiles_to_analyse)
# mark chosen profiles as part of the training set
all_uids = main.dbinteract('SELECT uid from ' + sys.argv[1] + ';')
for uid in all_uids:
uid = uid[0]
is_training = int(uid in df['uid'].astype(int).as_matrix())
query = "UPDATE " + sys.argv[1] + " SET training=" + str(is_training) + " WHERE uid=" + str(uid) + ";"
main.dbinteract(query)
# drop nondiscriminating tests
nondiscrim = []
cols = list(df.columns)
for c in cols:
if len(pandas.unique(df[c])) == 1:
nondiscrim.append(c)
cols = [t for t in cols if t not in nondiscrim]
df = df[cols]
print list(df)
testNames = df.columns[2:].values.tolist()
if verbose:
print 'Number of profiles from database was: ', len(df.index)
print 'Number of quality checks from database was: ', len(testNames)
# Convert to numpy structures and make inverse versions of tests if required.
# Any test with true positive rate of zero is discarded.
truth = df['Truth'].as_matrix()
tests = []
names = []
tprs = []
fprs = []
if with_reverses:
reverselist = [False, True]
else:
reverselist = [False]
for i, testname in enumerate(testNames):
for reversal in reverselist:
results = df[testname].as_matrix() != reversal
tpr, fpr, fnr, tnr = main.calcRates(results, truth)
if tpr > 0.0:
tests.append(results)
if reversal:
addtext = 'r'
else:
addtext = ''
names.append(addtext + testname)
tprs.append(tpr)
fprs.append(fpr)
del df # No further need for the data frame.
if verbose: print 'Number of quality checks after reverses and removing zero TPR was: ', len(names)
# Make combinations of the single checks and store.
assert n_combination_iterations <= 2, 'Setting n_combination_iterations > 2 results in a very large number of combinations'
if verbose: print 'Starting construction of combinations with number of iterations: ', n_combination_iterations
for its in range(n_combination_iterations):
ntests = len(names)
for i in range(ntests - 1):
if verbose: print 'Processing iteration ', its + 1, ' out of ', n_combination_iterations, ' step ', i + 1, ' out of ', ntests - 1, ' with number of tests now ', len(names)
for j in range(i + 1, ntests):
# Create the name for this combination.
newname = ('&').join(sorted((names[i] + '&' + names[j]).split('&')))
if newname in names: continue # Do not keep multiple copies of the same combination.
results = np.logical_and(tests[i], tests[j])
tpr, fpr, fnr, tnr = main.calcRates(results, truth)
if tpr > 0.0:
tests.append(results)
tprs.append(tpr)
fprs.append(fpr)
names.append(newname)
if verbose: print 'Completed generation of tests, now constructing roc from number of tests: ', len(names)
# Create storage to hold the roc curve.
cumulative = truth.copy()
cumulative[:] = False
currenttpr = 0.0
currentfpr = 0.0
used = np.zeros(len(names), dtype=bool)
r_fprs = []
r_tprs = []
# Create roc by keep adding tests in order of ratio of tpr/fpr change to get the highest
# gradient in the roc curve.
keepgoing = True
used = np.zeros(len(names), dtype=bool)
testcomb = []
while keepgoing:
keepgoing = False
besti = -1
bestratio = 0.0
bestncomb = 100000
bestdtpr = 0
bestdfpr = 100000
for i in range(len(names)):
if used[i]: continue
cumulativenew = np.logical_or(cumulative, tests[i])
tpr, fpr, fnr, tnr = main.calcRates(cumulativenew, truth)
dtpr = tpr - currenttpr
dfpr = max(fpr - currentfpr, 0.1 / len(cumulative)) # In case of 0 change in fpr.
ratio = dtpr / dfpr
newbest = False
if ratio >= bestratio and dtpr >= improve_threshold and dtpr > 0.0:
# If ration is better than found previously, use it else if it is
# the same then decide if to use it or not.
if ratio > bestratio:
newbest = True
elif dtpr >= bestdtpr:
if dtpr > bestdtpr:
newbest = True
elif len(names[i].split('&')) < bestncomb:
newbest = True
if newbest:
besti = i
bestratio = ratio
bestncomb = len(names[i].split('&'))
bestdtpr = dtpr
bestdfpr = dfpr
if besti >= 0:
keepgoing = True
used[besti] = True
cumulative = np.logical_or(cumulative, tests[besti])
currenttpr, currentfpr, fnr, tnr = main.calcRates(cumulative, truth)
testcomb.append(names[besti])
r_fprs.append(currentfpr)
r_tprs.append(currenttpr)
print 'ROC point: ', currenttpr, currentfpr, names[besti]
if plot_roc:
plt.plot(r_fprs, r_tprs)
plt.xlim(0, 100)
plt.ylim(0, 100)
plt.xlabel('False positive rate (%)')
plt.ylabel('True positive rate (%)')
plt.savefig('roc.png')
plt.close()
if write_roc:
f = open('roc.json', 'w')
r = {}
r['tpr'] = r_tprs
r['fpr'] = r_fprs
r['tests'] = testcomb
json.dump(r, f)
f.close()
for test in combo[1:]:
decision = decision & df[test]
name = '&'.join(combo)
return df.assign(xx=decision).rename(index=str, columns={'xx': name})
print '=============='
print sys.argv[1]
print '=============='
# Read QC test specifications if required.
groupdefinition = ar.read_qc_groups()
# Read data from database into a pandas data frame.
df = dbutils.db_to_df(sys.argv[1],
filter_on_wire_break_test = False,
filter_on_tests = groupdefinition,
n_to_extract = sys.argv[2])
testNames = df.columns[2:].values.tolist()
# declare some downstream constructs
accepted = []
unflagged = []
fprs = []
bad = df.loc[df['Truth']]
bad.reset_index(inplace=True, drop=True)
# mark chosen profiles as part of the training set
all_uids = main.dbinteract('SELECT uid from ' + sys.argv[1] + ';')
for uid in all_uids:
uid = uid[0]
is_training = int(uid in df['uid'].astype(int).as_matrix())
import sys
import util.dbutils as dbutils
import util.main as main
'''
Prints a simple summary of true and false, postive and negative rates.
Usage: python summarize-results.py databasetable
'''
if len(sys.argv) == 2:
df = dbutils.db_to_df(sys.argv[1])
testNames = df.columns[1:].values.tolist()
print('%35s %7s %7s %7s %7s %7s' %
('NAME OF TEST', 'FAILS', 'TPR', 'FPR', 'TNR', 'FNR'))
for test in testNames:
tpr, fpr, fnr, tnr = main.calcRates(df[test].tolist(),
df['Truth'].tolist())
print('%35s %7i %6.1f%1s %6.1f%1s %6.1f%1s %6.1f%1s' %
(test, sum(
df[test].tolist()), tpr, '%', fpr, '%', tnr, '%', fnr, '%'))
else:
print('Usage: python summarize-results.py databasetable')
print('-d <db table name to read from>')
print('-t <name of db file>')
print('-n <number of profiles to consider>')
print('-o <filename to write json results out to>')
print('-h print this help message and quit')
if samplesize is None:
print('please provide a sample size to consider with the -n flag')
print('-h to print usage')
# Read QC test specifications if required.
groupdefinition = ar.read_qc_groups()
# Read data from database into a pandas data frame.
df = dbutils.db_to_df(table=dbtable,
targetdb=targetdb,
filter_on_wire_break_test=False,
filter_on_tests=groupdefinition,
n_to_extract=samplesize)
testNames = df.columns[2:].values.tolist()
# declare some downstream constructs
accepted = []
unflagged = []
fprs = []
bad = df.loc[df['Truth']]
bad.reset_index(inplace=True, drop=True)
# mark chosen profiles as part of the training set
all_uids = main.dbinteract('SELECT uid from ' + dbtable + ';',
targetdb=targetdb)
for uid in all_uids:
Usage: python summarize-results.py -t <db filename> -d <table name>
'''
# parse options
options, remainder = getopt.getopt(sys.argv[1:], 't:d:h')
targetdb = 'iquod.db'
dbtable = 'iquod'
for opt, arg in options:
if opt == '-d':
dbtable = arg
if opt == '-t':
targetdb = arg
if opt == '-h':
print('usage:')
print('-d <db table name to read from>')
print('-t <name of db file>')
print('-h print this help message and quit')
df = dbutils.db_to_df(table=dbtable, targetdb=targetdb)
testNames = df.columns[1:].values.tolist()
print('{0:>35s} {1:>7s} {2:>7s} {3:>7s} {4:>7s} {5:>7s}'.format(
'NAME OF TEST', 'FAILS', 'TPR', 'FPR', 'TNR', 'FNR'))
for test in testNames:
tpr, fpr, fnr, tnr = main.calcRates(df[test].tolist(),
df['Truth'].tolist())
print(
'{0:>35s} {1:7d} {2:6.1f}{3:1s} {4:6.1f}{5:1s} {6:6.1f}{7:1s} {8:6.1f}{9:1s}'
.format(test, sum(df[test].tolist()), tpr, '%', fpr, '%', tnr, '%',
fnr, '%'))
def find_roc(table,
costratio=[2.5, 1.0],
filter_on_wire_break_test=False,
filter_from_file_spec=True,
enforce_types_of_check=True,
n_profiles_to_analyse=np.iinfo(np.int32).max,
n_combination_iterations=1,
with_reverses=False,
effectiveness_ratio=2.0,
improve_threshold=1.0,
verbose=True,
plot_roc=True,
write_roc=True):
'''
Generates a ROC curve from the database data in table by maximising the gradient
of the ROC curve. It will combine different tests together and invert the results
of tests if requested.
costratio - two element iterable that defines how the ROC curve is developed. Higher
numbers gives a ROC curve with lower false rates; the two elements allows
control over the shape of the ROC curve near the start and end. E.g. [2.5, 1.0].
filter_on_wire_break_test - filter out the impact of XBT wire breaks from results.
filter_from_file_spec - use specification from file to choose filtering.
enforce_types_of_check - use specification from file on particular types of checks to use.
n_profiles_to_analyse - restrict the number of profiles extracted from the database.
n_combination_iterations - AND tests together; restricted to max of 2 as otherwise
number of tests gets very large.
with_reverses - if True, a copy of each test with inverted results is made.
effectiveness_ratio - will give a warning if TPR / FPR is less than this value.
improve_threshold - ignores tests if they do not results in a change in true positive
rate (in %) of at least this amount.
verbose - if True, will print a lot of messages to screen.
plot_roc - if True, will save an image of the ROC to roc.png.
write_roc - if True, will save the ROC data to roc.json.
'''
# Read QC test specifications if required.
groupdefinition = {}
if filter_from_file_spec or enforce_types_of_check:
groupdefinition = read_qc_groups()
# Read data from database into a pandas data frame.
df = dbutils.db_to_df(sys.argv[1],
filter_on_wire_break_test = filter_on_wire_break_test,
filter_on_tests = groupdefinition,
n_to_extract = n_profiles_to_analyse)
# Drop nondiscriminating tests
nondiscrim = []
cols = list(df.columns)
for c in cols:
if len(pandas.unique(df[c])) == 1:
nondiscrim.append(c)
if verbose: print c + ' is nondiscriminating and will be removed'
cols = [t for t in cols if t not in nondiscrim]
df = df[cols]
print list(df)
testNames = df.columns[2:].values.tolist()
if verbose:
print 'Number of profiles is: ', len(df.index)
print 'Number of quality checks to process is: ', len(testNames)
# mark chosen profiles as part of the training set
all_uids = main.dbinteract('SELECT uid from ' + sys.argv[1] + ';')
for uid in all_uids:
uid = uid[0]
is_training = int(uid in df['uid'].astype(int).as_matrix())
query = "UPDATE " + sys.argv[1] + " SET training=" + str(is_training) + " WHERE uid=" + str(uid) + ";"
main.dbinteract(query)
# Convert to numpy structures and make inverse versions of tests if required.
# Any test with true positive rate of zero is discarded.
truth = df['Truth'].as_matrix()
tests = []
names = []
tprs = []
fprs = []
if with_reverses:
reverselist = [False, True]
else:
reverselist = [False]
for i, testname in enumerate(testNames):
for reversal in reverselist:
results = df[testname].as_matrix() != reversal
tpr, fpr, fnr, tnr = main.calcRates(results, truth)
if tpr > 0.0:
tests.append(results)
if reversal:
addtext = 'r'
else:
addtext = ''
names.append(addtext + testname)
tprs.append(tpr)
fprs.append(fpr)
del df # No further need for the data frame.
if verbose: print 'Number of quality checks after adding reverses and removing zero TPR was: ', len(names)
# Create storage to hold the roc curve.
cumulative = truth.copy()
cumulative[:] = False
currenttpr = 0.0
currentfpr = 0.0
r_fprs = [] # The false positive rate for each ROC point.
r_tprs = [] # True positive rate for each ROC point.
testcomb = [] # The QC test that was added at each ROC point.
groupsel = [] # Set to True if the ROC point was from an enforced group.
# Pre-select some tests if required.
if enforce_types_of_check:
if verbose: print 'Enforcing types of checks'
while len(groupdefinition['At least one from group']) > 0:
bestchoice = ''
bestgroup = ''
bestdist = np.sqrt(100.0**2 + 100.0**2)
besti = -1
for key in groupdefinition['At least one from group']:
for testname in groupdefinition['At least one from group'][key]:
# Need to check that the test exists - it may have been removed
# if it was non-discriminating.
if testname in names:
for itest, name in enumerate(names):
if name == testname:
cumulativenew = np.logical_or(cumulative, tests[itest])
tpr, fpr, fnr, tnr = main.calcRates(cumulativenew, truth)
newdist = return_cost(costratio, tpr, fpr)
print ' ', tpr, fpr, newdist, bestdist, testname
if newdist == bestdist:
if verbose:
print ' ' + bestchoice + ' and ' + testname + ' have the same results and the first is kept'
elif newdist < bestdist:
bestchoice = testname
bestdist = newdist
besti = itest
bestgroup = key
else:
if verbose: print ' ' + testname + ' not found and so was skipped'
#assert bestchoice != '', ' Error, did not make a choice in group ' + key
if verbose: print ' ' + bestchoice + ' was selected from group ' + bestgroup
if fprs[besti] > 0:
if tprs[besti] / fprs[besti] < effectiveness_ratio:
print 'WARNING - ' + bestchoice + ' TPR / FPR is below the effectiveness ratio limit: ', tprs[besti] / fprs[besti], effectiveness_ratio
cumulative = np.logical_or(cumulative, tests[besti])
currenttpr, currentfpr, fnr, tnr = main.calcRates(cumulative, truth)
testcomb.append(names[besti])
r_fprs.append(currentfpr)
r_tprs.append(currenttpr)
groupsel.append(True)
# Once a test has been added, it can be deleted so that it is not considered again.
del names[besti]
del tests[besti]
del fprs[besti]
del tprs[besti]
del groupdefinition['At least one from group'][bestgroup]
print 'ROC point from enforced group: ', currenttpr, currentfpr, testcomb[-1], bestgroup
# Make combinations of the single checks and store.
assert n_combination_iterations <= 2, 'Setting n_combination_iterations > 2 results in a very large number of combinations'
if verbose: print 'Starting construction of combinations with number of iterations: ', n_combination_iterations
for its in range(n_combination_iterations):
ntests = len(names)
for i in range(ntests - 1):
if verbose: print 'Processing iteration ', its + 1, ' out of ', n_combination_iterations, ' step ', i + 1, ' out of ', ntests - 1, ' with number of tests now ', len(names)
for j in range(i + 1, ntests):
# Create the name for this combination.
newname = ('&').join(sorted((names[i] + '&' + names[j]).split('&')))
if newname in names: continue # Do not keep multiple copies of the same combination.
results = np.logical_and(tests[i], tests[j])
tpr, fpr, fnr, tnr = main.calcRates(results, truth)
if tpr > 0.0:
tests.append(results)
tprs.append(tpr)
fprs.append(fpr)
names.append(newname)
if verbose: print 'Completed generation of tests, now constructing roc from number of tests: ', len(names)
# Create roc.
used = np.zeros(len(names), dtype=bool)
overallbest = return_cost(costratio, tpr, fpr)
keepgoing = True
while keepgoing:
keepgoing = False
besti = -1
bestcost = overallbest
bestncomb = 100000
bestdtpr = 0
bestdfpr = 100000
for i in range(len(names)):
if used[i]: continue
cumulativenew = np.logical_or(cumulative, tests[i])
tpr, fpr, fnr, tnr = main.calcRates(cumulativenew, truth)
dtpr = tpr - currenttpr
dfpr = fpr - currentfpr
newcost = return_cost(costratio, tpr, fpr)
newbest = False
if newcost <= bestcost and dtpr >= improve_threshold and dtpr > 0.0:
# If cost is better than found previously, use it else if it is
# the same then decide if to use it or not.
if newcost < bestcost:
newbest = True
elif dtpr >= bestdtpr:
if dtpr > bestdtpr:
newbest = True
elif len(names[i].split('&')) < bestncomb:
newbest = True
if newbest:
besti = i
bestcost = newcost
bestncomb = len(names[i].split('&'))
bestdtpr = dtpr
bestdfpr = dfpr
if besti >= 0:
keepgoing = True
used[besti] = True
overallbest = bestcost
cumulative = np.logical_or(cumulative, tests[besti])
currenttpr, currentfpr, fnr, tnr = main.calcRates(cumulative, truth)
testcomb.append(names[besti])
r_fprs.append(currentfpr)
r_tprs.append(currenttpr)
groupsel.append(False)
print 'ROC point: ', currenttpr, currentfpr, names[besti], overallbest
if plot_roc:
plt.plot(r_fprs, r_tprs, 'k')
for i in range(len(r_fprs)):
if groupsel[i]:
colour = 'r'
else:
colour = 'b'
plt.plot(r_fprs[i], r_tprs[i], colour + 'o')
plt.xlim(0, 100)
plt.ylim(0, 100)
plt.xlabel('False positive rate (%)')
plt.ylabel('True positive rate (%)')
plt.savefig('roc.png')
plt.close()
if write_roc:
f = open('roc.json', 'w')
r = {}
r['tpr'] = r_tprs
r['fpr'] = r_fprs
r['tests'] = testcomb
r['groupsel'] = groupsel
json.dump(r, f)
f.close()
