def test_nn_call_bag(self, use_train_data=False):
"""
Get a CallBag for the NN classifier on this plaster.run.
use_train_data=True when you want to look at over-fitting.
"""
if use_train_data:
true_pep_iz = self.test_nn.train_true_pep_iz
pred_pep_iz = self.test_nn.train_pred_pep_iz
check.affirm(
true_pep_iz is not None and pred_pep_iz is not None,
"The test_nn task was not run with the training_set",
)
cached_pr = self.test_nn.train_peps_pr
else:
true_pep_iz = self.test_nn.test_true_pep_iz
pred_pep_iz = self.test_nn.test_pred_pep_iz
cached_pr = self.test_nn.test_peps_pr
return CallBag(
true_pep_iz=true_pep_iz,
pred_pep_iz=pred_pep_iz,
scores=self.test_nn.test_scores,
prep_result=self.prep,
sim_result=self.sim,
cached_pr=cached_pr,
classifier_name="nn",
)
def test_rf_call_bag(self, use_train_data=False):
"""
Get a CallBag for the RF classifier on this plaster.run.
use_train_data=True when you want to look at over-fitting.
"""
if use_train_data:
true_pep_iz = self.test_rf.train_true_pep_iz
pred_pep_iz = self.test_rf.train_pred_pep_iz
scores = self.test_rf.train_scores
all_class_scores = self.test_rf.train_all_class_scores
cached_pr = self.test_rf.train_peps_pr
cached_pr_abund = self.test_rf.train_peps_pr_abund
check.affirm(
true_pep_iz is not None and pred_pep_iz is not None,
"The test_rf task was not run with the training_set",
)
else:
true_pep_iz = self.test_rf.test_true_pep_iz
pred_pep_iz = self.test_rf.test_pred_pep_iz
scores = self.test_rf.test_scores
all_class_scores = self.test_rf.test_all_class_scores
cached_pr = self.test_rf.test_peps_pr
cached_pr_abund = self.test_rf.test_peps_pr_abund
return CallBag(
true_pep_iz=true_pep_iz,
pred_pep_iz=pred_pep_iz,
scores=scores,
all_class_scores=all_class_scores,
prep_result=self.prep,
sim_result=self.sim,
cached_pr=cached_pr,
cached_pr_abund=cached_pr_abund,
classifier_name="rf",
)
def classify_rf_call_bag(self):
"""
Get a call bag for classification of real sigprocv2 data.
"""
return CallBag(
pred_pep_iz=self.classify_rf.pred_pep_iz,
scores=self.classify_rf.scores,
prep_result=self.prep,
classifier_name="classify_rf",
)
def zest_pr_curve_no_tied_scores_mean_recall():
"""
Testing situations with some right and some wrong calls, but no tied scores.
Adding to this different train_recall factors to test that those get included
in recall calculations.
Same fixtures as above except the means, see above for comments.
"""
# first entry is null peptide
stub_sim_result = Munch(train_recalls=np.array([-1.0, 0.1, 0.2, 0.3]))
stub_prep_result = Munch(n_peps=4)
true_pep_iz = np.array([1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3])
pred_pep_iz = np.array([1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 1])
scores = np.array(
[0.8, 0.9, 0.7, 0.6, 0.85, 0.53, 0.54, 0.55, 0.75, 0.4, 0.3, 0.35])
cb = CallBag(
sim_result=stub_sim_result,
prep_result=stub_prep_result,
true_pep_iz=true_pep_iz,
pred_pep_iz=pred_pep_iz,
scores=scores,
)
cum_sum_correct = np.array(
[0.0, 1.0, 2.0, 3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 5.0, 5.0, 6.0])
cum_sum_count = np.array(range(1, len(pred_pep_iz) + 1))
# precision at score threshold
prec_at_thresh = utils.np_safe_divide(cum_sum_correct,
cum_sum_count,
default=0)
prec_at_thresh = np.append([1.0], prec_at_thresh)
# recall at each threshold
recall_at_thresh = cum_sum_correct / len(pred_pep_iz)
recall_at_thresh = np.append([0.0], recall_at_thresh)
zest()
def test_nn(test_nn_params,
prep_result,
sim_result,
progress=None,
pipeline=None):
n_channels, n_cycles = sim_result.params.n_channels_and_cycles
n_phases = 6 if test_nn_params.include_training_set else 3
if pipeline is not None:
pipeline.set_phase(0, n_phases)
shape = sim_result.test_radmat.shape
assert len(shape) == 4
test_radmat = sim_result.test_radmat.reshape(
(shape[0] * shape[1], shape[2], shape[3]))
test_dyemat = sim_result.test_dyemat.reshape(
(shape[0] * shape[1], shape[2], shape[3]))
test_result = nn(
test_nn_params,
sim_result,
radmat=test_radmat,
true_dyemat=test_dyemat,
progress=progress,
)
test_result.true_pep_iz = ArrayResult(
filename="test_true_pep_iz",
shape=(shape[0] * shape[1], ),
dtype=IndexType,
mode="w+",
)
test_result.true_pep_iz[:] = np.repeat(
np.arange(shape[0]).astype(IndexType), shape[1])
check.t(test_result.true_pep_iz, ArrayResult)
check.t(test_result.pred_pep_iz, ArrayResult)
call_bag = CallBag(
true_pep_iz=test_result.true_pep_iz.arr(),
pred_pep_iz=test_result.pred_pep_iz.arr(),
scores=test_result.scores.arr(),
prep_result=prep_result,
sim_result=sim_result,
)
if pipeline is not None:
pipeline.set_phase(1, n_phases)
test_result.peps_pr = call_bag.pr_curve_by_pep(progress=progress)
# If there is abundance information, compute the abundance-adjusted PR
# This call returns None if there is no abundance info avail.
if pipeline is not None:
pipeline.set_phase(2, n_phases)
test_result.peps_pr_abund = call_bag.pr_curve_by_pep_with_abundance(
progress=progress)
if test_nn_params.include_training_set:
# Permit testing for over-fitting by classifying on the train data
if pipeline is not None:
pipeline.set_phase(3, n_phases)
real_pep_iz = prep_result.peps__no_decoys().pep_i.values
keep_rows = np.isin(sim_result.train_true_pep_iz, real_pep_iz)
train_radmat = sim_result.train_radmat[keep_rows]
train_dyemat = sim_result.train_dyemat[keep_rows]
assert train_radmat.shape == shape
train_result = nn(
test_nn_params.use_gmm,
sim_result,
radmat=train_radmat,
true_dyemat=train_dyemat,
progress=progress,
)
train_result.true_pep_iz = sim_result.train_true_pep_iz
train_result.true_pep_iz = ArrayResult(
filename="train_true_pep_iz",
shape=(shape[0] * shape[1], ),
dtype=IndexType,
mode="w+",
)
train_result.true_pep_iz[:] = np.repeat(
np.arange(shape[0]).astype(IndexType), shape[1])
check.t(train_result.true_pep_iz, ArrayResult)
check.t(train_result.pred_pep_iz, ArrayResult)
call_bag = CallBag(
true_pep_iz=train_result.true_pep_iz.arr(),
pred_pep_iz=train_result.pred_pep_iz.arr(),
scores=train_result.scores.arr(),
prep_result=prep_result,
sim_result=sim_result,
)
if pipeline is not None:
pipeline.set_phase(4, n_phases)
train_result.peps_pr = call_bag.pr_curve_by_pep(progress=progress)
if pipeline is not None:
pipeline.set_phase(5, n_phases)
train_result.peps_pr_abund = call_bag.pr_curve_by_pep_with_abundance(
progress=progress)
else:
train_result = {k: None for k in test_result.keys()}
def rename(d, prefix):
return {f"{prefix}{k}": v for k, v in d.items()}
return TestNNResult(
params=test_nn_params,
**rename(test_result, "test_"),
**rename(train_result, "train_"),
)
def zest_pr_curve_no_tied_scores():
"""
Testing situations with some right and some wrong calls, but no tied scores.
"""
# first entry is null peptide
stub_sim_result = Munch(train_recalls=np.array([-1.0] + [1.0] * 4))
stub_prep_result = Munch(n_peps=5)
# pep 1 is predicted correctly 1/4, #2 is 1/2, #3 is 3/4
true_pep_iz = np.array([1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3])
pred_pep_iz = np.array([1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 1])
scores = np.array(
[0.8, 0.9, 0.7, 0.6, 0.85, 0.53, 0.54, 0.55, 0.75, 0.4, 0.3, 0.35])
cb = CallBag(
sim_result=stub_sim_result,
prep_result=stub_prep_result,
true_pep_iz=true_pep_iz,
pred_pep_iz=pred_pep_iz,
scores=scores,
)
# sorted_i = np.argsort( scores )[::-1]
# sorted by score, highest->lowest
# t[1. 2. 1. 3. 1. 1. 2. 2. 2. 3. 3. 3.]
# p[2. 2. 1. 3. 2. 2. 3. 3. 2. 3. 1. 3.]
# s[0.9 0.85 0.8 0.75 0.7 0.6 0.55 0.54 0.53 0.4 0.35 0.3]
# cumulative sum of correct calls, cumulative call count
# [F. T. T. T. F. F. F. F. T. T. F. T.]
# cum_sum_correct = np.array(
# [0.0, 1.0, 2.0, 3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 5.0, 5.0, 6.0]
# )
# cum_sum_count = np.array(range(1, len(pred_pep_iz) + 1))
# Now using a linear stepper of the score so these are now:
# [F. T. T. T. F. F. F. F. T. T. F. T.]
# [0.0, 1.0, 2.0, 3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 5.0, 5.0, 6.0]
# .9 .8 .7 .6 .5 .4 .3
# Note that the new pr_curve trims out starting values of r=0
# So the p=0.0 and r=0./12. are removed
prec = np.array([
# 0.0 / 1.0, # 0.9
2.0 / 3.0, # 0.8
3.0 / 5.0, # 0.7
3.0 / 6.0, # 0.6
4.0 / 9.0, # 0.5
5.0 / 10.0, # 0.4
6.0 / 12.0, # 0.3
6.0 / 12.0, # 0.2
6.0 / 12.0, # 0.1
6.0 / 12.0, # 0.0
])
reca = np.array([
# 0.0 / 12.0, # 0.9
2.0 / 12.0, # 0.8
3.0 / 12.0, # 0.7
3.0 / 12.0, # 0.6
4.0 / 12.0, # 0.5
5.0 / 12.0, # 0.4
6.0 / 12.0, # 0.3
6.0 / 12.0, # 0.2
6.0 / 12.0, # 0.1
6.0 / 12.0, # 0.0
])
# precision at score threshold
# prec_at_thresh = utils.np_safe_divide(cum_sum_correct, cum_sum_count, default=0)
# prec_at_thresh = np.append([1.0], prec_at_thresh)
# recall at each threshold
# recall_at_thresh = cum_sum_correct / len(pred_pep_iz)
# recall_at_thresh = np.append([0.0], recall_at_thresh)
def it_computes_combined_pr():
p, r, s, a = cb.pr_curve(n_steps=10)
assert np.array_equal(p, prec)
assert np.allclose(s, np.linspace(0.8, 0.0, 9))
assert np.array_equal(r, reca)
def it_computes_subset_pr():
p, r, s, a = cb.pr_curve(pep_iz_subset=[1], n_steps=10)
# t[1. 1. 1. 1. 3. ]
# p[2. 1. 2. 2. 1. ]
# s[0.9 0.85 0.8 0.75 0.7 0.6 0.55 0.54 0.53 0.4 0.35 0.3]
prec = np.array([
# 0.0 / 1.0, # 0.9
1.0 / 1.0, # 0.8
1.0 / 1.0, # 0.7
1.0 / 1.0, # 0.6
1.0 / 1.0, # 0.5
1.0 / 1.0, # 0.4
1.0 / 2.0, # 0.3
1.0 / 2.0, # 0.2
1.0 / 2.0, # 0.1
1.0 / 2.0, # 0.0
])
reca = np.array([
# 0.0 / 1.0, # 0.9
1.0 / 4.0, # 0.8
1.0 / 4.0, # 0.7
1.0 / 4.0, # 0.6
1.0 / 4.0, # 0.5
1.0 / 4.0, # 0.4
1.0 / 4.0, # 0.3
1.0 / 4.0, # 0.2
1.0 / 4.0, # 0.1
1.0 / 4.0, # 0.0
])
# look at true & pred that contain 1
# t[1. 1. 1. 1. 3. ]
# p[2. 1. 2. 2. 1. ]
# s[0.9 0.8 0.7 0.6 0.35]
#
# calculated prec first, based only on predictions
# [1] out front is threshold added by sklearn
# prec = [1][ 1, 1/2]
#
# the scores for those are used as thresholds:
# scor = [1][ 8, 35 ]
# the recalls are found by looking at trues at threshold scores
# e.g. "what fraction of trues have been successfully found at score threshold .8?"
# reca = [0][ .25, .25]
# Note that sklearn truncates when full recall has been obtained,
# so we'll lose the last element of each of the above since max
# recall occurs at recall=0.25 -- last one is dropped. Said another
# way: "The number of true positives does not go up after the one found
# at score = 0.8, so stop reporting precision/recall/thresholds right there."
assert np.array_equal(p, prec)
assert np.array_equal(r, reca)
assert np.allclose(s, np.linspace(0.8, 0.0, 9))
#
# Test again for a different subset.
#
p, r, s, a = cb.pr_curve(pep_iz_subset=[2], n_steps=10)
# t[1. 2. 1. 1. 2. 2. 2. ]
# p[2. 2. 2. 2. 3. 3. 2. ]
# s[0.9 0.85 0.7 0.6 0.55 0.54 0.53]
#
# calculated prec first, based only on predictions
# prec = [1][0, .5, 1/3, 1/4, 2/5 ]
#
# the scores for those are used as thresholds:
# scor = [1][9, 85, 7, 6, 53 ]
# the recalls are found by looking trues and thresholds
# reca = [0][0, 1/4, 1/4, 1/4, 2/4 ]
# t[ 1. 2. 1. 1. 2. 2. 2. ]
# p[ 2. 2. 2. 2. 3. 3. 2. ]
# [.9 .8 .7 .6 .5 .4 .3
prec = np.array([
# 0.0 / 2.0, # 0.9
1.0 / 2.0, # 0.8
1.0 / 3.0, # 0.7
1.0 / 4.0, # 0.6
2.0 / 5.0, # 0.5
2.0 / 5.0, # 0.4
2.0 / 5.0, # 0.3
2.0 / 5.0, # 0.2
2.0 / 5.0, # 0.1
2.0 / 5.0, # 0.0
])
reca = np.array([
# 0.0 / 0.0, # 0.9
1.0 / 4.0, # 0.8
1.0 / 4.0, # 0.7
1.0 / 4.0, # 0.6
2.0 / 4.0, # 0.5
2.0 / 4.0, # 0.4
2.0 / 4.0, # 0.3
2.0 / 4.0, # 0.2
2.0 / 4.0, # 0.1
2.0 / 4.0, # 0.0
])
assert np.array_equal(p, prec)
assert np.array_equal(r, reca)
assert np.allclose(s, np.linspace(0.8, 0.0, 9))
zest()
def zest_pr_curve_edge_cases():
"""
Testing situations in which calls are all wrong or all right.
"""
# first entry is null peptide
stub_sim_result = Munch(train_recalls=np.array([-1.0, 1.0, 1.0]))
stub_prep_result = Munch(n_peps=3)
# CallBag: all right / all wrong
true_pep_iz = [1] * 10
pred_pep_iz_1 = [1] * 10
pred_pep_iz_2 = [2] * 10
scores = [0.5] * 10
cb_all_right = CallBag(
sim_result=stub_sim_result,
prep_result=stub_prep_result,
true_pep_iz=true_pep_iz,
pred_pep_iz=pred_pep_iz_1,
scores=scores,
)
cb_all_wrong = CallBag(
sim_result=stub_sim_result,
prep_result=stub_prep_result,
true_pep_iz=true_pep_iz,
pred_pep_iz=pred_pep_iz_2,
scores=scores,
)
zero_pr_result = [[0.0, 0.0], [0.0, 0.0], [0.5, 0.0], [0.0, 0.0]]
one_pr_result = [[1.0, 1.0], [1.0, 1.0], [0.5, 0.0], [1.0, 1.0]]
def it_computes_zero_pr():
p, r, s, a = cb_all_wrong.pr_curve(n_steps=2)
assert utils.np_array_same([p, r, s, a], zero_pr_result)
def it_computes_zero_pr_for_subset():
p, r, s, a = cb_all_wrong.pr_curve(pep_iz_subset=[1], n_steps=2)
assert utils.np_array_same([p, r, s, a], zero_pr_result)
p, r, s, a = cb_all_wrong.pr_curve(pep_iz_subset=[2], n_steps=2)
assert utils.np_array_same([p, r, s, a], zero_pr_result)
# peptide 2 does not show up in true/pred at all so should get zero pr curve
p, r, s, a = cb_all_right.pr_curve(pep_iz_subset=[2], n_steps=2)
assert utils.np_array_same([p, r, s, a], zero_pr_result)
def it_computes_one_pr():
p, r, s, a = cb_all_right.pr_curve(n_steps=2)
# ugh, can't use utils.flatten bc elems are ndarray, not list or tuple
compare = [a == b for a, b in zip([p, r, s, a], one_pr_result)]
compare = [
list(el) if type(el) is np.ndarray else el for el in compare
]
assert all(compare)
def it_computes_one_pr_for_subset():
p, r, s, a = cb_all_right.pr_curve(pep_iz_subset=[1], n_steps=2)
compare = [a == b for a, b in zip([p, r, s, a], one_pr_result)]
compare = [
list(el) if type(el) is np.ndarray else el for el in compare
]
assert all(compare)
def it_handles_all_rows_no_recall():
p, r, s, a = cb_all_wrong.pr_curve()
assert np.all(r == 0.0)
zest()
def it_computes_prs_with_ties():
p = np.array([0.3, 0.2, 0.2, 0.2, 0.1])
r = np.array([0.1, 0.2, 0.3, 0.4, 0.5])
s = np.array([0.9, 0.6, 0.6, 0.6, 0.5])
result = CallBag._prs_at_prec(0.2, p, r, s)
assert result == (0.2, 0.2, 0.6)
def test_rf(
test_rf_params,
prep_result,
sim_result,
train_rf_result,
progress=None,
pipeline=None,
):
n_phases = 6 if test_rf_params.include_training_set else 3
classifier = train_rf_result.classifier
if pipeline is not None:
pipeline.set_phase(0, n_phases)
test_pred_pep_iz, test_scores, test_all_class_scores = classifier.classify(
sim_result.flat_test_radmat(), test_rf_params.keep_all_class_scores,
progress)
test_true_pep_iz = sim_result.test_true_pep_iz()
# We do some PR calculation during the task so that this information is readily
# available in results & notebooks don't need to recompute it (costly).
# TODO: it is probably worth optimizing this by only doing PR for proteins of
# interest if this has been specified for the run, since otherwise we'll be
# computing full PR curves for every peptide in the background which is
# probably not interesting.
#
call_bag = CallBag(
true_pep_iz=test_true_pep_iz,
pred_pep_iz=test_pred_pep_iz,
scores=test_scores,
all_class_scores=test_all_class_scores,
prep_result=prep_result,
sim_result=sim_result,
)
if pipeline is not None:
pipeline.set_phase(1, n_phases)
if pipeline is not None:
pipeline.set_phase(2, n_phases)
test_peps_pr = call_bag.pr_curve_by_pep(progress=progress)
# If there is abundance information, compute the abundance-adjusted PR
# This call returns None if there is no abundance info avail.
test_peps_pr_abund = call_bag.pr_curve_by_pep_with_abundance(
progress=progress)
if test_rf_params.include_training_set:
# Permit testing for over-fitting by classifying on the train data
if pipeline is not None:
pipeline.set_phase(3, n_phases)
real_pep_iz = prep_result.peps__no_decoys().pep_i.values
keep_rows = np.isin(sim_result.train_true_pep_iz, real_pep_iz)
train_true_pep_iz = sim_result.train_true_pep_iz[keep_rows]
train_radmat = sim_result.train_radmat[keep_rows]
train_pred_pep_iz, train_scores, train_all_class_scores = classifier.classify(
train_radmat, test_rf_params.keep_all_class_scores, progress)
call_bag = CallBag(
true_pep_iz=train_true_pep_iz,
pred_pep_iz=train_pred_pep_iz,
scores=train_scores,
all_class_scores=train_all_class_scores,
prep_result=prep_result,
sim_result=sim_result,
)
if pipeline is not None:
pipeline.set_phase(4, n_phases)
train_peps_pr = call_bag.pr_curve_by_pep(progress=progress)
if pipeline is not None:
pipeline.set_phase(5, n_phases)
train_peps_pr_abund = call_bag.pr_curve_by_pep_with_abundance(
progress=progress)
else:
(
train_pred_pep_iz,
train_scores,
train_all_class_scores,
train_true_pep_iz,
train_peps_pr,
train_peps_pr_abund,
) = (
None,
None,
None,
None,
None,
None,
)
return TestRFResult(
params=test_rf_params,
test_true_pep_iz=test_true_pep_iz,
test_pred_pep_iz=test_pred_pep_iz,
test_scores=test_scores,
test_all_class_scores=test_all_class_scores,
test_peps_pr=test_peps_pr,
test_peps_pr_abund=test_peps_pr_abund,
train_true_pep_iz=train_true_pep_iz,
train_pred_pep_iz=train_pred_pep_iz,
train_scores=train_scores,
train_all_class_scores=train_all_class_scores,
train_peps_pr=train_peps_pr,
train_peps_pr_abund=train_peps_pr_abund,
)