def get_classification_roc(data_iter,classifier):
"""
Returns an roc for the given classification task
"""
num_frames=0
all_positive_scores = []
all_negative_scores = []
data_iter.reset_exp()
for datum_id in xrange(data_iter.num_data):
if datum_id % 10 == 0:
print "working on example", datum_id
if data_iter.next(compute_pattern_times=True,
max_template_length=classifier.window[1],
wait_for_positive_example=True):
pattern_times = data_iter.pattern_times
num_detections = data_iter.E.shape[1] - classifier.window[1]
num_frames += data_iter.E.shape[1]
scores = -np.inf * np.ones(num_detections)
esp._edge_map_threshold_segments(data_iter.E,
classifier.window[1],
1,
threshold=.3,
edge_orientations = data_iter.edge_orientations,
edge_feature_row_breaks = data_iter.edge_feature_row_breaks)
for d in xrange(num_detections):
E_segment = data_iter.E[:,d:d+classifier.window[1]]
scores[d] = classifier.score_no_bg(E_segment)
# get the positive and negative scores removed out
pos_scores = []
neg_indices = np.empty(scores.shape[0],dtype=bool)
neg_indices[:]=True
for pt in xrange(len(pattern_times)):
pos_scores.append(np.max(scores[pattern_times[pt][0]-int(np.ceil(classifier.window[1]/3.)):pattern_times[pt][0]+int(np.ceil(classifier.window[1]/3.))]))
neg_indices[pattern_times[pt][0]-int(np.ceil(classifier.window[1]/3.)):pattern_times[pt][1]] = False
# get rid of overlapping instances
neg_indices_non_overlap = remove_overlapping_examples(np.argsort(scores),
classifier.window[1],
int(allowed_overlap*classifier.window[1]))
neg_idx2 = np.empty(scores.shape[0],dtype=bool)
neg_idx2[neg_indices_non_overlap] =True
neg_indices_full = np.logical_and(neg_indices,neg_idx2)
all_positive_scores.extend(pos_scores)
all_negative_scores.extend(scores[neg_indices_full])
else:
break
return get_roc(np.sort(all_positive_scores)[::-1],
np.sort(all_negative_scores)[::-1],
num_frames)
def add_frames(self,E,edge_feature_row_breaks=None,
edge_orientations=None,abst_threshold=None):
new_E = E.copy()
if abst_threshold is not None:
esp._edge_map_threshold_segments(new_E,
40,
1,
threshold=.3,
edge_orientations = edge_orientations,
edge_feature_row_breaks = edge_feature_row_breaks)
if not self.processed_frames:
self.E = np.mean(new_E,axis=1)
self.processed_frames = True
else:
self.E = (self.E * self.num_frames + np.sum(new_E,axis=1))/(self.num_frames+new_E.shape[1])
self.num_frames += new_E.shape[1]
def get_roc_generous(data_iter, classifier,coarse_thresh=-np.inf,
allowed_overlap = .1,
edge_feature_row_breaks= np.array([ 0.,
45.,
90.,
138.,
186.,
231.,
276.,
321.,
366.]),
edge_orientations=np.array([[ 1., 0.],
[-1., 0.],
[ 0., 1.],
[ 0., -1.],
[ 1., 1.],
[-1., -1.],
[ 1., -1.],
[-1., 1.]]),
abst_threshold=np.array([.025,.025,.015,.015,
.02,.02,.02,.02]),
spread_radius=3):
"""
Computes an ROC curve for a classifier, do not remove positive examples that overlap with negative examples, simply take the max within the positive regions
"""
num_frames = 0
all_positive_scores = []
all_negative_scores = []
all_positive_counts = []
all_negative_counts = []
data_iter.reset_exp()
for datum_id in xrange(data_iter.num_data):
if datum_id % 10 == 0:
print "working on example", datum_id
if data_iter.next(compute_pattern_times=True,
max_template_length=classifier.window[1]):
pattern_times = data_iter.pattern_times
num_detections = data_iter.E.shape[1] - classifier.window[1]
num_frames += data_iter.E.shape[1]
scores = -np.inf * np.ones(num_detections)
coarse_count_scores = -np.inf *np.ones(num_detections)
coarse_scores = -np.inf * np.ones(num_detections)
esp._edge_map_threshold_segments(data_iter.E,
classifier.window[1],
1,
threshold=.3,
edge_orientations = data_iter.edge_orientations,
edge_feature_row_breaks = data_iter.edge_feature_row_breaks)
for d in xrange(num_detections):
E_segment = data_iter.E[:,d:d+classifier.window[1]]
scores[d] = classifier.score_no_bg(E_segment)
coarse_count_scores[d] = classifier.coarse_score_count(E_segment)
if d>1 and d<num_detections-1:
if (coarse_count_scores[d-1] > coarse_thresh) and \
((coarse_count_scores[d-1]>\
coarse_count_scores[d] and\
coarse_count_scores[d-1]>=\
coarse_count_scores[d-2]) or\
(coarse_count_scores[d-1]>=\
coarse_count_scores[d] and\
coarse_count_scores[d-1]>\
coarse_count_scores[d-2]) ):
coarse_scores[d] = classifier.score_no_bg(E_segment)
# get the positive and negative scores removed out
pos_counts =[]
pos_scores = []
neg_indices = np.empty(scores.shape[0],dtype=bool)
neg_indices[:]=True
for pt in xrange(len(pattern_times)):
pos_scores.append(np.max(scores[pattern_times[pt][0]-int(np.ceil(classifier.window[1]/3.)):pattern_times[pt][0]+int(np.ceil(classifier.window[1]/3.))]))
pos_counts.append(np.max(coarse_scores[pattern_times[pt][0]-int(np.ceil(classifier.window[1]/3.)):pattern_times[pt][0]+int(np.ceil(classifier.window[1]/3.))]))
neg_indices[pattern_times[pt][0]-int(np.ceil(classifier.window[1]/3.)):pattern_times[pt][1]] = False
# get rid of overlapping instances
neg_indices_counts_non_overlap = remove_overlapping_examples(np.argsort(coarse_scores),
classifier.coarse_length,
int(allowed_overlap*classifier.coarse_length))
neg_indices_non_overlap = remove_overlapping_examples(np.argsort(scores),
classifier.window[1],
int(allowed_overlap*classifier.window[1]))
neg_idx2 = np.empty(scores.shape[0],dtype=bool)
neg_idx2[neg_indices_non_overlap] =True
neg_indices_full = np.logical_and(neg_indices,neg_idx2)
neg_idx2 = np.empty(scores.shape[0],dtype=bool)
neg_idx2[neg_indices_counts_non_overlap] =True
neg_indices_coarse = np.logical_and(neg_indices,neg_idx2)
all_positive_scores.extend(pos_scores)
all_positive_counts.extend(pos_counts)
all_negative_scores.extend(scores[neg_indices_full])
all_negative_counts.extend(coarse_scores[neg_indices_coarse])
else:
break
like_roc = get_roc(np.sort(all_positive_scores)[::-1],
np.sort(all_negative_scores)[::-1],
num_frames)
count_roc = get_roc(np.sort(all_positive_counts)[::-1],
np.sort(all_negative_counts)[::-1],
num_frames)
return like_roc, count_roc
