def get_classification_roc(data_iter,classifier): """ Returns an roc for the given classification task """ num_frames=0 all_positive_scores = [] all_negative_scores = [] data_iter.reset_exp() for datum_id in xrange(data_iter.num_data): if datum_id % 10 == 0: print "working on example", datum_id if data_iter.next(compute_pattern_times=True, max_template_length=classifier.window[1], wait_for_positive_example=True): pattern_times = data_iter.pattern_times num_detections = data_iter.E.shape[1] - classifier.window[1] num_frames += data_iter.E.shape[1] scores = -np.inf * np.ones(num_detections) esp._edge_map_threshold_segments(data_iter.E, classifier.window[1], 1, threshold=.3, edge_orientations = data_iter.edge_orientations, edge_feature_row_breaks = data_iter.edge_feature_row_breaks) for d in xrange(num_detections): E_segment = data_iter.E[:,d:d+classifier.window[1]] scores[d] = classifier.score_no_bg(E_segment) # get the positive and negative scores removed out pos_scores = [] neg_indices = np.empty(scores.shape[0],dtype=bool) neg_indices[:]=True for pt in xrange(len(pattern_times)): pos_scores.append(np.max(scores[pattern_times[pt][0]-int(np.ceil(classifier.window[1]/3.)):pattern_times[pt][0]+int(np.ceil(classifier.window[1]/3.))])) neg_indices[pattern_times[pt][0]-int(np.ceil(classifier.window[1]/3.)):pattern_times[pt][1]] = False # get rid of overlapping instances neg_indices_non_overlap = remove_overlapping_examples(np.argsort(scores), classifier.window[1], int(allowed_overlap*classifier.window[1])) neg_idx2 = np.empty(scores.shape[0],dtype=bool) neg_idx2[neg_indices_non_overlap] =True neg_indices_full = np.logical_and(neg_indices,neg_idx2) all_positive_scores.extend(pos_scores) all_negative_scores.extend(scores[neg_indices_full]) else: break return get_roc(np.sort(all_positive_scores)[::-1], np.sort(all_negative_scores)[::-1], num_frames)
def add_frames(self,E,edge_feature_row_breaks=None, edge_orientations=None,abst_threshold=None): new_E = E.copy() if abst_threshold is not None: esp._edge_map_threshold_segments(new_E, 40, 1, threshold=.3, edge_orientations = edge_orientations, edge_feature_row_breaks = edge_feature_row_breaks) if not self.processed_frames: self.E = np.mean(new_E,axis=1) self.processed_frames = True else: self.E = (self.E * self.num_frames + np.sum(new_E,axis=1))/(self.num_frames+new_E.shape[1]) self.num_frames += new_E.shape[1]
def get_roc_generous(data_iter, classifier,coarse_thresh=-np.inf, allowed_overlap = .1, edge_feature_row_breaks= np.array([ 0., 45., 90., 138., 186., 231., 276., 321., 366.]), edge_orientations=np.array([[ 1., 0.], [-1., 0.], [ 0., 1.], [ 0., -1.], [ 1., 1.], [-1., -1.], [ 1., -1.], [-1., 1.]]), abst_threshold=np.array([.025,.025,.015,.015, .02,.02,.02,.02]), spread_radius=3): """ Computes an ROC curve for a classifier, do not remove positive examples that overlap with negative examples, simply take the max within the positive regions """ num_frames = 0 all_positive_scores = [] all_negative_scores = [] all_positive_counts = [] all_negative_counts = [] data_iter.reset_exp() for datum_id in xrange(data_iter.num_data): if datum_id % 10 == 0: print "working on example", datum_id if data_iter.next(compute_pattern_times=True, max_template_length=classifier.window[1]): pattern_times = data_iter.pattern_times num_detections = data_iter.E.shape[1] - classifier.window[1] num_frames += data_iter.E.shape[1] scores = -np.inf * np.ones(num_detections) coarse_count_scores = -np.inf *np.ones(num_detections) coarse_scores = -np.inf * np.ones(num_detections) esp._edge_map_threshold_segments(data_iter.E, classifier.window[1], 1, threshold=.3, edge_orientations = data_iter.edge_orientations, edge_feature_row_breaks = data_iter.edge_feature_row_breaks) for d in xrange(num_detections): E_segment = data_iter.E[:,d:d+classifier.window[1]] scores[d] = classifier.score_no_bg(E_segment) coarse_count_scores[d] = classifier.coarse_score_count(E_segment) if d>1 and d<num_detections-1: if (coarse_count_scores[d-1] > coarse_thresh) and \ ((coarse_count_scores[d-1]>\ coarse_count_scores[d] and\ coarse_count_scores[d-1]>=\ coarse_count_scores[d-2]) or\ (coarse_count_scores[d-1]>=\ coarse_count_scores[d] and\ coarse_count_scores[d-1]>\ coarse_count_scores[d-2]) ): coarse_scores[d] = classifier.score_no_bg(E_segment) # get the positive and negative scores removed out pos_counts =[] pos_scores = [] neg_indices = np.empty(scores.shape[0],dtype=bool) neg_indices[:]=True for pt in xrange(len(pattern_times)): pos_scores.append(np.max(scores[pattern_times[pt][0]-int(np.ceil(classifier.window[1]/3.)):pattern_times[pt][0]+int(np.ceil(classifier.window[1]/3.))])) pos_counts.append(np.max(coarse_scores[pattern_times[pt][0]-int(np.ceil(classifier.window[1]/3.)):pattern_times[pt][0]+int(np.ceil(classifier.window[1]/3.))])) neg_indices[pattern_times[pt][0]-int(np.ceil(classifier.window[1]/3.)):pattern_times[pt][1]] = False # get rid of overlapping instances neg_indices_counts_non_overlap = remove_overlapping_examples(np.argsort(coarse_scores), classifier.coarse_length, int(allowed_overlap*classifier.coarse_length)) neg_indices_non_overlap = remove_overlapping_examples(np.argsort(scores), classifier.window[1], int(allowed_overlap*classifier.window[1])) neg_idx2 = np.empty(scores.shape[0],dtype=bool) neg_idx2[neg_indices_non_overlap] =True neg_indices_full = np.logical_and(neg_indices,neg_idx2) neg_idx2 = np.empty(scores.shape[0],dtype=bool) neg_idx2[neg_indices_counts_non_overlap] =True neg_indices_coarse = np.logical_and(neg_indices,neg_idx2) all_positive_scores.extend(pos_scores) all_positive_counts.extend(pos_counts) all_negative_scores.extend(scores[neg_indices_full]) all_negative_counts.extend(coarse_scores[neg_indices_coarse]) else: break like_roc = get_roc(np.sort(all_positive_scores)[::-1], np.sort(all_negative_scores)[::-1], num_frames) count_roc = get_roc(np.sort(all_positive_counts)[::-1], np.sort(all_negative_counts)[::-1], num_frames) return like_roc, count_roc