def rf_uncertainty(self): """ Selcts [self.batch_size] new segments to label based on their distance from [self.thresh] First trains classifier on all labeled data, then predicts probability of all other segments being damage and chooses segments closest to [self.thresh] Returns ------- ndarray All indices of unlabeled data sorted in decreasing uncertainty """ model = ObjectClassifier(NZ = 0, verbose = 0) #train and predict segs of classifier training_sample = model.sample(self.training_labels, EVEN = 2) model.fit(self.train_map, self.training_labels, training_sample) proba_segs = model.predict_proba_segs(self.train_map) #If show, save figures of heatmap from prediction if self.show: self.show_selected() fig = plt.figure() n_labeled = np.where(self.training_labels != -1)[0].shape[0] img = self.train_map.seg_convert(self.seg, proba_segs) plt.imshow(img, cmap = 'seismic', norm = plt.Normalize(0,1)) fig.savefig('{}test_{}{}.png'.format(self.path, n_labeled, self.postfix), format='png') plt.close(fig) #choose indices whose predictions minus thresh were closest to zero unknown_indcs = np.where(self.training_labels == -1)[0] uncertainties = 1-np.abs(proba_segs-self.thresh) return self._uncertain_order(uncertainties.ravel(), unknown_indcs)
def test_progress(self): """Evaluates progress of active learning run by looking at results tested on testing map""" model = ObjectClassifier(NZ = 0, verbose = 0) #Pulls all training data thats been labeled and samples evenly between the classes training_sample = model.sample(self.training_labels, EVEN = 2) #Trains on training data and tests on test map model.fit(self.train_map, self.training_labels, training_sample) proba = model.predict_proba(self.test_map) #Uses majority vote as ground truth g_truth = self.labelers.majority_vote()[self.test_map.segmentations[self.seg]] n_labeled = np.where(self.training_labels > -1)[0].shape[0] #If show is true, saves the ROC curve if self.show: fig, AUC = analyze_results.ROC(g_truth.ravel(), proba.ravel(), 'Haiti Test')[:2] fig.savefig('{}ROC_{}{}.png'.format(self.path, n_labeled, self.postfix), format='png') plt.close(fig) #Evaluates progress by finding FPR at self.FNR and adding it to the fprs list FPR, thresh = analyze_results.FPR_from_FNR(g_truth.ravel(), proba.ravel(), TPR = self.TPR) self.fprs.append(FPR) #saves all fprs to document every iteration so results are not loss and progress can be seen mid-run np.save('{}fprs{}.npy'.format(self.path, self.postfix), self.fprs)