def evaluate(dataset, classifier, groundTruth): '''evaluate the classifier on the given dataset, using only the points that are in the groundTruth. It then returns the confusion matrix. - dataset is the dataset from which to get the points - classifier is a function which given a point returns its class - groundTruth is a map from the points to classify to their respective class ''' total = len(groundTruth) done = 0 confusion = ConfusionMatrix() for (pointId, expected) in groundTruth.items(): try: found = classifier(dataset.point(pointId)) confusion.add(expected, found, pointId) except Exception, e: print 'WARNING: Could not classify point', pointId, 'because', str( e) done += 1 print '\r[%d%%]' % int(done * 100 / total),
def evaluate(dataset, classifier, groundTruth): '''evaluate the classifier on the given dataset, using only the points that are in the groundTruth. It then returns the confusion matrix. - dataset is the dataset from which to get the points - classifier is a function which given a point returns its class - groundTruth is a map from the points to classify to their respective class ''' total = len(groundTruth) done = 0 confusion = ConfusionMatrix() for (pointId, expected) in groundTruth.items(): try: found = classifier(dataset.point(pointId)) confusion.add(expected, found, pointId) except Exception, e: print 'WARNING: Could not classify point', pointId, 'because', str(e) done += 1 print '\r[%d%%]' % int(done*100 / total),
class IoU(metric.Metric): """Computes the intersection over union (IoU) per class and corresponding mean (mIoU). Intersection over union (IoU) is a common evaluation metric for semantic segmentation. The predictions are first accumulated in a confusion matrix and the IoU is computed from it as follows: IoU = true_positive / (true_positive + false_positive + false_negative). Keyword arguments: - num_classes (int): number of classes in the classification problem - normalized (boolean, optional): Determines whether or not the confusion matrix is normalized or not. Default: False. - ignore_index (int or iterable, optional): Index of the classes to ignore when computing the IoU. Can be an int, or any iterable of ints. """ def __init__(self, num_classes, normalized=False, ignore_index=None): super(IoU, self).__init__() self.conf_metric = ConfusionMatrix(num_classes, normalized) if ignore_index is None: self.ignore_index = None elif isinstance(ignore_index, int): self.ignore_index = (ignore_index, ) else: try: self.ignore_index = tuple(ignore_index) except TypeError: raise ValueError("'ignore_index' must be an int or iterable") def reset(self): self.conf_metric.reset() def add(self, predicted, target): """Adds the predicted and target pair to the IoU metric. Keyword arguments: - predicted (Tensor): Can be a (N, K, H, W) tensor of predicted scores obtained from the model for N examples and K classes, or (N, H, W) tensor of integer values between 0 and K-1. - target (Tensor): Can be a (N, K, H, W) tensor of target scores for N examples and K classes, or (N, H, W) tensor of integer values between 0 and K-1. """ # Dimensions check assert predicted.size(0) == target.size(0), \ 'number of targets and predicted outputs do not match' assert predicted.dim() == 3 or predicted.dim() == 4, \ "predictions must be of dimension (N, H, W) or (N, K, H, W)" assert target.dim() == 3 or target.dim() == 4, \ "targets must be of dimension (N, H, W) or (N, K, H, W)" # If the tensor is in categorical format convert it to integer format # When the tensor is 4-dimensional, for each of the 3-dimensions of # class for each pixel, the value it contains is confidence of that # class. We take the "index" (class label) with the maximum confidence # among the classes (i.e 3 dimensions) if predicted.dim() == 4: _, predicted = predicted.max(1) if target.dim() == 4: _, target = target.max(1) # .view(-1) converts tensor to single dimensional tensor self.conf_metric.add(predicted.view(-1), target.view(-1)) def value(self): """Computes the IoU and mean IoU. The mean computation ignores NaN elements of the IoU array. Returns: Tuple: (IoU, mIoU). The first output is the per class IoU, for K classes it's numpy.ndarray with K elements. The second output, is the mean IoU. """ conf_matrix = self.conf_metric.value() if self.ignore_index is not None: for index in self.ignore_index: conf_matrix[:, self.ignore_index] = 0 conf_matrix[self.ignore_index, :] = 0 true_positive = np.diag(conf_matrix) false_positive = np.sum(conf_matrix, 0) - true_positive false_negative = np.sum(conf_matrix, 1) - true_positive # Just in case we get a division by 0, ignore/hide the error with np.errstate(divide='ignore', invalid='ignore'): iou = true_positive / (true_positive + false_positive + false_negative) return iou, np.nanmean(iou)