def train_skmodel(model, average_n_predictions=None, binarize=False, **kwargs): train_x = model.train_x valid_x = model.valid_x if model.pca: train_x = model.pca.fit_transform(train_x) valid_x = model.pca.transform(valid_x) model.c.fit(train_x, model.train_y) preds = model.c.predict(valid_x) if average_n_predictions: correct = 0 incorrect = 0 for y in np.unique(model.valid_y): y_preds = preds[np.where(model.valid_y == y)] for i in xrange(0, len(y_preds), average_n_predictions): if sum(y_preds[i:(i + average_n_predictions)] == y) > (average_n_predictions / 2): correct += 1 else: incorrect += 1 acc = correct / (correct + incorrect) else: acc = sum(np.equal(preds, model.valid_y)) / len(preds) _auc = auc(model.valid_y[:len(preds)], preds, pos_label=1) if binarize: binary_preds = np.greater_equal(preds, np.median(preds)) acc = sum(np.equal(binary_preds, model.valid_y)) / len(binary_preds) print("validation auc: {auc}".format(auc=_auc)) log('epoch 0, validation accuracy {acc:.2%}'.format(acc=acc), True) return acc, 0, model.serialize()
def evaluate(self, idxs, pred): '''scores the predictions of a given set of rows Args: idxs (int[]): the indices of the rows to be evaluated pred (float[]): the prediction for the label of that row Returns: float: an evaluation score (the higher the better) ''' # _correct_y is int-casted, go to owner op (int-cast) to get shared variable # as first input and get its value without copying the value out _y = self._correct_y.owner.inputs[0].get_value(borrow=True)[idxs] return auc(_y[:len(pred)], pred, pos_label=1)
def evaluate(self, idxs, pred): '''scores the predictions of a given set of rows Args: idxs (int[]): the indices of the rows to be evaluated pred (float[]): the prediction for the label of that row Returns: float: an evaluation score (the higher the better) ''' # _correct_y is int-casted, go to owner op (int-cast) to get shared variable # as first input and get its value without copying the value out _y = self._correct_y.owner.inputs[0].get_value(borrow=True)[idxs] return auc(_y[:len(pred)], pred, pos_label=1)
def valid_eval(idxs, pred): _y = correct_y.owner.inputs[0].get_value(borrow=True)[idxs] return auc(_y, pred, pos_label=1)
def valid_eval(idxs, pred): _y = correct_y.owner.inputs[0].get_value(borrow=True)[idxs] return auc(_y, pred, pos_label=1)
def evaluate(self, idxs, pred): y = self._ys.owner.inputs[0].get_value(borrow=True)[idxs] return auc(y[:len(pred)], pred, pos_label=1)
def auc(self, input_vec, true_y, pos_label=1): """Returns auc score for binary classification.""" res = self.infer(input_vec)[:, pos_label] return auc(true_y, self.infer(input_vec)[:, pos_label], pos_label=pos_label)
def evaluate(self, idxs, pred): y = self._ys.owner.inputs[0].get_value(borrow=True)[idxs] return auc(y[:len(pred)], pred, pos_label=1)