def print_prec_rec(y_test, pred): ''' y_test are the labels for the test data pred is the prediction prints a precision recall diagram for a binary classification prediction ''' precision, recall, _ = precision_recall_curve(y_test, pred) # In matplotlib < 1.5, plt.fill_between does not have a 'step' argument step_kwargs = ({ 'step': 'post' } if 'step' in signature(pyplot.fill_between).parameters else {}) pyplot.step(recall, precision, color='b', alpha=0.2, where='post') pyplot.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) print(type(recall)) print(recall) print(type(precision)) print(precision) pyplot.xlabel('Recall') pyplot.ylabel('Precision') pyplot.ylim([0.0, 1.05]) pyplot.xlim([0.0, 1.0]) pyplot.show()
def _handle_extra_syntax_parameters(self, params): """ Handles extra parameters given to the constructor such as *columns* or a role. """ # remove column_ for roles def clean_name(name): return DataRoles._allowed_attr.get(name, name) set_params = set(map(clean_name, params)) # Checks that extra parameters are allowed. sign = signature(self.__class__.__init__) allowed = set(sign.parameters) notin = set_params - allowed - \ BasePipelineItem._hidden_constructor_arguments if len(notin) > 0: allowed = "\n".join( wrap(", ".join(sorted(filter(lambda _: _ != 'self', allowed))))) if len(notin) == 1: raise NameError("Parameter '{0}' is not allowed for class '{" "1}'.\nAllowed: {2}".format( list(sorted(notin))[0], self.__class__.__name__, allowed)) else: raise NameError("Parameters {0} are not allowed for class '{" "1}'.\nAllowed: {2}".format( sorted(notin), self.__class__.__name__, allowed)) # Handles parameters columns. inputs = OrderedDict() cols = params.pop('columns', None) if cols: if isinstance(cols, dict): inputs.update(cols) else: self.set_inputs(cols, early=True) for role in DataRoles._allowed: name = DataRoles.to_attribute(role) if name in params: if cols is not None and role in cols and params[name] != \ cols[role]: raise AttributeError( "Attribute '{0}' is already set to '{1}', " "cannot be replaced by '{2}'".format( name, cols[role], params[name])) attr = DataRoles.to_attribute(role) if attr in allowed: setattr(self, attr, params[name]) else: inputs[role] = params[name] del params[name] if len(inputs) > 0: self.set_inputs(inputs, early=True)
def plot_precision_recall_curve(input_shp,groud_truth_shp,save_path): # from sklearn.metrics import precision_recall_curve from sklearn.utils.fixes import signature precision, recall, _ = precision_recall_curve_iou(input_shp,groud_truth_shp) average_precision = calculate_average_precision(precision,recall) # In matplotlib < 1.5, plt.fill_between does not have a 'step' argument step_pos = 'mid' # post step_kwargs = ({'step': step_pos} if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where=step_pos) plt.plot(recall, precision, 'r--') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([-0.01, 1.05]) plt.xlim([-0.01, 1.01]) plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format( average_precision)) # save average_precision to txt file txt_path = os.path.splitext(save_path)[0]+'_ap.txt' with open(txt_path,'w') as f_obj: f_obj.writelines('shape_file average_precision\n') f_obj.writelines('%s %.4lf\n' % (input_shp, average_precision)) # plt.show() plt.savefig(save_path,dpi=300) basic.outputlogMessage("Output figures to %s" % os.path.abspath(save_path))
def plot_precision_recall_curve(clf, X_test, y_test): if hasattr(clf, 'decision_function'): y_scores = clf.decision_function(X_test) elif hasattr(clf, 'predict_proba'): y_scores = clf.predict_proba(X_test)[:, 1] else: print('Precision recall curve not possible') return average_precision = average_precision_score(y_test, y_scores) print('Average precision-recall score: {0:0.2f}'.format(average_precision)) precision, recall, thresholds = precision_recall_curve(y_test, y_scores) # In matplotlib < 1.5, plt.fill_between does not have a 'step' argument step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format( average_precision))
def precision_recall_plot(targs, preds, figsize=(6,6)): """ Plots the precision recall curve ---------- targs: array-like true class labels preds: array-like predicted probabilities figsize: size of figure Returns: ------- A precision and recall curve """ average_precision = average_precision_score(targs, preds) precision, recall, _ = precision_recall_curve(targs, preds) plt.figure(figsize=figsize) step_kwargs = ({'step': 'post'} if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format( average_precision)) plt.show()
def plot_precision_recall(classifier, X_test, y_test, y_score): """ Function to plot an precision-recall curve knowing the features, resp and model """ from sklearn.metrics import precision_recall_curve import matplotlib.pyplot as plt from sklearn.utils.fixes import signature from sklearn.metrics import average_precision_score classifier.fit(X_train, y_train) predictions_probs = classifier.predict_proba(X_test) average_precision = average_precision_score(y_test, predictions_probs[:,1]) print('Average precision-recall score: {0:0.2f}'.format( average_precision)) precision, recall, _ = precision_recall_curve(y_test, predictions_probs[:,1]) # In matplotlib < 1.5, plt.fill_between does not have a 'step' argument step_kwargs = ({'step': 'post'} if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.title('Precision-Recall curve: AP={0:0.2f}'.format( average_precision))
def plotAUPRCForLogisticRegression(X_train, y_train, testData, y_pred, predictionProbability): import matplotlib.pyplot as plt4 # Create a simple classifier classifier = svm.LinearSVC() classifier.fit(X_train, y_train) y_score = classifier.decision_function(testData) from sklearn.metrics import average_precision_score average_precision = average_precision_score(y_train, y_score) from sklearn.metrics import precision_recall_curve import matplotlib.pyplot as plt from sklearn.utils.fixes import signature precision, recall, _ = precision_recall_curve(y_train, predictionProbability) # In matplotlib < 1.5, plt.fill_between does not have a 'step' argument step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) plt4.step(recall, precision, color='b', alpha=0.2, where='post') plt4.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt4.xlabel('Recall') plt4.ylabel('Precision') plt4.ylim([0.0, 1.05]) plt4.xlim([0.0, 1.0]) plt4.title( '2-class Precision-Recall(Logistic Regression) curve: AP= {}'.format( average_precision)) plt4.show()
def _get_param_names(cls): """Get parameter names for the estimator""" # fetch the constructor or the original constructor before # deprecation wrapping if any init = getattr(cls.__init__, 'deprecated_original', cls.__init__) if init is object.__init__: # No explicit constructor to introspect return [] # introspect the constructor arguments to find the model parameters # to represent init_signature = signature(init) # Consider the constructor parameters excluding 'self' parameters = [ p for p in init_signature.parameters.values() if p.name != 'self' and p.kind != p.VAR_KEYWORD ] # Extract and sort argument names excluding 'self' parameters = set([p.name for p in parameters]) # recurse for superclass in cls.__bases__: try: parameters.update(superclass._get_param_names()) except AttributeError: # object and pygsp.graphs.Graph don't have this method pass return parameters
def draw_pr_curve(y_true, y_pred, **kwargs): precision, recall, _ = precision_recall_curve(y_true, y_pred) step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) title = kwargs.get('title', None) plot_path = kwargs.get('plot_path', None) curve_auc = auc(recall, precision) if title is None: plt.title('AUC=%.2f' % curve_auc) else: plt.title('%s, AUC=%.2f' % (title, curve_auc)) if plot_path is None: plt.show() else: plt.savefig(plot_path) plt.close() return curve_auc
def _check_roles(self): """ Checks the consistency between defined roles and supported roles. """ if not hasattr(self, '_entrypoint'): raise SystemExit( 'One internal learner does not follow the new syntax.') params = signature(self._entrypoint).parameters for role in DataRoles._allowed: attr = DataRoles.to_attribute(role) if hasattr(self, attr) and getattr(self, attr) is not None and \ attr not in params: if role == Role.Label: # warnings instead of an exception but we should # really simplify the logic # in experiment.py. The model should know which # roles it supports. # current code makes it difficult to guess. # A minor modification in entrypoints.py should do the # trick. if self.type != "clusterer": warnings.warn( "Model '{0}' (type='{1}') does not support " "role '{2}' (for developers, check " "_allowed_roles is defined).".format( type(self), self.type, role)) else: raise RuntimeError( "Model '{0}' (type='{1}') does not support role " "'{2}' (for developers, check _allowed_roles is " "defined).".format(type(self), self.type, role))
def precisionRecall_microJoint_curve(self, precision, recall): ''' Function to plot precision recall curve with micro averaged scores # args precision: list of precision score for each class along with micro averaged score recall: list of recall score for each class along with micro averaged score # returns precision recall curve ''' plt.figure() step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall['micro'], precision['micro'], color='b', alpha=0.2, where='post') plt.fill_between(recall["micro"], precision["micro"], alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.title('Micro-averaged over all classes') return plt
def drawPRC(result, sp, methodlabel): ''' Accept a result dataframe whose 'cond' coulmn is binary series representing the ture condition and 'pred' column is the normalized score of predicton. ''' print("drawing prc curve...") sp.set_xlabel('Recall') sp.set_ylabel('Precision') sp.set_ylim([0.0, 1.0]) sp.set_xlim([0.0, 1.0]) sp.set_title('2-class Precision-Recall curve') precision, recall, threshold = precision_recall_curve( result['cond'], result['pred']) average_precision = average_precision_score(result['cond'], result['pred']) myauc = auc(recall, precision) step_kwargs = ({ 'step': 'post' } if 'step' in signature(sp.fill_between).parameters else {}) sp.step(recall, precision, alpha=0.2, where='post') sp.fill_between(recall, precision, alpha=0.2, label=methodlabel + " (AUC = %.3f)" % myauc, **step_kwargs) return myauc
def draw_precision_recall_curve(classifier, x_data, y_data): ''' Takes: classifier object, feature and target data, Returns: chart showing precision recall curve Reference: code drawn from Scikit-learn documentation, https://bit.ly/2WaYP2I ''' if isinstance(classifier, LinearSVC): pred_scores = classifier.decision_function(x_data) else: pred_scores = classifier.predict_proba(x_data)[:, 1] precision, recall, thresholds = precision_recall_curve(y_data, pred_scores) step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.title('Precision-Recall Curve') return
def _build_repr(self): # XXX This is copied from sklearn.BaseEstimator's get_params cls = self.__class__ init = getattr(cls.__init__, 'deprecated_original', cls.__init__) init_signature = signature(init) if init is object.__init__: args = [] else: args = sorted([p.name for p in init_signature.parameters.values() if p.name != 'self' and p.kind != p.VAR_KEYWORD]) class_name = self.__class__.__name__ params = dict() for key in args: warnings.simplefilter("always", DeprecationWarning) try: with warnings.catch_warnings(record=True) as w: value = getattr(self, key, None) if len(w) and w[0].category == DeprecationWarning: continue finally: warnings.filters.pop(0) params[key] = value return '%s(%s)' % (class_name, _pprint(params, offset=len(class_name)))
def _get_param_names(cls): """Get parameter names for the estimator""" # fetch the constructor or the original constructor before # deprecation wrapping if any init = getattr(cls.__init__, 'deprecated_original', cls.__init__) if init is object.__init__: # No explicit constructor to introspect return [] # introspect the constructor arguments to find the model parameters # to represent from sklearn.utils.fixes import signature init_signature = signature(init) # Consider the constructor parameters excluding 'self' parameters = [p for p in init_signature.parameters.values() if p.name != 'self' and p.kind != p.VAR_KEYWORD] for p in parameters: if p.kind == p.VAR_POSITIONAL: raise RuntimeError("scikit-learn estimators should always " "specify their parameters in the signature" " of their __init__ (no varargs)." " %s with constructor %s doesn't " " follow this convention." % (cls, init_signature)) # Extract and sort argument names excluding 'self' return sorted([p.name for p in parameters])
def test(model): _, _, x_score, y_score = get_data() y_pred = [] for im in x_score: im = im.reshape(2304) pred = np.dot(im, model.W) + model.b y_pred.append(pred) precision, recall, _ = precision_recall_curve(y_score, y_pred) step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) fig2 = plt.figure() plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) y_score = (y_score > 0.5) y_pred = (np.asarray(y_pred) > 0.5) ACA = accuracy_score(y_score, y_pred, normalize=True) f1 = f1_score(y_score, y_pred, 'binary') print('ACA; ' + str(ACA)) print('F1; ' + str(f1)) plt.title('2-class Precision-Recall curve: ACA={0:0.2f}'.format(ACA) + ' F1={0:0.2f}'.format(f1)) plt.show()
def plot_positionalPRC(positionalPRC_output): ''' accepts output dictionary from the positionalPRC function of the form: motif_name --> [precision,recall,auPRC] generates PRC curves for each motif on same coordinates ''' from sklearn.utils.fixes import signature for motif_name, values in positionalPRC_output.items(): recall = values[0] precision = values[1] auPRC = str(round(values[2], 3)) step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, label=motif_name + ":" + auPRC, where='post') #uncomment to fill the area below the curve, generally not desirable if multiple curves plotted on same axes. #plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.legend() plt.show()
def evaluation(self, df_true, df_predict, df_prob, output_eval_dir): run = Run.get_context() # precition-recall-curve average_precision = average_precision_score(df_true, df_predict) precision, recall, _ = precision_recall_curve(df_true, df_prob) step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) f1_plt = plt.figure(1) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0, 1.1]) plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format( average_precision)) run.log_image("precition/recall curve", plot=f1_plt) f1_plt.savefig(os.path.join(output_eval_dir, 'precition_recall.png')) f2_plt = plt.figure(2) metrics_name = ['precition', 'recall', 'F1-Score'] p = precision_score(df_true, df_predict, average='binary') r = recall_score(df_true, df_predict, average='binary') f1 = f1_score(df_true, df_predict, average='binary') values_list = [p, r, f1] plt.bar(metrics_name, values_list, width=0.8, facecolor="#ff9999", edgecolor="white") for x, y in zip(metrics_name, values_list): plt.text(x, y, '%.4f' % y, ha='center', va='bottom') plt.ylim([0, 1.1]) plt.ylabel('score') plt.title('Scores') run.log_image("scores", plot=f2_plt) f2_plt.savefig(os.path.join(output_eval_dir, 'scores.png')) f3_plt = plt.figure(3) # Compute fpr, tpr, thresholds and roc auc fpr, tpr, thresholds = roc_curve(df_true, df_prob) roc_auc = auc(df_true, df_prob) plt.plot(fpr, tpr, label='ROC curve (area = %0.3f)' % roc_auc) plt.plot([0, 1], [0, 1], 'k--') # random predictions curve plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.0]) plt.xlabel('False Positive Rate or (1 - Specifity)') plt.ylabel('True Positive Rate or (Sensitivity)') plt.title('ROC Curve') plt.legend(loc="lower right") run.log_image("ROC curve", plot=f3_plt) f3_plt.savefig(os.path.join(output_eval_dir, 'roc.png'))
def plot_precision_recall_curve(labels, predictions_prob, class_nums, average_function='weighted'): """ Plot precision recall curve :param labels: :param predictions_prob: :param class_nums: :param average_function: :return: """ labels = label_binarize(labels, classes=np.linspace(0, class_nums - 1, num=class_nums).tolist()) predictions_prob = np.array(predictions_prob, dtype=np.float32) precision = dict() recall = dict() average_precision = dict() for i in range(class_nums): precision[i], recall[i], _ = precision_recall_curve( labels[:, i], predictions_prob[:, i]) average_precision[i] = average_precision_score(labels[:, i], predictions_prob[:, i]) precision[average_function], recall[ average_function], _ = precision_recall_curve(labels.ravel(), predictions_prob.ravel()) average_precision[average_function] = average_precision_score( labels, predictions_prob, average=average_function) log.info('Average precision score, {:s}-averaged ' 'over all classes: {:.5f}'.format( average_function, average_precision[average_function])) plt.figure() plt.step(recall[average_function], precision[average_function], color='b', alpha=0.2, where='post') step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) plt.fill_between(recall[average_function], precision[average_function], alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.title('Average precision score, {:s}-averaged over ' 'all classes: AP={:.5f}'.format( average_function, average_precision[average_function]))
def plot_pr_curve(y, yhat): precision, recall, _ = precision_recall_curve(y.ravel(), yhat.ravel()) step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.grid(color='b', linestyle='--', linewidth=0.5, alpha=0.3)
def test_kernel_theta(): # Check that parameter vector theta of kernel is set correctly. for kernel in kernels: if isinstance(kernel, KernelOperator) \ or isinstance(kernel, Exponentiation): # skip non-basic kernels continue theta = kernel.theta _, K_gradient = kernel(X, eval_gradient=True) # Determine kernel parameters that contribute to theta init_sign = signature(kernel.__class__.__init__).parameters.values() args = [p.name for p in init_sign if p.name != 'self'] theta_vars = map(lambda s: s[0:-len("_bounds")], filter(lambda s: s.endswith("_bounds"), args)) assert_equal( set(hyperparameter.name for hyperparameter in kernel.hyperparameters), set(theta_vars)) # Check that values returned in theta are consistent with # hyperparameter values (being their logarithms) for i, hyperparameter in enumerate(kernel.hyperparameters): assert_equal(theta[i], np.log(getattr(kernel, hyperparameter.name))) # Fixed kernel parameters must be excluded from theta and gradient. for i, hyperparameter in enumerate(kernel.hyperparameters): # create copy with certain hyperparameter fixed params = kernel.get_params() params[hyperparameter.name + "_bounds"] = "fixed" kernel_class = kernel.__class__ new_kernel = kernel_class(**params) # Check that theta and K_gradient are identical with the fixed # dimension left out _, K_gradient_new = new_kernel(X, eval_gradient=True) assert_equal(theta.shape[0], new_kernel.theta.shape[0] + 1) assert_equal(K_gradient.shape[2], K_gradient_new.shape[2] + 1) if i > 0: assert_equal(theta[:i], new_kernel.theta[:i]) assert_array_equal(K_gradient[..., :i], K_gradient_new[..., :i]) if i + 1 < len(kernel.hyperparameters): assert_equal(theta[i + 1:], new_kernel.theta[i:]) assert_array_equal(K_gradient[..., i + 1:], K_gradient_new[..., i:]) # Check that values of theta are modified correctly for i, hyperparameter in enumerate(kernel.hyperparameters): theta[i] = np.log(42) kernel.theta = theta assert_almost_equal(getattr(kernel, hyperparameter.name), 42) setattr(kernel, hyperparameter.name, 43) assert_almost_equal(kernel.theta[i], np.log(43))
def get_params(self, deep=True): "Scikit-learn API, returns all parameters." sig = signature(self.__class__.__init__) params = [(p if p != 'columns' else '_columns', p) for p in sig.parameters if p not in ('self', 'params')] res = {p: getattr(self, att) for att, p in params if hasattr(self, att)} if hasattr(self, "_columns") and isinstance(self._columns, dict): res['columns'] = self._columns return res
def prec_rec_curve(prec, rec, avg_prec): step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) plt.step(rec, prec, color='b', alpha=0.2, where='post') plt.fill_between(rec, prec, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(avg_prec))
def plot_pr_curve(self, y_pred, y_score): precision, recall, thresholds = precision_recall_curve(self.y_test, y_score) step_kwargs = ({'step': 'post'} if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, here='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) average_precision = average_precision_score(self.y_test, y_score) plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(average_precision)) plt.show() return plt
def plot_prec_recall(precision, recall, thresholds): # In matplotlib < 1.5, plt.fill_between does not have a 'step' argument step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.title('2-class Precision-Recall curve')
def site_example(): # In matplotlib < 1.5, plt.fill_between does not have a 'step' argument step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.show()
def has_fit_parameter(estimator, parameter): """Checks whether the estimator's fit method supports the given parameter. Examples -------- >>> from sklearn.svm import SVC >>> has_fit_parameter(SVC(), "sample_weight") True """ # print("has_fit_parameter has been called") addfreq() return parameter in signature(estimator.fit).parameters
def validate(self): if len(self.models) == 0: raise Exception('Model has not been initialized') index = 1 AP = [] Majority = [] val_X = self.feature.val_X.copy() models = self.models for model in models: y_true = self.feature.val_y.copy() y_true[y_true != index] = 0 y_true[y_true != 0] = 1 if hasattr(model, 'decision_function'): y_pred = model.decision_function(val_X) else: y_pred = model.predict(val_X) average_precision = average_precision_score(y_true, y_pred) AP.append(average_precision) index += 1 from sklearn.metrics import precision_recall_curve import matplotlib.pyplot as plt from sklearn.utils.fixes import signature precision, recall, threshold = precision_recall_curve( y_true, y_pred) # In matplotlib < 1.5, plt.fill_between does not have a 'step' argument step_kwargs = ({ 'step': 'post' } if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format( average_precision)) plt.savefig('AUC' + self.feature.feat_types[0] + str(index) + '.png') mAP = np.mean(np.array(AP)) return AP, mAP
def clf_print_results(clf, Xtrain, ytrain, Xtest, ytest): """ Classifier results: * params: - clf: classifier - Xtrain, ytrain, Xtest, ytest * prints: - classification report - accuracy score - confustion matrix - AUC for ROC and ROC curve - AUC for precision recall and precision-recall curve """ clf.fit(Xtrain, ytrain) ypred = clf.predict(Xtest) print("Classification report:\n", classification_report(ytest,ypred)) accuracy = accuracy_score(ytest, ypred) conf_mat = confusion_matrix(ytest, ypred) print("Accuracy %.2f%%" % (accuracy*100)) print("---- Confusion matrix ----") print(conf_mat) ypred_proba = clf.predict_proba(Xtest)[:,1] fpr, tpr, thr_roc = roc_curve(ytest, ypred_proba) auc_roc = auc(fpr,tpr) precision, recall, thr_pr = precision_recall_curve(ytest, ypred_proba) auc_pr = auc(recall, precision) average_precision = average_precision_score(ytest, ypred_proba) #----------------- FIGURES AUC ----------------- fig, ax = plt.subplots(1,2, figsize=(14,6)) ax[0].plot(fpr, tpr, '-', lw=2, label='AUC-ROC=%.3f' % auc_roc) ax[0].plot([0, 1], [0, 1],linestyle='--') ax[0].set_xlabel('False Positive Rate', fontsize=16) ax[0].set_ylabel('True Positive Rate', fontsize=16) ax[0].set_title('ROC Curve',fontsize=16) ax[0].legend(loc="lower right", fontsize=14) plt.subplots_adjust(hspace = 0.3, top=0.92) step_kwargs = ({'step': 'post'} if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post', label='AUC-PR=%.3f' % auc_pr) plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) ax[1].set_xlabel('Recall', fontsize=16) ax[1].set_ylabel('Precision', fontsize=16) ax[1].set_title('Precision-Recall Curve', fontsize=16) #ax[1].plot([0, 1], [0.1, 0.1], linestyle='--') ax[1].legend(loc="upper right", fontsize=14) plt.show()
def basic_assess_AUC(scores, labels, plot_pr_idx=None): assert len(scores) == len(labels) if plot_pr_idx is not None: precision, recall, _ = precision_recall_curve(labels, scores[:, plot_pr_idx]) print(len(np.where(labels == 0)[0]), len(np.where(labels == 1)[0]), len(np.unique(precision)), len(np.unique(recall))) step_kwargs = ({'step': 'post'} if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('recall') plt.ylabel('precision') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.show() return np.array([roc_auc_score(labels, scores[:, i]) for i in range(scores.shape[1])]), np.array([average_precision_score(labels, scores[:, i]) for i in range(scores.shape[1])])
def plot_prc(plot_file, y_test, y_score, average_precision, txt): precision, recall, _ = sklearn.metrics.precision_recall_curve(y_test, y_score) step_kwargs = ({'step': 'post'} if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.title('{0} AP={1:0.2f}'.format(txt, average_precision)) plt.axes().set_aspect('equal') plt.savefig(plot_file, bbox_inches='tight')
def _get_args(function, varargs=False): """Helper to get function arguments""" try: params = signature(function).parameters except ValueError: # Error on builtin C function return [] args = [key for key, param in params.items() if param.kind not in (param.VAR_POSITIONAL, param.VAR_KEYWORD)] if varargs: varargs = [param.name for param in params.values() if param.kind == param.VAR_POSITIONAL] if len(varargs) == 0: varargs = None return args, varargs else: return args
def _serialize_cross_validator(o): ret = OrderedDict() parameters = OrderedDict() # XXX this is copied from sklearn.model_selection._split cls = o.__class__ init = getattr(cls.__init__, 'deprecated_original', cls.__init__) # Ignore varargs, kw and default values and pop self init_signature = signature(init) # Consider the constructor parameters excluding 'self' if init is object.__init__: args = [] else: args = sorted([p.name for p in init_signature.parameters.values() if p.name != 'self' and p.kind != p.VAR_KEYWORD]) for key in args: # We need deprecation warnings to always be on in order to # catch deprecated param values. # This is set in utils/__init__.py but it gets overwritten # when running under python3 somehow. warnings.simplefilter("always", DeprecationWarning) try: with warnings.catch_warnings(record=True) as w: value = getattr(o, key, None) if len(w) and w[0].category == DeprecationWarning: # if the parameter is deprecated, don't show it continue finally: warnings.filters.pop(0) if not (hasattr(value, '__len__') and len(value) == 0): value = json.dumps(value) parameters[key] = value else: parameters[key] = None ret['oml-python:serialized_object'] = 'cv_object' name = o.__module__ + "." + o.__class__.__name__ value = OrderedDict([['name', name], ['parameters', parameters]]) ret['value'] = value return ret
def plot_positionalPRC(positionalPRC_output): ''' accepts output dictionary from the positionalPRC function of the form: motif_name --> [precision,recall,auPRC] generates PRC curves for each motif on same coordinates ''' from sklearn.utils.fixes import signature for motif_name,values in positionalPRC_output.items(): recall=values[0] precision=values[1] auPRC=str(round(values[2],3)) step_kwargs = ({'step': 'post'} if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, label=motif_name+":"+auPRC,where='post') #uncomment to fill the area below the curve, generally not desirable if multiple curves plotted on same axes. #plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.legend() plt.show()
print('Average precision-recall score: {0:0.2f}'.format( average_precision)) ############################################################################### # Plot the Precision-Recall curve # ................................ from sklearn.metrics import precision_recall_curve import matplotlib.pyplot as plt from sklearn.utils.fixes import signature precision, recall, _ = precision_recall_curve(y_test, y_score) # In matplotlib < 1.5, plt.fill_between does not have a 'step' argument step_kwargs = ({'step': 'post'} if 'step' in signature(plt.fill_between).parameters else {}) plt.step(recall, precision, color='b', alpha=0.2, where='post') plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format( average_precision)) ############################################################################### # In multi-label settings # ------------------------